authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-01 23:43:16-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-02 00:38:51-07:00
log01e73fd4cffeccb05dd939ca75bcfb583fb252e2
treef6a28eddf5f2acff928b12e82be6f9967f290673
parent9d07a3bb194eae46161a67803138348d98d0b7d4
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

feat: record audio and video on iOS

Record Audio and Record Video join the page's Insert menu. AVFoundation records, and crates/mobile stores the result through canvas::recording, which now holds what the desktop stored: 16 kHz IMA ADPCM WAV and Motion JPEG AVI, the files OneNote 2010 plays. A bar over the page's foot shows the clock with Pause and Stop; a tap on a recording plays it with AVPlayer, a video's pictures in the bar, and See Playback follows it. Audio keeps recording in the background under the audio background mode, as a lecture outlasts the screen's timeout, and a call pauses it. The camera stops in the background, which ends a video recording and saves it. The simulator records video from a stand-in camera. Assisted-by: claude-opus-5.5

24 files changed, 2080 insertions(+), 819 deletions(-)

apps/ios/Info.plist+7-1
...@@ -22,7 +22,9 @@...@@ -22,7 +22,9 @@
22 <key>LSSupportsOpeningDocumentsInPlace</key>22 <key>LSSupportsOpeningDocumentsInPlace</key>
23 <true/>23 <true/>
24 <key>NSCameraUsageDescription</key>24 <key>NSCameraUsageDescription</key>
25 <string>Take a photo to put on a page.</string>25 <string>Take photos and record video on your pages.</string>
26 <key>NSMicrophoneUsageDescription</key>
27 <string>Record audio and video on your pages.</string>
26 <key>NSUbiquitousContainers</key>28 <key>NSUbiquitousContainers</key>
27 <dict>29 <dict>
28 <key>iCloud.net.paperclover.snowbound</key>30 <key>iCloud.net.paperclover.snowbound</key>
...@@ -42,6 +44,10 @@...@@ -42,6 +44,10 @@
42 <key>UIApplicationSupportsMultipleScenes</key>44 <key>UIApplicationSupportsMultipleScenes</key>
43 <false/>45 <false/>
44 </dict>46 </dict>
47 <key>UIBackgroundModes</key>
48 <array>
49 <string>audio</string>
50 </array>
45 <key>UIFileSharingEnabled</key>51 <key>UIFileSharingEnabled</key>
46 <true/>52 <true/>
47 <key>UILaunchScreen</key>53 <key>UILaunchScreen</key>
apps/ios/Snowbound/CanvasView.swift+70-3
...@@ -137,6 +137,8 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI...@@ -137,6 +137,8 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI
137 var onOpened: (() -> Void)?137 var onOpened: (() -> Void)?
138 /// Called when a tap on the page date asks to change it, with the date it shows.138 /// Called when a tap on the page date asks to change it, with the date it shows.
139 var onDate: ((DateField, Date) -> Void)?139 var onDate: ((DateField, Date) -> Void)?
140 /// Called when a tap on a recording, or beside a note linked to one, asks to play it.
141 var onPlay: ((PlayRequest) -> Void)?
140 /// Formatting above the keyboard.142 /// Formatting above the keyboard.
141 let formatBar = FormatBar()143 let formatBar = FormatBar()
142 /// Room above the page for a bar over it.144 /// Room above the page for a bar over it.
...@@ -147,6 +149,16 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI...@@ -147,6 +149,16 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI
147 }149 }
148 }150 }
149151
152 /// Room below the page, above the keyboard, for a bar over its foot.
153 var bottomInset: CGFloat = 0 {
154 didSet {
155 guard bottomInset != oldValue else { return }
156 contentInset.bottom += bottomInset - oldValue
157 verticalScrollIndicatorInsets.bottom = contentInset.bottom
158 revealCaret()
159 }
160 }
161
150 init(section: Section, page: String) {162 init(section: Section, page: String) {
151 self.section = section163 self.section = section
152 self.page = page164 self.page = page
...@@ -480,8 +492,8 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI...@@ -480,8 +492,8 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI
480 else { return }492 else { return }
481 let keyboard = convert(frame, from: window.screen.coordinateSpace)493 let keyboard = convert(frame, from: window.screen.coordinateSpace)
482 let overlap = max(0, bounds.maxY - keyboard.minY - safeAreaInsets.bottom)494 let overlap = max(0, bounds.maxY - keyboard.minY - safeAreaInsets.bottom)
483 contentInset.bottom = overlap495 contentInset.bottom = overlap + bottomInset
484 verticalScrollIndicatorInsets.bottom = overlap496 verticalScrollIndicatorInsets.bottom = contentInset.bottom
485 revealCaret()497 revealCaret()
486 }498 }
487499
...@@ -542,7 +554,9 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI...@@ -542,7 +554,9 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI
542 func tap(at point: CGPoint) {554 func tap(at point: CGPoint) {
543 press(at: point)555 press(at: point)
544 var seconds: Int64 = 0556 var seconds: Int64 = 0
545 if let handle, let field = DateField(rawValue: sb_view_date_request(handle, &seconds)) {557 if let request = playRequest() {
558 onPlay?(request)
559 } else if let handle, let field = DateField(rawValue: sb_view_date_request(handle, &seconds)) {
546 onDate?(field, Date(timeIntervalSince1970: TimeInterval(seconds)))560 onDate?(field, Date(timeIntervalSince1970: TimeInterval(seconds)))
547 } else if inkFrame != nil {561 } else if inkFrame != nil {
548 showInkMenu()562 showInkMenu()
...@@ -1142,6 +1156,59 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI...@@ -1142,6 +1156,59 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI
1142 }1156 }
1143 }1157 }
11441158
1159 // MARK: Recordings
1160
1161 /// Starts recording at the caret, under a line saying when; whether it started.
1162 func startRecording(video: Bool) -> Bool {
1163 guard let handle else { return false }
1164 let (day, time) = titleDate()
1165 return edit(external: true) { sb_view_start_recording(handle, video, day, time) }
1166 }
1167
1168 /// How far the recording has got in milliseconds, without its pauses, and whether it is
1169 /// paused; nil when nothing records.
1170 var recording: (ms: Int64, paused: Bool)? {
1171 guard let handle else { return nil }
1172 var paused = false
1173 let ms = sb_view_recording(handle, &paused)
1174 return ms < 0 ? nil : (ms, paused)
1175 }
1176
1177 /// Nothing typed while the recording is paused links to it.
1178 func pauseRecording(_ paused: Bool) {
1179 guard let handle else { return }
1180 sb_view_pause_recording(handle, paused)
1181 }
1182
1183 /// Puts the recording made where it started: a PCM WAV file, or with `video` the AVI
1184 /// file of `sb_movie_finish`. Nil forgets a recording whose file never came.
1185 func finishRecording(_ file: Data?, video: Bool) {
1186 guard let handle else { return }
1187 _ = edit(external: true) {
1188 guard let file else { return sb_view_finish_recording(handle, nil, 0, video) }
1189 return file.withUnsafeBytes { bytes in
1190 sb_view_finish_recording(handle, bytes.bindMemory(to: UInt8.self).baseAddress, bytes.count, video)
1191 }
1192 }
1193 }
1194
1195 /// The recording the last tap asked to play, with its files written out to play from.
1196 private func playRequest() -> PlayRequest? {
1197 let folder = FileManager.default.temporaryDirectory.appendingPathComponent("Playback")
1198 guard let handle, let json = sb_view_play_request(handle, folder.path) else { return nil }
1199 defer { sb_string_free(json) }
1200 let decoder = JSONDecoder()
1201 decoder.keyDecodingStrategy = .convertFromSnakeCase
1202 return try? decoder.decode(PlayRequest.self, from: Data(String(cString: json).utf8))
1203 }
1204
1205 /// See Playback: the note linked at `ms` into the recording playing is highlighted; nil
1206 /// ends playback.
1207 func played(at ms: Int64?) {
1208 guard let handle, sb_view_played(handle, ms ?? -1) else { return }
1209 dirty = true
1210 }
1211
1145 /// Commits marked text, so it is stored before the app leaves the screen.1212 /// Commits marked text, so it is stored before the app leaves the screen.
1146 func commitComposition() {1213 func commitComposition() {
1147 guard markedTextRange != nil else { return }1214 guard markedTextRange != nil else { return }
apps/ios/Snowbound/PageViewController.swift+116
...@@ -21,6 +21,10 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate...@@ -21,6 +21,10 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
21 private let titleFocus: Bool21 private let titleFocus: Bool
22 private lazy var canvas = CanvasView(section: section, page: page)22 private lazy var canvas = CanvasView(section: section, page: page)
23 private let bar = ConflictBar()23 private let bar = ConflictBar()
24 /// What records or plays on this page, over its foot.
25 private let media = MediaBar()
26 private var playback: Playback?
27 private var mediaClock: Timer?
24 /// Reading View, beside the page menu where the page has one.28 /// Reading View, beside the page menu where the page has one.
25 private var reader: UIBarButtonItem?29 private var reader: UIBarButtonItem?
26 /// Opens another page or conflict page of the section.30 /// Opens another page or conflict page of the section.
...@@ -62,7 +66,15 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate...@@ -62,7 +66,15 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
62 bar.translatesAutoresizingMaskIntoConstraints = false66 bar.translatesAutoresizingMaskIntoConstraints = false
63 view.addSubview(canvas)67 view.addSubview(canvas)
64 view.addSubview(bar)68 view.addSubview(bar)
69 media.translatesAutoresizingMaskIntoConstraints = false
70 view.addSubview(media)
71 let wide = media.widthAnchor.constraint(equalToConstant: 480)
72 wide.priority = .required - 1
65 NSLayoutConstraint.activate([73 NSLayoutConstraint.activate([
74 wide,
75 media.centerXAnchor.constraint(equalTo: view.safeAreaLayoutGuide.centerXAnchor),
76 media.leadingAnchor.constraint(greaterThanOrEqualTo: view.safeAreaLayoutGuide.leadingAnchor, constant: 12),
77 media.bottomAnchor.constraint(equalTo: view.keyboardLayoutGuide.topAnchor, constant: -12),
66 canvas.leadingAnchor.constraint(equalTo: view.leadingAnchor),78 canvas.leadingAnchor.constraint(equalTo: view.leadingAnchor),
67 canvas.trailingAnchor.constraint(equalTo: view.trailingAnchor),79 canvas.trailingAnchor.constraint(equalTo: view.trailingAnchor),
68 canvas.topAnchor.constraint(equalTo: view.topAnchor),80 canvas.topAnchor.constraint(equalTo: view.topAnchor),
...@@ -75,6 +87,10 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate...@@ -75,6 +87,10 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
75 canvas.onChange = { [weak self] in self?.editingChanged() }87 canvas.onChange = { [weak self] in self?.editingChanged() }
76 canvas.onOpened = { [weak self] in self?.opened() }88 canvas.onOpened = { [weak self] in self?.opened() }
77 canvas.onDate = { [weak self] field, date in self?.pickDate(field, date) }89 canvas.onDate = { [weak self] field, date in self?.pickDate(field, date) }
90 canvas.onPlay = { [weak self] request in self?.play(request) }
91 media.onPause = { [weak self] in self?.pauseMedia() }
92 media.onStop = { [weak self] in self?.stopMedia() }
93 NotificationCenter.default.addObserver(self, selector: #selector(showMedia), name: Recorder.changed, object: nil)
78 canvas.formatBar.onPicture = { [weak self] camera in self?.pickPicture(camera: camera) }94 canvas.formatBar.onPicture = { [weak self] camera in self?.pickPicture(camera: camera) }
79 bar.onAction = { [weak self] in self?.conflictAction() }95 bar.onAction = { [weak self] in self?.conflictAction() }
80 NotificationCenter.default.addObserver(96 NotificationCenter.default.addObserver(
...@@ -173,6 +189,16 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate...@@ -173,6 +189,16 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
173 UIAction(189 UIAction(
174 title: "Space", image: UIImage(systemName: "arrow.up.and.down.text.horizontal"), attributes: editable190 title: "Space", image: UIImage(systemName: "arrow.up.and.down.text.horizontal"), attributes: editable
175 ) { [weak self] _ in self?.canvas.insertSpace() },191 ) { [weak self] _ in self?.canvas.insertSpace() },
192 UIMenu(options: .displayInline, children: [
193 UIAction(
194 title: "Record Audio", image: UIImage(systemName: "mic"),
195 attributes: Recorder.current == nil ? editable : .disabled
196 ) { [weak self] _ in self?.record(video: false) },
197 UIAction(
198 title: "Record Video", image: UIImage(systemName: "video"),
199 attributes: Recorder.current == nil && Recorder.canRecordVideo ? editable : .disabled
200 ) { [weak self] _ in self?.record(video: true) },
201 ]),
176 ])202 ])
177 return (Prototype.reading ? [readingAction()] : []) + [203 return (Prototype.reading ? [readingAction()] : []) + [
178 insert,204 insert,
...@@ -217,6 +243,8 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate...@@ -217,6 +243,8 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
217 }243 }
218244
219 private func toggleReading() {245 private func toggleReading() {
246 // A recording goes on the page as laid out.
247 guard Recorder.current?.canvas !== canvas else { return }
220 canvas.showReading(!canvas.readingShown)248 canvas.showReading(!canvas.readingShown)
221 }249 }
222250
...@@ -289,6 +317,94 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate...@@ -289,6 +317,94 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
289317
290 @objc private func leaving() { canvas.commitComposition() }318 @objc private func leaving() { canvas.commitComposition() }
291319
320 override func viewWillDisappear(_ animated: Bool) {
321 super.viewWillDisappear(animated)
322 // A recording goes on the page it started on, as the desktop's does on leaving it.
323 if Recorder.current?.canvas === canvas { Recorder.current?.stop() }
324 stopPlayback()
325 }
326
327 // MARK: Recording and playback
328
329 func record(video: Bool) {
330 stopPlayback()
331 Recorder.start(video: video, on: canvas) { [weak self] title, message in self?.alert(title, message) }
332 }
333
334 func stopRecording() {
335 if Recorder.current?.canvas === canvas { Recorder.current?.stop() }
336 }
337
338 private func play(_ request: PlayRequest) {
339 guard Recorder.current == nil else { return }
340 playback = Playback(request)
341 showMedia()
342 }
343
344 private func stopPlayback() {
345 guard playback != nil else { return }
346 playback = nil
347 canvas.played(at: nil)
348 showMedia()
349 }
350
351 private func pauseMedia() {
352 if let recorder = Recorder.current, recorder.canvas === canvas {
353 recorder.pause(!(recorder.elapsed?.paused ?? false))
354 } else {
355 playback?.toggle()
356 }
357 showMedia()
358 }
359
360 private func stopMedia() {
361 if Recorder.current?.canvas === canvas {
362 Recorder.current?.stop()
363 } else {
364 stopPlayback()
365 }
366 }
367
368 /// The bar shows the clock of what records or plays, and a video's picture as it plays.
369 @objc private func showMedia() {
370 if let recorder = Recorder.current, recorder.canvas === canvas {
371 media.picture.isHidden = true
372 if recorder.saving {
373 media.showSaving(recorder.video ? "Saving video…" : "Saving audio…")
374 } else if let elapsed = recorder.elapsed {
375 let state = elapsed.paused ? "Paused" : recorder.video ? "Recording video" : "Recording"
376 media.show(
377 "\(state) \(MediaBar.clock(elapsed.ms))", recording: true,
378 paused: recorder.pausable ? elapsed.paused : nil)
379 }
380 } else if let playback {
381 if playback.failed {
382 stopPlayback()
383 return alert("Couldn't Play the Recording", "This kind of recording doesn't play on this device.")
384 }
385 let total = playback.durationMs.map { " / " + MediaBar.clock($0) } ?? ""
386 media.show(
387 "\(playback.request.name) \(MediaBar.clock(playback.ms))\(total)", recording: false,
388 paused: !playback.playing, playing: true)
389 if let picture = playback.picture() { media.picture.image = picture }
390 media.picture.isHidden = playback.request.video == nil
391 canvas.played(at: playback.ms)
392 } else {
393 media.isHidden = true
394 canvas.bottomInset = 0
395 mediaClock?.invalidate()
396 mediaClock = nil
397 return
398 }
399 // The line being written stays above the bar.
400 canvas.bottomInset = view.keyboardLayoutGuide.layoutFrame.minY - media.frame.minY
401 if mediaClock == nil {
402 mediaClock = Timer.scheduledTimer(withTimeInterval: 1.0 / 15, repeats: true) { [weak self] _ in
403 self?.showMedia()
404 }
405 }
406 }
407
292 @objc private func sectionChanged(_ notification: Notification) {408 @objc private func sectionChanged(_ notification: Notification) {
293 let flags = notification.userInfo?["flags"] as? UInt32 ?? 0409 let flags = notification.userInfo?["flags"] as? UInt32 ?? 0
294 if flags & Section.rejected != 0 {410 if flags & Section.rejected != 0 {
apps/ios/Snowbound/Recording.swift created+545
...@@ -0,0 +1,545 @@
1import AVFoundation
2import UIKit
3
4/// A recording `sb_view_play_request` asks to play: its sound file from `atMs`, and for a
5/// video the AVI file whose JPEG pictures `frames` gives, `frameUs` apart.
6struct PlayRequest: Decodable {
7 struct Video: Decodable {
8 let path: String
9 let frames: [[Int]]
10 let frameUs: UInt32
11 let size: [Int]
12 }
13 let name: String
14 let sound: String
15 let video: Video?
16 let atMs: UInt32
17}
18
19/// Record Audio and Record Video, stored as the desktop stores them: sound as mono PCM at
20/// `sb_recording_rate`, which the page keeps as IMA ADPCM, and video as a movie from the
21/// camera made into Motion JPEG AVI. Only one records at a time, and it keeps the page it
22/// records on until its file is there, whether or not the page is still shown.
23///
24/// Audio records on with the screen locked or the app in the background, under the audio
25/// background mode, as a lecture or meeting outlasts the screen's timeout; a call pauses it.
26/// The camera stops in the background, so a video recording then ends and is saved.
27final class Recorder: NSObject, AVCaptureFileOutputRecordingDelegate {
28 /// The recording under way or being saved.
29 private(set) static var current: Recorder?
30 /// Posted when the current recording pauses, resumes, saves or ends.
31 static let changed = Notification.Name("SnowboundRecordingChanged")
32
33 let video: Bool
34 let canvas: CanvasView
35 private let folder: URL
36 private var sound: AVAudioRecorder?
37 private var capture: AVCaptureSession?
38 private let output = AVCaptureMovieFileOutput()
39 #if targetEnvironment(simulator)
40 private var stub: StubCamera?
41 #endif
42 /// Stopped, its file being made and put on the page.
43 private(set) var saving = false
44
45 /// Whether video can be recorded here: with a camera, or the simulator's stand-in.
46 static var canRecordVideo: Bool {
47 #if targetEnvironment(simulator)
48 true
49 #else
50 AVCaptureDevice.default(for: .video) != nil
51 #endif
52 }
53
54 private init(video: Bool, canvas: CanvasView) {
55 self.video = video
56 self.canvas = canvas
57 folder = FileManager.default.temporaryDirectory.appendingPathComponent("Recording")
58 super.init()
59 }
60
61 /// Starts recording on `canvas` at its caret once the system allows the microphone, and
62 /// for video the camera; `failed` says why it could not.
63 static func start(video: Bool, on canvas: CanvasView, failed: @escaping (String, String) -> Void) {
64 guard current == nil else { return }
65 let title = video ? "Couldn't Record Video" : "Couldn't Record Audio"
66 AVAudioApplication.requestRecordPermission { microphone in
67 let begin = { (camera: Bool) in
68 DispatchQueue.main.async {
69 guard current == nil else { return }
70 if video && !camera {
71 return failed(title, "Allow Snowbound to use the camera in Settings, Privacy & Security, Camera.")
72 }
73 // A video without the microphone records silent, as on the desktop.
74 if !video && !microphone {
75 return failed(
76 title, "Allow Snowbound to use the microphone in Settings, Privacy & Security, Microphone.")
77 }
78 let recorder = Recorder(video: video, canvas: canvas)
79 do {
80 try recorder.begin(microphone: microphone)
81 } catch {
82 recorder.end()
83 return failed(title, "Try recording again.")
84 }
85 guard canvas.startRecording(video: video) else {
86 recorder.end()
87 return failed(title, "Try recording again.")
88 }
89 current = recorder
90 NotificationCenter.default.post(name: changed, object: recorder)
91 }
92 }
93 if video {
94 #if targetEnvironment(simulator)
95 begin(true)
96 #else
97 AVCaptureDevice.requestAccess(for: .video, completionHandler: begin)
98 #endif
99 } else {
100 begin(true)
101 }
102 }
103 }
104
105 private func begin(microphone: Bool) throws {
106 try? FileManager.default.removeItem(at: folder)
107 try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
108 let session = AVAudioSession.sharedInstance()
109 if microphone {
110 try session.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetoothHFP])
111 try session.setActive(true)
112 NotificationCenter.default.addObserver(
113 self, selector: #selector(interrupted), name: AVAudioSession.interruptionNotification, object: session)
114 }
115 #if targetEnvironment(simulator)
116 if video {
117 stub = try StubCamera(movie: folder.appendingPathComponent("camera.mov"))
118 }
119 #endif
120 if !video || isStubbed {
121 guard microphone else { return }
122 let settings: [String: Any] = [
123 AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: sb_recording_rate(), AVNumberOfChannelsKey: 1,
124 AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false,
125 ]
126 let sound = try AVAudioRecorder(url: soundFile, settings: settings)
127 guard sound.record() else { throw CocoaError(.fileWriteUnknown) }
128 self.sound = sound
129 return
130 }
131 let capture = AVCaptureSession()
132 capture.beginConfiguration()
133 if capture.canSetSessionPreset(.vga640x480) { capture.sessionPreset = .vga640x480 }
134 let devices = [AVCaptureDevice.default(for: .video)] + (microphone ? [AVCaptureDevice.default(for: .audio)] : [])
135 for (index, device) in devices.enumerated() {
136 guard let device, let input = try? AVCaptureDeviceInput(device: device), capture.canAddInput(input) else {
137 // Without the microphone the video records silent; without the camera, nothing.
138 if index == 0 { throw CocoaError(.fileWriteUnknown) }
139 continue
140 }
141 capture.addInput(input)
142 }
143 guard capture.canAddOutput(output) else { throw CocoaError(.fileWriteUnknown) }
144 capture.addOutput(output)
145 capture.commitConfiguration()
146 capture.startRunning()
147 output.startRecording(to: movieFile, recordingDelegate: self)
148 self.capture = capture
149 }
150
151 private var isStubbed: Bool {
152 #if targetEnvironment(simulator)
153 stub != nil
154 #else
155 false
156 #endif
157 }
158
159 private var soundFile: URL { folder.appendingPathComponent("sound.wav") }
160 private var movieFile: URL { folder.appendingPathComponent("camera.mov") }
161
162 /// How far the recording has got, its pauses left out, and whether it is paused.
163 var elapsed: (ms: Int64, paused: Bool)? { canvas.recording }
164
165 /// Whether Pause works: always but for a camera before iOS 18.
166 var pausable: Bool {
167 if capture == nil { return true }
168 if #available(iOS 18, *) { return true }
169 return false
170 }
171
172 func pause(_ paused: Bool) {
173 guard !saving, pausable else { return }
174 if paused {
175 sound?.pause()
176 } else {
177 sound?.record()
178 }
179 #if targetEnvironment(simulator)
180 stub?.paused = paused
181 #endif
182 if #available(iOS 18, *), capture != nil {
183 if paused { output.pauseRecording() } else { output.resumeRecording() }
184 }
185 canvas.pauseRecording(paused)
186 NotificationCenter.default.post(name: Self.changed, object: self)
187 }
188
189 /// A call or another app's audio takes the microphone: the recording pauses, and goes
190 /// on after where the system says it should.
191 @objc private func interrupted(_ notification: Notification) {
192 guard let raw = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt,
193 let type = AVAudioSession.InterruptionType(rawValue: raw)
194 else { return }
195 DispatchQueue.main.async { [self] in
196 switch type {
197 case .began where capture == nil: pause(true)
198 case .ended:
199 let options = (notification.userInfo?[AVAudioSessionInterruptionOptionKey] as? UInt)
200 .map(AVAudioSession.InterruptionOptions.init) ?? []
201 if options.contains(.shouldResume), capture == nil {
202 try? AVAudioSession.sharedInstance().setActive(true)
203 pause(false)
204 }
205 default: break
206 }
207 }
208 }
209
210 /// Stop: the recording's file is made and goes on the page where it started.
211 func stop() {
212 guard !saving else { return }
213 saving = true
214 canvas.pauseRecording(true)
215 NotificationCenter.default.post(name: Self.changed, object: self)
216 if capture != nil {
217 // The movie is saved once `fileOutput(_:didFinishRecordingTo:…)` hears so.
218 output.stopRecording()
219 return
220 }
221 sound?.stop()
222 #if targetEnvironment(simulator)
223 if let stub {
224 stub.finish { [self] in convert(movie: stub.movie) }
225 return
226 }
227 #endif
228 save(try? Data(contentsOf: soundFile))
229 }
230
231 /// The camera stopped: at Stop, or as the app left the screen or a call took the camera.
232 func fileOutput(
233 _ output: AVCaptureFileOutput, didFinishRecordingTo url: URL, from connections: [AVCaptureConnection],
234 error: Error?
235 ) {
236 let finished = (error as NSError?)?.userInfo[AVErrorRecordingSuccessfullyFinishedKey] as? Bool ?? (error == nil)
237 DispatchQueue.main.async { [self] in
238 saving = true
239 canvas.pauseRecording(true)
240 NotificationCenter.default.post(name: Self.changed, object: self)
241 capture?.stopRunning()
242 if finished { convert(movie: url) } else { save(nil) }
243 }
244 }
245
246 /// Makes the AVI file of `movie`, with the sound recorded beside it where there is.
247 private func convert(movie: URL) {
248 let sound = self.sound == nil ? movie : soundFile
249 // Finishing is not cut short by the app leaving the screen.
250 let task = UIApplication.shared.beginBackgroundTask()
251 Task.detached {
252 let avi = await avi(pictures: movie, sound: sound)
253 await MainActor.run {
254 self.save(avi)
255 UIApplication.shared.endBackgroundTask(task)
256 }
257 }
258 }
259
260 /// Puts `file` on the page, or with none forgets the recording.
261 private func save(_ file: Data?) {
262 canvas.finishRecording(file, video: video)
263 end()
264 Self.current = nil
265 NotificationCenter.default.post(name: Self.changed, object: self)
266 }
267
268 private func end() {
269 NotificationCenter.default.removeObserver(self)
270 capture?.stopRunning()
271 try? FileManager.default.removeItem(at: folder)
272 try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
273 }
274}
275
276/// The AVI file the desktop makes of a movie: the pictures of `pictures`' first video track
277/// and the sound of `sound`'s first audio track, mono PCM at `sb_recording_rate`.
278private func avi(pictures: URL, sound: URL) async -> Data? {
279 let movie = sb_movie_new()
280 let asset = AVURLAsset(url: pictures)
281 await read(asset, .video, [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]) { sample in
282 let time = CMSampleBufferGetPresentationTimeStamp(sample)
283 guard time.isValid, let buffer = CMSampleBufferGetImageBuffer(sample) else { return }
284 CVPixelBufferLockBaseAddress(buffer, .readOnly)
285 defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) }
286 guard let base = CVPixelBufferGetBaseAddress(buffer) else { return }
287 sb_movie_picture(
288 movie, UInt64(max(0, time.seconds) * 1_000_000), base.assumingMemoryBound(to: UInt8.self),
289 UInt32(CVPixelBufferGetWidth(buffer)), UInt32(CVPixelBufferGetHeight(buffer)),
290 CVPixelBufferGetBytesPerRow(buffer))
291 }
292 let pcm: [String: Any] = [
293 AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: sb_recording_rate(), AVNumberOfChannelsKey: 1,
294 AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false,
295 AVLinearPCMIsNonInterleaved: false,
296 ]
297 // A movie without sound has no sound track to read.
298 await read(AVURLAsset(url: sound), .audio, pcm) { sample in
299 guard let buffer = CMSampleBufferGetDataBuffer(sample) else { return }
300 let length = CMBlockBufferGetDataLength(buffer)
301 var bytes = [UInt8](repeating: 0, count: length)
302 if CMBlockBufferCopyDataBytes(buffer, atOffset: 0, dataLength: length, destination: &bytes) == noErr {
303 sb_movie_sound(movie, bytes, length)
304 }
305 }
306 let duration = (try? await asset.load(.duration))?.seconds ?? 0
307 var length = 0
308 guard let bytes = sb_movie_finish(movie, UInt64(max(0, duration) * 1_000_000), &length) else { return nil }
309 defer { sb_bytes_free(bytes, length) }
310 return Data(bytes: bytes, count: length)
311}
312
313/// Reads the first track of `kind` in `asset`, decoded as `settings` ask, a sample at a time.
314private func read(
315 _ asset: AVURLAsset, _ kind: AVMediaType, _ settings: [String: Any], _ sample: (CMSampleBuffer) -> Void
316) async {
317 guard let track = try? await asset.loadTracks(withMediaType: kind).first,
318 let reader = try? AVAssetReader(asset: asset)
319 else { return }
320 let output = AVAssetReaderTrackOutput(track: track, outputSettings: settings)
321 guard reader.canAdd(output) else { return }
322 reader.add(output)
323 guard reader.startReading() else { return }
324 while let next = output.copyNextSampleBuffer() {
325 sample(next)
326 }
327}
328
329#if targetEnvironment(simulator)
330/// The simulator's stand-in for a camera, which it has none of: moving colour bars and the
331/// time written as a movie, 15 pictures a second, while the microphone records beside it.
332private final class StubCamera {
333 let movie: URL
334 private let writer: AVAssetWriter
335 private let input: AVAssetWriterInput
336 private let adaptor: AVAssetWriterInputPixelBufferAdaptor
337 private var timer: Timer?
338 private var frame: Int64 = 0
339 var paused = false
340
341 init(movie: URL) throws {
342 self.movie = movie
343 writer = try AVAssetWriter(outputURL: movie, fileType: .mov)
344 input = AVAssetWriterInput(
345 mediaType: .video,
346 outputSettings: [AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: 640, AVVideoHeightKey: 480])
347 input.expectsMediaDataInRealTime = true
348 adaptor = AVAssetWriterInputPixelBufferAdaptor(
349 assetWriterInput: input,
350 sourcePixelBufferAttributes: [
351 kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
352 kCVPixelBufferWidthKey as String: 640, kCVPixelBufferHeightKey as String: 480,
353 ])
354 writer.add(input)
355 guard writer.startWriting() else { throw writer.error ?? CocoaError(.fileWriteUnknown) }
356 writer.startSession(atSourceTime: .zero)
357 timer = Timer.scheduledTimer(withTimeInterval: 1.0 / 15, repeats: true) { [weak self] _ in self?.shoot() }
358 }
359
360 private func shoot() {
361 guard !paused, input.isReadyForMoreMediaData, let pool = adaptor.pixelBufferPool else { return }
362 var made: CVPixelBuffer?
363 CVPixelBufferPoolCreatePixelBuffer(nil, pool, &made)
364 guard let buffer = made else { return }
365 CVPixelBufferLockBaseAddress(buffer, [])
366 let context = CGContext(
367 data: CVPixelBufferGetBaseAddress(buffer), width: 640, height: 480, bitsPerComponent: 8,
368 bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: CGColorSpaceCreateDeviceRGB(),
369 bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)
370 if let context {
371 let colors: [UIColor] = [.systemYellow, .systemTeal, .systemGreen, .systemPink, .systemRed, .systemBlue]
372 for bar in 0..<8 {
373 context.setFillColor(colors[(bar + Int(frame / 5)) % colors.count].cgColor)
374 context.fill(CGRect(x: bar * 80, y: 0, width: 80, height: 480))
375 }
376 UIGraphicsPushContext(context)
377 context.translateBy(x: 0, y: 480)
378 context.scaleBy(x: 1, y: -1)
379 let seconds = Double(frame) / 15
380 NSString(string: String(format: "%.1f s", seconds)).draw(
381 at: CGPoint(x: 24, y: 24),
382 withAttributes: [.font: UIFont.monospacedDigitSystemFont(ofSize: 72, weight: .bold), .foregroundColor: UIColor.black])
383 UIGraphicsPopContext()
384 }
385 CVPixelBufferUnlockBaseAddress(buffer, [])
386 adaptor.append(buffer, withPresentationTime: CMTime(value: frame, timescale: 15))
387 frame += 1
388 }
389
390 func finish(_ done: @escaping () -> Void) {
391 timer?.invalidate()
392 input.markAsFinished()
393 writer.endSession(atSourceTime: CMTime(value: frame, timescale: 15))
394 writer.finishWriting { DispatchQueue.main.async(execute: done) }
395 }
396}
397#endif
398
399/// Plays a recording: its sound, and a video's pictures in time with it.
400final class Playback {
401 let request: PlayRequest
402 let player: AVPlayer
403 private let pictures: Data?
404 private var shown = -1
405
406 init(_ request: PlayRequest) {
407 self.request = request
408 player = AVPlayer(url: URL(fileURLWithPath: request.sound))
409 pictures = request.video.flatMap { try? Data(contentsOf: URL(fileURLWithPath: $0.path), options: .alwaysMapped) }
410 try? AVAudioSession.sharedInstance().setCategory(.playback, mode: .spokenAudio)
411 try? AVAudioSession.sharedInstance().setActive(true)
412 player.seek(to: CMTime(value: CMTimeValue(request.atMs), timescale: 1000), toleranceBefore: .zero, toleranceAfter: .zero)
413 player.play()
414 }
415
416 deinit {
417 player.pause()
418 try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
419 }
420
421 /// Whether the player could not open the sound, as with OneNote's own Windows Media.
422 var failed: Bool { player.currentItem?.status == .failed }
423
424 var ms: Int64 { Int64(max(0, player.currentTime().seconds) * 1000) }
425
426 var durationMs: Int64? {
427 guard let duration = player.currentItem?.duration, duration.isNumeric else { return nil }
428 return Int64(duration.seconds * 1000)
429 }
430
431 var playing: Bool { player.rate != 0 }
432
433 func toggle() {
434 if playing {
435 player.pause()
436 } else {
437 if let durationMs, ms >= durationMs - 50 { player.seek(to: .zero) }
438 player.play()
439 }
440 }
441
442 /// The video's picture now, when it changed since last asked.
443 func picture() -> UIImage? {
444 guard let video = request.video, let pictures, !video.frames.isEmpty else { return nil }
445 let index = min(Int(ms * 1000 / Int64(max(1, video.frameUs))), video.frames.count - 1)
446 guard index != shown else { return nil }
447 shown = index
448 let range = video.frames[index]
449 guard range.count == 2, range[0] < range[1], range[1] <= pictures.count else { return nil }
450 return UIImage(data: pictures.subdata(in: range[0]..<range[1]))
451 }
452}
453
454/// The bar over the page's foot while something records or plays, as OneNote's Recording and
455/// Playback tabs: the clock, Pause, and Stop; a video's pictures above as it plays.
456final class MediaBar: UIView {
457 private let dot = UIImageView(image: UIImage(systemName: "record.circle.fill"))
458 private let label = UILabel()
459 private let pause = UIButton(configuration: .plain())
460 private let stop = UIButton(configuration: .plain())
461 let picture = UIImageView()
462 var onPause: (() -> Void)?
463 var onStop: (() -> Void)?
464
465 init() {
466 super.init(frame: .zero)
467 let blur = UIVisualEffectView(effect: UIBlurEffect(style: .systemThickMaterial))
468 blur.translatesAutoresizingMaskIntoConstraints = false
469 blur.layer.cornerRadius = 14
470 blur.clipsToBounds = true
471 addSubview(blur)
472 layer.shadowOpacity = 0.15
473 layer.shadowRadius = 8
474 layer.shadowOffset = CGSize(width: 0, height: 2)
475 dot.tintColor = .systemRed
476 dot.setContentHuggingPriority(.required, for: .horizontal)
477 label.font = .monospacedDigitSystemFont(ofSize: UIFont.preferredFont(forTextStyle: .subheadline).pointSize, weight: .medium)
478 label.adjustsFontForContentSizeCategory = true
479 label.lineBreakMode = .byTruncatingMiddle
480 label.accessibilityTraits.insert(.updatesFrequently)
481 stop.configuration?.image = UIImage(systemName: "stop.fill")
482 stop.accessibilityLabel = "Stop"
483 stop.addAction(UIAction { [weak self] _ in self?.onStop?() }, for: .primaryActionTriggered)
484 pause.addAction(UIAction { [weak self] _ in self?.onPause?() }, for: .primaryActionTriggered)
485 for button in [pause, stop] { button.setContentHuggingPriority(.required, for: .horizontal) }
486 picture.contentMode = .scaleAspectFit
487 picture.backgroundColor = .black
488 picture.layer.cornerRadius = 6
489 picture.clipsToBounds = true
490 picture.isHidden = true
491 let controls = UIStackView(arrangedSubviews: [dot, label, pause, stop])
492 controls.spacing = 8
493 controls.alignment = .center
494 let stack = UIStackView(arrangedSubviews: [picture, controls])
495 stack.axis = .vertical
496 stack.spacing = 8
497 stack.translatesAutoresizingMaskIntoConstraints = false
498 blur.contentView.addSubview(stack)
499 NSLayoutConstraint.activate([
500 blur.leadingAnchor.constraint(equalTo: leadingAnchor),
501 blur.trailingAnchor.constraint(equalTo: trailingAnchor),
502 blur.topAnchor.constraint(equalTo: topAnchor),
503 blur.bottomAnchor.constraint(equalTo: bottomAnchor),
504 stack.leadingAnchor.constraint(equalTo: blur.contentView.leadingAnchor, constant: 14),
505 stack.trailingAnchor.constraint(equalTo: blur.contentView.trailingAnchor, constant: -6),
506 stack.topAnchor.constraint(equalTo: blur.contentView.topAnchor, constant: 8),
507 stack.bottomAnchor.constraint(equalTo: blur.contentView.bottomAnchor, constant: -8),
508 picture.widthAnchor.constraint(lessThanOrEqualToConstant: 320),
509 ])
510 // Below required, as the stack gives the picture no height while it is hidden.
511 let aspect = picture.heightAnchor.constraint(equalTo: picture.widthAnchor, multiplier: 3.0 / 4)
512 aspect.priority = .defaultHigh
513 aspect.isActive = true
514 isHidden = true
515 }
516
517 required init?(coder: NSCoder) { fatalError() }
518
519 /// Shows `text`, the dot while recording, and Pause or Resume (or Play) where it works.
520 func show(_ text: String, recording: Bool, paused: Bool?, playing: Bool = false) {
521 isHidden = false
522 label.text = text
523 dot.isHidden = !recording
524 dot.alpha = paused == true ? 0.4 : 1
525 pause.isHidden = paused == nil
526 stop.isHidden = false
527 let resume = paused == true
528 pause.configuration?.image = UIImage(systemName: resume ? (playing ? "play.fill" : "record.circle") : "pause.fill")
529 pause.accessibilityLabel = resume ? (playing ? "Play" : "Resume") : "Pause"
530 }
531
532 /// Saving: only the clock's place says so.
533 func showSaving(_ text: String) {
534 show(text, recording: false, paused: nil)
535 stop.isHidden = true
536 }
537
538 /// `ms` as a clock: minutes and seconds, hours when there are some.
539 static func clock(_ ms: Int64) -> String {
540 let seconds = ms / 1000
541 return seconds >= 3600
542 ? String(format: "%d:%02d:%02d", seconds / 3600, seconds / 60 % 60, seconds % 60)
543 : String(format: "%d:%02d", seconds / 60, seconds % 60)
544 }
545}
apps/ios/Snowbound/Script.swift+13-3
...@@ -14,17 +14,22 @@ extension CanvasView {...@@ -14,17 +14,22 @@ extension CanvasView {
14 /// `tool:pen:N`, `tool:shape:N`, `tool:eraser` or `tool:lasso` gives it a tool, `picker` shows or hides the14 /// `tool:pen:N`, `tool:shape:N`, `tool:eraser` or `tool:lasso` gives it a tool, `picker` shows or hides the
15 /// drawing tools, `pencildoubletap` is the Pencil's double tap, and `deleteink` deletes15 /// drawing tools, `pencildoubletap` is the Pencil's double tap, and `deleteink` deletes
16 /// what the lasso picked; `rotate:landscape` or `rotate:portrait` turns the device; `done`16 /// what the lasso picked; `rotate:landscape` or `rotate:portrait` turns the device; `done`
17 /// ends editing, `reading` turns the reading view on or off, `tree` saves the view hierarchy to17 /// ends editing, `reading` turns the reading view on or off, `record:audio` or
18 /// `record:video` records, `pauserecording` pauses or resumes and `stoprecording` stops it,
19 /// `wait:N` waits N seconds more before the next step, `tree` saves the view hierarchy to
18 /// Documents/tree.txt, and `shot:a` the window to Documents/a.png, as a device has no20 /// Documents/tree.txt, and `shot:a` the window to Documents/a.png, as a device has no
19 /// screenshot command.21 /// screenshot command.
20 func runScript() {22 func runScript() {
21 // Only the first page shown runs it, not every page opened afterwards.23 // Only the first page shown runs it, not every page opened afterwards.
22 guard !scriptedPage, let script = ProcessInfo.processInfo.environment["SNOWBOUND_SCRIPT"] else { return }24 guard !scriptedPage, let script = ProcessInfo.processInfo.environment["SNOWBOUND_SCRIPT"] else { return }
23 scriptedPage = true25 scriptedPage = true
24 for (index, step) in script.split(separator: "|").enumerated() {26 var delay = 1.0
25 DispatchQueue.main.asyncAfter(deadline: .now() + 1 + Double(index) * 0.4) { [weak self] in27 for step in script.split(separator: "|") {
28 if step.hasPrefix("wait:") { delay += Double(step.dropFirst(5)) ?? 0 }
29 DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [weak self] in
26 self?.perform(String(step))30 self?.perform(String(step))
27 }31 }
32 delay += 0.4
28 }33 }
29 }34 }
3035
...@@ -90,6 +95,11 @@ extension CanvasView {...@@ -90,6 +95,11 @@ extension CanvasView {
90 let caret = caretRect(for: range.start)95 let caret = caretRect(for: range.start)
91 menu.presentEditMenu(with: UIEditMenuConfiguration(identifier: nil, sourcePoint: CGPoint(x: caret.midX, y: caret.minY)))96 menu.presentEditMenu(with: UIEditMenuConfiguration(identifier: nil, sourcePoint: CGPoint(x: caret.midX, y: caret.minY)))
92 case "done": _ = resignFirstResponder()97 case "done": _ = resignFirstResponder()
98 case "wait": break
99 case "record", "stoprecording":
100 let page = sequence(first: self as UIResponder, next: \.next).lazy.compactMap { $0 as? PageViewController }.first
101 if command == "record" { page?.record(video: argument == "video") } else { page?.stopRecording() }
102 case "pauserecording": Recorder.current.map { $0.pause(!($0.elapsed?.paused ?? false)) }
93 case "selectall": selectAll(nil)103 case "selectall": selectAll(nil)
94 case "reading": showReading(!readingShown)104 case "reading": showReading(!readingShown)
95 case "format": apply(UInt8(argument) ?? 0)105 case "format": apply(UInt8(argument) ?? 0)
arc/canvas.md+2-1
...@@ -167,7 +167,8 @@ scrolling it into view when it changes....@@ -167,7 +167,8 @@ scrolling it into view when it changes.
167167
168The host records audio as 16 kHz WAV and stores it as IMA ADPCM, and video as168The host records audio as 16 kHz WAV and stores it as IMA ADPCM, and video as
169AVI of Motion JPEG at 320 by 240 and 15 pictures a second with PCM sound, both169AVI of Motion JPEG at 320 by 240 and 15 pictures a second with PCM sound, both
170of which OneNote plays (`corpus/recording/video`). GStreamer writes that AVI170of which OneNote plays (`corpus/recording/video`); `canvas::recording` makes both,
171so every host stores the same bytes. GStreamer writes that AVI
171directly; on macOS a capture session records a movie that `AVAssetReader` reads172directly; on macOS a capture session records a movie that `AVAssetReader` reads
172back into it off the main thread after Stop, the transport saying so and the page173back into it off the main thread after Stop, the transport saying so and the page
173editable meanwhile. Mac OS X 10.6, without AVFoundation, opens recordings in the174editable meanwhile. Mac OS X 10.6, without AVFoundation, opens recordings in the
arc/platforms.md+7
...@@ -208,6 +208,13 @@ the same ops. Notebooks from Files...@@ -208,6 +208,13 @@ the same ops. Notebooks from Files
208are read and written under `NSFileCoordinator`, so file providers see every208are read and written under `NSFileCoordinator`, so file providers see every
209change.209change.
210210
211Record Audio and Record Video record through AVFoundation and hand `crates/mobile` the
212WAV file, or the camera's pictures and sound, which it stores as the desktop does. Audio
213keeps recording in the background and with the screen locked, under the `audio`
214background mode, as a lecture outlasts the screen's timeout; a call pauses it until the
215system says to go on. The camera stops in the background, which ends a video recording
216and saves it. A tap on a recording plays it with `AVPlayer` in a bar over the page's foot.
217
211## What stays shared, on purpose218## What stays shared, on purpose
212219
213- **Behaviour**: the editor, hit-testing, the placement grid, undo, conflicts220- **Behaviour**: the editor, hit-testing, the placement grid, undo, conflicts
crates/canvas/src/editor/recording.rs+5
...@@ -161,6 +161,11 @@ impl CanvasEditor {...@@ -161,6 +161,11 @@ impl CanvasEditor {
161 Ok(id)161 Ok(id)
162 }162 }
163163
164 /// Forgets the recording under way, whose file never came; its line stays.
165 pub fn cancel_recording(&mut self) {
166 self.recording = None;
167 }
168
164 /// Stops recording: `file`, the recording made, goes where it started, linked to its169 /// Stops recording: `file`, the recording made, goes where it started, linked to its
165 /// start, or at the caret when that place is gone.170 /// start, or at the caret when that place is gone.
166 pub fn finish_recording(171 pub fn finish_recording(
crates/canvas/src/gpu/page.rs+3
...@@ -203,6 +203,9 @@ fn between(from: [f32; 2], middle: [f32; 2], to: [f32; 2]) -> bool {...@@ -203,6 +203,9 @@ fn between(from: [f32; 2], middle: [f32; 2], to: [f32; 2]) -> bool {
203 (0.0..=1.0).contains(&along) && (mx * dy - my * dx).abs() / length.sqrt() <= 0.1203 (0.0..=1.0).contains(&along) && (mx * dy - my * dx).abs() / length.sqrt() <= 0.1
204}204}
205205
206/// The size OneNote 2010 shows a file's icon at, in points.
207pub const ICON_SIZE: [f32; 2] = [24.0, 24.0];
208
206const FILE_ICON: &str = include_str!("../../assets/file.svg");209const FILE_ICON: &str = include_str!("../../assets/file.svg");
207210
208/// The blank page for a file without an icon of its own, as the 32 pixel PNG OneNote 2010211/// The blank page for a file without an icon of its own, as the 32 pixel PNG OneNote 2010
crates/canvas/src/interaction/mod.rs+8
...@@ -1261,6 +1261,14 @@ impl PageView {...@@ -1261,6 +1261,14 @@ impl PageView {
1261 Some(Hit::File(id)) => {1261 Some(Hit::File(id)) => {
1262 self.set_object_focus(Some(ObjectFocus::File(id)));1262 self.set_object_focus(Some(ObjectFocus::File(id)));
1263 self.drag = None;1263 self.drag = None;
1264 // A finger plays a recording with a tap, having no play button to hover.
1265 if self.touch
1266 && let Some(file) = self.editor.attachment(id)
1267 && file.recording.is_some()
1268 {
1269 let file = file.clone();
1270 return Ok(Response::request(Request::Play { file, at_ms: 0 }));
1271 }
1264 if unit != SelectionUnit::Grapheme1272 if unit != SelectionUnit::Grapheme
1265 && let Some(file) = self.editor.attachment(id)1273 && let Some(file) = self.editor.attachment(id)
1266 {1274 {
crates/canvas/src/lib.rs+2
...@@ -17,6 +17,8 @@ pub mod gpu;...@@ -17,6 +17,8 @@ pub mod gpu;
17pub mod interaction;17pub mod interaction;
18#[cfg(feature = "pdf")]18#[cfg(feature = "pdf")]
19pub mod print;19pub mod print;
20#[cfg(feature = "gpu")]
21pub mod recording;
2022
21/// Where an object stored with `layout` lies; an unset coordinate is zero.23/// Where an object stored with `layout` lies; an unset coordinate is zero.
22pub(crate) fn origin(layout: &onestore::document::Layout) -> [f32; 2] {24pub(crate) fn origin(layout: &onestore::document::Layout) -> [f32; 2] {
crates/canvas/src/recording.rs created+424
...@@ -0,0 +1,424 @@
1//! Recordings as Snowbound stores them, which OneNote 2010 plays (`corpus/recording`):
2//! audio as 16 kHz IMA ADPCM WAV, video as the Motion JPEG AVI files of `video`.
3
4pub mod video;
5
6use onestore::page::{Attachment, Recording};
7
8/// The rate audio is recorded at: wideband speech, as OneNote's own 8 kHz WMA is narrowband.
9pub const RATE: u32 = 16_000;
10
11/// The line saying when a recording started, as OneNote writes it from the long `date` and
12/// short `time`.
13pub fn label(video: bool, date: &str, time: &str) -> String {
14 let kind = if video { "Video" } else { "Audio" };
15 format!("{kind} recording started: {time} {date}")
16}
17
18/// The file recording `id` on page `title` is stored as, from what was recorded: a mono
19/// 16-bit PCM WAV file, which is compressed, or a `video` AVI file. It has no icon yet.
20pub fn file(
21 id: [u8; 16],
22 title: &str,
23 video: bool,
24 recorded: Vec<u8>,
25) -> Result<Attachment, onestore::page::text::EditError> {
26 let (bytes, duration_ms) = if video {
27 let duration = video::Movie::parse(&recorded).map(|movie| movie.duration_ms());
28 (recorded, duration)
29 } else {
30 match compress(&recorded) {
31 Some((bytes, duration)) => (bytes, Some(duration)),
32 None => {
33 let duration = Wave::parse(&recorded).map(|wave| wave.duration_ms());
34 (recorded, duration)
35 }
36 }
37 };
38 let name: String = title
39 .chars()
40 .map(|c| {
41 if matches!(c, '/' | '\\' | '\0') {
42 '_'
43 } else {
44 c
45 }
46 })
47 .collect();
48 let name = if name.trim().is_empty() {
49 if video {
50 "Video recording"
51 } else {
52 "Audio recording"
53 }
54 .to_owned()
55 } else {
56 name
57 };
58 let extension = if video { "avi" } else { "wav" };
59 Ok(Attachment {
60 id: onestore::page::text::new_id()?,
61 filename: format!("{name}.{extension}"),
62 source_path: None,
63 size: Some(crate::gpu::page::ICON_SIZE),
64 layout: Default::default(),
65 bytes: Some(bytes.into()),
66 preview: None,
67 recording: Some(Recording {
68 id,
69 kind: if video { 2 } else { 1 },
70 duration_ms,
71 }),
72 tags: Vec::new(),
73 })
74}
75
76/// A recording's sound as a 16-bit PCM WAV file every platform's player decodes: IMA ADPCM
77/// decoded, a video's sound taken out of it. None for other files, which play as they are.
78pub fn sound(bytes: &[u8]) -> Option<Vec<u8>> {
79 let sound = match video::Movie::parse(bytes) {
80 Some(movie) => movie
81 .wave(bytes)
82 .unwrap_or_else(|| silence(movie.duration_ms())),
83 None => return decompress(bytes),
84 };
85 Some(decompress(&sound).unwrap_or(sound))
86}
87
88/// Silence as long as `ms`, the clock of a video without sound.
89fn silence(ms: u32) -> Vec<u8> {
90 wave(8_000, &vec![0; ms as usize * 8])
91}
92
93/// A mono 16-bit PCM WAV file of `samples` at `rate`, as the platform recorders write.
94pub fn wave(rate: u32, samples: &[i16]) -> Vec<u8> {
95 let data: Vec<u8> = samples.iter().flat_map(|s| s.to_le_bytes()).collect();
96 [
97 &b"RIFF"[..],
98 &(36 + data.len() as u32).to_le_bytes(),
99 b"WAVEfmt ",
100 &16_u32.to_le_bytes(),
101 &1_u16.to_le_bytes(),
102 &1_u16.to_le_bytes(),
103 &rate.to_le_bytes(),
104 &(rate * 2).to_le_bytes(),
105 &2_u16.to_le_bytes(),
106 &16_u16.to_le_bytes(),
107 b"data",
108 &(data.len() as u32).to_le_bytes(),
109 &data,
110 ]
111 .concat()
112}
113
114/// A recording for a file attached under `filename`, as OneNote 2010 makes one of every
115/// audio and video file it attaches, with the length of a WAV file's `bytes`.
116pub fn attached(filename: &str, bytes: &[u8]) -> Option<Recording> {
117 Some(Recording {
118 id: onestore::page::text::new_guid().ok()?,
119 kind: Recording::kind_of(filename)?,
120 duration_ms: Wave::parse(bytes).map(|wave| wave.duration_ms()),
121 })
122}
123
124/// The chunks of a RIFF WAVE file this module reads.
125struct Wave<'a> {
126 format: u16,
127 channels: u16,
128 rate: u32,
129 bytes_per_second: u32,
130 block: u16,
131 bits: u16,
132 /// The sample count a compressed file's `fact` chunk states.
133 samples: Option<u32>,
134 data: &'a [u8],
135}
136
137impl<'a> Wave<'a> {
138 fn parse(bytes: &'a [u8]) -> Option<Self> {
139 if bytes.get(..4)? != b"RIFF" || bytes.get(8..12)? != b"WAVE" {
140 return None;
141 }
142 let u16_at = |data: &[u8], at: usize| {
143 Some(u16::from_le_bytes(data.get(at..at + 2)?.try_into().ok()?))
144 };
145 let u32_at = |data: &[u8], at: usize| {
146 Some(u32::from_le_bytes(data.get(at..at + 4)?.try_into().ok()?))
147 };
148 let (mut fmt, mut samples, mut data) = (None, None, None);
149 let mut at = 12;
150 while at + 8 <= bytes.len() {
151 let size = u32_at(bytes, at + 4)? as usize;
152 let body = &bytes[at + 8..(at + 8).saturating_add(size).min(bytes.len())];
153 match &bytes[at..at + 4] {
154 b"fmt " => fmt = Some(body),
155 b"fact" => samples = u32_at(body, 0),
156 b"data" => data = Some(body),
157 _ => {}
158 }
159 at = at + 8 + size + size % 2;
160 }
161 let fmt = fmt?;
162 Some(Self {
163 format: u16_at(fmt, 0)?,
164 channels: u16_at(fmt, 2)?,
165 rate: u32_at(fmt, 4)?,
166 bytes_per_second: u32_at(fmt, 8)?,
167 block: u16_at(fmt, 12)?,
168 bits: u16_at(fmt, 14)?,
169 samples,
170 data: data?,
171 })
172 }
173
174 fn duration_ms(&self) -> u32 {
175 let ms = match self.samples.filter(|_| self.format != 1) {
176 Some(samples) => u64::from(samples) * 1000 / u64::from(self.rate.max(1)),
177 None => self.data.len() as u64 * 1000 / u64::from(self.bytes_per_second.max(1)),
178 };
179 u32::try_from(ms).unwrap_or(u32::MAX)
180 }
181}
182
183/// IMA ADPCM's step sizes and how each code moves along them.
184const STEPS: [i32; 89] = [
185 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66,
186 73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449,
187 494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, 2272,
188 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358, 5894, 6484, 7132, 7845, 8630, 9493,
189 10442, 11487, 12635, 13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767,
190];
191const INDEX_STEPS: [i32; 8] = [-1, -1, -1, -1, 2, 4, 6, 8];
192
193/// Bytes per block: Windows' IMA ADPCM codec's choice for 16 kHz.
194const BLOCK: usize = 512;
195/// Samples per block: the header's, then two per remaining byte.
196const BLOCK_SAMPLES: usize = (BLOCK - 4) * 2 + 1;
197
198/// A mono 16-bit PCM WAV file as 4-bit IMA ADPCM (WAVE_FORMAT_DVI_ADPCM), which Windows
199/// and macOS decode, a quarter of the size; and its length. None for other files.
200pub fn compress(wav: &[u8]) -> Option<(Vec<u8>, u32)> {
201 let wave = Wave::parse(wav)?;
202 if wave.format != 1 || wave.channels != 1 || wave.bits != 16 {
203 return None;
204 }
205 let samples: Vec<i16> = wave
206 .data
207 .chunks_exact(2)
208 .map(|pair| i16::from_le_bytes([pair[0], pair[1]]))
209 .collect();
210 let mut data = Vec::with_capacity(samples.len() / 2 + BLOCK);
211 let mut index = 0_i32;
212 for block in samples.chunks(BLOCK_SAMPLES) {
213 let mut predicted = i32::from(block[0]);
214 data.extend_from_slice(&block[0].to_le_bytes());
215 data.extend_from_slice(&[index as u8, 0]);
216 // A short last block is padded with its last sample; `fact` gives the true length.
217 let last = *block.last().unwrap();
218 let rest = block[1..]
219 .iter()
220 .copied()
221 .chain(std::iter::repeat(last))
222 .take(BLOCK_SAMPLES - 1)
223 .collect::<Vec<_>>();
224 for pair in rest.chunks_exact(2) {
225 let mut byte = 0;
226 for (shift, sample) in [0, 4].into_iter().zip(pair) {
227 let code = encode(i32::from(*sample), &mut predicted, &mut index);
228 byte |= code << shift;
229 }
230 data.push(byte);
231 }
232 }
233 let count = u32::try_from(samples.len()).ok()?;
234 let bytes_per_second = wave.rate * BLOCK as u32 / BLOCK_SAMPLES as u32;
235 let mut out = Vec::with_capacity(data.len() + 60);
236 let fmt: Vec<u8> = [
237 &0x11_u16.to_le_bytes()[..],
238 &1_u16.to_le_bytes(),
239 &wave.rate.to_le_bytes(),
240 &bytes_per_second.to_le_bytes(),
241 &(BLOCK as u16).to_le_bytes(),
242 &4_u16.to_le_bytes(),
243 &2_u16.to_le_bytes(),
244 &(BLOCK_SAMPLES as u16).to_le_bytes(),
245 ]
246 .concat();
247 let riff = 4 + (8 + fmt.len()) + (8 + 4) + (8 + data.len());
248 out.extend_from_slice(b"RIFF");
249 out.extend_from_slice(&u32::try_from(riff).ok()?.to_le_bytes());
250 out.extend_from_slice(b"WAVEfmt ");
251 out.extend_from_slice(&(fmt.len() as u32).to_le_bytes());
252 out.extend_from_slice(&fmt);
253 out.extend_from_slice(b"fact");
254 out.extend_from_slice(&4_u32.to_le_bytes());
255 out.extend_from_slice(&count.to_le_bytes());
256 out.extend_from_slice(b"data");
257 out.extend_from_slice(&u32::try_from(data.len()).ok()?.to_le_bytes());
258 out.extend_from_slice(&data);
259 let duration = u32::try_from(u64::from(count) * 1000 / u64::from(wave.rate.max(1))).ok()?;
260 Some((out, duration))
261}
262
263/// The 4-bit code nearest `sample` from `predicted`, which it and `index` then follow as a
264/// decoder does.
265fn encode(sample: i32, predicted: &mut i32, index: &mut i32) -> u8 {
266 let step = STEPS[*index as usize];
267 let mut difference = sample - *predicted;
268 let mut code = 0;
269 if difference < 0 {
270 code = 8;
271 difference = -difference;
272 }
273 let mut delta = step >> 3;
274 for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] {
275 if difference >= part {
276 code |= bit;
277 difference -= part;
278 delta += part;
279 }
280 }
281 *predicted = if code & 8 != 0 {
282 *predicted - delta
283 } else {
284 *predicted + delta
285 }
286 .clamp(-32768, 32767);
287 *index = (*index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88);
288 code
289}
290
291/// A mono IMA ADPCM WAV file as 16-bit PCM, which every platform's player decodes; none
292/// for other files.
293pub fn decompress(wav: &[u8]) -> Option<Vec<u8>> {
294 let wave = Wave::parse(wav).filter(|wave| wave.format == 0x11 && wave.channels == 1)?;
295 Some(self::wave(wave.rate, &decode(&wave)))
296}
297
298/// Each block's header sample and index, then its codes.
299fn decode(wave: &Wave<'_>) -> Vec<i16> {
300 let mut out = Vec::new();
301 for block in wave.data.chunks(usize::from(wave.block.max(4))) {
302 let Some(header) = block.get(..4) else {
303 break;
304 };
305 let mut predicted = i32::from(i16::from_le_bytes([header[0], header[1]]));
306 let mut index = i32::from(header[2]).clamp(0, 88);
307 out.push(predicted as i16);
308 for byte in &block[4..] {
309 for code in [byte & 15, byte >> 4] {
310 let step = STEPS[index as usize];
311 let mut delta = step >> 3;
312 for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] {
313 if code & bit != 0 {
314 delta += part;
315 }
316 }
317 predicted = if code & 8 != 0 {
318 predicted - delta
319 } else {
320 predicted + delta
321 }
322 .clamp(-32768, 32767);
323 index = (index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88);
324 out.push(predicted as i16);
325 }
326 }
327 }
328 if let Some(samples) = wave.samples {
329 out.truncate(samples as usize);
330 }
331 out
332}
333
334#[cfg(test)]
335mod tests {
336 use super::*;
337
338 /// Two seconds of a spoken-pitch chord.
339 fn tone() -> Vec<i16> {
340 (0..2 * RATE)
341 .map(|n| {
342 let t = n as f32 / RATE as f32;
343 let wave = (t * 220.0 * std::f32::consts::TAU).sin() * 0.5
344 + (t * 330.0 * std::f32::consts::TAU).sin() * 0.3;
345 (wave * 12_000.0) as i16
346 })
347 .collect()
348 }
349
350 #[test]
351 fn a_recording_compresses_to_ima_adpcm_that_decodes_close_to_what_was_heard() {
352 let samples = tone();
353 let (bytes, duration) = compress(&wave(RATE, &samples)).unwrap();
354 assert_eq!(duration, 2000);
355 let wave = Wave::parse(&bytes).unwrap();
356 assert_eq!(
357 (wave.format, wave.channels, wave.rate, wave.bits),
358 (0x11, 1, RATE, 4)
359 );
360 assert_eq!(wave.data.len() % BLOCK, 0);
361 assert_eq!(wave.duration_ms(), 2000);
362 assert!(bytes.len() * 3 < samples.len() * 2);
363 let decoded = decode(&wave);
364 assert_eq!(
365 Wave::parse(&decompress(&bytes).unwrap())
366 .unwrap()
367 .data
368 .len(),
369 decoded.len() * 2
370 );
371 assert_eq!(decoded.len(), samples.len());
372 let noise: f64 = samples
373 .iter()
374 .zip(&decoded)
375 .map(|(a, b)| (f64::from(*a) - f64::from(*b)).powi(2))
376 .sum();
377 let signal: f64 = samples.iter().map(|a| f64::from(*a).powi(2)).sum();
378 let snr = 10.0 * (signal / noise).log10();
379 assert!(snr > 20.0, "{snr} dB");
380 }
381
382 #[test]
383 fn only_mono_16_bit_pcm_compresses_and_attached_wav_files_know_their_length() {
384 let stereo = {
385 let mut bytes = wave(RATE, &[0; 64]);
386 bytes[22] = 2;
387 bytes
388 };
389 assert!(compress(&stereo).is_none());
390 assert!(compress(b"not a wave").is_none());
391 let silence = include_bytes!(
392 "../../../corpus/m6/native-features-01/notebook/fixture-data/silence.wav"
393 );
394 // OneNote 2010 stored 1000 ms for this file when attached (`corpus/recording`).
395 let recording = attached("silence.WAV", silence).unwrap();
396 assert_eq!((recording.kind, recording.duration_ms), (1, Some(1000)));
397 assert_eq!(attached("clip.wmv", b"").unwrap().kind, 2);
398 assert!(attached("notes.txt", b"").is_none());
399 }
400
401 #[test]
402 fn a_recording_is_stored_named_after_its_page_and_plays_as_pcm() {
403 let id = [7; 16];
404 let audio = file(id, "Lecture 3/4", false, wave(RATE, &tone())).unwrap();
405 assert_eq!(audio.filename, "Lecture 3_4.wav");
406 let recording = audio.recording.unwrap();
407 assert_eq!(
408 (recording.id, recording.kind, recording.duration_ms),
409 (id, 1, Some(2000))
410 );
411 let stored = audio.bytes.unwrap();
412 assert_eq!(Wave::parse(&stored).unwrap().format, 0x11);
413 let played = sound(&stored).unwrap();
414 assert_eq!(Wave::parse(&played).unwrap().format, 1);
415 assert_eq!(
416 file(id, " ", true, Vec::new()).unwrap().filename,
417 "Video recording.avi"
418 );
419 assert_eq!(
420 label(false, "Tuesday, September 29, 2026", "5:18 PM"),
421 "Audio recording started: 5:18 PM Tuesday, September 29, 2026"
422 );
423 }
424}
crates/canvas/src/recording/video.rs created+422
...@@ -0,0 +1,422 @@
1//! Video recordings as AVI files of Motion JPEG pictures and PCM sound, which OneNote 2010
2//! plays and lists as recordings (`corpus/recording/video`), and reading them back to play.
3
4use std::ops::Range;
5
6/// Pictures a second in a recording: OneNote 2010's own profiles record at 15.
7pub const FPS: u32 = 15;
8/// A recording's picture size, OneNote 2010's own.
9pub const SIZE: [u32; 2] = [320, 240];
10
11/// Interleaved signed 16-bit little-endian sound.
12pub struct Pcm {
13 pub rate: u32,
14 pub channels: u16,
15 pub data: Vec<u8>,
16}
17
18fn chunk(out: &mut Vec<u8>, id: &[u8; 4], body: &[u8]) {
19 out.extend_from_slice(id);
20 out.extend_from_slice(&(body.len() as u32).to_le_bytes());
21 out.extend_from_slice(body);
22 if body.len() % 2 == 1 {
23 out.push(0);
24 }
25}
26
27fn list(kind: &[u8; 4], body: &[u8]) -> Vec<u8> {
28 let mut out = Vec::with_capacity(body.len() + 12);
29 chunk(&mut out, b"LIST", &[&kind[..], body].concat());
30 out
31}
32
33fn words(values: &[u32]) -> Vec<u8> {
34 values
35 .iter()
36 .flat_map(|value| value.to_le_bytes())
37 .collect()
38}
39
40/// An AVI file of `frames`, JPEG pictures of `size` shown `FPS` a second, where an empty one
41/// repeats the picture before it, and `sound` alongside. None past AVI's 4 GiB.
42pub fn write(size: [u32; 2], frames: &[Vec<u8>], sound: &Pcm) -> Option<Vec<u8>> {
43 let block = u32::from(sound.channels) * 2;
44 let bytes_per_second = sound.rate * block;
45 let samples = sound.data.len() as u64 / u64::from(block);
46 let count = frames.len() as u64;
47 // The sound up to frame `index`, whole samples.
48 let sound_at = |index: u64| -> usize {
49 let before = if index >= count {
50 samples
51 } else {
52 (index * u64::from(sound.rate) / u64::from(FPS)).min(samples)
53 };
54 (before * u64::from(block)) as usize
55 };
56 let mut movi = b"movi".to_vec();
57 let mut index = Vec::new();
58 let [mut largest_frame, mut largest_sound] = [0, 0];
59 for (at, frame) in frames.iter().enumerate() {
60 let entries: [(&[u8; 4], &[u8]); 2] = [
61 (b"00dc", frame),
62 (
63 b"01wb",
64 &sound.data[sound_at(at as u64)..sound_at(at as u64 + 1)],
65 ),
66 ];
67 for (id, body) in entries {
68 if id == b"01wb" && body.is_empty() {
69 continue;
70 }
71 // Keyframes: every picture and all sound; a repeat is none.
72 let flags: u32 = if body.is_empty() { 0 } else { 0x10 };
73 index.extend_from_slice(id);
74 index.extend(words(&[
75 flags,
76 u32::try_from(movi.len()).ok()?,
77 body.len() as u32,
78 ]));
79 chunk(&mut movi, id, body);
80 let largest = if id == b"00dc" {
81 &mut largest_frame
82 } else {
83 &mut largest_sound
84 };
85 *largest = (*largest).max(body.len() as u32);
86 }
87 }
88 let [width, height] = size;
89 let frame_count = u32::try_from(count).ok()?;
90 let rect = [0u16, 0, width as u16, height as u16]
91 .iter()
92 .flat_map(|side| side.to_le_bytes())
93 .collect::<Vec<u8>>();
94 let mut video = Vec::new();
95 let strh = [
96 &b"vidsMJPG"[..],
97 &words(&[0, 0, 0, 1, FPS, 0, frame_count, largest_frame, u32::MAX, 0]),
98 &rect,
99 ]
100 .concat();
101 chunk(&mut video, b"strh", &strh);
102 let strf = [
103 words(&[40, width, height]),
104 [&1u16.to_le_bytes()[..], &24u16.to_le_bytes()].concat(),
105 b"MJPG".to_vec(),
106 words(&[width * height * 3, 0, 0, 0, 0]),
107 ]
108 .concat();
109 chunk(&mut video, b"strf", &strf);
110 let mut streams = list(b"strl", &video);
111 if samples > 0 {
112 let mut audio = Vec::new();
113 let strh = [
114 &b"auds\0\0\0\0"[..],
115 &words(&[
116 0,
117 0,
118 0,
119 block,
120 bytes_per_second,
121 0,
122 u32::try_from(samples).ok()?,
123 largest_sound,
124 u32::MAX,
125 block,
126 ]),
127 &[0; 8],
128 ]
129 .concat();
130 chunk(&mut audio, b"strh", &strh);
131 let strf = [
132 &1u16.to_le_bytes()[..],
133 &sound.channels.to_le_bytes(),
134 &sound.rate.to_le_bytes(),
135 &bytes_per_second.to_le_bytes(),
136 &(block as u16).to_le_bytes(),
137 &16u16.to_le_bytes(),
138 &0u16.to_le_bytes(),
139 ]
140 .concat();
141 chunk(&mut audio, b"strf", &strf);
142 streams.extend(list(b"strl", &audio));
143 }
144 let mut avih = Vec::new();
145 chunk(
146 &mut avih,
147 b"avih",
148 &words(&[
149 1_000_000 / FPS,
150 largest_frame
151 .saturating_add(largest_sound)
152 .saturating_mul(FPS),
153 0,
154 // AVIF_HASINDEX | AVIF_ISINTERLEAVED
155 0x110,
156 frame_count,
157 0,
158 if samples > 0 { 2 } else { 1 },
159 largest_frame.max(largest_sound),
160 width,
161 height,
162 0,
163 0,
164 0,
165 0,
166 ]),
167 );
168 let header = list(b"hdrl", &[avih, streams].concat());
169 let mut body = b"AVI ".to_vec();
170 body.extend(header);
171 let movi_size = u32::try_from(movi.len()).ok()?;
172 body.extend_from_slice(b"LIST");
173 body.extend_from_slice(&movi_size.to_le_bytes());
174 body.extend(movi);
175 chunk(&mut body, b"idx1", &index);
176 u32::try_from(body.len()).ok()?;
177 let mut out = Vec::with_capacity(body.len() + 8);
178 chunk(&mut out, b"RIFF", &body);
179 Some(out)
180}
181
182/// An AVI file's pictures and sound, as byte ranges of the file.
183pub struct Movie {
184 /// Each picture's JPEG; an empty range repeats the one before.
185 pub frames: Vec<Range<usize>>,
186 pub frame_us: u32,
187 /// The sound's WAVEFORMATEX and its data, in order.
188 pub sound: Option<(Range<usize>, Vec<Range<usize>>)>,
189}
190
191impl Movie {
192 /// A Motion JPEG AVI file's contents; none for other files.
193 pub fn parse(bytes: &[u8]) -> Option<Self> {
194 let mut movie = Self {
195 frames: Vec::new(),
196 frame_us: 0,
197 sound: None,
198 };
199 let mut kinds = Vec::new();
200 let mut first = true;
201 for (id, body) in Chunks::new(bytes, 0..bytes.len()) {
202 let kind = bytes.get(body.start..body.start + 4)?;
203 if id != *b"RIFF" || (kind != b"AVI " && !(kind == b"AVIX" && !first)) {
204 return (!first).then_some(movie).filter(|movie| movie.frame_us > 0);
205 }
206 first = false;
207 movie.walk(bytes, body.start + 4..body.end, &mut kinds)?;
208 }
209 (movie.frame_us > 0).then_some(movie)
210 }
211
212 fn walk(&mut self, bytes: &[u8], range: Range<usize>, kinds: &mut Vec<[u8; 4]>) -> Option<()> {
213 for (id, body) in Chunks::new(bytes, range) {
214 let u32_at = |at: usize| {
215 Some(u32::from_le_bytes(
216 bytes
217 .get(body.start + at..body.start + at + 4)?
218 .try_into()
219 .ok()?,
220 ))
221 };
222 match &id {
223 b"LIST" => {
224 let inner = body.start + 4..body.end;
225 self.walk(bytes, inner, kinds)?;
226 }
227 b"strh" => {
228 let kind: [u8; 4] = bytes.get(body.start..body.start + 4)?.try_into().ok()?;
229 if &kind == b"vids" {
230 let [scale, rate] = [u32_at(20)?, u32_at(24)?];
231 self.frame_us =
232 u32::try_from(u64::from(scale) * 1_000_000 / u64::from(rate.max(1)))
233 .ok()?;
234 }
235 kinds.push(kind);
236 }
237 b"strf" if kinds.last() == Some(b"vids") => {
238 if bytes.get(body.start + 16..body.start + 20)? != b"MJPG" {
239 return None;
240 }
241 }
242 b"strf" if kinds.last() == Some(b"auds") => {
243 self.sound = Some((body.clone(), Vec::new()));
244 }
245 [a, b, c, d] if a.is_ascii_digit() && b.is_ascii_digit() => {
246 let stream = usize::from((a - b'0') * 10 + (b - b'0'));
247 match (kinds.get(stream).map(|kind| &kind[..]), [c, d]) {
248 (Some(b"vids"), [b'd', b'c' | b'b']) => self.frames.push(body),
249 (Some(b"auds"), [b'w', b'b']) => {
250 if let Some((_, data)) = &mut self.sound {
251 data.push(body);
252 }
253 }
254 _ => {}
255 }
256 }
257 _ => {}
258 }
259 }
260 Some(())
261 }
262
263 pub fn duration_ms(&self) -> u32 {
264 let ms = (self.frames.len() as u64 * u64::from(self.frame_us)).div_ceil(1000);
265 u32::try_from(ms).unwrap_or(u32::MAX)
266 }
267
268 /// The picture showing `at_ms` in: the last stored one at or before it.
269 pub fn frame(&self, at_ms: u32) -> Option<Range<usize>> {
270 let at = (u64::from(at_ms) * 1000 / u64::from(self.frame_us.max(1))) as usize;
271 self.frames[..=at.min(self.frames.len().checked_sub(1)?)]
272 .iter()
273 .rev()
274 .find(|frame| !frame.is_empty())
275 .cloned()
276 }
277
278 /// The sound as a WAV file, or none where the file has none.
279 pub fn wave(&self, bytes: &[u8]) -> Option<Vec<u8>> {
280 let (format, data) = self.sound.as_ref()?;
281 let format = &bytes[format.clone()];
282 let length: usize = data.iter().map(ExactSizeIterator::len).sum();
283 let mut out = Vec::with_capacity(length + 64);
284 out.extend_from_slice(b"RIFF");
285 out.extend_from_slice(&((4 + 8 + format.len() + 8 + length) as u32).to_le_bytes());
286 out.extend_from_slice(b"WAVE");
287 chunk(&mut out, b"fmt ", format);
288 out.extend_from_slice(b"data");
289 out.extend_from_slice(&(length as u32).to_le_bytes());
290 for range in data {
291 out.extend_from_slice(&bytes[range.clone()]);
292 }
293 Some(out)
294 }
295}
296
297/// The chunks in `range` of a RIFF file: each one's id and body.
298struct Chunks<'a> {
299 bytes: &'a [u8],
300 at: usize,
301 end: usize,
302}
303
304impl<'a> Chunks<'a> {
305 fn new(bytes: &'a [u8], range: Range<usize>) -> Self {
306 Self {
307 bytes,
308 at: range.start,
309 end: range.end.min(bytes.len()),
310 }
311 }
312}
313
314impl Iterator for Chunks<'_> {
315 type Item = ([u8; 4], Range<usize>);
316
317 fn next(&mut self) -> Option<Self::Item> {
318 let header = self.bytes.get(self.at..self.at + 8)?;
319 if self.at + 8 > self.end {
320 return None;
321 }
322 let id: [u8; 4] = header[..4].try_into().ok()?;
323 let size = u32::from_le_bytes(header[4..].try_into().ok()?) as usize;
324 let start = self.at + 8;
325 let end = start.saturating_add(size).min(self.end);
326 self.at = end + size % 2;
327 Some((id, start..end))
328 }
329}
330
331/// Pictures taken at uneven moments, as a camera takes them, made the `FPS` a second of a
332/// recording: each one shows until the next, fitted to `SIZE`.
333#[derive(Default)]
334pub struct Pictures {
335 frames: Vec<Vec<u8>>,
336 last: Option<image::RgbImage>,
337 /// Whether the last picture is stored yet, which the frames after it repeat.
338 stored: bool,
339}
340
341impl Pictures {
342 /// The picture taken `at_us` microseconds in; the first shows from the start.
343 pub fn take(&mut self, at_us: u64, picture: image::RgbImage) {
344 self.fill(at_us);
345 self.last = Some(fit(picture));
346 self.stored = false;
347 }
348
349 /// A picture as Core Video gives one: `height` rows of `stride` bytes of BGRA pixels.
350 pub fn take_bgra(
351 &mut self,
352 at_us: u64,
353 [width, height]: [usize; 2],
354 stride: usize,
355 bgra: &[u8],
356 ) {
357 let rgb = bgra
358 .chunks_exact(stride.max(1))
359 .take(height)
360 .filter_map(|row| row.get(..width * 4))
361 .flat_map(|row| {
362 row.chunks_exact(4)
363 .flat_map(|pixel| [pixel[2], pixel[1], pixel[0]])
364 })
365 .collect();
366 if let Some(picture) = image::RgbImage::from_raw(width as u32, height as u32, rgb) {
367 self.take(at_us, picture);
368 }
369 }
370
371 /// The frames of a recording `end_us` microseconds long.
372 pub fn finish(mut self, end_us: u64) -> Vec<Vec<u8>> {
373 self.fill(end_us);
374 self.frames
375 }
376
377 /// Frames up to `until_us`, of the last picture.
378 fn fill(&mut self, until_us: u64) {
379 let Some(last) = &self.last else {
380 return;
381 };
382 while (self.frames.len() as u64) * 1_000_000 / u64::from(FPS) < until_us {
383 let frame = if self.stored {
384 Vec::new()
385 } else {
386 self.stored = true;
387 let mut jpeg = Vec::new();
388 let encoded = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 70)
389 .encode_image(last);
390 if encoded.is_err() {
391 jpeg.clear();
392 }
393 jpeg
394 };
395 self.frames.push(frame);
396 }
397 }
398}
399
400/// `picture` scaled to fit `SIZE`, centred on black.
401fn fit(picture: image::RgbImage) -> image::RgbImage {
402 let [width, height] = SIZE;
403 if picture.dimensions() == (width, height) {
404 return picture;
405 }
406 let scale =
407 (width as f32 / picture.width() as f32).min(height as f32 / picture.height() as f32);
408 let fitted = image::imageops::resize(
409 &picture,
410 ((picture.width() as f32 * scale) as u32).clamp(1, width),
411 ((picture.height() as f32 * scale) as u32).clamp(1, height),
412 image::imageops::FilterType::Triangle,
413 );
414 let mut out = image::RgbImage::new(width, height);
415 image::imageops::overlay(
416 &mut out,
417 &fitted,
418 i64::from((width - fitted.width()) / 2),
419 i64::from((height - fitted.height()) / 2),
420 );
421 out
422}
crates/mobile/include/snowbound.h+14
...@@ -109,6 +109,20 @@ int8_t sb_view_date_request(View *view, int64_t *seconds);...@@ -109,6 +109,20 @@ int8_t sb_view_date_request(View *view, int64_t *seconds);
109bool sb_view_change_date(View *view, int64_t seconds, const char *date, const char *time);109bool sb_view_change_date(View *view, int64_t seconds, const char *date, const char *time);
110bool sb_view_insert_picture(View *view, const uint8_t *bytes, size_t length, float width, float height);110bool sb_view_insert_picture(View *view, const uint8_t *bytes, size_t length, float width, float height);
111111
112typedef struct Movie Movie;
113uint32_t sb_recording_rate(void);
114bool sb_view_start_recording(View *view, bool video, const char *date, const char *time);
115int64_t sb_view_recording(const View *view, bool *paused);
116void sb_view_pause_recording(View *view, bool paused);
117bool sb_view_finish_recording(View *view, const uint8_t *bytes, size_t length, bool video);
118char *sb_view_play_request(View *view, const char *folder);
119bool sb_view_played(View *view, int64_t at_ms);
120Movie *sb_movie_new(void);
121void sb_movie_picture(Movie *movie, uint64_t at_us, const uint8_t *bgra, uint32_t width, uint32_t height,
122 size_t stride);
123void sb_movie_sound(Movie *movie, const uint8_t *pcm, size_t length);
124uint8_t *sb_movie_finish(Movie *movie, uint64_t duration_us, size_t *length);
125
112uint32_t sb_text_length(const View *view);126uint32_t sb_text_length(const View *view);
113char *sb_text(const View *view, uint32_t start, uint32_t end);127char *sb_text(const View *view, uint32_t start, uint32_t end);
114void sb_selection(const View *view, uint32_t range[2]);128void sb_selection(const View *view, uint32_t range[2]);
crates/mobile/src/lib.rs+15-4
...@@ -8,6 +8,7 @@...@@ -8,6 +8,7 @@
8//! the host redraws and rereads `sb_view_content`.8//! the host redraws and rereads `sb_view_content`.
99
10mod library;10mod library;
11mod recording;
11#[cfg(target_os = "ios")]12#[cfg(target_os = "ios")]
12mod spell;13mod spell;
1314
...@@ -119,6 +120,10 @@ struct Canvas {...@@ -119,6 +120,10 @@ struct Canvas {
119 scale: f32,120 scale: f32,
120 /// The page date field a tap asked to change, for the host's picker.121 /// The page date field a tap asked to change, for the host's picker.
121 asked: Option<DateField>,122 asked: Option<DateField>,
123 /// The recording a tap asked to play, and from when.
124 play: Option<(onestore::page::Attachment, u32)>,
125 /// The recording playing, whose linked notes See Playback follows.
126 playing: Option<[u8; 16]>,
122}127}
123128
124impl Canvas {129impl Canvas {
...@@ -156,6 +161,8 @@ impl Canvas {...@@ -156,6 +161,8 @@ impl Canvas {
156 paper: Paper::WHITE,161 paper: Paper::WHITE,
157 scale,162 scale,
158 asked: None,163 asked: None,
164 play: None,
165 playing: None,
159 })166 })
160 }167 }
161168
...@@ -263,12 +270,16 @@ impl Canvas {...@@ -263,12 +270,16 @@ impl Canvas {
263 Ok(self.respond(pressed) || changed)270 Ok(self.respond(pressed) || changed)
264 }271 }
265272
266 /// Whether `response` moved the page, keeping the date field it asks to change.273 /// Whether `response` moved the page, keeping the date field it asks to change or the
274 /// recording it asks to play.
267 fn respond(&mut self, response: Response) -> bool {275 fn respond(&mut self, response: Response) -> bool {
268 if let Some(Request::EditDate(field)) = response.request {276 let changed = response.changed || response.moved;
269 self.asked = Some(field);277 match response.request {
278 Some(Request::EditDate(field)) => self.asked = Some(field),
279 Some(Request::Play { file, at_ms }) => self.play = Some((file, at_ms)),
280 _ => {}
270 }281 }
271 moved(response)282 changed
272 }283 }
273284
274 fn drag(&mut self, point: [f32; 2]) -> Result<bool> {285 fn drag(&mut self, point: [f32; 2]) -> Result<bool> {
crates/mobile/src/recording.rs created+274
...@@ -0,0 +1,274 @@
1//! Record Audio, Record Video and playback, stored as the desktop stores them
2//! (`canvas::recording`): the host records with the platform's devices and hands over a PCM
3//! WAV file, or a video's pictures and sound as `Movie` takes them; a recording plays from
4//! the files `sb_view_play_request` writes.
5
6use super::*;
7use canvas::recording::{self, video};
8use std::path::Path;
9
10impl Canvas {
11 /// Starts recording at the caret, its line saying when from the long `date` and short
12 /// `time`; whether it started.
13 pub(crate) fn start_recording(&mut self, video: bool, date: &str, time: &str) -> Result<bool> {
14 let label = recording::label(video, date, time);
15 Ok(self.page.start_recording(&label)?.0.is_some())
16 }
17
18 /// Puts the recording made, `recorded`, where it started: a PCM WAV file, or with
19 /// `video` an AVI file from `Movie`. None forgets the recording, whose file never came.
20 pub(crate) fn finish_recording(
21 &mut self,
22 recorded: Option<Vec<u8>>,
23 video: bool,
24 ) -> Result<bool> {
25 let editor = &mut self.page.editor;
26 let Some(id) = editor.recording() else {
27 return Ok(false);
28 };
29 let Some(recorded) = recorded else {
30 editor.cancel_recording();
31 return Ok(false);
32 };
33 let file = recording::file(id, &editor.page()?.title, video, recorded)?;
34 Ok(moved(self.page.finish_recording(file)?))
35 }
36
37 /// The recording a tap asked to play, taken, with its files written to `folder` to play
38 /// from: see `sb_view_play_request`.
39 pub(crate) fn take_play(&mut self, folder: &Path) -> Result<Option<serde_json::Value>> {
40 let Some((file, at)) = self.play.take() else {
41 return Ok(None);
42 };
43 let bytes = file.bytes.clone().ok_or("The recording has no data")?;
44 let stem = Path::new(&file.filename)
45 .file_stem()
46 .map_or_else(String::new, |stem| stem.to_string_lossy().into_owned());
47 std::fs::create_dir_all(folder)?;
48 let sound = match recording::sound(&bytes) {
49 Some(wave) => {
50 let path = folder.join(format!("{stem}.wav"));
51 std::fs::write(&path, wave)?;
52 path
53 }
54 None => {
55 let path = folder.join(file.filename.replace('/', "_"));
56 std::fs::write(&path, &bytes)?;
57 path
58 }
59 };
60 let movie = video::Movie::parse(&bytes)
61 .map(|movie| -> Result<_> {
62 let path = folder.join(format!("{stem}.avi"));
63 std::fs::write(&path, &bytes)?;
64 // An empty frame repeats the picture before it.
65 let mut shown = [0; 2];
66 let frames: Vec<[usize; 2]> = movie
67 .frames
68 .iter()
69 .map(|range| {
70 if !range.is_empty() {
71 shown = [range.start, range.end];
72 }
73 shown
74 })
75 .collect();
76 Ok(serde_json::json!({
77 "path": path,
78 "frames": frames,
79 "frame_us": movie.frame_us,
80 "size": video::SIZE,
81 }))
82 })
83 .transpose()?;
84 self.playing = file.recording.map(|recording| recording.id);
85 Ok(Some(serde_json::json!({
86 "name": stem,
87 "sound": sound,
88 "video": movie,
89 "at_ms": at,
90 })))
91 }
92
93 /// See Playback: highlights the note linked last at or before `at_ms` into the recording
94 /// playing, or with none ends playback.
95 pub(crate) fn played(&mut self, at_ms: Option<u32>) -> Result<bool> {
96 if at_ms.is_none() {
97 self.playing = None;
98 }
99 let note = self
100 .playing
101 .zip(at_ms)
102 .and_then(|(id, at)| self.page.editor.played_note(id, at))
103 .and_then(|note| canvas::search::paragraph_match(&self.page.editor, note));
104 Ok(moved(self.page.set_played(note)?))
105 }
106}
107
108/// Starts recording audio, or with `video` video, at the caret, its line saying when from
109/// the long `date` and short `time`; whether it started.
110///
111/// # Safety
112/// `date` and `time` are NUL-terminated UTF-8.
113#[unsafe(no_mangle)]
114pub unsafe extern "C" fn sb_view_start_recording(
115 view: &mut View,
116 video: bool,
117 date: *const c_char,
118 time: *const c_char,
119) -> bool {
120 if view.read_only || view.reading.is_some() {
121 return false;
122 }
123 let started = view
124 .canvas
125 .start_recording(video, &string(date), &string(time));
126 let started = report(started).unwrap_or(false);
127 view.stored(Ok(started));
128 started
129}
130
131/// How far the recording under way has got in milliseconds, its pauses left out, or -1;
132/// `paused` gets whether it is paused.
133#[unsafe(no_mangle)]
134pub extern "C" fn sb_view_recording(view: &View, paused: &mut bool) -> i64 {
135 let editor = &view.canvas.page.editor;
136 *paused = editor.recording_paused();
137 editor.recording_ms().map_or(-1, i64::from)
138}
139
140/// Pauses or resumes the recording: nothing written while it is paused links to it.
141#[unsafe(no_mangle)]
142pub extern "C" fn sb_view_pause_recording(view: &mut View, paused: bool) {
143 view.canvas.page.editor.pause_recording(paused);
144}
145
146/// Stops recording: the file recorded, `length` bytes of mono 16-bit PCM WAV or with
147/// `video` the AVI file `sb_movie_finish` made, goes where the recording started. Null
148/// `bytes` forgets the recording.
149///
150/// # Safety
151/// `bytes` is null or holds `length` bytes.
152#[unsafe(no_mangle)]
153pub unsafe extern "C" fn sb_view_finish_recording(
154 view: &mut View,
155 bytes: *const u8,
156 length: usize,
157 video: bool,
158) -> bool {
159 // SAFETY: the caller's contract.
160 let recorded = (!bytes.is_null()).then(|| unsafe { std::slice::from_raw_parts(bytes, length) });
161 let result = view
162 .canvas
163 .finish_recording(recorded.map(<[u8]>::to_vec), video);
164 view.stored(result)
165}
166
167/// The recording the last tap asked to play, taken, as JSON: its `name`, the `sound` file
168/// written to `folder` that the platform's player plays, from `at_ms`; and for a video its
169/// `video` file `path`, the byte range of the JPEG picture each of its `frames` shows,
170/// `frame_us` apart, and their `size`. Null when none was asked for.
171///
172/// # Safety
173/// `folder` is NUL-terminated UTF-8.
174#[unsafe(no_mangle)]
175pub unsafe extern "C" fn sb_view_play_request(
176 view: &mut View,
177 folder: *const c_char,
178) -> *mut c_char {
179 let request = view.canvas.take_play(Path::new(&string(folder)));
180 report(request)
181 .flatten()
182 .map_or(std::ptr::null_mut(), |json| owned(json.to_string()))
183}
184
185/// The recording last asked to play is at `at_ms`, whose linked note the page highlights;
186/// -1 ends playback. Whether the page changed.
187#[unsafe(no_mangle)]
188pub extern "C" fn sb_view_played(view: &mut View, at_ms: i64) -> bool {
189 let at = u32::try_from(at_ms).ok();
190 report(view.canvas.played(at)).unwrap_or(false)
191}
192
193/// The rate the host records sound at, as mono 16-bit PCM: the desktop's.
194#[unsafe(no_mangle)]
195pub extern "C" fn sb_recording_rate() -> u32 {
196 recording::RATE
197}
198
199/// A video recording's pictures and sound as the camera gave them, which become the AVI
200/// file the desktop records.
201#[derive(Default)]
202pub struct Movie {
203 pictures: video::Pictures,
204 sound: Vec<u8>,
205}
206
207#[unsafe(no_mangle)]
208pub extern "C" fn sb_movie_new() -> *mut Movie {
209 Box::into_raw(Box::default())
210}
211
212/// The picture taken `at_us` microseconds in: `height` rows of `stride` bytes of BGRA.
213///
214/// # Safety
215/// `bgra` holds `height` × `stride` bytes.
216#[unsafe(no_mangle)]
217pub unsafe extern "C" fn sb_movie_picture(
218 movie: &mut Movie,
219 at_us: u64,
220 bgra: *const u8,
221 width: u32,
222 height: u32,
223 stride: usize,
224) {
225 // SAFETY: the caller's contract.
226 let bgra = unsafe { std::slice::from_raw_parts(bgra, height as usize * stride) };
227 movie.pictures.take_bgra(
228 at_us,
229 [width, height].map(|side| side as usize),
230 stride,
231 bgra,
232 );
233}
234
235/// More of the sound, mono 16-bit little-endian PCM at `canvas::recording::RATE`.
236///
237/// # Safety
238/// `pcm` holds `length` bytes.
239#[unsafe(no_mangle)]
240pub unsafe extern "C" fn sb_movie_sound(movie: &mut Movie, pcm: *const u8, length: usize) {
241 // SAFETY: the caller's contract.
242 movie
243 .sound
244 .extend_from_slice(unsafe { std::slice::from_raw_parts(pcm, length) });
245}
246
247/// The AVI file of a movie `duration_us` long, `length` bytes freed with `sb_bytes_free`,
248/// or null when it has no pictures or passes 4 GiB.
249///
250/// # Safety
251/// `movie` came from `sb_movie_new` and is not used again.
252#[unsafe(no_mangle)]
253pub unsafe extern "C" fn sb_movie_finish(
254 movie: *mut Movie,
255 duration_us: u64,
256 length: &mut usize,
257) -> *mut u8 {
258 // SAFETY: the caller's contract.
259 let movie = unsafe { Box::from_raw(movie) };
260 let frames = movie.pictures.finish(duration_us);
261 if frames.is_empty() {
262 return std::ptr::null_mut();
263 }
264 let sound = video::Pcm {
265 rate: recording::RATE,
266 channels: 1,
267 data: movie.sound,
268 };
269 let Some(avi) = video::write(video::SIZE, &frames, &sound) else {
270 return std::ptr::null_mut();
271 };
272 *length = avi.len();
273 Box::into_raw(avi.into_boxed_slice()).cast()
274}
crates/mobile/src/tests.rs+107
...@@ -1129,3 +1129,110 @@ fn the_page_prints_as_a_pdf() {...@@ -1129,3 +1129,110 @@ fn the_page_prints_as_a_pdf() {
1129 &text[..400]1129 &text[..400]
1130 );1130 );
1131}1131}
1132
1133#[test]
1134fn a_recording_goes_in_as_the_desktop_stores_it_and_a_tap_plays_it() {
1135 use onestore::page::{PageObject, ParagraphContent};
1136 let (directory, section) = features();
1137 let mut canvas = canvas(&section, "Paragraph controls");
1138 focus(&mut canvas, "Collapsed parent");
1139 canvas.edit().unwrap();
1140 let date = ["Tuesday, September 29, 2026", "5:18 PM"];
1141 assert!(canvas.start_recording(false, date[0], date[1]).unwrap());
1142 canvas.insert("Linked note".into()).unwrap();
1143 let samples: Vec<i16> = (0..16_000).map(|n| (n % 64 * 256 - 8192) as i16).collect();
1144 let wav = canvas::recording::wave(canvas::recording::RATE, &samples);
1145 assert!(canvas.finish_recording(Some(wav), false).unwrap());
1146 section
1147 .shared
1148 .apply(canvas.edit().unwrap().unwrap())
1149 .unwrap();
1150
1151 let page = section.shared.page(canvas.space).unwrap().0;
1152 let paragraphs: Vec<_> = page
1153 .objects
1154 .iter()
1155 .filter_map(|object| match object {
1156 PageObject::Outline(outline) => Some(&outline.paragraphs),
1157 _ => None,
1158 })
1159 .flatten()
1160 .collect();
1161 let file = paragraphs
1162 .iter()
1163 .find_map(|paragraph| match &paragraph.content {
1164 ParagraphContent::Attachment(file) => Some(file),
1165 _ => None,
1166 })
1167 .unwrap();
1168 let recording = file.recording.unwrap();
1169 assert_eq!(file.filename, format!("{}.wav", page.title));
1170 assert_eq!((recording.kind, recording.duration_ms), (1, Some(1000)));
1171 // IMA ADPCM, as the desktop stores audio for OneNote (`corpus/recording`).
1172 let bytes = file.bytes.as_deref().unwrap();
1173 assert_eq!(u16::from_le_bytes([bytes[20], bytes[21]]), 0x11);
1174 let linked = |text: &str| {
1175 paragraphs
1176 .iter()
1177 .find(|paragraph| {
1178 paragraph
1179 .text()
1180 .is_some_and(|t| t.text.text().starts_with(text))
1181 })
1182 .map(|paragraph| (paragraph.media.recordings.clone(), paragraph.media.time_ms))
1183 .unwrap()
1184 };
1185 let line = linked("Audio recording started: 5:18 PM Tuesday, September 29, 2026");
1186 assert_eq!(line, (vec![recording.id], Some(0)));
1187 assert_eq!(linked("Linked note").0, [recording.id]);
1188
1189 // A tap on the recording plays it as PCM from the start.
1190 let tapped = (0..800)
1191 .flat_map(|y| (0..300).map(move |x| [x as f32 * 2.0, y as f32 * 2.0]))
1192 .find(|&point| {
1193 matches!(
1194 canvas.page.hit(canvas.device(point.map(|v| v * POINT))),
1195 Some(Hit::File(_))
1196 )
1197 })
1198 .unwrap()
1199 .map(|v| v * POINT);
1200 canvas.press(tapped).unwrap();
1201 canvas.release().unwrap();
1202 let folder = directory.path().join("playback");
1203 let request = canvas.take_play(&folder).unwrap().unwrap();
1204 assert_eq!(request["at_ms"], 0);
1205 assert!(request["video"].is_null());
1206 let sound = std::fs::read(request["sound"].as_str().unwrap()).unwrap();
1207 assert_eq!(u16::from_le_bytes([sound[20], sound[21]]), 1);
1208 assert!(canvas.take_play(&folder).unwrap().is_none());
1209 canvas.played(Some(5_000)).unwrap();
1210 canvas.played(None).unwrap();
1211}
1212
1213#[test]
1214fn a_movie_from_the_camera_becomes_the_desktop_s_avi() {
1215 let movie = recording::sb_movie_new();
1216 let bgra: Vec<u8> = (0..480 * 640 * 4).map(|n| (n % 251) as u8).collect();
1217 for index in 0..20u64 {
1218 unsafe {
1219 recording::sb_movie_picture(
1220 &mut *movie,
1221 index * 50_000,
1222 bgra.as_ptr(),
1223 640,
1224 480,
1225 640 * 4,
1226 )
1227 };
1228 }
1229 let pcm = vec![0u8; canvas::recording::RATE as usize * 2];
1230 unsafe { recording::sb_movie_sound(&mut *movie, pcm.as_ptr(), pcm.len()) };
1231 let mut length = 0;
1232 let avi = unsafe { recording::sb_movie_finish(movie, 1_000_000, &mut length) };
1233 let bytes = unsafe { std::slice::from_raw_parts(avi, length) }.to_vec();
1234 unsafe { sb_bytes_free(avi, length) };
1235 let parsed = canvas::recording::video::Movie::parse(&bytes).unwrap();
1236 assert_eq!((parsed.frames.len(), parsed.duration_ms()), (15, 1000));
1237 assert!(parsed.sound.is_some());
1238}
crates/snowbound/Cargo.toml+6-2
...@@ -14,8 +14,6 @@ canvas = { path = "../canvas", features = ["interaction", "pdf"] }...@@ -14,8 +14,6 @@ canvas = { path = "../canvas", features = ["interaction", "pdf"] }
14draw = { path = "../draw", default-features = false, features = ["render"] }14draw = { path = "../draw", default-features = false, features = ["render"] }
15ui = { path = "../ui" }15ui = { path = "../ui" }
16png = "0.18"16png = "0.18"
17# Encodes a video recording's pictures as JPEG, which OneNote 2010 plays.
18image = { version = "0.25", default-features = false, features = ["jpeg"] }
19onestore = { path = "../onestore" }17onestore = { path = "../onestore" }
20notebook = { path = "../notebook" }18notebook = { path = "../notebook" }
21serde = { version = "1", features = ["derive"] }19serde = { version = "1", features = ["derive"] }
...@@ -94,6 +92,8 @@ windows-sys = { version = "0.61", features = [...@@ -94,6 +92,8 @@ windows-sys = { version = "0.61", features = [
94[target.'cfg(target_os = "linux")'.dependencies]92[target.'cfg(target_os = "linux")'.dependencies]
95accesskit_winit = { version = "0.34.0", default-features = false, features = ["accesskit_unix", "async-io"] }93accesskit_winit = { version = "0.34.0", default-features = false, features = ["accesskit_unix", "async-io"] }
96smithay-clipboard = "0.7.3"94smithay-clipboard = "0.7.3"
95# Scales the window's icon.
96image = { version = "0.25", default-features = false }
97# Loads fontconfig at run time, so builds need no fontconfig headers or library.97# Loads fontconfig at run time, so builds need no fontconfig headers or library.
98fontique = { workspace = true, features = ["fontconfig-dlopen"] }98fontique = { workspace = true, features = ["fontconfig-dlopen"] }
99libc = "0.2"99libc = "0.2"
...@@ -106,3 +106,7 @@ sctk-adwaita = { version = "0.10.1", default-features = false }...@@ -106,3 +106,7 @@ sctk-adwaita = { version = "0.10.1", default-features = false }
106wayland-backend = { version = "0.3.17", features = ["client_system"] }106wayland-backend = { version = "0.3.17", features = ["client_system"] }
107wayland-client = "0.31.15"107wayland-client = "0.31.15"
108wayland-protocols = { version = "0.32.13", features = ["client", "staging"] }108wayland-protocols = { version = "0.32.13", features = ["client", "staging"] }
109
110[dev-dependencies]
111# Draws the pictures of the video recording the tests store.
112image = { version = "0.25", default-features = false }
crates/snowbound/src/attachment.rs+2-5
...@@ -4,9 +4,6 @@ use crate::{State, platform};...@@ -4,9 +4,6 @@ use crate::{State, platform};
4use onestore::page::Attachment;4use onestore::page::Attachment;
5use std::{error::Error, path::Path};5use std::{error::Error, path::Path};
66
7/// The size OneNote 2010 shows a file's icon at, in points.
8pub(crate) const ICON_SIZE: [f32; 2] = [24.0, 24.0];
9
10impl State {7impl State {
11 /// Attach File, or a file dropped at `at`, a window point: a copy of the file's bytes8 /// Attach File, or a file dropped at `at`, a window point: a copy of the file's bytes
12 /// at the caret or at `at`, with the icon the system shows for it; audio and video are9 /// at the caret or at `at`, with the icon the system shows for it; audio and video are
...@@ -30,10 +27,10 @@ impl State {...@@ -30,10 +27,10 @@ impl State {
30 id: onestore::page::text::new_id()?,27 id: onestore::page::text::new_id()?,
31 filename: name.to_owned(),28 filename: name.to_owned(),
32 source_path: path.to_str().map(str::to_owned),29 source_path: path.to_str().map(str::to_owned),
33 size: Some(ICON_SIZE),30 size: Some(canvas::gpu::page::ICON_SIZE),
34 layout: Default::default(),31 layout: Default::default(),
35 preview: platform::file_icon(path).map(Into::into),32 preview: platform::file_icon(path).map(Into::into),
36 recording: crate::recording::attached(name, &bytes),33 recording: canvas::recording::attached(name, &bytes),
37 bytes: Some(bytes.into()),34 bytes: Some(bytes.into()),
38 tags: Vec::new(),35 tags: Vec::new(),
39 };36 };
crates/snowbound/src/main.rs-1
...@@ -84,7 +84,6 @@ mod undo;...@@ -84,7 +84,6 @@ mod undo;
84mod unpack;84mod unpack;
85mod unread;85mod unread;
86mod update;86mod update;
87mod video;
88#[cfg_attr(target_arch = "wasm32", path = "watch_web.rs")]87#[cfg_attr(target_arch = "wasm32", path = "watch_web.rs")]
89mod watch;88mod watch;
9089
crates/snowbound/src/media_linux.rs+8-6
...@@ -3,7 +3,8 @@...@@ -3,7 +3,8 @@
3//! with gst-libav), the default microphone records as PCM WAV, and the default camera with3//! with gst-libav), the default microphone records as PCM WAV, and the default camera with
4//! it as the Motion JPEG AVI file OneNote plays. Recording needs the Base and Good plug-ins.4//! it as the Motion JPEG AVI file OneNote plays. Recording needs the Base and Good plug-ins.
55
6use crate::{platform::symbol, video};6use crate::platform::symbol;
7use canvas::recording::video;
7use std::{8use std::{
8 cell::Cell,9 cell::Cell,
9 ffi::{CStr, CString, c_char, c_int, c_void},10 ffi::{CStr, CString, c_char, c_int, c_void},
...@@ -313,7 +314,7 @@ impl Recorder {...@@ -313,7 +314,7 @@ impl Recorder {
313 Self::start(&movie(314 Self::start(&movie(
314 "autovideosrc",315 "autovideosrc",
315 "autoaudiosrc",316 "autoaudiosrc",
316 crate::recording::RATE,317 canvas::recording::RATE,
317 location,318 location,
318 ))319 ))
319 }320 }
...@@ -395,17 +396,18 @@ mod tests {...@@ -395,17 +396,18 @@ mod tests {
395 "{} ! wavenc ! filesink location=\"{location}\"",396 "{} ! wavenc ! filesink location=\"{location}\"",
396 sound(397 sound(
397 "audiotestsrc is-live=true wave=sine",398 "audiotestsrc is-live=true wave=sine",
398 crate::recording::RATE399 canvas::recording::RATE
399 )400 )
400 ),401 ),
401 2,402 2,
402 true,403 true,
403 );404 );
404 let (bytes, duration) = crate::recording::compress(&std::fs::read(&path).unwrap()).unwrap();405 let (bytes, duration) =
406 canvas::recording::compress(&std::fs::read(&path).unwrap()).unwrap();
405 // Two seconds recorded; the paused one is left out.407 // Two seconds recorded; the paused one is left out.
406 assert!(duration.abs_diff(2000) < 300, "{duration}");408 assert!(duration.abs_diff(2000) < 300, "{duration}");
407 // Snowbound's IMA ADPCM, decoded as playback decodes it, plays at its length.409 // Snowbound's IMA ADPCM, decoded as playback decodes it, plays at its length.
408 let wave = crate::recording::decompress(&bytes).unwrap();410 let wave = canvas::recording::decompress(&bytes).unwrap();
409 let played = scratch.0.join("played.wav");411 let played = scratch.0.join("played.wav");
410 std::fs::write(&played, wave).unwrap();412 std::fs::write(&played, wave).unwrap();
411 let mut player = Player::open(&played, false).unwrap();413 let mut player = Player::open(&played, false).unwrap();
...@@ -432,7 +434,7 @@ mod tests {...@@ -432,7 +434,7 @@ mod tests {
432 &movie(434 &movie(
433 "videotestsrc is-live=true pattern=smpte",435 "videotestsrc is-live=true pattern=smpte",
434 "audiotestsrc is-live=true wave=sine",436 "audiotestsrc is-live=true wave=sine",
435 crate::recording::RATE,437 canvas::recording::RATE,
436 quoted(&path).unwrap(),438 quoted(&path).unwrap(),
437 ),439 ),
438 2,440 2,
crates/snowbound/src/media_macos.rs+8-15
...@@ -3,7 +3,7 @@...@@ -3,7 +3,7 @@
3//! PCM WAV, and a capture session records the default camera and microphone as a movie,3//! PCM WAV, and a capture session records the default camera and microphone as a movie,
4//! which `AVAssetReader` reads back into the AVI file OneNote plays.4//! which `AVAssetReader` reads back into the AVI file OneNote plays.
55
6use crate::video;6use canvas::recording::video;
7use objc2::{7use objc2::{
8 msg_send, msg_send_id,8 msg_send, msg_send_id,
9 rc::{Allocated, Retained},9 rc::{Allocated, Retained},
...@@ -432,7 +432,7 @@ fn delegate() -> Retained<AnyObject> {...@@ -432,7 +432,7 @@ fn delegate() -> Retained<AnyObject> {
432}432}
433433
434/// The movie at `path`, as AVFoundation reads it, as a Motion JPEG AVI file with its sound434/// The movie at `path`, as AVFoundation reads it, as a Motion JPEG AVI file with its sound
435/// in mono PCM at `crate::recording::RATE`.435/// in mono PCM at `canvas::recording::RATE`.
436fn avi(path: &Path) -> Option<Vec<u8>> {436fn avi(path: &Path) -> Option<Vec<u8>> {
437 unsafe {437 unsafe {
438 let asset: Retained<AnyObject> = msg_send_id![class("AVURLAsset"), URLAssetWithURL: &*url(path)?, options: std::ptr::null::<AnyObject>()];438 let asset: Retained<AnyObject> = msg_send_id![class("AVURLAsset"), URLAssetWithURL: &*url(path)?, options: std::ptr::null::<AnyObject>()];
...@@ -453,22 +453,14 @@ fn avi(path: &Path) -> Option<Vec<u8>> {...@@ -453,22 +453,14 @@ fn avi(path: &Path) -> Option<Vec<u8>> {
453 CVPixelBufferGetBytesPerRow(buffer),453 CVPixelBufferGetBytesPerRow(buffer),
454 ];454 ];
455 let base = CVPixelBufferGetBaseAddress(buffer);455 let base = CVPixelBufferGetBaseAddress(buffer);
456 let mut rgb = Vec::with_capacity(width * height * 3);
457 if !base.is_null() {456 if !base.is_null() {
458 let rows = std::slice::from_raw_parts(base, stride * height);457 let rows = std::slice::from_raw_parts(base, stride * height);
459 for row in rows.chunks_exact(stride) {458 pictures.take_bgra(at, [width, height], stride, rows);
460 for bgra in row[..width * 4].chunks_exact(4) {
461 rgb.extend_from_slice(&[bgra[2], bgra[1], bgra[0]]);
462 }
463 }
464 }459 }
465 CVPixelBufferUnlockBaseAddress(buffer, 1);460 CVPixelBufferUnlockBaseAddress(buffer, 1);
466 if let Some(picture) = image::RgbImage::from_raw(width as u32, height as u32, rgb) {
467 pictures.take(at, picture);
468 }
469 })?;461 })?;
470 let mut sound = video::Pcm {462 let mut sound = video::Pcm {
471 rate: crate::recording::RATE,463 rate: canvas::recording::RATE,
472 channels: 1,464 channels: 1,
473 data: Vec::new(),465 data: Vec::new(),
474 };466 };
...@@ -550,8 +542,9 @@ mod tests {...@@ -550,8 +542,9 @@ mod tests {
550 /// What Snowbound records, compressed, plays back through Core Audio at its length.542 /// What Snowbound records, compressed, plays back through Core Audio at its length.
551 #[test]543 #[test]
552 fn core_audio_plays_the_compressed_recordings() {544 fn core_audio_plays_the_compressed_recordings() {
553 let pcm = crate::recording::wave(crate::recording::RATE, &crate::recording::tests::tone());545 let pcm =
554 let (bytes, duration) = crate::recording::compress(&pcm).unwrap();546 canvas::recording::wave(canvas::recording::RATE, &crate::recording::tests::tone());
547 let (bytes, duration) = canvas::recording::compress(&pcm).unwrap();
555 let folder = std::env::temp_dir().join(format!("snowbound-player-{}", std::process::id()));548 let folder = std::env::temp_dir().join(format!("snowbound-player-{}", std::process::id()));
556 std::fs::create_dir_all(&folder).unwrap();549 std::fs::create_dir_all(&folder).unwrap();
557 let path = folder.join("tone.wav");550 let path = folder.join("tone.wav");
...@@ -611,7 +604,7 @@ mod tests {...@@ -611,7 +604,7 @@ mod tests {
611 .iter()604 .iter()
612 .map(ExactSizeIterator::len)605 .map(ExactSizeIterator::len)
613 .sum();606 .sum();
614 let expected = 2 * crate::recording::RATE as usize * 2;607 let expected = 2 * canvas::recording::RATE as usize * 2;
615 assert!(length.abs_diff(expected) < expected / 20, "{length}");608 assert!(length.abs_diff(expected) < expected / 20, "{length}");
616 }609 }
617}610}
crates/snowbound/src/recording.rs+22-371
...@@ -3,15 +3,13 @@...@@ -3,15 +3,13 @@
3//! as a file OneNote plays; a recording, or a note linked to a moment in one, plays in a3//! as a file OneNote plays; a recording, or a note linked to a moment in one, plays in a
4//! transport over the page. What Snowbound cannot play opens in the system's player.4//! transport over the page. What Snowbound cannot play opens in the system's player.
55
6use crate::{State, media, platform, video};6use crate::{State, media, platform};
7use onestore::page::{Attachment, Recording};7use canvas::recording::{RATE, video};
8use onestore::page::Attachment;
8use std::{error::Error, ops::Range, path::Path, sync::Arc, time::Duration};9use std::{error::Error, ops::Range, path::Path, sync::Arc, time::Duration};
9use ui::{Axis, Flags, Spec, Theme, children, fit, px};10use ui::{Axis, Flags, Spec, Theme, children, fit, px};
10use winit::keyboard::NamedKey;11use winit::keyboard::NamedKey;
1112
12/// The rate audio is recorded at: wideband speech, as OneNote's own 8 kHz WMA is narrowband.
13pub(crate) const RATE: u32 = 16_000;
14
15/// What records or plays now.13/// What records or plays now.
16#[derive(Default)]14#[derive(Default)]
17pub(crate) enum Media {15pub(crate) enum Media {
...@@ -26,7 +24,7 @@ pub(crate) enum Media {...@@ -26,7 +24,7 @@ pub(crate) enum Media {
26 Saving {24 Saving {
27 video: bool,25 video: bool,
28 path: std::path::PathBuf,26 path: std::path::PathBuf,
29 job: std::thread::JoinHandle<Result<Vec<u8>, String>>,27 job: std::thread::JoinHandle<Result<Attachment, String>>,
30 },28 },
31 Playing(Box<Playback>),29 Playing(Box<Playback>),
32}30}
...@@ -94,8 +92,7 @@ impl State {...@@ -94,8 +92,7 @@ impl State {
94 }92 }
95 };93 };
96 let [date, time] = platform::date_text(crate::filetime());94 let [date, time] = platform::date_text(crate::filetime());
97 let kind = if video { "Video" } else { "Audio" };95 let label = canvas::recording::label(video, &date, &time);
98 let label = format!("{kind} recording started: {time} {date}");
99 let (id, response) = self.view.start_recording(&label)?;96 let (id, response) = self.view.start_recording(&label)?;
100 self.respond(response);97 self.respond(response);
101 if id.is_none() {98 if id.is_none() {
...@@ -129,15 +126,15 @@ impl State {...@@ -129,15 +126,15 @@ impl State {
129 }126 }
130 };127 };
131 let recorded = path.clone();128 let recorded = path.clone();
129 let id = self.view.editor.recording();
130 let title = self.view.editor.page()?.title;
132 let job = std::thread::spawn(move || {131 let job = std::thread::spawn(move || {
133 finish()?;132 finish()?;
134 let recorded =133 let recorded =
135 notebook::fs::read(&recorded).map_err(|error| error.to_string())?;134 notebook::fs::read(&recorded).map_err(|error| error.to_string())?;
136 Ok(if video {135 let id = id.ok_or("Try recording again.")?;
137 recorded136 canvas::recording::file(id, &title, video, recorded)
138 } else {137 .map_err(|error| error.to_string())
139 compress(&recorded).map_or(recorded, |(bytes, _)| bytes)
140 })
141 });138 });
142 self.media = Media::Saving { video, path, job };139 self.media = Media::Saving { video, path, job };
143 }140 }
...@@ -147,66 +144,24 @@ impl State {...@@ -147,66 +144,24 @@ impl State {
147 Media::Saving { job, .. } if wait || job.is_finished() => {}144 Media::Saving { job, .. } if wait || job.is_finished() => {}
148 _ => return Ok(()),145 _ => return Ok(()),
149 }146 }
150 let Media::Saving { video, path, job } = std::mem::take(&mut self.media) else {147 let Media::Saving { path, job, .. } = std::mem::take(&mut self.media) else {
151 unreachable!("saving, as matched")148 unreachable!("saving, as matched")
152 };149 };
153 let saved = job150 let saved = job
154 .join()151 .join()
155 .unwrap_or_else(|_| Err("Try recording again.".into()));152 .unwrap_or_else(|_| Err("Try recording again.".into()));
156 let Some(id) = self.view.editor.recording() else {153 if self.view.editor.recording().is_none() {
157 return Ok(());154 return Ok(());
158 };155 }
159 let bytes = match saved {156 let mut file = match saved {
160 Ok(bytes) => bytes,157 Ok(file) => file,
161 Err(detail) => {158 Err(detail) => {
162 platform::alert("Couldn't finish the recording", &detail);159 platform::alert("Couldn't finish the recording", &detail);
163 return Ok(());160 return Ok(());
164 }161 }
165 };162 };
166 let duration_ms = if video {163 file.preview = platform::file_icon(&path).map(Into::into);
167 video::Movie::parse(&bytes).map(|movie| movie.duration_ms())
168 } else {
169 Wave::parse(&bytes).map(|wave| wave.duration_ms())
170 };
171 let preview = platform::file_icon(&path);
172 notebook::fs::remove_file(&path)?;164 notebook::fs::remove_file(&path)?;
173 let title = self.view.editor.page()?.title;
174 let name: String = title
175 .chars()
176 .map(|c| {
177 if matches!(c, '/' | '\\' | '\0') {
178 '_'
179 } else {
180 c
181 }
182 })
183 .collect();
184 let name = if name.trim().is_empty() {
185 if video {
186 "Video recording"
187 } else {
188 "Audio recording"
189 }
190 .to_owned()
191 } else {
192 name
193 };
194 let extension = if video { "avi" } else { "wav" };
195 let file = Attachment {
196 id: onestore::page::text::new_id()?,
197 filename: format!("{name}.{extension}"),
198 source_path: None,
199 size: Some(crate::attachment::ICON_SIZE),
200 layout: Default::default(),
201 bytes: Some(bytes.into()),
202 preview: preview.map(Into::into),
203 recording: Some(Recording {
204 id,
205 kind: if video { 2 } else { 1 },
206 duration_ms,
207 }),
208 tags: Vec::new(),
209 };
210 let response = self.view.finish_recording(file)?;165 let response = self.view.finish_recording(file)?;
211 self.respond(response);166 self.respond(response);
212 Ok(())167 Ok(())
...@@ -226,19 +181,10 @@ impl State {...@@ -226,19 +181,10 @@ impl State {
226 let name = Path::new(&file.filename)181 let name = Path::new(&file.filename)
227 .file_stem()182 .file_stem()
228 .map_or_else(String::new, |stem| stem.to_string_lossy().into_owned());183 .map_or_else(String::new, |stem| stem.to_string_lossy().into_owned());
229 let movie = video::Movie::parse(&bytes);184 let playable = match canvas::recording::sound(&bytes) {
230 // The sound as a WAV file every platform plays: IMA ADPCM decoded, a video's sound
231 // taken out of it.
232 let sound = movie.as_ref().map(|movie| {
233 movie
234 .wave(&bytes)
235 .unwrap_or_else(|| silence(movie.duration_ms()))
236 });
237 let sound = sound.map_or_else(|| decompress(&bytes), Some);
238 let playable = match sound {
239 Some(wave) => Attachment {185 Some(wave) => Attachment {
240 filename: format!("{name}.wav"),186 filename: format!("{name}.wav"),
241 bytes: Some(decompress(&wave).unwrap_or(wave).into()),187 bytes: Some(wave.into()),
242 ..file.clone()188 ..file.clone()
243 },189 },
244 None => file.clone(),190 None => file.clone(),
...@@ -260,7 +206,7 @@ impl State {...@@ -260,7 +206,7 @@ impl State {
260 player,206 player,
261 name,207 name,
262 recording: file.recording.map(|recording| recording.id),208 recording: file.recording.map(|recording| recording.id),
263 video: movie.map(|movie| Video {209 video: video::Movie::parse(&bytes).map(|movie| Video {
264 bytes,210 bytes,
265 movie,211 movie,
266 shown: None,212 shown: None,
...@@ -607,255 +553,11 @@ fn moment(text: &str) -> Option<u32> {...@@ -607,255 +553,11 @@ fn moment(text: &str) -> Option<u32> {
607 seconds.checked_mul(1000)553 seconds.checked_mul(1000)
608}554}
609555
610/// Silence as long as `ms`, the clock of a video without sound.
611fn silence(ms: u32) -> Vec<u8> {
612 wave(8_000, &vec![0; ms as usize * 8])
613}
614
615/// A mono 16-bit PCM WAV file of `samples` at `rate`, as the platform recorders write.
616pub(crate) fn wave(rate: u32, samples: &[i16]) -> Vec<u8> {
617 let data: Vec<u8> = samples.iter().flat_map(|s| s.to_le_bytes()).collect();
618 [
619 &b"RIFF"[..],
620 &(36 + data.len() as u32).to_le_bytes(),
621 b"WAVEfmt ",
622 &16_u32.to_le_bytes(),
623 &1_u16.to_le_bytes(),
624 &1_u16.to_le_bytes(),
625 &rate.to_le_bytes(),
626 &(rate * 2).to_le_bytes(),
627 &2_u16.to_le_bytes(),
628 &16_u16.to_le_bytes(),
629 b"data",
630 &(data.len() as u32).to_le_bytes(),
631 &data,
632 ]
633 .concat()
634}
635
636/// A recording for a file attached under `filename`, as OneNote 2010 makes one of every
637/// audio and video file it attaches, with the length of a WAV file's `bytes`.
638pub(crate) fn attached(filename: &str, bytes: &[u8]) -> Option<Recording> {
639 Some(Recording {
640 id: onestore::page::text::new_guid().ok()?,
641 kind: Recording::kind_of(filename)?,
642 duration_ms: Wave::parse(bytes).map(|wave| wave.duration_ms()),
643 })
644}
645
646/// The chunks of a RIFF WAVE file this module reads.
647struct Wave<'a> {
648 format: u16,
649 channels: u16,
650 rate: u32,
651 bytes_per_second: u32,
652 block: u16,
653 bits: u16,
654 /// The sample count a compressed file's `fact` chunk states.
655 samples: Option<u32>,
656 data: &'a [u8],
657}
658
659impl<'a> Wave<'a> {
660 fn parse(bytes: &'a [u8]) -> Option<Self> {
661 if bytes.get(..4)? != b"RIFF" || bytes.get(8..12)? != b"WAVE" {
662 return None;
663 }
664 let u16_at = |data: &[u8], at: usize| {
665 Some(u16::from_le_bytes(data.get(at..at + 2)?.try_into().ok()?))
666 };
667 let u32_at = |data: &[u8], at: usize| {
668 Some(u32::from_le_bytes(data.get(at..at + 4)?.try_into().ok()?))
669 };
670 let (mut fmt, mut samples, mut data) = (None, None, None);
671 let mut at = 12;
672 while at + 8 <= bytes.len() {
673 let size = u32_at(bytes, at + 4)? as usize;
674 let body = &bytes[at + 8..(at + 8).saturating_add(size).min(bytes.len())];
675 match &bytes[at..at + 4] {
676 b"fmt " => fmt = Some(body),
677 b"fact" => samples = u32_at(body, 0),
678 b"data" => data = Some(body),
679 _ => {}
680 }
681 at = at + 8 + size + size % 2;
682 }
683 let fmt = fmt?;
684 Some(Self {
685 format: u16_at(fmt, 0)?,
686 channels: u16_at(fmt, 2)?,
687 rate: u32_at(fmt, 4)?,
688 bytes_per_second: u32_at(fmt, 8)?,
689 block: u16_at(fmt, 12)?,
690 bits: u16_at(fmt, 14)?,
691 samples,
692 data: data?,
693 })
694 }
695
696 fn duration_ms(&self) -> u32 {
697 let ms = match self.samples.filter(|_| self.format != 1) {
698 Some(samples) => u64::from(samples) * 1000 / u64::from(self.rate.max(1)),
699 None => self.data.len() as u64 * 1000 / u64::from(self.bytes_per_second.max(1)),
700 };
701 u32::try_from(ms).unwrap_or(u32::MAX)
702 }
703}
704
705/// IMA ADPCM's step sizes and how each code moves along them.
706const STEPS: [i32; 89] = [
707 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66,
708 73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449,
709 494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, 2272,
710 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358, 5894, 6484, 7132, 7845, 8630, 9493,
711 10442, 11487, 12635, 13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767,
712];
713const INDEX_STEPS: [i32; 8] = [-1, -1, -1, -1, 2, 4, 6, 8];
714
715/// Bytes per block: Windows' IMA ADPCM codec's choice for 16 kHz.
716const BLOCK: usize = 512;
717/// Samples per block: the header's, then two per remaining byte.
718const BLOCK_SAMPLES: usize = (BLOCK - 4) * 2 + 1;
719
720/// A mono 16-bit PCM WAV file as 4-bit IMA ADPCM (WAVE_FORMAT_DVI_ADPCM), which Windows
721/// and macOS decode, a quarter of the size; and its length. None for other files.
722pub(crate) fn compress(wav: &[u8]) -> Option<(Vec<u8>, u32)> {
723 let wave = Wave::parse(wav)?;
724 if wave.format != 1 || wave.channels != 1 || wave.bits != 16 {
725 return None;
726 }
727 let samples: Vec<i16> = wave
728 .data
729 .chunks_exact(2)
730 .map(|pair| i16::from_le_bytes([pair[0], pair[1]]))
731 .collect();
732 let mut data = Vec::with_capacity(samples.len() / 2 + BLOCK);
733 let mut index = 0_i32;
734 for block in samples.chunks(BLOCK_SAMPLES) {
735 let mut predicted = i32::from(block[0]);
736 data.extend_from_slice(&block[0].to_le_bytes());
737 data.extend_from_slice(&[index as u8, 0]);
738 // A short last block is padded with its last sample; `fact` gives the true length.
739 let last = *block.last().unwrap();
740 let rest = block[1..]
741 .iter()
742 .copied()
743 .chain(std::iter::repeat(last))
744 .take(BLOCK_SAMPLES - 1)
745 .collect::<Vec<_>>();
746 for pair in rest.chunks_exact(2) {
747 let mut byte = 0;
748 for (shift, sample) in [0, 4].into_iter().zip(pair) {
749 let code = encode(i32::from(*sample), &mut predicted, &mut index);
750 byte |= code << shift;
751 }
752 data.push(byte);
753 }
754 }
755 let count = u32::try_from(samples.len()).ok()?;
756 let bytes_per_second = wave.rate * BLOCK as u32 / BLOCK_SAMPLES as u32;
757 let mut out = Vec::with_capacity(data.len() + 60);
758 let fmt: Vec<u8> = [
759 &0x11_u16.to_le_bytes()[..],
760 &1_u16.to_le_bytes(),
761 &wave.rate.to_le_bytes(),
762 &bytes_per_second.to_le_bytes(),
763 &(BLOCK as u16).to_le_bytes(),
764 &4_u16.to_le_bytes(),
765 &2_u16.to_le_bytes(),
766 &(BLOCK_SAMPLES as u16).to_le_bytes(),
767 ]
768 .concat();
769 let riff = 4 + (8 + fmt.len()) + (8 + 4) + (8 + data.len());
770 out.extend_from_slice(b"RIFF");
771 out.extend_from_slice(&u32::try_from(riff).ok()?.to_le_bytes());
772 out.extend_from_slice(b"WAVEfmt ");
773 out.extend_from_slice(&(fmt.len() as u32).to_le_bytes());
774 out.extend_from_slice(&fmt);
775 out.extend_from_slice(b"fact");
776 out.extend_from_slice(&4_u32.to_le_bytes());
777 out.extend_from_slice(&count.to_le_bytes());
778 out.extend_from_slice(b"data");
779 out.extend_from_slice(&u32::try_from(data.len()).ok()?.to_le_bytes());
780 out.extend_from_slice(&data);
781 let duration = u32::try_from(u64::from(count) * 1000 / u64::from(wave.rate.max(1))).ok()?;
782 Some((out, duration))
783}
784
785/// The 4-bit code nearest `sample` from `predicted`, which it and `index` then follow as a
786/// decoder does.
787fn encode(sample: i32, predicted: &mut i32, index: &mut i32) -> u8 {
788 let step = STEPS[*index as usize];
789 let mut difference = sample - *predicted;
790 let mut code = 0;
791 if difference < 0 {
792 code = 8;
793 difference = -difference;
794 }
795 let mut delta = step >> 3;
796 for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] {
797 if difference >= part {
798 code |= bit;
799 difference -= part;
800 delta += part;
801 }
802 }
803 *predicted = if code & 8 != 0 {
804 *predicted - delta
805 } else {
806 *predicted + delta
807 }
808 .clamp(-32768, 32767);
809 *index = (*index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88);
810 code
811}
812
813/// A mono IMA ADPCM WAV file as 16-bit PCM, which every platform's player decodes; none
814/// for other files.
815pub(crate) fn decompress(wav: &[u8]) -> Option<Vec<u8>> {
816 let wave = Wave::parse(wav).filter(|wave| wave.format == 0x11 && wave.channels == 1)?;
817 Some(self::wave(wave.rate, &decode(&wave)))
818}
819
820/// Each block's header sample and index, then its codes.
821fn decode(wave: &Wave<'_>) -> Vec<i16> {
822 let mut out = Vec::new();
823 for block in wave.data.chunks(usize::from(wave.block.max(4))) {
824 let Some(header) = block.get(..4) else {
825 break;
826 };
827 let mut predicted = i32::from(i16::from_le_bytes([header[0], header[1]]));
828 let mut index = i32::from(header[2]).clamp(0, 88);
829 out.push(predicted as i16);
830 for byte in &block[4..] {
831 for code in [byte & 15, byte >> 4] {
832 let step = STEPS[index as usize];
833 let mut delta = step >> 3;
834 for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] {
835 if code & bit != 0 {
836 delta += part;
837 }
838 }
839 predicted = if code & 8 != 0 {
840 predicted - delta
841 } else {
842 predicted + delta
843 }
844 .clamp(-32768, 32767);
845 index = (index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88);
846 out.push(predicted as i16);
847 }
848 }
849 }
850 if let Some(samples) = wave.samples {
851 out.truncate(samples as usize);
852 }
853 out
854}
855
856#[cfg(test)]556#[cfg(test)]
857pub(crate) mod tests {557pub(crate) mod tests {
858 use super::*;558 use super::*;
559 use canvas::recording::{compress, wave};
560 use onestore::page::Recording;
859 use std::path::PathBuf;561 use std::path::PathBuf;
860562
861 /// Two seconds of a spoken-pitch chord.563 /// Two seconds of a spoken-pitch chord.
...@@ -870,57 +572,6 @@ pub(crate) mod tests {...@@ -870,57 +572,6 @@ pub(crate) mod tests {
870 .collect()572 .collect()
871 }573 }
872574
873 #[test]
874 fn a_recording_compresses_to_ima_adpcm_that_decodes_close_to_what_was_heard() {
875 let samples = tone();
876 let (bytes, duration) = compress(&wave(RATE, &samples)).unwrap();
877 assert_eq!(duration, 2000);
878 let wave = Wave::parse(&bytes).unwrap();
879 assert_eq!(
880 (wave.format, wave.channels, wave.rate, wave.bits),
881 (0x11, 1, RATE, 4)
882 );
883 assert_eq!(wave.data.len() % BLOCK, 0);
884 assert_eq!(wave.duration_ms(), 2000);
885 assert!(bytes.len() * 3 < samples.len() * 2);
886 let decoded = decode(&wave);
887 assert_eq!(
888 Wave::parse(&decompress(&bytes).unwrap())
889 .unwrap()
890 .data
891 .len(),
892 decoded.len() * 2
893 );
894 assert_eq!(decoded.len(), samples.len());
895 let noise: f64 = samples
896 .iter()
897 .zip(&decoded)
898 .map(|(a, b)| (f64::from(*a) - f64::from(*b)).powi(2))
899 .sum();
900 let signal: f64 = samples.iter().map(|a| f64::from(*a).powi(2)).sum();
901 let snr = 10.0 * (signal / noise).log10();
902 assert!(snr > 20.0, "{snr} dB");
903 }
904
905 #[test]
906 fn only_mono_16_bit_pcm_compresses_and_attached_wav_files_know_their_length() {
907 let stereo = {
908 let mut bytes = wave(RATE, &[0; 64]);
909 bytes[22] = 2;
910 bytes
911 };
912 assert!(compress(&stereo).is_none());
913 assert!(compress(b"not a wave").is_none());
914 let silence = include_bytes!(
915 "../../../corpus/m6/native-features-01/notebook/fixture-data/silence.wav"
916 );
917 // OneNote 2010 stored 1000 ms for this file when attached (`corpus/recording`).
918 let recording = attached("silence.WAV", silence).unwrap();
919 assert_eq!((recording.kind, recording.duration_ms), (1, Some(1000)));
920 assert_eq!(attached("clip.wmv", b"").unwrap().kind, 2);
921 assert!(attached("notes.txt", b"").is_none());
922 }
923
924 /// A recording made through the editor at the end of "Before recording", two notes575 /// A recording made through the editor at the end of "Before recording", two notes
925 /// written while it records, stored as the host stores it; `export` names a variable576 /// written while it records, stored as the host stores it; `export` names a variable
926 /// naming a directory receiving the notebook for a cold read in OneNote 2010.577 /// naming a directory receiving the notebook for a cold read in OneNote 2010.
...@@ -976,7 +627,7 @@ pub(crate) mod tests {...@@ -976,7 +627,7 @@ pub(crate) mod tests {
976 id: onestore::page::text::new_id().unwrap(),627 id: onestore::page::text::new_id().unwrap(),
977 filename: filename.into(),628 filename: filename.into(),
978 source_path: None,629 source_path: None,
979 size: Some(crate::attachment::ICON_SIZE),630 size: Some(canvas::gpu::page::ICON_SIZE),
980 layout: Default::default(),631 layout: Default::default(),
981 bytes: Some(bytes.into()),632 bytes: Some(bytes.into()),
982 preview: Some(canvas::gpu::page::file_icon().into()),633 preview: Some(canvas::gpu::page::file_icon().into()),
crates/snowbound/src/video.rs deleted-407
...@@ -1,407 +0,0 @@
1//! Video recordings as AVI files of Motion JPEG pictures and PCM sound, which OneNote 2010
2//! plays and lists as recordings (`corpus/recording/video`), and reading them back to play.
3
4use std::ops::Range;
5
6/// Pictures a second in a recording: OneNote 2010's own profiles record at 15.
7pub(crate) const FPS: u32 = 15;
8/// A recording's picture size, OneNote 2010's own.
9pub(crate) const SIZE: [u32; 2] = [320, 240];
10
11/// Interleaved signed 16-bit little-endian sound.
12#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
13pub(crate) struct Pcm {
14 pub rate: u32,
15 pub channels: u16,
16 pub data: Vec<u8>,
17}
18
19fn chunk(out: &mut Vec<u8>, id: &[u8; 4], body: &[u8]) {
20 out.extend_from_slice(id);
21 out.extend_from_slice(&(body.len() as u32).to_le_bytes());
22 out.extend_from_slice(body);
23 if body.len() % 2 == 1 {
24 out.push(0);
25 }
26}
27
28#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
29fn list(kind: &[u8; 4], body: &[u8]) -> Vec<u8> {
30 let mut out = Vec::with_capacity(body.len() + 12);
31 chunk(&mut out, b"LIST", &[&kind[..], body].concat());
32 out
33}
34
35#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
36fn words(values: &[u32]) -> Vec<u8> {
37 values
38 .iter()
39 .flat_map(|value| value.to_le_bytes())
40 .collect()
41}
42
43/// An AVI file of `frames`, JPEG pictures of `size` shown `FPS` a second, where an empty one
44/// repeats the picture before it, and `sound` alongside. None past AVI's 4 GiB.
45#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
46pub(crate) fn write(size: [u32; 2], frames: &[Vec<u8>], sound: &Pcm) -> Option<Vec<u8>> {
47 let block = u32::from(sound.channels) * 2;
48 let bytes_per_second = sound.rate * block;
49 let samples = sound.data.len() as u64 / u64::from(block);
50 let count = frames.len() as u64;
51 // The sound up to frame `index`, whole samples.
52 let sound_at = |index: u64| -> usize {
53 let before = if index >= count {
54 samples
55 } else {
56 (index * u64::from(sound.rate) / u64::from(FPS)).min(samples)
57 };
58 (before * u64::from(block)) as usize
59 };
60 let mut movi = b"movi".to_vec();
61 let mut index = Vec::new();
62 let [mut largest_frame, mut largest_sound] = [0, 0];
63 for (at, frame) in frames.iter().enumerate() {
64 let entries: [(&[u8; 4], &[u8]); 2] = [
65 (b"00dc", frame),
66 (
67 b"01wb",
68 &sound.data[sound_at(at as u64)..sound_at(at as u64 + 1)],
69 ),
70 ];
71 for (id, body) in entries {
72 if id == b"01wb" && body.is_empty() {
73 continue;
74 }
75 // Keyframes: every picture and all sound; a repeat is none.
76 let flags: u32 = if body.is_empty() { 0 } else { 0x10 };
77 index.extend_from_slice(id);
78 index.extend(words(&[
79 flags,
80 u32::try_from(movi.len()).ok()?,
81 body.len() as u32,
82 ]));
83 chunk(&mut movi, id, body);
84 let largest = if id == b"00dc" {
85 &mut largest_frame
86 } else {
87 &mut largest_sound
88 };
89 *largest = (*largest).max(body.len() as u32);
90 }
91 }
92 let [width, height] = size;
93 let frame_count = u32::try_from(count).ok()?;
94 let rect = [0u16, 0, width as u16, height as u16]
95 .iter()
96 .flat_map(|side| side.to_le_bytes())
97 .collect::<Vec<u8>>();
98 let mut video = Vec::new();
99 let strh = [
100 &b"vidsMJPG"[..],
101 &words(&[0, 0, 0, 1, FPS, 0, frame_count, largest_frame, u32::MAX, 0]),
102 &rect,
103 ]
104 .concat();
105 chunk(&mut video, b"strh", &strh);
106 let strf = [
107 words(&[40, width, height]),
108 [&1u16.to_le_bytes()[..], &24u16.to_le_bytes()].concat(),
109 b"MJPG".to_vec(),
110 words(&[width * height * 3, 0, 0, 0, 0]),
111 ]
112 .concat();
113 chunk(&mut video, b"strf", &strf);
114 let mut streams = list(b"strl", &video);
115 if samples > 0 {
116 let mut audio = Vec::new();
117 let strh = [
118 &b"auds\0\0\0\0"[..],
119 &words(&[
120 0,
121 0,
122 0,
123 block,
124 bytes_per_second,
125 0,
126 u32::try_from(samples).ok()?,
127 largest_sound,
128 u32::MAX,
129 block,
130 ]),
131 &[0; 8],
132 ]
133 .concat();
134 chunk(&mut audio, b"strh", &strh);
135 let strf = [
136 &1u16.to_le_bytes()[..],
137 &sound.channels.to_le_bytes(),
138 &sound.rate.to_le_bytes(),
139 &bytes_per_second.to_le_bytes(),
140 &(block as u16).to_le_bytes(),
141 &16u16.to_le_bytes(),
142 &0u16.to_le_bytes(),
143 ]
144 .concat();
145 chunk(&mut audio, b"strf", &strf);
146 streams.extend(list(b"strl", &audio));
147 }
148 let mut avih = Vec::new();
149 chunk(
150 &mut avih,
151 b"avih",
152 &words(&[
153 1_000_000 / FPS,
154 largest_frame
155 .saturating_add(largest_sound)
156 .saturating_mul(FPS),
157 0,
158 // AVIF_HASINDEX | AVIF_ISINTERLEAVED
159 0x110,
160 frame_count,
161 0,
162 if samples > 0 { 2 } else { 1 },
163 largest_frame.max(largest_sound),
164 width,
165 height,
166 0,
167 0,
168 0,
169 0,
170 ]),
171 );
172 let header = list(b"hdrl", &[avih, streams].concat());
173 let mut body = b"AVI ".to_vec();
174 body.extend(header);
175 let movi_size = u32::try_from(movi.len()).ok()?;
176 body.extend_from_slice(b"LIST");
177 body.extend_from_slice(&movi_size.to_le_bytes());
178 body.extend(movi);
179 chunk(&mut body, b"idx1", &index);
180 u32::try_from(body.len()).ok()?;
181 let mut out = Vec::with_capacity(body.len() + 8);
182 chunk(&mut out, b"RIFF", &body);
183 Some(out)
184}
185
186/// An AVI file's pictures and sound, as byte ranges of the file.
187pub(crate) struct Movie {
188 /// Each picture's JPEG; an empty range repeats the one before.
189 pub frames: Vec<Range<usize>>,
190 pub frame_us: u32,
191 /// The sound's WAVEFORMATEX and its data, in order.
192 pub sound: Option<(Range<usize>, Vec<Range<usize>>)>,
193}
194
195impl Movie {
196 /// A Motion JPEG AVI file's contents; none for other files.
197 pub(crate) fn parse(bytes: &[u8]) -> Option<Self> {
198 let mut movie = Self {
199 frames: Vec::new(),
200 frame_us: 0,
201 sound: None,
202 };
203 let mut kinds = Vec::new();
204 let mut first = true;
205 for (id, body) in Chunks::new(bytes, 0..bytes.len()) {
206 let kind = bytes.get(body.start..body.start + 4)?;
207 if id != *b"RIFF" || (kind != b"AVI " && !(kind == b"AVIX" && !first)) {
208 return (!first).then_some(movie).filter(|movie| movie.frame_us > 0);
209 }
210 first = false;
211 movie.walk(bytes, body.start + 4..body.end, &mut kinds)?;
212 }
213 (movie.frame_us > 0).then_some(movie)
214 }
215
216 fn walk(&mut self, bytes: &[u8], range: Range<usize>, kinds: &mut Vec<[u8; 4]>) -> Option<()> {
217 for (id, body) in Chunks::new(bytes, range) {
218 let u32_at = |at: usize| {
219 Some(u32::from_le_bytes(
220 bytes
221 .get(body.start + at..body.start + at + 4)?
222 .try_into()
223 .ok()?,
224 ))
225 };
226 match &id {
227 b"LIST" => {
228 let inner = body.start + 4..body.end;
229 self.walk(bytes, inner, kinds)?;
230 }
231 b"strh" => {
232 let kind: [u8; 4] = bytes.get(body.start..body.start + 4)?.try_into().ok()?;
233 if &kind == b"vids" {
234 let [scale, rate] = [u32_at(20)?, u32_at(24)?];
235 self.frame_us =
236 u32::try_from(u64::from(scale) * 1_000_000 / u64::from(rate.max(1)))
237 .ok()?;
238 }
239 kinds.push(kind);
240 }
241 b"strf" if kinds.last() == Some(b"vids") => {
242 if bytes.get(body.start + 16..body.start + 20)? != b"MJPG" {
243 return None;
244 }
245 }
246 b"strf" if kinds.last() == Some(b"auds") => {
247 self.sound = Some((body.clone(), Vec::new()));
248 }
249 [a, b, c, d] if a.is_ascii_digit() && b.is_ascii_digit() => {
250 let stream = usize::from((a - b'0') * 10 + (b - b'0'));
251 match (kinds.get(stream).map(|kind| &kind[..]), [c, d]) {
252 (Some(b"vids"), [b'd', b'c' | b'b']) => self.frames.push(body),
253 (Some(b"auds"), [b'w', b'b']) => {
254 if let Some((_, data)) = &mut self.sound {
255 data.push(body);
256 }
257 }
258 _ => {}
259 }
260 }
261 _ => {}
262 }
263 }
264 Some(())
265 }
266
267 pub(crate) fn duration_ms(&self) -> u32 {
268 let ms = (self.frames.len() as u64 * u64::from(self.frame_us)).div_ceil(1000);
269 u32::try_from(ms).unwrap_or(u32::MAX)
270 }
271
272 /// The picture showing `at_ms` in: the last stored one at or before it.
273 pub(crate) fn frame(&self, at_ms: u32) -> Option<Range<usize>> {
274 let at = (u64::from(at_ms) * 1000 / u64::from(self.frame_us.max(1))) as usize;
275 self.frames[..=at.min(self.frames.len().checked_sub(1)?)]
276 .iter()
277 .rev()
278 .find(|frame| !frame.is_empty())
279 .cloned()
280 }
281
282 /// The sound as a WAV file, or none where the file has none.
283 pub(crate) fn wave(&self, bytes: &[u8]) -> Option<Vec<u8>> {
284 let (format, data) = self.sound.as_ref()?;
285 let format = &bytes[format.clone()];
286 let length: usize = data.iter().map(ExactSizeIterator::len).sum();
287 let mut out = Vec::with_capacity(length + 64);
288 out.extend_from_slice(b"RIFF");
289 out.extend_from_slice(&((4 + 8 + format.len() + 8 + length) as u32).to_le_bytes());
290 out.extend_from_slice(b"WAVE");
291 chunk(&mut out, b"fmt ", format);
292 out.extend_from_slice(b"data");
293 out.extend_from_slice(&(length as u32).to_le_bytes());
294 for range in data {
295 out.extend_from_slice(&bytes[range.clone()]);
296 }
297 Some(out)
298 }
299}
300
301/// The chunks in `range` of a RIFF file: each one's id and body.
302struct Chunks<'a> {
303 bytes: &'a [u8],
304 at: usize,
305 end: usize,
306}
307
308impl<'a> Chunks<'a> {
309 fn new(bytes: &'a [u8], range: Range<usize>) -> Self {
310 Self {
311 bytes,
312 at: range.start,
313 end: range.end.min(bytes.len()),
314 }
315 }
316}
317
318impl Iterator for Chunks<'_> {
319 type Item = ([u8; 4], Range<usize>);
320
321 fn next(&mut self) -> Option<Self::Item> {
322 let header = self.bytes.get(self.at..self.at + 8)?;
323 if self.at + 8 > self.end {
324 return None;
325 }
326 let id: [u8; 4] = header[..4].try_into().ok()?;
327 let size = u32::from_le_bytes(header[4..].try_into().ok()?) as usize;
328 let start = self.at + 8;
329 let end = start.saturating_add(size).min(self.end);
330 self.at = end + size % 2;
331 Some((id, start..end))
332 }
333}
334
335/// Pictures taken at uneven moments, as a camera takes them, made the `FPS` a second of a
336/// recording: each one shows until the next, fitted to `SIZE`.
337#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
338#[derive(Default)]
339pub(crate) struct Pictures {
340 frames: Vec<Vec<u8>>,
341 last: Option<image::RgbImage>,
342 /// Whether the last picture is stored yet, which the frames after it repeat.
343 stored: bool,
344}
345
346#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
347impl Pictures {
348 /// The picture taken `at_us` microseconds in; the first shows from the start.
349 pub(crate) fn take(&mut self, at_us: u64, picture: image::RgbImage) {
350 self.fill(at_us);
351 self.last = Some(fit(picture));
352 self.stored = false;
353 }
354
355 /// The frames of a recording `end_us` microseconds long.
356 pub(crate) fn finish(mut self, end_us: u64) -> Vec<Vec<u8>> {
357 self.fill(end_us);
358 self.frames
359 }
360
361 /// Frames up to `until_us`, of the last picture.
362 fn fill(&mut self, until_us: u64) {
363 let Some(last) = &self.last else {
364 return;
365 };
366 while (self.frames.len() as u64) * 1_000_000 / u64::from(FPS) < until_us {
367 let frame = if self.stored {
368 Vec::new()
369 } else {
370 self.stored = true;
371 let mut jpeg = Vec::new();
372 let encoded = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 70)
373 .encode_image(last);
374 if encoded.is_err() {
375 jpeg.clear();
376 }
377 jpeg
378 };
379 self.frames.push(frame);
380 }
381 }
382}
383
384/// `picture` scaled to fit `SIZE`, centred on black.
385#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
386fn fit(picture: image::RgbImage) -> image::RgbImage {
387 let [width, height] = SIZE;
388 if picture.dimensions() == (width, height) {
389 return picture;
390 }
391 let scale =
392 (width as f32 / picture.width() as f32).min(height as f32 / picture.height() as f32);
393 let fitted = image::imageops::resize(
394 &picture,
395 ((picture.width() as f32 * scale) as u32).clamp(1, width),
396 ((picture.height() as f32 * scale) as u32).clamp(1, height),
397 image::imageops::FilterType::Triangle,
398 );
399 let mut out = image::RgbImage::new(width, height);
400 image::imageops::overlay(
401 &mut out,
402 &fitted,
403 i64::from((width - fitted.width()) / 2),
404 i64::from((height - fitted.height()) / 2),
405 );
406 out
407}