Commit 01e73fd4cffeccb05dd939ca75bcfb583fb252e2 | snowbound

author	clover caruso <git@paperclover.net> 2026-10-01 23:43:16-07:00
committer	clover caruso <git@paperclover.net> 2026-10-02 00:38:51-07:00
log	01e73fd4cffeccb05dd939ca75bcfb583fb252e2
tree	f6a28eddf5f2acff928b12e82be6f9967f290673
parent	9d07a3bb194eae46161a67803138348d98d0b7d4
signature	Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

feat: record audio and video on iOS

Record Audio and Record Video join the page's Insert menu. AVFoundation records, and
crates/mobile stores the result through canvas::recording, which now holds what the
desktop stored: 16 kHz IMA ADPCM WAV and Motion JPEG AVI, the files OneNote 2010 plays.
A bar over the page's foot shows the clock with Pause and Stop; a tap on a recording plays
it with AVPlayer, a video's pictures in the bar, and See Playback follows it.

Audio keeps recording in the background under the audio background mode, as a lecture
outlasts the screen's timeout, and a call pauses it. The camera stops in the background,
which ends a video recording and saves it. The simulator records video from a stand-in
camera.

Assisted-by: claude-opus-5.5

24 files changed, 2080 insertions(+), 819 deletions(-)

apps
- ios
  - Info.plist
  - Snowbound
    CanvasView.swift
    PageViewController.swift
    Recording.swift
    Script.swift
arc
- canvas.md
- platforms.md
crates
- canvas
  - src
    editor
    recording.rs
    gpu
    page.rs
    interaction
    mod.rs
    lib.rs
    recording.rs
    recording
    video.rs
- mobile
  - include
    snowbound.h
  - src
    lib.rs
    recording.rs
    tests.rs
- snowbound
  - Cargo.toml
  - src
    attachment.rs
    main.rs
    media_linux.rs
    media_macos.rs
    recording.rs
    video.rs

apps/ios/Info.plist+7-1

...	...	@@ -22,7 +22,9 @@
22	22	<key>LSSupportsOpeningDocumentsInPlace</key>
23	23	<true/>
24	24	<key>NSCameraUsageDescription</key>
25		<string>Take a photo to put on a page.</string>
	25	<string>Take photos and record video on your pages.</string>
	26	<key>NSMicrophoneUsageDescription</key>
	27	<string>Record audio and video on your pages.</string>
26	28	<key>NSUbiquitousContainers</key>
27	29	<dict>
28	30	<key>iCloud.net.paperclover.snowbound</key>
...	...	@@ -42,6 +44,10 @@
42	44	<key>UIApplicationSupportsMultipleScenes</key>
43	45	<false/>
44	46	</dict>
	47	<key>UIBackgroundModes</key>
	48	<array>
	49	<string>audio</string>
	50	</array>
45	51	<key>UIFileSharingEnabled</key>
46	52	<true/>
47	53	<key>UILaunchScreen</key>

apps/ios/Snowbound/CanvasView.swift+70-3

...	...	@@ -137,6 +137,8 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI
137	137	var onOpened: (() -> Void)?
138	138	/// Called when a tap on the page date asks to change it, with the date it shows.
139	139	var onDate: ((DateField, Date) -> Void)?
	140	/// Called when a tap on a recording, or beside a note linked to one, asks to play it.
	141	var onPlay: ((PlayRequest) -> Void)?
140	142	/// Formatting above the keyboard.
141	143	let formatBar = FormatBar()
142	144	/// Room above the page for a bar over it.
...	...	@@ -147,6 +149,16 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI
147	149	}
148	150	}
149	151
	152	/// Room below the page, above the keyboard, for a bar over its foot.
	153	var bottomInset: CGFloat = 0 {
	154	didSet {
	155	guard bottomInset != oldValue else { return }
	156	contentInset.bottom += bottomInset - oldValue
	157	verticalScrollIndicatorInsets.bottom = contentInset.bottom
	158	revealCaret()
	159	}
	160	}
	161
150	162	init(section: Section, page: String) {
151	163	self.section = section
152	164	self.page = page
...	...	@@ -480,8 +492,8 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI
480	492	else { return }
481	493	let keyboard = convert(frame, from: window.screen.coordinateSpace)
482	494	let overlap = max(0, bounds.maxY - keyboard.minY - safeAreaInsets.bottom)
483		contentInset.bottom = overlap
484		verticalScrollIndicatorInsets.bottom = overlap
	495	contentInset.bottom = overlap + bottomInset
	496	verticalScrollIndicatorInsets.bottom = contentInset.bottom
485	497	revealCaret()
486	498	}
487	499
...	...	@@ -542,7 +554,9 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI
542	554	func tap(at point: CGPoint) {
543	555	press(at: point)
544	556	var seconds: Int64 = 0
545		if let handle, let field = DateField(rawValue: sb_view_date_request(handle, &seconds)) {
	557	if let request = playRequest() {
	558	onPlay?(request)
	559	} else if let handle, let field = DateField(rawValue: sb_view_date_request(handle, &seconds)) {
546	560	onDate?(field, Date(timeIntervalSince1970: TimeInterval(seconds)))
547	561	} else if inkFrame != nil {
548	562	showInkMenu()
...	...	@@ -1142,6 +1156,59 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI
1142	1156	}
1143	1157	}
1144	1158
	1159	// MARK: Recordings
	1160
	1161	/// Starts recording at the caret, under a line saying when; whether it started.
	1162	func startRecording(video: Bool) -> Bool {
	1163	guard let handle else { return false }
	1164	let (day, time) = titleDate()
	1165	return edit(external: true) { sb_view_start_recording(handle, video, day, time) }
	1166	}
	1167
	1168	/// How far the recording has got in milliseconds, without its pauses, and whether it is
	1169	/// paused; nil when nothing records.
	1170	var recording: (ms: Int64, paused: Bool)? {
	1171	guard let handle else { return nil }
	1172	var paused = false
	1173	let ms = sb_view_recording(handle, &paused)
	1174	return ms < 0 ? nil : (ms, paused)
	1175	}
	1176
	1177	/// Nothing typed while the recording is paused links to it.
	1178	func pauseRecording(_ paused: Bool) {
	1179	guard let handle else { return }
	1180	sb_view_pause_recording(handle, paused)
	1181	}
	1182
	1183	/// Puts the recording made where it started: a PCM WAV file, or with `video` the AVI
	1184	/// file of `sb_movie_finish`. Nil forgets a recording whose file never came.
	1185	func finishRecording(_ file: Data?, video: Bool) {
	1186	guard let handle else { return }
	1187	_ = edit(external: true) {
	1188	guard let file else { return sb_view_finish_recording(handle, nil, 0, video) }
	1189	return file.withUnsafeBytes { bytes in
	1190	sb_view_finish_recording(handle, bytes.bindMemory(to: UInt8.self).baseAddress, bytes.count, video)
	1191	}
	1192	}
	1193	}
	1194
	1195	/// The recording the last tap asked to play, with its files written out to play from.
	1196	private func playRequest() -> PlayRequest? {
	1197	let folder = FileManager.default.temporaryDirectory.appendingPathComponent("Playback")
	1198	guard let handle, let json = sb_view_play_request(handle, folder.path) else { return nil }
	1199	defer { sb_string_free(json) }
	1200	let decoder = JSONDecoder()
	1201	decoder.keyDecodingStrategy = .convertFromSnakeCase
	1202	return try? decoder.decode(PlayRequest.self, from: Data(String(cString: json).utf8))
	1203	}
	1204
	1205	/// See Playback: the note linked at `ms` into the recording playing is highlighted; nil
	1206	/// ends playback.
	1207	func played(at ms: Int64?) {
	1208	guard let handle, sb_view_played(handle, ms ?? -1) else { return }
	1209	dirty = true
	1210	}
	1211
1145	1212	/// Commits marked text, so it is stored before the app leaves the screen.
1146	1213	func commitComposition() {
1147	1214	guard markedTextRange != nil else { return }

apps/ios/Snowbound/PageViewController.swift+116

...	...	@@ -21,6 +21,10 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
21	21	private let titleFocus: Bool
22	22	private lazy var canvas = CanvasView(section: section, page: page)
23	23	private let bar = ConflictBar()
	24	/// What records or plays on this page, over its foot.
	25	private let media = MediaBar()
	26	private var playback: Playback?
	27	private var mediaClock: Timer?
24	28	/// Reading View, beside the page menu where the page has one.
25	29	private var reader: UIBarButtonItem?
26	30	/// Opens another page or conflict page of the section.
...	...	@@ -62,7 +66,15 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
62	66	bar.translatesAutoresizingMaskIntoConstraints = false
63	67	view.addSubview(canvas)
64	68	view.addSubview(bar)
	69	media.translatesAutoresizingMaskIntoConstraints = false
	70	view.addSubview(media)
	71	let wide = media.widthAnchor.constraint(equalToConstant: 480)
	72	wide.priority = .required - 1
65	73	NSLayoutConstraint.activate([
	74	wide,
	75	media.centerXAnchor.constraint(equalTo: view.safeAreaLayoutGuide.centerXAnchor),
	76	media.leadingAnchor.constraint(greaterThanOrEqualTo: view.safeAreaLayoutGuide.leadingAnchor, constant: 12),
	77	media.bottomAnchor.constraint(equalTo: view.keyboardLayoutGuide.topAnchor, constant: -12),
66	78	canvas.leadingAnchor.constraint(equalTo: view.leadingAnchor),
67	79	canvas.trailingAnchor.constraint(equalTo: view.trailingAnchor),
68	80	canvas.topAnchor.constraint(equalTo: view.topAnchor),
...	...	@@ -75,6 +87,10 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
75	87	canvas.onChange = { [weak self] in self?.editingChanged() }
76	88	canvas.onOpened = { [weak self] in self?.opened() }
77	89	canvas.onDate = { [weak self] field, date in self?.pickDate(field, date) }
	90	canvas.onPlay = { [weak self] request in self?.play(request) }
	91	media.onPause = { [weak self] in self?.pauseMedia() }
	92	media.onStop = { [weak self] in self?.stopMedia() }
	93	NotificationCenter.default.addObserver(self, selector: #selector(showMedia), name: Recorder.changed, object: nil)
78	94	canvas.formatBar.onPicture = { [weak self] camera in self?.pickPicture(camera: camera) }
79	95	bar.onAction = { [weak self] in self?.conflictAction() }
80	96	NotificationCenter.default.addObserver(
...	...	@@ -173,6 +189,16 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
173	189	UIAction(
174	190	title: "Space", image: UIImage(systemName: "arrow.up.and.down.text.horizontal"), attributes: editable
175	191	) { [weak self] _ in self?.canvas.insertSpace() },
	192	UIMenu(options: .displayInline, children: [
	193	UIAction(
	194	title: "Record Audio", image: UIImage(systemName: "mic"),
	195	attributes: Recorder.current == nil ? editable : .disabled
	196	) { [weak self] _ in self?.record(video: false) },
	197	UIAction(
	198	title: "Record Video", image: UIImage(systemName: "video"),
	199	attributes: Recorder.current == nil && Recorder.canRecordVideo ? editable : .disabled
	200	) { [weak self] _ in self?.record(video: true) },
	201	]),
176	202	])
177	203	return (Prototype.reading ? [readingAction()] : []) + [
178	204	insert,
...	...	@@ -217,6 +243,8 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
217	243	}
218	244
219	245	private func toggleReading() {
	246	// A recording goes on the page as laid out.
	247	guard Recorder.current?.canvas !== canvas else { return }
220	248	canvas.showReading(!canvas.readingShown)
221	249	}
222	250
...	...	@@ -289,6 +317,94 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate
289	317
290	318	@objc private func leaving() { canvas.commitComposition() }
291	319
	320	override func viewWillDisappear(_ animated: Bool) {
	321	super.viewWillDisappear(animated)
	322	// A recording goes on the page it started on, as the desktop's does on leaving it.
	323	if Recorder.current?.canvas === canvas { Recorder.current?.stop() }
	324	stopPlayback()
	325	}
	326
	327	// MARK: Recording and playback
	328
	329	func record(video: Bool) {
	330	stopPlayback()
	331	Recorder.start(video: video, on: canvas) { [weak self] title, message in self?.alert(title, message) }
	332	}
	333
	334	func stopRecording() {
	335	if Recorder.current?.canvas === canvas { Recorder.current?.stop() }
	336	}
	337
	338	private func play(_ request: PlayRequest) {
	339	guard Recorder.current == nil else { return }
	340	playback = Playback(request)
	341	showMedia()
	342	}
	343
	344	private func stopPlayback() {
	345	guard playback != nil else { return }
	346	playback = nil
	347	canvas.played(at: nil)
	348	showMedia()
	349	}
	350
	351	private func pauseMedia() {
	352	if let recorder = Recorder.current, recorder.canvas === canvas {
	353	recorder.pause(!(recorder.elapsed?.paused ?? false))
	354	} else {
	355	playback?.toggle()
	356	}
	357	showMedia()
	358	}
	359
	360	private func stopMedia() {
	361	if Recorder.current?.canvas === canvas {
	362	Recorder.current?.stop()
	363	} else {
	364	stopPlayback()
	365	}
	366	}
	367
	368	/// The bar shows the clock of what records or plays, and a video's picture as it plays.
	369	@objc private func showMedia() {
	370	if let recorder = Recorder.current, recorder.canvas === canvas {
	371	media.picture.isHidden = true
	372	if recorder.saving {
	373	media.showSaving(recorder.video ? "Saving video…" : "Saving audio…")
	374	} else if let elapsed = recorder.elapsed {
	375	let state = elapsed.paused ? "Paused" : recorder.video ? "Recording video" : "Recording"
	376	media.show(
	377	"\(state) \(MediaBar.clock(elapsed.ms))", recording: true,
	378	paused: recorder.pausable ? elapsed.paused : nil)
	379	}
	380	} else if let playback {
	381	if playback.failed {
	382	stopPlayback()
	383	return alert("Couldn't Play the Recording", "This kind of recording doesn't play on this device.")
	384	}
	385	let total = playback.durationMs.map { " / " + MediaBar.clock($0) } ?? ""
	386	media.show(
	387	"\(playback.request.name) \(MediaBar.clock(playback.ms))\(total)", recording: false,
	388	paused: !playback.playing, playing: true)
	389	if let picture = playback.picture() { media.picture.image = picture }
	390	media.picture.isHidden = playback.request.video == nil
	391	canvas.played(at: playback.ms)
	392	} else {
	393	media.isHidden = true
	394	canvas.bottomInset = 0
	395	mediaClock?.invalidate()
	396	mediaClock = nil
	397	return
	398	}
	399	// The line being written stays above the bar.
	400	canvas.bottomInset = view.keyboardLayoutGuide.layoutFrame.minY - media.frame.minY
	401	if mediaClock == nil {
	402	mediaClock = Timer.scheduledTimer(withTimeInterval: 1.0 / 15, repeats: true) { [weak self] _ in
	403	self?.showMedia()
	404	}
	405	}
	406	}
	407
292	408	@objc private func sectionChanged(_ notification: Notification) {
293	409	let flags = notification.userInfo?["flags"] as? UInt32 ?? 0
294	410	if flags & Section.rejected != 0 {

apps/ios/Snowbound/Recording.swift created+545

...	...	@@ -0,0 +1,545 @@
	1	import AVFoundation
	2	import UIKit
	3
	4	/// A recording `sb_view_play_request` asks to play: its sound file from `atMs`, and for a
	5	/// video the AVI file whose JPEG pictures `frames` gives, `frameUs` apart.
	6	struct PlayRequest: Decodable {
	7	struct Video: Decodable {
	8	let path: String
	9	let frames: [[Int]]
	10	let frameUs: UInt32
	11	let size: [Int]
	12	}
	13	let name: String
	14	let sound: String
	15	let video: Video?
	16	let atMs: UInt32
	17	}
	18
	19	/// Record Audio and Record Video, stored as the desktop stores them: sound as mono PCM at
	20	/// `sb_recording_rate`, which the page keeps as IMA ADPCM, and video as a movie from the
	21	/// camera made into Motion JPEG AVI. Only one records at a time, and it keeps the page it
	22	/// records on until its file is there, whether or not the page is still shown.
	23	///
	24	/// Audio records on with the screen locked or the app in the background, under the audio
	25	/// background mode, as a lecture or meeting outlasts the screen's timeout; a call pauses it.
	26	/// The camera stops in the background, so a video recording then ends and is saved.
	27	final class Recorder: NSObject, AVCaptureFileOutputRecordingDelegate {
	28	/// The recording under way or being saved.
	29	private(set) static var current: Recorder?
	30	/// Posted when the current recording pauses, resumes, saves or ends.
	31	static let changed = Notification.Name("SnowboundRecordingChanged")
	32
	33	let video: Bool
	34	let canvas: CanvasView
	35	private let folder: URL
	36	private var sound: AVAudioRecorder?
	37	private var capture: AVCaptureSession?
	38	private let output = AVCaptureMovieFileOutput()
	39	#if targetEnvironment(simulator)
	40	private var stub: StubCamera?
	41	#endif
	42	/// Stopped, its file being made and put on the page.
	43	private(set) var saving = false
	44
	45	/// Whether video can be recorded here: with a camera, or the simulator's stand-in.
	46	static var canRecordVideo: Bool {
	47	#if targetEnvironment(simulator)
	48	true
	49	#else
	50	AVCaptureDevice.default(for: .video) != nil
	51	#endif
	52	}
	53
	54	private init(video: Bool, canvas: CanvasView) {
	55	self.video = video
	56	self.canvas = canvas
	57	folder = FileManager.default.temporaryDirectory.appendingPathComponent("Recording")
	58	super.init()
	59	}
	60
	61	/// Starts recording on `canvas` at its caret once the system allows the microphone, and
	62	/// for video the camera; `failed` says why it could not.
	63	static func start(video: Bool, on canvas: CanvasView, failed: @escaping (String, String) -> Void) {
	64	guard current == nil else { return }
	65	let title = video ? "Couldn't Record Video" : "Couldn't Record Audio"
	66	AVAudioApplication.requestRecordPermission { microphone in
	67	let begin = { (camera: Bool) in
	68	DispatchQueue.main.async {
	69	guard current == nil else { return }
	70	if video && !camera {
	71	return failed(title, "Allow Snowbound to use the camera in Settings, Privacy & Security, Camera.")
	72	}
	73	// A video without the microphone records silent, as on the desktop.
	74	if !video && !microphone {
	75	return failed(
	76	title, "Allow Snowbound to use the microphone in Settings, Privacy & Security, Microphone.")
	77	}
	78	let recorder = Recorder(video: video, canvas: canvas)
	79	do {
	80	try recorder.begin(microphone: microphone)
	81	} catch {
	82	recorder.end()
	83	return failed(title, "Try recording again.")
	84	}
	85	guard canvas.startRecording(video: video) else {
	86	recorder.end()
	87	return failed(title, "Try recording again.")
	88	}
	89	current = recorder
	90	NotificationCenter.default.post(name: changed, object: recorder)
	91	}
	92	}
	93	if video {
	94	#if targetEnvironment(simulator)
	95	begin(true)
	96	#else
	97	AVCaptureDevice.requestAccess(for: .video, completionHandler: begin)
	98	#endif
	99	} else {
	100	begin(true)
	101	}
	102	}
	103	}
	104
	105	private func begin(microphone: Bool) throws {
	106	try? FileManager.default.removeItem(at: folder)
	107	try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
	108	let session = AVAudioSession.sharedInstance()
	109	if microphone {
	110	try session.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetoothHFP])
	111	try session.setActive(true)
	112	NotificationCenter.default.addObserver(
	113	self, selector: #selector(interrupted), name: AVAudioSession.interruptionNotification, object: session)
	114	}
	115	#if targetEnvironment(simulator)
	116	if video {
	117	stub = try StubCamera(movie: folder.appendingPathComponent("camera.mov"))
	118	}
	119	#endif
	120	if !video \|\| isStubbed {
	121	guard microphone else { return }
	122	let settings: [String: Any] = [
	123	AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: sb_recording_rate(), AVNumberOfChannelsKey: 1,
	124	AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false,
	125	]
	126	let sound = try AVAudioRecorder(url: soundFile, settings: settings)
	127	guard sound.record() else { throw CocoaError(.fileWriteUnknown) }
	128	self.sound = sound
	129	return
	130	}
	131	let capture = AVCaptureSession()
	132	capture.beginConfiguration()
	133	if capture.canSetSessionPreset(.vga640x480) { capture.sessionPreset = .vga640x480 }
	134	let devices = [AVCaptureDevice.default(for: .video)] + (microphone ? [AVCaptureDevice.default(for: .audio)] : [])
	135	for (index, device) in devices.enumerated() {
	136	guard let device, let input = try? AVCaptureDeviceInput(device: device), capture.canAddInput(input) else {
	137	// Without the microphone the video records silent; without the camera, nothing.
	138	if index == 0 { throw CocoaError(.fileWriteUnknown) }
	139	continue
	140	}
	141	capture.addInput(input)
	142	}
	143	guard capture.canAddOutput(output) else { throw CocoaError(.fileWriteUnknown) }
	144	capture.addOutput(output)
	145	capture.commitConfiguration()
	146	capture.startRunning()
	147	output.startRecording(to: movieFile, recordingDelegate: self)
	148	self.capture = capture
	149	}
	150
	151	private var isStubbed: Bool {
	152	#if targetEnvironment(simulator)
	153	stub != nil
	154	#else
	155	false
	156	#endif
	157	}
	158
	159	private var soundFile: URL { folder.appendingPathComponent("sound.wav") }
	160	private var movieFile: URL { folder.appendingPathComponent("camera.mov") }
	161
	162	/// How far the recording has got, its pauses left out, and whether it is paused.
	163	var elapsed: (ms: Int64, paused: Bool)? { canvas.recording }
	164
	165	/// Whether Pause works: always but for a camera before iOS 18.
	166	var pausable: Bool {
	167	if capture == nil { return true }
	168	if #available(iOS 18, *) { return true }
	169	return false
	170	}
	171
	172	func pause(_ paused: Bool) {
	173	guard !saving, pausable else { return }
	174	if paused {
	175	sound?.pause()
	176	} else {
	177	sound?.record()
	178	}
	179	#if targetEnvironment(simulator)
	180	stub?.paused = paused
	181	#endif
	182	if #available(iOS 18, *), capture != nil {
	183	if paused { output.pauseRecording() } else { output.resumeRecording() }
	184	}
	185	canvas.pauseRecording(paused)
	186	NotificationCenter.default.post(name: Self.changed, object: self)
	187	}
	188
	189	/// A call or another app's audio takes the microphone: the recording pauses, and goes
	190	/// on after where the system says it should.
	191	@objc private func interrupted(_ notification: Notification) {
	192	guard let raw = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt,
	193	let type = AVAudioSession.InterruptionType(rawValue: raw)
	194	else { return }
	195	DispatchQueue.main.async { [self] in
	196	switch type {
	197	case .began where capture == nil: pause(true)
	198	case .ended:
	199	let options = (notification.userInfo?[AVAudioSessionInterruptionOptionKey] as? UInt)
	200	.map(AVAudioSession.InterruptionOptions.init) ?? []
	201	if options.contains(.shouldResume), capture == nil {
	202	try? AVAudioSession.sharedInstance().setActive(true)
	203	pause(false)
	204	}
	205	default: break
	206	}
	207	}
	208	}
	209
	210	/// Stop: the recording's file is made and goes on the page where it started.
	211	func stop() {
	212	guard !saving else { return }
	213	saving = true
	214	canvas.pauseRecording(true)
	215	NotificationCenter.default.post(name: Self.changed, object: self)
	216	if capture != nil {
	217	// The movie is saved once `fileOutput(_:didFinishRecordingTo:…)` hears so.
	218	output.stopRecording()
	219	return
	220	}
	221	sound?.stop()
	222	#if targetEnvironment(simulator)
	223	if let stub {
	224	stub.finish { [self] in convert(movie: stub.movie) }
	225	return
	226	}
	227	#endif
	228	save(try? Data(contentsOf: soundFile))
	229	}
	230
	231	/// The camera stopped: at Stop, or as the app left the screen or a call took the camera.
	232	func fileOutput(
	233	_ output: AVCaptureFileOutput, didFinishRecordingTo url: URL, from connections: [AVCaptureConnection],
	234	error: Error?
	235	) {
	236	let finished = (error as NSError?)?.userInfo[AVErrorRecordingSuccessfullyFinishedKey] as? Bool ?? (error == nil)
	237	DispatchQueue.main.async { [self] in
	238	saving = true
	239	canvas.pauseRecording(true)
	240	NotificationCenter.default.post(name: Self.changed, object: self)
	241	capture?.stopRunning()
	242	if finished { convert(movie: url) } else { save(nil) }
	243	}
	244	}
	245
	246	/// Makes the AVI file of `movie`, with the sound recorded beside it where there is.
	247	private func convert(movie: URL) {
	248	let sound = self.sound == nil ? movie : soundFile
	249	// Finishing is not cut short by the app leaving the screen.
	250	let task = UIApplication.shared.beginBackgroundTask()
	251	Task.detached {
	252	let avi = await avi(pictures: movie, sound: sound)
	253	await MainActor.run {
	254	self.save(avi)
	255	UIApplication.shared.endBackgroundTask(task)
	256	}
	257	}
	258	}
	259
	260	/// Puts `file` on the page, or with none forgets the recording.
	261	private func save(_ file: Data?) {
	262	canvas.finishRecording(file, video: video)
	263	end()
	264	Self.current = nil
	265	NotificationCenter.default.post(name: Self.changed, object: self)
	266	}
	267
	268	private func end() {
	269	NotificationCenter.default.removeObserver(self)
	270	capture?.stopRunning()
	271	try? FileManager.default.removeItem(at: folder)
	272	try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
	273	}
	274	}
	275
	276	/// The AVI file the desktop makes of a movie: the pictures of `pictures`' first video track
	277	/// and the sound of `sound`'s first audio track, mono PCM at `sb_recording_rate`.
	278	private func avi(pictures: URL, sound: URL) async -> Data? {
	279	let movie = sb_movie_new()
	280	let asset = AVURLAsset(url: pictures)
	281	await read(asset, .video, [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]) { sample in
	282	let time = CMSampleBufferGetPresentationTimeStamp(sample)
	283	guard time.isValid, let buffer = CMSampleBufferGetImageBuffer(sample) else { return }
	284	CVPixelBufferLockBaseAddress(buffer, .readOnly)
	285	defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) }
	286	guard let base = CVPixelBufferGetBaseAddress(buffer) else { return }
	287	sb_movie_picture(
	288	movie, UInt64(max(0, time.seconds) * 1_000_000), base.assumingMemoryBound(to: UInt8.self),
	289	UInt32(CVPixelBufferGetWidth(buffer)), UInt32(CVPixelBufferGetHeight(buffer)),
	290	CVPixelBufferGetBytesPerRow(buffer))
	291	}
	292	let pcm: [String: Any] = [
	293	AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: sb_recording_rate(), AVNumberOfChannelsKey: 1,
	294	AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false,
	295	AVLinearPCMIsNonInterleaved: false,
	296	]
	297	// A movie without sound has no sound track to read.
	298	await read(AVURLAsset(url: sound), .audio, pcm) { sample in
	299	guard let buffer = CMSampleBufferGetDataBuffer(sample) else { return }
	300	let length = CMBlockBufferGetDataLength(buffer)
	301	var bytes = [UInt8](repeating: 0, count: length)
	302	if CMBlockBufferCopyDataBytes(buffer, atOffset: 0, dataLength: length, destination: &bytes) == noErr {
	303	sb_movie_sound(movie, bytes, length)
	304	}
	305	}
	306	let duration = (try? await asset.load(.duration))?.seconds ?? 0
	307	var length = 0
	308	guard let bytes = sb_movie_finish(movie, UInt64(max(0, duration) * 1_000_000), &length) else { return nil }
	309	defer { sb_bytes_free(bytes, length) }
	310	return Data(bytes: bytes, count: length)
	311	}
	312
	313	/// Reads the first track of `kind` in `asset`, decoded as `settings` ask, a sample at a time.
	314	private func read(
	315	_ asset: AVURLAsset, _ kind: AVMediaType, _ settings: [String: Any], _ sample: (CMSampleBuffer) -> Void
	316	) async {
	317	guard let track = try? await asset.loadTracks(withMediaType: kind).first,
	318	let reader = try? AVAssetReader(asset: asset)
	319	else { return }
	320	let output = AVAssetReaderTrackOutput(track: track, outputSettings: settings)
	321	guard reader.canAdd(output) else { return }
	322	reader.add(output)
	323	guard reader.startReading() else { return }
	324	while let next = output.copyNextSampleBuffer() {
	325	sample(next)
	326	}
	327	}
	328
	329	#if targetEnvironment(simulator)
	330	/// The simulator's stand-in for a camera, which it has none of: moving colour bars and the
	331	/// time written as a movie, 15 pictures a second, while the microphone records beside it.
	332	private final class StubCamera {
	333	let movie: URL
	334	private let writer: AVAssetWriter
	335	private let input: AVAssetWriterInput
	336	private let adaptor: AVAssetWriterInputPixelBufferAdaptor
	337	private var timer: Timer?
	338	private var frame: Int64 = 0
	339	var paused = false
	340
	341	init(movie: URL) throws {
	342	self.movie = movie
	343	writer = try AVAssetWriter(outputURL: movie, fileType: .mov)
	344	input = AVAssetWriterInput(
	345	mediaType: .video,
	346	outputSettings: [AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: 640, AVVideoHeightKey: 480])
	347	input.expectsMediaDataInRealTime = true
	348	adaptor = AVAssetWriterInputPixelBufferAdaptor(
	349	assetWriterInput: input,
	350	sourcePixelBufferAttributes: [
	351	kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
	352	kCVPixelBufferWidthKey as String: 640, kCVPixelBufferHeightKey as String: 480,
	353	])
	354	writer.add(input)
	355	guard writer.startWriting() else { throw writer.error ?? CocoaError(.fileWriteUnknown) }
	356	writer.startSession(atSourceTime: .zero)
	357	timer = Timer.scheduledTimer(withTimeInterval: 1.0 / 15, repeats: true) { [weak self] _ in self?.shoot() }
	358	}
	359
	360	private func shoot() {
	361	guard !paused, input.isReadyForMoreMediaData, let pool = adaptor.pixelBufferPool else { return }
	362	var made: CVPixelBuffer?
	363	CVPixelBufferPoolCreatePixelBuffer(nil, pool, &made)
	364	guard let buffer = made else { return }
	365	CVPixelBufferLockBaseAddress(buffer, [])
	366	let context = CGContext(
	367	data: CVPixelBufferGetBaseAddress(buffer), width: 640, height: 480, bitsPerComponent: 8,
	368	bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: CGColorSpaceCreateDeviceRGB(),
	369	bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue \| CGBitmapInfo.byteOrder32Little.rawValue)
	370	if let context {
	371	let colors: [UIColor] = [.systemYellow, .systemTeal, .systemGreen, .systemPink, .systemRed, .systemBlue]
	372	for bar in 0..<8 {
	373	context.setFillColor(colors[(bar + Int(frame / 5)) % colors.count].cgColor)
	374	context.fill(CGRect(x: bar * 80, y: 0, width: 80, height: 480))
	375	}
	376	UIGraphicsPushContext(context)
	377	context.translateBy(x: 0, y: 480)
	378	context.scaleBy(x: 1, y: -1)
	379	let seconds = Double(frame) / 15
	380	NSString(string: String(format: "%.1f s", seconds)).draw(
	381	at: CGPoint(x: 24, y: 24),
	382	withAttributes: [.font: UIFont.monospacedDigitSystemFont(ofSize: 72, weight: .bold), .foregroundColor: UIColor.black])
	383	UIGraphicsPopContext()
	384	}
	385	CVPixelBufferUnlockBaseAddress(buffer, [])
	386	adaptor.append(buffer, withPresentationTime: CMTime(value: frame, timescale: 15))
	387	frame += 1
	388	}
	389
	390	func finish(_ done: @escaping () -> Void) {
	391	timer?.invalidate()
	392	input.markAsFinished()
	393	writer.endSession(atSourceTime: CMTime(value: frame, timescale: 15))
	394	writer.finishWriting { DispatchQueue.main.async(execute: done) }
	395	}
	396	}
	397	#endif
	398
	399	/// Plays a recording: its sound, and a video's pictures in time with it.
	400	final class Playback {
	401	let request: PlayRequest
	402	let player: AVPlayer
	403	private let pictures: Data?
	404	private var shown = -1
	405
	406	init(_ request: PlayRequest) {
	407	self.request = request
	408	player = AVPlayer(url: URL(fileURLWithPath: request.sound))
	409	pictures = request.video.flatMap { try? Data(contentsOf: URL(fileURLWithPath: $0.path), options: .alwaysMapped) }
	410	try? AVAudioSession.sharedInstance().setCategory(.playback, mode: .spokenAudio)
	411	try? AVAudioSession.sharedInstance().setActive(true)
	412	player.seek(to: CMTime(value: CMTimeValue(request.atMs), timescale: 1000), toleranceBefore: .zero, toleranceAfter: .zero)
	413	player.play()
	414	}
	415
	416	deinit {
	417	player.pause()
	418	try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
	419	}
	420
	421	/// Whether the player could not open the sound, as with OneNote's own Windows Media.
	422	var failed: Bool { player.currentItem?.status == .failed }
	423
	424	var ms: Int64 { Int64(max(0, player.currentTime().seconds) * 1000) }
	425
	426	var durationMs: Int64? {
	427	guard let duration = player.currentItem?.duration, duration.isNumeric else { return nil }
	428	return Int64(duration.seconds * 1000)
	429	}
	430
	431	var playing: Bool { player.rate != 0 }
	432
	433	func toggle() {
	434	if playing {
	435	player.pause()
	436	} else {
	437	if let durationMs, ms >= durationMs - 50 { player.seek(to: .zero) }
	438	player.play()
	439	}
	440	}
	441
	442	/// The video's picture now, when it changed since last asked.
	443	func picture() -> UIImage? {
	444	guard let video = request.video, let pictures, !video.frames.isEmpty else { return nil }
	445	let index = min(Int(ms * 1000 / Int64(max(1, video.frameUs))), video.frames.count - 1)
	446	guard index != shown else { return nil }
	447	shown = index
	448	let range = video.frames[index]
	449	guard range.count == 2, range[0] < range[1], range[1] <= pictures.count else { return nil }
	450	return UIImage(data: pictures.subdata(in: range[0]..<range[1]))
	451	}
	452	}
	453
	454	/// The bar over the page's foot while something records or plays, as OneNote's Recording and
	455	/// Playback tabs: the clock, Pause, and Stop; a video's pictures above as it plays.
	456	final class MediaBar: UIView {
	457	private let dot = UIImageView(image: UIImage(systemName: "record.circle.fill"))
	458	private let label = UILabel()
	459	private let pause = UIButton(configuration: .plain())
	460	private let stop = UIButton(configuration: .plain())
	461	let picture = UIImageView()
	462	var onPause: (() -> Void)?
	463	var onStop: (() -> Void)?
	464
	465	init() {
	466	super.init(frame: .zero)
	467	let blur = UIVisualEffectView(effect: UIBlurEffect(style: .systemThickMaterial))
	468	blur.translatesAutoresizingMaskIntoConstraints = false
	469	blur.layer.cornerRadius = 14
	470	blur.clipsToBounds = true
	471	addSubview(blur)
	472	layer.shadowOpacity = 0.15
	473	layer.shadowRadius = 8
	474	layer.shadowOffset = CGSize(width: 0, height: 2)
	475	dot.tintColor = .systemRed
	476	dot.setContentHuggingPriority(.required, for: .horizontal)
	477	label.font = .monospacedDigitSystemFont(ofSize: UIFont.preferredFont(forTextStyle: .subheadline).pointSize, weight: .medium)
	478	label.adjustsFontForContentSizeCategory = true
	479	label.lineBreakMode = .byTruncatingMiddle
	480	label.accessibilityTraits.insert(.updatesFrequently)
	481	stop.configuration?.image = UIImage(systemName: "stop.fill")
	482	stop.accessibilityLabel = "Stop"
	483	stop.addAction(UIAction { [weak self] _ in self?.onStop?() }, for: .primaryActionTriggered)
	484	pause.addAction(UIAction { [weak self] _ in self?.onPause?() }, for: .primaryActionTriggered)
	485	for button in [pause, stop] { button.setContentHuggingPriority(.required, for: .horizontal) }
	486	picture.contentMode = .scaleAspectFit
	487	picture.backgroundColor = .black
	488	picture.layer.cornerRadius = 6
	489	picture.clipsToBounds = true
	490	picture.isHidden = true
	491	let controls = UIStackView(arrangedSubviews: [dot, label, pause, stop])
	492	controls.spacing = 8
	493	controls.alignment = .center
	494	let stack = UIStackView(arrangedSubviews: [picture, controls])
	495	stack.axis = .vertical
	496	stack.spacing = 8
	497	stack.translatesAutoresizingMaskIntoConstraints = false
	498	blur.contentView.addSubview(stack)
	499	NSLayoutConstraint.activate([
	500	blur.leadingAnchor.constraint(equalTo: leadingAnchor),
	501	blur.trailingAnchor.constraint(equalTo: trailingAnchor),
	502	blur.topAnchor.constraint(equalTo: topAnchor),
	503	blur.bottomAnchor.constraint(equalTo: bottomAnchor),
	504	stack.leadingAnchor.constraint(equalTo: blur.contentView.leadingAnchor, constant: 14),
	505	stack.trailingAnchor.constraint(equalTo: blur.contentView.trailingAnchor, constant: -6),
	506	stack.topAnchor.constraint(equalTo: blur.contentView.topAnchor, constant: 8),
	507	stack.bottomAnchor.constraint(equalTo: blur.contentView.bottomAnchor, constant: -8),
	508	picture.widthAnchor.constraint(lessThanOrEqualToConstant: 320),
	509	])
	510	// Below required, as the stack gives the picture no height while it is hidden.
	511	let aspect = picture.heightAnchor.constraint(equalTo: picture.widthAnchor, multiplier: 3.0 / 4)
	512	aspect.priority = .defaultHigh
	513	aspect.isActive = true
	514	isHidden = true
	515	}
	516
	517	required init?(coder: NSCoder) { fatalError() }
	518
	519	/// Shows `text`, the dot while recording, and Pause or Resume (or Play) where it works.
	520	func show(_ text: String, recording: Bool, paused: Bool?, playing: Bool = false) {
	521	isHidden = false
	522	label.text = text
	523	dot.isHidden = !recording
	524	dot.alpha = paused == true ? 0.4 : 1
	525	pause.isHidden = paused == nil
	526	stop.isHidden = false
	527	let resume = paused == true
	528	pause.configuration?.image = UIImage(systemName: resume ? (playing ? "play.fill" : "record.circle") : "pause.fill")
	529	pause.accessibilityLabel = resume ? (playing ? "Play" : "Resume") : "Pause"
	530	}
	531
	532	/// Saving: only the clock's place says so.
	533	func showSaving(_ text: String) {
	534	show(text, recording: false, paused: nil)
	535	stop.isHidden = true
	536	}
	537
	538	/// `ms` as a clock: minutes and seconds, hours when there are some.
	539	static func clock(_ ms: Int64) -> String {
	540	let seconds = ms / 1000
	541	return seconds >= 3600
	542	? String(format: "%d:%02d:%02d", seconds / 3600, seconds / 60 % 60, seconds % 60)
	543	: String(format: "%d:%02d", seconds / 60, seconds % 60)
	544	}
	545	}

apps/ios/Snowbound/Script.swift+13-3

...	...	@@ -14,17 +14,22 @@ extension CanvasView {
14	14	/// `tool:pen:N`, `tool:shape:N`, `tool:eraser` or `tool:lasso` gives it a tool, `picker` shows or hides the
15	15	/// drawing tools, `pencildoubletap` is the Pencil's double tap, and `deleteink` deletes
16	16	/// what the lasso picked; `rotate:landscape` or `rotate:portrait` turns the device; `done`
17		/// ends editing, `reading` turns the reading view on or off, `tree` saves the view hierarchy to
	17	/// ends editing, `reading` turns the reading view on or off, `record:audio` or
	18	/// `record:video` records, `pauserecording` pauses or resumes and `stoprecording` stops it,
	19	/// `wait:N` waits N seconds more before the next step, `tree` saves the view hierarchy to
18	20	/// Documents/tree.txt, and `shot:a` the window to Documents/a.png, as a device has no
19	21	/// screenshot command.
20	22	func runScript() {
21	23	// Only the first page shown runs it, not every page opened afterwards.
22	24	guard !scriptedPage, let script = ProcessInfo.processInfo.environment["SNOWBOUND_SCRIPT"] else { return }
23	25	scriptedPage = true
24		for (index, step) in script.split(separator: "\|").enumerated() {
25		DispatchQueue.main.asyncAfter(deadline: .now() + 1 + Double(index) * 0.4) { [weak self] in
	26	var delay = 1.0
	27	for step in script.split(separator: "\|") {
	28	if step.hasPrefix("wait:") { delay += Double(step.dropFirst(5)) ?? 0 }
	29	DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [weak self] in
26	30	self?.perform(String(step))
27	31	}
	32	delay += 0.4
28	33	}
29	34	}
30	35
...	...	@@ -90,6 +95,11 @@ extension CanvasView {
90	95	let caret = caretRect(for: range.start)
91	96	menu.presentEditMenu(with: UIEditMenuConfiguration(identifier: nil, sourcePoint: CGPoint(x: caret.midX, y: caret.minY)))
92	97	case "done": _ = resignFirstResponder()
	98	case "wait": break
	99	case "record", "stoprecording":
	100	let page = sequence(first: self as UIResponder, next: \.next).lazy.compactMap { $0 as? PageViewController }.first
	101	if command == "record" { page?.record(video: argument == "video") } else { page?.stopRecording() }
	102	case "pauserecording": Recorder.current.map { $0.pause(!($0.elapsed?.paused ?? false)) }
93	103	case "selectall": selectAll(nil)
94	104	case "reading": showReading(!readingShown)
95	105	case "format": apply(UInt8(argument) ?? 0)

arc/canvas.md+2-1

...	...	@@ -167,7 +167,8 @@ scrolling it into view when it changes.
167	167
168	168	The host records audio as 16 kHz WAV and stores it as IMA ADPCM, and video as
169	169	AVI of Motion JPEG at 320 by 240 and 15 pictures a second with PCM sound, both
170		of which OneNote plays (`corpus/recording/video`). GStreamer writes that AVI
	170	of which OneNote plays (`corpus/recording/video`); `canvas::recording` makes both,
	171	so every host stores the same bytes. GStreamer writes that AVI
171	172	directly; on macOS a capture session records a movie that `AVAssetReader` reads
172	173	back into it off the main thread after Stop, the transport saying so and the page
173	174	editable meanwhile. Mac OS X 10.6, without AVFoundation, opens recordings in the

arc/platforms.md+7

...	...	@@ -208,6 +208,13 @@ the same ops. Notebooks from Files
208	208	are read and written under `NSFileCoordinator`, so file providers see every
209	209	change.
210	210
	211	Record Audio and Record Video record through AVFoundation and hand `crates/mobile` the
	212	WAV file, or the camera's pictures and sound, which it stores as the desktop does. Audio
	213	keeps recording in the background and with the screen locked, under the `audio`
	214	background mode, as a lecture outlasts the screen's timeout; a call pauses it until the
	215	system says to go on. The camera stops in the background, which ends a video recording
	216	and saves it. A tap on a recording plays it with `AVPlayer` in a bar over the page's foot.
	217
211	218	## What stays shared, on purpose
212	219
213	220	- Behaviour: the editor, hit-testing, the placement grid, undo, conflicts

crates/canvas/src/editor/recording.rs+5

...	...	@@ -161,6 +161,11 @@ impl CanvasEditor {
161	161	Ok(id)
162	162	}
163	163
	164	/// Forgets the recording under way, whose file never came; its line stays.
	165	pub fn cancel_recording(&mut self) {
	166	self.recording = None;
	167	}
	168
164	169	/// Stops recording: `file`, the recording made, goes where it started, linked to its
165	170	/// start, or at the caret when that place is gone.
166	171	pub fn finish_recording(

crates/canvas/src/gpu/page.rs+3

...	...	@@ -203,6 +203,9 @@ fn between(from: [f32; 2], middle: [f32; 2], to: [f32; 2]) -> bool {
203	203	(0.0..=1.0).contains(&along) && (mx * dy - my * dx).abs() / length.sqrt() <= 0.1
204	204	}
205	205
	206	/// The size OneNote 2010 shows a file's icon at, in points.
	207	pub const ICON_SIZE: [f32; 2] = [24.0, 24.0];
	208
206	209	const FILE_ICON: &str = include_str!("../../assets/file.svg");
207	210
208	211	/// The blank page for a file without an icon of its own, as the 32 pixel PNG OneNote 2010

crates/canvas/src/interaction/mod.rs+8

...	...	@@ -1261,6 +1261,14 @@ impl PageView {
1261	1261	Some(Hit::File(id)) => {
1262	1262	self.set_object_focus(Some(ObjectFocus::File(id)));
1263	1263	self.drag = None;
	1264	// A finger plays a recording with a tap, having no play button to hover.
	1265	if self.touch
	1266	&& let Some(file) = self.editor.attachment(id)
	1267	&& file.recording.is_some()
	1268	{
	1269	let file = file.clone();
	1270	return Ok(Response::request(Request::Play { file, at_ms: 0 }));
	1271	}
1264	1272	if unit != SelectionUnit::Grapheme
1265	1273	&& let Some(file) = self.editor.attachment(id)
1266	1274	{

crates/canvas/src/lib.rs+2

...	...	@@ -17,6 +17,8 @@ pub mod gpu;
17	17	pub mod interaction;
18	18	#[cfg(feature = "pdf")]
19	19	pub mod print;
	20	#[cfg(feature = "gpu")]
	21	pub mod recording;
20	22
21	23	/// Where an object stored with `layout` lies; an unset coordinate is zero.
22	24	pub(crate) fn origin(layout: &onestore::document::Layout) -> [f32; 2] {

crates/canvas/src/recording.rs created+424

...	...	@@ -0,0 +1,424 @@
	1	//! Recordings as Snowbound stores them, which OneNote 2010 plays (`corpus/recording`):
	2	//! audio as 16 kHz IMA ADPCM WAV, video as the Motion JPEG AVI files of `video`.
	3
	4	pub mod video;
	5
	6	use onestore::page::{Attachment, Recording};
	7
	8	/// The rate audio is recorded at: wideband speech, as OneNote's own 8 kHz WMA is narrowband.
	9	pub const RATE: u32 = 16_000;
	10
	11	/// The line saying when a recording started, as OneNote writes it from the long `date` and
	12	/// short `time`.
	13	pub fn label(video: bool, date: &str, time: &str) -> String {
	14	let kind = if video { "Video" } else { "Audio" };
	15	format!("{kind} recording started: {time} {date}")
	16	}
	17
	18	/// The file recording `id` on page `title` is stored as, from what was recorded: a mono
	19	/// 16-bit PCM WAV file, which is compressed, or a `video` AVI file. It has no icon yet.
	20	pub fn file(
	21	id: [u8; 16],
	22	title: &str,
	23	video: bool,
	24	recorded: Vec<u8>,
	25	) -> Result<Attachment, onestore::page::text::EditError> {
	26	let (bytes, duration_ms) = if video {
	27	let duration = video::Movie::parse(&recorded).map(\|movie\| movie.duration_ms());
	28	(recorded, duration)
	29	} else {
	30	match compress(&recorded) {
	31	Some((bytes, duration)) => (bytes, Some(duration)),
	32	None => {
	33	let duration = Wave::parse(&recorded).map(\|wave\| wave.duration_ms());
	34	(recorded, duration)
	35	}
	36	}
	37	};
	38	let name: String = title
	39	.chars()
	40	.map(\|c\| {
	41	if matches!(c, '/' \| '\\' \| '\0') {
	42	'_'
	43	} else {
	44	c
	45	}
	46	})
	47	.collect();
	48	let name = if name.trim().is_empty() {
	49	if video {
	50	"Video recording"
	51	} else {
	52	"Audio recording"
	53	}
	54	.to_owned()
	55	} else {
	56	name
	57	};
	58	let extension = if video { "avi" } else { "wav" };
	59	Ok(Attachment {
	60	id: onestore::page::text::new_id()?,
	61	filename: format!("{name}.{extension}"),
	62	source_path: None,
	63	size: Some(crate::gpu::page::ICON_SIZE),
	64	layout: Default::default(),
	65	bytes: Some(bytes.into()),
	66	preview: None,
	67	recording: Some(Recording {
	68	id,
	69	kind: if video { 2 } else { 1 },
	70	duration_ms,
	71	}),
	72	tags: Vec::new(),
	73	})
	74	}
	75
	76	/// A recording's sound as a 16-bit PCM WAV file every platform's player decodes: IMA ADPCM
	77	/// decoded, a video's sound taken out of it. None for other files, which play as they are.
	78	pub fn sound(bytes: &[u8]) -> Option<Vec<u8>> {
	79	let sound = match video::Movie::parse(bytes) {
	80	Some(movie) => movie
	81	.wave(bytes)
	82	.unwrap_or_else(\|\| silence(movie.duration_ms())),
	83	None => return decompress(bytes),
	84	};
	85	Some(decompress(&sound).unwrap_or(sound))
	86	}
	87
	88	/// Silence as long as `ms`, the clock of a video without sound.
	89	fn silence(ms: u32) -> Vec<u8> {
	90	wave(8_000, &vec![0; ms as usize * 8])
	91	}
	92
	93	/// A mono 16-bit PCM WAV file of `samples` at `rate`, as the platform recorders write.
	94	pub fn wave(rate: u32, samples: &[i16]) -> Vec<u8> {
	95	let data: Vec<u8> = samples.iter().flat_map(\|s\| s.to_le_bytes()).collect();
	96	[
	97	&b"RIFF"[..],
	98	&(36 + data.len() as u32).to_le_bytes(),
	99	b"WAVEfmt ",
	100	&16_u32.to_le_bytes(),
	101	&1_u16.to_le_bytes(),
	102	&1_u16.to_le_bytes(),
	103	&rate.to_le_bytes(),
	104	&(rate * 2).to_le_bytes(),
	105	&2_u16.to_le_bytes(),
	106	&16_u16.to_le_bytes(),
	107	b"data",
	108	&(data.len() as u32).to_le_bytes(),
	109	&data,
	110	]
	111	.concat()
	112	}
	113
	114	/// A recording for a file attached under `filename`, as OneNote 2010 makes one of every
	115	/// audio and video file it attaches, with the length of a WAV file's `bytes`.
	116	pub fn attached(filename: &str, bytes: &[u8]) -> Option<Recording> {
	117	Some(Recording {
	118	id: onestore::page::text::new_guid().ok()?,
	119	kind: Recording::kind_of(filename)?,
	120	duration_ms: Wave::parse(bytes).map(\|wave\| wave.duration_ms()),
	121	})
	122	}
	123
	124	/// The chunks of a RIFF WAVE file this module reads.
	125	struct Wave<'a> {
	126	format: u16,
	127	channels: u16,
	128	rate: u32,
	129	bytes_per_second: u32,
	130	block: u16,
	131	bits: u16,
	132	/// The sample count a compressed file's `fact` chunk states.
	133	samples: Option<u32>,
	134	data: &'a [u8],
	135	}
	136
	137	impl<'a> Wave<'a> {
	138	fn parse(bytes: &'a [u8]) -> Option<Self> {
	139	if bytes.get(..4)? != b"RIFF" \|\| bytes.get(8..12)? != b"WAVE" {
	140	return None;
	141	}
	142	let u16_at = \|data: &[u8], at: usize\| {
	143	Some(u16::from_le_bytes(data.get(at..at + 2)?.try_into().ok()?))
	144	};
	145	let u32_at = \|data: &[u8], at: usize\| {
	146	Some(u32::from_le_bytes(data.get(at..at + 4)?.try_into().ok()?))
	147	};
	148	let (mut fmt, mut samples, mut data) = (None, None, None);
	149	let mut at = 12;
	150	while at + 8 <= bytes.len() {
	151	let size = u32_at(bytes, at + 4)? as usize;
	152	let body = &bytes[at + 8..(at + 8).saturating_add(size).min(bytes.len())];
	153	match &bytes[at..at + 4] {
	154	b"fmt " => fmt = Some(body),
	155	b"fact" => samples = u32_at(body, 0),
	156	b"data" => data = Some(body),
	157	_ => {}
	158	}
	159	at = at + 8 + size + size % 2;
	160	}
	161	let fmt = fmt?;
	162	Some(Self {
	163	format: u16_at(fmt, 0)?,
	164	channels: u16_at(fmt, 2)?,
	165	rate: u32_at(fmt, 4)?,
	166	bytes_per_second: u32_at(fmt, 8)?,
	167	block: u16_at(fmt, 12)?,
	168	bits: u16_at(fmt, 14)?,
	169	samples,
	170	data: data?,
	171	})
	172	}
	173
	174	fn duration_ms(&self) -> u32 {
	175	let ms = match self.samples.filter(\|_\| self.format != 1) {
	176	Some(samples) => u64::from(samples) * 1000 / u64::from(self.rate.max(1)),
	177	None => self.data.len() as u64 * 1000 / u64::from(self.bytes_per_second.max(1)),
	178	};
	179	u32::try_from(ms).unwrap_or(u32::MAX)
	180	}
	181	}
	182
	183	/// IMA ADPCM's step sizes and how each code moves along them.
	184	const STEPS: [i32; 89] = [
	185	7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66,
	186	73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449,
	187	494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, 2272,
	188	2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358, 5894, 6484, 7132, 7845, 8630, 9493,
	189	10442, 11487, 12635, 13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767,
	190	];
	191	const INDEX_STEPS: [i32; 8] = [-1, -1, -1, -1, 2, 4, 6, 8];
	192
	193	/// Bytes per block: Windows' IMA ADPCM codec's choice for 16 kHz.
	194	const BLOCK: usize = 512;
	195	/// Samples per block: the header's, then two per remaining byte.
	196	const BLOCK_SAMPLES: usize = (BLOCK - 4) * 2 + 1;
	197
	198	/// A mono 16-bit PCM WAV file as 4-bit IMA ADPCM (WAVE_FORMAT_DVI_ADPCM), which Windows
	199	/// and macOS decode, a quarter of the size; and its length. None for other files.
	200	pub fn compress(wav: &[u8]) -> Option<(Vec<u8>, u32)> {
	201	let wave = Wave::parse(wav)?;
	202	if wave.format != 1 \|\| wave.channels != 1 \|\| wave.bits != 16 {
	203	return None;
	204	}
	205	let samples: Vec<i16> = wave
	206	.data
	207	.chunks_exact(2)
	208	.map(\|pair\| i16::from_le_bytes([pair[0], pair[1]]))
	209	.collect();
	210	let mut data = Vec::with_capacity(samples.len() / 2 + BLOCK);
	211	let mut index = 0_i32;
	212	for block in samples.chunks(BLOCK_SAMPLES) {
	213	let mut predicted = i32::from(block[0]);
	214	data.extend_from_slice(&block[0].to_le_bytes());
	215	data.extend_from_slice(&[index as u8, 0]);
	216	// A short last block is padded with its last sample; `fact` gives the true length.
	217	let last = *block.last().unwrap();
	218	let rest = block[1..]
	219	.iter()
	220	.copied()
	221	.chain(std::iter::repeat(last))
	222	.take(BLOCK_SAMPLES - 1)
	223	.collect::<Vec<_>>();
	224	for pair in rest.chunks_exact(2) {
	225	let mut byte = 0;
	226	for (shift, sample) in [0, 4].into_iter().zip(pair) {
	227	let code = encode(i32::from(*sample), &mut predicted, &mut index);
	228	byte \|= code << shift;
	229	}
	230	data.push(byte);
	231	}
	232	}
	233	let count = u32::try_from(samples.len()).ok()?;
	234	let bytes_per_second = wave.rate * BLOCK as u32 / BLOCK_SAMPLES as u32;
	235	let mut out = Vec::with_capacity(data.len() + 60);
	236	let fmt: Vec<u8> = [
	237	&0x11_u16.to_le_bytes()[..],
	238	&1_u16.to_le_bytes(),
	239	&wave.rate.to_le_bytes(),
	240	&bytes_per_second.to_le_bytes(),
	241	&(BLOCK as u16).to_le_bytes(),
	242	&4_u16.to_le_bytes(),
	243	&2_u16.to_le_bytes(),
	244	&(BLOCK_SAMPLES as u16).to_le_bytes(),
	245	]
	246	.concat();
	247	let riff = 4 + (8 + fmt.len()) + (8 + 4) + (8 + data.len());
	248	out.extend_from_slice(b"RIFF");
	249	out.extend_from_slice(&u32::try_from(riff).ok()?.to_le_bytes());
	250	out.extend_from_slice(b"WAVEfmt ");
	251	out.extend_from_slice(&(fmt.len() as u32).to_le_bytes());
	252	out.extend_from_slice(&fmt);
	253	out.extend_from_slice(b"fact");
	254	out.extend_from_slice(&4_u32.to_le_bytes());
	255	out.extend_from_slice(&count.to_le_bytes());
	256	out.extend_from_slice(b"data");
	257	out.extend_from_slice(&u32::try_from(data.len()).ok()?.to_le_bytes());
	258	out.extend_from_slice(&data);
	259	let duration = u32::try_from(u64::from(count) * 1000 / u64::from(wave.rate.max(1))).ok()?;
	260	Some((out, duration))
	261	}
	262
	263	/// The 4-bit code nearest `sample` from `predicted`, which it and `index` then follow as a
	264	/// decoder does.
	265	fn encode(sample: i32, predicted: &mut i32, index: &mut i32) -> u8 {
	266	let step = STEPS[*index as usize];
	267	let mut difference = sample - *predicted;
	268	let mut code = 0;
	269	if difference < 0 {
	270	code = 8;
	271	difference = -difference;
	272	}
	273	let mut delta = step >> 3;
	274	for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] {
	275	if difference >= part {
	276	code \|= bit;
	277	difference -= part;
	278	delta += part;
	279	}
	280	}
	281	*predicted = if code & 8 != 0 {
	282	*predicted - delta
	283	} else {
	284	*predicted + delta
	285	}
	286	.clamp(-32768, 32767);
	287	index = (index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88);
	288	code
	289	}
	290
	291	/// A mono IMA ADPCM WAV file as 16-bit PCM, which every platform's player decodes; none
	292	/// for other files.
	293	pub fn decompress(wav: &[u8]) -> Option<Vec<u8>> {
	294	let wave = Wave::parse(wav).filter(\|wave\| wave.format == 0x11 && wave.channels == 1)?;
	295	Some(self::wave(wave.rate, &decode(&wave)))
	296	}
	297
	298	/// Each block's header sample and index, then its codes.
	299	fn decode(wave: &Wave<'_>) -> Vec<i16> {
	300	let mut out = Vec::new();
	301	for block in wave.data.chunks(usize::from(wave.block.max(4))) {
	302	let Some(header) = block.get(..4) else {
	303	break;
	304	};
	305	let mut predicted = i32::from(i16::from_le_bytes([header[0], header[1]]));
	306	let mut index = i32::from(header[2]).clamp(0, 88);
	307	out.push(predicted as i16);
	308	for byte in &block[4..] {
	309	for code in [byte & 15, byte >> 4] {
	310	let step = STEPS[index as usize];
	311	let mut delta = step >> 3;
	312	for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] {
	313	if code & bit != 0 {
	314	delta += part;
	315	}
	316	}
	317	predicted = if code & 8 != 0 {
	318	predicted - delta
	319	} else {
	320	predicted + delta
	321	}
	322	.clamp(-32768, 32767);
	323	index = (index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88);
	324	out.push(predicted as i16);
	325	}
	326	}
	327	}
	328	if let Some(samples) = wave.samples {
	329	out.truncate(samples as usize);
	330	}
	331	out
	332	}
	333
	334	#[cfg(test)]
	335	mod tests {
	336	use super::*;
	337
	338	/// Two seconds of a spoken-pitch chord.
	339	fn tone() -> Vec<i16> {
	340	(0..2 * RATE)
	341	.map(\|n\| {
	342	let t = n as f32 / RATE as f32;
	343	let wave = (t * 220.0 * std::f32::consts::TAU).sin() * 0.5
	344	+ (t * 330.0 * std::f32::consts::TAU).sin() * 0.3;
	345	(wave * 12_000.0) as i16
	346	})
	347	.collect()
	348	}
	349
	350	#[test]
	351	fn a_recording_compresses_to_ima_adpcm_that_decodes_close_to_what_was_heard() {
	352	let samples = tone();
	353	let (bytes, duration) = compress(&wave(RATE, &samples)).unwrap();
	354	assert_eq!(duration, 2000);
	355	let wave = Wave::parse(&bytes).unwrap();
	356	assert_eq!(
	357	(wave.format, wave.channels, wave.rate, wave.bits),
	358	(0x11, 1, RATE, 4)
	359	);
	360	assert_eq!(wave.data.len() % BLOCK, 0);
	361	assert_eq!(wave.duration_ms(), 2000);
	362	assert!(bytes.len() * 3 < samples.len() * 2);
	363	let decoded = decode(&wave);
	364	assert_eq!(
	365	Wave::parse(&decompress(&bytes).unwrap())
	366	.unwrap()
	367	.data
	368	.len(),
	369	decoded.len() * 2
	370	);
	371	assert_eq!(decoded.len(), samples.len());
	372	let noise: f64 = samples
	373	.iter()
	374	.zip(&decoded)
	375	.map(\|(a, b)\| (f64::from(a) - f64::from(b)).powi(2))
	376	.sum();
	377	let signal: f64 = samples.iter().map(\|a\| f64::from(*a).powi(2)).sum();
	378	let snr = 10.0 * (signal / noise).log10();
	379	assert!(snr > 20.0, "{snr} dB");
	380	}
	381
	382	#[test]
	383	fn only_mono_16_bit_pcm_compresses_and_attached_wav_files_know_their_length() {
	384	let stereo = {
	385	let mut bytes = wave(RATE, &[0; 64]);
	386	bytes[22] = 2;
	387	bytes
	388	};
	389	assert!(compress(&stereo).is_none());
	390	assert!(compress(b"not a wave").is_none());
	391	let silence = include_bytes!(
	392	"../../../corpus/m6/native-features-01/notebook/fixture-data/silence.wav"
	393	);
	394	// OneNote 2010 stored 1000 ms for this file when attached (`corpus/recording`).
	395	let recording = attached("silence.WAV", silence).unwrap();
	396	assert_eq!((recording.kind, recording.duration_ms), (1, Some(1000)));
	397	assert_eq!(attached("clip.wmv", b"").unwrap().kind, 2);
	398	assert!(attached("notes.txt", b"").is_none());
	399	}
	400
	401	#[test]
	402	fn a_recording_is_stored_named_after_its_page_and_plays_as_pcm() {
	403	let id = [7; 16];
	404	let audio = file(id, "Lecture 3/4", false, wave(RATE, &tone())).unwrap();
	405	assert_eq!(audio.filename, "Lecture 3_4.wav");
	406	let recording = audio.recording.unwrap();
	407	assert_eq!(
	408	(recording.id, recording.kind, recording.duration_ms),
	409	(id, 1, Some(2000))
	410	);
	411	let stored = audio.bytes.unwrap();
	412	assert_eq!(Wave::parse(&stored).unwrap().format, 0x11);
	413	let played = sound(&stored).unwrap();
	414	assert_eq!(Wave::parse(&played).unwrap().format, 1);
	415	assert_eq!(
	416	file(id, " ", true, Vec::new()).unwrap().filename,
	417	"Video recording.avi"
	418	);
	419	assert_eq!(
	420	label(false, "Tuesday, September 29, 2026", "5:18 PM"),
	421	"Audio recording started: 5:18 PM Tuesday, September 29, 2026"
	422	);
	423	}
	424	}

crates/canvas/src/recording/video.rs created+422

...	...	@@ -0,0 +1,422 @@
	1	//! Video recordings as AVI files of Motion JPEG pictures and PCM sound, which OneNote 2010
	2	//! plays and lists as recordings (`corpus/recording/video`), and reading them back to play.
	3
	4	use std::ops::Range;
	5
	6	/// Pictures a second in a recording: OneNote 2010's own profiles record at 15.
	7	pub const FPS: u32 = 15;
	8	/// A recording's picture size, OneNote 2010's own.
	9	pub const SIZE: [u32; 2] = [320, 240];
	10
	11	/// Interleaved signed 16-bit little-endian sound.
	12	pub struct Pcm {
	13	pub rate: u32,
	14	pub channels: u16,
	15	pub data: Vec<u8>,
	16	}
	17
	18	fn chunk(out: &mut Vec<u8>, id: &[u8; 4], body: &[u8]) {
	19	out.extend_from_slice(id);
	20	out.extend_from_slice(&(body.len() as u32).to_le_bytes());
	21	out.extend_from_slice(body);
	22	if body.len() % 2 == 1 {
	23	out.push(0);
	24	}
	25	}
	26
	27	fn list(kind: &[u8; 4], body: &[u8]) -> Vec<u8> {
	28	let mut out = Vec::with_capacity(body.len() + 12);
	29	chunk(&mut out, b"LIST", &[&kind[..], body].concat());
	30	out
	31	}
	32
	33	fn words(values: &[u32]) -> Vec<u8> {
	34	values
	35	.iter()
	36	.flat_map(\|value\| value.to_le_bytes())
	37	.collect()
	38	}
	39
	40	/// An AVI file of `frames`, JPEG pictures of `size` shown `FPS` a second, where an empty one
	41	/// repeats the picture before it, and `sound` alongside. None past AVI's 4 GiB.
	42	pub fn write(size: [u32; 2], frames: &[Vec<u8>], sound: &Pcm) -> Option<Vec<u8>> {
	43	let block = u32::from(sound.channels) * 2;
	44	let bytes_per_second = sound.rate * block;
	45	let samples = sound.data.len() as u64 / u64::from(block);
	46	let count = frames.len() as u64;
	47	// The sound up to frame `index`, whole samples.
	48	let sound_at = \|index: u64\| -> usize {
	49	let before = if index >= count {
	50	samples
	51	} else {
	52	(index * u64::from(sound.rate) / u64::from(FPS)).min(samples)
	53	};
	54	(before * u64::from(block)) as usize
	55	};
	56	let mut movi = b"movi".to_vec();
	57	let mut index = Vec::new();
	58	let [mut largest_frame, mut largest_sound] = [0, 0];
	59	for (at, frame) in frames.iter().enumerate() {
	60	let entries: [(&[u8; 4], &[u8]); 2] = [
	61	(b"00dc", frame),
	62	(
	63	b"01wb",
	64	&sound.data[sound_at(at as u64)..sound_at(at as u64 + 1)],
	65	),
	66	];
	67	for (id, body) in entries {
	68	if id == b"01wb" && body.is_empty() {
	69	continue;
	70	}
	71	// Keyframes: every picture and all sound; a repeat is none.
	72	let flags: u32 = if body.is_empty() { 0 } else { 0x10 };
	73	index.extend_from_slice(id);
	74	index.extend(words(&[
	75	flags,
	76	u32::try_from(movi.len()).ok()?,
	77	body.len() as u32,
	78	]));
	79	chunk(&mut movi, id, body);
	80	let largest = if id == b"00dc" {
	81	&mut largest_frame
	82	} else {
	83	&mut largest_sound
	84	};
	85	largest = (largest).max(body.len() as u32);
	86	}
	87	}
	88	let [width, height] = size;
	89	let frame_count = u32::try_from(count).ok()?;
	90	let rect = [0u16, 0, width as u16, height as u16]
	91	.iter()
	92	.flat_map(\|side\| side.to_le_bytes())
	93	.collect::<Vec<u8>>();
	94	let mut video = Vec::new();
	95	let strh = [
	96	&b"vidsMJPG"[..],
	97	&words(&[0, 0, 0, 1, FPS, 0, frame_count, largest_frame, u32::MAX, 0]),
	98	&rect,
	99	]
	100	.concat();
	101	chunk(&mut video, b"strh", &strh);
	102	let strf = [
	103	words(&[40, width, height]),
	104	[&1u16.to_le_bytes()[..], &24u16.to_le_bytes()].concat(),
	105	b"MJPG".to_vec(),
	106	words(&[width * height * 3, 0, 0, 0, 0]),
	107	]
	108	.concat();
	109	chunk(&mut video, b"strf", &strf);
	110	let mut streams = list(b"strl", &video);
	111	if samples > 0 {
	112	let mut audio = Vec::new();
	113	let strh = [
	114	&b"auds\0\0\0\0"[..],
	115	&words(&[
	116	0,
	117	0,
	118	0,
	119	block,
	120	bytes_per_second,
	121	0,
	122	u32::try_from(samples).ok()?,
	123	largest_sound,
	124	u32::MAX,
	125	block,
	126	]),
	127	&[0; 8],
	128	]
	129	.concat();
	130	chunk(&mut audio, b"strh", &strh);
	131	let strf = [
	132	&1u16.to_le_bytes()[..],
	133	&sound.channels.to_le_bytes(),
	134	&sound.rate.to_le_bytes(),
	135	&bytes_per_second.to_le_bytes(),
	136	&(block as u16).to_le_bytes(),
	137	&16u16.to_le_bytes(),
	138	&0u16.to_le_bytes(),
	139	]
	140	.concat();
	141	chunk(&mut audio, b"strf", &strf);
	142	streams.extend(list(b"strl", &audio));
	143	}
	144	let mut avih = Vec::new();
	145	chunk(
	146	&mut avih,
	147	b"avih",
	148	&words(&[
	149	1_000_000 / FPS,
	150	largest_frame
	151	.saturating_add(largest_sound)
	152	.saturating_mul(FPS),
	153	0,
	154	// AVIF_HASINDEX \| AVIF_ISINTERLEAVED
	155	0x110,
	156	frame_count,
	157	0,
	158	if samples > 0 { 2 } else { 1 },
	159	largest_frame.max(largest_sound),
	160	width,
	161	height,
	162	0,
	163	0,
	164	0,
	165	0,
	166	]),
	167	);
	168	let header = list(b"hdrl", &[avih, streams].concat());
	169	let mut body = b"AVI ".to_vec();
	170	body.extend(header);
	171	let movi_size = u32::try_from(movi.len()).ok()?;
	172	body.extend_from_slice(b"LIST");
	173	body.extend_from_slice(&movi_size.to_le_bytes());
	174	body.extend(movi);
	175	chunk(&mut body, b"idx1", &index);
	176	u32::try_from(body.len()).ok()?;
	177	let mut out = Vec::with_capacity(body.len() + 8);
	178	chunk(&mut out, b"RIFF", &body);
	179	Some(out)
	180	}
	181
	182	/// An AVI file's pictures and sound, as byte ranges of the file.
	183	pub struct Movie {
	184	/// Each picture's JPEG; an empty range repeats the one before.
	185	pub frames: Vec<Range<usize>>,
	186	pub frame_us: u32,
	187	/// The sound's WAVEFORMATEX and its data, in order.
	188	pub sound: Option<(Range<usize>, Vec<Range<usize>>)>,
	189	}
	190
	191	impl Movie {
	192	/// A Motion JPEG AVI file's contents; none for other files.
	193	pub fn parse(bytes: &[u8]) -> Option<Self> {
	194	let mut movie = Self {
	195	frames: Vec::new(),
	196	frame_us: 0,
	197	sound: None,
	198	};
	199	let mut kinds = Vec::new();
	200	let mut first = true;
	201	for (id, body) in Chunks::new(bytes, 0..bytes.len()) {
	202	let kind = bytes.get(body.start..body.start + 4)?;
	203	if id != *b"RIFF" \|\| (kind != b"AVI " && !(kind == b"AVIX" && !first)) {
	204	return (!first).then_some(movie).filter(\|movie\| movie.frame_us > 0);
	205	}
	206	first = false;
	207	movie.walk(bytes, body.start + 4..body.end, &mut kinds)?;
	208	}
	209	(movie.frame_us > 0).then_some(movie)
	210	}
	211
	212	fn walk(&mut self, bytes: &[u8], range: Range<usize>, kinds: &mut Vec<[u8; 4]>) -> Option<()> {
	213	for (id, body) in Chunks::new(bytes, range) {
	214	let u32_at = \|at: usize\| {
	215	Some(u32::from_le_bytes(
	216	bytes
	217	.get(body.start + at..body.start + at + 4)?
	218	.try_into()
	219	.ok()?,
	220	))
	221	};
	222	match &id {
	223	b"LIST" => {
	224	let inner = body.start + 4..body.end;
	225	self.walk(bytes, inner, kinds)?;
	226	}
	227	b"strh" => {
	228	let kind: [u8; 4] = bytes.get(body.start..body.start + 4)?.try_into().ok()?;
	229	if &kind == b"vids" {
	230	let [scale, rate] = [u32_at(20)?, u32_at(24)?];
	231	self.frame_us =
	232	u32::try_from(u64::from(scale) * 1_000_000 / u64::from(rate.max(1)))
	233	.ok()?;
	234	}
	235	kinds.push(kind);
	236	}
	237	b"strf" if kinds.last() == Some(b"vids") => {
	238	if bytes.get(body.start + 16..body.start + 20)? != b"MJPG" {
	239	return None;
	240	}
	241	}
	242	b"strf" if kinds.last() == Some(b"auds") => {
	243	self.sound = Some((body.clone(), Vec::new()));
	244	}
	245	[a, b, c, d] if a.is_ascii_digit() && b.is_ascii_digit() => {
	246	let stream = usize::from((a - b'0') * 10 + (b - b'0'));
	247	match (kinds.get(stream).map(\|kind\| &kind[..]), [c, d]) {
	248	(Some(b"vids"), [b'd', b'c' \| b'b']) => self.frames.push(body),
	249	(Some(b"auds"), [b'w', b'b']) => {
	250	if let Some((_, data)) = &mut self.sound {
	251	data.push(body);
	252	}
	253	}
	254	_ => {}
	255	}
	256	}
	257	_ => {}
	258	}
	259	}
	260	Some(())
	261	}
	262
	263	pub fn duration_ms(&self) -> u32 {
	264	let ms = (self.frames.len() as u64 * u64::from(self.frame_us)).div_ceil(1000);
	265	u32::try_from(ms).unwrap_or(u32::MAX)
	266	}
	267
	268	/// The picture showing `at_ms` in: the last stored one at or before it.
	269	pub fn frame(&self, at_ms: u32) -> Option<Range<usize>> {
	270	let at = (u64::from(at_ms) * 1000 / u64::from(self.frame_us.max(1))) as usize;
	271	self.frames[..=at.min(self.frames.len().checked_sub(1)?)]
	272	.iter()
	273	.rev()
	274	.find(\|frame\| !frame.is_empty())
	275	.cloned()
	276	}
	277
	278	/// The sound as a WAV file, or none where the file has none.
	279	pub fn wave(&self, bytes: &[u8]) -> Option<Vec<u8>> {
	280	let (format, data) = self.sound.as_ref()?;
	281	let format = &bytes[format.clone()];
	282	let length: usize = data.iter().map(ExactSizeIterator::len).sum();
	283	let mut out = Vec::with_capacity(length + 64);
	284	out.extend_from_slice(b"RIFF");
	285	out.extend_from_slice(&((4 + 8 + format.len() + 8 + length) as u32).to_le_bytes());
	286	out.extend_from_slice(b"WAVE");
	287	chunk(&mut out, b"fmt ", format);
	288	out.extend_from_slice(b"data");
	289	out.extend_from_slice(&(length as u32).to_le_bytes());
	290	for range in data {
	291	out.extend_from_slice(&bytes[range.clone()]);
	292	}
	293	Some(out)
	294	}
	295	}
	296
	297	/// The chunks in `range` of a RIFF file: each one's id and body.
	298	struct Chunks<'a> {
	299	bytes: &'a [u8],
	300	at: usize,
	301	end: usize,
	302	}
	303
	304	impl<'a> Chunks<'a> {
	305	fn new(bytes: &'a [u8], range: Range<usize>) -> Self {
	306	Self {
	307	bytes,
	308	at: range.start,
	309	end: range.end.min(bytes.len()),
	310	}
	311	}
	312	}
	313
	314	impl Iterator for Chunks<'_> {
	315	type Item = ([u8; 4], Range<usize>);
	316
	317	fn next(&mut self) -> Option<Self::Item> {
	318	let header = self.bytes.get(self.at..self.at + 8)?;
	319	if self.at + 8 > self.end {
	320	return None;
	321	}
	322	let id: [u8; 4] = header[..4].try_into().ok()?;
	323	let size = u32::from_le_bytes(header[4..].try_into().ok()?) as usize;
	324	let start = self.at + 8;
	325	let end = start.saturating_add(size).min(self.end);
	326	self.at = end + size % 2;
	327	Some((id, start..end))
	328	}
	329	}
	330
	331	/// Pictures taken at uneven moments, as a camera takes them, made the `FPS` a second of a
	332	/// recording: each one shows until the next, fitted to `SIZE`.
	333	#[derive(Default)]
	334	pub struct Pictures {
	335	frames: Vec<Vec<u8>>,
	336	last: Option<image::RgbImage>,
	337	/// Whether the last picture is stored yet, which the frames after it repeat.
	338	stored: bool,
	339	}
	340
	341	impl Pictures {
	342	/// The picture taken `at_us` microseconds in; the first shows from the start.
	343	pub fn take(&mut self, at_us: u64, picture: image::RgbImage) {
	344	self.fill(at_us);
	345	self.last = Some(fit(picture));
	346	self.stored = false;
	347	}
	348
	349	/// A picture as Core Video gives one: `height` rows of `stride` bytes of BGRA pixels.
	350	pub fn take_bgra(
	351	&mut self,
	352	at_us: u64,
	353	[width, height]: [usize; 2],
	354	stride: usize,
	355	bgra: &[u8],
	356	) {
	357	let rgb = bgra
	358	.chunks_exact(stride.max(1))
	359	.take(height)
	360	.filter_map(\|row\| row.get(..width * 4))
	361	.flat_map(\|row\| {
	362	row.chunks_exact(4)
	363	.flat_map(\|pixel\| [pixel[2], pixel[1], pixel[0]])
	364	})
	365	.collect();
	366	if let Some(picture) = image::RgbImage::from_raw(width as u32, height as u32, rgb) {
	367	self.take(at_us, picture);
	368	}
	369	}
	370
	371	/// The frames of a recording `end_us` microseconds long.
	372	pub fn finish(mut self, end_us: u64) -> Vec<Vec<u8>> {
	373	self.fill(end_us);
	374	self.frames
	375	}
	376
	377	/// Frames up to `until_us`, of the last picture.
	378	fn fill(&mut self, until_us: u64) {
	379	let Some(last) = &self.last else {
	380	return;
	381	};
	382	while (self.frames.len() as u64) * 1_000_000 / u64::from(FPS) < until_us {
	383	let frame = if self.stored {
	384	Vec::new()
	385	} else {
	386	self.stored = true;
	387	let mut jpeg = Vec::new();
	388	let encoded = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 70)
	389	.encode_image(last);
	390	if encoded.is_err() {
	391	jpeg.clear();
	392	}
	393	jpeg
	394	};
	395	self.frames.push(frame);
	396	}
	397	}
	398	}
	399
	400	/// `picture` scaled to fit `SIZE`, centred on black.
	401	fn fit(picture: image::RgbImage) -> image::RgbImage {
	402	let [width, height] = SIZE;
	403	if picture.dimensions() == (width, height) {
	404	return picture;
	405	}
	406	let scale =
	407	(width as f32 / picture.width() as f32).min(height as f32 / picture.height() as f32);
	408	let fitted = image::imageops::resize(
	409	&picture,
	410	((picture.width() as f32 * scale) as u32).clamp(1, width),
	411	((picture.height() as f32 * scale) as u32).clamp(1, height),
	412	image::imageops::FilterType::Triangle,
	413	);
	414	let mut out = image::RgbImage::new(width, height);
	415	image::imageops::overlay(
	416	&mut out,
	417	&fitted,
	418	i64::from((width - fitted.width()) / 2),
	419	i64::from((height - fitted.height()) / 2),
	420	);
	421	out
	422	}

crates/mobile/include/snowbound.h+14

...	...	@@ -109,6 +109,20 @@ int8_t sb_view_date_request(View view, int64_t seconds);
109	109	bool sb_view_change_date(View view, int64_t seconds, const char date, const char *time);
110	110	bool sb_view_insert_picture(View view, const uint8_t bytes, size_t length, float width, float height);
111	111
	112	typedef struct Movie Movie;
	113	uint32_t sb_recording_rate(void);
	114	bool sb_view_start_recording(View view, bool video, const char date, const char *time);
	115	int64_t sb_view_recording(const View view, bool paused);
	116	void sb_view_pause_recording(View *view, bool paused);
	117	bool sb_view_finish_recording(View view, const uint8_t bytes, size_t length, bool video);
	118	char sb_view_play_request(View view, const char *folder);
	119	bool sb_view_played(View *view, int64_t at_ms);
	120	Movie *sb_movie_new(void);
	121	void sb_movie_picture(Movie movie, uint64_t at_us, const uint8_t bgra, uint32_t width, uint32_t height,
	122	size_t stride);
	123	void sb_movie_sound(Movie movie, const uint8_t pcm, size_t length);
	124	uint8_t sb_movie_finish(Movie movie, uint64_t duration_us, size_t *length);
	125
112	126	uint32_t sb_text_length(const View *view);
113	127	char sb_text(const View view, uint32_t start, uint32_t end);
114	128	void sb_selection(const View *view, uint32_t range[2]);

crates/mobile/src/lib.rs+15-4

...	...	@@ -8,6 +8,7 @@
8	8	//! the host redraws and rereads `sb_view_content`.
9	9
10	10	mod library;
	11	mod recording;
11	12	#[cfg(target_os = "ios")]
12	13	mod spell;
13	14
...	...	@@ -119,6 +120,10 @@ struct Canvas {
119	120	scale: f32,
120	121	/// The page date field a tap asked to change, for the host's picker.
121	122	asked: Option<DateField>,
	123	/// The recording a tap asked to play, and from when.
	124	play: Option<(onestore::page::Attachment, u32)>,
	125	/// The recording playing, whose linked notes See Playback follows.
	126	playing: Option<[u8; 16]>,
122	127	}
123	128
124	129	impl Canvas {
...	...	@@ -156,6 +161,8 @@ impl Canvas {
156	161	paper: Paper::WHITE,
157	162	scale,
158	163	asked: None,
	164	play: None,
	165	playing: None,
159	166	})
160	167	}
161	168
...	...	@@ -263,12 +270,16 @@ impl Canvas {
263	270	Ok(self.respond(pressed) \|\| changed)
264	271	}
265	272
266		/// Whether `response` moved the page, keeping the date field it asks to change.
	273	/// Whether `response` moved the page, keeping the date field it asks to change or the
	274	/// recording it asks to play.
267	275	fn respond(&mut self, response: Response) -> bool {
268		if let Some(Request::EditDate(field)) = response.request {
269		self.asked = Some(field);
	276	let changed = response.changed \|\| response.moved;
	277	match response.request {
	278	Some(Request::EditDate(field)) => self.asked = Some(field),
	279	Some(Request::Play { file, at_ms }) => self.play = Some((file, at_ms)),
	280	_ => {}
270	281	}
271		moved(response)
	282	changed
272	283	}
273	284
274	285	fn drag(&mut self, point: [f32; 2]) -> Result<bool> {

crates/mobile/src/recording.rs created+274

...	...	@@ -0,0 +1,274 @@
	1	//! Record Audio, Record Video and playback, stored as the desktop stores them
	2	//! (`canvas::recording`): the host records with the platform's devices and hands over a PCM
	3	//! WAV file, or a video's pictures and sound as `Movie` takes them; a recording plays from
	4	//! the files `sb_view_play_request` writes.
	5
	6	use super::*;
	7	use canvas::recording::{self, video};
	8	use std::path::Path;
	9
	10	impl Canvas {
	11	/// Starts recording at the caret, its line saying when from the long `date` and short
	12	/// `time`; whether it started.
	13	pub(crate) fn start_recording(&mut self, video: bool, date: &str, time: &str) -> Result<bool> {
	14	let label = recording::label(video, date, time);
	15	Ok(self.page.start_recording(&label)?.0.is_some())
	16	}
	17
	18	/// Puts the recording made, `recorded`, where it started: a PCM WAV file, or with
	19	/// `video` an AVI file from `Movie`. None forgets the recording, whose file never came.
	20	pub(crate) fn finish_recording(
	21	&mut self,
	22	recorded: Option<Vec<u8>>,
	23	video: bool,
	24	) -> Result<bool> {
	25	let editor = &mut self.page.editor;
	26	let Some(id) = editor.recording() else {
	27	return Ok(false);
	28	};
	29	let Some(recorded) = recorded else {
	30	editor.cancel_recording();
	31	return Ok(false);
	32	};
	33	let file = recording::file(id, &editor.page()?.title, video, recorded)?;
	34	Ok(moved(self.page.finish_recording(file)?))
	35	}
	36
	37	/// The recording a tap asked to play, taken, with its files written to `folder` to play
	38	/// from: see `sb_view_play_request`.
	39	pub(crate) fn take_play(&mut self, folder: &Path) -> Result<Option<serde_json::Value>> {
	40	let Some((file, at)) = self.play.take() else {
	41	return Ok(None);
	42	};
	43	let bytes = file.bytes.clone().ok_or("The recording has no data")?;
	44	let stem = Path::new(&file.filename)
	45	.file_stem()
	46	.map_or_else(String::new, \|stem\| stem.to_string_lossy().into_owned());
	47	std::fs::create_dir_all(folder)?;
	48	let sound = match recording::sound(&bytes) {
	49	Some(wave) => {
	50	let path = folder.join(format!("{stem}.wav"));
	51	std::fs::write(&path, wave)?;
	52	path
	53	}
	54	None => {
	55	let path = folder.join(file.filename.replace('/', "_"));
	56	std::fs::write(&path, &bytes)?;
	57	path
	58	}
	59	};
	60	let movie = video::Movie::parse(&bytes)
	61	.map(\|movie\| -> Result<_> {
	62	let path = folder.join(format!("{stem}.avi"));
	63	std::fs::write(&path, &bytes)?;
	64	// An empty frame repeats the picture before it.
	65	let mut shown = [0; 2];
	66	let frames: Vec<[usize; 2]> = movie
	67	.frames
	68	.iter()
	69	.map(\|range\| {
	70	if !range.is_empty() {
	71	shown = [range.start, range.end];
	72	}
	73	shown
	74	})
	75	.collect();
	76	Ok(serde_json::json!({
	77	"path": path,
	78	"frames": frames,
	79	"frame_us": movie.frame_us,
	80	"size": video::SIZE,
	81	}))
	82	})
	83	.transpose()?;
	84	self.playing = file.recording.map(\|recording\| recording.id);
	85	Ok(Some(serde_json::json!({
	86	"name": stem,
	87	"sound": sound,
	88	"video": movie,
	89	"at_ms": at,
	90	})))
	91	}
	92
	93	/// See Playback: highlights the note linked last at or before `at_ms` into the recording
	94	/// playing, or with none ends playback.
	95	pub(crate) fn played(&mut self, at_ms: Option<u32>) -> Result<bool> {
	96	if at_ms.is_none() {
	97	self.playing = None;
	98	}
	99	let note = self
	100	.playing
	101	.zip(at_ms)
	102	.and_then(\|(id, at)\| self.page.editor.played_note(id, at))
	103	.and_then(\|note\| canvas::search::paragraph_match(&self.page.editor, note));
	104	Ok(moved(self.page.set_played(note)?))
	105	}
	106	}
	107
	108	/// Starts recording audio, or with `video` video, at the caret, its line saying when from
	109	/// the long `date` and short `time`; whether it started.
	110	///
	111	/// # Safety
	112	/// `date` and `time` are NUL-terminated UTF-8.
	113	#[unsafe(no_mangle)]
	114	pub unsafe extern "C" fn sb_view_start_recording(
	115	view: &mut View,
	116	video: bool,
	117	date: *const c_char,
	118	time: *const c_char,
	119	) -> bool {
	120	if view.read_only \|\| view.reading.is_some() {
	121	return false;
	122	}
	123	let started = view
	124	.canvas
	125	.start_recording(video, &string(date), &string(time));
	126	let started = report(started).unwrap_or(false);
	127	view.stored(Ok(started));
	128	started
	129	}
	130
	131	/// How far the recording under way has got in milliseconds, its pauses left out, or -1;
	132	/// `paused` gets whether it is paused.
	133	#[unsafe(no_mangle)]
	134	pub extern "C" fn sb_view_recording(view: &View, paused: &mut bool) -> i64 {
	135	let editor = &view.canvas.page.editor;
	136	*paused = editor.recording_paused();
	137	editor.recording_ms().map_or(-1, i64::from)
	138	}
	139
	140	/// Pauses or resumes the recording: nothing written while it is paused links to it.
	141	#[unsafe(no_mangle)]
	142	pub extern "C" fn sb_view_pause_recording(view: &mut View, paused: bool) {
	143	view.canvas.page.editor.pause_recording(paused);
	144	}
	145
	146	/// Stops recording: the file recorded, `length` bytes of mono 16-bit PCM WAV or with
	147	/// `video` the AVI file `sb_movie_finish` made, goes where the recording started. Null
	148	/// `bytes` forgets the recording.
	149	///
	150	/// # Safety
	151	/// `bytes` is null or holds `length` bytes.
	152	#[unsafe(no_mangle)]
	153	pub unsafe extern "C" fn sb_view_finish_recording(
	154	view: &mut View,
	155	bytes: *const u8,
	156	length: usize,
	157	video: bool,
	158	) -> bool {
	159	// SAFETY: the caller's contract.
	160	let recorded = (!bytes.is_null()).then(\|\| unsafe { std::slice::from_raw_parts(bytes, length) });
	161	let result = view
	162	.canvas
	163	.finish_recording(recorded.map(<[u8]>::to_vec), video);
	164	view.stored(result)
	165	}
	166
	167	/// The recording the last tap asked to play, taken, as JSON: its `name`, the `sound` file
	168	/// written to `folder` that the platform's player plays, from `at_ms`; and for a video its
	169	/// `video` file `path`, the byte range of the JPEG picture each of its `frames` shows,
	170	/// `frame_us` apart, and their `size`. Null when none was asked for.
	171	///
	172	/// # Safety
	173	/// `folder` is NUL-terminated UTF-8.
	174	#[unsafe(no_mangle)]
	175	pub unsafe extern "C" fn sb_view_play_request(
	176	view: &mut View,
	177	folder: *const c_char,
	178	) -> *mut c_char {
	179	let request = view.canvas.take_play(Path::new(&string(folder)));
	180	report(request)
	181	.flatten()
	182	.map_or(std::ptr::null_mut(), \|json\| owned(json.to_string()))
	183	}
	184
	185	/// The recording last asked to play is at `at_ms`, whose linked note the page highlights;
	186	/// -1 ends playback. Whether the page changed.
	187	#[unsafe(no_mangle)]
	188	pub extern "C" fn sb_view_played(view: &mut View, at_ms: i64) -> bool {
	189	let at = u32::try_from(at_ms).ok();
	190	report(view.canvas.played(at)).unwrap_or(false)
	191	}
	192
	193	/// The rate the host records sound at, as mono 16-bit PCM: the desktop's.
	194	#[unsafe(no_mangle)]
	195	pub extern "C" fn sb_recording_rate() -> u32 {
	196	recording::RATE
	197	}
	198
	199	/// A video recording's pictures and sound as the camera gave them, which become the AVI
	200	/// file the desktop records.
	201	#[derive(Default)]
	202	pub struct Movie {
	203	pictures: video::Pictures,
	204	sound: Vec<u8>,
	205	}
	206
	207	#[unsafe(no_mangle)]
	208	pub extern "C" fn sb_movie_new() -> *mut Movie {
	209	Box::into_raw(Box::default())
	210	}
	211
	212	/// The picture taken `at_us` microseconds in: `height` rows of `stride` bytes of BGRA.
	213	///
	214	/// # Safety
	215	/// `bgra` holds `height` × `stride` bytes.
	216	#[unsafe(no_mangle)]
	217	pub unsafe extern "C" fn sb_movie_picture(
	218	movie: &mut Movie,
	219	at_us: u64,
	220	bgra: *const u8,
	221	width: u32,
	222	height: u32,
	223	stride: usize,
	224	) {
	225	// SAFETY: the caller's contract.
	226	let bgra = unsafe { std::slice::from_raw_parts(bgra, height as usize * stride) };
	227	movie.pictures.take_bgra(
	228	at_us,
	229	[width, height].map(\|side\| side as usize),
	230	stride,
	231	bgra,
	232	);
	233	}
	234
	235	/// More of the sound, mono 16-bit little-endian PCM at `canvas::recording::RATE`.
	236	///
	237	/// # Safety
	238	/// `pcm` holds `length` bytes.
	239	#[unsafe(no_mangle)]
	240	pub unsafe extern "C" fn sb_movie_sound(movie: &mut Movie, pcm: *const u8, length: usize) {
	241	// SAFETY: the caller's contract.
	242	movie
	243	.sound
	244	.extend_from_slice(unsafe { std::slice::from_raw_parts(pcm, length) });
	245	}
	246
	247	/// The AVI file of a movie `duration_us` long, `length` bytes freed with `sb_bytes_free`,
	248	/// or null when it has no pictures or passes 4 GiB.
	249	///
	250	/// # Safety
	251	/// `movie` came from `sb_movie_new` and is not used again.
	252	#[unsafe(no_mangle)]
	253	pub unsafe extern "C" fn sb_movie_finish(
	254	movie: *mut Movie,
	255	duration_us: u64,
	256	length: &mut usize,
	257	) -> *mut u8 {
	258	// SAFETY: the caller's contract.
	259	let movie = unsafe { Box::from_raw(movie) };
	260	let frames = movie.pictures.finish(duration_us);
	261	if frames.is_empty() {
	262	return std::ptr::null_mut();
	263	}
	264	let sound = video::Pcm {
	265	rate: recording::RATE,
	266	channels: 1,
	267	data: movie.sound,
	268	};
	269	let Some(avi) = video::write(video::SIZE, &frames, &sound) else {
	270	return std::ptr::null_mut();
	271	};
	272	*length = avi.len();
	273	Box::into_raw(avi.into_boxed_slice()).cast()
	274	}

crates/mobile/src/tests.rs+107

...	...	@@ -1129,3 +1129,110 @@ fn the_page_prints_as_a_pdf() {
1129	1129	&text[..400]
1130	1130	);
1131	1131	}
	1132
	1133	#[test]
	1134	fn a_recording_goes_in_as_the_desktop_stores_it_and_a_tap_plays_it() {
	1135	use onestore::page::{PageObject, ParagraphContent};
	1136	let (directory, section) = features();
	1137	let mut canvas = canvas(&section, "Paragraph controls");
	1138	focus(&mut canvas, "Collapsed parent");
	1139	canvas.edit().unwrap();
	1140	let date = ["Tuesday, September 29, 2026", "5:18 PM"];
	1141	assert!(canvas.start_recording(false, date[0], date[1]).unwrap());
	1142	canvas.insert("Linked note".into()).unwrap();
	1143	let samples: Vec<i16> = (0..16_000).map(\|n\| (n % 64 * 256 - 8192) as i16).collect();
	1144	let wav = canvas::recording::wave(canvas::recording::RATE, &samples);
	1145	assert!(canvas.finish_recording(Some(wav), false).unwrap());
	1146	section
	1147	.shared
	1148	.apply(canvas.edit().unwrap().unwrap())
	1149	.unwrap();
	1150
	1151	let page = section.shared.page(canvas.space).unwrap().0;
	1152	let paragraphs: Vec<_> = page
	1153	.objects
	1154	.iter()
	1155	.filter_map(\|object\| match object {
	1156	PageObject::Outline(outline) => Some(&outline.paragraphs),
	1157	_ => None,
	1158	})
	1159	.flatten()
	1160	.collect();
	1161	let file = paragraphs
	1162	.iter()
	1163	.find_map(\|paragraph\| match &paragraph.content {
	1164	ParagraphContent::Attachment(file) => Some(file),
	1165	_ => None,
	1166	})
	1167	.unwrap();
	1168	let recording = file.recording.unwrap();
	1169	assert_eq!(file.filename, format!("{}.wav", page.title));
	1170	assert_eq!((recording.kind, recording.duration_ms), (1, Some(1000)));
	1171	// IMA ADPCM, as the desktop stores audio for OneNote (`corpus/recording`).
	1172	let bytes = file.bytes.as_deref().unwrap();
	1173	assert_eq!(u16::from_le_bytes([bytes[20], bytes[21]]), 0x11);
	1174	let linked = \|text: &str\| {
	1175	paragraphs
	1176	.iter()
	1177	.find(\|paragraph\| {
	1178	paragraph
	1179	.text()
	1180	.is_some_and(\|t\| t.text.text().starts_with(text))
	1181	})
	1182	.map(\|paragraph\| (paragraph.media.recordings.clone(), paragraph.media.time_ms))
	1183	.unwrap()
	1184	};
	1185	let line = linked("Audio recording started: 5:18 PM Tuesday, September 29, 2026");
	1186	assert_eq!(line, (vec![recording.id], Some(0)));
	1187	assert_eq!(linked("Linked note").0, [recording.id]);
	1188
	1189	// A tap on the recording plays it as PCM from the start.
	1190	let tapped = (0..800)
	1191	.flat_map(\|y\| (0..300).map(move \|x\| [x as f32 * 2.0, y as f32 * 2.0]))
	1192	.find(\|&point\| {
	1193	matches!(
	1194	canvas.page.hit(canvas.device(point.map(\|v\| v * POINT))),
	1195	Some(Hit::File(_))
	1196	)
	1197	})
	1198	.unwrap()
	1199	.map(\|v\| v * POINT);
	1200	canvas.press(tapped).unwrap();
	1201	canvas.release().unwrap();
	1202	let folder = directory.path().join("playback");
	1203	let request = canvas.take_play(&folder).unwrap().unwrap();
	1204	assert_eq!(request["at_ms"], 0);
	1205	assert!(request["video"].is_null());
	1206	let sound = std::fs::read(request["sound"].as_str().unwrap()).unwrap();
	1207	assert_eq!(u16::from_le_bytes([sound[20], sound[21]]), 1);
	1208	assert!(canvas.take_play(&folder).unwrap().is_none());
	1209	canvas.played(Some(5_000)).unwrap();
	1210	canvas.played(None).unwrap();
	1211	}
	1212
	1213	#[test]
	1214	fn a_movie_from_the_camera_becomes_the_desktop_s_avi() {
	1215	let movie = recording::sb_movie_new();
	1216	let bgra: Vec<u8> = (0..480 * 640 * 4).map(\|n\| (n % 251) as u8).collect();
	1217	for index in 0..20u64 {
	1218	unsafe {
	1219	recording::sb_movie_picture(
	1220	&mut *movie,
	1221	index * 50_000,
	1222	bgra.as_ptr(),
	1223	640,
	1224	480,
	1225	640 * 4,
	1226	)
	1227	};
	1228	}
	1229	let pcm = vec![0u8; canvas::recording::RATE as usize * 2];
	1230	unsafe { recording::sb_movie_sound(&mut *movie, pcm.as_ptr(), pcm.len()) };
	1231	let mut length = 0;
	1232	let avi = unsafe { recording::sb_movie_finish(movie, 1_000_000, &mut length) };
	1233	let bytes = unsafe { std::slice::from_raw_parts(avi, length) }.to_vec();
	1234	unsafe { sb_bytes_free(avi, length) };
	1235	let parsed = canvas::recording::video::Movie::parse(&bytes).unwrap();
	1236	assert_eq!((parsed.frames.len(), parsed.duration_ms()), (15, 1000));
	1237	assert!(parsed.sound.is_some());
	1238	}

crates/snowbound/Cargo.toml+6-2

...	...	@@ -14,8 +14,6 @@ canvas = { path = "../canvas", features = ["interaction", "pdf"] }
14	14	draw = { path = "../draw", default-features = false, features = ["render"] }
15	15	ui = { path = "../ui" }
16	16	png = "0.18"
17		# Encodes a video recording's pictures as JPEG, which OneNote 2010 plays.
18		image = { version = "0.25", default-features = false, features = ["jpeg"] }
19	17	onestore = { path = "../onestore" }
20	18	notebook = { path = "../notebook" }
21	19	serde = { version = "1", features = ["derive"] }
...	...	@@ -94,6 +92,8 @@ windows-sys = { version = "0.61", features = [
94	92	[target.'cfg(target_os = "linux")'.dependencies]
95	93	accesskit_winit = { version = "0.34.0", default-features = false, features = ["accesskit_unix", "async-io"] }
96	94	smithay-clipboard = "0.7.3"
	95	# Scales the window's icon.
	96	image = { version = "0.25", default-features = false }
97	97	# Loads fontconfig at run time, so builds need no fontconfig headers or library.
98	98	fontique = { workspace = true, features = ["fontconfig-dlopen"] }
99	99	libc = "0.2"
...	...	@@ -106,3 +106,7 @@ sctk-adwaita = { version = "0.10.1", default-features = false }
106	106	wayland-backend = { version = "0.3.17", features = ["client_system"] }
107	107	wayland-client = "0.31.15"
108	108	wayland-protocols = { version = "0.32.13", features = ["client", "staging"] }
	109
	110	[dev-dependencies]
	111	# Draws the pictures of the video recording the tests store.
	112	image = { version = "0.25", default-features = false }

crates/snowbound/src/attachment.rs+2-5

...	...	@@ -4,9 +4,6 @@ use crate::{State, platform};
4	4	use onestore::page::Attachment;
5	5	use std::{error::Error, path::Path};
6	6
7		/// The size OneNote 2010 shows a file's icon at, in points.
8		pub(crate) const ICON_SIZE: [f32; 2] = [24.0, 24.0];
9
10	7	impl State {
11	8	/// Attach File, or a file dropped at `at`, a window point: a copy of the file's bytes
12	9	/// at the caret or at `at`, with the icon the system shows for it; audio and video are
...	...	@@ -30,10 +27,10 @@ impl State {
30	27	id: onestore::page::text::new_id()?,
31	28	filename: name.to_owned(),
32	29	source_path: path.to_str().map(str::to_owned),
33		size: Some(ICON_SIZE),
	30	size: Some(canvas::gpu::page::ICON_SIZE),
34	31	layout: Default::default(),
35	32	preview: platform::file_icon(path).map(Into::into),
36		recording: crate::recording::attached(name, &bytes),
	33	recording: canvas::recording::attached(name, &bytes),
37	34	bytes: Some(bytes.into()),
38	35	tags: Vec::new(),
39	36	};

crates/snowbound/src/main.rs-1

...	...	@@ -84,7 +84,6 @@ mod undo;
84	84	mod unpack;
85	85	mod unread;
86	86	mod update;
87		mod video;
88	87	#[cfg_attr(target_arch = "wasm32", path = "watch_web.rs")]
89	88	mod watch;
90	89

crates/snowbound/src/media_linux.rs+8-6

...	...	@@ -3,7 +3,8 @@
3	3	//! with gst-libav), the default microphone records as PCM WAV, and the default camera with
4	4	//! it as the Motion JPEG AVI file OneNote plays. Recording needs the Base and Good plug-ins.
5	5
6		use crate::{platform::symbol, video};
	6	use crate::platform::symbol;
	7	use canvas::recording::video;
7	8	use std::{
8	9	cell::Cell,
9	10	ffi::{CStr, CString, c_char, c_int, c_void},
...	...	@@ -313,7 +314,7 @@ impl Recorder {
313	314	Self::start(&movie(
314	315	"autovideosrc",
315	316	"autoaudiosrc",
316		crate::recording::RATE,
	317	canvas::recording::RATE,
317	318	location,
318	319	))
319	320	}
...	...	@@ -395,17 +396,18 @@ mod tests {
395	396	"{} ! wavenc ! filesink location=\"{location}\"",
396	397	sound(
397	398	"audiotestsrc is-live=true wave=sine",
398		crate::recording::RATE
	399	canvas::recording::RATE
399	400	)
400	401	),
401	402	2,
402	403	true,
403	404	);
404		let (bytes, duration) = crate::recording::compress(&std::fs::read(&path).unwrap()).unwrap();
	405	let (bytes, duration) =
	406	canvas::recording::compress(&std::fs::read(&path).unwrap()).unwrap();
405	407	// Two seconds recorded; the paused one is left out.
406	408	assert!(duration.abs_diff(2000) < 300, "{duration}");
407	409	// Snowbound's IMA ADPCM, decoded as playback decodes it, plays at its length.
408		let wave = crate::recording::decompress(&bytes).unwrap();
	410	let wave = canvas::recording::decompress(&bytes).unwrap();
409	411	let played = scratch.0.join("played.wav");
410	412	std::fs::write(&played, wave).unwrap();
411	413	let mut player = Player::open(&played, false).unwrap();
...	...	@@ -432,7 +434,7 @@ mod tests {
432	434	&movie(
433	435	"videotestsrc is-live=true pattern=smpte",
434	436	"audiotestsrc is-live=true wave=sine",
435		crate::recording::RATE,
	437	canvas::recording::RATE,
436	438	quoted(&path).unwrap(),
437	439	),
438	440	2,

crates/snowbound/src/media_macos.rs+8-15

...	...	@@ -3,7 +3,7 @@
3	3	//! PCM WAV, and a capture session records the default camera and microphone as a movie,
4	4	//! which `AVAssetReader` reads back into the AVI file OneNote plays.
5	5
6		use crate::video;
	6	use canvas::recording::video;
7	7	use objc2::{
8	8	msg_send, msg_send_id,
9	9	rc::{Allocated, Retained},
...	...	@@ -432,7 +432,7 @@ fn delegate() -> Retained<AnyObject> {
432	432	}
433	433
434	434	/// The movie at `path`, as AVFoundation reads it, as a Motion JPEG AVI file with its sound
435		/// in mono PCM at `crate::recording::RATE`.
	435	/// in mono PCM at `canvas::recording::RATE`.
436	436	fn avi(path: &Path) -> Option<Vec<u8>> {
437	437	unsafe {
438	438	let asset: Retained<AnyObject> = msg_send_id![class("AVURLAsset"), URLAssetWithURL: &*url(path)?, options: std::ptr::null::<AnyObject>()];
...	...	@@ -453,22 +453,14 @@ fn avi(path: &Path) -> Option<Vec<u8>> {
453	453	CVPixelBufferGetBytesPerRow(buffer),
454	454	];
455	455	let base = CVPixelBufferGetBaseAddress(buffer);
456		let mut rgb = Vec::with_capacity(width * height * 3);
457	456	if !base.is_null() {
458	457	let rows = std::slice::from_raw_parts(base, stride * height);
459		for row in rows.chunks_exact(stride) {
460		for bgra in row[..width * 4].chunks_exact(4) {
461		rgb.extend_from_slice(&[bgra[2], bgra[1], bgra[0]]);
462		}
463		}
	458	pictures.take_bgra(at, [width, height], stride, rows);
464	459	}
465	460	CVPixelBufferUnlockBaseAddress(buffer, 1);
466		if let Some(picture) = image::RgbImage::from_raw(width as u32, height as u32, rgb) {
467		pictures.take(at, picture);
468		}
469	461	})?;
470	462	let mut sound = video::Pcm {
471		rate: crate::recording::RATE,
	463	rate: canvas::recording::RATE,
472	464	channels: 1,
473	465	data: Vec::new(),
474	466	};
...	...	@@ -550,8 +542,9 @@ mod tests {
550	542	/// What Snowbound records, compressed, plays back through Core Audio at its length.
551	543	#[test]
552	544	fn core_audio_plays_the_compressed_recordings() {
553		let pcm = crate::recording::wave(crate::recording::RATE, &crate::recording::tests::tone());
554		let (bytes, duration) = crate::recording::compress(&pcm).unwrap();
	545	let pcm =
	546	canvas::recording::wave(canvas::recording::RATE, &crate::recording::tests::tone());
	547	let (bytes, duration) = canvas::recording::compress(&pcm).unwrap();
555	548	let folder = std::env::temp_dir().join(format!("snowbound-player-{}", std::process::id()));
556	549	std::fs::create_dir_all(&folder).unwrap();
557	550	let path = folder.join("tone.wav");
...	...	@@ -611,7 +604,7 @@ mod tests {
611	604	.iter()
612	605	.map(ExactSizeIterator::len)
613	606	.sum();
614		let expected = 2 * crate::recording::RATE as usize * 2;
	607	let expected = 2 * canvas::recording::RATE as usize * 2;
615	608	assert!(length.abs_diff(expected) < expected / 20, "{length}");
616	609	}
617	610	}

crates/snowbound/src/recording.rs+22-371

...	...	@@ -3,15 +3,13 @@
3	3	//! as a file OneNote plays; a recording, or a note linked to a moment in one, plays in a
4	4	//! transport over the page. What Snowbound cannot play opens in the system's player.
5	5
6		use crate::{State, media, platform, video};
7		use onestore::page::{Attachment, Recording};
	6	use crate::{State, media, platform};
	7	use canvas::recording::{RATE, video};
	8	use onestore::page::Attachment;
8	9	use std::{error::Error, ops::Range, path::Path, sync::Arc, time::Duration};
9	10	use ui::{Axis, Flags, Spec, Theme, children, fit, px};
10	11	use winit::keyboard::NamedKey;
11	12
12		/// The rate audio is recorded at: wideband speech, as OneNote's own 8 kHz WMA is narrowband.
13		pub(crate) const RATE: u32 = 16_000;
14
15	13	/// What records or plays now.
16	14	#[derive(Default)]
17	15	pub(crate) enum Media {
...	...	@@ -26,7 +24,7 @@ pub(crate) enum Media {
26	24	Saving {
27	25	video: bool,
28	26	path: std::path::PathBuf,
29		job: std::thread::JoinHandle<Result<Vec<u8>, String>>,
	27	job: std::thread::JoinHandle<Result<Attachment, String>>,
30	28	},
31	29	Playing(Box<Playback>),
32	30	}
...	...	@@ -94,8 +92,7 @@ impl State {
94	92	}
95	93	};
96	94	let [date, time] = platform::date_text(crate::filetime());
97		let kind = if video { "Video" } else { "Audio" };
98		let label = format!("{kind} recording started: {time} {date}");
	95	let label = canvas::recording::label(video, &date, &time);
99	96	let (id, response) = self.view.start_recording(&label)?;
100	97	self.respond(response);
101	98	if id.is_none() {
...	...	@@ -129,15 +126,15 @@ impl State {
129	126	}
130	127	};
131	128	let recorded = path.clone();
	129	let id = self.view.editor.recording();
	130	let title = self.view.editor.page()?.title;
132	131	let job = std::thread::spawn(move \|\| {
133	132	finish()?;
134	133	let recorded =
135	134	notebook::fs::read(&recorded).map_err(\|error\| error.to_string())?;
136		Ok(if video {
137		recorded
138		} else {
139		compress(&recorded).map_or(recorded, \|(bytes, _)\| bytes)
140		})
	135	let id = id.ok_or("Try recording again.")?;
	136	canvas::recording::file(id, &title, video, recorded)
	137	.map_err(\|error\| error.to_string())
141	138	});
142	139	self.media = Media::Saving { video, path, job };
143	140	}
...	...	@@ -147,66 +144,24 @@ impl State {
147	144	Media::Saving { job, .. } if wait \|\| job.is_finished() => {}
148	145	_ => return Ok(()),
149	146	}
150		let Media::Saving { video, path, job } = std::mem::take(&mut self.media) else {
	147	let Media::Saving { path, job, .. } = std::mem::take(&mut self.media) else {
151	148	unreachable!("saving, as matched")
152	149	};
153	150	let saved = job
154	151	.join()
155	152	.unwrap_or_else(\|_\| Err("Try recording again.".into()));
156		let Some(id) = self.view.editor.recording() else {
	153	if self.view.editor.recording().is_none() {
157	154	return Ok(());
158		};
159		let bytes = match saved {
160		Ok(bytes) => bytes,
	155	}
	156	let mut file = match saved {
	157	Ok(file) => file,
161	158	Err(detail) => {
162	159	platform::alert("Couldn't finish the recording", &detail);
163	160	return Ok(());
164	161	}
165	162	};
166		let duration_ms = if video {
167		video::Movie::parse(&bytes).map(\|movie\| movie.duration_ms())
168		} else {
169		Wave::parse(&bytes).map(\|wave\| wave.duration_ms())
170		};
171		let preview = platform::file_icon(&path);
	163	file.preview = platform::file_icon(&path).map(Into::into);
172	164	notebook::fs::remove_file(&path)?;
173		let title = self.view.editor.page()?.title;
174		let name: String = title
175		.chars()
176		.map(\|c\| {
177		if matches!(c, '/' \| '\\' \| '\0') {
178		'_'
179		} else {
180		c
181		}
182		})
183		.collect();
184		let name = if name.trim().is_empty() {
185		if video {
186		"Video recording"
187		} else {
188		"Audio recording"
189		}
190		.to_owned()
191		} else {
192		name
193		};
194		let extension = if video { "avi" } else { "wav" };
195		let file = Attachment {
196		id: onestore::page::text::new_id()?,
197		filename: format!("{name}.{extension}"),
198		source_path: None,
199		size: Some(crate::attachment::ICON_SIZE),
200		layout: Default::default(),
201		bytes: Some(bytes.into()),
202		preview: preview.map(Into::into),
203		recording: Some(Recording {
204		id,
205		kind: if video { 2 } else { 1 },
206		duration_ms,
207		}),
208		tags: Vec::new(),
209		};
210	165	let response = self.view.finish_recording(file)?;
211	166	self.respond(response);
212	167	Ok(())
...	...	@@ -226,19 +181,10 @@ impl State {
226	181	let name = Path::new(&file.filename)
227	182	.file_stem()
228	183	.map_or_else(String::new, \|stem\| stem.to_string_lossy().into_owned());
229		let movie = video::Movie::parse(&bytes);
230		// The sound as a WAV file every platform plays: IMA ADPCM decoded, a video's sound
231		// taken out of it.
232		let sound = movie.as_ref().map(\|movie\| {
233		movie
234		.wave(&bytes)
235		.unwrap_or_else(\|\| silence(movie.duration_ms()))
236		});
237		let sound = sound.map_or_else(\|\| decompress(&bytes), Some);
238		let playable = match sound {
	184	let playable = match canvas::recording::sound(&bytes) {
239	185	Some(wave) => Attachment {
240	186	filename: format!("{name}.wav"),
241		bytes: Some(decompress(&wave).unwrap_or(wave).into()),
	187	bytes: Some(wave.into()),
242	188	..file.clone()
243	189	},
244	190	None => file.clone(),
...	...	@@ -260,7 +206,7 @@ impl State {
260	206	player,
261	207	name,
262	208	recording: file.recording.map(\|recording\| recording.id),
263		video: movie.map(\|movie\| Video {
	209	video: video::Movie::parse(&bytes).map(\|movie\| Video {
264	210	bytes,
265	211	movie,
266	212	shown: None,
...	...	@@ -607,255 +553,11 @@ fn moment(text: &str) -> Option<u32> {
607	553	seconds.checked_mul(1000)
608	554	}
609	555
610		/// Silence as long as `ms`, the clock of a video without sound.
611		fn silence(ms: u32) -> Vec<u8> {
612		wave(8_000, &vec![0; ms as usize * 8])
613		}
614
615		/// A mono 16-bit PCM WAV file of `samples` at `rate`, as the platform recorders write.
616		pub(crate) fn wave(rate: u32, samples: &[i16]) -> Vec<u8> {
617		let data: Vec<u8> = samples.iter().flat_map(\|s\| s.to_le_bytes()).collect();
618		[
619		&b"RIFF"[..],
620		&(36 + data.len() as u32).to_le_bytes(),
621		b"WAVEfmt ",
622		&16_u32.to_le_bytes(),
623		&1_u16.to_le_bytes(),
624		&1_u16.to_le_bytes(),
625		&rate.to_le_bytes(),
626		&(rate * 2).to_le_bytes(),
627		&2_u16.to_le_bytes(),
628		&16_u16.to_le_bytes(),
629		b"data",
630		&(data.len() as u32).to_le_bytes(),
631		&data,
632		]
633		.concat()
634		}
635
636		/// A recording for a file attached under `filename`, as OneNote 2010 makes one of every
637		/// audio and video file it attaches, with the length of a WAV file's `bytes`.
638		pub(crate) fn attached(filename: &str, bytes: &[u8]) -> Option<Recording> {
639		Some(Recording {
640		id: onestore::page::text::new_guid().ok()?,
641		kind: Recording::kind_of(filename)?,
642		duration_ms: Wave::parse(bytes).map(\|wave\| wave.duration_ms()),
643		})
644		}
645
646		/// The chunks of a RIFF WAVE file this module reads.
647		struct Wave<'a> {
648		format: u16,
649		channels: u16,
650		rate: u32,
651		bytes_per_second: u32,
652		block: u16,
653		bits: u16,
654		/// The sample count a compressed file's `fact` chunk states.
655		samples: Option<u32>,
656		data: &'a [u8],
657		}
658
659		impl<'a> Wave<'a> {
660		fn parse(bytes: &'a [u8]) -> Option<Self> {
661		if bytes.get(..4)? != b"RIFF" \|\| bytes.get(8..12)? != b"WAVE" {
662		return None;
663		}
664		let u16_at = \|data: &[u8], at: usize\| {
665		Some(u16::from_le_bytes(data.get(at..at + 2)?.try_into().ok()?))
666		};
667		let u32_at = \|data: &[u8], at: usize\| {
668		Some(u32::from_le_bytes(data.get(at..at + 4)?.try_into().ok()?))
669		};
670		let (mut fmt, mut samples, mut data) = (None, None, None);
671		let mut at = 12;
672		while at + 8 <= bytes.len() {
673		let size = u32_at(bytes, at + 4)? as usize;
674		let body = &bytes[at + 8..(at + 8).saturating_add(size).min(bytes.len())];
675		match &bytes[at..at + 4] {
676		b"fmt " => fmt = Some(body),
677		b"fact" => samples = u32_at(body, 0),
678		b"data" => data = Some(body),
679		_ => {}
680		}
681		at = at + 8 + size + size % 2;
682		}
683		let fmt = fmt?;
684		Some(Self {
685		format: u16_at(fmt, 0)?,
686		channels: u16_at(fmt, 2)?,
687		rate: u32_at(fmt, 4)?,
688		bytes_per_second: u32_at(fmt, 8)?,
689		block: u16_at(fmt, 12)?,
690		bits: u16_at(fmt, 14)?,
691		samples,
692		data: data?,
693		})
694		}
695
696		fn duration_ms(&self) -> u32 {
697		let ms = match self.samples.filter(\|_\| self.format != 1) {
698		Some(samples) => u64::from(samples) * 1000 / u64::from(self.rate.max(1)),
699		None => self.data.len() as u64 * 1000 / u64::from(self.bytes_per_second.max(1)),
700		};
701		u32::try_from(ms).unwrap_or(u32::MAX)
702		}
703		}
704
705		/// IMA ADPCM's step sizes and how each code moves along them.
706		const STEPS: [i32; 89] = [
707		7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66,
708		73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449,
709		494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, 2272,
710		2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358, 5894, 6484, 7132, 7845, 8630, 9493,
711		10442, 11487, 12635, 13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767,
712		];
713		const INDEX_STEPS: [i32; 8] = [-1, -1, -1, -1, 2, 4, 6, 8];
714
715		/// Bytes per block: Windows' IMA ADPCM codec's choice for 16 kHz.
716		const BLOCK: usize = 512;
717		/// Samples per block: the header's, then two per remaining byte.
718		const BLOCK_SAMPLES: usize = (BLOCK - 4) * 2 + 1;
719
720		/// A mono 16-bit PCM WAV file as 4-bit IMA ADPCM (WAVE_FORMAT_DVI_ADPCM), which Windows
721		/// and macOS decode, a quarter of the size; and its length. None for other files.
722		pub(crate) fn compress(wav: &[u8]) -> Option<(Vec<u8>, u32)> {
723		let wave = Wave::parse(wav)?;
724		if wave.format != 1 \|\| wave.channels != 1 \|\| wave.bits != 16 {
725		return None;
726		}
727		let samples: Vec<i16> = wave
728		.data
729		.chunks_exact(2)
730		.map(\|pair\| i16::from_le_bytes([pair[0], pair[1]]))
731		.collect();
732		let mut data = Vec::with_capacity(samples.len() / 2 + BLOCK);
733		let mut index = 0_i32;
734		for block in samples.chunks(BLOCK_SAMPLES) {
735		let mut predicted = i32::from(block[0]);
736		data.extend_from_slice(&block[0].to_le_bytes());
737		data.extend_from_slice(&[index as u8, 0]);
738		// A short last block is padded with its last sample; `fact` gives the true length.
739		let last = *block.last().unwrap();
740		let rest = block[1..]
741		.iter()
742		.copied()
743		.chain(std::iter::repeat(last))
744		.take(BLOCK_SAMPLES - 1)
745		.collect::<Vec<_>>();
746		for pair in rest.chunks_exact(2) {
747		let mut byte = 0;
748		for (shift, sample) in [0, 4].into_iter().zip(pair) {
749		let code = encode(i32::from(*sample), &mut predicted, &mut index);
750		byte \|= code << shift;
751		}
752		data.push(byte);
753		}
754		}
755		let count = u32::try_from(samples.len()).ok()?;
756		let bytes_per_second = wave.rate * BLOCK as u32 / BLOCK_SAMPLES as u32;
757		let mut out = Vec::with_capacity(data.len() + 60);
758		let fmt: Vec<u8> = [
759		&0x11_u16.to_le_bytes()[..],
760		&1_u16.to_le_bytes(),
761		&wave.rate.to_le_bytes(),
762		&bytes_per_second.to_le_bytes(),
763		&(BLOCK as u16).to_le_bytes(),
764		&4_u16.to_le_bytes(),
765		&2_u16.to_le_bytes(),
766		&(BLOCK_SAMPLES as u16).to_le_bytes(),
767		]
768		.concat();
769		let riff = 4 + (8 + fmt.len()) + (8 + 4) + (8 + data.len());
770		out.extend_from_slice(b"RIFF");
771		out.extend_from_slice(&u32::try_from(riff).ok()?.to_le_bytes());
772		out.extend_from_slice(b"WAVEfmt ");
773		out.extend_from_slice(&(fmt.len() as u32).to_le_bytes());
774		out.extend_from_slice(&fmt);
775		out.extend_from_slice(b"fact");
776		out.extend_from_slice(&4_u32.to_le_bytes());
777		out.extend_from_slice(&count.to_le_bytes());
778		out.extend_from_slice(b"data");
779		out.extend_from_slice(&u32::try_from(data.len()).ok()?.to_le_bytes());
780		out.extend_from_slice(&data);
781		let duration = u32::try_from(u64::from(count) * 1000 / u64::from(wave.rate.max(1))).ok()?;
782		Some((out, duration))
783		}
784
785		/// The 4-bit code nearest `sample` from `predicted`, which it and `index` then follow as a
786		/// decoder does.
787		fn encode(sample: i32, predicted: &mut i32, index: &mut i32) -> u8 {
788		let step = STEPS[*index as usize];
789		let mut difference = sample - *predicted;
790		let mut code = 0;
791		if difference < 0 {
792		code = 8;
793		difference = -difference;
794		}
795		let mut delta = step >> 3;
796		for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] {
797		if difference >= part {
798		code \|= bit;
799		difference -= part;
800		delta += part;
801		}
802		}
803		*predicted = if code & 8 != 0 {
804		*predicted - delta
805		} else {
806		*predicted + delta
807		}
808		.clamp(-32768, 32767);
809		index = (index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88);
810		code
811		}
812
813		/// A mono IMA ADPCM WAV file as 16-bit PCM, which every platform's player decodes; none
814		/// for other files.
815		pub(crate) fn decompress(wav: &[u8]) -> Option<Vec<u8>> {
816		let wave = Wave::parse(wav).filter(\|wave\| wave.format == 0x11 && wave.channels == 1)?;
817		Some(self::wave(wave.rate, &decode(&wave)))
818		}
819
820		/// Each block's header sample and index, then its codes.
821		fn decode(wave: &Wave<'_>) -> Vec<i16> {
822		let mut out = Vec::new();
823		for block in wave.data.chunks(usize::from(wave.block.max(4))) {
824		let Some(header) = block.get(..4) else {
825		break;
826		};
827		let mut predicted = i32::from(i16::from_le_bytes([header[0], header[1]]));
828		let mut index = i32::from(header[2]).clamp(0, 88);
829		out.push(predicted as i16);
830		for byte in &block[4..] {
831		for code in [byte & 15, byte >> 4] {
832		let step = STEPS[index as usize];
833		let mut delta = step >> 3;
834		for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] {
835		if code & bit != 0 {
836		delta += part;
837		}
838		}
839		predicted = if code & 8 != 0 {
840		predicted - delta
841		} else {
842		predicted + delta
843		}
844		.clamp(-32768, 32767);
845		index = (index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88);
846		out.push(predicted as i16);
847		}
848		}
849		}
850		if let Some(samples) = wave.samples {
851		out.truncate(samples as usize);
852		}
853		out
854		}
855
856	556	#[cfg(test)]
857	557	pub(crate) mod tests {
858	558	use super::*;
	559	use canvas::recording::{compress, wave};
	560	use onestore::page::Recording;
859	561	use std::path::PathBuf;
860	562
861	563	/// Two seconds of a spoken-pitch chord.
...	...	@@ -870,57 +572,6 @@ pub(crate) mod tests {
870	572	.collect()
871	573	}
872	574
873		#[test]
874		fn a_recording_compresses_to_ima_adpcm_that_decodes_close_to_what_was_heard() {
875		let samples = tone();
876		let (bytes, duration) = compress(&wave(RATE, &samples)).unwrap();
877		assert_eq!(duration, 2000);
878		let wave = Wave::parse(&bytes).unwrap();
879		assert_eq!(
880		(wave.format, wave.channels, wave.rate, wave.bits),
881		(0x11, 1, RATE, 4)
882		);
883		assert_eq!(wave.data.len() % BLOCK, 0);
884		assert_eq!(wave.duration_ms(), 2000);
885		assert!(bytes.len() * 3 < samples.len() * 2);
886		let decoded = decode(&wave);
887		assert_eq!(
888		Wave::parse(&decompress(&bytes).unwrap())
889		.unwrap()
890		.data
891		.len(),
892		decoded.len() * 2
893		);
894		assert_eq!(decoded.len(), samples.len());
895		let noise: f64 = samples
896		.iter()
897		.zip(&decoded)
898		.map(\|(a, b)\| (f64::from(a) - f64::from(b)).powi(2))
899		.sum();
900		let signal: f64 = samples.iter().map(\|a\| f64::from(*a).powi(2)).sum();
901		let snr = 10.0 * (signal / noise).log10();
902		assert!(snr > 20.0, "{snr} dB");
903		}
904
905		#[test]
906		fn only_mono_16_bit_pcm_compresses_and_attached_wav_files_know_their_length() {
907		let stereo = {
908		let mut bytes = wave(RATE, &[0; 64]);
909		bytes[22] = 2;
910		bytes
911		};
912		assert!(compress(&stereo).is_none());
913		assert!(compress(b"not a wave").is_none());
914		let silence = include_bytes!(
915		"../../../corpus/m6/native-features-01/notebook/fixture-data/silence.wav"
916		);
917		// OneNote 2010 stored 1000 ms for this file when attached (`corpus/recording`).
918		let recording = attached("silence.WAV", silence).unwrap();
919		assert_eq!((recording.kind, recording.duration_ms), (1, Some(1000)));
920		assert_eq!(attached("clip.wmv", b"").unwrap().kind, 2);
921		assert!(attached("notes.txt", b"").is_none());
922		}
923
924	575	/// A recording made through the editor at the end of "Before recording", two notes
925	576	/// written while it records, stored as the host stores it; `export` names a variable
926	577	/// naming a directory receiving the notebook for a cold read in OneNote 2010.
...	...	@@ -976,7 +627,7 @@ pub(crate) mod tests {
976	627	id: onestore::page::text::new_id().unwrap(),
977	628	filename: filename.into(),
978	629	source_path: None,
979		size: Some(crate::attachment::ICON_SIZE),
	630	size: Some(canvas::gpu::page::ICON_SIZE),
980	631	layout: Default::default(),
981	632	bytes: Some(bytes.into()),
982	633	preview: Some(canvas::gpu::page::file_icon().into()),

crates/snowbound/src/video.rs deleted-407

...	...	@@ -1,407 +0,0 @@
1		//! Video recordings as AVI files of Motion JPEG pictures and PCM sound, which OneNote 2010
2		//! plays and lists as recordings (`corpus/recording/video`), and reading them back to play.
3
4		use std::ops::Range;
5
6		/// Pictures a second in a recording: OneNote 2010's own profiles record at 15.
7		pub(crate) const FPS: u32 = 15;
8		/// A recording's picture size, OneNote 2010's own.
9		pub(crate) const SIZE: [u32; 2] = [320, 240];
10
11		/// Interleaved signed 16-bit little-endian sound.
12		#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
13		pub(crate) struct Pcm {
14		pub rate: u32,
15		pub channels: u16,
16		pub data: Vec<u8>,
17		}
18
19		fn chunk(out: &mut Vec<u8>, id: &[u8; 4], body: &[u8]) {
20		out.extend_from_slice(id);
21		out.extend_from_slice(&(body.len() as u32).to_le_bytes());
22		out.extend_from_slice(body);
23		if body.len() % 2 == 1 {
24		out.push(0);
25		}
26		}
27
28		#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
29		fn list(kind: &[u8; 4], body: &[u8]) -> Vec<u8> {
30		let mut out = Vec::with_capacity(body.len() + 12);
31		chunk(&mut out, b"LIST", &[&kind[..], body].concat());
32		out
33		}
34
35		#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
36		fn words(values: &[u32]) -> Vec<u8> {
37		values
38		.iter()
39		.flat_map(\|value\| value.to_le_bytes())
40		.collect()
41		}
42
43		/// An AVI file of `frames`, JPEG pictures of `size` shown `FPS` a second, where an empty one
44		/// repeats the picture before it, and `sound` alongside. None past AVI's 4 GiB.
45		#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
46		pub(crate) fn write(size: [u32; 2], frames: &[Vec<u8>], sound: &Pcm) -> Option<Vec<u8>> {
47		let block = u32::from(sound.channels) * 2;
48		let bytes_per_second = sound.rate * block;
49		let samples = sound.data.len() as u64 / u64::from(block);
50		let count = frames.len() as u64;
51		// The sound up to frame `index`, whole samples.
52		let sound_at = \|index: u64\| -> usize {
53		let before = if index >= count {
54		samples
55		} else {
56		(index * u64::from(sound.rate) / u64::from(FPS)).min(samples)
57		};
58		(before * u64::from(block)) as usize
59		};
60		let mut movi = b"movi".to_vec();
61		let mut index = Vec::new();
62		let [mut largest_frame, mut largest_sound] = [0, 0];
63		for (at, frame) in frames.iter().enumerate() {
64		let entries: [(&[u8; 4], &[u8]); 2] = [
65		(b"00dc", frame),
66		(
67		b"01wb",
68		&sound.data[sound_at(at as u64)..sound_at(at as u64 + 1)],
69		),
70		];
71		for (id, body) in entries {
72		if id == b"01wb" && body.is_empty() {
73		continue;
74		}
75		// Keyframes: every picture and all sound; a repeat is none.
76		let flags: u32 = if body.is_empty() { 0 } else { 0x10 };
77		index.extend_from_slice(id);
78		index.extend(words(&[
79		flags,
80		u32::try_from(movi.len()).ok()?,
81		body.len() as u32,
82		]));
83		chunk(&mut movi, id, body);
84		let largest = if id == b"00dc" {
85		&mut largest_frame
86		} else {
87		&mut largest_sound
88		};
89		largest = (largest).max(body.len() as u32);
90		}
91		}
92		let [width, height] = size;
93		let frame_count = u32::try_from(count).ok()?;
94		let rect = [0u16, 0, width as u16, height as u16]
95		.iter()
96		.flat_map(\|side\| side.to_le_bytes())
97		.collect::<Vec<u8>>();
98		let mut video = Vec::new();
99		let strh = [
100		&b"vidsMJPG"[..],
101		&words(&[0, 0, 0, 1, FPS, 0, frame_count, largest_frame, u32::MAX, 0]),
102		&rect,
103		]
104		.concat();
105		chunk(&mut video, b"strh", &strh);
106		let strf = [
107		words(&[40, width, height]),
108		[&1u16.to_le_bytes()[..], &24u16.to_le_bytes()].concat(),
109		b"MJPG".to_vec(),
110		words(&[width * height * 3, 0, 0, 0, 0]),
111		]
112		.concat();
113		chunk(&mut video, b"strf", &strf);
114		let mut streams = list(b"strl", &video);
115		if samples > 0 {
116		let mut audio = Vec::new();
117		let strh = [
118		&b"auds\0\0\0\0"[..],
119		&words(&[
120		0,
121		0,
122		0,
123		block,
124		bytes_per_second,
125		0,
126		u32::try_from(samples).ok()?,
127		largest_sound,
128		u32::MAX,
129		block,
130		]),
131		&[0; 8],
132		]
133		.concat();
134		chunk(&mut audio, b"strh", &strh);
135		let strf = [
136		&1u16.to_le_bytes()[..],
137		&sound.channels.to_le_bytes(),
138		&sound.rate.to_le_bytes(),
139		&bytes_per_second.to_le_bytes(),
140		&(block as u16).to_le_bytes(),
141		&16u16.to_le_bytes(),
142		&0u16.to_le_bytes(),
143		]
144		.concat();
145		chunk(&mut audio, b"strf", &strf);
146		streams.extend(list(b"strl", &audio));
147		}
148		let mut avih = Vec::new();
149		chunk(
150		&mut avih,
151		b"avih",
152		&words(&[
153		1_000_000 / FPS,
154		largest_frame
155		.saturating_add(largest_sound)
156		.saturating_mul(FPS),
157		0,
158		// AVIF_HASINDEX \| AVIF_ISINTERLEAVED
159		0x110,
160		frame_count,
161		0,
162		if samples > 0 { 2 } else { 1 },
163		largest_frame.max(largest_sound),
164		width,
165		height,
166		0,
167		0,
168		0,
169		0,
170		]),
171		);
172		let header = list(b"hdrl", &[avih, streams].concat());
173		let mut body = b"AVI ".to_vec();
174		body.extend(header);
175		let movi_size = u32::try_from(movi.len()).ok()?;
176		body.extend_from_slice(b"LIST");
177		body.extend_from_slice(&movi_size.to_le_bytes());
178		body.extend(movi);
179		chunk(&mut body, b"idx1", &index);
180		u32::try_from(body.len()).ok()?;
181		let mut out = Vec::with_capacity(body.len() + 8);
182		chunk(&mut out, b"RIFF", &body);
183		Some(out)
184		}
185
186		/// An AVI file's pictures and sound, as byte ranges of the file.
187		pub(crate) struct Movie {
188		/// Each picture's JPEG; an empty range repeats the one before.
189		pub frames: Vec<Range<usize>>,
190		pub frame_us: u32,
191		/// The sound's WAVEFORMATEX and its data, in order.
192		pub sound: Option<(Range<usize>, Vec<Range<usize>>)>,
193		}
194
195		impl Movie {
196		/// A Motion JPEG AVI file's contents; none for other files.
197		pub(crate) fn parse(bytes: &[u8]) -> Option<Self> {
198		let mut movie = Self {
199		frames: Vec::new(),
200		frame_us: 0,
201		sound: None,
202		};
203		let mut kinds = Vec::new();
204		let mut first = true;
205		for (id, body) in Chunks::new(bytes, 0..bytes.len()) {
206		let kind = bytes.get(body.start..body.start + 4)?;
207		if id != *b"RIFF" \|\| (kind != b"AVI " && !(kind == b"AVIX" && !first)) {
208		return (!first).then_some(movie).filter(\|movie\| movie.frame_us > 0);
209		}
210		first = false;
211		movie.walk(bytes, body.start + 4..body.end, &mut kinds)?;
212		}
213		(movie.frame_us > 0).then_some(movie)
214		}
215
216		fn walk(&mut self, bytes: &[u8], range: Range<usize>, kinds: &mut Vec<[u8; 4]>) -> Option<()> {
217		for (id, body) in Chunks::new(bytes, range) {
218		let u32_at = \|at: usize\| {
219		Some(u32::from_le_bytes(
220		bytes
221		.get(body.start + at..body.start + at + 4)?
222		.try_into()
223		.ok()?,
224		))
225		};
226		match &id {
227		b"LIST" => {
228		let inner = body.start + 4..body.end;
229		self.walk(bytes, inner, kinds)?;
230		}
231		b"strh" => {
232		let kind: [u8; 4] = bytes.get(body.start..body.start + 4)?.try_into().ok()?;
233		if &kind == b"vids" {
234		let [scale, rate] = [u32_at(20)?, u32_at(24)?];
235		self.frame_us =
236		u32::try_from(u64::from(scale) * 1_000_000 / u64::from(rate.max(1)))
237		.ok()?;
238		}
239		kinds.push(kind);
240		}
241		b"strf" if kinds.last() == Some(b"vids") => {
242		if bytes.get(body.start + 16..body.start + 20)? != b"MJPG" {
243		return None;
244		}
245		}
246		b"strf" if kinds.last() == Some(b"auds") => {
247		self.sound = Some((body.clone(), Vec::new()));
248		}
249		[a, b, c, d] if a.is_ascii_digit() && b.is_ascii_digit() => {
250		let stream = usize::from((a - b'0') * 10 + (b - b'0'));
251		match (kinds.get(stream).map(\|kind\| &kind[..]), [c, d]) {
252		(Some(b"vids"), [b'd', b'c' \| b'b']) => self.frames.push(body),
253		(Some(b"auds"), [b'w', b'b']) => {
254		if let Some((_, data)) = &mut self.sound {
255		data.push(body);
256		}
257		}
258		_ => {}
259		}
260		}
261		_ => {}
262		}
263		}
264		Some(())
265		}
266
267		pub(crate) fn duration_ms(&self) -> u32 {
268		let ms = (self.frames.len() as u64 * u64::from(self.frame_us)).div_ceil(1000);
269		u32::try_from(ms).unwrap_or(u32::MAX)
270		}
271
272		/// The picture showing `at_ms` in: the last stored one at or before it.
273		pub(crate) fn frame(&self, at_ms: u32) -> Option<Range<usize>> {
274		let at = (u64::from(at_ms) * 1000 / u64::from(self.frame_us.max(1))) as usize;
275		self.frames[..=at.min(self.frames.len().checked_sub(1)?)]
276		.iter()
277		.rev()
278		.find(\|frame\| !frame.is_empty())
279		.cloned()
280		}
281
282		/// The sound as a WAV file, or none where the file has none.
283		pub(crate) fn wave(&self, bytes: &[u8]) -> Option<Vec<u8>> {
284		let (format, data) = self.sound.as_ref()?;
285		let format = &bytes[format.clone()];
286		let length: usize = data.iter().map(ExactSizeIterator::len).sum();
287		let mut out = Vec::with_capacity(length + 64);
288		out.extend_from_slice(b"RIFF");
289		out.extend_from_slice(&((4 + 8 + format.len() + 8 + length) as u32).to_le_bytes());
290		out.extend_from_slice(b"WAVE");
291		chunk(&mut out, b"fmt ", format);
292		out.extend_from_slice(b"data");
293		out.extend_from_slice(&(length as u32).to_le_bytes());
294		for range in data {
295		out.extend_from_slice(&bytes[range.clone()]);
296		}
297		Some(out)
298		}
299		}
300
301		/// The chunks in `range` of a RIFF file: each one's id and body.
302		struct Chunks<'a> {
303		bytes: &'a [u8],
304		at: usize,
305		end: usize,
306		}
307
308		impl<'a> Chunks<'a> {
309		fn new(bytes: &'a [u8], range: Range<usize>) -> Self {
310		Self {
311		bytes,
312		at: range.start,
313		end: range.end.min(bytes.len()),
314		}
315		}
316		}
317
318		impl Iterator for Chunks<'_> {
319		type Item = ([u8; 4], Range<usize>);
320
321		fn next(&mut self) -> Option<Self::Item> {
322		let header = self.bytes.get(self.at..self.at + 8)?;
323		if self.at + 8 > self.end {
324		return None;
325		}
326		let id: [u8; 4] = header[..4].try_into().ok()?;
327		let size = u32::from_le_bytes(header[4..].try_into().ok()?) as usize;
328		let start = self.at + 8;
329		let end = start.saturating_add(size).min(self.end);
330		self.at = end + size % 2;
331		Some((id, start..end))
332		}
333		}
334
335		/// Pictures taken at uneven moments, as a camera takes them, made the `FPS` a second of a
336		/// recording: each one shows until the next, fitted to `SIZE`.
337		#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
338		#[derive(Default)]
339		pub(crate) struct Pictures {
340		frames: Vec<Vec<u8>>,
341		last: Option<image::RgbImage>,
342		/// Whether the last picture is stored yet, which the frames after it repeat.
343		stored: bool,
344		}
345
346		#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
347		impl Pictures {
348		/// The picture taken `at_us` microseconds in; the first shows from the start.
349		pub(crate) fn take(&mut self, at_us: u64, picture: image::RgbImage) {
350		self.fill(at_us);
351		self.last = Some(fit(picture));
352		self.stored = false;
353		}
354
355		/// The frames of a recording `end_us` microseconds long.
356		pub(crate) fn finish(mut self, end_us: u64) -> Vec<Vec<u8>> {
357		self.fill(end_us);
358		self.frames
359		}
360
361		/// Frames up to `until_us`, of the last picture.
362		fn fill(&mut self, until_us: u64) {
363		let Some(last) = &self.last else {
364		return;
365		};
366		while (self.frames.len() as u64) * 1_000_000 / u64::from(FPS) < until_us {
367		let frame = if self.stored {
368		Vec::new()
369		} else {
370		self.stored = true;
371		let mut jpeg = Vec::new();
372		let encoded = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 70)
373		.encode_image(last);
374		if encoded.is_err() {
375		jpeg.clear();
376		}
377		jpeg
378		};
379		self.frames.push(frame);
380		}
381		}
382		}
383
384		/// `picture` scaled to fit `SIZE`, centred on black.
385		#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
386		fn fit(picture: image::RgbImage) -> image::RgbImage {
387		let [width, height] = SIZE;
388		if picture.dimensions() == (width, height) {
389		return picture;
390		}
391		let scale =
392		(width as f32 / picture.width() as f32).min(height as f32 / picture.height() as f32);
393		let fitted = image::imageops::resize(
394		&picture,
395		((picture.width() as f32 * scale) as u32).clamp(1, width),
396		((picture.height() as f32 * scale) as u32).clamp(1, height),
397		image::imageops::FilterType::Triangle,
398		);
399		let mut out = image::RgbImage::new(width, height);
400		image::imageops::overlay(
401		&mut out,
402		&fitted,
403		i64::from((width - fitted.width()) / 2),
404		i64::from((height - fitted.height()) / 2),
405		);
406		out
407		}