From 01e73fd4cffeccb05dd939ca75bcfb583fb252e2 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Thu, 1 Oct 2026 23:43:16 -0700 Subject: [PATCH] feat: record audio and video on iOS Record Audio and Record Video join the page's Insert menu. AVFoundation records, and crates/mobile stores the result through canvas::recording, which now holds what the desktop stored: 16 kHz IMA ADPCM WAV and Motion JPEG AVI, the files OneNote 2010 plays. A bar over the page's foot shows the clock with Pause and Stop; a tap on a recording plays it with AVPlayer, a video's pictures in the bar, and See Playback follows it. Audio keeps recording in the background under the audio background mode, as a lecture outlasts the screen's timeout, and a call pauses it. The camera stops in the background, which ends a video recording and saves it. The simulator records video from a stand-in camera. Assisted-by: claude-opus-5.5 --- apps/ios/Info.plist | 8 +- apps/ios/Snowbound/CanvasView.swift | 73 ++- apps/ios/Snowbound/PageViewController.swift | 116 ++++ apps/ios/Snowbound/Recording.swift | 545 ++++++++++++++++++ apps/ios/Snowbound/Script.swift | 16 +- arc/canvas.md | 3 +- arc/platforms.md | 7 + crates/canvas/src/editor/recording.rs | 5 + crates/canvas/src/gpu/page.rs | 3 + crates/canvas/src/interaction/mod.rs | 8 + crates/canvas/src/lib.rs | 2 + crates/canvas/src/recording.rs | 424 ++++++++++++++ .../src => canvas/src/recording}/video.rs | 53 +- crates/mobile/include/snowbound.h | 14 + crates/mobile/src/lib.rs | 19 +- crates/mobile/src/recording.rs | 274 +++++++++ crates/mobile/src/tests.rs | 107 ++++ crates/snowbound/Cargo.toml | 8 +- crates/snowbound/src/attachment.rs | 7 +- crates/snowbound/src/main.rs | 1 - crates/snowbound/src/media_linux.rs | 14 +- crates/snowbound/src/media_macos.rs | 23 +- crates/snowbound/src/recording.rs | 393 +------------ 23 files changed, 1692 insertions(+), 431 deletions(-) create mode 100644 apps/ios/Snowbound/Recording.swift create mode 100644 crates/canvas/src/recording.rs rename crates/{snowbound/src => canvas/src/recording}/video.rs (91%) create mode 100644 crates/mobile/src/recording.rs diff --git a/apps/ios/Info.plist b/apps/ios/Info.plist index ef1cb4094c7a988eb5bb7a1aeeeda6ca3b0f84ed..49da15c8defcc47f7ac1c1ba7c4b2e5a6b3d4ef2 100644 --- a/apps/ios/Info.plist +++ b/apps/ios/Info.plist @@ -22,7 +22,9 @@ LSSupportsOpeningDocumentsInPlace NSCameraUsageDescription - Take a photo to put on a page. + Take photos and record video on your pages. + NSMicrophoneUsageDescription + Record audio and video on your pages. NSUbiquitousContainers iCloud.net.paperclover.snowbound @@ -42,6 +44,10 @@ UIApplicationSupportsMultipleScenes + UIBackgroundModes + + audio + UIFileSharingEnabled UILaunchScreen diff --git a/apps/ios/Snowbound/CanvasView.swift b/apps/ios/Snowbound/CanvasView.swift index fa18181abb96ee5c25c55b192731f2100eeb97b4..46c6708ed55f07b0adc4b55a2378dd44ca5b3d56 100644 --- a/apps/ios/Snowbound/CanvasView.swift +++ b/apps/ios/Snowbound/CanvasView.swift @@ -137,6 +137,8 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI var onOpened: (() -> Void)? /// Called when a tap on the page date asks to change it, with the date it shows. var onDate: ((DateField, Date) -> Void)? + /// Called when a tap on a recording, or beside a note linked to one, asks to play it. + var onPlay: ((PlayRequest) -> Void)? /// Formatting above the keyboard. let formatBar = FormatBar() /// Room above the page for a bar over it. @@ -147,6 +149,16 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI } } + /// Room below the page, above the keyboard, for a bar over its foot. + var bottomInset: CGFloat = 0 { + didSet { + guard bottomInset != oldValue else { return } + contentInset.bottom += bottomInset - oldValue + verticalScrollIndicatorInsets.bottom = contentInset.bottom + revealCaret() + } + } + init(section: Section, page: String) { self.section = section self.page = page @@ -480,8 +492,8 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI else { return } let keyboard = convert(frame, from: window.screen.coordinateSpace) let overlap = max(0, bounds.maxY - keyboard.minY - safeAreaInsets.bottom) - contentInset.bottom = overlap - verticalScrollIndicatorInsets.bottom = overlap + contentInset.bottom = overlap + bottomInset + verticalScrollIndicatorInsets.bottom = contentInset.bottom revealCaret() } @@ -542,7 +554,9 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI func tap(at point: CGPoint) { press(at: point) var seconds: Int64 = 0 - if let handle, let field = DateField(rawValue: sb_view_date_request(handle, &seconds)) { + if let request = playRequest() { + onPlay?(request) + } else if let handle, let field = DateField(rawValue: sb_view_date_request(handle, &seconds)) { onDate?(field, Date(timeIntervalSince1970: TimeInterval(seconds))) } else if inkFrame != nil { showInkMenu() @@ -1142,6 +1156,59 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI } } + // MARK: Recordings + + /// Starts recording at the caret, under a line saying when; whether it started. + func startRecording(video: Bool) -> Bool { + guard let handle else { return false } + let (day, time) = titleDate() + return edit(external: true) { sb_view_start_recording(handle, video, day, time) } + } + + /// How far the recording has got in milliseconds, without its pauses, and whether it is + /// paused; nil when nothing records. + var recording: (ms: Int64, paused: Bool)? { + guard let handle else { return nil } + var paused = false + let ms = sb_view_recording(handle, &paused) + return ms < 0 ? nil : (ms, paused) + } + + /// Nothing typed while the recording is paused links to it. + func pauseRecording(_ paused: Bool) { + guard let handle else { return } + sb_view_pause_recording(handle, paused) + } + + /// Puts the recording made where it started: a PCM WAV file, or with `video` the AVI + /// file of `sb_movie_finish`. Nil forgets a recording whose file never came. + func finishRecording(_ file: Data?, video: Bool) { + guard let handle else { return } + _ = edit(external: true) { + guard let file else { return sb_view_finish_recording(handle, nil, 0, video) } + return file.withUnsafeBytes { bytes in + sb_view_finish_recording(handle, bytes.bindMemory(to: UInt8.self).baseAddress, bytes.count, video) + } + } + } + + /// The recording the last tap asked to play, with its files written out to play from. + private func playRequest() -> PlayRequest? { + let folder = FileManager.default.temporaryDirectory.appendingPathComponent("Playback") + guard let handle, let json = sb_view_play_request(handle, folder.path) else { return nil } + defer { sb_string_free(json) } + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + return try? decoder.decode(PlayRequest.self, from: Data(String(cString: json).utf8)) + } + + /// See Playback: the note linked at `ms` into the recording playing is highlighted; nil + /// ends playback. + func played(at ms: Int64?) { + guard let handle, sb_view_played(handle, ms ?? -1) else { return } + dirty = true + } + /// Commits marked text, so it is stored before the app leaves the screen. func commitComposition() { guard markedTextRange != nil else { return } diff --git a/apps/ios/Snowbound/PageViewController.swift b/apps/ios/Snowbound/PageViewController.swift index 04379b01e3b1d20aa072d6efc34eec5ac4cd1a11..b7a7e6ab1f65218c26b5a428891f4a54c5b0520e 100644 --- a/apps/ios/Snowbound/PageViewController.swift +++ b/apps/ios/Snowbound/PageViewController.swift @@ -21,6 +21,10 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate private let titleFocus: Bool private lazy var canvas = CanvasView(section: section, page: page) private let bar = ConflictBar() + /// What records or plays on this page, over its foot. + private let media = MediaBar() + private var playback: Playback? + private var mediaClock: Timer? /// Reading View, beside the page menu where the page has one. private var reader: UIBarButtonItem? /// Opens another page or conflict page of the section. @@ -62,7 +66,15 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate bar.translatesAutoresizingMaskIntoConstraints = false view.addSubview(canvas) view.addSubview(bar) + media.translatesAutoresizingMaskIntoConstraints = false + view.addSubview(media) + let wide = media.widthAnchor.constraint(equalToConstant: 480) + wide.priority = .required - 1 NSLayoutConstraint.activate([ + wide, + media.centerXAnchor.constraint(equalTo: view.safeAreaLayoutGuide.centerXAnchor), + media.leadingAnchor.constraint(greaterThanOrEqualTo: view.safeAreaLayoutGuide.leadingAnchor, constant: 12), + media.bottomAnchor.constraint(equalTo: view.keyboardLayoutGuide.topAnchor, constant: -12), canvas.leadingAnchor.constraint(equalTo: view.leadingAnchor), canvas.trailingAnchor.constraint(equalTo: view.trailingAnchor), canvas.topAnchor.constraint(equalTo: view.topAnchor), @@ -75,6 +87,10 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate canvas.onChange = { [weak self] in self?.editingChanged() } canvas.onOpened = { [weak self] in self?.opened() } canvas.onDate = { [weak self] field, date in self?.pickDate(field, date) } + canvas.onPlay = { [weak self] request in self?.play(request) } + media.onPause = { [weak self] in self?.pauseMedia() } + media.onStop = { [weak self] in self?.stopMedia() } + NotificationCenter.default.addObserver(self, selector: #selector(showMedia), name: Recorder.changed, object: nil) canvas.formatBar.onPicture = { [weak self] camera in self?.pickPicture(camera: camera) } bar.onAction = { [weak self] in self?.conflictAction() } NotificationCenter.default.addObserver( @@ -173,6 +189,16 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate UIAction( title: "Space", image: UIImage(systemName: "arrow.up.and.down.text.horizontal"), attributes: editable ) { [weak self] _ in self?.canvas.insertSpace() }, + UIMenu(options: .displayInline, children: [ + UIAction( + title: "Record Audio", image: UIImage(systemName: "mic"), + attributes: Recorder.current == nil ? editable : .disabled + ) { [weak self] _ in self?.record(video: false) }, + UIAction( + title: "Record Video", image: UIImage(systemName: "video"), + attributes: Recorder.current == nil && Recorder.canRecordVideo ? editable : .disabled + ) { [weak self] _ in self?.record(video: true) }, + ]), ]) return (Prototype.reading ? [readingAction()] : []) + [ insert, @@ -217,6 +243,8 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate } private func toggleReading() { + // A recording goes on the page as laid out. + guard Recorder.current?.canvas !== canvas else { return } canvas.showReading(!canvas.readingShown) } @@ -289,6 +317,94 @@ final class PageViewController: UIViewController, PHPickerViewControllerDelegate @objc private func leaving() { canvas.commitComposition() } + override func viewWillDisappear(_ animated: Bool) { + super.viewWillDisappear(animated) + // A recording goes on the page it started on, as the desktop's does on leaving it. + if Recorder.current?.canvas === canvas { Recorder.current?.stop() } + stopPlayback() + } + + // MARK: Recording and playback + + func record(video: Bool) { + stopPlayback() + Recorder.start(video: video, on: canvas) { [weak self] title, message in self?.alert(title, message) } + } + + func stopRecording() { + if Recorder.current?.canvas === canvas { Recorder.current?.stop() } + } + + private func play(_ request: PlayRequest) { + guard Recorder.current == nil else { return } + playback = Playback(request) + showMedia() + } + + private func stopPlayback() { + guard playback != nil else { return } + playback = nil + canvas.played(at: nil) + showMedia() + } + + private func pauseMedia() { + if let recorder = Recorder.current, recorder.canvas === canvas { + recorder.pause(!(recorder.elapsed?.paused ?? false)) + } else { + playback?.toggle() + } + showMedia() + } + + private func stopMedia() { + if Recorder.current?.canvas === canvas { + Recorder.current?.stop() + } else { + stopPlayback() + } + } + + /// The bar shows the clock of what records or plays, and a video's picture as it plays. + @objc private func showMedia() { + if let recorder = Recorder.current, recorder.canvas === canvas { + media.picture.isHidden = true + if recorder.saving { + media.showSaving(recorder.video ? "Saving video…" : "Saving audio…") + } else if let elapsed = recorder.elapsed { + let state = elapsed.paused ? "Paused" : recorder.video ? "Recording video" : "Recording" + media.show( + "\(state) \(MediaBar.clock(elapsed.ms))", recording: true, + paused: recorder.pausable ? elapsed.paused : nil) + } + } else if let playback { + if playback.failed { + stopPlayback() + return alert("Couldn't Play the Recording", "This kind of recording doesn't play on this device.") + } + let total = playback.durationMs.map { " / " + MediaBar.clock($0) } ?? "" + media.show( + "\(playback.request.name) \(MediaBar.clock(playback.ms))\(total)", recording: false, + paused: !playback.playing, playing: true) + if let picture = playback.picture() { media.picture.image = picture } + media.picture.isHidden = playback.request.video == nil + canvas.played(at: playback.ms) + } else { + media.isHidden = true + canvas.bottomInset = 0 + mediaClock?.invalidate() + mediaClock = nil + return + } + // The line being written stays above the bar. + canvas.bottomInset = view.keyboardLayoutGuide.layoutFrame.minY - media.frame.minY + if mediaClock == nil { + mediaClock = Timer.scheduledTimer(withTimeInterval: 1.0 / 15, repeats: true) { [weak self] _ in + self?.showMedia() + } + } + } + @objc private func sectionChanged(_ notification: Notification) { let flags = notification.userInfo?["flags"] as? UInt32 ?? 0 if flags & Section.rejected != 0 { diff --git a/apps/ios/Snowbound/Recording.swift b/apps/ios/Snowbound/Recording.swift new file mode 100644 index 0000000000000000000000000000000000000000..f04e6f27d66fb73a851b9801663267aaebdab96d --- /dev/null +++ b/apps/ios/Snowbound/Recording.swift @@ -0,0 +1,545 @@ +import AVFoundation +import UIKit + +/// A recording `sb_view_play_request` asks to play: its sound file from `atMs`, and for a +/// video the AVI file whose JPEG pictures `frames` gives, `frameUs` apart. +struct PlayRequest: Decodable { + struct Video: Decodable { + let path: String + let frames: [[Int]] + let frameUs: UInt32 + let size: [Int] + } + let name: String + let sound: String + let video: Video? + let atMs: UInt32 +} + +/// Record Audio and Record Video, stored as the desktop stores them: sound as mono PCM at +/// `sb_recording_rate`, which the page keeps as IMA ADPCM, and video as a movie from the +/// camera made into Motion JPEG AVI. Only one records at a time, and it keeps the page it +/// records on until its file is there, whether or not the page is still shown. +/// +/// Audio records on with the screen locked or the app in the background, under the audio +/// background mode, as a lecture or meeting outlasts the screen's timeout; a call pauses it. +/// The camera stops in the background, so a video recording then ends and is saved. +final class Recorder: NSObject, AVCaptureFileOutputRecordingDelegate { + /// The recording under way or being saved. + private(set) static var current: Recorder? + /// Posted when the current recording pauses, resumes, saves or ends. + static let changed = Notification.Name("SnowboundRecordingChanged") + + let video: Bool + let canvas: CanvasView + private let folder: URL + private var sound: AVAudioRecorder? + private var capture: AVCaptureSession? + private let output = AVCaptureMovieFileOutput() + #if targetEnvironment(simulator) + private var stub: StubCamera? + #endif + /// Stopped, its file being made and put on the page. + private(set) var saving = false + + /// Whether video can be recorded here: with a camera, or the simulator's stand-in. + static var canRecordVideo: Bool { + #if targetEnvironment(simulator) + true + #else + AVCaptureDevice.default(for: .video) != nil + #endif + } + + private init(video: Bool, canvas: CanvasView) { + self.video = video + self.canvas = canvas + folder = FileManager.default.temporaryDirectory.appendingPathComponent("Recording") + super.init() + } + + /// Starts recording on `canvas` at its caret once the system allows the microphone, and + /// for video the camera; `failed` says why it could not. + static func start(video: Bool, on canvas: CanvasView, failed: @escaping (String, String) -> Void) { + guard current == nil else { return } + let title = video ? "Couldn't Record Video" : "Couldn't Record Audio" + AVAudioApplication.requestRecordPermission { microphone in + let begin = { (camera: Bool) in + DispatchQueue.main.async { + guard current == nil else { return } + if video && !camera { + return failed(title, "Allow Snowbound to use the camera in Settings, Privacy & Security, Camera.") + } + // A video without the microphone records silent, as on the desktop. + if !video && !microphone { + return failed( + title, "Allow Snowbound to use the microphone in Settings, Privacy & Security, Microphone.") + } + let recorder = Recorder(video: video, canvas: canvas) + do { + try recorder.begin(microphone: microphone) + } catch { + recorder.end() + return failed(title, "Try recording again.") + } + guard canvas.startRecording(video: video) else { + recorder.end() + return failed(title, "Try recording again.") + } + current = recorder + NotificationCenter.default.post(name: changed, object: recorder) + } + } + if video { + #if targetEnvironment(simulator) + begin(true) + #else + AVCaptureDevice.requestAccess(for: .video, completionHandler: begin) + #endif + } else { + begin(true) + } + } + } + + private func begin(microphone: Bool) throws { + try? FileManager.default.removeItem(at: folder) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + let session = AVAudioSession.sharedInstance() + if microphone { + try session.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetoothHFP]) + try session.setActive(true) + NotificationCenter.default.addObserver( + self, selector: #selector(interrupted), name: AVAudioSession.interruptionNotification, object: session) + } + #if targetEnvironment(simulator) + if video { + stub = try StubCamera(movie: folder.appendingPathComponent("camera.mov")) + } + #endif + if !video || isStubbed { + guard microphone else { return } + let settings: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: sb_recording_rate(), AVNumberOfChannelsKey: 1, + AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false, + ] + let sound = try AVAudioRecorder(url: soundFile, settings: settings) + guard sound.record() else { throw CocoaError(.fileWriteUnknown) } + self.sound = sound + return + } + let capture = AVCaptureSession() + capture.beginConfiguration() + if capture.canSetSessionPreset(.vga640x480) { capture.sessionPreset = .vga640x480 } + let devices = [AVCaptureDevice.default(for: .video)] + (microphone ? [AVCaptureDevice.default(for: .audio)] : []) + for (index, device) in devices.enumerated() { + guard let device, let input = try? AVCaptureDeviceInput(device: device), capture.canAddInput(input) else { + // Without the microphone the video records silent; without the camera, nothing. + if index == 0 { throw CocoaError(.fileWriteUnknown) } + continue + } + capture.addInput(input) + } + guard capture.canAddOutput(output) else { throw CocoaError(.fileWriteUnknown) } + capture.addOutput(output) + capture.commitConfiguration() + capture.startRunning() + output.startRecording(to: movieFile, recordingDelegate: self) + self.capture = capture + } + + private var isStubbed: Bool { + #if targetEnvironment(simulator) + stub != nil + #else + false + #endif + } + + private var soundFile: URL { folder.appendingPathComponent("sound.wav") } + private var movieFile: URL { folder.appendingPathComponent("camera.mov") } + + /// How far the recording has got, its pauses left out, and whether it is paused. + var elapsed: (ms: Int64, paused: Bool)? { canvas.recording } + + /// Whether Pause works: always but for a camera before iOS 18. + var pausable: Bool { + if capture == nil { return true } + if #available(iOS 18, *) { return true } + return false + } + + func pause(_ paused: Bool) { + guard !saving, pausable else { return } + if paused { + sound?.pause() + } else { + sound?.record() + } + #if targetEnvironment(simulator) + stub?.paused = paused + #endif + if #available(iOS 18, *), capture != nil { + if paused { output.pauseRecording() } else { output.resumeRecording() } + } + canvas.pauseRecording(paused) + NotificationCenter.default.post(name: Self.changed, object: self) + } + + /// A call or another app's audio takes the microphone: the recording pauses, and goes + /// on after where the system says it should. + @objc private func interrupted(_ notification: Notification) { + guard let raw = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt, + let type = AVAudioSession.InterruptionType(rawValue: raw) + else { return } + DispatchQueue.main.async { [self] in + switch type { + case .began where capture == nil: pause(true) + case .ended: + let options = (notification.userInfo?[AVAudioSessionInterruptionOptionKey] as? UInt) + .map(AVAudioSession.InterruptionOptions.init) ?? [] + if options.contains(.shouldResume), capture == nil { + try? AVAudioSession.sharedInstance().setActive(true) + pause(false) + } + default: break + } + } + } + + /// Stop: the recording's file is made and goes on the page where it started. + func stop() { + guard !saving else { return } + saving = true + canvas.pauseRecording(true) + NotificationCenter.default.post(name: Self.changed, object: self) + if capture != nil { + // The movie is saved once `fileOutput(_:didFinishRecordingTo:…)` hears so. + output.stopRecording() + return + } + sound?.stop() + #if targetEnvironment(simulator) + if let stub { + stub.finish { [self] in convert(movie: stub.movie) } + return + } + #endif + save(try? Data(contentsOf: soundFile)) + } + + /// The camera stopped: at Stop, or as the app left the screen or a call took the camera. + func fileOutput( + _ output: AVCaptureFileOutput, didFinishRecordingTo url: URL, from connections: [AVCaptureConnection], + error: Error? + ) { + let finished = (error as NSError?)?.userInfo[AVErrorRecordingSuccessfullyFinishedKey] as? Bool ?? (error == nil) + DispatchQueue.main.async { [self] in + saving = true + canvas.pauseRecording(true) + NotificationCenter.default.post(name: Self.changed, object: self) + capture?.stopRunning() + if finished { convert(movie: url) } else { save(nil) } + } + } + + /// Makes the AVI file of `movie`, with the sound recorded beside it where there is. + private func convert(movie: URL) { + let sound = self.sound == nil ? movie : soundFile + // Finishing is not cut short by the app leaving the screen. + let task = UIApplication.shared.beginBackgroundTask() + Task.detached { + let avi = await avi(pictures: movie, sound: sound) + await MainActor.run { + self.save(avi) + UIApplication.shared.endBackgroundTask(task) + } + } + } + + /// Puts `file` on the page, or with none forgets the recording. + private func save(_ file: Data?) { + canvas.finishRecording(file, video: video) + end() + Self.current = nil + NotificationCenter.default.post(name: Self.changed, object: self) + } + + private func end() { + NotificationCenter.default.removeObserver(self) + capture?.stopRunning() + try? FileManager.default.removeItem(at: folder) + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + } +} + +/// The AVI file the desktop makes of a movie: the pictures of `pictures`' first video track +/// and the sound of `sound`'s first audio track, mono PCM at `sb_recording_rate`. +private func avi(pictures: URL, sound: URL) async -> Data? { + let movie = sb_movie_new() + let asset = AVURLAsset(url: pictures) + await read(asset, .video, [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]) { sample in + let time = CMSampleBufferGetPresentationTimeStamp(sample) + guard time.isValid, let buffer = CMSampleBufferGetImageBuffer(sample) else { return } + CVPixelBufferLockBaseAddress(buffer, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) } + guard let base = CVPixelBufferGetBaseAddress(buffer) else { return } + sb_movie_picture( + movie, UInt64(max(0, time.seconds) * 1_000_000), base.assumingMemoryBound(to: UInt8.self), + UInt32(CVPixelBufferGetWidth(buffer)), UInt32(CVPixelBufferGetHeight(buffer)), + CVPixelBufferGetBytesPerRow(buffer)) + } + let pcm: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: sb_recording_rate(), AVNumberOfChannelsKey: 1, + AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false, + AVLinearPCMIsNonInterleaved: false, + ] + // A movie without sound has no sound track to read. + await read(AVURLAsset(url: sound), .audio, pcm) { sample in + guard let buffer = CMSampleBufferGetDataBuffer(sample) else { return } + let length = CMBlockBufferGetDataLength(buffer) + var bytes = [UInt8](repeating: 0, count: length) + if CMBlockBufferCopyDataBytes(buffer, atOffset: 0, dataLength: length, destination: &bytes) == noErr { + sb_movie_sound(movie, bytes, length) + } + } + let duration = (try? await asset.load(.duration))?.seconds ?? 0 + var length = 0 + guard let bytes = sb_movie_finish(movie, UInt64(max(0, duration) * 1_000_000), &length) else { return nil } + defer { sb_bytes_free(bytes, length) } + return Data(bytes: bytes, count: length) +} + +/// Reads the first track of `kind` in `asset`, decoded as `settings` ask, a sample at a time. +private func read( + _ asset: AVURLAsset, _ kind: AVMediaType, _ settings: [String: Any], _ sample: (CMSampleBuffer) -> Void +) async { + guard let track = try? await asset.loadTracks(withMediaType: kind).first, + let reader = try? AVAssetReader(asset: asset) + else { return } + let output = AVAssetReaderTrackOutput(track: track, outputSettings: settings) + guard reader.canAdd(output) else { return } + reader.add(output) + guard reader.startReading() else { return } + while let next = output.copyNextSampleBuffer() { + sample(next) + } +} + +#if targetEnvironment(simulator) +/// The simulator's stand-in for a camera, which it has none of: moving colour bars and the +/// time written as a movie, 15 pictures a second, while the microphone records beside it. +private final class StubCamera { + let movie: URL + private let writer: AVAssetWriter + private let input: AVAssetWriterInput + private let adaptor: AVAssetWriterInputPixelBufferAdaptor + private var timer: Timer? + private var frame: Int64 = 0 + var paused = false + + init(movie: URL) throws { + self.movie = movie + writer = try AVAssetWriter(outputURL: movie, fileType: .mov) + input = AVAssetWriterInput( + mediaType: .video, + outputSettings: [AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: 640, AVVideoHeightKey: 480]) + input.expectsMediaDataInRealTime = true + adaptor = AVAssetWriterInputPixelBufferAdaptor( + assetWriterInput: input, + sourcePixelBufferAttributes: [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey as String: 640, kCVPixelBufferHeightKey as String: 480, + ]) + writer.add(input) + guard writer.startWriting() else { throw writer.error ?? CocoaError(.fileWriteUnknown) } + writer.startSession(atSourceTime: .zero) + timer = Timer.scheduledTimer(withTimeInterval: 1.0 / 15, repeats: true) { [weak self] _ in self?.shoot() } + } + + private func shoot() { + guard !paused, input.isReadyForMoreMediaData, let pool = adaptor.pixelBufferPool else { return } + var made: CVPixelBuffer? + CVPixelBufferPoolCreatePixelBuffer(nil, pool, &made) + guard let buffer = made else { return } + CVPixelBufferLockBaseAddress(buffer, []) + let context = CGContext( + data: CVPixelBufferGetBaseAddress(buffer), width: 640, height: 480, bitsPerComponent: 8, + bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue) + if let context { + let colors: [UIColor] = [.systemYellow, .systemTeal, .systemGreen, .systemPink, .systemRed, .systemBlue] + for bar in 0..<8 { + context.setFillColor(colors[(bar + Int(frame / 5)) % colors.count].cgColor) + context.fill(CGRect(x: bar * 80, y: 0, width: 80, height: 480)) + } + UIGraphicsPushContext(context) + context.translateBy(x: 0, y: 480) + context.scaleBy(x: 1, y: -1) + let seconds = Double(frame) / 15 + NSString(string: String(format: "%.1f s", seconds)).draw( + at: CGPoint(x: 24, y: 24), + withAttributes: [.font: UIFont.monospacedDigitSystemFont(ofSize: 72, weight: .bold), .foregroundColor: UIColor.black]) + UIGraphicsPopContext() + } + CVPixelBufferUnlockBaseAddress(buffer, []) + adaptor.append(buffer, withPresentationTime: CMTime(value: frame, timescale: 15)) + frame += 1 + } + + func finish(_ done: @escaping () -> Void) { + timer?.invalidate() + input.markAsFinished() + writer.endSession(atSourceTime: CMTime(value: frame, timescale: 15)) + writer.finishWriting { DispatchQueue.main.async(execute: done) } + } +} +#endif + +/// Plays a recording: its sound, and a video's pictures in time with it. +final class Playback { + let request: PlayRequest + let player: AVPlayer + private let pictures: Data? + private var shown = -1 + + init(_ request: PlayRequest) { + self.request = request + player = AVPlayer(url: URL(fileURLWithPath: request.sound)) + pictures = request.video.flatMap { try? Data(contentsOf: URL(fileURLWithPath: $0.path), options: .alwaysMapped) } + try? AVAudioSession.sharedInstance().setCategory(.playback, mode: .spokenAudio) + try? AVAudioSession.sharedInstance().setActive(true) + player.seek(to: CMTime(value: CMTimeValue(request.atMs), timescale: 1000), toleranceBefore: .zero, toleranceAfter: .zero) + player.play() + } + + deinit { + player.pause() + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + } + + /// Whether the player could not open the sound, as with OneNote's own Windows Media. + var failed: Bool { player.currentItem?.status == .failed } + + var ms: Int64 { Int64(max(0, player.currentTime().seconds) * 1000) } + + var durationMs: Int64? { + guard let duration = player.currentItem?.duration, duration.isNumeric else { return nil } + return Int64(duration.seconds * 1000) + } + + var playing: Bool { player.rate != 0 } + + func toggle() { + if playing { + player.pause() + } else { + if let durationMs, ms >= durationMs - 50 { player.seek(to: .zero) } + player.play() + } + } + + /// The video's picture now, when it changed since last asked. + func picture() -> UIImage? { + guard let video = request.video, let pictures, !video.frames.isEmpty else { return nil } + let index = min(Int(ms * 1000 / Int64(max(1, video.frameUs))), video.frames.count - 1) + guard index != shown else { return nil } + shown = index + let range = video.frames[index] + guard range.count == 2, range[0] < range[1], range[1] <= pictures.count else { return nil } + return UIImage(data: pictures.subdata(in: range[0].. Void)? + var onStop: (() -> Void)? + + init() { + super.init(frame: .zero) + let blur = UIVisualEffectView(effect: UIBlurEffect(style: .systemThickMaterial)) + blur.translatesAutoresizingMaskIntoConstraints = false + blur.layer.cornerRadius = 14 + blur.clipsToBounds = true + addSubview(blur) + layer.shadowOpacity = 0.15 + layer.shadowRadius = 8 + layer.shadowOffset = CGSize(width: 0, height: 2) + dot.tintColor = .systemRed + dot.setContentHuggingPriority(.required, for: .horizontal) + label.font = .monospacedDigitSystemFont(ofSize: UIFont.preferredFont(forTextStyle: .subheadline).pointSize, weight: .medium) + label.adjustsFontForContentSizeCategory = true + label.lineBreakMode = .byTruncatingMiddle + label.accessibilityTraits.insert(.updatesFrequently) + stop.configuration?.image = UIImage(systemName: "stop.fill") + stop.accessibilityLabel = "Stop" + stop.addAction(UIAction { [weak self] _ in self?.onStop?() }, for: .primaryActionTriggered) + pause.addAction(UIAction { [weak self] _ in self?.onPause?() }, for: .primaryActionTriggered) + for button in [pause, stop] { button.setContentHuggingPriority(.required, for: .horizontal) } + picture.contentMode = .scaleAspectFit + picture.backgroundColor = .black + picture.layer.cornerRadius = 6 + picture.clipsToBounds = true + picture.isHidden = true + let controls = UIStackView(arrangedSubviews: [dot, label, pause, stop]) + controls.spacing = 8 + controls.alignment = .center + let stack = UIStackView(arrangedSubviews: [picture, controls]) + stack.axis = .vertical + stack.spacing = 8 + stack.translatesAutoresizingMaskIntoConstraints = false + blur.contentView.addSubview(stack) + NSLayoutConstraint.activate([ + blur.leadingAnchor.constraint(equalTo: leadingAnchor), + blur.trailingAnchor.constraint(equalTo: trailingAnchor), + blur.topAnchor.constraint(equalTo: topAnchor), + blur.bottomAnchor.constraint(equalTo: bottomAnchor), + stack.leadingAnchor.constraint(equalTo: blur.contentView.leadingAnchor, constant: 14), + stack.trailingAnchor.constraint(equalTo: blur.contentView.trailingAnchor, constant: -6), + stack.topAnchor.constraint(equalTo: blur.contentView.topAnchor, constant: 8), + stack.bottomAnchor.constraint(equalTo: blur.contentView.bottomAnchor, constant: -8), + picture.widthAnchor.constraint(lessThanOrEqualToConstant: 320), + ]) + // Below required, as the stack gives the picture no height while it is hidden. + let aspect = picture.heightAnchor.constraint(equalTo: picture.widthAnchor, multiplier: 3.0 / 4) + aspect.priority = .defaultHigh + aspect.isActive = true + isHidden = true + } + + required init?(coder: NSCoder) { fatalError() } + + /// Shows `text`, the dot while recording, and Pause or Resume (or Play) where it works. + func show(_ text: String, recording: Bool, paused: Bool?, playing: Bool = false) { + isHidden = false + label.text = text + dot.isHidden = !recording + dot.alpha = paused == true ? 0.4 : 1 + pause.isHidden = paused == nil + stop.isHidden = false + let resume = paused == true + pause.configuration?.image = UIImage(systemName: resume ? (playing ? "play.fill" : "record.circle") : "pause.fill") + pause.accessibilityLabel = resume ? (playing ? "Play" : "Resume") : "Pause" + } + + /// Saving: only the clock's place says so. + func showSaving(_ text: String) { + show(text, recording: false, paused: nil) + stop.isHidden = true + } + + /// `ms` as a clock: minutes and seconds, hours when there are some. + static func clock(_ ms: Int64) -> String { + let seconds = ms / 1000 + return seconds >= 3600 + ? String(format: "%d:%02d:%02d", seconds / 3600, seconds / 60 % 60, seconds % 60) + : String(format: "%d:%02d", seconds / 60, seconds % 60) + } +} diff --git a/apps/ios/Snowbound/Script.swift b/apps/ios/Snowbound/Script.swift index 42926d5eeba70461f40701f6d2a4d77e64ea9671..7d16411dce6c27f6bd147f9c3a1d64099d95fcd1 100644 --- a/apps/ios/Snowbound/Script.swift +++ b/apps/ios/Snowbound/Script.swift @@ -14,17 +14,22 @@ extension CanvasView { /// `tool:pen:N`, `tool:shape:N`, `tool:eraser` or `tool:lasso` gives it a tool, `picker` shows or hides the /// drawing tools, `pencildoubletap` is the Pencil's double tap, and `deleteink` deletes /// what the lasso picked; `rotate:landscape` or `rotate:portrait` turns the device; `done` - /// ends editing, `reading` turns the reading view on or off, `tree` saves the view hierarchy to + /// ends editing, `reading` turns the reading view on or off, `record:audio` or + /// `record:video` records, `pauserecording` pauses or resumes and `stoprecording` stops it, + /// `wait:N` waits N seconds more before the next step, `tree` saves the view hierarchy to /// Documents/tree.txt, and `shot:a` the window to Documents/a.png, as a device has no /// screenshot command. func runScript() { // Only the first page shown runs it, not every page opened afterwards. guard !scriptedPage, let script = ProcessInfo.processInfo.environment["SNOWBOUND_SCRIPT"] else { return } scriptedPage = true - for (index, step) in script.split(separator: "|").enumerated() { - DispatchQueue.main.asyncAfter(deadline: .now() + 1 + Double(index) * 0.4) { [weak self] in + var delay = 1.0 + for step in script.split(separator: "|") { + if step.hasPrefix("wait:") { delay += Double(step.dropFirst(5)) ?? 0 } + DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [weak self] in self?.perform(String(step)) } + delay += 0.4 } } @@ -90,6 +95,11 @@ extension CanvasView { let caret = caretRect(for: range.start) menu.presentEditMenu(with: UIEditMenuConfiguration(identifier: nil, sourcePoint: CGPoint(x: caret.midX, y: caret.minY))) case "done": _ = resignFirstResponder() + case "wait": break + case "record", "stoprecording": + let page = sequence(first: self as UIResponder, next: \.next).lazy.compactMap { $0 as? PageViewController }.first + if command == "record" { page?.record(video: argument == "video") } else { page?.stopRecording() } + case "pauserecording": Recorder.current.map { $0.pause(!($0.elapsed?.paused ?? false)) } case "selectall": selectAll(nil) case "reading": showReading(!readingShown) case "format": apply(UInt8(argument) ?? 0) diff --git a/arc/canvas.md b/arc/canvas.md index 9e35300d4d9d3a458ae5af171f2059302be4a350..0e9e5bd858acc26be02b28077c72c1cca61571e8 100644 --- a/arc/canvas.md +++ b/arc/canvas.md @@ -167,7 +167,8 @@ scrolling it into view when it changes. The host records audio as 16 kHz WAV and stores it as IMA ADPCM, and video as AVI of Motion JPEG at 320 by 240 and 15 pictures a second with PCM sound, both -of which OneNote plays (`corpus/recording/video`). GStreamer writes that AVI +of which OneNote plays (`corpus/recording/video`); `canvas::recording` makes both, +so every host stores the same bytes. GStreamer writes that AVI directly; on macOS a capture session records a movie that `AVAssetReader` reads back into it off the main thread after Stop, the transport saying so and the page editable meanwhile. Mac OS X 10.6, without AVFoundation, opens recordings in the diff --git a/arc/platforms.md b/arc/platforms.md index c2e64c1e7b34ba5721931390a7643397448cd544..f45d4b0f571d352b02117fd883b0b9b5cfd06d9d 100644 --- a/arc/platforms.md +++ b/arc/platforms.md @@ -208,6 +208,13 @@ the same ops. Notebooks from Files are read and written under `NSFileCoordinator`, so file providers see every change. +Record Audio and Record Video record through AVFoundation and hand `crates/mobile` the +WAV file, or the camera's pictures and sound, which it stores as the desktop does. Audio +keeps recording in the background and with the screen locked, under the `audio` +background mode, as a lecture outlasts the screen's timeout; a call pauses it until the +system says to go on. The camera stops in the background, which ends a video recording +and saves it. A tap on a recording plays it with `AVPlayer` in a bar over the page's foot. + ## What stays shared, on purpose - **Behaviour**: the editor, hit-testing, the placement grid, undo, conflicts diff --git a/crates/canvas/src/editor/recording.rs b/crates/canvas/src/editor/recording.rs index 1c967e00d9df445a25a158e053aa38b520192bab..df0773dffb9b80f129e0d05e15291ed696f3c693 100644 --- a/crates/canvas/src/editor/recording.rs +++ b/crates/canvas/src/editor/recording.rs @@ -161,6 +161,11 @@ impl CanvasEditor { Ok(id) } + /// Forgets the recording under way, whose file never came; its line stays. + pub fn cancel_recording(&mut self) { + self.recording = None; + } + /// Stops recording: `file`, the recording made, goes where it started, linked to its /// start, or at the caret when that place is gone. pub fn finish_recording( diff --git a/crates/canvas/src/gpu/page.rs b/crates/canvas/src/gpu/page.rs index 7eb353153b15af151f8e4d1e11657995d6c5e813..316210aedf4f61634cf86b4071aa2a5da5d4979f 100644 --- a/crates/canvas/src/gpu/page.rs +++ b/crates/canvas/src/gpu/page.rs @@ -203,6 +203,9 @@ fn between(from: [f32; 2], middle: [f32; 2], to: [f32; 2]) -> bool { (0.0..=1.0).contains(&along) && (mx * dy - my * dx).abs() / length.sqrt() <= 0.1 } +/// The size OneNote 2010 shows a file's icon at, in points. +pub const ICON_SIZE: [f32; 2] = [24.0, 24.0]; + const FILE_ICON: &str = include_str!("../../assets/file.svg"); /// The blank page for a file without an icon of its own, as the 32 pixel PNG OneNote 2010 diff --git a/crates/canvas/src/interaction/mod.rs b/crates/canvas/src/interaction/mod.rs index 21d76cc70adc1f2440695a09bc6406dea58da358..c112218f04bbd083718baa643370f02dc4bd81b6 100644 --- a/crates/canvas/src/interaction/mod.rs +++ b/crates/canvas/src/interaction/mod.rs @@ -1261,6 +1261,14 @@ impl PageView { Some(Hit::File(id)) => { self.set_object_focus(Some(ObjectFocus::File(id))); self.drag = None; + // A finger plays a recording with a tap, having no play button to hover. + if self.touch + && let Some(file) = self.editor.attachment(id) + && file.recording.is_some() + { + let file = file.clone(); + return Ok(Response::request(Request::Play { file, at_ms: 0 })); + } if unit != SelectionUnit::Grapheme && let Some(file) = self.editor.attachment(id) { diff --git a/crates/canvas/src/lib.rs b/crates/canvas/src/lib.rs index 840d8486a761dcb1e2c2e0358360e860ec7d939f..5e3d6613cc9a93c9bb22b7017805d27cf7ec53b9 100644 --- a/crates/canvas/src/lib.rs +++ b/crates/canvas/src/lib.rs @@ -17,6 +17,8 @@ pub mod gpu; pub mod interaction; #[cfg(feature = "pdf")] pub mod print; +#[cfg(feature = "gpu")] +pub mod recording; /// Where an object stored with `layout` lies; an unset coordinate is zero. pub(crate) fn origin(layout: &onestore::document::Layout) -> [f32; 2] { diff --git a/crates/canvas/src/recording.rs b/crates/canvas/src/recording.rs new file mode 100644 index 0000000000000000000000000000000000000000..9c271378e5cafc1ea5b3bc3041ced3f9957b4de9 --- /dev/null +++ b/crates/canvas/src/recording.rs @@ -0,0 +1,424 @@ +//! Recordings as Snowbound stores them, which OneNote 2010 plays (`corpus/recording`): +//! audio as 16 kHz IMA ADPCM WAV, video as the Motion JPEG AVI files of `video`. + +pub mod video; + +use onestore::page::{Attachment, Recording}; + +/// The rate audio is recorded at: wideband speech, as OneNote's own 8 kHz WMA is narrowband. +pub const RATE: u32 = 16_000; + +/// The line saying when a recording started, as OneNote writes it from the long `date` and +/// short `time`. +pub fn label(video: bool, date: &str, time: &str) -> String { + let kind = if video { "Video" } else { "Audio" }; + format!("{kind} recording started: {time} {date}") +} + +/// The file recording `id` on page `title` is stored as, from what was recorded: a mono +/// 16-bit PCM WAV file, which is compressed, or a `video` AVI file. It has no icon yet. +pub fn file( + id: [u8; 16], + title: &str, + video: bool, + recorded: Vec, +) -> Result { + let (bytes, duration_ms) = if video { + let duration = video::Movie::parse(&recorded).map(|movie| movie.duration_ms()); + (recorded, duration) + } else { + match compress(&recorded) { + Some((bytes, duration)) => (bytes, Some(duration)), + None => { + let duration = Wave::parse(&recorded).map(|wave| wave.duration_ms()); + (recorded, duration) + } + } + }; + let name: String = title + .chars() + .map(|c| { + if matches!(c, '/' | '\\' | '\0') { + '_' + } else { + c + } + }) + .collect(); + let name = if name.trim().is_empty() { + if video { + "Video recording" + } else { + "Audio recording" + } + .to_owned() + } else { + name + }; + let extension = if video { "avi" } else { "wav" }; + Ok(Attachment { + id: onestore::page::text::new_id()?, + filename: format!("{name}.{extension}"), + source_path: None, + size: Some(crate::gpu::page::ICON_SIZE), + layout: Default::default(), + bytes: Some(bytes.into()), + preview: None, + recording: Some(Recording { + id, + kind: if video { 2 } else { 1 }, + duration_ms, + }), + tags: Vec::new(), + }) +} + +/// A recording's sound as a 16-bit PCM WAV file every platform's player decodes: IMA ADPCM +/// decoded, a video's sound taken out of it. None for other files, which play as they are. +pub fn sound(bytes: &[u8]) -> Option> { + let sound = match video::Movie::parse(bytes) { + Some(movie) => movie + .wave(bytes) + .unwrap_or_else(|| silence(movie.duration_ms())), + None => return decompress(bytes), + }; + Some(decompress(&sound).unwrap_or(sound)) +} + +/// Silence as long as `ms`, the clock of a video without sound. +fn silence(ms: u32) -> Vec { + wave(8_000, &vec![0; ms as usize * 8]) +} + +/// A mono 16-bit PCM WAV file of `samples` at `rate`, as the platform recorders write. +pub fn wave(rate: u32, samples: &[i16]) -> Vec { + let data: Vec = samples.iter().flat_map(|s| s.to_le_bytes()).collect(); + [ + &b"RIFF"[..], + &(36 + data.len() as u32).to_le_bytes(), + b"WAVEfmt ", + &16_u32.to_le_bytes(), + &1_u16.to_le_bytes(), + &1_u16.to_le_bytes(), + &rate.to_le_bytes(), + &(rate * 2).to_le_bytes(), + &2_u16.to_le_bytes(), + &16_u16.to_le_bytes(), + b"data", + &(data.len() as u32).to_le_bytes(), + &data, + ] + .concat() +} + +/// A recording for a file attached under `filename`, as OneNote 2010 makes one of every +/// audio and video file it attaches, with the length of a WAV file's `bytes`. +pub fn attached(filename: &str, bytes: &[u8]) -> Option { + Some(Recording { + id: onestore::page::text::new_guid().ok()?, + kind: Recording::kind_of(filename)?, + duration_ms: Wave::parse(bytes).map(|wave| wave.duration_ms()), + }) +} + +/// The chunks of a RIFF WAVE file this module reads. +struct Wave<'a> { + format: u16, + channels: u16, + rate: u32, + bytes_per_second: u32, + block: u16, + bits: u16, + /// The sample count a compressed file's `fact` chunk states. + samples: Option, + data: &'a [u8], +} + +impl<'a> Wave<'a> { + fn parse(bytes: &'a [u8]) -> Option { + if bytes.get(..4)? != b"RIFF" || bytes.get(8..12)? != b"WAVE" { + return None; + } + let u16_at = |data: &[u8], at: usize| { + Some(u16::from_le_bytes(data.get(at..at + 2)?.try_into().ok()?)) + }; + let u32_at = |data: &[u8], at: usize| { + Some(u32::from_le_bytes(data.get(at..at + 4)?.try_into().ok()?)) + }; + let (mut fmt, mut samples, mut data) = (None, None, None); + let mut at = 12; + while at + 8 <= bytes.len() { + let size = u32_at(bytes, at + 4)? as usize; + let body = &bytes[at + 8..(at + 8).saturating_add(size).min(bytes.len())]; + match &bytes[at..at + 4] { + b"fmt " => fmt = Some(body), + b"fact" => samples = u32_at(body, 0), + b"data" => data = Some(body), + _ => {} + } + at = at + 8 + size + size % 2; + } + let fmt = fmt?; + Some(Self { + format: u16_at(fmt, 0)?, + channels: u16_at(fmt, 2)?, + rate: u32_at(fmt, 4)?, + bytes_per_second: u32_at(fmt, 8)?, + block: u16_at(fmt, 12)?, + bits: u16_at(fmt, 14)?, + samples, + data: data?, + }) + } + + fn duration_ms(&self) -> u32 { + let ms = match self.samples.filter(|_| self.format != 1) { + Some(samples) => u64::from(samples) * 1000 / u64::from(self.rate.max(1)), + None => self.data.len() as u64 * 1000 / u64::from(self.bytes_per_second.max(1)), + }; + u32::try_from(ms).unwrap_or(u32::MAX) + } +} + +/// IMA ADPCM's step sizes and how each code moves along them. +const STEPS: [i32; 89] = [ + 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66, + 73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449, + 494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, 2272, + 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358, 5894, 6484, 7132, 7845, 8630, 9493, + 10442, 11487, 12635, 13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767, +]; +const INDEX_STEPS: [i32; 8] = [-1, -1, -1, -1, 2, 4, 6, 8]; + +/// Bytes per block: Windows' IMA ADPCM codec's choice for 16 kHz. +const BLOCK: usize = 512; +/// Samples per block: the header's, then two per remaining byte. +const BLOCK_SAMPLES: usize = (BLOCK - 4) * 2 + 1; + +/// A mono 16-bit PCM WAV file as 4-bit IMA ADPCM (WAVE_FORMAT_DVI_ADPCM), which Windows +/// and macOS decode, a quarter of the size; and its length. None for other files. +pub fn compress(wav: &[u8]) -> Option<(Vec, u32)> { + let wave = Wave::parse(wav)?; + if wave.format != 1 || wave.channels != 1 || wave.bits != 16 { + return None; + } + let samples: Vec = wave + .data + .chunks_exact(2) + .map(|pair| i16::from_le_bytes([pair[0], pair[1]])) + .collect(); + let mut data = Vec::with_capacity(samples.len() / 2 + BLOCK); + let mut index = 0_i32; + for block in samples.chunks(BLOCK_SAMPLES) { + let mut predicted = i32::from(block[0]); + data.extend_from_slice(&block[0].to_le_bytes()); + data.extend_from_slice(&[index as u8, 0]); + // A short last block is padded with its last sample; `fact` gives the true length. + let last = *block.last().unwrap(); + let rest = block[1..] + .iter() + .copied() + .chain(std::iter::repeat(last)) + .take(BLOCK_SAMPLES - 1) + .collect::>(); + for pair in rest.chunks_exact(2) { + let mut byte = 0; + for (shift, sample) in [0, 4].into_iter().zip(pair) { + let code = encode(i32::from(*sample), &mut predicted, &mut index); + byte |= code << shift; + } + data.push(byte); + } + } + let count = u32::try_from(samples.len()).ok()?; + let bytes_per_second = wave.rate * BLOCK as u32 / BLOCK_SAMPLES as u32; + let mut out = Vec::with_capacity(data.len() + 60); + let fmt: Vec = [ + &0x11_u16.to_le_bytes()[..], + &1_u16.to_le_bytes(), + &wave.rate.to_le_bytes(), + &bytes_per_second.to_le_bytes(), + &(BLOCK as u16).to_le_bytes(), + &4_u16.to_le_bytes(), + &2_u16.to_le_bytes(), + &(BLOCK_SAMPLES as u16).to_le_bytes(), + ] + .concat(); + let riff = 4 + (8 + fmt.len()) + (8 + 4) + (8 + data.len()); + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&u32::try_from(riff).ok()?.to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&(fmt.len() as u32).to_le_bytes()); + out.extend_from_slice(&fmt); + out.extend_from_slice(b"fact"); + out.extend_from_slice(&4_u32.to_le_bytes()); + out.extend_from_slice(&count.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&u32::try_from(data.len()).ok()?.to_le_bytes()); + out.extend_from_slice(&data); + let duration = u32::try_from(u64::from(count) * 1000 / u64::from(wave.rate.max(1))).ok()?; + Some((out, duration)) +} + +/// The 4-bit code nearest `sample` from `predicted`, which it and `index` then follow as a +/// decoder does. +fn encode(sample: i32, predicted: &mut i32, index: &mut i32) -> u8 { + let step = STEPS[*index as usize]; + let mut difference = sample - *predicted; + let mut code = 0; + if difference < 0 { + code = 8; + difference = -difference; + } + let mut delta = step >> 3; + for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] { + if difference >= part { + code |= bit; + difference -= part; + delta += part; + } + } + *predicted = if code & 8 != 0 { + *predicted - delta + } else { + *predicted + delta + } + .clamp(-32768, 32767); + *index = (*index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88); + code +} + +/// A mono IMA ADPCM WAV file as 16-bit PCM, which every platform's player decodes; none +/// for other files. +pub fn decompress(wav: &[u8]) -> Option> { + let wave = Wave::parse(wav).filter(|wave| wave.format == 0x11 && wave.channels == 1)?; + Some(self::wave(wave.rate, &decode(&wave))) +} + +/// Each block's header sample and index, then its codes. +fn decode(wave: &Wave<'_>) -> Vec { + let mut out = Vec::new(); + for block in wave.data.chunks(usize::from(wave.block.max(4))) { + let Some(header) = block.get(..4) else { + break; + }; + let mut predicted = i32::from(i16::from_le_bytes([header[0], header[1]])); + let mut index = i32::from(header[2]).clamp(0, 88); + out.push(predicted as i16); + for byte in &block[4..] { + for code in [byte & 15, byte >> 4] { + let step = STEPS[index as usize]; + let mut delta = step >> 3; + for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] { + if code & bit != 0 { + delta += part; + } + } + predicted = if code & 8 != 0 { + predicted - delta + } else { + predicted + delta + } + .clamp(-32768, 32767); + index = (index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88); + out.push(predicted as i16); + } + } + } + if let Some(samples) = wave.samples { + out.truncate(samples as usize); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Two seconds of a spoken-pitch chord. + fn tone() -> Vec { + (0..2 * RATE) + .map(|n| { + let t = n as f32 / RATE as f32; + let wave = (t * 220.0 * std::f32::consts::TAU).sin() * 0.5 + + (t * 330.0 * std::f32::consts::TAU).sin() * 0.3; + (wave * 12_000.0) as i16 + }) + .collect() + } + + #[test] + fn a_recording_compresses_to_ima_adpcm_that_decodes_close_to_what_was_heard() { + let samples = tone(); + let (bytes, duration) = compress(&wave(RATE, &samples)).unwrap(); + assert_eq!(duration, 2000); + let wave = Wave::parse(&bytes).unwrap(); + assert_eq!( + (wave.format, wave.channels, wave.rate, wave.bits), + (0x11, 1, RATE, 4) + ); + assert_eq!(wave.data.len() % BLOCK, 0); + assert_eq!(wave.duration_ms(), 2000); + assert!(bytes.len() * 3 < samples.len() * 2); + let decoded = decode(&wave); + assert_eq!( + Wave::parse(&decompress(&bytes).unwrap()) + .unwrap() + .data + .len(), + decoded.len() * 2 + ); + assert_eq!(decoded.len(), samples.len()); + let noise: f64 = samples + .iter() + .zip(&decoded) + .map(|(a, b)| (f64::from(*a) - f64::from(*b)).powi(2)) + .sum(); + let signal: f64 = samples.iter().map(|a| f64::from(*a).powi(2)).sum(); + let snr = 10.0 * (signal / noise).log10(); + assert!(snr > 20.0, "{snr} dB"); + } + + #[test] + fn only_mono_16_bit_pcm_compresses_and_attached_wav_files_know_their_length() { + let stereo = { + let mut bytes = wave(RATE, &[0; 64]); + bytes[22] = 2; + bytes + }; + assert!(compress(&stereo).is_none()); + assert!(compress(b"not a wave").is_none()); + let silence = include_bytes!( + "../../../corpus/m6/native-features-01/notebook/fixture-data/silence.wav" + ); + // OneNote 2010 stored 1000 ms for this file when attached (`corpus/recording`). + let recording = attached("silence.WAV", silence).unwrap(); + assert_eq!((recording.kind, recording.duration_ms), (1, Some(1000))); + assert_eq!(attached("clip.wmv", b"").unwrap().kind, 2); + assert!(attached("notes.txt", b"").is_none()); + } + + #[test] + fn a_recording_is_stored_named_after_its_page_and_plays_as_pcm() { + let id = [7; 16]; + let audio = file(id, "Lecture 3/4", false, wave(RATE, &tone())).unwrap(); + assert_eq!(audio.filename, "Lecture 3_4.wav"); + let recording = audio.recording.unwrap(); + assert_eq!( + (recording.id, recording.kind, recording.duration_ms), + (id, 1, Some(2000)) + ); + let stored = audio.bytes.unwrap(); + assert_eq!(Wave::parse(&stored).unwrap().format, 0x11); + let played = sound(&stored).unwrap(); + assert_eq!(Wave::parse(&played).unwrap().format, 1); + assert_eq!( + file(id, " ", true, Vec::new()).unwrap().filename, + "Video recording.avi" + ); + assert_eq!( + label(false, "Tuesday, September 29, 2026", "5:18 PM"), + "Audio recording started: 5:18 PM Tuesday, September 29, 2026" + ); + } +} diff --git a/crates/snowbound/src/video.rs b/crates/canvas/src/recording/video.rs similarity index 91% rename from crates/snowbound/src/video.rs rename to crates/canvas/src/recording/video.rs index c5e7e3e2b8839240bcf9375035caf97be375793a..14621d3d325d380295ffabace61ae09372acc489 100644 --- a/crates/snowbound/src/video.rs +++ b/crates/canvas/src/recording/video.rs @@ -4,13 +4,12 @@ use std::ops::Range; /// Pictures a second in a recording: OneNote 2010's own profiles record at 15. -pub(crate) const FPS: u32 = 15; +pub const FPS: u32 = 15; /// A recording's picture size, OneNote 2010's own. -pub(crate) const SIZE: [u32; 2] = [320, 240]; +pub const SIZE: [u32; 2] = [320, 240]; /// Interleaved signed 16-bit little-endian sound. -#[cfg_attr(not(target_os = "macos"), allow(dead_code))] -pub(crate) struct Pcm { +pub struct Pcm { pub rate: u32, pub channels: u16, pub data: Vec, @@ -25,14 +24,12 @@ fn chunk(out: &mut Vec, id: &[u8; 4], body: &[u8]) { } } -#[cfg_attr(not(target_os = "macos"), allow(dead_code))] fn list(kind: &[u8; 4], body: &[u8]) -> Vec { let mut out = Vec::with_capacity(body.len() + 12); chunk(&mut out, b"LIST", &[&kind[..], body].concat()); out } -#[cfg_attr(not(target_os = "macos"), allow(dead_code))] fn words(values: &[u32]) -> Vec { values .iter() @@ -42,8 +39,7 @@ fn words(values: &[u32]) -> Vec { /// An AVI file of `frames`, JPEG pictures of `size` shown `FPS` a second, where an empty one /// repeats the picture before it, and `sound` alongside. None past AVI's 4 GiB. -#[cfg_attr(not(target_os = "macos"), allow(dead_code))] -pub(crate) fn write(size: [u32; 2], frames: &[Vec], sound: &Pcm) -> Option> { +pub fn write(size: [u32; 2], frames: &[Vec], sound: &Pcm) -> Option> { let block = u32::from(sound.channels) * 2; let bytes_per_second = sound.rate * block; let samples = sound.data.len() as u64 / u64::from(block); @@ -184,7 +180,7 @@ pub(crate) fn write(size: [u32; 2], frames: &[Vec], sound: &Pcm) -> Option>, pub frame_us: u32, @@ -194,7 +190,7 @@ pub(crate) struct Movie { impl Movie { /// A Motion JPEG AVI file's contents; none for other files. - pub(crate) fn parse(bytes: &[u8]) -> Option { + pub fn parse(bytes: &[u8]) -> Option { let mut movie = Self { frames: Vec::new(), frame_us: 0, @@ -264,13 +260,13 @@ impl Movie { Some(()) } - pub(crate) fn duration_ms(&self) -> u32 { + pub fn duration_ms(&self) -> u32 { let ms = (self.frames.len() as u64 * u64::from(self.frame_us)).div_ceil(1000); u32::try_from(ms).unwrap_or(u32::MAX) } /// The picture showing `at_ms` in: the last stored one at or before it. - pub(crate) fn frame(&self, at_ms: u32) -> Option> { + pub fn frame(&self, at_ms: u32) -> Option> { let at = (u64::from(at_ms) * 1000 / u64::from(self.frame_us.max(1))) as usize; self.frames[..=at.min(self.frames.len().checked_sub(1)?)] .iter() @@ -280,7 +276,7 @@ impl Movie { } /// The sound as a WAV file, or none where the file has none. - pub(crate) fn wave(&self, bytes: &[u8]) -> Option> { + pub fn wave(&self, bytes: &[u8]) -> Option> { let (format, data) = self.sound.as_ref()?; let format = &bytes[format.clone()]; let length: usize = data.iter().map(ExactSizeIterator::len).sum(); @@ -334,26 +330,46 @@ impl Iterator for Chunks<'_> { /// Pictures taken at uneven moments, as a camera takes them, made the `FPS` a second of a /// recording: each one shows until the next, fitted to `SIZE`. -#[cfg_attr(not(target_os = "macos"), allow(dead_code))] #[derive(Default)] -pub(crate) struct Pictures { +pub struct Pictures { frames: Vec>, last: Option, /// Whether the last picture is stored yet, which the frames after it repeat. stored: bool, } -#[cfg_attr(not(target_os = "macos"), allow(dead_code))] impl Pictures { /// The picture taken `at_us` microseconds in; the first shows from the start. - pub(crate) fn take(&mut self, at_us: u64, picture: image::RgbImage) { + pub fn take(&mut self, at_us: u64, picture: image::RgbImage) { self.fill(at_us); self.last = Some(fit(picture)); self.stored = false; } + /// A picture as Core Video gives one: `height` rows of `stride` bytes of BGRA pixels. + pub fn take_bgra( + &mut self, + at_us: u64, + [width, height]: [usize; 2], + stride: usize, + bgra: &[u8], + ) { + let rgb = bgra + .chunks_exact(stride.max(1)) + .take(height) + .filter_map(|row| row.get(..width * 4)) + .flat_map(|row| { + row.chunks_exact(4) + .flat_map(|pixel| [pixel[2], pixel[1], pixel[0]]) + }) + .collect(); + if let Some(picture) = image::RgbImage::from_raw(width as u32, height as u32, rgb) { + self.take(at_us, picture); + } + } + /// The frames of a recording `end_us` microseconds long. - pub(crate) fn finish(mut self, end_us: u64) -> Vec> { + pub fn finish(mut self, end_us: u64) -> Vec> { self.fill(end_us); self.frames } @@ -382,7 +398,6 @@ impl Pictures { } /// `picture` scaled to fit `SIZE`, centred on black. -#[cfg_attr(not(target_os = "macos"), allow(dead_code))] fn fit(picture: image::RgbImage) -> image::RgbImage { let [width, height] = SIZE; if picture.dimensions() == (width, height) { diff --git a/crates/mobile/include/snowbound.h b/crates/mobile/include/snowbound.h index a8355fb8c626d69e498f71f5aea64adb2248b5cc..d008f053c78056e82bdd5bb9dfdefb807d138975 100644 --- a/crates/mobile/include/snowbound.h +++ b/crates/mobile/include/snowbound.h @@ -109,6 +109,20 @@ int8_t sb_view_date_request(View *view, int64_t *seconds); bool sb_view_change_date(View *view, int64_t seconds, const char *date, const char *time); bool sb_view_insert_picture(View *view, const uint8_t *bytes, size_t length, float width, float height); +typedef struct Movie Movie; +uint32_t sb_recording_rate(void); +bool sb_view_start_recording(View *view, bool video, const char *date, const char *time); +int64_t sb_view_recording(const View *view, bool *paused); +void sb_view_pause_recording(View *view, bool paused); +bool sb_view_finish_recording(View *view, const uint8_t *bytes, size_t length, bool video); +char *sb_view_play_request(View *view, const char *folder); +bool sb_view_played(View *view, int64_t at_ms); +Movie *sb_movie_new(void); +void sb_movie_picture(Movie *movie, uint64_t at_us, const uint8_t *bgra, uint32_t width, uint32_t height, + size_t stride); +void sb_movie_sound(Movie *movie, const uint8_t *pcm, size_t length); +uint8_t *sb_movie_finish(Movie *movie, uint64_t duration_us, size_t *length); + uint32_t sb_text_length(const View *view); char *sb_text(const View *view, uint32_t start, uint32_t end); void sb_selection(const View *view, uint32_t range[2]); diff --git a/crates/mobile/src/lib.rs b/crates/mobile/src/lib.rs index f21d0ed19663ab86f042a66c3f9e0719e4fc4725..79e34f05565ebfd60e89dcd9205f4de403875b0f 100644 --- a/crates/mobile/src/lib.rs +++ b/crates/mobile/src/lib.rs @@ -8,6 +8,7 @@ //! the host redraws and rereads `sb_view_content`. mod library; +mod recording; #[cfg(target_os = "ios")] mod spell; @@ -119,6 +120,10 @@ struct Canvas { scale: f32, /// The page date field a tap asked to change, for the host's picker. asked: Option, + /// The recording a tap asked to play, and from when. + play: Option<(onestore::page::Attachment, u32)>, + /// The recording playing, whose linked notes See Playback follows. + playing: Option<[u8; 16]>, } impl Canvas { @@ -156,6 +161,8 @@ impl Canvas { paper: Paper::WHITE, scale, asked: None, + play: None, + playing: None, }) } @@ -263,12 +270,16 @@ impl Canvas { Ok(self.respond(pressed) || changed) } - /// Whether `response` moved the page, keeping the date field it asks to change. + /// Whether `response` moved the page, keeping the date field it asks to change or the + /// recording it asks to play. fn respond(&mut self, response: Response) -> bool { - if let Some(Request::EditDate(field)) = response.request { - self.asked = Some(field); + let changed = response.changed || response.moved; + match response.request { + Some(Request::EditDate(field)) => self.asked = Some(field), + Some(Request::Play { file, at_ms }) => self.play = Some((file, at_ms)), + _ => {} } - moved(response) + changed } fn drag(&mut self, point: [f32; 2]) -> Result { diff --git a/crates/mobile/src/recording.rs b/crates/mobile/src/recording.rs new file mode 100644 index 0000000000000000000000000000000000000000..605180f002c014588f0e1eac1ae8388a5ccff9e1 --- /dev/null +++ b/crates/mobile/src/recording.rs @@ -0,0 +1,274 @@ +//! Record Audio, Record Video and playback, stored as the desktop stores them +//! (`canvas::recording`): the host records with the platform's devices and hands over a PCM +//! WAV file, or a video's pictures and sound as `Movie` takes them; a recording plays from +//! the files `sb_view_play_request` writes. + +use super::*; +use canvas::recording::{self, video}; +use std::path::Path; + +impl Canvas { + /// Starts recording at the caret, its line saying when from the long `date` and short + /// `time`; whether it started. + pub(crate) fn start_recording(&mut self, video: bool, date: &str, time: &str) -> Result { + let label = recording::label(video, date, time); + Ok(self.page.start_recording(&label)?.0.is_some()) + } + + /// Puts the recording made, `recorded`, where it started: a PCM WAV file, or with + /// `video` an AVI file from `Movie`. None forgets the recording, whose file never came. + pub(crate) fn finish_recording( + &mut self, + recorded: Option>, + video: bool, + ) -> Result { + let editor = &mut self.page.editor; + let Some(id) = editor.recording() else { + return Ok(false); + }; + let Some(recorded) = recorded else { + editor.cancel_recording(); + return Ok(false); + }; + let file = recording::file(id, &editor.page()?.title, video, recorded)?; + Ok(moved(self.page.finish_recording(file)?)) + } + + /// The recording a tap asked to play, taken, with its files written to `folder` to play + /// from: see `sb_view_play_request`. + pub(crate) fn take_play(&mut self, folder: &Path) -> Result> { + let Some((file, at)) = self.play.take() else { + return Ok(None); + }; + let bytes = file.bytes.clone().ok_or("The recording has no data")?; + let stem = Path::new(&file.filename) + .file_stem() + .map_or_else(String::new, |stem| stem.to_string_lossy().into_owned()); + std::fs::create_dir_all(folder)?; + let sound = match recording::sound(&bytes) { + Some(wave) => { + let path = folder.join(format!("{stem}.wav")); + std::fs::write(&path, wave)?; + path + } + None => { + let path = folder.join(file.filename.replace('/', "_")); + std::fs::write(&path, &bytes)?; + path + } + }; + let movie = video::Movie::parse(&bytes) + .map(|movie| -> Result<_> { + let path = folder.join(format!("{stem}.avi")); + std::fs::write(&path, &bytes)?; + // An empty frame repeats the picture before it. + let mut shown = [0; 2]; + let frames: Vec<[usize; 2]> = movie + .frames + .iter() + .map(|range| { + if !range.is_empty() { + shown = [range.start, range.end]; + } + shown + }) + .collect(); + Ok(serde_json::json!({ + "path": path, + "frames": frames, + "frame_us": movie.frame_us, + "size": video::SIZE, + })) + }) + .transpose()?; + self.playing = file.recording.map(|recording| recording.id); + Ok(Some(serde_json::json!({ + "name": stem, + "sound": sound, + "video": movie, + "at_ms": at, + }))) + } + + /// See Playback: highlights the note linked last at or before `at_ms` into the recording + /// playing, or with none ends playback. + pub(crate) fn played(&mut self, at_ms: Option) -> Result { + if at_ms.is_none() { + self.playing = None; + } + let note = self + .playing + .zip(at_ms) + .and_then(|(id, at)| self.page.editor.played_note(id, at)) + .and_then(|note| canvas::search::paragraph_match(&self.page.editor, note)); + Ok(moved(self.page.set_played(note)?)) + } +} + +/// Starts recording audio, or with `video` video, at the caret, its line saying when from +/// the long `date` and short `time`; whether it started. +/// +/// # Safety +/// `date` and `time` are NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn sb_view_start_recording( + view: &mut View, + video: bool, + date: *const c_char, + time: *const c_char, +) -> bool { + if view.read_only || view.reading.is_some() { + return false; + } + let started = view + .canvas + .start_recording(video, &string(date), &string(time)); + let started = report(started).unwrap_or(false); + view.stored(Ok(started)); + started +} + +/// How far the recording under way has got in milliseconds, its pauses left out, or -1; +/// `paused` gets whether it is paused. +#[unsafe(no_mangle)] +pub extern "C" fn sb_view_recording(view: &View, paused: &mut bool) -> i64 { + let editor = &view.canvas.page.editor; + *paused = editor.recording_paused(); + editor.recording_ms().map_or(-1, i64::from) +} + +/// Pauses or resumes the recording: nothing written while it is paused links to it. +#[unsafe(no_mangle)] +pub extern "C" fn sb_view_pause_recording(view: &mut View, paused: bool) { + view.canvas.page.editor.pause_recording(paused); +} + +/// Stops recording: the file recorded, `length` bytes of mono 16-bit PCM WAV or with +/// `video` the AVI file `sb_movie_finish` made, goes where the recording started. Null +/// `bytes` forgets the recording. +/// +/// # Safety +/// `bytes` is null or holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn sb_view_finish_recording( + view: &mut View, + bytes: *const u8, + length: usize, + video: bool, +) -> bool { + // SAFETY: the caller's contract. + let recorded = (!bytes.is_null()).then(|| unsafe { std::slice::from_raw_parts(bytes, length) }); + let result = view + .canvas + .finish_recording(recorded.map(<[u8]>::to_vec), video); + view.stored(result) +} + +/// The recording the last tap asked to play, taken, as JSON: its `name`, the `sound` file +/// written to `folder` that the platform's player plays, from `at_ms`; and for a video its +/// `video` file `path`, the byte range of the JPEG picture each of its `frames` shows, +/// `frame_us` apart, and their `size`. Null when none was asked for. +/// +/// # Safety +/// `folder` is NUL-terminated UTF-8. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn sb_view_play_request( + view: &mut View, + folder: *const c_char, +) -> *mut c_char { + let request = view.canvas.take_play(Path::new(&string(folder))); + report(request) + .flatten() + .map_or(std::ptr::null_mut(), |json| owned(json.to_string())) +} + +/// The recording last asked to play is at `at_ms`, whose linked note the page highlights; +/// -1 ends playback. Whether the page changed. +#[unsafe(no_mangle)] +pub extern "C" fn sb_view_played(view: &mut View, at_ms: i64) -> bool { + let at = u32::try_from(at_ms).ok(); + report(view.canvas.played(at)).unwrap_or(false) +} + +/// The rate the host records sound at, as mono 16-bit PCM: the desktop's. +#[unsafe(no_mangle)] +pub extern "C" fn sb_recording_rate() -> u32 { + recording::RATE +} + +/// A video recording's pictures and sound as the camera gave them, which become the AVI +/// file the desktop records. +#[derive(Default)] +pub struct Movie { + pictures: video::Pictures, + sound: Vec, +} + +#[unsafe(no_mangle)] +pub extern "C" fn sb_movie_new() -> *mut Movie { + Box::into_raw(Box::default()) +} + +/// The picture taken `at_us` microseconds in: `height` rows of `stride` bytes of BGRA. +/// +/// # Safety +/// `bgra` holds `height` × `stride` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn sb_movie_picture( + movie: &mut Movie, + at_us: u64, + bgra: *const u8, + width: u32, + height: u32, + stride: usize, +) { + // SAFETY: the caller's contract. + let bgra = unsafe { std::slice::from_raw_parts(bgra, height as usize * stride) }; + movie.pictures.take_bgra( + at_us, + [width, height].map(|side| side as usize), + stride, + bgra, + ); +} + +/// More of the sound, mono 16-bit little-endian PCM at `canvas::recording::RATE`. +/// +/// # Safety +/// `pcm` holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn sb_movie_sound(movie: &mut Movie, pcm: *const u8, length: usize) { + // SAFETY: the caller's contract. + movie + .sound + .extend_from_slice(unsafe { std::slice::from_raw_parts(pcm, length) }); +} + +/// The AVI file of a movie `duration_us` long, `length` bytes freed with `sb_bytes_free`, +/// or null when it has no pictures or passes 4 GiB. +/// +/// # Safety +/// `movie` came from `sb_movie_new` and is not used again. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn sb_movie_finish( + movie: *mut Movie, + duration_us: u64, + length: &mut usize, +) -> *mut u8 { + // SAFETY: the caller's contract. + let movie = unsafe { Box::from_raw(movie) }; + let frames = movie.pictures.finish(duration_us); + if frames.is_empty() { + return std::ptr::null_mut(); + } + let sound = video::Pcm { + rate: recording::RATE, + channels: 1, + data: movie.sound, + }; + let Some(avi) = video::write(video::SIZE, &frames, &sound) else { + return std::ptr::null_mut(); + }; + *length = avi.len(); + Box::into_raw(avi.into_boxed_slice()).cast() +} diff --git a/crates/mobile/src/tests.rs b/crates/mobile/src/tests.rs index 9d562ec7e0227b9ccca29422c54ba0b480f091ae..e817e21cc56bde88e0510f0657d731e549f7e7b3 100644 --- a/crates/mobile/src/tests.rs +++ b/crates/mobile/src/tests.rs @@ -1129,3 +1129,110 @@ fn the_page_prints_as_a_pdf() { &text[..400] ); } + +#[test] +fn a_recording_goes_in_as_the_desktop_stores_it_and_a_tap_plays_it() { + use onestore::page::{PageObject, ParagraphContent}; + let (directory, section) = features(); + let mut canvas = canvas(§ion, "Paragraph controls"); + focus(&mut canvas, "Collapsed parent"); + canvas.edit().unwrap(); + let date = ["Tuesday, September 29, 2026", "5:18 PM"]; + assert!(canvas.start_recording(false, date[0], date[1]).unwrap()); + canvas.insert("Linked note".into()).unwrap(); + let samples: Vec = (0..16_000).map(|n| (n % 64 * 256 - 8192) as i16).collect(); + let wav = canvas::recording::wave(canvas::recording::RATE, &samples); + assert!(canvas.finish_recording(Some(wav), false).unwrap()); + section + .shared + .apply(canvas.edit().unwrap().unwrap()) + .unwrap(); + + let page = section.shared.page(canvas.space).unwrap().0; + let paragraphs: Vec<_> = page + .objects + .iter() + .filter_map(|object| match object { + PageObject::Outline(outline) => Some(&outline.paragraphs), + _ => None, + }) + .flatten() + .collect(); + let file = paragraphs + .iter() + .find_map(|paragraph| match ¶graph.content { + ParagraphContent::Attachment(file) => Some(file), + _ => None, + }) + .unwrap(); + let recording = file.recording.unwrap(); + assert_eq!(file.filename, format!("{}.wav", page.title)); + assert_eq!((recording.kind, recording.duration_ms), (1, Some(1000))); + // IMA ADPCM, as the desktop stores audio for OneNote (`corpus/recording`). + let bytes = file.bytes.as_deref().unwrap(); + assert_eq!(u16::from_le_bytes([bytes[20], bytes[21]]), 0x11); + let linked = |text: &str| { + paragraphs + .iter() + .find(|paragraph| { + paragraph + .text() + .is_some_and(|t| t.text.text().starts_with(text)) + }) + .map(|paragraph| (paragraph.media.recordings.clone(), paragraph.media.time_ms)) + .unwrap() + }; + let line = linked("Audio recording started: 5:18 PM Tuesday, September 29, 2026"); + assert_eq!(line, (vec![recording.id], Some(0))); + assert_eq!(linked("Linked note").0, [recording.id]); + + // A tap on the recording plays it as PCM from the start. + let tapped = (0..800) + .flat_map(|y| (0..300).map(move |x| [x as f32 * 2.0, y as f32 * 2.0])) + .find(|&point| { + matches!( + canvas.page.hit(canvas.device(point.map(|v| v * POINT))), + Some(Hit::File(_)) + ) + }) + .unwrap() + .map(|v| v * POINT); + canvas.press(tapped).unwrap(); + canvas.release().unwrap(); + let folder = directory.path().join("playback"); + let request = canvas.take_play(&folder).unwrap().unwrap(); + assert_eq!(request["at_ms"], 0); + assert!(request["video"].is_null()); + let sound = std::fs::read(request["sound"].as_str().unwrap()).unwrap(); + assert_eq!(u16::from_le_bytes([sound[20], sound[21]]), 1); + assert!(canvas.take_play(&folder).unwrap().is_none()); + canvas.played(Some(5_000)).unwrap(); + canvas.played(None).unwrap(); +} + +#[test] +fn a_movie_from_the_camera_becomes_the_desktop_s_avi() { + let movie = recording::sb_movie_new(); + let bgra: Vec = (0..480 * 640 * 4).map(|n| (n % 251) as u8).collect(); + for index in 0..20u64 { + unsafe { + recording::sb_movie_picture( + &mut *movie, + index * 50_000, + bgra.as_ptr(), + 640, + 480, + 640 * 4, + ) + }; + } + let pcm = vec![0u8; canvas::recording::RATE as usize * 2]; + unsafe { recording::sb_movie_sound(&mut *movie, pcm.as_ptr(), pcm.len()) }; + let mut length = 0; + let avi = unsafe { recording::sb_movie_finish(movie, 1_000_000, &mut length) }; + let bytes = unsafe { std::slice::from_raw_parts(avi, length) }.to_vec(); + unsafe { sb_bytes_free(avi, length) }; + let parsed = canvas::recording::video::Movie::parse(&bytes).unwrap(); + assert_eq!((parsed.frames.len(), parsed.duration_ms()), (15, 1000)); + assert!(parsed.sound.is_some()); +} diff --git a/crates/snowbound/Cargo.toml b/crates/snowbound/Cargo.toml index d576e08385d25ed14d32d4f64adc831d09db7baa..594b3aa119add07135d763c973d321d7734d51a5 100644 --- a/crates/snowbound/Cargo.toml +++ b/crates/snowbound/Cargo.toml @@ -14,8 +14,6 @@ canvas = { path = "../canvas", features = ["interaction", "pdf"] } draw = { path = "../draw", default-features = false, features = ["render"] } ui = { path = "../ui" } png = "0.18" -# Encodes a video recording's pictures as JPEG, which OneNote 2010 plays. -image = { version = "0.25", default-features = false, features = ["jpeg"] } onestore = { path = "../onestore" } notebook = { path = "../notebook" } serde = { version = "1", features = ["derive"] } @@ -94,6 +92,8 @@ windows-sys = { version = "0.61", features = [ [target.'cfg(target_os = "linux")'.dependencies] accesskit_winit = { version = "0.34.0", default-features = false, features = ["accesskit_unix", "async-io"] } smithay-clipboard = "0.7.3" +# Scales the window's icon. +image = { version = "0.25", default-features = false } # Loads fontconfig at run time, so builds need no fontconfig headers or library. fontique = { workspace = true, features = ["fontconfig-dlopen"] } libc = "0.2" @@ -106,3 +106,7 @@ sctk-adwaita = { version = "0.10.1", default-features = false } wayland-backend = { version = "0.3.17", features = ["client_system"] } wayland-client = "0.31.15" wayland-protocols = { version = "0.32.13", features = ["client", "staging"] } + +[dev-dependencies] +# Draws the pictures of the video recording the tests store. +image = { version = "0.25", default-features = false } diff --git a/crates/snowbound/src/attachment.rs b/crates/snowbound/src/attachment.rs index 2f76105c5854f8567e25e0afed31e4e6c7a17cb9..c2e32d4f92505b83372c1e3fa1a94030211c4596 100644 --- a/crates/snowbound/src/attachment.rs +++ b/crates/snowbound/src/attachment.rs @@ -4,9 +4,6 @@ use crate::{State, platform}; use onestore::page::Attachment; use std::{error::Error, path::Path}; -/// The size OneNote 2010 shows a file's icon at, in points. -pub(crate) const ICON_SIZE: [f32; 2] = [24.0, 24.0]; - impl State { /// Attach File, or a file dropped at `at`, a window point: a copy of the file's bytes /// at the caret or at `at`, with the icon the system shows for it; audio and video are @@ -30,10 +27,10 @@ impl State { id: onestore::page::text::new_id()?, filename: name.to_owned(), source_path: path.to_str().map(str::to_owned), - size: Some(ICON_SIZE), + size: Some(canvas::gpu::page::ICON_SIZE), layout: Default::default(), preview: platform::file_icon(path).map(Into::into), - recording: crate::recording::attached(name, &bytes), + recording: canvas::recording::attached(name, &bytes), bytes: Some(bytes.into()), tags: Vec::new(), }; diff --git a/crates/snowbound/src/main.rs b/crates/snowbound/src/main.rs index 2c7f02854f5173a8b2e245145058f402c8639a25..ddbd1cd7cead8ab8e398e74dbfe68e626ca9cb04 100644 --- a/crates/snowbound/src/main.rs +++ b/crates/snowbound/src/main.rs @@ -84,7 +84,6 @@ mod undo; mod unpack; mod unread; mod update; -mod video; #[cfg_attr(target_arch = "wasm32", path = "watch_web.rs")] mod watch; diff --git a/crates/snowbound/src/media_linux.rs b/crates/snowbound/src/media_linux.rs index 906fda14dfdc579b2a2a37fbfe2ca37c1b4b0e82..9865df2cd23fd96490f6d7606010e365d60a343c 100644 --- a/crates/snowbound/src/media_linux.rs +++ b/crates/snowbound/src/media_linux.rs @@ -3,7 +3,8 @@ //! with gst-libav), the default microphone records as PCM WAV, and the default camera with //! it as the Motion JPEG AVI file OneNote plays. Recording needs the Base and Good plug-ins. -use crate::{platform::symbol, video}; +use crate::platform::symbol; +use canvas::recording::video; use std::{ cell::Cell, ffi::{CStr, CString, c_char, c_int, c_void}, @@ -313,7 +314,7 @@ impl Recorder { Self::start(&movie( "autovideosrc", "autoaudiosrc", - crate::recording::RATE, + canvas::recording::RATE, location, )) } @@ -395,17 +396,18 @@ mod tests { "{} ! wavenc ! filesink location=\"{location}\"", sound( "audiotestsrc is-live=true wave=sine", - crate::recording::RATE + canvas::recording::RATE ) ), 2, true, ); - let (bytes, duration) = crate::recording::compress(&std::fs::read(&path).unwrap()).unwrap(); + let (bytes, duration) = + canvas::recording::compress(&std::fs::read(&path).unwrap()).unwrap(); // Two seconds recorded; the paused one is left out. assert!(duration.abs_diff(2000) < 300, "{duration}"); // Snowbound's IMA ADPCM, decoded as playback decodes it, plays at its length. - let wave = crate::recording::decompress(&bytes).unwrap(); + let wave = canvas::recording::decompress(&bytes).unwrap(); let played = scratch.0.join("played.wav"); std::fs::write(&played, wave).unwrap(); let mut player = Player::open(&played, false).unwrap(); @@ -432,7 +434,7 @@ mod tests { &movie( "videotestsrc is-live=true pattern=smpte", "audiotestsrc is-live=true wave=sine", - crate::recording::RATE, + canvas::recording::RATE, quoted(&path).unwrap(), ), 2, diff --git a/crates/snowbound/src/media_macos.rs b/crates/snowbound/src/media_macos.rs index 6ac39e6f8f33f91cf68bc02282c02abc61cb3d18..e2ee6bb13f73f3f8049f6725c0a2d6f81da53066 100644 --- a/crates/snowbound/src/media_macos.rs +++ b/crates/snowbound/src/media_macos.rs @@ -3,7 +3,7 @@ //! PCM WAV, and a capture session records the default camera and microphone as a movie, //! which `AVAssetReader` reads back into the AVI file OneNote plays. -use crate::video; +use canvas::recording::video; use objc2::{ msg_send, msg_send_id, rc::{Allocated, Retained}, @@ -432,7 +432,7 @@ fn delegate() -> Retained { } /// The movie at `path`, as AVFoundation reads it, as a Motion JPEG AVI file with its sound -/// in mono PCM at `crate::recording::RATE`. +/// in mono PCM at `canvas::recording::RATE`. fn avi(path: &Path) -> Option> { unsafe { let asset: Retained = msg_send_id![class("AVURLAsset"), URLAssetWithURL: &*url(path)?, options: std::ptr::null::()]; @@ -453,22 +453,14 @@ fn avi(path: &Path) -> Option> { CVPixelBufferGetBytesPerRow(buffer), ]; let base = CVPixelBufferGetBaseAddress(buffer); - let mut rgb = Vec::with_capacity(width * height * 3); if !base.is_null() { let rows = std::slice::from_raw_parts(base, stride * height); - for row in rows.chunks_exact(stride) { - for bgra in row[..width * 4].chunks_exact(4) { - rgb.extend_from_slice(&[bgra[2], bgra[1], bgra[0]]); - } - } + pictures.take_bgra(at, [width, height], stride, rows); } CVPixelBufferUnlockBaseAddress(buffer, 1); - if let Some(picture) = image::RgbImage::from_raw(width as u32, height as u32, rgb) { - pictures.take(at, picture); - } })?; let mut sound = video::Pcm { - rate: crate::recording::RATE, + rate: canvas::recording::RATE, channels: 1, data: Vec::new(), }; @@ -550,8 +542,9 @@ mod tests { /// What Snowbound records, compressed, plays back through Core Audio at its length. #[test] fn core_audio_plays_the_compressed_recordings() { - let pcm = crate::recording::wave(crate::recording::RATE, &crate::recording::tests::tone()); - let (bytes, duration) = crate::recording::compress(&pcm).unwrap(); + let pcm = + canvas::recording::wave(canvas::recording::RATE, &crate::recording::tests::tone()); + let (bytes, duration) = canvas::recording::compress(&pcm).unwrap(); let folder = std::env::temp_dir().join(format!("snowbound-player-{}", std::process::id())); std::fs::create_dir_all(&folder).unwrap(); let path = folder.join("tone.wav"); @@ -611,7 +604,7 @@ mod tests { .iter() .map(ExactSizeIterator::len) .sum(); - let expected = 2 * crate::recording::RATE as usize * 2; + let expected = 2 * canvas::recording::RATE as usize * 2; assert!(length.abs_diff(expected) < expected / 20, "{length}"); } } diff --git a/crates/snowbound/src/recording.rs b/crates/snowbound/src/recording.rs index a20c608909ca9d25f957b9068e95ea50269cc414..6576327f92c2947c1f34d6312f19bc925a59dcbf 100644 --- a/crates/snowbound/src/recording.rs +++ b/crates/snowbound/src/recording.rs @@ -3,15 +3,13 @@ //! as a file OneNote plays; a recording, or a note linked to a moment in one, plays in a //! transport over the page. What Snowbound cannot play opens in the system's player. -use crate::{State, media, platform, video}; -use onestore::page::{Attachment, Recording}; +use crate::{State, media, platform}; +use canvas::recording::{RATE, video}; +use onestore::page::Attachment; use std::{error::Error, ops::Range, path::Path, sync::Arc, time::Duration}; use ui::{Axis, Flags, Spec, Theme, children, fit, px}; use winit::keyboard::NamedKey; -/// The rate audio is recorded at: wideband speech, as OneNote's own 8 kHz WMA is narrowband. -pub(crate) const RATE: u32 = 16_000; - /// What records or plays now. #[derive(Default)] pub(crate) enum Media { @@ -26,7 +24,7 @@ pub(crate) enum Media { Saving { video: bool, path: std::path::PathBuf, - job: std::thread::JoinHandle, String>>, + job: std::thread::JoinHandle>, }, Playing(Box), } @@ -94,8 +92,7 @@ impl State { } }; let [date, time] = platform::date_text(crate::filetime()); - let kind = if video { "Video" } else { "Audio" }; - let label = format!("{kind} recording started: {time} {date}"); + let label = canvas::recording::label(video, &date, &time); let (id, response) = self.view.start_recording(&label)?; self.respond(response); if id.is_none() { @@ -129,15 +126,15 @@ impl State { } }; let recorded = path.clone(); + let id = self.view.editor.recording(); + let title = self.view.editor.page()?.title; let job = std::thread::spawn(move || { finish()?; let recorded = notebook::fs::read(&recorded).map_err(|error| error.to_string())?; - Ok(if video { - recorded - } else { - compress(&recorded).map_or(recorded, |(bytes, _)| bytes) - }) + let id = id.ok_or("Try recording again.")?; + canvas::recording::file(id, &title, video, recorded) + .map_err(|error| error.to_string()) }); self.media = Media::Saving { video, path, job }; } @@ -147,66 +144,24 @@ impl State { Media::Saving { job, .. } if wait || job.is_finished() => {} _ => return Ok(()), } - let Media::Saving { video, path, job } = std::mem::take(&mut self.media) else { + let Media::Saving { path, job, .. } = std::mem::take(&mut self.media) else { unreachable!("saving, as matched") }; let saved = job .join() .unwrap_or_else(|_| Err("Try recording again.".into())); - let Some(id) = self.view.editor.recording() else { + if self.view.editor.recording().is_none() { return Ok(()); - }; - let bytes = match saved { - Ok(bytes) => bytes, + } + let mut file = match saved { + Ok(file) => file, Err(detail) => { platform::alert("Couldn't finish the recording", &detail); return Ok(()); } }; - let duration_ms = if video { - video::Movie::parse(&bytes).map(|movie| movie.duration_ms()) - } else { - Wave::parse(&bytes).map(|wave| wave.duration_ms()) - }; - let preview = platform::file_icon(&path); + file.preview = platform::file_icon(&path).map(Into::into); notebook::fs::remove_file(&path)?; - let title = self.view.editor.page()?.title; - let name: String = title - .chars() - .map(|c| { - if matches!(c, '/' | '\\' | '\0') { - '_' - } else { - c - } - }) - .collect(); - let name = if name.trim().is_empty() { - if video { - "Video recording" - } else { - "Audio recording" - } - .to_owned() - } else { - name - }; - let extension = if video { "avi" } else { "wav" }; - let file = Attachment { - id: onestore::page::text::new_id()?, - filename: format!("{name}.{extension}"), - source_path: None, - size: Some(crate::attachment::ICON_SIZE), - layout: Default::default(), - bytes: Some(bytes.into()), - preview: preview.map(Into::into), - recording: Some(Recording { - id, - kind: if video { 2 } else { 1 }, - duration_ms, - }), - tags: Vec::new(), - }; let response = self.view.finish_recording(file)?; self.respond(response); Ok(()) @@ -226,19 +181,10 @@ impl State { let name = Path::new(&file.filename) .file_stem() .map_or_else(String::new, |stem| stem.to_string_lossy().into_owned()); - let movie = video::Movie::parse(&bytes); - // The sound as a WAV file every platform plays: IMA ADPCM decoded, a video's sound - // taken out of it. - let sound = movie.as_ref().map(|movie| { - movie - .wave(&bytes) - .unwrap_or_else(|| silence(movie.duration_ms())) - }); - let sound = sound.map_or_else(|| decompress(&bytes), Some); - let playable = match sound { + let playable = match canvas::recording::sound(&bytes) { Some(wave) => Attachment { filename: format!("{name}.wav"), - bytes: Some(decompress(&wave).unwrap_or(wave).into()), + bytes: Some(wave.into()), ..file.clone() }, None => file.clone(), @@ -260,7 +206,7 @@ impl State { player, name, recording: file.recording.map(|recording| recording.id), - video: movie.map(|movie| Video { + video: video::Movie::parse(&bytes).map(|movie| Video { bytes, movie, shown: None, @@ -607,255 +553,11 @@ fn moment(text: &str) -> Option { seconds.checked_mul(1000) } -/// Silence as long as `ms`, the clock of a video without sound. -fn silence(ms: u32) -> Vec { - wave(8_000, &vec![0; ms as usize * 8]) -} - -/// A mono 16-bit PCM WAV file of `samples` at `rate`, as the platform recorders write. -pub(crate) fn wave(rate: u32, samples: &[i16]) -> Vec { - let data: Vec = samples.iter().flat_map(|s| s.to_le_bytes()).collect(); - [ - &b"RIFF"[..], - &(36 + data.len() as u32).to_le_bytes(), - b"WAVEfmt ", - &16_u32.to_le_bytes(), - &1_u16.to_le_bytes(), - &1_u16.to_le_bytes(), - &rate.to_le_bytes(), - &(rate * 2).to_le_bytes(), - &2_u16.to_le_bytes(), - &16_u16.to_le_bytes(), - b"data", - &(data.len() as u32).to_le_bytes(), - &data, - ] - .concat() -} - -/// A recording for a file attached under `filename`, as OneNote 2010 makes one of every -/// audio and video file it attaches, with the length of a WAV file's `bytes`. -pub(crate) fn attached(filename: &str, bytes: &[u8]) -> Option { - Some(Recording { - id: onestore::page::text::new_guid().ok()?, - kind: Recording::kind_of(filename)?, - duration_ms: Wave::parse(bytes).map(|wave| wave.duration_ms()), - }) -} - -/// The chunks of a RIFF WAVE file this module reads. -struct Wave<'a> { - format: u16, - channels: u16, - rate: u32, - bytes_per_second: u32, - block: u16, - bits: u16, - /// The sample count a compressed file's `fact` chunk states. - samples: Option, - data: &'a [u8], -} - -impl<'a> Wave<'a> { - fn parse(bytes: &'a [u8]) -> Option { - if bytes.get(..4)? != b"RIFF" || bytes.get(8..12)? != b"WAVE" { - return None; - } - let u16_at = |data: &[u8], at: usize| { - Some(u16::from_le_bytes(data.get(at..at + 2)?.try_into().ok()?)) - }; - let u32_at = |data: &[u8], at: usize| { - Some(u32::from_le_bytes(data.get(at..at + 4)?.try_into().ok()?)) - }; - let (mut fmt, mut samples, mut data) = (None, None, None); - let mut at = 12; - while at + 8 <= bytes.len() { - let size = u32_at(bytes, at + 4)? as usize; - let body = &bytes[at + 8..(at + 8).saturating_add(size).min(bytes.len())]; - match &bytes[at..at + 4] { - b"fmt " => fmt = Some(body), - b"fact" => samples = u32_at(body, 0), - b"data" => data = Some(body), - _ => {} - } - at = at + 8 + size + size % 2; - } - let fmt = fmt?; - Some(Self { - format: u16_at(fmt, 0)?, - channels: u16_at(fmt, 2)?, - rate: u32_at(fmt, 4)?, - bytes_per_second: u32_at(fmt, 8)?, - block: u16_at(fmt, 12)?, - bits: u16_at(fmt, 14)?, - samples, - data: data?, - }) - } - - fn duration_ms(&self) -> u32 { - let ms = match self.samples.filter(|_| self.format != 1) { - Some(samples) => u64::from(samples) * 1000 / u64::from(self.rate.max(1)), - None => self.data.len() as u64 * 1000 / u64::from(self.bytes_per_second.max(1)), - }; - u32::try_from(ms).unwrap_or(u32::MAX) - } -} - -/// IMA ADPCM's step sizes and how each code moves along them. -const STEPS: [i32; 89] = [ - 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66, - 73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, 337, 371, 408, 449, - 494, 544, 598, 658, 724, 796, 876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, 2272, - 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358, 5894, 6484, 7132, 7845, 8630, 9493, - 10442, 11487, 12635, 13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767, -]; -const INDEX_STEPS: [i32; 8] = [-1, -1, -1, -1, 2, 4, 6, 8]; - -/// Bytes per block: Windows' IMA ADPCM codec's choice for 16 kHz. -const BLOCK: usize = 512; -/// Samples per block: the header's, then two per remaining byte. -const BLOCK_SAMPLES: usize = (BLOCK - 4) * 2 + 1; - -/// A mono 16-bit PCM WAV file as 4-bit IMA ADPCM (WAVE_FORMAT_DVI_ADPCM), which Windows -/// and macOS decode, a quarter of the size; and its length. None for other files. -pub(crate) fn compress(wav: &[u8]) -> Option<(Vec, u32)> { - let wave = Wave::parse(wav)?; - if wave.format != 1 || wave.channels != 1 || wave.bits != 16 { - return None; - } - let samples: Vec = wave - .data - .chunks_exact(2) - .map(|pair| i16::from_le_bytes([pair[0], pair[1]])) - .collect(); - let mut data = Vec::with_capacity(samples.len() / 2 + BLOCK); - let mut index = 0_i32; - for block in samples.chunks(BLOCK_SAMPLES) { - let mut predicted = i32::from(block[0]); - data.extend_from_slice(&block[0].to_le_bytes()); - data.extend_from_slice(&[index as u8, 0]); - // A short last block is padded with its last sample; `fact` gives the true length. - let last = *block.last().unwrap(); - let rest = block[1..] - .iter() - .copied() - .chain(std::iter::repeat(last)) - .take(BLOCK_SAMPLES - 1) - .collect::>(); - for pair in rest.chunks_exact(2) { - let mut byte = 0; - for (shift, sample) in [0, 4].into_iter().zip(pair) { - let code = encode(i32::from(*sample), &mut predicted, &mut index); - byte |= code << shift; - } - data.push(byte); - } - } - let count = u32::try_from(samples.len()).ok()?; - let bytes_per_second = wave.rate * BLOCK as u32 / BLOCK_SAMPLES as u32; - let mut out = Vec::with_capacity(data.len() + 60); - let fmt: Vec = [ - &0x11_u16.to_le_bytes()[..], - &1_u16.to_le_bytes(), - &wave.rate.to_le_bytes(), - &bytes_per_second.to_le_bytes(), - &(BLOCK as u16).to_le_bytes(), - &4_u16.to_le_bytes(), - &2_u16.to_le_bytes(), - &(BLOCK_SAMPLES as u16).to_le_bytes(), - ] - .concat(); - let riff = 4 + (8 + fmt.len()) + (8 + 4) + (8 + data.len()); - out.extend_from_slice(b"RIFF"); - out.extend_from_slice(&u32::try_from(riff).ok()?.to_le_bytes()); - out.extend_from_slice(b"WAVEfmt "); - out.extend_from_slice(&(fmt.len() as u32).to_le_bytes()); - out.extend_from_slice(&fmt); - out.extend_from_slice(b"fact"); - out.extend_from_slice(&4_u32.to_le_bytes()); - out.extend_from_slice(&count.to_le_bytes()); - out.extend_from_slice(b"data"); - out.extend_from_slice(&u32::try_from(data.len()).ok()?.to_le_bytes()); - out.extend_from_slice(&data); - let duration = u32::try_from(u64::from(count) * 1000 / u64::from(wave.rate.max(1))).ok()?; - Some((out, duration)) -} - -/// The 4-bit code nearest `sample` from `predicted`, which it and `index` then follow as a -/// decoder does. -fn encode(sample: i32, predicted: &mut i32, index: &mut i32) -> u8 { - let step = STEPS[*index as usize]; - let mut difference = sample - *predicted; - let mut code = 0; - if difference < 0 { - code = 8; - difference = -difference; - } - let mut delta = step >> 3; - for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] { - if difference >= part { - code |= bit; - difference -= part; - delta += part; - } - } - *predicted = if code & 8 != 0 { - *predicted - delta - } else { - *predicted + delta - } - .clamp(-32768, 32767); - *index = (*index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88); - code -} - -/// A mono IMA ADPCM WAV file as 16-bit PCM, which every platform's player decodes; none -/// for other files. -pub(crate) fn decompress(wav: &[u8]) -> Option> { - let wave = Wave::parse(wav).filter(|wave| wave.format == 0x11 && wave.channels == 1)?; - Some(self::wave(wave.rate, &decode(&wave))) -} - -/// Each block's header sample and index, then its codes. -fn decode(wave: &Wave<'_>) -> Vec { - let mut out = Vec::new(); - for block in wave.data.chunks(usize::from(wave.block.max(4))) { - let Some(header) = block.get(..4) else { - break; - }; - let mut predicted = i32::from(i16::from_le_bytes([header[0], header[1]])); - let mut index = i32::from(header[2]).clamp(0, 88); - out.push(predicted as i16); - for byte in &block[4..] { - for code in [byte & 15, byte >> 4] { - let step = STEPS[index as usize]; - let mut delta = step >> 3; - for (bit, part) in [(4, step), (2, step >> 1), (1, step >> 2)] { - if code & bit != 0 { - delta += part; - } - } - predicted = if code & 8 != 0 { - predicted - delta - } else { - predicted + delta - } - .clamp(-32768, 32767); - index = (index + INDEX_STEPS[usize::from(code & 7)]).clamp(0, 88); - out.push(predicted as i16); - } - } - } - if let Some(samples) = wave.samples { - out.truncate(samples as usize); - } - out -} - #[cfg(test)] pub(crate) mod tests { use super::*; + use canvas::recording::{compress, wave}; + use onestore::page::Recording; use std::path::PathBuf; /// Two seconds of a spoken-pitch chord. @@ -870,57 +572,6 @@ pub(crate) mod tests { .collect() } - #[test] - fn a_recording_compresses_to_ima_adpcm_that_decodes_close_to_what_was_heard() { - let samples = tone(); - let (bytes, duration) = compress(&wave(RATE, &samples)).unwrap(); - assert_eq!(duration, 2000); - let wave = Wave::parse(&bytes).unwrap(); - assert_eq!( - (wave.format, wave.channels, wave.rate, wave.bits), - (0x11, 1, RATE, 4) - ); - assert_eq!(wave.data.len() % BLOCK, 0); - assert_eq!(wave.duration_ms(), 2000); - assert!(bytes.len() * 3 < samples.len() * 2); - let decoded = decode(&wave); - assert_eq!( - Wave::parse(&decompress(&bytes).unwrap()) - .unwrap() - .data - .len(), - decoded.len() * 2 - ); - assert_eq!(decoded.len(), samples.len()); - let noise: f64 = samples - .iter() - .zip(&decoded) - .map(|(a, b)| (f64::from(*a) - f64::from(*b)).powi(2)) - .sum(); - let signal: f64 = samples.iter().map(|a| f64::from(*a).powi(2)).sum(); - let snr = 10.0 * (signal / noise).log10(); - assert!(snr > 20.0, "{snr} dB"); - } - - #[test] - fn only_mono_16_bit_pcm_compresses_and_attached_wav_files_know_their_length() { - let stereo = { - let mut bytes = wave(RATE, &[0; 64]); - bytes[22] = 2; - bytes - }; - assert!(compress(&stereo).is_none()); - assert!(compress(b"not a wave").is_none()); - let silence = include_bytes!( - "../../../corpus/m6/native-features-01/notebook/fixture-data/silence.wav" - ); - // OneNote 2010 stored 1000 ms for this file when attached (`corpus/recording`). - let recording = attached("silence.WAV", silence).unwrap(); - assert_eq!((recording.kind, recording.duration_ms), (1, Some(1000))); - assert_eq!(attached("clip.wmv", b"").unwrap().kind, 2); - assert!(attached("notes.txt", b"").is_none()); - } - /// A recording made through the editor at the end of "Before recording", two notes /// written while it records, stored as the host stores it; `export` names a variable /// naming a directory receiving the notebook for a cold read in OneNote 2010. @@ -976,7 +627,7 @@ pub(crate) mod tests { id: onestore::page::text::new_id().unwrap(), filename: filename.into(), source_path: None, - size: Some(crate::attachment::ICON_SIZE), + size: Some(canvas::gpu::page::ICON_SIZE), layout: Default::default(), bytes: Some(bytes.into()), preview: Some(canvas::gpu::page::file_icon().into()), -- 2.54.0