1import AVFoundation
2import UIKit
3
4/// A recording `sb_view_play_request` asks to play: its sound file from `atMs`, and for a
5/// video the AVI file whose JPEG pictures `frames` gives, `frameUs` apart.
6struct PlayRequest: Decodable {
7 struct Video: Decodable {
8 let path: String
9 let frames: [[Int]]
10 let frameUs: UInt32
11 let size: [Int]
12 }
13 let name: String
14 let sound: String
15 let video: Video?
16 let atMs: UInt32
17}
18
19/// Record Audio and Record Video, stored as the desktop stores them: sound as mono PCM at
20/// `sb_recording_rate`, which the page keeps as IMA ADPCM, and video as a movie from the
21/// camera made into Motion JPEG AVI. Only one records at a time, and it keeps the page it
22/// records on until its file is there, whether or not the page is still shown.
23///
24/// Audio records on with the screen locked or the app in the background, under the audio
25/// background mode, as a lecture or meeting outlasts the screen's timeout; a call pauses it.
26/// The camera stops in the background, so a video recording then ends and is saved.
27final class Recorder: NSObject, AVCaptureFileOutputRecordingDelegate {
28 /// The recording under way or being saved.
29 private(set) static var current: Recorder?
30 /// Posted when the current recording pauses, resumes, saves or ends.
31 static let changed = Notification.Name("SnowboundRecordingChanged")
32
33 let video: Bool
34 let canvas: CanvasView
35 private let folder: URL
36 private var sound: AVAudioRecorder?
37 private var capture: AVCaptureSession?
38 private let output = AVCaptureMovieFileOutput()
39 #if targetEnvironment(simulator)
40 private var stub: StubCamera?
41 #endif
42 /// Stopped, its file being made and put on the page.
43 private(set) var saving = false
44
45 /// Whether video can be recorded here: with a camera, or the simulator's stand-in.
46 static var canRecordVideo: Bool {
47 #if targetEnvironment(simulator)
48 true
49 #else
50 AVCaptureDevice.default(for: .video) != nil
51 #endif
52 }
53
54 private init(video: Bool, canvas: CanvasView) {
55 self.video = video
56 self.canvas = canvas
57 folder = FileManager.default.temporaryDirectory.appendingPathComponent("Recording")
58 super.init()
59 }
60
61 /// Starts recording on `canvas` at its caret once the system allows the microphone, and
62 /// for video the camera; `failed` says why it could not.
63 static func start(video: Bool, on canvas: CanvasView, failed: @escaping (String, String) -> Void) {
64 guard current == nil else { return }
65 let title = video ? "Couldn't Record Video" : "Couldn't Record Audio"
66 AVAudioApplication.requestRecordPermission { microphone in
67 let begin = { (camera: Bool) in
68 DispatchQueue.main.async {
69 guard current == nil else { return }
70 if video && !camera {
71 return failed(title, "Allow Snowbound to use the camera in Settings, Privacy & Security, Camera.")
72 }
73 // A video without the microphone records silent, as on the desktop.
74 if !video && !microphone {
75 return failed(
76 title, "Allow Snowbound to use the microphone in Settings, Privacy & Security, Microphone.")
77 }
78 let recorder = Recorder(video: video, canvas: canvas)
79 do {
80 try recorder.begin(microphone: microphone)
81 } catch {
82 recorder.end()
83 return failed(title, "Try recording again.")
84 }
85 guard canvas.startRecording(video: video) else {
86 recorder.end()
87 return failed(title, "Try recording again.")
88 }
89 current = recorder
90 NotificationCenter.default.post(name: changed, object: recorder)
91 }
92 }
93 if video {
94 #if targetEnvironment(simulator)
95 begin(true)
96 #else
97 AVCaptureDevice.requestAccess(for: .video, completionHandler: begin)
98 #endif
99 } else {
100 begin(true)
101 }
102 }
103 }
104
105 private func begin(microphone: Bool) throws {
106 try? FileManager.default.removeItem(at: folder)
107 try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
108 let session = AVAudioSession.sharedInstance()
109 if microphone {
110 try session.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetoothHFP])
111 try session.setActive(true)
112 NotificationCenter.default.addObserver(
113 self, selector: #selector(interrupted), name: AVAudioSession.interruptionNotification, object: session)
114 }
115 #if targetEnvironment(simulator)
116 if video {
117 stub = try StubCamera(movie: folder.appendingPathComponent("camera.mov"))
118 }
119 #endif
120 if !video || isStubbed {
121 guard microphone else { return }
122 let settings: [String: Any] = [
123 AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: sb_recording_rate(), AVNumberOfChannelsKey: 1,
124 AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false,
125 ]
126 let sound = try AVAudioRecorder(url: soundFile, settings: settings)
127 guard sound.record() else { throw CocoaError(.fileWriteUnknown) }
128 self.sound = sound
129 return
130 }
131 let capture = AVCaptureSession()
132 capture.beginConfiguration()
133 if capture.canSetSessionPreset(.vga640x480) { capture.sessionPreset = .vga640x480 }
134 let devices = [AVCaptureDevice.default(for: .video)] + (microphone ? [AVCaptureDevice.default(for: .audio)] : [])
135 for (index, device) in devices.enumerated() {
136 guard let device, let input = try? AVCaptureDeviceInput(device: device), capture.canAddInput(input) else {
137 // Without the microphone the video records silent; without the camera, nothing.
138 if index == 0 { throw CocoaError(.fileWriteUnknown) }
139 continue
140 }
141 capture.addInput(input)
142 }
143 guard capture.canAddOutput(output) else { throw CocoaError(.fileWriteUnknown) }
144 capture.addOutput(output)
145 capture.commitConfiguration()
146 capture.startRunning()
147 output.startRecording(to: movieFile, recordingDelegate: self)
148 self.capture = capture
149 }
150
151 private var isStubbed: Bool {
152 #if targetEnvironment(simulator)
153 stub != nil
154 #else
155 false
156 #endif
157 }
158
159 private var soundFile: URL { folder.appendingPathComponent("sound.wav") }
160 private var movieFile: URL { folder.appendingPathComponent("camera.mov") }
161
162 /// How far the recording has got, its pauses left out, and whether it is paused.
163 var elapsed: (ms: Int64, paused: Bool)? { canvas.recording }
164
165 /// Whether Pause works: always but for a camera before iOS 18.
166 var pausable: Bool {
167 if capture == nil { return true }
168 if #available(iOS 18, *) { return true }
169 return false
170 }
171
172 func pause(_ paused: Bool) {
173 guard !saving, pausable else { return }
174 if paused {
175 sound?.pause()
176 } else {
177 sound?.record()
178 }
179 #if targetEnvironment(simulator)
180 stub?.paused = paused
181 #endif
182 if #available(iOS 18, *), capture != nil {
183 if paused { output.pauseRecording() } else { output.resumeRecording() }
184 }
185 canvas.pauseRecording(paused)
186 NotificationCenter.default.post(name: Self.changed, object: self)
187 }
188
189 /// A call or another app's audio takes the microphone: the recording pauses, and goes
190 /// on after where the system says it should.
191 @objc private func interrupted(_ notification: Notification) {
192 guard let raw = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt,
193 let type = AVAudioSession.InterruptionType(rawValue: raw)
194 else { return }
195 DispatchQueue.main.async { [self] in
196 switch type {
197 case .began where capture == nil: pause(true)
198 case .ended:
199 let options = (notification.userInfo?[AVAudioSessionInterruptionOptionKey] as? UInt)
200 .map(AVAudioSession.InterruptionOptions.init) ?? []
201 if options.contains(.shouldResume), capture == nil {
202 try? AVAudioSession.sharedInstance().setActive(true)
203 pause(false)
204 }
205 default: break
206 }
207 }
208 }
209
210 /// Stop: the recording's file is made and goes on the page where it started.
211 func stop() {
212 guard !saving else { return }
213 saving = true
214 canvas.pauseRecording(true)
215 NotificationCenter.default.post(name: Self.changed, object: self)
216 if capture != nil {
217 // The movie is saved once `fileOutput(_:didFinishRecordingTo:…)` hears so.
218 output.stopRecording()
219 return
220 }
221 sound?.stop()
222 #if targetEnvironment(simulator)
223 if let stub {
224 stub.finish { [self] in convert(movie: stub.movie) }
225 return
226 }
227 #endif
228 save(try? Data(contentsOf: soundFile))
229 }
230
231 /// The camera stopped: at Stop, or as the app left the screen or a call took the camera.
232 func fileOutput(
233 _ output: AVCaptureFileOutput, didFinishRecordingTo url: URL, from connections: [AVCaptureConnection],
234 error: Error?
235 ) {
236 let finished = (error as NSError?)?.userInfo[AVErrorRecordingSuccessfullyFinishedKey] as? Bool ?? (error == nil)
237 DispatchQueue.main.async { [self] in
238 saving = true
239 canvas.pauseRecording(true)
240 NotificationCenter.default.post(name: Self.changed, object: self)
241 capture?.stopRunning()
242 if finished { convert(movie: url) } else { save(nil) }
243 }
244 }
245
246 /// Makes the AVI file of `movie`, with the sound recorded beside it where there is.
247 private func convert(movie: URL) {
248 let sound = self.sound == nil ? movie : soundFile
249 // Finishing is not cut short by the app leaving the screen.
250 let task = UIApplication.shared.beginBackgroundTask()
251 Task.detached {
252 let avi = await avi(pictures: movie, sound: sound)
253 await MainActor.run {
254 self.save(avi)
255 UIApplication.shared.endBackgroundTask(task)
256 }
257 }
258 }
259
260 /// Puts `file` on the page, or with none forgets the recording.
261 private func save(_ file: Data?) {
262 canvas.finishRecording(file, video: video)
263 end()
264 Self.current = nil
265 NotificationCenter.default.post(name: Self.changed, object: self)
266 }
267
268 private func end() {
269 NotificationCenter.default.removeObserver(self)
270 capture?.stopRunning()
271 try? FileManager.default.removeItem(at: folder)
272 try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
273 }
274}
275
276/// The AVI file the desktop makes of a movie: the pictures of `pictures`' first video track
277/// and the sound of `sound`'s first audio track, mono PCM at `sb_recording_rate`.
278private func avi(pictures: URL, sound: URL) async -> Data? {
279 let movie = sb_movie_new()
280 let asset = AVURLAsset(url: pictures)
281 await read(asset, .video, [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]) { sample in
282 let time = CMSampleBufferGetPresentationTimeStamp(sample)
283 guard time.isValid, let buffer = CMSampleBufferGetImageBuffer(sample) else { return }
284 CVPixelBufferLockBaseAddress(buffer, .readOnly)
285 defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) }
286 guard let base = CVPixelBufferGetBaseAddress(buffer) else { return }
287 sb_movie_picture(
288 movie, UInt64(max(0, time.seconds) * 1_000_000), base.assumingMemoryBound(to: UInt8.self),
289 UInt32(CVPixelBufferGetWidth(buffer)), UInt32(CVPixelBufferGetHeight(buffer)),
290 CVPixelBufferGetBytesPerRow(buffer))
291 }
292 let pcm: [String: Any] = [
293 AVFormatIDKey: kAudioFormatLinearPCM, AVSampleRateKey: sb_recording_rate(), AVNumberOfChannelsKey: 1,
294 AVLinearPCMBitDepthKey: 16, AVLinearPCMIsFloatKey: false, AVLinearPCMIsBigEndianKey: false,
295 AVLinearPCMIsNonInterleaved: false,
296 ]
297 // A movie without sound has no sound track to read.
298 await read(AVURLAsset(url: sound), .audio, pcm) { sample in
299 guard let buffer = CMSampleBufferGetDataBuffer(sample) else { return }
300 let length = CMBlockBufferGetDataLength(buffer)
301 var bytes = [UInt8](repeating: 0, count: length)
302 if CMBlockBufferCopyDataBytes(buffer, atOffset: 0, dataLength: length, destination: &bytes) == noErr {
303 sb_movie_sound(movie, bytes, length)
304 }
305 }
306 let duration = (try? await asset.load(.duration))?.seconds ?? 0
307 var length = 0
308 guard let bytes = sb_movie_finish(movie, UInt64(max(0, duration) * 1_000_000), &length) else { return nil }
309 defer { sb_bytes_free(bytes, length) }
310 return Data(bytes: bytes, count: length)
311}
312
313/// Reads the first track of `kind` in `asset`, decoded as `settings` ask, a sample at a time.
314private func read(
315 _ asset: AVURLAsset, _ kind: AVMediaType, _ settings: [String: Any], _ sample: (CMSampleBuffer) -> Void
316) async {
317 guard let track = try? await asset.loadTracks(withMediaType: kind).first,
318 let reader = try? AVAssetReader(asset: asset)
319 else { return }
320 let output = AVAssetReaderTrackOutput(track: track, outputSettings: settings)
321 guard reader.canAdd(output) else { return }
322 reader.add(output)
323 guard reader.startReading() else { return }
324 while let next = output.copyNextSampleBuffer() {
325 sample(next)
326 }
327}
328
329#if targetEnvironment(simulator)
330/// The simulator's stand-in for a camera, which it has none of: moving colour bars and the
331/// time written as a movie, 15 pictures a second, while the microphone records beside it.
332private final class StubCamera {
333 let movie: URL
334 private let writer: AVAssetWriter
335 private let input: AVAssetWriterInput
336 private let adaptor: AVAssetWriterInputPixelBufferAdaptor
337 private var timer: Timer?
338 private var frame: Int64 = 0
339 var paused = false
340
341 init(movie: URL) throws {
342 self.movie = movie
343 writer = try AVAssetWriter(outputURL: movie, fileType: .mov)
344 input = AVAssetWriterInput(
345 mediaType: .video,
346 outputSettings: [AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: 640, AVVideoHeightKey: 480])
347 input.expectsMediaDataInRealTime = true
348 adaptor = AVAssetWriterInputPixelBufferAdaptor(
349 assetWriterInput: input,
350 sourcePixelBufferAttributes: [
351 kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
352 kCVPixelBufferWidthKey as String: 640, kCVPixelBufferHeightKey as String: 480,
353 ])
354 writer.add(input)
355 guard writer.startWriting() else { throw writer.error ?? CocoaError(.fileWriteUnknown) }
356 writer.startSession(atSourceTime: .zero)
357 timer = Timer.scheduledTimer(withTimeInterval: 1.0 / 15, repeats: true) { [weak self] _ in self?.shoot() }
358 }
359
360 private func shoot() {
361 guard !paused, input.isReadyForMoreMediaData, let pool = adaptor.pixelBufferPool else { return }
362 var made: CVPixelBuffer?
363 CVPixelBufferPoolCreatePixelBuffer(nil, pool, &made)
364 guard let buffer = made else { return }
365 CVPixelBufferLockBaseAddress(buffer, [])
366 let context = CGContext(
367 data: CVPixelBufferGetBaseAddress(buffer), width: 640, height: 480, bitsPerComponent: 8,
368 bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: CGColorSpaceCreateDeviceRGB(),
369 bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)
370 if let context {
371 let colors: [UIColor] = [.systemYellow, .systemTeal, .systemGreen, .systemPink, .systemRed, .systemBlue]
372 for bar in 0..<8 {
373 context.setFillColor(colors[(bar + Int(frame / 5)) % colors.count].cgColor)
374 context.fill(CGRect(x: bar * 80, y: 0, width: 80, height: 480))
375 }
376 UIGraphicsPushContext(context)
377 context.translateBy(x: 0, y: 480)
378 context.scaleBy(x: 1, y: -1)
379 let seconds = Double(frame) / 15
380 NSString(string: String(format: "%.1f s", seconds)).draw(
381 at: CGPoint(x: 24, y: 24),
382 withAttributes: [.font: UIFont.monospacedDigitSystemFont(ofSize: 72, weight: .bold), .foregroundColor: UIColor.black])
383 UIGraphicsPopContext()
384 }
385 CVPixelBufferUnlockBaseAddress(buffer, [])
386 adaptor.append(buffer, withPresentationTime: CMTime(value: frame, timescale: 15))
387 frame += 1
388 }
389
390 func finish(_ done: @escaping () -> Void) {
391 timer?.invalidate()
392 input.markAsFinished()
393 writer.endSession(atSourceTime: CMTime(value: frame, timescale: 15))
394 writer.finishWriting { DispatchQueue.main.async(execute: done) }
395 }
396}
397#endif
398
399/// Plays a recording: its sound, and a video's pictures in time with it.
400final class Playback {
401 let request: PlayRequest
402 let player: AVPlayer
403 private let pictures: Data?
404 private var shown = -1
405
406 init(_ request: PlayRequest) {
407 self.request = request
408 player = AVPlayer(url: URL(fileURLWithPath: request.sound))
409 pictures = request.video.flatMap { try? Data(contentsOf: URL(fileURLWithPath: $0.path), options: .alwaysMapped) }
410 try? AVAudioSession.sharedInstance().setCategory(.playback, mode: .spokenAudio)
411 try? AVAudioSession.sharedInstance().setActive(true)
412 player.seek(to: CMTime(value: CMTimeValue(request.atMs), timescale: 1000), toleranceBefore: .zero, toleranceAfter: .zero)
413 player.play()
414 }
415
416 deinit {
417 player.pause()
418 try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
419 }
420
421 /// Whether the player could not open the sound, as with OneNote's own Windows Media.
422 var failed: Bool { player.currentItem?.status == .failed }
423
424 var ms: Int64 { Int64(max(0, player.currentTime().seconds) * 1000) }
425
426 var durationMs: Int64? {
427 guard let duration = player.currentItem?.duration, duration.isNumeric else { return nil }
428 return Int64(duration.seconds * 1000)
429 }
430
431 var playing: Bool { player.rate != 0 }
432
433 func toggle() {
434 if playing {
435 player.pause()
436 } else {
437 if let durationMs, ms >= durationMs - 50 { player.seek(to: .zero) }
438 player.play()
439 }
440 }
441
442 /// The video's picture now, when it changed since last asked.
443 func picture() -> UIImage? {
444 guard let video = request.video, let pictures, !video.frames.isEmpty else { return nil }
445 let index = min(Int(ms * 1000 / Int64(max(1, video.frameUs))), video.frames.count - 1)
446 guard index != shown else { return nil }
447 shown = index
448 let range = video.frames[index]
449 guard range.count == 2, range[0] < range[1], range[1] <= pictures.count else { return nil }
450 return UIImage(data: pictures.subdata(in: range[0]..<range[1]))
451 }
452}
453
454/// The bar over the page's foot while something records or plays, as OneNote's Recording and
455/// Playback tabs: the clock, Pause, and Stop; a video's pictures above as it plays.
456final class MediaBar: UIView {
457 private let dot = UIImageView(image: UIImage(systemName: "record.circle.fill"))
458 private let label = UILabel()
459 private let pause = UIButton(configuration: .plain())
460 private let stop = UIButton(configuration: .plain())
461 let picture = UIImageView()
462 var onPause: (() -> Void)?
463 var onStop: (() -> Void)?
464
465 init() {
466 super.init(frame: .zero)
467 let blur = UIVisualEffectView(effect: UIBlurEffect(style: .systemThickMaterial))
468 blur.translatesAutoresizingMaskIntoConstraints = false
469 blur.layer.cornerRadius = 14
470 blur.clipsToBounds = true
471 addSubview(blur)
472 layer.shadowOpacity = 0.15
473 layer.shadowRadius = 8
474 layer.shadowOffset = CGSize(width: 0, height: 2)
475 dot.tintColor = .systemRed
476 dot.setContentHuggingPriority(.required, for: .horizontal)
477 label.font = .monospacedDigitSystemFont(ofSize: UIFont.preferredFont(forTextStyle: .subheadline).pointSize, weight: .medium)
478 label.adjustsFontForContentSizeCategory = true
479 label.lineBreakMode = .byTruncatingMiddle
480 label.accessibilityTraits.insert(.updatesFrequently)
481 stop.configuration?.image = UIImage(systemName: "stop.fill")
482 stop.accessibilityLabel = "Stop"
483 stop.addAction(UIAction { [weak self] _ in self?.onStop?() }, for: .primaryActionTriggered)
484 pause.addAction(UIAction { [weak self] _ in self?.onPause?() }, for: .primaryActionTriggered)
485 for button in [pause, stop] { button.setContentHuggingPriority(.required, for: .horizontal) }
486 picture.contentMode = .scaleAspectFit
487 picture.backgroundColor = .black
488 picture.layer.cornerRadius = 6
489 picture.clipsToBounds = true
490 picture.isHidden = true
491 let controls = UIStackView(arrangedSubviews: [dot, label, pause, stop])
492 controls.spacing = 8
493 controls.alignment = .center
494 let stack = UIStackView(arrangedSubviews: [picture, controls])
495 stack.axis = .vertical
496 stack.spacing = 8
497 stack.translatesAutoresizingMaskIntoConstraints = false
498 blur.contentView.addSubview(stack)
499 NSLayoutConstraint.activate([
500 blur.leadingAnchor.constraint(equalTo: leadingAnchor),
501 blur.trailingAnchor.constraint(equalTo: trailingAnchor),
502 blur.topAnchor.constraint(equalTo: topAnchor),
503 blur.bottomAnchor.constraint(equalTo: bottomAnchor),
504 stack.leadingAnchor.constraint(equalTo: blur.contentView.leadingAnchor, constant: 14),
505 stack.trailingAnchor.constraint(equalTo: blur.contentView.trailingAnchor, constant: -6),
506 stack.topAnchor.constraint(equalTo: blur.contentView.topAnchor, constant: 8),
507 stack.bottomAnchor.constraint(equalTo: blur.contentView.bottomAnchor, constant: -8),
508 picture.widthAnchor.constraint(lessThanOrEqualToConstant: 320),
509 ])
510 // Below required, as the stack gives the picture no height while it is hidden.
511 let aspect = picture.heightAnchor.constraint(equalTo: picture.widthAnchor, multiplier: 3.0 / 4)
512 aspect.priority = .defaultHigh
513 aspect.isActive = true
514 isHidden = true
515 }
516
517 required init?(coder: NSCoder) { fatalError() }
518
519 /// Shows `text`, the dot while recording, and Pause or Resume (or Play) where it works.
520 func show(_ text: String, recording: Bool, paused: Bool?, playing: Bool = false) {
521 isHidden = false
522 label.text = text
523 dot.isHidden = !recording
524 dot.alpha = paused == true ? 0.4 : 1
525 pause.isHidden = paused == nil
526 stop.isHidden = false
527 let resume = paused == true
528 pause.configuration?.image = UIImage(systemName: resume ? (playing ? "play.fill" : "record.circle") : "pause.fill")
529 pause.accessibilityLabel = resume ? (playing ? "Play" : "Resume") : "Pause"
530 }
531
532 /// Saving: only the clock's place says so.
533 func showSaving(_ text: String) {
534 show(text, recording: false, paused: nil)
535 stop.isHidden = true
536 }
537
538 /// `ms` as a clock: minutes and seconds, hours when there are some.
539 static func clock(_ ms: Int64) -> String {
540 let seconds = ms / 1000
541 return seconds >= 3600
542 ? String(format: "%d:%02d:%02d", seconds / 3600, seconds / 60 % 60, seconds % 60)
543 : String(format: "%d:%02d", seconds / 60, seconds % 60)
544 }
545}