1//! Video recordings as AVI files of Motion JPEG pictures and PCM sound, which OneNote 2010
2//! plays and lists as recordings (`corpus/recording/video`), and reading them back to play.
3
4use std::ops::Range;
5
6/// Pictures a second in a recording: OneNote 2010's own profiles record at 15.
7pub const FPS: u32 = 15;
8/// A recording's picture size, OneNote 2010's own.
9pub const SIZE: [u32; 2] = [320, 240];
10
11/// Interleaved signed 16-bit little-endian sound.
12pub struct Pcm {
13 pub rate: u32,
14 pub channels: u16,
15 pub data: Vec<u8>,
16}
17
18fn chunk(out: &mut Vec<u8>, id: &[u8; 4], body: &[u8]) {
19 out.extend_from_slice(id);
20 out.extend_from_slice(&(body.len() as u32).to_le_bytes());
21 out.extend_from_slice(body);
22 if body.len() % 2 == 1 {
23 out.push(0);
24 }
25}
26
27fn list(kind: &[u8; 4], body: &[u8]) -> Vec<u8> {
28 let mut out = Vec::with_capacity(body.len() + 12);
29 chunk(&mut out, b"LIST", &[&kind[..], body].concat());
30 out
31}
32
33fn words(values: &[u32]) -> Vec<u8> {
34 values
35 .iter()
36 .flat_map(|value| value.to_le_bytes())
37 .collect()
38}
39
40/// An AVI file of `frames`, JPEG pictures of `size` shown `FPS` a second, where an empty one
41/// repeats the picture before it, and `sound` alongside. None past AVI's 4 GiB.
42pub fn write(size: [u32; 2], frames: &[Vec<u8>], sound: &Pcm) -> Option<Vec<u8>> {
43 let block = u32::from(sound.channels) * 2;
44 let bytes_per_second = sound.rate * block;
45 let samples = sound.data.len() as u64 / u64::from(block);
46 let count = frames.len() as u64;
47 // The sound up to frame `index`, whole samples.
48 let sound_at = |index: u64| -> usize {
49 let before = if index >= count {
50 samples
51 } else {
52 (index * u64::from(sound.rate) / u64::from(FPS)).min(samples)
53 };
54 (before * u64::from(block)) as usize
55 };
56 let mut movi = b"movi".to_vec();
57 let mut index = Vec::new();
58 let [mut largest_frame, mut largest_sound] = [0, 0];
59 for (at, frame) in frames.iter().enumerate() {
60 let entries: [(&[u8; 4], &[u8]); 2] = [
61 (b"00dc", frame),
62 (
63 b"01wb",
64 &sound.data[sound_at(at as u64)..sound_at(at as u64 + 1)],
65 ),
66 ];
67 for (id, body) in entries {
68 if id == b"01wb" && body.is_empty() {
69 continue;
70 }
71 // Keyframes: every picture and all sound; a repeat is none.
72 let flags: u32 = if body.is_empty() { 0 } else { 0x10 };
73 index.extend_from_slice(id);
74 index.extend(words(&[
75 flags,
76 u32::try_from(movi.len()).ok()?,
77 body.len() as u32,
78 ]));
79 chunk(&mut movi, id, body);
80 let largest = if id == b"00dc" {
81 &mut largest_frame
82 } else {
83 &mut largest_sound
84 };
85 *largest = (*largest).max(body.len() as u32);
86 }
87 }
88 let [width, height] = size;
89 let frame_count = u32::try_from(count).ok()?;
90 let rect = [0u16, 0, width as u16, height as u16]
91 .iter()
92 .flat_map(|side| side.to_le_bytes())
93 .collect::<Vec<u8>>();
94 let mut video = Vec::new();
95 let strh = [
96 &b"vidsMJPG"[..],
97 &words(&[0, 0, 0, 1, FPS, 0, frame_count, largest_frame, u32::MAX, 0]),
98 &rect,
99 ]
100 .concat();
101 chunk(&mut video, b"strh", &strh);
102 let strf = [
103 words(&[40, width, height]),
104 [&1u16.to_le_bytes()[..], &24u16.to_le_bytes()].concat(),
105 b"MJPG".to_vec(),
106 words(&[width * height * 3, 0, 0, 0, 0]),
107 ]
108 .concat();
109 chunk(&mut video, b"strf", &strf);
110 let mut streams = list(b"strl", &video);
111 if samples > 0 {
112 let mut audio = Vec::new();
113 let strh = [
114 &b"auds\0\0\0\0"[..],
115 &words(&[
116 0,
117 0,
118 0,
119 block,
120 bytes_per_second,
121 0,
122 u32::try_from(samples).ok()?,
123 largest_sound,
124 u32::MAX,
125 block,
126 ]),
127 &[0; 8],
128 ]
129 .concat();
130 chunk(&mut audio, b"strh", &strh);
131 let strf = [
132 &1u16.to_le_bytes()[..],
133 &sound.channels.to_le_bytes(),
134 &sound.rate.to_le_bytes(),
135 &bytes_per_second.to_le_bytes(),
136 &(block as u16).to_le_bytes(),
137 &16u16.to_le_bytes(),
138 &0u16.to_le_bytes(),
139 ]
140 .concat();
141 chunk(&mut audio, b"strf", &strf);
142 streams.extend(list(b"strl", &audio));
143 }
144 let mut avih = Vec::new();
145 chunk(
146 &mut avih,
147 b"avih",
148 &words(&[
149 1_000_000 / FPS,
150 largest_frame
151 .saturating_add(largest_sound)
152 .saturating_mul(FPS),
153 0,
154 // AVIF_HASINDEX | AVIF_ISINTERLEAVED
155 0x110,
156 frame_count,
157 0,
158 if samples > 0 { 2 } else { 1 },
159 largest_frame.max(largest_sound),
160 width,
161 height,
162 0,
163 0,
164 0,
165 0,
166 ]),
167 );
168 let header = list(b"hdrl", &[avih, streams].concat());
169 let mut body = b"AVI ".to_vec();
170 body.extend(header);
171 let movi_size = u32::try_from(movi.len()).ok()?;
172 body.extend_from_slice(b"LIST");
173 body.extend_from_slice(&movi_size.to_le_bytes());
174 body.extend(movi);
175 chunk(&mut body, b"idx1", &index);
176 u32::try_from(body.len()).ok()?;
177 let mut out = Vec::with_capacity(body.len() + 8);
178 chunk(&mut out, b"RIFF", &body);
179 Some(out)
180}
181
182/// An AVI file's pictures and sound, as byte ranges of the file.
183pub struct Movie {
184 /// Each picture's JPEG; an empty range repeats the one before.
185 pub frames: Vec<Range<usize>>,
186 pub frame_us: u32,
187 /// The sound's WAVEFORMATEX and its data, in order.
188 pub sound: Option<(Range<usize>, Vec<Range<usize>>)>,
189}
190
191impl Movie {
192 /// A Motion JPEG AVI file's contents; none for other files.
193 pub fn parse(bytes: &[u8]) -> Option<Self> {
194 let mut movie = Self {
195 frames: Vec::new(),
196 frame_us: 0,
197 sound: None,
198 };
199 let mut kinds = Vec::new();
200 let mut first = true;
201 for (id, body) in Chunks::new(bytes, 0..bytes.len()) {
202 let kind = bytes.get(body.start..body.start + 4)?;
203 if id != *b"RIFF" || (kind != b"AVI " && !(kind == b"AVIX" && !first)) {
204 return (!first).then_some(movie).filter(|movie| movie.frame_us > 0);
205 }
206 first = false;
207 movie.walk(bytes, body.start + 4..body.end, &mut kinds)?;
208 }
209 (movie.frame_us > 0).then_some(movie)
210 }
211
212 fn walk(&mut self, bytes: &[u8], range: Range<usize>, kinds: &mut Vec<[u8; 4]>) -> Option<()> {
213 for (id, body) in Chunks::new(bytes, range) {
214 let u32_at = |at: usize| {
215 Some(u32::from_le_bytes(
216 bytes
217 .get(body.start + at..body.start + at + 4)?
218 .try_into()
219 .ok()?,
220 ))
221 };
222 match &id {
223 b"LIST" => {
224 let inner = body.start + 4..body.end;
225 self.walk(bytes, inner, kinds)?;
226 }
227 b"strh" => {
228 let kind: [u8; 4] = bytes.get(body.start..body.start + 4)?.try_into().ok()?;
229 if &kind == b"vids" {
230 let [scale, rate] = [u32_at(20)?, u32_at(24)?];
231 self.frame_us =
232 u32::try_from(u64::from(scale) * 1_000_000 / u64::from(rate.max(1)))
233 .ok()?;
234 }
235 kinds.push(kind);
236 }
237 b"strf" if kinds.last() == Some(b"vids") => {
238 if bytes.get(body.start + 16..body.start + 20)? != b"MJPG" {
239 return None;
240 }
241 }
242 b"strf" if kinds.last() == Some(b"auds") => {
243 self.sound = Some((body.clone(), Vec::new()));
244 }
245 [a, b, c, d] if a.is_ascii_digit() && b.is_ascii_digit() => {
246 let stream = usize::from((a - b'0') * 10 + (b - b'0'));
247 match (kinds.get(stream).map(|kind| &kind[..]), [c, d]) {
248 (Some(b"vids"), [b'd', b'c' | b'b']) => self.frames.push(body),
249 (Some(b"auds"), [b'w', b'b']) => {
250 if let Some((_, data)) = &mut self.sound {
251 data.push(body);
252 }
253 }
254 _ => {}
255 }
256 }
257 _ => {}
258 }
259 }
260 Some(())
261 }
262
263 pub fn duration_ms(&self) -> u32 {
264 let ms = (self.frames.len() as u64 * u64::from(self.frame_us)).div_ceil(1000);
265 u32::try_from(ms).unwrap_or(u32::MAX)
266 }
267
268 /// The picture showing `at_ms` in: the last stored one at or before it.
269 pub fn frame(&self, at_ms: u32) -> Option<Range<usize>> {
270 let at = (u64::from(at_ms) * 1000 / u64::from(self.frame_us.max(1))) as usize;
271 self.frames[..=at.min(self.frames.len().checked_sub(1)?)]
272 .iter()
273 .rev()
274 .find(|frame| !frame.is_empty())
275 .cloned()
276 }
277
278 /// The sound as a WAV file, or none where the file has none.
279 pub fn wave(&self, bytes: &[u8]) -> Option<Vec<u8>> {
280 let (format, data) = self.sound.as_ref()?;
281 let format = &bytes[format.clone()];
282 let length: usize = data.iter().map(ExactSizeIterator::len).sum();
283 let mut out = Vec::with_capacity(length + 64);
284 out.extend_from_slice(b"RIFF");
285 out.extend_from_slice(&((4 + 8 + format.len() + 8 + length) as u32).to_le_bytes());
286 out.extend_from_slice(b"WAVE");
287 chunk(&mut out, b"fmt ", format);
288 out.extend_from_slice(b"data");
289 out.extend_from_slice(&(length as u32).to_le_bytes());
290 for range in data {
291 out.extend_from_slice(&bytes[range.clone()]);
292 }
293 Some(out)
294 }
295}
296
297/// The chunks in `range` of a RIFF file: each one's id and body.
298struct Chunks<'a> {
299 bytes: &'a [u8],
300 at: usize,
301 end: usize,
302}
303
304impl<'a> Chunks<'a> {
305 fn new(bytes: &'a [u8], range: Range<usize>) -> Self {
306 Self {
307 bytes,
308 at: range.start,
309 end: range.end.min(bytes.len()),
310 }
311 }
312}
313
314impl Iterator for Chunks<'_> {
315 type Item = ([u8; 4], Range<usize>);
316
317 fn next(&mut self) -> Option<Self::Item> {
318 let header = self.bytes.get(self.at..self.at + 8)?;
319 if self.at + 8 > self.end {
320 return None;
321 }
322 let id: [u8; 4] = header[..4].try_into().ok()?;
323 let size = u32::from_le_bytes(header[4..].try_into().ok()?) as usize;
324 let start = self.at + 8;
325 let end = start.saturating_add(size).min(self.end);
326 self.at = end + size % 2;
327 Some((id, start..end))
328 }
329}
330
331/// Pictures taken at uneven moments, as a camera takes them, made the `FPS` a second of a
332/// recording: each one shows until the next, fitted to `SIZE`.
333#[derive(Default)]
334pub struct Pictures {
335 frames: Vec<Vec<u8>>,
336 last: Option<image::RgbImage>,
337 /// Whether the last picture is stored yet, which the frames after it repeat.
338 stored: bool,
339}
340
341impl Pictures {
342 /// The picture taken `at_us` microseconds in; the first shows from the start.
343 pub fn take(&mut self, at_us: u64, picture: image::RgbImage) {
344 self.fill(at_us);
345 self.last = Some(fit(picture));
346 self.stored = false;
347 }
348
349 /// A picture as Core Video gives one: `height` rows of `stride` bytes of BGRA pixels.
350 pub fn take_bgra(
351 &mut self,
352 at_us: u64,
353 [width, height]: [usize; 2],
354 stride: usize,
355 bgra: &[u8],
356 ) {
357 let rgb = bgra
358 .chunks_exact(stride.max(1))
359 .take(height)
360 .filter_map(|row| row.get(..width * 4))
361 .flat_map(|row| {
362 row.chunks_exact(4)
363 .flat_map(|pixel| [pixel[2], pixel[1], pixel[0]])
364 })
365 .collect();
366 if let Some(picture) = image::RgbImage::from_raw(width as u32, height as u32, rgb) {
367 self.take(at_us, picture);
368 }
369 }
370
371 /// The frames of a recording `end_us` microseconds long.
372 pub fn finish(mut self, end_us: u64) -> Vec<Vec<u8>> {
373 self.fill(end_us);
374 self.frames
375 }
376
377 /// Frames up to `until_us`, of the last picture.
378 fn fill(&mut self, until_us: u64) {
379 let Some(last) = &self.last else {
380 return;
381 };
382 while (self.frames.len() as u64) * 1_000_000 / u64::from(FPS) < until_us {
383 let frame = if self.stored {
384 Vec::new()
385 } else {
386 self.stored = true;
387 let mut jpeg = Vec::new();
388 let encoded = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 70)
389 .encode_image(last);
390 if encoded.is_err() {
391 jpeg.clear();
392 }
393 jpeg
394 };
395 self.frames.push(frame);
396 }
397 }
398}
399
400/// `picture` scaled to fit `SIZE`, centred on black.
401fn fit(picture: image::RgbImage) -> image::RgbImage {
402 let [width, height] = SIZE;
403 if picture.dimensions() == (width, height) {
404 return picture;
405 }
406 let scale =
407 (width as f32 / picture.width() as f32).min(height as f32 / picture.height() as f32);
408 let fitted = image::imageops::resize(
409 &picture,
410 ((picture.width() as f32 * scale) as u32).clamp(1, width),
411 ((picture.height() as f32 * scale) as u32).clamp(1, height),
412 image::imageops::FilterType::Triangle,
413 );
414 let mut out = image::RgbImage::new(width, height);
415 image::imageops::overlay(
416 &mut out,
417 &fitted,
418 i64::from((width - fitted.width()) / 2),
419 i64::from((height - fitted.height()) / 2),
420 );
421 out
422}