1import AppKit
2
3// Clover-recorder transcript integration: when a `cam.mov` has a sibling
4// `transcript.json` (the format the recorder writes), the viewer shows
5// synchronized two-line karaoke captions over the picture.
6//
7// All transcript times are in the media's OWN (source) timebase — seconds from
8// the start of cam.mov — so a clip's `sourceTime(at:)` maps the playhead into
9// them, respecting trim (srcIn) and speed.
10
11/// One spoken word with its timing (source seconds) and which segment
12/// (sentence) it came from — segment changes force a line break so sentences
13/// don't run together.
14struct TranscriptWord {
15 var text: String
16 var start: Double
17 var end: Double
18 var seg: Int
19}
20
21/// A parsed clover `transcript.json`: words flattened across segments in time
22/// order. `key` is the file path — its identity for layout caching.
23struct Transcript {
24 var words: [TranscriptWord]
25 var key: String
26
27 // MARK: Decoding shapes (tolerant — unknown/missing fields are skipped)
28
29 private struct RawWord: Decodable { var word: String?; var start: Double?; var end: Double? }
30 private struct RawSeg: Decodable {
31 var start: Double?; var end: Double?; var text: String?; var words: [RawWord]?
32 }
33 private struct RawTranscript: Decodable { var segments: [RawSeg]? }
34
35 /// Parse raw JSON. Returns nil if there's nothing usable to show.
36 static func parse(_ data: Data, key: String) -> Transcript? {
37 guard let raw = try? JSONDecoder().decode(RawTranscript.self, from: data),
38 let segments = raw.segments else { return nil }
39 var words: [TranscriptWord] = []
40 for (si, seg) in segments.enumerated() {
41 let wordList = seg.words ?? []
42 var added = false
43 for rw in wordList {
44 guard let start = rw.start else { continue }
45 let text = (rw.word ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
46 guard !text.isEmpty else { continue }
47 words.append(TranscriptWord(text: text, start: start,
48 end: max(start, rw.end ?? start), seg: si))
49 added = true
50 }
51 // A segment with no per-word timing still shows as one block spanning
52 // the segment, so nothing goes silently missing.
53 if !added, let start = seg.start {
54 let text = (seg.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
55 if !text.isEmpty {
56 words.append(TranscriptWord(text: text, start: start,
57 end: max(start, seg.end ?? start), seg: si))
58 }
59 }
60 }
61 guard !words.isEmpty else { return nil }
62 // Recorder output is monotonic in start time; sort defensively (stable,
63 // so an already-sorted file keeps its segment contiguity) so the
64 // active-word binary search below is always valid.
65 words.sort { $0.start < $1.start }
66 return Transcript(words: words, key: key)
67 }
68
69 // MARK: Sibling-file loading (cached)
70
71 private static var cache: [String: Transcript?] = [:]
72
73 /// The transcript for a media file, or nil unless it is named `cam.mov` and
74 /// has a readable `transcript.json` in the same folder. Cached by path so
75 /// the per-frame visibility probe is cheap.
76 static func load(forVideo url: URL) -> Transcript? {
77 guard url.lastPathComponent.lowercased() == "cam.mov" else { return nil }
78 let path = url.deletingLastPathComponent()
79 .appendingPathComponent("transcript.json").path
80 if let hit = cache[path] { return hit }
81 let parsed = (try? Data(contentsOf: URL(fileURLWithPath: path)))
82 .flatMap { parse($0, key: path) }
83 cache[path] = parsed
84 return parsed
85 }
86
87 /// Drop cached parses so a re-imported/edited transcript is re-read.
88 static func clearCache() { cache.removeAll() }
89
90 // MARK: Lookup
91
92 /// Index of the word active at source time `s` — the last word that has
93 /// started. A word stays "current" until the next one begins, so there are
94 /// no gaps. Returns -1 before the first word.
95 func activeIndex(at s: Double) -> Int {
96 var lo = 0, hi = words.count - 1, res = -1
97 while lo <= hi {
98 let mid = (lo + hi) / 2
99 if words[mid].start <= s { res = mid; lo = mid + 1 } else { hi = mid - 1 }
100 }
101 return res
102 }
103}
104
105/// Two-line synchronized captions drawn over the viewer: white text on a black
106/// clipped background, the active word in magenta, always on the TOP line.
107/// New lines rise from the bottom and leave above the top as speech advances.
108///
109/// **The whole thing is a pure function of the playhead.** Nothing here uses a
110/// timer, Core Animation, or wall-clock tween: the vertical scroll offset is
111/// computed straight from the source time, so stepping frame-by-frame steps the
112/// animation and seeking lands with no motion (exactly what was asked for).
113final class SubtitleOverlay: NSView {
114 var ctx: DocumentContext = .headless {
115 didSet {
116 guard oldValue !== ctx else { return }
117 oldValue.notify.removeObserver(self, name: .playheadChanged, object: nil)
118 ctx.notify.addObserver(self, selector: #selector(sync),
119 name: .playheadChanged, object: nil)
120 sync()
121 }
122 }
123 private var store: Store { ctx.store }
124 private var playback: PlaybackController { ctx.playback }
125 private var session: SessionState { ctx.session }
126
127 /// The transcript being shown and the source time to render it at — both
128 /// recomputed on every playhead move in `sync()`.
129 private var transcript: Transcript?
130 private var sourceTime: Double = 0
131 /// The clip the transcript is being shown for — kept so a word click can map
132 /// the word's source time back to a timeline moment (inverse of the scroll).
133 private var clip: Clip?
134
135 /// Laid-out lines (each an array of positioned words) cached per transcript
136 /// + box width. `lineOfWord[i]` is the line word `i` landed on.
137 private struct LaidWord { var index: Int; var text: String; var x: CGFloat; var width: CGFloat }
138 private var lines: [[LaidWord]] = []
139 private var lineOfWord: [Int] = []
140 /// Natural content width (points) of each laid-out line — drives the black
141 /// backing that hugs the visible two lines.
142 private var lineWidths: [CGFloat] = []
143 private var laidKey: String?
144 private var laidWidth: CGFloat = -1
145
146 /// What to mark at the current source time: a spoken word painted magenta,
147 /// or — during a real pause between two words — a magenta caret sitting
148 /// after the last spoken word, signalling that nothing is being said.
149 private enum Mark { case word(Int); case caret(after: Int); case none }
150
151 /// Gaps up to this (seconds) hold the previous word lit instead of blinking
152 /// off; longer gaps show the caret. Small inter-word silences are noise and
153 /// shouldn't flicker the highlight.
154 private static let extendGap = 0.25
155
156 private static let magenta = NSColor(srgbRed: 1.0, green: 0.22, blue: 0.86, alpha: 1)
157
158 override init(frame: NSRect) {
159 super.init(frame: frame)
160 wantsLayer = true
161 layer?.backgroundColor = NSColor.clear.cgColor
162 isHidden = true
163 NotificationCenter.default.addObserver(self, selector: #selector(sync),
164 name: .projectChanged, object: nil)
165 NotificationCenter.default.addObserver(self, selector: #selector(sync),
166 name: .viewOptionsChanged, object: nil)
167 NotificationCenter.default.addObserver(self, selector: #selector(sync),
168 name: .viewerNeedsRefresh, object: nil)
169 ctx.notify.addObserver(self, selector: #selector(sync),
170 name: .playheadChanged, object: nil)
171 }
172 required init?(coder: NSCoder) { fatalError() }
173
174 override var isFlipped: Bool { true } // y grows downward: top line at small y
175
176 // Swallow clicks only when they land on a word — a word click scrubs the
177 // playhead to that word (see `mouseDown`). Clicks on the box's gaps/padding
178 // pass through to the cell beneath so clip selection still works there.
179 override func hitTest(_ point: NSPoint) -> NSView? {
180 guard !isHidden, transcript != nil else { return nil }
181 return wordIndex(atLocal: convert(point, from: superview)) != nil ? self : nil
182 }
183
184 /// Scrub to the clicked word's start.
185 override func mouseDown(with event: NSEvent) {
186 let local = convert(event.locationInWindow, from: nil)
187 guard let transcript, let clip,
188 let idx = wordIndex(atLocal: local) else { return }
189 playback.seek(to: clip.timelineTime(forSource: transcript.words[idx].start))
190 }
191
192 /// The transcript word whose drawn rect contains `p` (this view's flipped,
193 /// local coords), or nil. Mirrors `draw`'s layout so hit-testing and painting
194 /// never disagree.
195 private func wordIndex(atLocal p: NSPoint) -> Int? {
196 guard transcript != nil, bounds.width > 40 else { return nil }
197 rebuildLayoutIfNeeded()
198 guard !lines.isEmpty else { return nil }
199 let m = Metrics(width: bounds.width)
200 // Only the visible two-line band is clickable (matches what's drawn).
201 guard p.y >= m.vpad, p.y <= bounds.height - m.vpad else { return nil }
202 let scroll = scrollLines(at: sourceTime)
203 for (li, line) in lines.enumerated() {
204 let topY = m.vpad + (CGFloat(li) - scroll) * m.lineH
205 guard p.y >= topY, p.y < topY + m.lineH else { continue }
206 for lw in line where p.x >= lw.x && p.x < lw.x + lw.width {
207 return lw.index
208 }
209 }
210 return nil
211 }
212
213 // MARK: Geometry
214
215 /// Box metrics for a given width. Font scales gently with the box so the
216 /// captions stay legible on small viewers without ballooning on large ones.
217 struct Metrics {
218 let fontSize, lineH, vpad, hpad, height: CGFloat
219 init(width: CGFloat) {
220 fontSize = min(26, max(14, width / 30))
221 lineH = (fontSize * 1.32).rounded(.up)
222 vpad = (lineH * 0.34).rounded()
223 hpad = 16
224 height = lineH * 2 + vpad * 2
225 }
226 }
227
228 private static func font(_ size: CGFloat) -> NSFont {
229 .systemFont(ofSize: size, weight: .semibold)
230 }
231
232 /// Where the caption box sits inside the viewer: centred horizontally, near
233 /// the bottom. Sized only from the viewer bounds, so the parent can position
234 /// it in `layout()` without knowing the content.
235 func preferredFrame(in bounds: NSRect) -> NSRect {
236 let w = min(max(bounds.width * 0.72, 260), 860)
237 let m = Metrics(width: w)
238 let x = ((bounds.width - w) / 2).rounded()
239 let margin = max(16, bounds.height * 0.045)
240 let y = max(m.vpad, (bounds.height - m.height - margin).rounded())
241 return NSRect(x: x, y: y, width: w, height: m.height)
242 }
243
244 // MARK: State
245
246 /// Recompute which transcript/clip is under the playhead and at what source
247 /// time, then redraw. Hidden whenever the toggle is off or nothing under the
248 /// playhead carries a transcript.
249 @objc func sync() {
250 guard session.subtitlesEnabled, let (clip, media) = transcriptClipUnderPlayhead() else {
251 if !isHidden { isHidden = true }
252 transcript = nil
253 self.clip = nil
254 return
255 }
256 transcript = Transcript.load(forVideo: media.url)
257 self.clip = clip
258 sourceTime = max(0, clip.sourceTime(at: playback.playhead))
259 isHidden = transcript == nil
260 needsDisplay = true
261 }
262
263 /// The visible video clip under the playhead whose media is a transcript-
264 /// bearing `cam.mov`. Mirrors the viewer's own hide/focus visibility rule;
265 /// the Priority pane wins when it qualifies.
266 private func transcriptClipUnderPlayhead() -> (Clip, MediaItem)? {
267 let project = store.project
268 let t = playback.playhead
269 let focusActive = !session.focusedTracks.isEmpty || session.fusionFocus
270 func visible(_ ref: TrackRef) -> Bool {
271 focusActive ? session.focusedTracks.contains(ref)
272 : !session.hiddenTracks.contains(ref)
273 }
274 var found: (TrackRef, Clip, MediaItem)?
275 for ref in project.laneRefs where visible(ref) {
276 guard let clip = project.clipAt(track: ref, time: t, kind: .video),
277 let m = project.media(clip.mediaId),
278 Transcript.load(forVideo: m.url) != nil else { continue }
279 if ref == session.priorityPane { return (clip, m) }
280 if found == nil { found = (ref, clip, m) }
281 }
282 return found.map { ($0.1, $0.2) }
283 }
284
285 // MARK: Layout
286
287 /// Wrap the transcript into center-aligned lines that fit the box, breaking
288 /// at segment (sentence) boundaries. Cached until the box width or the
289 /// transcript changes.
290 private func rebuildLayoutIfNeeded() {
291 guard let transcript else { lines = []; lineOfWord = []; lineWidths = []; return }
292 let m = Metrics(width: bounds.width)
293 if laidKey == transcript.key && abs(laidWidth - bounds.width) < 0.5 { return }
294 laidKey = transcript.key
295 laidWidth = bounds.width
296
297 let font = Self.font(m.fontSize)
298 let attrs: [NSAttributedString.Key: Any] = [.font: font]
299 func measure(_ s: String) -> CGFloat { (s as NSString).size(withAttributes: attrs).width }
300 let space = measure(" ")
301 let maxW = bounds.width - m.hpad * 2
302
303 var built: [[LaidWord]] = []
304 var widths: [CGFloat] = []
305 var lineIdx = [Int](repeating: 0, count: transcript.words.count)
306 var cur: [LaidWord] = []
307 var penX: CGFloat = 0
308
309 func flush() {
310 guard !cur.isEmpty else { return }
311 let lineW = (cur.last?.x ?? 0) + (cur.last?.width ?? 0)
312 let off = (m.hpad + max(0, (maxW - lineW) / 2)).rounded()
313 built.append(cur.map { LaidWord(index: $0.index, text: $0.text,
314 x: $0.x + off, width: $0.width) })
315 widths.append(lineW)
316 cur = []
317 penX = 0
318 }
319
320 for (gi, w) in transcript.words.enumerated() {
321 let width = measure(w.text)
322 let newSegment = gi > 0 && w.seg != transcript.words[gi - 1].seg
323 let x = cur.isEmpty ? 0 : penX + space
324 if newSegment || (!cur.isEmpty && x + width > maxW) {
325 flush()
326 }
327 let placeX = cur.isEmpty ? 0 : penX + space
328 cur.append(LaidWord(index: gi, text: w.text, x: placeX, width: width))
329 penX = placeX + width
330 lineIdx[gi] = built.count // the line this word will land on once flushed
331 }
332 flush()
333 lines = built
334 lineOfWord = lineIdx
335 lineWidths = widths
336 }
337
338 /// Content width (points) to draw the black backing at for a given vertical
339 /// scroll. At rest on line `k` this is the wider of the two visible lines
340 /// (`k` and `k+1`); mid-slide it blends the outgoing and incoming pairs, so
341 /// the box width eases in lockstep with the scroll — same playhead-driven,
342 /// frame-steppable motion, no Core Animation.
343 private func contentWidth(atScroll s: CGFloat) -> CGFloat {
344 guard !lineWidths.isEmpty else { return 0 }
345 func pair(_ k: Int) -> CGFloat {
346 let a = (k >= 0 && k < lineWidths.count) ? lineWidths[k] : 0
347 let b = (k + 1 >= 0 && k + 1 < lineWidths.count) ? lineWidths[k + 1] : 0
348 return max(a, b)
349 }
350 let k = Int(s.rounded(.down))
351 let f = s - CGFloat(k)
352 return pair(k) + (pair(k + 1) - pair(k)) * f
353 }
354
355 /// Continuous vertical scroll, in line units, as a pure function of source
356 /// time. Holds on the active word's line, then slides up over ~0.3s when a
357 /// new line begins — so the focused word slides into the top row and stays
358 /// there. Interpolating over time (not a CA animation) is what makes it
359 /// frame-steppable and animation-free on a seek.
360 private func scrollLines(at s: Double) -> CGFloat {
361 guard let transcript, !lineOfWord.isEmpty else { return 0 }
362 let i = transcript.activeIndex(at: s)
363 guard i >= 0 else { return 0 }
364 let cur = lineOfWord[i]
365 let prev = i > 0 ? lineOfWord[i - 1] : cur
366 if cur == prev { return CGFloat(cur) }
367 let startI = transcript.words[i].start
368 let nextStart = i + 1 < transcript.words.count
369 ? transcript.words[i + 1].start : transcript.words[i].end
370 let slide = min(0.30, max(0.0001, nextStart - startI))
371 let t = min(1, max(0, (s - startI) / slide))
372 let e = t * t * (3 - 2 * t) // smoothstep
373 return CGFloat(prev) + (CGFloat(cur) - CGFloat(prev)) * e
374 }
375
376 /// Resolve what to highlight at source time `s`. While a word is being
377 /// spoken it lights up. In the gap after it: a *small* gap holds the word lit
378 /// right up to the next one (so brief silences don't flicker); a *longer* gap
379 /// shows a caret between the two words, marking the pause. Trailing silence
380 /// after the final word fades to nothing.
381 private func mark(at s: Double) -> Mark {
382 guard let transcript else { return .none }
383 let i = transcript.activeIndex(at: s)
384 guard i >= 0 else { return .none }
385 let w = transcript.words[i]
386 if s <= w.end { return .word(i) } // still being spoken
387 if i + 1 < transcript.words.count {
388 let gap = transcript.words[i + 1].start - w.end
389 return gap <= Self.extendGap ? .word(i) // tiny gap: extend it
390 : .caret(after: i) // real pause: caret
391 }
392 return s <= w.end + 0.05 ? .word(i) : .none // trailing silence
393 }
394
395 // MARK: Draw
396
397 override func draw(_ dirty: NSRect) {
398 guard transcript != nil, bounds.width > 40 else { return }
399 rebuildLayoutIfNeeded()
400 guard !lines.isEmpty else { return }
401 let m = Metrics(width: bounds.width)
402
403 let font = Self.font(m.fontSize)
404 let scroll = scrollLines(at: sourceTime)
405 let markResult = mark(at: sourceTime)
406 var highlight = -1
407 if case .word(let idx) = markResult { highlight = idx }
408
409 // Black rounded backing, sized to hug the two visible lines and centred
410 // in the (fixed, transparent) container — width eases with the scroll.
411 let boxW = min(bounds.width, (contentWidth(atScroll: scroll) + m.hpad * 2).rounded())
412 let boxX = ((bounds.width - boxW) / 2).rounded()
413 let boxRect = NSRect(x: boxX, y: 0, width: boxW, height: bounds.height)
414 let bg = NSBezierPath(roundedRect: boxRect, xRadius: 10, yRadius: 10)
415 NSColor(calibratedWhite: 0, alpha: 0.8).setFill()
416 bg.fill()
417 NSGraphicsContext.saveGraphicsState()
418 bg.addClip()
419 // Clip text to the inner TWO-line band (inset by the vertical padding),
420 // not the full padded box. This is what keeps the focused word pinned to
421 // the top line: without it, a settled neighbour line's descenders bleed
422 // into the top/bottom padding and the active line reads as the 2nd row.
423 // Lines still animate through this band — they're simply cut off cleanly
424 // at its top/bottom edges as they rise away / come up from below. The
425 // rounded-box clip above also trims lines sliding through a narrower box.
426 NSBezierPath(rect: NSRect(x: 0, y: m.vpad,
427 width: bounds.width,
428 height: bounds.height - m.vpad * 2)).addClip()
429
430 // Vertical inset so the glyphs sit centred in their line box.
431 let glyphH = font.ascender - font.descender
432 let textInset = ((m.lineH - glyphH) / 2).rounded()
433
434 // A caret (magenta cursor) sits just after the last spoken word during a
435 // real pause. Resolve its line/x from that word's laid-out position.
436 var caretLine = -1
437 var caretX: CGFloat = 0
438 if case .caret(let after) = markResult, after < lineOfWord.count {
439 let li = lineOfWord[after]
440 if li < lines.count, let prev = lines[li].first(where: { $0.index == after }) {
441 caretLine = li
442 let rightEdge = prev.x + prev.width
443 // Centre the caret in the gap to the next word when it shares
444 // this line; if the next word wrapped away, sit just past this one.
445 if let next = lines[li].first(where: { $0.index == after + 1 }) {
446 caretX = ((rightEdge + next.x) / 2).rounded()
447 } else {
448 caretX = (rightEdge + 3).rounded()
449 }
450 }
451 }
452
453 for (li, line) in lines.enumerated() {
454 let topY = m.vpad + (CGFloat(li) - scroll) * m.lineH
455 if topY > bounds.height || topY + m.lineH < 0 { continue } // fully clipped
456 let baselineY = topY + textInset
457 for lw in line {
458 let color = lw.index == highlight ? Self.magenta : NSColor.white
459 let attrs: [NSAttributedString.Key: Any] = [.font: font, .foregroundColor: color]
460 (lw.text as NSString).draw(at: NSPoint(x: lw.x, y: baselineY), withAttributes: attrs)
461 }
462 if li == caretLine {
463 Self.magenta.setFill()
464 NSBezierPath(rect: NSRect(x: caretX - 1, y: baselineY, width: 2, height: glyphH)).fill()
465 }
466 }
467 NSGraphicsContext.restoreGraphicsState()
468 }
469}