magic-lantern

1import type { EngineInterface, Register } from 'claude-code'
2
3const TOOL = 'mcp__magic-lantern__project'
4const PANE = 'magic-lantern'
5
6const CANVAS = { width: 1280, height: 720 }
7const RENDER_SCRIPT_BUDGET_MS = 3000
8// A motion frame's pictures are captured in sheets: one page stacking up to
9// SHEET_PICTURES copies of the frame, each paused at its own moment, shot by
10// one Chrome launch and cut into pictures by ffmpeg. Launching Chrome is what
11// costs, so a sheet holds as many pictures as Chrome screenshots in one go.
12const SHEET_PICTURES = 20
13const SHEET_MAX_HEIGHT = 16_000
14// A motion picture is swapped many times a second, so it is captured at a density that
15// keeps its RGBA pixels under 2 MiB: terminals drawn with the Iced UI library upload a
16// larger picture in the background and show nothing in its place until it lands, which
17// flickers on every swap. Stills, shown once, keep the full density.
18const MOTION_MAX_PIXELS = Math.floor((1.8 * 1024 * 1024) / 4)
19const SHEET_SCRIPT_BUDGET_MS = 1000
20const RENDER_TIMEOUT_MS = 60_000
21// Chrome runs this many renders at once.
22const RENDER_CONCURRENCY = 4
23
24// A terminal cell is about twice as tall as it is wide.
25const CELL_ASPECT = 2
26// The pane's layout: the picture centred with this many columns free on each side, and
27// under it, rows for a blank, the spoken line (up to three rows), a blank, the progress
28// segments, a blank and the controls: Pause and a button per frame while it plays (two
29// rows where a narrow pane wraps them), Replay and Close once it has ended.
30const PICTURE_MARGIN_COLUMNS = 2
31const ROWS_UNDER_PICTURE = 10
32// One progress segment per frame, at most this many cells long; a single frame gets one
33// longer bar, which reads as the clip's own progress.
34const SEGMENT_MAX_CELLS = 10
35const SINGLE_BAR_CELLS = 40
36const ACCENT = '#f0a24a'
37
38// Playback advances on a clock tick, which also moves the current segment's fill; the
39// pause between one frame's line and the next frame is counted in ticks.
40const TICK_MS = 100
41const GAP_MS = 400
42// Where a line cannot play, a frame stays as long as the line takes to read.
43const WORDS_PER_SECOND = 2.5
44
45// The voice that reads a narration is the best one macOS has for its language,
46// which the speech framework knows and `say` does not: left to itself, `say`
47// speaks every language in the one system voice. A Premium or Enhanced voice the
48// person downloaded wins over the language's default, which is a compact one even
49// then; among voices of one quality the default stays, so a Mac with compact voices
50// only never speaks in a novelty voice. The voice goes to `say` by its identifier:
51// names repeat across languages, and `say` speaks a name it does not know in another
52// voice without failing. The language arrives as an argument, never inside the script.
53const DEFAULT_VOICE_SCRIPT = `function run(argv) {
54  ObjC.import('AVFoundation')
55  const language = argv[0].toLowerCase()
56  const fallback = $.AVSpeechSynthesisVoice.voiceWithLanguage(argv[0])
57  let best = fallback.isNil() ? undefined : fallback
58  const voices = $.AVSpeechSynthesisVoice.speechVoices
59  for (let i = 0; i < voices.count; i++) {
60    const voice = voices.objectAtIndex(i)
61    if (voice.language.js.toLowerCase() !== language) continue
62    if (!best || voice.quality > best.quality) best = voice
63  }
64  return best ? best.identifier.js : ''
65}`
66// Narration is spoken to a WAV file ahead of playback, so a line can be
67// stopped mid-sentence when the pane closes.
68const SPEECH_FORMAT = 'LEI16@22050'
69// That format's bytes a second (16-bit mono at 22050 Hz), and the WAV header `say`
70// writes: not the bare 44 bytes, but padded so the samples start at 4 KiB, whatever
71// the voice or the length of the line.
72const SPEECH_BYTES_PER_SECOND = 44_100
73const WAV_HEADER_BYTES = 4096
74
75// Appended to a motion frame's page: it pauses every animation at the moment
76// the URL's fragment names, in milliseconds (`#t=480`).
77const CAPTURE_SCRIPT = `<script>
78const at = Number(new URLSearchParams(location.hash.slice(1)).get('t'))
79for (const animation of document.getAnimations()) { animation.pause(); animation.currentTime = at }
80</script>`
81
82const CHROME_PATHS = [
83  '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
84  '/Applications/Chromium.app/Contents/MacOS/Chromium',
85]
86const CHROME_COMMANDS = ['google-chrome', 'google-chrome-stable', 'chromium', 'chromium-browser']
87const FFMPEG_COMMANDS = ['/opt/homebrew/bin/ffmpeg', '/usr/local/bin/ffmpeg', 'ffmpeg']
88
89const DESCRIPTION = `Project a short narrated slideshow in a pane beside the conversation: a few frames you compose as HTML, shown in order, each with one line a voice speaks while it is on screen. Load the magic-lantern skill first: it says when a projection fits.
90
91Each frame is a whole HTML document drawn at width x height CSS pixels (default 1280x720): one idea per frame, large type, few words on screen, since the narration carries the sentence. Keep each narration line to one or two spoken sentences. 3 to 8 frames.
92
93A frame can move: give it motion { seconds, fps } and animate its HTML with CSS animations or the Web Animations API, starting at time 0. The animation is captured at that frame rate and played once while its line is spoken, then holds its last picture.
94
95The call returns once the frames are rendered and playback has started; the pane closes by itself after the last frame, and closing it stops the voice.`
96
97type Motion = { seconds: number; fps: number }
98type Frame = { html: string; narration: string; motion?: Motion }
99type ExplainInput = { title: string; language: string; frames: Frame[]; width?: number; height?: number; voice?: string }
100
101const INPUT_SCHEMA = {
102  type: 'object',
103  required: ['title', 'language', 'frames'],
104  properties: {
105    title: { type: 'string', description: 'The pane title: what the projection is about, a few words.' },
106    language: { type: 'string', description: 'The language the narration is written in, as a BCP 47 tag (en-US, en-GB, it-IT); it picks the voice.' },
107    frames: {
108      type: 'array',
109      minItems: 1,
110      maxItems: 12,
111      items: {
112        type: 'object',
113        required: ['html', 'narration'],
114        properties: {
115          html: { type: 'string', description: 'A whole HTML document for this frame.' },
116          narration: { type: 'string', description: 'What the voice says while this frame is on screen, as plain text.' },
117          motion: {
118            type: 'object',
119            required: ['seconds', 'fps'],
120            description: "Play the frame's animation instead of a still picture.",
121            properties: {
122              seconds: { type: 'number', minimum: 0.1, maximum: 4, description: 'How long the animation runs.' },
123              fps: { type: 'integer', minimum: 1, maximum: 60, description: 'Pictures a second, captured and played.' },
124            },
125          },
126        },
127      },
128    },
129    width: { type: 'integer', minimum: 320, maximum: 2560, description: 'The width every frame is drawn in, in CSS pixels; default 1280.' },
130    height: { type: 'integer', minimum: 180, maximum: 2560, description: 'The height every frame is drawn in, in CSS pixels; default 720.' },
131    voice: { type: 'string', description: "A system voice by its exact name, in place of the language's own." },
132  },
133}
134
135/** How a motion frame played: what the terminal took, and how fast. */
136type ClipStats = {
137  frame: number
138  fps: number
139  pictures: number
140  captureMs: number
141  sent: number
142  taken: number
143  denied: number
144  firstDenial?: string
145  startedAt?: number
146  endedAt?: number
147  achievedFps?: number
148}
149
150type PlaybackFrame = {
151  path: string
152  narration: string
153  /** The spoken line as WAV bytes, base64; absent where `say` could not speak it. */
154  speech?: string
155  clip?: { paths: string[]; fps: number; stats: ClipStats }
156  /** How long the frame plays: its spoken line, or its clip where that runs longer. */
157  durationMs: number
158}
159
160type Playback = {
161  title: string
162  dir: string
163  canvas: { width: number; height: number }
164  frames: PlaybackFrame[]
165  index: number
166  /** The picture on screen: the frame's own, or the clip's latest. */
167  shown: string
168  // speaking: the line plays; moving: the clip plays; waiting: ticks left
169  // before the next frame.
170  isSpeaking: boolean
171  isMoving: boolean
172  waitTicks: number
173  /** When the current frame started, and how far through it playback is, 0 to 1. */
174  frameStartedAt: number
175  progress: number
176  /** The last frame has played: the pane holds it, with Replay and Close. */
177  isEnded: boolean
178  /** The pane is not drawn yet: the terminal is too narrow for a pane nobody asked for. */
179  isAwaitingOpen: boolean
180  /** Playback is manually paused; clock advances are skipped. */
181  isPaused: boolean
182  /** Aborted when the pane closes: stops the line being spoken. */
183  stop: AbortController
184}
185
186let playback: Playback | undefined
187
188const reason = (error: unknown) => (error instanceof Error ? error.message : String(error))
189
190const fileUrl = (path: string, fragment = '') => `file://${encodeURI(path)}${fragment}`
191
192let chrome: string | undefined
193const findChrome = async ($: EngineInterface) => {
194  if (chrome) return chrome
195  for (const path of CHROME_PATHS) {
196    if (await $.fs.exists(path)) return (chrome = path)
197  }
198  for (const command of CHROME_COMMANDS) {
199    const answered = await $.process.run([command, '--version']).catch(() => undefined)
200    if (answered?.exitCode === 0) return (chrome = command)
201  }
202  return undefined
203}
204
205let ffmpeg: string | undefined
206const findFfmpeg = async ($: EngineInterface) => {
207  if (ffmpeg) return ffmpeg
208  for (const command of FFMPEG_COMMANDS) {
209    const answered = await $.process.run([command, '-version']).catch(() => undefined)
210    if (answered?.exitCode === 0) return (ffmpeg = command)
211  }
212  return undefined
213}
214
215const sheetPage = (urls: string[], canvas: { width: number; height: number }) =>
216  `<!doctype html><meta charset="utf-8"><style>html,body{margin:0}iframe{display:block;border:0;width:${canvas.width}px;height:${canvas.height}px}</style>` +
217  urls.map(url => `<iframe src="${url}"></iframe>`).join('')
218
219const defaultVoice = async ($: EngineInterface, language: string) => {
220  const ran = await $.process
221    .run(['osascript', '-l', 'JavaScript', '-e', DEFAULT_VOICE_SCRIPT, language])
222    .catch(() => undefined)
223  const name = ran?.exitCode === 0 ? ran.stdout.trim() : ''
224  return name || undefined
225}
226
227const speakToFile = async ($: EngineInterface, voice: string, text: string, base: string) => {
228  await $.fs.write(`${base}.txt`, text)
229  const ran = await $.process
230    .run(['say', '-v', voice, '-f', `${base}.txt`, '-o', `${base}.wav`, `--data-format=${SPEECH_FORMAT}`], { timeoutMs: RENDER_TIMEOUT_MS })
231    .catch(() => undefined)
232  if (ran?.exitCode !== 0) return undefined
233  const read = await $.fs.read(`${base}.wav`, { as: 'bytes' }).catch(() => undefined)
234  return read?.base64
235}
236
237const render = async (
238  $: EngineInterface,
239  browser: string,
240  url: string,
241  pngPath: string,
242  canvas: Playback['canvas'],
243  budgetMs: number,
244  scale = 1,
245) => {
246  const ran = await $.process
247    .run(
248      [
249        browser,
250        '--headless',
251        '--disable-gpu',
252        '--hide-scrollbars',
253        `--window-size=${canvas.width},${canvas.height}`,
254        `--force-device-scale-factor=${scale}`,
255        `--virtual-time-budget=${budgetMs}`,
256        `--screenshot=${pngPath}`,
257        url,
258      ],
259      { timeoutMs: RENDER_TIMEOUT_MS },
260    )
261    .catch((error: unknown) => reason(error))
262  if (typeof ran === 'string') return ran
263  if (ran.exitCode !== 0) return `exit ${ran.exitCode}: ${ran.stderr.slice(-300)}`
264  return undefined
265}
266
267/**
268 * The density a motion frame is captured at: the largest whole number of pixel rows
269 * whose picture stays within MOTION_MAX_PIXELS, so a sheet of them cuts evenly.
270 */
271const motionScale = (canvas: Playback['canvas']) => {
272  const rows = Math.min(canvas.height, Math.floor(Math.sqrt((MOTION_MAX_PIXELS * canvas.height) / canvas.width)))
273  return rows / canvas.height
274}
275
276/** Shoots one sheet and cuts it into its pictures, numbered from `first`. */
277const renderSheet = async (
278  $: EngineInterface,
279  browser: string,
280  cutter: string,
281  sheet: { page: string; png: string; count: number; first: number; pattern: string },
282  canvas: Playback['canvas'],
283) => {
284  const sheetCanvas = { width: canvas.width, height: canvas.height * sheet.count }
285  const shot = await render($, browser, fileUrl(sheet.page), sheet.png, sheetCanvas, SHEET_SCRIPT_BUDGET_MS, motionScale(canvas))
286  if (shot) return shot
287  const cut = await $.process
288    .run([cutter, '-v', 'error', '-y', '-i', sheet.png, '-vf', `untile=1x${sheet.count}`, '-start_number', String(sheet.first), sheet.pattern], { timeoutMs: RENDER_TIMEOUT_MS })
289    .catch((error: unknown) => reason(error))
290  if (typeof cut === 'string') return cut
291  if (cut.exitCode !== 0) return `ffmpeg exit ${cut.exitCode}: ${cut.stderr.slice(-300)}`
292  return undefined
293}
294
295/** Runs the jobs at most `limit` at a time, answers in their order. */
296const pooled = async <T,>(jobs: (() => Promise<T>)[], limit: number) => {
297  const answers: T[] = new Array(jobs.length)
298  let nextJob = 0
299  const worker = async () => {
300    while (nextJob < jobs.length) {
301      const i = nextJob++
302      answers[i] = await jobs[i]!()
303    }
304  }
305  await Promise.all(Array.from({ length: Math.min(limit, jobs.length) }, worker))
306  return answers
307}
308
309const ticks = (ms: number) => Math.ceil(ms / TICK_MS)
310const readingTimeMs = (text: string) => (text.split(' ').length / WORDS_PER_SECOND) * 1000
311
312/** How long a WAV in SPEECH_FORMAT plays, from its base64 bytes. */
313const speechMs = (base64: string) => {
314  const padding = base64.endsWith('==') ? 2 : base64.endsWith('=') ? 1 : 0
315  const bytes = (base64.length * 3) / 4 - padding
316  return (Math.max(0, bytes - WAV_HEADER_BYTES) / SPEECH_BYTES_PER_SECOND) * 1000
317}
318
319/** Back to the first frame, as the tool call left it. */
320const rewind = (current: Playback) => {
321  current.stop.abort()
322  current.stop = new AbortController()
323  current.index = -1
324  current.shown = current.frames[0]!.path
325  current.isSpeaking = false
326  current.isMoving = false
327  current.waitTicks = 0
328  current.progress = 0
329  current.isEnded = false
330  current.isPaused = false
331}
332
333/** Freezes playback on the frame shown: the line stops, the clip holds its picture. */
334const pause = (current: Playback) => {
335  current.stop.abort()
336  current.stop = new AbortController()
337  current.isSpeaking = false
338  current.isMoving = false
339  current.isPaused = true
340}
341
342/**
343 * Puts frame `index` on screen and starts its line and its clip; the clock moves on
344 * once both have ended. Nothing in it waits, so no tick can come between.
345 */
346const startFrame = ($: EngineInterface, current: Playback, index: number, now: number) => {
347  const frame = current.frames[index]
348  if (!frame) return
349  current.index = index
350  current.shown = frame.clip?.paths[0] ?? frame.path
351  current.frameStartedAt = now
352  current.progress = 0
353  $.ui.invalidate('ui.render')
354
355  // A pause, a jump or a replay aborts this frame's signal and sets playback up
356  // itself: the line and the clip they cut short then leave it alone, however late
357  // they answer.
358  const stopped = current.stop.signal
359  if (frame.speech) {
360    current.isSpeaking = true
361    void $.audio.play({ base64: frame.speech, mime: 'audio/wav' }, { signal: stopped }).then(
362      () => {
363        if (stopped.aborted) return
364        current.isSpeaking = false
365        current.waitTicks = ticks(GAP_MS)
366      },
367      () => {
368        if (stopped.aborted) return
369        current.isSpeaking = false
370        current.waitTicks = ticks(readingTimeMs(frame.narration))
371      },
372    )
373  } else {
374    current.waitTicks = ticks(readingTimeMs(frame.narration))
375  }
376
377  const clip = frame.clip
378  if (!clip) return
379  current.isMoving = true
380  clip.stats.startedAt = now
381  let picture = 1
382  const timer = $.clock.every(Math.round(1000 / clip.fps), async () => {
383    if (playback !== current || current.index !== index || stopped.aborted) {
384      timer.cancel()
385      return
386    }
387    if (picture >= clip.paths.length) {
388      timer.cancel()
389      current.isMoving = false
390      const stats = clip.stats
391      stats.endedAt = await $.clock.now()
392      const seconds = (stats.endedAt - (stats.startedAt ?? stats.endedAt)) / 1000
393      stats.achievedFps = seconds > 0 ? Math.round((stats.taken / seconds) * 10) / 10 : undefined
394      await $.fs.write(`${current.dir}/clip-${stats.frame}-stats.json`, JSON.stringify(stats, null, 2))
395      return
396    }
397    const path = clip.paths[picture++]!
398    current.shown = path
399    clip.stats.sent += 1
400    void $.ui.blit({ requestId: PANE, key: 'frame', source: { file: path, format: 'png' } }).then(answer => {
401      if (answer.deny) {
402        clip.stats.denied += 1
403        clip.stats.firstDenial ??= answer.deny
404      } else {
405        clip.stats.taken += 1
406      }
407    })
408  })
409}
410
411/**
412 * Cuts short what plays and plays frame `target` from its start, on screen at once. The
413 * time is read first: from there to the frame starting nothing waits, so a tick cannot
414 * move playback on in between.
415 */
416const jumpToFrame = async ($: EngineInterface, current: Playback, target: number) => {
417  const now = await $.clock.now()
418  if (playback !== current) return
419  current.stop.abort()
420  current.stop = new AbortController()
421  current.isSpeaking = false
422  current.isMoving = false
423  current.waitTicks = 0
424  current.isPaused = false
425  current.isEnded = false
426  startFrame($, current, Math.max(0, Math.min(target, current.frames.length - 1)), now)
427}
428
429// The clock that moves playback on: a tick does nothing unless slides play, and
430// advances them once the line and the clip have ended and the pause has run. A reload
431// of the plugins can stop it without the mod's start running again, so every place
432// that starts or resumes a projection makes sure one is ticking; a clock that finds a
433// newer one started stops itself.
434let clockId = 0
435let lastTickAt = 0
436const ensureClock = async ($: EngineInterface) => {
437  if ((await $.clock.now()) - lastTickAt < TICK_MS * 5) return
438  const id = ++clockId
439  lastTickAt = await $.clock.now()
440  const clock = $.clock.every(TICK_MS, async () => {
441    if (id !== clockId) {
442      clock.cancel()
443      return
444    }
445      const now = await $.clock.now()
446      lastTickAt = now
447      const current = playback
448      if (!current || current.isAwaitingOpen || current.isEnded || current.isPaused) return
449      if (playback !== current) return
450      const playing = current.frames[current.index]
451      if (playing) {
452        const progress = Math.min(1, (now - current.frameStartedAt) / Math.max(1, playing.durationMs))
453        if (Math.round(progress * 100) !== Math.round(current.progress * 100)) {
454          current.progress = progress
455          $.ui.invalidate('ui.render')
456        }
457      }
458      if (current.isSpeaking || current.isMoving) return
459      if (current.waitTicks > 0) {
460        current.waitTicks -= 1
461        return
462      }
463      const index = current.index + 1
464      if (index >= current.frames.length) {
465        current.isEnded = true
466        current.progress = 1
467        $.ui.invalidate('ui.render')
468        return
469      }
470      startFrame($, current, index, now)
471  })
472}
473
474export const register: Register = on => {
475  on('session.start', async ($, e, next) => {
476    await $.tool.register({ name: 'project', description: DESCRIPTION, inputSchema: INPUT_SCHEMA, isDeferred: false })
477    await $.command.register({ name: 'magic-lantern', description: 'Open the magic lantern pane and project what is waiting in it' })
478    await ensureClock($)
479    return next(e)
480  })
481
482  on('tool.call', { tool: TOOL }, async ($, e) => {
483    const input = e as unknown as ExplainInput & { tool_use_id: string }
484    if (!input.frames?.length) return { deny: 'Give at least one frame.' }
485    const voice = input.voice ?? (await defaultVoice($, input.language))
486    if (!voice) return { deny: `macOS has no voice for ${input.language}. Give another language tag, or a voice by name.` }
487    const browser = await findChrome($)
488    if (!browser) return { deny: 'No Chrome or Chromium found to render the frames.' }
489
490    const temp = ((await $.env.get('TMPDIR')) ?? '/tmp').replace(/\/+$/, '')
491    const dir = `${temp}/magic-lantern/${input.tool_use_id}`
492    const canvas = { width: input.width ?? CANVAS.width, height: input.height ?? CANVAS.height }
493
494    const cutter = input.frames.some(frame => frame.motion) ? await findFfmpeg($) : undefined
495    if (input.frames.some(frame => frame.motion) && !cutter) {
496      return { deny: 'Motion frames need ffmpeg, to cut the captured sheets into pictures. Install it, or give the frames without motion.' }
497    }
498
499    const frames: PlaybackFrame[] = []
500    const renders: { frame: number; run: () => Promise<string | undefined> }[] = []
501    for (const [i, frame] of input.frames.entries()) {
502      const base = `${dir}/frame-${i + 1}`
503      await $.fs.write(`${base}.html`, frame.html)
504      renders.push({ frame: i + 1, run: () => render($, browser, fileUrl(`${base}.html`), `${base}.png`, canvas, RENDER_SCRIPT_BUDGET_MS) })
505      const playing: PlaybackFrame = { path: `${base}.png`, narration: frame.narration, durationMs: 0 }
506      if (frame.motion) {
507        const { seconds, fps } = frame.motion
508        await $.fs.write(`${base}.capture.html`, frame.html + CAPTURE_SCRIPT)
509        const count = Math.max(1, Math.round(seconds * fps))
510        const paths = Array.from({ length: count }, (_, k) => `${base}.clip-${String(k + 1).padStart(3, '0')}.png`)
511        const perSheet = Math.max(1, Math.min(SHEET_PICTURES, Math.floor(SHEET_MAX_HEIGHT / canvas.height)))
512        for (let first = 0; first < count; first += perSheet) {
513          const sheetCount = Math.min(perSheet, count - first)
514          const urls = Array.from({ length: sheetCount }, (_, k) => fileUrl(`${base}.capture.html`, `#t=${Math.round(((first + k) * 1000) / fps)}`))
515          const sheet = {
516            page: `${base}.sheet-${first + 1}.html`,
517            png: `${base}.sheet-${first + 1}.png`,
518            count: sheetCount,
519            first: first + 1,
520            pattern: `${base}.clip-%03d.png`,
521          }
522          await $.fs.write(sheet.page, sheetPage(urls, canvas))
523          renders.push({ frame: i + 1, run: () => renderSheet($, browser, cutter!, sheet, canvas) })
524        }
525        playing.clip = { paths, fps, stats: { frame: i + 1, fps, pictures: count, captureMs: 0, sent: 0, taken: 0, denied: 0 } }
526      }
527      frames.push(playing)
528    }
529
530    const startedAt = await $.clock.now()
531    const [failures, speech] = await Promise.all([
532      pooled(renders.map(job => job.run), RENDER_CONCURRENCY),
533      Promise.all(input.frames.map((frame, i) => speakToFile($, voice, frame.narration, `${dir}/frame-${i + 1}`))),
534    ])
535    const renderMs = (await $.clock.now()) - startedAt
536    const failed = failures.findIndex(failure => failure !== undefined)
537    if (failed !== -1) return { deny: `Chrome could not render frame ${renders[failed]!.frame}: ${failures[failed]}` }
538    frames.forEach((frame, i) => {
539      frame.speech = speech[i]
540      const lineMs = frame.speech ? speechMs(frame.speech) : readingTimeMs(frame.narration)
541      frame.durationMs = Math.max(lineMs, frame.clip ? (frame.clip.paths.length / frame.clip.fps) * 1000 : 0)
542      if (frame.clip) frame.clip.stats.captureMs = renderMs
543    })
544
545    playback = {
546      title: input.title,
547      dir,
548      canvas,
549      frames,
550      index: -1,
551      shown: frames[0]!.path,
552      isSpeaking: false,
553      isMoving: false,
554      waitTicks: 0,
555      frameStartedAt: 0,
556      progress: 0,
557      isEnded: false,
558      isAwaitingOpen: false,
559      isPaused: false,
560      stop: new AbortController(),
561    }
562    // A pane opened from a tool call is one nobody asked for, which a narrow terminal
563    // leaves undrawn; the slides then wait for the person's /magic-lantern.
564    await ensureClock($)
565    const opened = await $.ui.open({ id: PANE, title: input.title })
566    if (!opened.isPlaced) {
567      playback.isAwaitingOpen = true
568      return {
569        result: `The slides are ready but the pane is not drawn: ${opened.reason}. Ask the person to run /magic-lantern, which opens it at any width and starts playback.`,
570      }
571    }
572    const silent = speech.filter(line => line === undefined).length
573    const silentNote = silent ? ` ${silent} line(s) could not be spoken and are shown only.` : ''
574    const clips = frames.filter(frame => frame.clip).length
575    const clipNote = clips ? ` ${clips} motion frame(s); each writes clip-<frame>-stats.json there once played.` : ''
576    return {
577      result: `Projecting ${frames.length} frames in the magic lantern pane, voice ${voice}; the pane stays open on the last frame until the person closes it. ${renders.length} Chrome renders (stills and sheets) in ${renderMs} ms. Frames: ${dir}.${silentNote}${clipNote}`,
578    }
579  }).catch(($, e, next) => ({
580    deny: `magic-lantern could not project (${next.error.kind}${next.error.message ? `: ${next.error.message}` : ''}).`,
581  }))
582
583  on('command.run', { command: 'magic-lantern' }, async $ => {
584    if (!playback) return { text: 'Nothing is waiting to be projected.' }
585    const opened = await $.ui.open({ id: PANE, title: playback.title })
586    if (!opened.isPlaced) return { text: `The magic lantern pane could not open: ${opened.reason}` }
587    playback.isAwaitingOpen = false
588    await ensureClock($)
589    return { text: 'Projecting.' }
590  })
591
592  // The person closing the pane stops the playback and the line being spoken.
593  on('ui.close', { id: PANE }, async (_$, e, next) => {
594    playback?.stop.abort()
595    playback = undefined
596    return next(e)
597  })
598
599  // The call's row is left out of the transcript: the pane is what the person watches,
600  // and the call's markup only crowds the conversation. ctrl+o shows it whole, and a
601  // failed call keeps its row so the failure is seen.
602  on('ui.render', { component: 'ToolUse', props: { tool: TOOL } }, async ($, e, next) => {
603    if (e.props.isExpanded || e.props.isErrored || e.props.isInterrupted) return next(e)
604    const { Box } = $.ui.resolve(e)
605    return <Box />
606  })
607
608  // Frames are pictures, which only the terminal draws; elsewhere the pane
609  // carries the spoken line alone. In the terminal: the picture centred, the line as a
610  // subtitle under it (with a speaker while the line is heard), a progress segment per
611  // frame (the current one filling as it plays), Pause and a button per frame to jump
612  // to, and once the last frame has played, Replay and Close.
613  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
614    const current = playback
615    const index = current ? Math.min(Math.max(current.index, 0), current.frames.length - 1) : 0
616    const frame = current?.frames[index]
617    if (!current || !frame) {
618      const { Text } = $.ui.resolve(e)
619      return <Text dimColor>Nothing playing.</Text>
620    }
621    if (e.surface !== 'terminal') {
622      const { Box, Text } = $.ui.resolve(e)
623      return (
624        <Box flexDirection="column">
625          <Text>{frame.narration}</Text>
626          <Text dimColor>
627            {index + 1} / {current.frames.length}
628          </Text>
629        </Box>
630      )
631    }
632    const { Box, Text, Image, Button } = $.ui.resolve(e)
633    const { width, height } = current.canvas
634    let columns = Math.min(255, Math.max(1, e.props.bodyColumns - 2 * PICTURE_MARGIN_COLUMNS))
635    let rows = Math.max(1, Math.round((columns * height) / width / CELL_ASPECT))
636    const room = Math.max(4, e.props.scroll.bodyRows - ROWS_UNDER_PICTURE)
637    if (rows > room) {
638      rows = room
639      columns = Math.max(1, Math.round((rows * CELL_ASPECT * width) / height))
640    }
641
642    const count = current.frames.length
643    const cells =
644      count === 1
645        ? Math.min(columns, SINGLE_BAR_CELLS)
646        : Math.max(2, Math.min(SEGMENT_MAX_CELLS, Math.floor((columns - (count - 1)) / count)))
647    const bar = (length: number) => '━'.repeat(Math.max(0, length))
648    const segments = current.frames.map((_, i) => {
649      if (current.isEnded || i < index) return <Text>{bar(cells)}</Text>
650      if (i > index || current.index < 0) return <Text dimColor>{bar(cells)}</Text>
651      const lit = Math.round(cells * current.progress)
652      return (
653        <Box>
654          <Text color={ACCENT}>{bar(lit)}</Text>
655          <Text dimColor>{bar(cells - lit)}</Text>
656        </Box>
657      )
658    })
659
660    // Playing again replays the paused frame from its start, so its line is heard whole.
661    const play = (target: number) => {
662      if (playback !== current) return
663      void jumpToFrame($, current, target).then(() => ensureClock($))
664    }
665    const togglePause = () => {
666      if (playback !== current) return
667      if (current.isPaused) return play(index)
668      pause(current)
669      $.ui.invalidate('ui.render')
670    }
671
672    // The speaker has a slot of its own, so the line does not move as it comes and goes.
673    // The labels are plain words: the glyphs for play and pause are drawn one cell wide
674    // by some terminals and two by others, which breaks the row they sit in.
675    return (
676      <Box flexDirection="column" alignItems="center" paddingTop={1}>
677        <Image key="frame" source={{ file: current.shown, format: 'png' }} columns={columns} rows={rows} alt={frame.narration} />
678        {!current.isEnded && (
679          <Box marginTop={1} width={columns} justifyContent="center" gap={1}>
680            <Box width={2} flexShrink={0}>
681              <Text>{current.isSpeaking ? '🔊' : ''}</Text>
682            </Box>
683            <Text>{frame.narration}</Text>
684          </Box>
685        )}
686        <Box marginTop={1} gap={1}>
687          {segments}
688        </Box>
689        {!current.isEnded && (
690          <Box marginTop={1} width={columns} justifyContent="center" flexWrap="wrap" gap={1}>
691            <Button key="pause" hotkey="p" onPress={togglePause}>
692              {current.isPaused ? 'Play' : 'Pause'}
693            </Button>
694            {current.frames.map((_, i) => (
695              <Button
696                key={`frame-${i + 1}`}
697                hotkey={i < 9 ? String(i + 1) : undefined}
698                variant={i === index ? 'primary' : undefined}
699                onPress={() => play(i)}
700              >
701                {String(i + 1)}
702              </Button>
703            ))}
704          </Box>
705        )}
706        {current.isEnded && (
707          <Box marginTop={1} gap={2}>
708            <Button
709              hotkey="r"
710              variant="primary"
711              onPress={() => {
712                if (playback !== current) return
713                rewind(current)
714                $.ui.invalidate('ui.render')
715                void ensureClock($)
716              }}
717            >
718              Replay
719            </Button>
720            <Button role="dismiss" onPress={() => void $.ui.close({ id: PANE })}>
721              Close
722            </Button>
723          </Box>
724        )}
725      </Box>
726    )
727  })
728}
729