Replaycast

1// replaycast: a podcast-style player for Claude's replies.
2//
3// A Listen button under each reply, and a now-playing bar above the prompt with
4// a progress bar, play/pause, ±10 s, previous/next sentence, speed 1x-4x and voice.
5//
6// Two engines, both fully offline:
7//   native   Windows. engine/reader.py synthesizes with the Windows voice (SAPI)
8//            or a local Piper model, time-stretches with ffmpeg (atempo keeps the
9//            pitch) and plays through Windows MCI. Seek, speed and voice switching.
10//   builtin  Any platform. Claude Code's own $.audio.speak, one sentence at a time.
11//            Play/pause and sentence skipping; speed is the system voice's own.
12// No network calls. Nothing leaves the machine.
13
14const ACCENT = '#D97757'
15const SPEEDS = [1, 1.25, 1.5, 1.75, 2, 2.25, 2.5, 2.75, 3, 3.25, 3.5, 3.75, 4]
16const CHARS_PER_SEC = 13 // rough listening rate at 1x, for estimates
17
18// Resolved once from userConfig and the environment
19let cfg = null
20// 'native' | 'builtin'
21let engine = null
22let stateDir = ''
23
24let speed = 1.5
25let voice = 'sapi'
26let autoRead = false
27let lastAnswer = ''
28
29// The reply being played: { id, text, title }
30let active = null
31// Player state, from the native engine's state.json or the builtin loop
32let st = null
33let child = null
34let poller = null
35let lastSig = ''
36
37// Builtin engine: sentence spans and a token that invalidates stale loops
38let spans = []
39let cursor = 0
40let token = 0
41
42export function label(s) {
43  return s + 'x'
44}
45
46export function clock(ms) {
47  const t = Math.max(0, Math.round((ms || 0) / 1000))
48  return Math.floor(t / 60) + ':' + String(t % 60).padStart(2, '0')
49}
50
51function estimate(text, s) {
52  return clock((speakable(text).length / CHARS_PER_SEC / s) * 1000)
53}
54
55// Turn markdown into something that sounds right when spoken
56export function speakable(md) {
57  return String(md || '')
58    .replace(/```[\s\S]*?```/g, ' (code block skipped.) ')
59    .replace(/`([^`]*)`/g, '$1')
60    .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
61    .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
62    .replace(/^\s*\|?[\s:|-]+\|[\s:|-]*$/gm, '')
63    .replace(/^\s*\|/gm, '')
64    .replace(/\|\s*$/gm, '.')
65    .replace(/\|/g, ', ')
66    .replace(/^#{1,6}\s*(.*)$/gm, '$1.')
67    .replace(/^\s*(?:[-*•]|\d+\.)\s+(.*)$/gm, '$1.')
68    .replace(/[*_~>#]+/g, '')
69    .replace(/[→⇒]/g, ' to ')
70    .replace(/[★✓✗☐●○■▶⏸]/g, '')
71    .replace(/\.(\s*\.)+/g, '.')
72    .replace(/[^\S\n]+/g, ' ')
73    .replace(/\n{2,}/g, '\n')
74    .trim()
75}
76
77// Sentence spans [start, end) for the builtin engine, each under 400 characters
78export function sentences(text) {
79  const out = []
80  const re = /[^.!?;:\n]+(?:[.!?;:]+|\n|$)/g
81  let m
82  while ((m = re.exec(text)) !== null) {
83    if (!m[0].trim()) continue
84    let s = m.index
85    const e = m.index + m[0].length
86    while (e - s > 400) {
87      const cut = text.lastIndexOf(' ', s + 400)
88      const at = cut > s ? cut : s + 400
89      out.push([s, at])
90      s = at
91    }
92    out.push([s, e])
93  }
94  return out
95}
96
97function titleOf(text) {
98  const first = speakable(text).split('\n').find((l) => l.trim().length > 0) || 'Claude reply'
99  return first.length > 64 ? first.slice(0, 61) + '…' : first
100}
101
102function barSvg(frac, w) {
103  const x = Math.round(Math.max(0, Math.min(1, frac)) * w)
104  const knob = Math.max(6, Math.min(w - 6, x))
105  return (
106    '<svg xmlns="http://www.w3.org/2000/svg" width="' + w + '" height="14" viewBox="0 0 ' + w + ' 14">' +
107    '<rect x="0" y="5" width="' + w + '" height="4" rx="2" fill="rgba(128,128,128,0.35)"/>' +
108    '<rect x="0" y="5" width="' + x + '" height="4" rx="2" fill="' + ACCENT + '"/>' +
109    '<circle cx="' + knob + '" cy="7" r="6" fill="' + ACCENT + '"/></svg>'
110  )
111}
112
113function barText(frac, w) {
114  const x = Math.round(Math.max(0, Math.min(1, frac)) * w)
115  return '━'.repeat(x) + '●' + '─'.repeat(Math.max(0, w - x))
116}
117
118// ---------- configuration ----------
119
120async function resolveEngine($) {
121  if (engine) return engine
122  const isWindows = (await $.env.get('OS')) === 'Windows_NT'
123  const want = cfg.engine || 'auto'
124  engine = want === 'builtin' || (want === 'auto' && !isWindows) ? 'builtin' : 'native'
125  if (engine === 'native' && !isWindows) {
126    $.ui.toast('replaycast: the native engine is Windows-only; using the built-in voice')
127    engine = 'builtin'
128  }
129  if (engine === 'native') {
130    const base = (await $.env.get('LOCALAPPDATA')) || $.plugin.root
131    stateDir = String(base).replace(/\\/g, '/') + '/replaycast'
132  }
133  return engine
134}
135
136function voices() {
137  if (engine === 'builtin') return [{ value: 'system', label: cfg.builtin_voice || 'System voice' }]
138  const list = [{ value: 'sapi', label: (cfg.sapi_voice || 'Windows voice') + ' · offline' }]
139  if (cfg.piper && cfg.piper_model) list.unshift({ value: 'piper', label: 'Piper neural · offline' })
140  return list
141}
142
143function voiceName(v) {
144  const hit = voices().find((x) => x.value === v)
145  return hit ? hit.label.split(' · ')[0] : v
146}
147
148// ---------- native engine (Windows) ----------
149
150async function sendCmd($, line) {
151  if (engine === 'builtin') return builtinCmd($, line)
152  try {
153    const path = stateDir + '/cmd.txt'
154    const prev = (await $.fs.exists(path)) ? String(await $.fs.read(path)) : ''
155    await $.fs.write(path, prev + line + '\n')
156  } catch (err) {
157    $.ui.toast('replaycast: could not send ' + line)
158  }
159}
160
161async function readState($) {
162  try {
163    return JSON.parse(String(await $.fs.read(stateDir + '/state.json')))
164  } catch (err) {
165    return null
166  }
167}
168
169function engineArgs(extra) {
170  const argv = [
171    cfg.python || 'pythonw', $root + '/engine/reader.py',
172    '--state', stateDir,
173    '--voice', voice,
174    '--speed', String(speed),
175    '--ffmpeg', cfg.ffmpeg || 'ffmpeg',
176  ]
177  if (cfg.sapi_voice) argv.push('--sapi-voice', cfg.sapi_voice)
178  if (cfg.piper && cfg.piper_model) argv.push('--piper', cfg.piper, '--piper-model', cfg.piper_model)
179  return argv.concat(extra)
180}
181
182// The plugin directory, captured from $.plugin.root on first use
183let $root = ''
184
185function runNative($, offset) {
186  $root = $.plugin.root.replace(/\\/g, '/')
187  const stream = $.process.spawn({
188    argv: engineArgs(['--text', stateDir + '/text.txt', '--offset', String(offset || 0)]),
189    cwd: $root,
190  })
191  child = stream
192  void (async () => {
193    try {
194      for await (const piece of stream) {
195        if (piece && piece.stream === 'stderr' && piece.text) $.ui.log('replaycast: ' + piece.text, { to: 'debug' })
196      }
197    } catch (err) {
198      // No Python, or the engine crashed before it started: fall back to the built-in voice
199      if (child === stream && st && st.status === 'loading') {
200        $.ui.toast('replaycast: the Windows engine could not start (' + err.message + '); using the built-in voice')
201        engine = 'builtin'
202        voice = 'system'
203        child = null
204        if (active) await play($, active.id, active.text, 0)
205        return
206      }
207    }
208    if (child === stream) child = null
209  })()
210}
211
212async function killNative($) {
213  const old = child
214  child = null
215  if (!old) return
216  await sendCmd($, 'stop')
217  if (typeof old.return === 'function') {
218    try {
219      await old.return()
220    } catch (err) {
221      // already gone
222    }
223  }
224}
225
226// Fill the audio cache for a fresh reply so Listen starts at once (Piper only; SAPI is instant)
227async function prefetch($, text) {
228  if (engine !== 'native' || voice !== 'piper') return
229  const clean = speakable(text)
230  if (!clean) return
231  try {
232    await $.fs.write(stateDir + '/prefetch.txt', clean)
233  } catch (err) {
234    return
235  }
236  $root = $.plugin.root.replace(/\\/g, '/')
237  const stream = $.process.spawn({ argv: engineArgs(['--prefetch', '--text', stateDir + '/prefetch.txt']), cwd: $root })
238  void (async () => {
239    try {
240      for await (const piece of stream) void piece
241    } catch (err) {
242      // a failed prefetch only costs the first-play wait
243    }
244  })()
245}
246
247function startPolling($) {
248  if (poller) poller.cancel()
249  poller = $.clock.every(300, async () => {
250    const next = await readState($)
251    if (next && engine === 'native') st = next
252    const sig = st ? [st.status, st.paused, Math.round((st.elapsedMs || 0) / 250), st.totalMs, st.speed, st.voice].join('|') : ''
253    if (sig !== lastSig) {
254      lastSig = sig
255      $.ui.invalidate('ui.render')
256    }
257    if (st && (st.status === 'done' || st.status === 'stopped' || st.status === 'error') && !child) {
258      if (poller) poller.cancel()
259      poller = null
260    }
261  })
262}
263
264// ---------- builtin engine (any platform) ----------
265
266function builtinState(status) {
267  const text = active ? speakable(active.text) : ''
268  const before = cursor < spans.length ? spans[cursor][0] : text.length
269  st = {
270    status,
271    paused: status === 'paused',
272    elapsedMs: (before / CHARS_PER_SEC) * 1000,
273    totalMs: (text.length / CHARS_PER_SEC) * 1000,
274    speed: 1,
275    voice: 'system',
276    engine: 'builtin',
277    charPos: before,
278  }
279}
280
281async function builtinLoop($) {
282  const mine = ++token
283  const text = speakable(active.text)
284  builtinState('playing')
285  $.ui.invalidate('ui.render')
286  while (mine === token && cursor < spans.length) {
287    const at = cursor
288    const [s, e] = spans[at]
289    try {
290      await $.audio.speak(text.slice(s, e).trim(), cfg.builtin_voice ? { voice: cfg.builtin_voice } : undefined)
291    } catch (err) {
292      if (mine === token) {
293        builtinState('error')
294        st.lastError = err.message
295        $.ui.toast('replaycast: the system voice failed (' + err.message + ')')
296        $.ui.invalidate('ui.render')
297      }
298      return
299    }
300    if (mine !== token) return
301    // A prev/next during the sentence already moved the cursor
302    if (cursor === at) cursor = at + 1
303    builtinState(cursor < spans.length ? 'playing' : 'done')
304    $.ui.invalidate('ui.render')
305  }
306}
307
308async function builtinCmd($, line) {
309  const [cmd, arg] = line.split(' ')
310  if (cmd === 'stop') {
311    token++
312    return
313  }
314  if (cmd === 'toggle') {
315    if (st && st.status === 'paused') {
316      void builtinLoop($)
317    } else {
318      // The sentence being spoken finishes; nothing after it starts
319      token++
320      builtinState('paused')
321    }
322  } else if (cmd === 'next') {
323    cursor = Math.min(spans.length - 1, cursor + 1)
324  } else if (cmd === 'prev') {
325    cursor = Math.max(0, cursor - 1)
326  } else if (cmd === 'back' || cmd === 'fwd') {
327    // About one sentence per 10 s step at 1x
328    const step = Math.max(1, Math.round(Number(arg || 10) / 10))
329    cursor = Math.max(0, Math.min(spans.length - 1, cursor + (cmd === 'back' ? -step : step)))
330  }
331  if ((cmd === 'next' || cmd === 'prev' || cmd === 'back' || cmd === 'fwd') && st && st.status !== 'paused') {
332    builtinState('playing')
333  }
334  $.ui.invalidate('ui.render')
335}
336
337// ---------- shared player ----------
338
339async function play($, id, text, offset) {
340  await resolveEngine($)
341  const clean = speakable(text)
342  if (!clean) {
343    $.ui.toast('Nothing to read in this reply')
344    return
345  }
346  await stopAll($)
347  if (!voices().some((x) => x.value === voice)) voice = voices()[0].value
348  active = { id, text, title: titleOf(text) }
349  if (engine === 'builtin') {
350    spans = sentences(clean)
351    cursor = Math.max(0, spans.findIndex((sp) => sp[1] > (offset || 0)))
352    void builtinLoop($)
353    return
354  }
355  try {
356    await $.fs.write(stateDir + '/cmd.txt', '')
357    await $.fs.write(stateDir + '/text.txt', clean)
358  } catch (err) {
359    $.ui.toast('replaycast: could not write to ' + stateDir)
360    active = null
361    return
362  }
363  st = { status: 'loading', paused: false, elapsedMs: 0, totalMs: 0, speed, voice }
364  runNative($, offset || 0)
365  startPolling($)
366  $.ui.invalidate('ui.render')
367}
368
369async function stopAll($) {
370  if (engine === 'builtin') token++
371  else await killNative($)
372}
373
374async function togglePause($) {
375  if (!st) return
376  if (engine === 'native') {
377    st = { ...st, paused: !st.paused, status: st.status === 'loading' ? 'loading' : st.paused ? 'playing' : 'paused' }
378  }
379  await sendCmd($, 'toggle')
380  $.ui.invalidate('ui.render')
381}
382
383async function close($) {
384  await stopAll($)
385  active = null
386  st = null
387  if (poller) poller.cancel()
388  poller = null
389  $.ui.invalidate('ui.render')
390}
391
392async function setSpeed($, s) {
393  speed = s
394  await $.store.set('speed', s)
395  if (active && child) {
396    await sendCmd($, 'speed ' + s)
397    if (st) st = { ...st, speed: s }
398  }
399  $.ui.invalidate('ui.render')
400}
401
402async function setVoice($, v) {
403  voice = v
404  await $.store.set('voice', v)
405  // A new voice needs new audio: restart from where we are
406  if (active && st && st.status !== 'done') {
407    await play($, active.id, active.text, st.charPos || 0)
408  } else {
409    $.ui.invalidate('ui.render')
410  }
411}
412
413async function replay($) {
414  if (active) await play($, active.id, active.text, 0)
415}
416
417// ---------- hooks ----------
418
419export function register(on, options) {
420  cfg = options || {}
421  if (typeof cfg.default_speed === 'number') speed = Math.max(1, Math.min(4, cfg.default_speed))
422
423  on('session.start', async ($, e, next) => {
424    await resolveEngine($)
425    const s = await $.store.get('speed')
426    if (typeof s === 'number' && s >= 1 && s <= 4) speed = s
427    const v = await $.store.get('voice')
428    if (typeof v === 'string' && voices().some((x) => x.value === v)) voice = v
429    else voice = voices()[0].value
430    autoRead = (await $.store.get('autoRead')) === true
431    try {
432      await $.command.register({ name: 'listen', description: 'Listen to the last Claude reply' })
433      await $.command.register({ name: 'listen-speed', description: 'Set listening speed, 1 to 4', argumentHint: '[1-4]' })
434      await $.command.register({ name: 'listen-pause', description: 'Pause or resume listening' })
435      await $.command.register({ name: 'listen-stop', description: 'Stop listening and close the player' })
436      await $.command.register({ name: 'listen-auto', description: 'Toggle listening to each new reply automatically' })
437    } catch (err) {
438      $.ui.log('replaycast: could not register a command: ' + err.message)
439    }
440    return next(e)
441  })
442
443  on('turn.complete', async ($, e, next) => {
444    if (!e.agentId && !e.isAborted && e.answer) {
445      lastAnswer = e.answer
446      if (autoRead) await play($, 'last', e.answer, 0)
447      else await prefetch($, e.answer)
448    }
449    return next(e)
450  })
451
452  on('command.run', { command: 'listen' }, async ($) => {
453    if (!lastAnswer) return { text: 'No reply yet in this session. Use Listen under a reply.' }
454    await play($, 'last', lastAnswer, 0)
455    return {}
456  })
457
458  on('command.run', { command: 'listen-speed' }, async ($, e) => {
459    const v = parseFloat(String(e.args || '').replace('x', ''))
460    if (!(v >= 1 && v <= 4)) return { text: 'Speed is ' + label(speed) + '. Use /listen-speed 1 to /listen-speed 4.' }
461    await setSpeed($, Math.round(v * 4) / 4)
462    return { text: 'Speed ' + label(speed) }
463  })
464
465  on('command.run', { command: 'listen-pause' }, async ($) => {
466    if (!active) return { text: 'Nothing is playing' }
467    await togglePause($)
468    return {}
469  })
470
471  on('command.run', { command: 'listen-stop' }, async ($) => {
472    await close($)
473    return {}
474  })
475
476  on('command.run', { command: 'listen-auto' }, async ($) => {
477    autoRead = !autoRead
478    await $.store.set('autoRead', autoRead)
479    return { text: 'Auto-listen is ' + (autoRead ? 'on' : 'off') }
480  })
481
482  // Under each reply: Listen, with how long it takes at the current speed
483  on('ui.render', { component: 'AssistantMessage' }, async ($, e, next) => {
484    const theirs = await next(e)
485    const text = e.props && typeof e.props.text === 'string' ? e.props.text : ''
486    if (speakable(text).length < 2) return theirs
487    const { Box, Button, Text } = $.ui.resolve(e)
488    const id = e.requestId
489    const isThis = active && active.id === id
490    const pace = engine === 'builtin' ? 1 : speed
491    const row = isThis
492      ? [
493          Button({
494            key: 'rc-row-toggle-' + id,
495            label: st && st.paused ? '▶  Resume' : '⏸  Pause',
496            variant: 'secondary',
497            onPress: async () => { await togglePause($) },
498          }),
499          Text({ dimColor: true, children: [st && st.status === 'loading' ? 'Preparing audio…' : 'Now playing'] }),
500        ]
501      : [
502          Button({
503            key: 'rc-listen-' + id,
504            label: '▶  Listen',
505            variant: 'secondary',
506            onPress: async () => { await play($, id, text, 0) },
507          }),
508          Text({ dimColor: true, children: [estimate(text, pace) + ' at ' + label(pace)] }),
509        ]
510    return Box({
511      flexDirection: 'column',
512      children: [theirs, Box({ flexDirection: 'row', columnGap: 1, alignItems: 'center', children: row })],
513    })
514  })
515
516  // The now-playing bar above the prompt
517  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
518    if (!active) return next(e)
519    const { Box, Text, Button, Select, Svg } = $.ui.resolve(e)
520    const desktop = e.surface === 'desktop'
521    const native = engine === 'native'
522    const s = st || {}
523    const loading = s.status === 'loading'
524    const finished = s.status === 'done' || s.status === 'stopped'
525    const frac = s.totalMs ? (s.elapsedMs || 0) / s.totalMs : 0
526    const paused = !!s.paused
527
528    const status = s.status === 'error'
529      ? 'Could not make audio' + (s.lastError ? ': ' + String(s.lastError).slice(0, 80) : '')
530      : loading
531        ? 'Preparing audio…'
532        : finished
533          ? 'Finished'
534          : (paused ? 'Paused' : 'Playing') + ' · ' + voiceName(s.voice || voice) +
535            (native && s.engine === 'sapi' && voice === 'piper' ? ' (fallback)' : '')
536
537    const header = Box({
538      flexDirection: 'row',
539      columnGap: 1,
540      children: [
541        Text({ color: ACCENT, bold: true, children: ['🎧'] }),
542        Text({ bold: true, wrap: 'truncate-end', children: [active.title] }),
543      ],
544    })
545
546    const progress = Box({
547      flexDirection: 'row',
548      columnGap: 1,
549      alignItems: 'center',
550      children: [
551        Text({ dimColor: true, children: [clock(s.elapsedMs)] }),
552        desktop
553          ? Svg({ source: barSvg(frac, 420), alt: Math.round(frac * 100) + '% played', width: 420, height: 14 })
554          : Text({ color: ACCENT, children: [barText(frac, 30)] }),
555        Text({ dimColor: true, children: [clock(s.totalMs)] }),
556        Text({ dimColor: true, children: ['· ' + status] }),
557      ],
558    })
559
560    const controls = [
561      Button({ key: 'rc-prev', label: '⏮', plain: true, onPress: async () => { await sendCmd($, 'prev') } }),
562      Button({ key: 'rc-back', label: '↺ 10', plain: true, onPress: async () => { await sendCmd($, 'back 10') } }),
563      finished
564        ? Button({ key: 'rc-replay', label: '↻  Replay', variant: 'primary', onPress: async () => { await replay($) } })
565        : Button({
566            key: 'rc-toggle',
567            label: paused ? '▶  Play' : '⏸  Pause',
568            variant: 'primary',
569            onPress: async () => { await togglePause($) },
570          }),
571      Button({ key: 'rc-fwd', label: '10 ↻', plain: true, onPress: async () => { await sendCmd($, 'fwd 10') } }),
572      Button({ key: 'rc-next', label: '⏭', plain: true, onPress: async () => { await sendCmd($, 'next') } }),
573    ]
574    if (native) {
575      controls.push(
576        Select({
577          key: 'rc-speed',
578          label: 'Speed',
579          value: String(speed),
580          options: SPEEDS.map((x) => ({ value: String(x), label: label(x) })),
581          onSelect: async (v) => { await setSpeed($, parseFloat(v)) },
582        }),
583      )
584    }
585    if (voices().length > 1) {
586      controls.push(
587        Select({
588          key: 'rc-voice',
589          label: 'Voice',
590          value: voice,
591          options: voices(),
592          onSelect: async (v) => { await setVoice($, v) },
593        }),
594      )
595    }
596    controls.push(Button({ key: 'rc-close', label: 'Close player', role: 'dismiss', onPress: async () => { await close($) } }))
597
598    return Box({
599      flexDirection: 'column',
600      rowGap: 0,
601      paddingX: 1,
602      borderStyle: 'round',
603      borderColor: ACCENT,
604      children: [header, progress, Box({ flexDirection: 'row', columnGap: 1, alignItems: 'center', children: controls })],
605    })
606  })
607}
608