play

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, HookStream, ProcessSpawnChunk, ProcessSpawnResult, Register } from 'claude-code'
3import type { Playback } from '../types'
4import { forTheEar, hash, lastReply, SCRIPT_PROMPT } from './speech'
5
6const VOICE = 'bD9maNcCuQQS75DGuteM'
7const MODEL = 'eleven_v4_turbo'
8const KEY_REF = 'pass://Nik/.env/ELEVENLABS_API_TOKEN'
9// Every character is billed; a longer reply needs `/play all`.
10const MAX_CHARS = 3000
11const REQUEST_LIMIT = 10000
12const REWIND_SECONDS = 10
13
14const IDLE: Playback = { phase: 'idle', chars: 0 }
15const playback = atom({ plugin: 'play', key: 'playback' } as const, IDLE)
16
17let player: HookStream<ProcessSpawnChunk, ProcessSpawnResult> | undefined
18let dir: string | undefined
19let key: string | undefined
20
21// Unix socket paths stay under ~104 bytes on macOS, so the socket lives in the
22// short cache dir, not a temp dir.
23async function cacheDir($: EngineInterface) {
24  dir ??= `${(await $.process.run(['sh', '-c', 'printf %s "$HOME"'])).stdout}/.cache/claude-play`
25  return dir
26}
27
28async function mpv($: EngineInterface, command: unknown[]) {
29  const socket = `${await cacheDir($)}/mpv.sock`
30  await $.process.run(['nc', '-U', '-w', '1', socket], { stdin: `${JSON.stringify({ command })}\n` })
31}
32
33async function setPhase($: EngineInterface, phase: Playback['phase'], chars?: number) {
34  await update($, playback, p => ({ phase, chars: chars ?? p.chars }))
35}
36
37async function stop($: EngineInterface) {
38  void player?.return({ code: null, signal: 'SIGTERM' })
39  player = undefined
40  await setPhase($, 'idle')
41}
42
43async function togglePause($: EngineInterface) {
44  const { phase } = await read($, playback)
45  if (phase !== 'playing' && phase !== 'paused') return false
46  const pause = phase === 'playing'
47  await mpv($, ['set_property', 'pause', pause])
48  await setPhase($, pause ? 'paused' : 'playing')
49  return true
50}
51
52async function rewind($: EngineInterface) {
53  const { phase } = await read($, playback)
54  if (phase !== 'playing' && phase !== 'paused') return false
55  await mpv($, ['seek', -REWIND_SECONDS])
56  return true
57}
58
59// Falls back to the plain cleanup when the model call fails, so a reply is
60// always read.
61async function toScript($: EngineInterface, reply: string) {
62  const result = await $.model.complete({
63    model: 'haiku',
64    system: SCRIPT_PROMPT,
65    prompt: `<reply>\n${reply}\n</reply>`,
66    maxTokens: 4000,
67    timeoutMs: 30_000,
68  })
69  return result.isAnswered && result.text.trim() ? result.text.trim() : forTheEar(reply)
70}
71
72// The key from Proton Pass, read once per session and held in memory only.
73async function apiKey($: EngineInterface) {
74  if (key) return key
75  const { exitCode, stdout } = await $.process.run(['pass-cli', 'item', 'view', KEY_REF])
76  if (exitCode !== 0 || !stdout.trim()) throw new Error(`could not read ${KEY_REF}`)
77  key = stdout.trim()
78  return key
79}
80
81// Calls the API with a key: the `elevenlabs` CLI signs in through the browser
82// and is billed at app rates, five times the API's on v4 Turbo.
83// Answers the error, or undefined once the mp3 is written.
84async function synthesize($: EngineInterface, text: string, file: string) {
85  const body = `${file}.json`
86  await $.process.run(['mkdir', '-p', await cacheDir($)])
87  await $.process.run(['sh', '-c', 'cat > "$0"', body], { stdin: JSON.stringify({ text, model_id: MODEL }) })
88  // The key travels on stdin, never in argv, where `ps` would show it.
89  const { exitCode, stderr } = await $.process.run(
90    ['curl', '-sS', '--fail-with-body', '-H', '@-', '-H', 'Content-Type: application/json',
91      '--data-binary', `@${body}`, '-o', file, `https://api.elevenlabs.io/v1/text-to-speech/${VOICE}`],
92    { stdin: `xi-api-key: ${await apiKey($)}\n`, timeoutMs: 120_000 },
93  )
94  await $.process.run(['rm', '-f', body])
95  if (exitCode === 0) return undefined
96  const response = (await $.process.run(['cat', file])).stdout
97  await $.process.run(['rm', '-f', file])
98  return response.trim() || stderr.trim() || `curl exited ${exitCode}`
99}
100
101async function speak($: EngineInterface, reply: string, estimate: number, file: string, cached: boolean) {
102  try {
103    if (!cached) {
104      await setPhase($, 'preparing', estimate)
105      const text = await toScript($, reply)
106      if (text.length > REQUEST_LIMIT) {
107        $.ui.toast(`play: ${text.length} chars, over the ${REQUEST_LIMIT} a request takes.`)
108        await setPhase($, 'idle')
109        return
110      }
111      await setPhase($, 'generating', text.length)
112      const error = await synthesize($, text, file)
113      if (error) {
114        $.ui.toast(`play: ElevenLabs failed: ${error}`)
115        await setPhase($, 'idle')
116        return
117      }
118    }
119    const stream = $.process.spawn({
120      argv: ['mpv', '--no-video', '--no-terminal', '--af=loudnorm', `--input-ipc-server=${await cacheDir($)}/mpv.sock`, file],
121    })
122    player = stream
123    await setPhase($, 'playing')
124    for await (const _ of stream) {
125    }
126    if (player === stream) {
127      player = undefined
128      await setPhase($, 'idle')
129    }
130  } catch (error) {
131    $.ui.toast(`play: ${error instanceof Error ? error.message : String(error)}`)
132    await setPhase($, 'idle')
133  }
134}
135
136async function playLast($: EngineInterface, isAllowedLong: boolean) {
137  const reply = lastReply(await $.session.messages())
138  // The plain cleanup's length stands in for the script's, so the guard
139  // answers before any model call.
140  const estimate = forTheEar(reply).length
141  if (!estimate) return 'No reply to read.'
142  if (estimate > MAX_CHARS && !isAllowedLong) return `~${estimate} chars. Run /play all to spend them.`
143
144  const file = `${await cacheDir($)}/${hash(`${MODEL}:${VOICE}:${reply}`)}.mp3`
145  const cached = (await $.process.run(['test', '-s', file])).exitCode === 0
146
147  await stop($)
148  void speak($, reply, estimate, file, cached)
149
150  return cached ? 'Playing from cache.' : `Preparing ~${estimate} chars…`
151}
152
153export const register: Register = on => {
154  on('session.start', async ($, e, next) => {
155    await $.command.register({
156      name: 'play',
157      description: 'Read the last reply aloud (/play pause, back, stop, all)',
158    })
159    return next(e)
160  })
161
162  on('command.run', { command: 'play' }, async ($, e) => {
163    const arg = e.args.trim()
164    if (arg === 'stop') {
165      const wasPlaying = player !== undefined
166      await stop($)
167      return { text: wasPlaying ? 'Stopped.' : 'Nothing is playing.' }
168    }
169    if (arg === 'pause') return { text: (await togglePause($)) ? 'Toggled pause.' : 'Nothing is playing.' }
170    if (arg === 'back') return { text: (await rewind($)) ? `Back ${REWIND_SECONDS}s.` : 'Nothing is playing.' }
171
172    return { text: await playLast($, arg === 'all') }
173  })
174
175  // The band is shared: the other mods' rows stay, this one sits below them.
176  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
177    const rest = await next(e)
178    if (e.props.hasSurvey) return rest
179
180    const { phase, chars } = await read($, playback)
181    const { Box, Button, Text } = $.ui.resolve(e)
182    const isBusy = phase === 'preparing' || phase === 'generating'
183    const isPlaying = phase === 'playing' || phase === 'paused'
184
185    return (
186      <Box flexDirection="column">
187        {rest}
188        <Box gap={1}>
189          <Text dimColor>♪ play</Text>
190          {phase === 'idle' && (
191            <Button key="play" label="▶ Play last reply" onPress={async () => $.ui.toast(`play: ${await playLast($, false)}`)} />
192          )}
193          {isBusy && <Text dimColor>· {phase} {chars} chars…</Text>}
194          {isPlaying && <Text dimColor>· {phase} {chars} chars</Text>}
195          {isPlaying && <Button key="back" label={`⏪ ${REWIND_SECONDS}s`} onPress={() => rewind($)} />}
196          {isPlaying && (
197            <Button key="pause" label={phase === 'playing' ? '⏸ Pause' : '▶ Resume'} onPress={() => togglePause($)} />
198          )}
199          {(isBusy || isPlaying) && <Button key="stop" label="■ Stop" onPress={() => stop($)} />}
200        </Box>
201      </Box>
202    )
203  })
204}
205

1import type { SessionMessage } from 'claude-code'
2
3// The assistant text since the last prompt the person typed: one reply, however
4// many tool steps it took.
5export function lastReply(messages: readonly SessionMessage[]): string {
6  const parts: string[] = []
7  let i = messages.length - 1
8  while (i >= 0 && !(messages[i]?.role === 'assistant' && messages[i]?.text.trim())) i--
9  for (; i >= 0; i--) {
10    const m = messages[i]
11    if (!m) continue
12    if (m.role === 'user' && !m.toolResults?.length) break
13    if (m.role === 'assistant' && m.text.trim()) parts.unshift(m.text)
14  }
15  return parts.join('\n\n')
16}
17
18export const SCRIPT_PROMPT = `You turn an assistant's reply from a coding session into a script that a text-to-speech voice reads aloud to the person who asked. Answer with the script alone: plain sentences, no Markdown, no preamble.
19
20- Keep the substance, the order and the first-person voice. Rephrase only what does not work spoken. Do not summarize prose, and add no commentary of your own.
21- The reply may open with short progress notes written while working ("Checking the config."). Drop them, unless one carries a finding the rest of the reply does not repeat; then keep that finding in one sentence.
22- Never read code. Replace a code block with one short sentence saying what it is ("Then a shell command that installs mpv.").
23- Turn a table into a few sentences carrying its point: what it compares and what stands out.
24- Never read a URL. A list of links becomes one sentence naming what they point to.
25- Say a file path by its last part and where it lives ("register.tsx in the play mod").
26- Write identifiers and abbreviations the way a person says them: "CLI" stays CLI, "v1.3" becomes version 1.3, "snake_case_name" becomes snake case name.
27- Drop emojis.`
28
29// Markdown to sentences: what the listener would not want read out goes.
30export function forTheEar(markdown: string): string {
31  return markdown
32    .replace(/^(```|~~~)[^\n]*\n[\s\S]*?^\1[^\n]*$/gm, '')
33    .replace(/^[ \t]*\|.*$/gm, '')
34    .replace(/^[ \t]*(-{3,}|\*{3,})[ \t]*$/gm, '')
35    .replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1')
36    .replace(/\[\[(?:[^\]|]*\|)?([^\]]*)\]\]/g, '$1')
37    .replace(/https?:\/\/\S+/g, '')
38    .replace(/`([^`\n]*)`/g, (_, code: string) => spokenCode(code))
39    .replace(/(^|\s)(~|\.{0,2})\/[\w.@~/-]+/g, '$1')
40    .replace(/^[ \t]{0,3}#{1,6}\s+(.*)$/gm, (_, h: string) => endSentence(h))
41    .replace(/^[ \t]*>\s?/gm, '')
42    .replace(/^[ \t]*(?:[-*+]|\d+\.)\s+(.*)$/gm, (_, item: string) => endSentence(item))
43    .replace(/(\*\*|__|\*|_)(\S(?:.*?\S)?)\1/g, '$2')
44    .replace(/<[^>]+>/g, '')
45    .replace(/[ \t]+/g, ' ')
46    .replace(/ +([.,;:!?])/g, '$1')
47    .replace(/\n{3,}/g, '\n\n')
48    .trim()
49}
50
51// A slash command keeps its name, a path goes, anything else reads as words.
52function spokenCode(code: string): string {
53  if (/^\/[\w:-]+$/.test(code)) return code.slice(1)
54  if (code.includes('/')) return ''
55  return code.replace(/_/g, ' ')
56}
57
58function endSentence(line: string): string {
59  const trimmed = line.trim()
60  return /[.!?:;,]$/.test(trimmed) ? trimmed : `${trimmed}.`
61}
62
63// cyrb53: enough to name a cache file, not a security hash.
64export function hash(text: string): string {
65  let h1 = 0xdeadbeef
66  let h2 = 0x41c6ce57
67  for (let i = 0; i < text.length; i++) {
68    const c = text.charCodeAt(i)
69    h1 = Math.imul(h1 ^ c, 2654435761)
70    h2 = Math.imul(h2 ^ c, 1597334677)
71  }
72  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909)
73  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909)
74  return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(36)
75}
76

1export type Playback = { phase: 'idle' | 'preparing' | 'generating' | 'playing' | 'paused'; chars: number }
2
3declare module 'claude-code' {
4  interface PluginState {
5    play: { playback: Playback }
6  }
7}
8