voice-replies

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { VoiceEngine, VoiceStatus } from '../types'
5
6const isOn = atom({ plugin: 'voice-replies', key: 'isOn' } as const, true)
7const status = atom({ plugin: 'voice-replies', key: 'status' } as const, 'idle' as VoiceStatus)
8const voice = atom({ plugin: 'voice-replies', key: 'voice' } as const, 'af_heart')
9const speed = atom({ plugin: 'voice-replies', key: 'speed' } as const, '1.0')
10const engine = atom({ plugin: 'voice-replies', key: 'engine' } as const, 'starting' as VoiceEngine)
11
12const MAX_SPOKEN_CHARS = 1500
13const SERVER = 'http://127.0.0.1:47861'
14
15// Kokoro-82M English voices. First letter: a = American, b = British. Second: f/m.
16const VOICES = [
17  'af_heart', 'af_alloy', 'af_aoede', 'af_bella', 'af_jessica', 'af_kore', 'af_nicole',
18  'af_nova', 'af_river', 'af_sarah', 'af_sky',
19  'am_adam', 'am_echo', 'am_eric', 'am_fenrir', 'am_liam', 'am_michael', 'am_onyx',
20  'am_puck', 'am_santa',
21  'bf_alice', 'bf_emma', 'bf_isabella', 'bf_lily',
22  'bm_daniel', 'bm_fable', 'bm_george', 'bm_lewis',
23] as const
24const SPEEDS = ['0.8', '0.9', '1.0', '1.1', '1.2', '1.3', '1.5'] as const
25
26const SPEECH_RULES = `You turn a coding assistant's chat reply into a short script that a text-to-speech voice reads aloud to the person who asked.
27
28Rules:
29- Output plain spoken sentences only. No markdown, bullets, headings, tables, code, emojis or URLs.
30- Lead with the outcome: what was done, whether it worked, what the person needs to do next.
31- Keep it under 80 words. A short, plain reply can stay almost word for word.
32- Never read code, commands, file paths, IDs or hashes character by character. Say what they are ("the config file", "a git command") instead.
33- Say numbers, versions and counts naturally.
34- If the reply asks the person a question, end with that question.
35- Output only the script, nothing before or after it.`
36
37// A speech run in flight. `runId` drops stale work; `speakerPid` lets Stop kill the Windows voice.
38let runId = 0
39let speakerPid: number | undefined
40let hasWarnedSetup = false
41
42function voiceLabel(name: string): string {
43  const accent = name.startsWith('b') ? 'UK' : 'US'
44  const gender = name[1] === 'm' ? 'male' : 'female'
45  const first = name.slice(3)
46  return `${first.charAt(0).toUpperCase()}${first.slice(1)} (${accent} ${gender})`
47}
48
49export const register: Register = on => {
50  on('session.start', async ($, e, next) => {
51    const saved = await Promise.all([
52      $.store.get('isOn'),
53      $.store.get('voice'),
54      $.store.get('speed'),
55    ])
56    const [savedOn, savedVoice, savedSpeed] = saved
57    if (typeof savedOn === 'boolean') await update($, isOn, () => savedOn)
58    if (typeof savedVoice === 'string' && (VOICES as readonly string[]).includes(savedVoice)) {
59      await update($, voice, () => savedVoice)
60    }
61    if (typeof savedSpeed === 'string' && (SPEEDS as readonly string[]).includes(savedSpeed)) {
62      await update($, speed, () => savedSpeed)
63    }
64    await update($, status, () => 'idle')
65    await $.command.register({
66      name: 'voice',
67      description: 'Voice replies: on, off, stop, voice <name>, speed <0.8-1.5>, voices, test',
68      argumentHint: '[on|off|stop|voice <name>|speed <n>|voices|test]',
69    })
70    // Warm the server up only where someone is watching; elsewhere it starts on first use.
71    if ((await $.session.surfaces()).length > 0) $.clock.after(0, () => void ensureServer($))
72
73    return next(e)
74  })
75
76  on('command.run', { command: 'voice' }, async ($, e) => {
77    const [word = '', value = ''] = e.args.trim().toLowerCase().split(/\s+/)
78    if (word === 'stop') {
79      await stopSpeaking($)
80      return { text: 'Stopped speaking.' }
81    }
82    if (word === 'voices') {
83      return { text: `Voices: ${VOICES.map(v => `${v} = ${voiceLabel(v)}`).join(', ')}` }
84    }
85    if (word === 'voice') {
86      if (!(VOICES as readonly string[]).includes(value)) {
87        return { text: `Unknown voice "${value}". Type /voice voices to list them.` }
88      }
89      await setVoice($, value)
90      return { text: `Voice set to ${voiceLabel(value)}.` }
91    }
92    if (word === 'speed') {
93      if (!(SPEEDS as readonly string[]).includes(value)) {
94        return { text: `Speed must be one of ${SPEEDS.join(', ')}.` }
95      }
96      await setSpeed($, value)
97      return { text: `Speed set to ${value}.` }
98    }
99    if (word === 'test') {
100      $.clock.after(0, () => void speakScript($, 'This is how voice replies will sound.'))
101      return { text: 'Playing a test phrase.' }
102    }
103    const want = word === 'on' ? true : word === 'off' ? false : !(await read($, isOn))
104    await setOn($, want)
105
106    return { text: want ? 'Voice replies are on.' : 'Voice replies are off.' }
107  })
108
109  // A new prompt means the person moved on: stop talking over them.
110  on('prompt.submit', ($, e, next) => {
111    void stopSpeaking($)
112
113    return next(e)
114  })
115
116  on('turn.complete', async ($, e, next) => {
117    const result = await next(e)
118    const isMainAnswer = e.agentId === undefined && e.reason === 'answer' && !e.isAborted
119    if (!isMainAnswer || e.answer.trim() === '' || !(await read($, isOn))) return result
120    // Stay silent where nobody is watching: `claude -p` runs, scheduled tasks, agents
121    // driven over the SDK with no app attached.
122    if ((await $.session.surfaces()).length === 0) return result
123
124    // Speaking outlasts the hook's 10 s budget, so it runs on a timer of its own.
125    const answer = e.answer
126    $.clock.after(0, () => void speakReply($, answer))
127
128    return result
129  })
130
131  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
132    if (e.props.hasSurvey) return next(e)
133
134    const ui = $.ui.resolve(e)
135    const { Box, Button, Text } = ui
136    const Select = 'Select' in ui ? ui.Select : undefined
137    const enabled = await read($, isOn)
138    const now = await read($, status)
139    const chosen = await read($, voice)
140    const pace = await read($, speed)
141    const via = await read($, engine)
142    const state = !enabled ? 'off' : now === 'writing' ? 'preparing' : now === 'speaking' ? 'speaking' : 'on'
143    const viaText = via === 'kokoro' ? 'Kokoro GPU' : via === 'windows' ? 'Windows voice' : 'starting Kokoro'
144
145    return (
146      <Box flexDirection="row" gap={1} alignItems="center">
147        <Text dimColor>
148          Voice replies: {state} ({viaText})
149        </Text>
150        <Button
151          key="toggle"
152          label={enabled ? 'Turn off' : 'Turn on'}
153          dimColor
154          onPress={() => setOn($, !enabled)}
155        />
156        {now !== 'idle' && <Button key="stop" label="Stop" onPress={() => stopSpeaking($)} />}
157        {enabled && Select !== undefined && (
158          <Select
159            key="voice"
160            label="Voice"
161            value={chosen}
162            options={VOICES.map(v => ({ value: v, label: voiceLabel(v) }))}
163            onSelect={(value: string) => void setVoice($, value)}
164          />
165        )}
166        {enabled && Select !== undefined && (
167          <Select
168            key="speed"
169            label="Speed"
170            value={pace}
171            options={SPEEDS.map(s => ({ value: s, label: `${s}x` }))}
172            onSelect={(value: string) => void setSpeed($, value)}
173          />
174        )}
175      </Box>
176    )
177  })
178}
179
180async function setOn($: EngineInterface, value: boolean): Promise<void> {
181  await update($, isOn, () => value)
182  await $.store.set('isOn', value)
183  if (!value) await stopSpeaking($)
184}
185
186async function setVoice($: EngineInterface, value: string): Promise<void> {
187  await update($, voice, () => value)
188  await $.store.set('voice', value)
189}
190
191async function setSpeed($: EngineInterface, value: string): Promise<void> {
192  await update($, speed, () => value)
193  await $.store.set('speed', value)
194}
195
196async function isServerUp($: EngineInterface): Promise<boolean> {
197  try {
198    const res = await $.http.fetch(`${SERVER}/health`)
199    return res.ok
200  } catch {
201    return false
202  }
203}
204
205// Starts the shared Kokoro server if it is not running. It runs hidden, outlives this
206// session, and exits by itself after two idle hours.
207async function ensureServer($: EngineInterface): Promise<boolean> {
208  if (await isServerUp($)) {
209    await update($, engine, () => 'kokoro')
210    return true
211  }
212  const home = await $.env.get('USERPROFILE')
213  if (home === undefined) {
214    await update($, engine, () => 'windows')
215    return false
216  }
217  const dir = `${home}\\.claude\\voice-replies`
218  const isSetUp = await $.fs.stat(`${dir}\\venv\\Scripts\\pythonw.exe`).then(
219    () => true,
220    () => false,
221  )
222  if (!isSetUp) {
223    await update($, engine, () => 'windows')
224    if (!hasWarnedSetup) {
225      hasWarnedSetup = true
226      $.ui.toast('Kokoro is not set up, so replies use the Windows voice. Run server\\setup.ps1 from the voice-replies plugin.')
227    }
228    return false
229  }
230  await update($, engine, () => 'starting')
231  try {
232    await $.process.run(
233      [
234        'powershell.exe', '-NoProfile', '-NonInteractive', '-Command',
235        `Start-Process -WindowStyle Hidden -FilePath '${dir}\\venv\\Scripts\\pythonw.exe' -ArgumentList '"${dir}\\server.py"' -WorkingDirectory '${dir}'`,
236      ],
237      { timeoutMs: 15000 },
238    )
239  } catch {
240    await update($, engine, () => 'windows')
241    return false
242  }
243  // The model takes a few seconds to load onto the GPU.
244  for (let i = 0; i < 60; i += 1) {
245    await wait($, 1000)
246    if (await isServerUp($)) {
247      await update($, engine, () => 'kokoro')
248      return true
249    }
250  }
251  await update($, engine, () => 'windows')
252  return false
253}
254
255// A pause built on $.clock.after, which runs outside any hook's time budget.
256function wait($: EngineInterface, ms: number): Promise<void> {
257  return new Promise(resolve => void $.clock.after(ms, resolve))
258}
259
260async function stopSpeaking($: EngineInterface): Promise<void> {
261  runId += 1
262  const pid = speakerPid
263  speakerPid = undefined
264  await update($, status, () => 'idle')
265  try {
266    await $.http.fetch(`${SERVER}/stop`, { method: 'POST' })
267  } catch {
268    // Server not running: nothing to stop there.
269  }
270  if (pid === undefined) return
271  try {
272    await $.process.run(['taskkill', '/PID', String(pid), '/T', '/F'], { timeoutMs: 5000 })
273  } catch {
274    // Already gone, or not Windows: nothing to stop.
275  }
276}
277
278async function speakReply($: EngineInterface, answer: string): Promise<void> {
279  await stopSpeaking($)
280  const id = runId
281  await update($, status, () => 'writing')
282
283  const script = await toSpeech($, answer)
284  if (id !== runId) return
285  await playScript($, script, id)
286}
287
288async function speakScript($: EngineInterface, script: string): Promise<void> {
289  await stopSpeaking($)
290  await playScript($, script, runId)
291}
292
293async function playScript($: EngineInterface, script: string, id: number): Promise<void> {
294  if (script === '') {
295    await update($, status, () => 'idle')
296    return
297  }
298  await update($, status, () => 'speaking')
299  try {
300    const spoke = await speakKokoro($, script, id)
301    if (!spoke && id === runId) await speakWindows($, script, id)
302  } catch (err) {
303    $.ui.log(`voice-replies: could not speak: ${String(err)}`, { to: 'debug' })
304  } finally {
305    if (id === runId) await update($, status, () => 'idle')
306  }
307}
308
309async function toSpeech($: EngineInterface, answer: string): Promise<string> {
310  const reply = await $.model.complete({
311    model: 'haiku',
312    system: [{ text: SPEECH_RULES, cache: true }],
313    prompt: `Reply to convert:\n\n${answer.slice(0, 20000)}`,
314    maxTokens: 400,
315    effort: 'low',
316    timeoutMs: 15000,
317  })
318  const text = reply.isAnswered ? reply.text : stripMarkdown(answer)
319
320  return text.trim().slice(0, MAX_SPOKEN_CHARS)
321}
322
323// Kokoro on the GPU. Returns false only when the server did not take the text, so the
324// caller falls back to the Windows voice. Once Kokoro has taken the text it never falls
325// back: the server answers /speak at once and this polls /status until the voice ends.
326async function speakKokoro($: EngineInterface, text: string, id: number): Promise<boolean> {
327  if (!(await isServerUp($)) && !(await ensureServer($))) return false
328  let job: number
329  try {
330    const res = await $.http.fetch(`${SERVER}/speak`, {
331      method: 'POST',
332      headers: { 'Content-Type': 'application/json' },
333      body: JSON.stringify({
334        text,
335        voice: await read($, voice),
336        speed: Number(await read($, speed)),
337      }),
338    })
339    const body = res.ok ? (JSON.parse(res.text) as { id?: unknown }) : {}
340    if (typeof body.id !== 'number') {
341      $.ui.log(`voice-replies: Kokoro said ${res.status}: ${res.text}`, { to: 'debug' })
342      await update($, engine, () => 'windows')
343      return false
344    }
345    job = body.id
346  } catch {
347    await update($, engine, () => 'windows')
348    return false
349  }
350  await update($, engine, () => 'kokoro')
351
352  // Kokoro has it now. Wait until it ends, is stopped, or the server goes quiet.
353  let misses = 0
354  for (let i = 0; i < 1200 && id === runId; i += 1) {
355    await wait($, 500)
356    try {
357      const res = await $.http.fetch(`${SERVER}/status?id=${job}`)
358      const { state } = JSON.parse(res.text) as { state?: string }
359      misses = 0
360      if (state !== 'queued' && state !== 'speaking') break
361    } catch {
362      misses += 1
363      if (misses >= 6) break
364    }
365  }
366  return true
367}
368
369// Windows: the built-in SAPI voice through PowerShell, which prints its PID so Stop can kill it.
370// Elsewhere (no powershell.exe): the platform voice through $.audio.speak.
371const PS_SPEAK = [
372  '[Console]::InputEncoding = [Text.Encoding]::UTF8',
373  '[Console]::Out.WriteLine($PID); [Console]::Out.Flush()',
374  '$t = [Console]::In.ReadToEnd()',
375  'Add-Type -AssemblyName System.Speech',
376  '$s = New-Object System.Speech.Synthesis.SpeechSynthesizer',
377  '$s.Speak($t)',
378].join('; ')
379
380async function speakWindows($: EngineInterface, text: string, id: number): Promise<void> {
381  let started = false
382  try {
383    const child = $.process.spawn({
384      argv: ['powershell.exe', '-NoProfile', '-NonInteractive', '-Command', PS_SPEAK],
385      input: text,
386    })
387    for await (const { stream, text: out } of child) {
388      started = true
389      if (stream !== 'stdout' || speakerPid !== undefined) continue
390      const pid = Number.parseInt(out.trim(), 10)
391      if (Number.isFinite(pid)) speakerPid = pid
392      if (id !== runId) await stopSpeaking($)
393    }
394    if (id === runId) speakerPid = undefined
395    return
396  } catch (err) {
397    if (started) throw err
398  }
399  await $.audio.speak(text)
400}
401
402function stripMarkdown(md: string): string {
403  return md
404    .replace(/```[\s\S]*?```/g, ' (code omitted) ')
405    .replace(/`([^`]*)`/g, '$1')
406    .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
407    .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
408    .replace(/^\s{0,3}#{1,6}\s+/gm, '')
409    .replace(/^\s*[-*+]\s+/gm, '')
410    .replace(/^\s*\|.*\|\s*$/gm, '')
411    .replace(/[*_~>]+/g, '')
412    .replace(/https?:\/\/\S+/g, 'a link')
413    .replace(/\n{2,}/g, '. ')
414    .replace(/\s+/g, ' ')
415    .trim()
416}
417

1export type VoiceStatus = 'idle' | 'writing' | 'speaking'
2export type VoiceEngine = 'kokoro' | 'windows' | 'starting'
3
4declare module 'claude-code' {
5  interface PluginState {
6    'voice-replies': {
7      isOn: boolean
8      status: VoiceStatus
9      voice: string
10      speed: string
11      engine: VoiceEngine
12    }
13  }
14}
15