live-voice

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { Phase } from '../types'
5import { LineReader, forSpeech, parseEvent } from './speech'
6import type { ListenerEvent } from './speech'
7
8const phase = atom({ plugin: 'live-voice', key: 'phase' } as const, 'off')
9const heard = atom({ plugin: 'live-voice', key: 'heard' } as const, '')
10
11// A pause this long after the last words sends them; talking again inside it adds to them.
12export const SEND_AFTER_MS = 1500
13const SPOKEN_CHARS = 700
14const MODELS = ['tiny', 'base', 'small', 'medium']
15const HINT = 'on | off | status | setup | engine openai|local | language nl | speak on|off | voice <name> | device <name> | model small'
16
17const LABELS: Record<Phase, string> = {
18  off: 'off',
19  loading: 'starting…',
20  listening: 'listening',
21  hearing: 'hearing you…',
22  transcribing: 'writing down what you said…',
23  thinking: 'Claude is working',
24  speaking: 'speaking',
25}
26
27/** openai: OpenAI hears and speaks (gpt-4o-transcribe, gpt-4o-mini-tts); local: Whisper and a Windows voice. */
28export type Engine = 'openai' | 'local'
29export type Settings = { engine: Engine; language: string; speak: boolean; device: string; model: string; voice: string }
30const DEFAULTS: Settings = { engine: 'openai', language: 'nl', speak: true, device: '', model: 'small', voice: '' }
31const OPENAI_VOICE = 'marin'
32
33type Stream = ReturnType<EngineInterface['process']['spawn']>
34
35export type State = {
36  settings: Settings
37  isWindows: boolean
38  /** ~/.claude-live-voice: the venv, the models and the control files. */
39  dir: string
40  /** The file listen.py and speak.ps1 poll: listen, pause, hush or stop. */
41  control: string
42  listener: Stream | null
43  ended: Promise<void>
44  speaking: boolean
45  isStopping: boolean
46  /** What was said since the last send, waiting out SEND_AFTER_MS. */
47  words: string[]
48  sendTimer: { cancel: () => void } | null
49}
50
51export function freshState(): State {
52  return {
53    settings: { ...DEFAULTS }, isWindows: false, dir: '', control: '', listener: null, ended: Promise.resolve(),
54    speaking: false, isStopping: false, words: [], sendTimer: null,
55  }
56}
57
58export function settingsFrom(raw: unknown): Settings {
59  const v = typeof raw === 'object' && raw !== null ? (raw as Record<string, unknown>) : {}
60  const text = (key: 'language' | 'device' | 'model' | 'voice') => (typeof v[key] === 'string' ? (v[key] as string) : DEFAULTS[key])
61  return {
62    engine: v.engine === 'local' || v.engine === 'openai' ? v.engine : DEFAULTS.engine,
63    language: text('language'),
64    speak: typeof v.speak === 'boolean' ? v.speak : DEFAULTS.speak,
65    device: text('device'),
66    model: text('model'),
67    voice: text('voice'),
68  }
69}
70
71function clip(text: string, max: number): string {
72  return text.length <= max ? text : '…' + text.slice(text.length - max + 1)
73}
74
75function python(s: State): string {
76  return s.isWindows ? `${s.dir}/venv/Scripts/python.exe` : `${s.dir}/venv/bin/python`
77}
78
79function describe(s: State): string {
80  const { engine, language, model, speak, device, voice } = s.settings
81  const ears = engine === 'openai' ? 'OpenAI' : `Whisper ${model}`
82  const mouth = engine === 'openai' ? voice || OPENAI_VOICE : voice || 'Windows'
83  return `language ${language} · ${ears} · read aloud ${speak ? `on (${mouth})` : 'off'} · microphone ${device || 'default'}`
84}
85
86/** The API key the scripts will read, by either name; never stored or shown. */
87async function hasKey($: EngineInterface): Promise<boolean> {
88  return Boolean((await $.env.get('OPENAI_API_KEY')) || (await $.env.get('CHATGPT_API_KEY')))
89}
90
91function cancelSend(s: State) {
92  s.sendTimer?.cancel()
93  s.sendTimer = null
94}
95
96async function setPhase($: EngineInterface, p: Phase) {
97  await update($, phase, () => p)
98}
99
100async function setControl($: EngineInterface, s: State, word: 'listen' | 'pause' | 'hush' | 'stop') {
101  await $.fs.write(s.control, word)
102}
103
104async function start($: EngineInterface, s: State): Promise<string> {
105  if (s.listener) return 'live voice is already on'
106  const isOpenai = s.settings.engine === 'openai'
107  if (!(await $.fs.exists(python(s)))) {
108    return isOpenai
109      ? 'live voice needs its setup first: /listen setup (a minute; a Python environment with sounddevice)'
110      : 'live voice needs its speech setup first: /listen setup (a few minutes; Python packages and the Whisper model, about 600 MB)'
111  }
112  if (isOpenai && !(await hasKey($))) {
113    return 'live voice with OpenAI needs an API key in the environment variable OPENAI_API_KEY (or CHATGPT_API_KEY). ' +
114      'Set it, restart Claude, and /listen again; or /listen engine local for Whisper and a Windows voice.'
115  }
116  s.isStopping = false
117  await setControl($, s, 'listen')
118  await setPhase($, 'loading')
119  const argv = [python(s), '-I', `${$.plugin.root}/scripts/listen.py`, '--engine', s.settings.engine,
120    '--language', s.settings.language, '--control', s.control]
121  if (!isOpenai) argv.push('--models', `${s.dir}/models`, '--model', s.settings.model)
122  if (s.settings.device) argv.push('--device', s.settings.device)
123  const stream = $.process.spawn({ argv, env: { HF_HUB_OFFLINE: '1', HF_HUB_DISABLE_SYMLINKS_WARNING: '1' } })
124  s.listener = stream
125  s.ended = follow($, s, stream)
126  return `live voice starting: ${describe(s)}. Talk once the band says listening; /listen off stops it.`
127}
128
129/** Reads listen.py for as long as it runs; its end, for whatever reason, turns voice off. */
130async function follow($: EngineInterface, s: State, stream: Stream) {
131  const lines = new LineReader()
132  const errors: string[] = []
133  try {
134    for await (const { stream: pipe, text } of stream) {
135      if (pipe === 'stderr') {
136        errors.push(text)
137        continue
138      }
139      for (const line of lines.push(text)) {
140        const event = parseEvent(line)
141        if (event) await onEvent($, s, event)
142      }
143    }
144  } catch (err) {
145    errors.push(err instanceof Error ? err.message : String(err))
146  } finally {
147    if (s.listener === stream) s.listener = null
148    cancelSend(s)
149    s.words = []
150    try {
151      await update($, heard, () => '')
152      await setPhase($, 'off')
153      if (!s.isStopping) {
154        const why = errors.join('').trim().split(/\r?\n/).slice(-2).join(' ')
155        $.ui.log(`live voice stopped${why ? `: ${why}` : ''}`)
156      }
157    } catch {
158      // The module is unloading; nothing is left to draw on.
159    }
160  }
161}
162
163async function stop($: EngineInterface, s: State): Promise<string> {
164  const stream = s.listener
165  if (!stream) return 'live voice is off'
166  s.isStopping = true
167  cancelSend(s)
168  await setControl($, s, 'stop')
169  await Promise.race([s.ended, $.clock.sleep(3000)])
170  if (s.listener === stream) void stream.return(undefined)
171  return 'live voice off'
172}
173
174async function onEvent($: EngineInterface, s: State, event: ListenerEvent) {
175  switch (event.type) {
176    case 'ready':
177      await setPhase($, 'listening')
178      $.ui.toast('live voice: listening')
179      return
180    case 'speech':
181      cancelSend(s)
182      await setPhase($, 'hearing')
183      return
184    case 'heard':
185      await setPhase($, 'transcribing')
186      return
187    case 'ignored':
188      await setPhase($, 'listening')
189      if (s.words.length) schedule($, s)
190      return
191    case 'error':
192      $.ui.log(`live voice: ${event.message}`)
193      return
194    case 'text':
195      s.words.push(event.text)
196      await update($, heard, () => s.words.join(' '))
197      await setPhase($, 'listening')
198      schedule($, s)
199  }
200}
201
202function schedule($: EngineInterface, s: State) {
203  cancelSend(s)
204  s.sendTimer = $.clock.after(SEND_AFTER_MS, () => {
205    void send($, s)
206  })
207}
208
209async function send($: EngineInterface, s: State) {
210  s.sendTimer = null
211  const text = s.words.join(' ').trim()
212  s.words = []
213  if (!text || !s.listener) return
214  await setControl($, s, 'pause')
215  await setPhase($, 'thinking')
216  // The person's own words, so Claude reads them bare, not framed as a plugin's message.
217  const entered = await $.prompt.submit({ text, asUser: true })
218  if (entered.drop !== undefined) {
219    $.ui.log(`live voice: not sent, ${entered.drop}`)
220    await resume($, s)
221  }
222}
223
224async function resume($: EngineInterface, s: State) {
225  if (!s.listener) return
226  await update($, heard, () => '')
227  await setControl($, s, 'listen')
228  await setPhase($, 'listening')
229}
230
231/** The command that speaks on this machine with this engine; null where the engine's own speech is used. */
232function speaker(root: string, s: State): string[] | null {
233  const language = s.settings.language === 'auto' ? 'nl' : s.settings.language
234  if (s.settings.engine === 'openai') {
235    return [python(s), '-I', `${root}/scripts/speak.py`, '--voice', s.settings.voice || OPENAI_VOICE,
236      '--language', language, '--control', s.control]
237  }
238  if (!s.isWindows) return null
239  const argv = ['powershell.exe', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
240    '-File', `${root}/scripts/speak.ps1`, '-Language', language, '-StopFile', s.control]
241  if (s.settings.voice) argv.push('-Voice', s.settings.voice)
242  return argv
243}
244
245async function say($: EngineInterface, s: State, text: string) {
246  s.speaking = true
247  try {
248    const argv = speaker($.plugin.root, s)
249    if (argv) {
250      let errors = ''
251      for await (const { stream, text: out } of $.process.spawn({ argv, input: text })) {
252        if (stream === 'stderr') errors += out
253      }
254      if (errors.trim()) $.ui.log(`live voice could not speak: ${errors.trim().split(/\r?\n/).at(-1)}`)
255    } else {
256      await $.audio.speak(text, s.settings.voice ? { voice: s.settings.voice } : undefined)
257    }
258  } catch (err) {
259    $.ui.log(`live voice could not speak: ${err instanceof Error ? err.message : String(err)}`)
260  } finally {
261    s.speaking = false
262  }
263}
264
265/** After a turn: read the reply aloud, then listen again. */
266async function answer($: EngineInterface, s: State, text: string) {
267  if (text) {
268    await setPhase($, 'speaking')
269    await say($, s, text)
270  }
271  await resume($, s)
272}
273
274async function setup($: EngineInterface, s: State): Promise<string> {
275  if (!s.isWindows) return 'setup runs on Windows only for now; elsewhere create the venv by hand (see the README)'
276  $.ui.toast(`live voice: setting up for ${s.settings.engine === 'openai' ? 'OpenAI, a minute' : 'Whisper, a few minutes'}…`)
277  const r = await $.process.run(['powershell.exe', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
278    '-File', `${$.plugin.root}/scripts/setup.ps1`, '-Dir', s.dir, '-Engine', s.settings.engine, '-Model', s.settings.model],
279  { timeoutMs: 600_000 })
280  if (r.exitCode === 0) return 'live voice is set up. /listen to start.'
281  const tail = `${r.stdout}\n${r.stderr}`.trim().split(/\r?\n/).slice(-4).join('\n')
282  return `live voice setup failed (exit ${r.exitCode}):\n${tail}`
283}
284
285async function save($: EngineInterface, s: State, change: Partial<Settings>, restart: boolean): Promise<string> {
286  s.settings = { ...s.settings, ...change }
287  await $.store.set('settings', s.settings)
288  if (!restart || !s.listener) return describe(s)
289  await stop($, s)
290  return start($, s)
291}
292
293async function command($: EngineInterface, s: State, args: string): Promise<string> {
294  const [word, ...rest] = args.trim().split(/\s+/).filter(Boolean)
295  const arg = rest.join(' ')
296  switch (word) {
297    case undefined:
298      return s.listener ? stop($, s) : start($, s)
299    case 'on':
300      return start($, s)
301    case 'off':
302      return stop($, s)
303    case 'status':
304      return `live voice ${LABELS[await read($, phase)]} · ${describe(s)}`
305    case 'setup':
306      return setup($, s)
307    case 'engine':
308      if (arg !== 'openai' && arg !== 'local') return '/listen engine takes openai (OpenAI hears and speaks) or local (Whisper and a Windows voice)'
309      return save($, s, { engine: arg, voice: '' }, true)
310    case 'language':
311      if (!/^([a-z]{2}|auto)$/.test(arg)) return '/listen language takes a code such as nl or en, or auto'
312      return save($, s, { language: arg }, true)
313    case 'speak':
314      if (arg !== 'on' && arg !== 'off') return '/listen speak takes on or off'
315      return save($, s, { speak: arg === 'on' }, false)
316    case 'device':
317      if (!arg) return '/listen device takes part of the microphone\'s name, its number, or default'
318      return save($, s, { device: arg === 'default' ? '' : arg }, true)
319    case 'model':
320      if (!MODELS.includes(arg)) return `/listen model takes ${MODELS.join(', ')}; a new one needs /listen setup to download it`
321      return save($, s, { model: arg }, true)
322    case 'voice':
323      if (!arg) {
324        return s.settings.engine === 'openai'
325          ? '/listen voice takes an OpenAI voice (marin, cedar, coral, sage, alloy, ash, ballad, echo, fable, nova, onyx, shimmer, verse) or default'
326          : '/listen voice takes a Windows voice\'s name (Microsoft Frank) or default'
327      }
328      return save($, s, { voice: arg === 'default' ? '' : arg }, false)
329    default:
330      return `/listen [${HINT}]`
331  }
332}
333
334export const register: Register = on => {
335  const s = freshState()
336
337  on('session.start', async ($, e, next) => {
338    const started = await next(e)
339    s.isWindows = (await $.env.get('OS')) === 'Windows_NT'
340    const home = (await $.env.get('USERPROFILE')) || (await $.env.get('HOME')) || '.'
341    s.dir = `${home}/.claude-live-voice`
342    s.control = `${s.dir}/control-${await $.session.id()}.txt`
343    s.settings = settingsFrom(await $.store.get('settings'))
344    await $.command.register({
345      name: 'listen',
346      description: 'Hands-free voice: talk to Claude and hear the reply (live-voice)',
347      argumentHint: `[${HINT}]`,
348      immediate: true,
349    })
350    return started
351  })
352
353  on('command.run', { command: 'listen' }, async ($, e) => ({ text: await command($, s, String(e.args ?? '')) }))
354
355  // A prompt the person types while voice is on stops the microphone for the turn too.
356  on('turn.start', async ($, e, next) => {
357    if (s.listener) {
358      cancelSend(s)
359      s.words = []
360      await setControl($, s, 'pause')
361      await setPhase($, 'thinking')
362    }
363    return next(e)
364  })
365
366  on('turn.complete', async ($, e, next) => {
367    const done = await next(e)
368    if (e.agentId || !s.listener) return done
369    const reply = 'answer' in e && typeof e.answer === 'string' ? e.answer : ''
370    void answer($, s, e.isAborted || !s.settings.speak ? '' : forSpeech(reply, SPOKEN_CHARS))
371    return done
372  })
373
374  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
375    const rest = await next(e)
376    const now = await read($, phase)
377    if (now === 'off' || e.props.hasSurvey) return rest
378    const said = await read($, heard)
379    const { Box, Button, Text } = $.ui.resolve(e)
380    const live = now === 'listening' || now === 'hearing'
381    return (
382      <Box flexDirection="column">
383        {rest}
384        <Box flexDirection="row">
385          <Text>
386            <Text bold color={live ? 'success' : undefined}>● live voice</Text>
387            <Text dimColor>{` · ${LABELS[now]}${said ? ` · “${clip(said, 80)}”` : ''} `}</Text>
388          </Text>
389          {now === 'speaking' && s.speaking ? (
390            <Button key="hush" label="Stop talking" onPress={() => setControl($, s, 'hush')} />
391          ) : null}
392          <Button key="off" label="Off" onPress={() => stop($, s)} />
393        </Box>
394      </Box>
395    )
396  })
397}
398

1// What crosses between the mod and its two helpers: listen.py's JSON lines in,
2// plain speakable text out.
3
4export type ListenerEvent =
5  | { type: 'ready' | 'speech' | 'heard' | 'ignored' }
6  | { type: 'text'; text: string }
7  | { type: 'error'; message: string }
8
9/** One line of listen.py's output; null for anything that is not one of its events. */
10export function parseEvent(line: string): ListenerEvent | null {
11  let value: unknown
12  try {
13    value = JSON.parse(line)
14  } catch {
15    return null
16  }
17  if (typeof value !== 'object' || value === null) return null
18  const v = value as Record<string, unknown>
19  if (v.type === 'text') return typeof v.text === 'string' && v.text.trim() ? { type: 'text', text: v.text.trim() } : null
20  if (v.type === 'error') return { type: 'error', message: typeof v.message === 'string' ? v.message : 'unknown error' }
21  if (v.type === 'ready' || v.type === 'speech' || v.type === 'heard' || v.type === 'ignored') return { type: v.type }
22  return null
23}
24
25/** Cuts a stream of text pieces into whole lines; a piece may end mid-line. */
26export class LineReader {
27  private rest = ''
28
29  push(piece: string): string[] {
30    const lines = (this.rest + piece).split(/\r?\n/)
31    this.rest = lines.pop() ?? ''
32    return lines.filter(line => line.trim() !== '')
33  }
34}
35
36/**
37 * Turns a markdown reply into text worth reading aloud: code blocks, tables and
38 * links' targets go, the markup goes, and the result stops at a sentence end
39 * within `max` characters.
40 */
41export function forSpeech(markdown: string, max: number): string {
42  const text = markdown
43    .replace(/```[\s\S]*?(```|$)/g, ' ')
44    .split('\n')
45    .filter(line => !/^\s*\|/.test(line))
46    .join('\n')
47    .replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1')
48    .replace(/https?:\/\/\S+/g, ' ')
49    .replace(/`([^`]*)`/g, '$1')
50    .replace(/^\s{0,3}#{1,6}\s+/gm, '')
51    .replace(/^\s*(?:[-*+]|\d+[.)])\s+/gm, '')
52    .replace(/^\s*>\s?/gm, '')
53    .replace(/(\*\*|__|\*|_|~~)(?=\S)([^*_~\n]+?)(?<=\S)\1/g, '$2')
54    .replace(/\s+/g, ' ')
55    .trim()
56  if (text.length <= max) return text
57  const cut = text.slice(0, max)
58  const end = Math.max(cut.lastIndexOf('. '), cut.lastIndexOf('? '), cut.lastIndexOf('! '))
59  return end > max / 3 ? cut.slice(0, end + 1) : cut.replace(/\s+\S*$/, '') + '…'
60}
61

1/** Where the conversation stands: drawn in the band above the prompt. */
2export type Phase = 'off' | 'loading' | 'listening' | 'hearing' | 'transcribing' | 'thinking' | 'speaking'
3
4declare module 'claude-code' {
5  interface PluginState {
6    'live-voice': { phase: Phase; heard: string }
7  }
8}
9