voice-grill

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { ChatLine, Phase, VoiceSettings } from '../types'
5
6const phase = atom({ plugin: 'voice-grill', key: 'phase' } as const, 'off' as Phase)
7const live = atom({ plugin: 'voice-grill', key: 'live' } as const, '')
8const detail = atom({ plugin: 'voice-grill', key: 'detail' } as const, '')
9const chat = atom({ plugin: 'voice-grill', key: 'chat' } as const, [] as ChatLine[])
10const settings = atom({ plugin: 'voice-grill', key: 'settings' } as const, null as VoiceSettings | null)
11// The index of the word being read aloud in Claude's last line; -1 when not reading.
12const word = atom({ plugin: 'voice-grill', key: 'word' } as const, -1)
13// Reading frozen mid-word, or the mic muted, until Space (or Resume) again.
14const paused = atom({ plugin: 'voice-grill', key: 'paused' } as const, false)
15
16const PANE = 'voice-grill'
17// Only the voice-grill skill turns voice mode on; the grill skills it runs are left as they are.
18const START_SKILLS = ['voice-grill']
19const RULED_SKILLS = [...START_SKILLS, 'grilling', 'grill-me', 'grill-with-docs']
20const COMMAND = 'voice-mode'
21const DONE = '[voice-grill:done]'
22const FIRST_LOAD_MS = 20 * 60_000 // the first start downloads the models
23const CHAT_LINES = 30
24const CHAT_SHOWN = 8
25const PHASE_LABEL: Record<Phase, [string, string]> = {
26  off: ['Voice off', 'inactive'],
27  waiting: ['Waiting for Claude', 'subtle'],
28  speaking: ['Speaking', 'claude'],
29  listening: ['Listening', 'success'],
30  sending: ['Sending', 'suggestion'],
31}
32const SPEED_STEP = 0.05
33const VOLUME_STEP = 0.1
34// Kokoro's English voices by prefix: American/British, woman/man.
35const ACCENTS: Record<string, string> = { af: 'American woman', am: 'American man', bf: 'British woman', bm: 'British man' }
36
37function voiceLabel(voice: string) {
38  const [prefix = '', name = voice] = voice.split('_')
39  const accent = ACCENTS[prefix]
40  return accent ? `${name[0]?.toUpperCase()}${name.slice(1)} (${accent})` : voice
41}
42
43const RULES = `VOICE GRILLING MODE is on. It overrides the grilling skill's round format.
44The user hears each of your messages through text-to-speech and answers by voice; their replies are speech-to-text transcripts.
45- Ask exactly ONE question per message: the most important question on the current frontier. Never a numbered round.
46- Keep it speakable: plain sentences, no markdown, headings, tables, code blocks or emoji, roughly 80 words at most.
47- Give your recommended answer in one sentence starting with "My recommendation:".
48- Track the rest of the design tree yourself and move to the next frontier question after each answer.
49- Transcripts may contain filler words or misheard terms: read past them, and ask to confirm if one looks garbled.
50- Look facts up yourself as the skill says, but still end every message with exactly one question.
51- When a picture would help (a diagram, chart, layout or comparison), add one SVG in a \`\`\`svg block (with a viewBox and no script). It is drawn in the user's side panel and never read aloud; say in a few words what it shows.
52- When the frontier is empty and the user has confirmed shared understanding, end that message with the line ${DONE}`
53
54// Markdown and emoji read badly aloud.
55const speakable = (md: string): string =>
56  md
57    .replaceAll(DONE, '')
58    .replace(/```svg[\s\S]*?```/g, ' ')
59    .replace(/<svg[\s\S]*?<\/svg>/g, ' ')
60    .replace(/```[\s\S]*?```/g, ' (code omitted) ')
61    .replace(/`([^`]*)`/g, '$1')
62    .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
63    .replace(/➡️/g, 'My recommendation:')
64    .replace(/\bQ\d+\s*-\s*/g, '')
65    .replace(/^\s*(#+|>|-{3,}|[-*]\s)/gm, '')
66    .replace(/(\*\*|__|\*|_)/g, '')
67    .replace(/[\u{1F000}-\u{1FAFF}\u{2600}-\u{27BF}\u{2B00}-\u{2BFF}\u{FE0F}\u{200D}]/gu, '')
68    .replace(/\n{2,}/g, '\n')
69    .trim()
70
71type Mood = 'talking' | 'listening' | 'thinking' | 'idle' | 'paused'
72
73// Clawd for the terminal, in quarter blocks: at rest, and arms up (talking alternates the two).
74const CLAWD_REST = [' ▐▛███▜▌ ', '▝▜█████▛▘', '  ▘▘ ▝▝  ']
75const CLAWD_WAVE = ['▗▐▛███▜▌▖', ' ▜█████▛ ', '  ▘▘ ▝▝  ']
76
77// Clawd for the desktop: an 11 by 6 pixel sprite, animated by mood (SMIL, no script).
78function clawdSvg(mood: Mood) {
79  const anim = (attr: string, values: string, dur: string) =>
80    `<animate attributeName="${attr}" values="${values}" dur="${dur}" repeatCount="indefinite"/>`
81  const bob =
82    mood === 'paused'
83      ? '0 0;0 0'
84      : mood === 'talking' ? '0 0;0 -0.5;0 0' : mood === 'thinking' ? '0 0;0 -0.3;0 0' : '0 0;0 0'
85  const bobDur = mood === 'talking' ? '0.35s' : '1.6s'
86  const arm = (x: number) =>
87    `<rect x="${x}" y="2" width="2" height="1">${mood === 'talking' ? anim('y', '2;1;2', '0.35s') : ''}</rect>`
88  // Listening: he blinks now and then; thinking: his eyes look up.
89  const eye = (x: number) =>
90    `<rect x="${x}" y="${mood === 'thinking' ? 0.6 : mood === 'paused' ? 1.6 : 1}" width="1" height="${mood === 'paused' ? 0.4 : 1}" fill="#1f1e1d">${
91      mood === 'listening' ? anim('height', '1;1;1;0.15;1', '2.4s') : ''
92    }</rect>`
93  return `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 -1.5 11 7" width="66" height="42" shape-rendering="crispEdges">
94<g fill="#D97757" opacity="${mood === 'idle' ? 0.6 : 1}">
95<animateTransform attributeName="transform" type="translate" values="${bob}" dur="${bobDur}" repeatCount="indefinite"/>
96<rect x="2" y="0" width="7" height="4"/>${arm(0)}${arm(9)}
97<rect x="2" y="4" width="1" height="1"/><rect x="4" y="4" width="1" height="1"/>
98<rect x="6" y="4" width="1" height="1"/><rect x="8" y="4" width="1" height="1"/>
99${eye(3)}${eye(7)}
100</g></svg>`
101}
102
103// The volume as a small bar and a percentage.
104function volumeBar(volume: number) {
105  const filled = Math.round(Math.min(1.5, Math.max(0, volume)) * 6)
106  return `${'▮'.repeat(filled)}${'▯'.repeat(9 - filled)} ${Math.round(volume * 100)}%`
107}
108
109// Splits text around its index-th word (words as the helper counts them: split on whitespace).
110function splitAtWord(text: string, index: number) {
111  const pieces = text.split(/(\s+)/)
112  let seen = -1
113  for (let i = 0; i < pieces.length; i += 1) {
114    if (/^\s*$/.test(pieces[i] ?? '')) continue
115    seen += 1
116    if (seen === index) {
117      return { before: pieces.slice(0, i).join(''), current: pieces[i] ?? '', after: pieces.slice(i + 1).join('') }
118    }
119  }
120  return { before: text, current: '', after: '' }
121}
122
123type VoiceEvent = { seq: number; type: string; text?: string; reason?: string; index?: number; id?: string | null }
124// A question for the driver: one already streaming to the helper as Claude writes it, or a whole answer.
125type Job = { kind: 'stream'; id: string } | { kind: 'answer'; answer: string }
126// Claude's reply as it streams: sentences go to the helper as soon as they are complete.
127type Outgoing = { stream: Stream; piece: string } | { stream: Stream; end: string }
128type Stream = { id: string; raw: string; sent: number; pieces: string[]; block: number; opened: boolean; off: boolean; done: boolean }
129type Health = { ready: boolean; loading: string; error: string; stt: string; tts: string; config: string }
130
131const run = {
132  // Bumped whenever a running cycle should stop caring about its own results.
133  generation: 0,
134  uvPath: 'uv',
135  socket: '',
136  daemon: 'off' as 'off' | 'starting' | 'up',
137  seq: 0,
138  // What the mod last put in the prompt box; anything else there is the person's.
139  lastFilled: '',
140  // Set while an Enter waits for the final transcript.
141  enterWaiter: undefined as ((text: string | undefined) => void) | undefined,
142  // Claude's answers waiting for the driver, and the driver's wake-up.
143  queue: [] as Job[],
144  stream: undefined as Stream | undefined,
145  // Pieces of streaming replies (and their ends) for the sender loop, in order.
146  outbox: [] as Outgoing[],
147  wakeSender: undefined as (() => void) | undefined,
148  speechCount: 0,
149  wake: undefined as (() => void) | undefined,
150  traceLines: [] as string[],
151}
152
153// A trace of what the voice loop did, in <mod>/.trace.log, for debugging.
154function trace($: EngineInterface, message: string) {
155  const stamp = new Date().toISOString().slice(11, 23)
156  run.traceLines = [...run.traceLines, `${stamp} ${message}`].slice(-300)
157  void $.fs.write(`${$.plugin.root}/.trace.log`, `${run.traceLines.join('\n')}\n`).catch(() => undefined)
158}
159
160function helper($: EngineInterface) {
161  return `${$.plugin.root}/bin/voice.py`
162}
163
164function delay($: EngineInterface, ms: number) {
165  return new Promise<void>(resolve => {
166    $.clock.after(ms, () => resolve())
167  })
168}
169
170async function api($: EngineInterface, path: string, body?: unknown): Promise<any> {
171  const response = await $.http.fetch(`http://voice-grill${path}`, {
172    method: body === undefined ? 'GET' : 'POST',
173    headers: { 'Content-Type': 'application/json' },
174    body: body === undefined ? undefined : JSON.stringify(body),
175    socketPath: run.socket,
176  })
177  return JSON.parse(response.text)
178}
179
180function control($: EngineInterface, cmd: string) {
181  trace($, `control ${cmd}`)
182  return api($, '/control', { cmd }).catch(() => undefined)
183}
184
185async function setPhase($: EngineInterface, next: Phase) {
186  trace($, `phase ${next}`)
187  await update($, paused, () => false) // the helper starts each new step unpaused
188  return update($, phase, () => next)
189}
190
191// Pauses or resumes what the helper is doing: reading (frozen mid-word) or listening (mic muted).
192async function togglePause($: EngineInterface) {
193  const now = await read($, phase)
194  if ((now !== 'speaking' && now !== 'listening') || run.daemon !== 'up') return
195  const answer = await api($, '/control', { cmd: 'toggle-pause' }).catch(() => undefined)
196  if (answer?.ok) await update($, paused, () => Boolean(answer.paused))
197  trace($, `pause ${answer?.paused}`)
198}
199
200// Changes the voice or speed in the running helper (and the config file): no restart.
201async function changeSettings($: EngineInterface, change: { voice?: string; speed?: number; volume?: number; autoSend?: boolean }) {
202  if (run.daemon !== 'up') {
203    $.ui.toast('The speech helper is still starting; try again in a moment')
204    return
205  }
206  const next = await api($, '/settings', change).catch(err => ({ error: String(err) }))
207  if (next.error) {
208    $.ui.toast(`Could not change it: ${next.error}`)
209    return
210  }
211  trace($, `settings ${JSON.stringify(change)}`)
212  await update($, settings, () => next as VoiceSettings)
213}
214
215function say($: EngineInterface, who: ChatLine['who'], text: string, visuals?: string[]) {
216  return update($, chat, lines => [...lines, { who, text, visuals }].slice(-CHAT_LINES))
217}
218
219// Starts the speech daemon once and waits until its models are loaded.
220async function ensureDaemon($: EngineInterface): Promise<Health> {
221  if (run.daemon === 'off') {
222    run.daemon = 'starting'
223    trace($, 'daemon starting')
224    void (async () => {
225      const child = $.process.spawn({
226        argv: [run.uvPath, 'run', '--quiet', '--script', helper($), 'serve', '--socket', run.socket],
227      })
228      for await (const { text } of child) trace($, `daemon: ${text.trimEnd()}`)
229      run.daemon = 'off'
230      trace($, 'daemon exited')
231    })().catch(err => {
232      run.daemon = 'off'
233      trace($, `daemon failed: ${String(err)}`)
234    })
235  }
236  const startedAt = await $.clock.now()
237  for (;;) {
238    const health: Health | undefined = await api($, '/health').catch(() => undefined)
239    if (health?.error) throw new Error(health.error)
240    if (health?.ready) {
241      run.daemon = 'up'
242      await update($, detail, () => `${health.stt} · ${health.tts}`)
243      const current = await api($, '/settings').catch(() => null)
244      await update($, settings, () => current as VoiceSettings | null)
245      return health
246    }
247    if (health) await update($, detail, () => `${health.loading}…`)
248    // The spawn loop above sets it back to off when the helper exits.
249    if ((run.daemon as string) === 'off') throw new Error('the speech helper exited; run voice.py doctor')
250    if ((await $.clock.now()) - startedAt > FIRST_LOAD_MS) throw new Error('the speech helper did not start')
251    await delay($, 700)
252  }
253}
254
255// Long-polls the daemon until an event of one of `types` arrives; undefined once stale.
256async function waitFor(
257  $: EngineInterface,
258  mine: number,
259  types: readonly string[],
260  on: { partial?: (text: string) => Promise<void>; word?: (index: number) => Promise<void>; id?: string } = {},
261): Promise<VoiceEvent | undefined> {
262  for (;;) {
263    if (mine !== run.generation) return undefined
264    const got: { events: VoiceEvent[] } = await api($, `/events?after=${run.seq}&wait=1000`)
265    for (const event of got.events) {
266      run.seq = Math.max(run.seq, event.seq)
267      if (mine !== run.generation) return undefined
268      // A speech's events carry its id; another speech's (one that was cut off) are not ours.
269      if (on.id !== undefined && (event.type === 'word' || event.type === 'spoken') && event.id !== on.id) continue
270      if (event.type === 'partial' && on.partial) await on.partial(event.text ?? '')
271      if (event.type === 'word' && on.word) await on.word(event.index ?? 0)
272      if (event.type === 'error' || types.includes(event.type)) {
273        trace($, `event ${event.type} ${event.reason ?? ''} ${(event.text ?? '').slice(0, 80)}`)
274        return event
275      }
276    }
277  }
278}
279
280async function activate($: EngineInterface) {
281  if ((await read($, phase)) !== 'off') return
282  trace($, 'activate')
283  await update($, chat, () => [])
284  await setPhase($, 'waiting')
285  await $.ui.open({ id: PANE, title: 'Voice grilling', closeOnEscape: true })
286  $.ui.toast('Voice grilling on: talk, then press Enter (or say "send it"). Esc leaves.')
287  // Warm the models while Claude writes the first question.
288  void ensureDaemon($).catch(err => fail($, 'the speech helper failed', String(err)))
289}
290
291async function deactivate($: EngineInterface, why?: string, isPaneClosed?: boolean) {
292  trace($, `deactivate ${why ?? ''}`)
293  run.generation += 1
294  run.queue = []
295  if (run.daemon === 'up') await control($, 'off')
296  run.enterWaiter?.(undefined)
297  const box = await $.prompt.read()
298  if (run.lastFilled && box.text === run.lastFilled) await $.prompt.fill({ text: '' })
299  run.lastFilled = ''
300  await setPhase($, 'off')
301  await update($, live, () => '')
302  await update($, word, () => -1)
303  if (!isPaneClosed) await $.ui.close({ id: PANE })
304  if (why) $.ui.toast(why)
305}
306
307async function fail($: EngineInterface, what: string, err: string) {
308  trace($, `FAIL ${what}: ${err}`)
309  $.ui.log(`voice-grill: ${what}: ${err}`)
310  await deactivate($, `Voice grilling stopped: ${what}. ${err}`.slice(0, 300))
311}
312
313// The live transcript goes in the prompt box until the person edits it.
314async function fillLive($: EngineInterface, text: string) {
315  await update($, live, () => text)
316  const box = await $.prompt.read()
317  if (box.text !== run.lastFilled) {
318    trace($, `box edited by the person (${JSON.stringify(box.text.slice(0, 40))})`)
319    await control($, 'cancel') // they took over: what is in the box is theirs
320    return
321  }
322  await $.prompt.fill({ text })
323  run.lastFilled = text
324}
325
326// Waits until the helper has read speech `id`, marking each word as it is said.
327async function heard($: EngineInterface, id: string, mine: number): Promise<string | undefined> {
328  await setPhase($, 'speaking')
329  const event = await waitFor($, mine, ['spoken'], {
330    id,
331    word: async at => {
332      await update($, word, () => at)
333    },
334  })
335  await update($, word, () => -1)
336  if (event?.type === 'error') throw new Error(event.text)
337  return event?.reason
338}
339
340async function speak($: EngineInterface, text: string, mine: number): Promise<string | undefined> {
341  run.speechCount += 1
342  const id = `say-${run.speechCount}`
343  const started = await api($, '/speak', { text, id })
344  run.seq = Math.max(run.seq, started.seq)
345  return heard($, id, mine)
346}
347
348// The SVG pictures in a message, drawn in the side panel instead of read.
349function visualsOf(text: string) {
350  const fenced = [...text.matchAll(/```svg\s*([\s\S]*?)```/g)].map(found => found[1]?.trim() ?? '')
351  const bare = [...text.replace(/```[\s\S]*?```/g, '').matchAll(/<svg[\s\S]*?<\/svg>/g)].map(found => found[0])
352  return [...fenced, ...bare].filter(svg => svg.startsWith('<svg') && svg.length <= 131_072)
353}
354
355// Where the last complete sentence of `raw` ends, at or after `from`; never inside a code
356// block or SVG still being written, which is held back until it closes.
357function sentenceEnd(raw: string, from: number) {
358  const boundary = /[.!?:;](?=\s)|\n/g
359  boundary.lastIndex = from
360  let cut = from
361  for (let found = boundary.exec(raw); found; found = boundary.exec(raw)) cut = found.index + found[0].length
362  const open = /```|<svg/g
363  open.lastIndex = from
364  for (let found = open.exec(raw); found; found = open.exec(raw)) {
365    const close = found[0] === '```' ? raw.indexOf('```', found.index + 3) : raw.indexOf('</svg>', found.index)
366    if (close === -1) return Math.min(cut, found.index)
367    open.lastIndex = close + (found[0] === '```' ? 3 : 6)
368  }
369  return cut
370}
371
372// Queues the next complete sentences of a streaming reply for the helper; `isEnd` queues the rest.
373function feedStream(stream: Stream, isEnd: boolean) {
374  if (stream.off) return
375  const cut = isEnd ? stream.raw.length : sentenceEnd(stream.raw, stream.sent)
376  if (cut <= stream.sent) return
377  const piece = speakable(stream.raw.slice(stream.sent, cut))
378  stream.sent = cut
379  if (piece === '') return
380  stream.pieces.push(piece)
381  run.outbox.push({ stream, piece })
382  run.wakeSender?.()
383}
384
385// The other long-lived loop (session.start's $ outlives any one event): sends what
386// feedStream queued, so a hook never waits on the helper.
387async function sender($: EngineInterface) {
388  for (;;) {
389    const next = run.outbox.shift()
390    if (next === undefined) {
391      await new Promise<void>(resolve => {
392        run.wakeSender = resolve
393      })
394      run.wakeSender = undefined
395      continue
396    }
397    try {
398      if ('piece' in next) await sendPiece($, next.stream, next.piece)
399      else await endStream($, next.stream, next.end)
400    } catch (err) {
401      next.stream.off = true
402      trace($, `stream failed: ${String(err)}`)
403    }
404  }
405}
406
407async function sendPiece($: EngineInterface, stream: Stream, piece: string) {
408  if (stream.off) return
409  const text = stream.pieces.join(' ')
410  const visuals = visualsOf(stream.raw)
411  if (stream.opened) {
412    await api($, '/speak', { text: piece, id: stream.id, more: true })
413    await update($, chat, lines => lines.map(line => (line.id === stream.id ? { ...line, text, visuals } : line)))
414    return
415  }
416  // The first question can arrive while the models still load: wait for them.
417  if (run.daemon !== 'up') await ensureDaemon($)
418  // The first sentence: a new question starts, and anything still running for the last one stops.
419  stream.opened = true
420  run.generation += 1
421  const started = await api($, '/speak', { text: piece, id: stream.id, more: true })
422  run.seq = Math.max(run.seq, started.seq)
423  trace($, `streaming ${stream.id}`)
424  await update($, chat, lines => [...lines, { who: 'claude' as const, text, id: stream.id, visuals }].slice(-CHAT_LINES))
425  run.queue = [{ kind: 'stream', id: stream.id }]
426  run.wake?.()
427}
428
429// The reply is all in: the helper reads what is left and then says it is done. A reply
430// that never reached the helper is read whole instead.
431async function endStream($: EngineInterface, stream: Stream, answer: string) {
432  stream.done = answer.includes(DONE) || stream.raw.includes(DONE)
433  if (!stream.opened || stream.off) {
434    if (stream.opened || !answer.trim()) return
435    await say($, 'claude', speakable(answer), visualsOf(answer))
436    run.generation += 1
437    run.queue = [{ kind: 'answer', answer }]
438    run.wake?.()
439    return
440  }
441  const visuals = visualsOf(stream.raw)
442  await update($, chat, lines => lines.map(line => (line.id === stream.id ? { ...line, visuals } : line)))
443  await api($, '/speak', { id: stream.id, more: false })
444}
445
446// One question: read it aloud, then listen until the answer is sent.
447async function cycle($: EngineInterface, job: Job) {
448  const mine = run.generation
449  trace($, `cycle ${mine}: ${job.kind === 'stream' ? `stream ${job.id}` : job.answer.slice(0, 60)}`)
450  await ensureDaemon($)
451  if (mine !== run.generation) return
452
453  const stream = job.kind === 'stream' && run.stream?.id === job.id ? run.stream : undefined
454  const spoken = stream ? stream.pieces.join(' ') : job.kind === 'answer' ? speakable(job.answer) : ''
455  let reason: string | undefined
456  if (job.kind === 'stream') reason = await heard($, job.id, mine)
457  else if (spoken) reason = await speak($, spoken, mine)
458  if (reason === 'off') return deactivate($, 'Voice grilling off')
459  if (mine !== run.generation) return
460  if (stream ? stream.done : job.kind === 'answer' && job.answer.includes(DONE)) {
461    return deactivate($, 'Grilling finished: voice off')
462  }
463
464  for (;;) {
465    await setPhase($, 'listening')
466    await update($, live, () => '')
467    const box = await $.prompt.read()
468    run.lastFilled = box.text // an empty box, or what the person left there
469    const started = await api($, '/listen', {})
470    run.seq = Math.max(run.seq, started.seq)
471    const final = await waitFor($, mine, ['final'], { partial: text => fillLive($, text) })
472    if (!final) return
473    if (final.type === 'error') throw new Error(final.text)
474    const text = (final.text ?? '').trim()
475
476    if (final.reason === 'enter') {
477      run.enterWaiter?.(text)
478      await setPhase($, 'waiting')
479      return
480    }
481    if (final.reason === 'cancel') {
482      await setPhase($, 'waiting')
483      return
484    }
485    if (final.reason === 'repeat') {
486      if ((await speak($, spoken, mine)) === 'off') return deactivate($, 'Voice grilling off')
487      continue
488    }
489    if (final.reason === 'off') {
490      // Keep what was said so far in the box to send by hand.
491      run.lastFilled = ''
492      return deactivate($, 'Voice grilling off')
493    }
494    if (!text) {
495      $.ui.toast("Didn't catch anything; still listening")
496      continue
497    }
498    // Sent by the 1 key or the send phrase.
499    await setPhase($, 'sending')
500    const now = await $.prompt.read()
501    if (now.text === run.lastFilled) await $.prompt.fill({ text: '' })
502    run.lastFilled = ''
503    await update($, live, () => '')
504    await say($, 'you', text)
505    await $.prompt.submit({ text, asUser: true })
506    if (mine === run.generation) await setPhase($, 'waiting')
507    return
508  }
509}
510
511// The one long-lived loop: started by session.start, whose $ outlives its dispatch,
512// it takes each of Claude's answers in turn and runs its question cycle.
513async function driver($: EngineInterface) {
514  trace($, 'driver started')
515  for (;;) {
516    const job = run.queue.shift()
517    if (job === undefined) {
518      await new Promise<void>(resolve => {
519        run.wake = resolve
520      })
521      run.wake = undefined
522      continue
523    }
524    try {
525      await cycle($, job)
526    } catch (err) {
527      await fail($, 'voice loop failed', String(err))
528    }
529  }
530}
531
532export const register: Register = on => {
533  on('session.start', async ($, e, next) => {
534    await $.command.register({
535      name: COMMAND,
536      description: 'Voice grilling controls (/voice-grill starts it)',
537      argumentHint: '[on|off|status|config|restart]',
538    })
539    const found = await $.process
540      .run(['sh', '-c', 'command -v uv || echo "$HOME/.local/bin/uv"; echo "${XDG_RUNTIME_DIR:-/tmp}"'])
541      .catch(() => undefined)
542    const [uv, runtime] = found?.stdout.trim().split('\n') ?? []
543    run.uvPath = uv || 'uv'
544    const id = Math.random().toString(36).slice(2, 10)
545    run.socket = `${runtime || '/tmp'}/voice-grill-${id}.sock`
546    // A reload drops any cycle that was running; wait for the next answer.
547    const now = await read($, phase)
548    if (now !== 'off' && now !== 'waiting') await setPhase($, 'waiting')
549    const started = await next(e)
550    void driver($)
551    void sender($)
552
553    return started
554  })
555
556  on('command.run', { command: COMMAND }, async ($, e) => {
557    const want = e.args.trim().toLowerCase()
558    const isOn = (await read($, phase)) !== 'off'
559    if (want === 'status' || want === 'config') {
560      const health: Health | undefined =
561        run.daemon === 'up' ? await api($, '/health').catch(() => undefined) : undefined
562      const where = await $.process.run([run.uvPath, 'run', '--quiet', '--script', helper($), 'config'], {
563        timeoutMs: 120_000,
564      })
565      return {
566        text: [
567          `Voice grilling is ${isOn ? 'on' : 'off'}; speech helper ${run.daemon}.`,
568          health ? `Engines: ${health.stt} · ${health.tts}` : '',
569          `Config: ${where.stdout.trim()} (changes apply after /voice-mode restart)`,
570          `Check it with: ${helper($)} doctor`,
571          `Trace: ${$.plugin.root}/.trace.log`,
572        ]
573          .filter(Boolean)
574          .join('\n'),
575      }
576    }
577    if (want === 'restart') {
578      if (run.daemon !== 'off') await api($, '/shutdown', {}).catch(() => undefined)
579      run.daemon = 'off'
580      if (isOn) void ensureDaemon($).catch(err => fail($, 'the speech helper failed', String(err)))
581      return { text: 'Speech helper restarting with the current config.' }
582    }
583    if (want === 'off' || (want === '' && isOn)) {
584      await deactivate($)
585      return { text: 'Voice grilling off.' }
586    }
587    await activate($)
588    return { text: 'Voice grilling on. It starts reading after the next question.' }
589  })
590
591  on('skill.prompt', async ($, e, next) => {
592    if (START_SKILLS.includes(e.skill)) await activate($)
593    const result = await next(e)
594    if (!RULED_SKILLS.includes(e.skill) || (await read($, phase)) === 'off') return result
595
596    return { text: `${result.text}\n\n${RULES}` }
597  })
598
599  on('prompt.compose', async ($, e, next) => {
600    const composed = await next(e)
601    if ((await read($, phase)) === 'off') return composed
602
603    return {
604      sections: [...composed.sections, { id: 'voice-grill:rules', text: RULES, scope: 'session' }],
605    }
606  })
607
608  // Enter on the live transcript sends the final transcript; typing your own takes over.
609  on('prompt.submit', async ($, e, next) => {
610    if (e.origin.kind === 'plugin') return next(e)
611    try {
612      const now = await read($, phase)
613      if (now === 'listening' && e.text.trim() && e.text === run.lastFilled) {
614        trace($, 'enter on the live transcript')
615        const final = await new Promise<string | undefined>(resolve => {
616          run.enterWaiter = resolve
617          $.clock.after(8000, () => resolve(undefined))
618          void control($, 'enter')
619        })
620        run.enterWaiter = undefined
621        run.lastFilled = ''
622        await update($, live, () => '')
623        await say($, 'you', final || e.text)
624        return next({ ...e, text: final || e.text })
625      }
626      if (now !== 'off') await say($, 'you', e.text)
627      if (now === 'speaking' || now === 'listening') {
628        trace($, 'typed prompt takes over')
629        run.generation += 1
630        run.lastFilled = ''
631        await control($, 'cancel')
632        await update($, live, () => '')
633        await setPhase($, 'waiting')
634      }
635    } catch (err) {
636      trace($, `prompt.submit: ${String(err)}`)
637    }
638
639    return next(e)
640  })
641
642  // Claude's reply streams to the helper sentence by sentence, so reading starts while it writes.
643  on('turn.step', async function* ($, e, next) {
644    const steps = next(e)
645    if (e.agentId || (await read($, phase)) === 'off') return yield* steps
646    if (run.stream?.id !== e.turnId) {
647      run.stream = { id: e.turnId, raw: '', sent: 0, pieces: [], block: -1, opened: false, off: false, done: false }
648    }
649    const stream = run.stream
650    for (;;) {
651      const step = await steps.next()
652      if (step.done) {
653        feedStream(stream, true) // a step's text is complete when it ends
654        return step.value
655      }
656      const chunk = step.value
657      if (chunk.kind === 'text') {
658        if (chunk.index !== stream.block && stream.raw !== '') stream.raw += '\n'
659        stream.block = chunk.index
660        stream.raw += chunk.text
661        feedStream(stream, false)
662      }
663      yield chunk
664    }
665  })
666
667  // Space in the composer while it holds only what the mod put there pauses or resumes.
668  on('prompt.edit', async ($, e, next) => {
669    if (e.key?.key !== ' ' || e.inputText !== ' ' || e.key.ctrl || e.key.meta) return next(e)
670    const now = await read($, phase)
671    if ((now !== 'speaking' && now !== 'listening') || e.text !== run.lastFilled) return next(e)
672    await togglePause($)
673    return next({ ...e, inputText: '' })
674  })
675
676  on('turn.complete', async ($, e, next) => {
677    const result = await next(e)
678    if (e.agentId || (await read($, phase)) === 'off') return result
679    if (e.isAborted) {
680      // Esc during a turn leaves voice mode too.
681      await deactivate($, 'Voice grilling off')
682      return result
683    }
684    const stream = run.stream?.id === e.turnId ? run.stream : undefined
685    if (stream && !stream.off && stream.pieces.length > 0) {
686      // Being read already (or about to be, once the models load): the sender finishes it.
687      feedStream(stream, true)
688      run.outbox.push({ stream, end: e.answer })
689      run.wakeSender?.()
690      return result
691    }
692    if (e.reason !== 'answer' || !e.answer.trim()) return result
693    await say($, 'claude', speakable(e.answer), visualsOf(e.answer))
694    // Hand the answer to the driver; a cycle still running for an older one stops.
695    run.generation += 1
696    run.queue = [{ kind: 'answer', answer: e.answer }]
697    trace($, `queued answer (driver ${run.wake ? 'idle' : 'busy'})`)
698    run.wake?.()
699
700    return result
701  })
702
703  // Esc at an empty, idle prompt closes the pane, and that ends voice mode.
704  on('ui.close', async ($, e, next) => {
705    if (e.id === PANE && e.origin.kind === 'person' && (await read($, phase)) !== 'off') {
706      await deactivate($, 'Voice grilling off', true)
707    }
708    return next(e)
709  })
710
711  // Ctrl+D in the pane's key catcher: leave voice mode and type instead.
712  on('ui.message', async ($, e, next) => {
713    if (e.requestId !== PANE || e.element !== 'keys') return next(e)
714    const data = e.data as { type?: string } | null
715    if (data?.type === 'stop') await deactivate($, 'Voice grilling off: type away')
716    if (data?.type === 'pause') await togglePause($)
717    return {}
718  })
719
720  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
721    const elements = $.ui.resolve(e)
722    const { Box, Button, Text } = elements
723    // The key catcher needs a Client, which the terminal and desktop draw; Svg draws Clawd where it can.
724    const Client = 'Client' in elements ? elements.Client : undefined
725    const Select = 'Select' in elements ? elements.Select : undefined
726    // The terminal's table has an Svg that draws nothing there, so it gets the block-character Clawd.
727    const Svg = e.surface !== 'terminal' && 'Svg' in elements ? elements.Svg : undefined
728    const now = await read($, phase)
729    const heardText = await read($, live)
730    const engines = await read($, detail)
731    const lines = await read($, chat)
732    const tts = await read($, settings)
733    const saying = await read($, word)
734    const english = (tts?.voices ?? []).filter(voice => (voice.split('_')[0] ?? '') in ACCENTS)
735    const at = english.indexOf(tts?.voice ?? '')
736    const step = (by: number) => english[(at + by + english.length) % english.length] ?? english[0]
737    const shown = lines.slice(-CHAT_SHOWN)
738    const lastClaude = shown.map(line => line.who).lastIndexOf('claude')
739    const isPaused = await read($, paused)
740    const [label, tone] = isPaused ? ['Paused', 'warning'] : PHASE_LABEL[now]
741    const thinking = (now === 'waiting' || now === 'sending') && shown[shown.length - 1]?.who === 'you'
742
743    // Claude's line being read aloud: said words plain, the current one marked, the rest dim.
744    const reading = (text: string) => {
745      const { before, current, after } = splitAtWord(text, saying)
746      return (
747        <Text>
748          {before}
749          <Text bold color="inverseText" backgroundColor="claude">
750            {current}
751          </Text>
752          <Text dimColor>{after}</Text>
753        </Text>
754      )
755    }
756    // Clawd: animated SVG on the desktop; on the terminal his arms wave word by word.
757    const mood: Mood = isPaused
758      ? 'paused'
759      : now === 'speaking' ? 'talking' : now === 'listening' ? 'listening' : thinking ? 'thinking' : 'idle'
760    const clawd = Svg ? (
761      <Svg key="clawd" source={clawdSvg(mood)} alt={`Clawd, ${mood}`} width={66} height={48} isInteractive />
762    ) : (
763      <Text color="claude" dimColor={mood === 'idle'}>
764        {(mood === 'talking' && saying % 2 === 1 ? CLAWD_WAVE : CLAWD_REST).join('\n')}
765      </Text>
766    )
767
768    return (
769      <Box flexDirection="column" gap={1}>
770        <Box borderStyle="round" borderColor="claude" paddingX={1} gap={2} alignItems="center">
771          {clawd}
772          <Box flexDirection="column">
773            <Text bold>Voice grilling</Text>
774            <Box alignItems="center" gap={1}>
775              <Text>
776                <Text color={tone}>● </Text>
777                {label}
778              </Text>
779              {(now === 'speaking' || now === 'listening') && (
780                <Button key="pause" label={isPaused ? '▶ Resume' : '❚❚ Pause'} onPress={() => togglePause($)} />
781              )}
782            </Box>
783            {engines !== '' && <Text dimColor>{engines}</Text>}
784            {now !== 'off' && Client && <Client key="keys" module="./keys.tsx" props={{}} />}
785          </Box>
786        </Box>
787
788        {now !== 'off' && tts && (
789          <Box flexDirection="column" paddingX={1}>
790            {english.length > 0 && !tts.voiceNeedsRestart && Select && (
791              <Select
792                key="voice"
793                label="Voice"
794                value={tts.voice}
795                options={english.map(voice => ({ key: voice, value: voice, label: voiceLabel(voice) }))}
796                onSelect={(voice: string) => changeSettings($, { voice })}
797              />
798            )}
799            {english.length > 0 && !tts.voiceNeedsRestart && !Select && (
800              <Box alignItems="center" gap={1}>
801                <Text dimColor>Voice </Text>
802                <Button key="voice-prev" label="◀" onPress={() => changeSettings($, { voice: step(-1) })} />
803                <Text>{voiceLabel(tts.voice)}</Text>
804                <Button key="voice-next" label="▶" onPress={() => changeSettings($, { voice: step(1) })} />
805              </Box>
806            )}
807            {tts.voiceNeedsRestart && <Text dimColor>Voice {tts.voice} (change it in the config, then /voice-mode restart)</Text>}
808            <Box alignItems="center" gap={1} flexWrap="wrap">
809              <Text dimColor>Speed </Text>
810              <Button key="slower" label="−" onPress={() => changeSettings($, { speed: tts.speed - SPEED_STEP })} />
811              <Text bold>{tts.speed.toFixed(2)}×</Text>
812              <Button key="faster" label="+" onPress={() => changeSettings($, { speed: tts.speed + SPEED_STEP })} />
813              <Text dimColor>   Volume </Text>
814              <Button
815                key="quieter"
816                label="−"
817                onPress={() => changeSettings($, { volume: (tts.volume ?? 1) - VOLUME_STEP })}
818              />
819              <Text bold>{volumeBar(tts.volume ?? 1)}</Text>
820              <Button
821                key="louder"
822                label="+"
823                onPress={() => changeSettings($, { volume: (tts.volume ?? 1) + VOLUME_STEP })}
824              />
825            </Box>
826            <Box>
827              <Button
828                key="auto-send"
829                label={`${tts.autoSend ? '☑' : '☐'} Auto-send after a ${tts.autoSendSeconds ?? 2.5}s pause`}
830                onPress={() => changeSettings($, { autoSend: !tts.autoSend })}
831              />
832            </Box>
833          </Box>
834        )}
835
836        <Box flexDirection="column" gap={1} paddingX={1}>
837          {shown.map((line, index) =>
838            line.who === 'claude' ? (
839              <Box key={`line-${index}`} flexDirection="column" gap={1}>
840                <Box>
841                  <Text color={now === 'speaking' && index === lastClaude ? 'claude' : 'text'}>⏺ </Text>
842                  <Box flexShrink={1}>
843                    {now === 'speaking' && index === lastClaude && saying >= 0 ? reading(line.text) : <Text>{line.text}</Text>}
844                  </Box>
845                </Box>
846                {Svg &&
847                  (line.visuals ?? []).map((svg, at) => (
848                    <Svg key={`visual-${index}-${at}`} source={svg} alt="A picture from Claude" isInteractive />
849                  ))}
850              </Box>
851            ) : (
852              <Box key={`line-${index}`}>
853                <Text dimColor>&gt; </Text>
854                <Box flexShrink={1}>
855                  <Text dimColor>{line.text}</Text>
856                </Box>
857              </Box>
858            ),
859          )}
860        </Box>
861
862        {now !== 'off' && (
863          <Box borderStyle="round" borderColor={now === 'listening' ? 'success' : 'promptBorder'} paddingX={1}>
864            <Text>
865              <Text color={now === 'listening' ? 'success' : 'subtle'}>&gt; </Text>
866              {now === 'listening' && <Text>{heardText === '' ? '▍' : `${heardText} ▍`}</Text>}
867            </Text>
868          </Box>
869        )}
870      </Box>
871    )
872  })
873
874  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
875    const now = await read($, phase)
876    if (now === 'off' || e.props.hasSurvey) return next(e)
877
878    const { Box, Button, Text } = $.ui.resolve(e)
879    const engines = await read($, detail)
880    const heard = await read($, live)
881    const isPaused = await read($, paused)
882    const status: Record<Phase, string> = {
883      off: '',
884      waiting: 'waiting for the next question…',
885      speaking: 'reading the question…',
886      listening: 'listening: press Enter or 1, or say "send it"',
887      sending: 'sending your answer…',
888    }
889
890    return (
891      <Box flexDirection="column">
892        <Text>
893          <Text color="red">● </Text>
894          Voice grilling: {isPaused ? 'paused (Space resumes)' : status[now]}
895          {now === 'waiting' && engines.endsWith('…') ? ` (${engines})` : ''}
896        </Text>
897        {now === 'listening' && heard !== '' && <Text dimColor>“{heard}”</Text>}
898        <Box>
899          {now === 'speaking' && (
900            <Button key="skip" hotkey="1" label="Skip to answering" onPress={() => control($, 'skip')} />
901          )}
902          {now === 'listening' && (
903            <Button key="send" hotkey="1" label="Send answer" onPress={() => control($, 'send')} />
904          )}
905          {now === 'listening' && (
906            <Button key="repeat" hotkey="2" label="Repeat question" onPress={() => control($, 'repeat')} />
907          )}
908          {(now === 'speaking' || now === 'listening') && (
909            <Button key="pause" hotkey="4" label={isPaused ? 'Resume' : 'Pause'} onPress={() => togglePause($)} />
910          )}
911          <Button key="off" hotkey="3" label="Voice off" onPress={() => deactivate($, 'Voice grilling off')} />
912        </Box>
913      </Box>
914    )
915  })
916}
917

1import type { ClientModule } from 'claude-code'
2
3// Catches keys while the pane has the focus (click it first): Ctrl+D leaves voice mode, Space pauses.
4const Keys: ClientModule = (_props, surface) => {
5  const { Text } = surface.elements
6  surface.onKey(event => {
7    if (event.ctrl && event.key.toLowerCase() === 'd') surface.post({ type: 'stop' })
8    if (event.key === ' ' && !event.ctrl && !event.meta) surface.post({ type: 'pause' })
9  })
10
11  return <Text dimColor>Space pauses · Ctrl+D types instead</Text>
12}
13
14export default Keys
15

1export type Phase = 'off' | 'waiting' | 'speaking' | 'listening' | 'sending'
2export type ChatLine = { who: 'claude' | 'you'; text: string; id?: string; visuals?: string[] }
3export type VoiceSettings = { voice: string; speed: number; volume: number; autoSend: boolean; autoSendSeconds: number; voices: string[]; voiceNeedsRestart: boolean }
4
5declare module 'claude-code' {
6  interface PluginState {
7    'voice-grill': { phase: Phase; live: string; detail: string; chat: ChatLine[]; settings: VoiceSettings | null; word: number; paused: boolean }
8  }
9}
10