talk-mode

1import { atom, read, update } from 'claude-code'
2import type { Register } from 'claude-code'
3
4import { Speakable } from './speakable'
5
6
7const isOn = atom({ plugin: 'talk-mode', key: 'isOn' } as const, false)
8const status = atom({ plugin: 'talk-mode', key: 'status' } as const, '')
9
10const VOICE_PROMPT = [
11  'Talk mode is on: the person is speaking to you and hears your replies read aloud.',
12  'Messages from the talk-mode plugin are transcribed speech and can contain recognition errors; if one looks cut off or garbled, ask a short question instead of guessing.',
13  'Write replies as plain spoken sentences: short, no headings, tables, bullet lists or emphasis marks.',
14  'Before using tools, first say in one sentence what you heard and what you will do.',
15  'Put code, commands and paths in code blocks; they are not read aloud, so say in words what they do.',
16].join(' ')
17
18let port = 0
19let turnId: string | undefined
20let spoke = false
21let quitAsked = false // End talk pressed before the helper was ready
22let modelsDir = '' // the modelsDir option; empty means ~/.talk-mode/models
23
24/** Bun from its default install folder, else from PATH. */
25async function bunPath($: any): Promise<string> {
26  const isWin = (await $.env.get('OS')) === 'Windows_NT'
27  const home = isWin ? await $.env.get('USERPROFILE') : await $.env.get('HOME')
28  const base = (await $.env.get('BUN_INSTALL')) ?? `${home ?? ''}/.bun`
29  const exe = `${base}/bin/bun${isWin ? '.exe' : ''}`
30  return (await $.fs.stat(exe).then(() => true, () => false)) ? exe : 'bun'
31}
32
33async function post($: any, path: string, body: object) {
34  if (!port) return
35  try {
36    await $.http.fetch(`http://127.0.0.1:${port}${path}`, {
37      method: 'POST',
38      headers: { 'content-type': 'application/json' },
39      body: JSON.stringify(body),
40    })
41  } catch {
42    // helper gone: the spawn loop below notices and turns talk mode off
43  }
44}
45
46function setStatus($: any, text: string) {
47  return update($, status, () => text)
48}
49
50async function start($: any) {
51  if (await read($, isOn)) return
52  quitAsked = false
53  await update($, isOn, () => true)
54  await setStatus($, 'starting')
55  void (async () => {
56    let rest = ''
57    let errors = ''
58    try {
59      const env: Record<string, string> = modelsDir ? { TALK_MODELS: modelsDir } : {}
60      const child = $.process.spawn({ argv: [await bunPath($), `${$.plugin.root}/sidecar/start.ts`], env })
61      let step = await child.next()
62      for (; !step.done; step = await child.next()) {
63        const piece = step.value
64        if (piece.stream !== 'stdout') { errors = (errors + piece.text).slice(-600); continue }
65        rest += piece.text
66        let nl: number
67        while ((nl = rest.indexOf('\n')) >= 0) {
68          const line = rest.slice(0, nl)
69          rest = rest.slice(nl + 1)
70          let m: any
71          try { m = JSON.parse(line) } catch { continue }
72          if (m.t === 'setup') await setStatus($, m.msg)
73          else if (m.t === 'ready') {
74            port = m.port
75            if (quitAsked) await post($, '/quit', {})
76            else await setStatus($, 'loading speech models')
77          }
78          else if (m.t === 'connected') await setStatus($, 'talk window open')
79          else if (m.t === 'heard') await setStatus($, 'hearing you')
80          else if (m.t === 'closed') { await post($, '/quit', {}); return }
81          else if (m.t === 'text') {
82            await setStatus($, 'listening')
83            if (turnId) await $.turn.abort({ turnId }).catch(() => {})
84            void $.prompt.submit({ text: m.text })
85          } else if (m.t === 'log') $.ui.log(`talk-mode: ${m.msg}`, { to: 'debug' })
86        }
87      }
88      if (step.value.code !== 0 && !quitAsked) {
89        const why = errors.trim().split(/\r?\n/).filter(Boolean).pop() ?? `exit code ${step.value.code}`
90        $.ui.toast(`Talk mode stopped: ${why}`)
91      }
92    } catch (err) {
93      $.ui.toast(`Talk mode could not start: ${String(err)}`)
94    } finally {
95      port = 0
96      await update($, isOn, () => false)
97      await setStatus($, '')
98    }
99  })()
100}
101
102async function stop($: any) {
103  quitAsked = true
104  await post($, '/quit', {})
105}
106
107
108export const register: Register = (on, options) => {
109  modelsDir = String((options as { modelsDir?: string }).modelsDir ?? '').trim()
110
111  on('session.start', async ($, e, next) => {
112    await $.command.register({ name: 'talk', description: 'Talk to Claude by voice; run it again to stop' })
113    return next(e)
114  })
115
116  on('command.run', { command: 'talk' }, async $ => {
117    if (await read($, isOn)) {
118      await stop($)
119      return { text: 'Talk mode stopped.' }
120    }
121    await start($)
122    return { text: 'Talk mode starting: a Talk window opens; press Start there.' }
123  })
124
125  on('session.end', async ($, e, next) => {
126    await stop($)
127    return next(e)
128  })
129
130  on('prompt.compose', async ($, e, next) => {
131    const out = await next(e)
132    if (!(await read($, isOn))) return out
133    return { sections: [...out.sections, { id: 'talk-mode:voice', text: VOICE_PROMPT, scope: 'session' as const }] }
134  })
135
136  on('turn.start', async ($, e, next) => {
137    if (e.agentId === undefined) {
138      turnId = e.turnId
139      spoke = false
140      if (port) void post($, '/state', { state: 'think' })
141    }
142    return next(e)
143  })
144
145  on('turn.step', async function* ($, e, next) {
146    const stream = next(e)
147    const voiced = e.agentId === undefined && port !== 0
148    const speak = new Speakable()
149    const say = (list: string[]) => {
150      for (const text of list) { spoke = true; void post($, '/say', { text }) }
151    }
152    let r = await stream.next()
153    while (!r.done) {
154      const chunk = r.value
155      if (voiced && chunk.kind === 'text') say(speak.push(chunk.text))
156      yield chunk
157      r = await stream.next()
158    }
159    if (voiced) say(speak.end())
160    return r.value
161  })
162
163  on('turn.complete', async ($, e, next) => {
164    if (e.agentId === undefined) {
165      turnId = undefined
166      if (port && !spoke) void post($, '/state', { state: 'idle' })
167    }
168    return next(e)
169  })
170
171  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
172    const { Box, Button, Text } = $.ui.resolve(e)
173    const on_ = await read($, isOn)
174    const text = await read($, status)
175    return (
176      <Box>
177        <Button
178          key="talk"
179          label={on_ ? 'End talk' : 'Talk'}
180          variant={on_ ? 'secondary' : 'primary'}
181          onPress={() => (on_ ? stop($) : start($))}
182        />
183        {text ? <Text dimColor> {text}</Text> : null}
184      </Box>
185    )
186  })
187}
188

1// Turns streamed reply text into sentences worth speaking: code blocks are
2// skipped, Markdown marks removed, sentences cut at . ! ? or a line end.
3
4const clean = (s: string) =>
5  s
6    .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // [label](url) -> label
7    .replace(/https?:\/\/\S+/g, 'a link')
8    .replace(/`([^`]+)`/g, '$1')
9    .replace(/^\s*(#{1,6}|[-*+]|\d+\.)\s+/, '')
10    .replace(/[*_~|>]+/g, '')
11    .replace(/\s+/g, ' ')
12    .trim()
13
14export class Speakable {
15  private line = ''
16  private sentence = ''
17  private inFence = false
18
19  /** Adds streamed text; returns the sentences it completes. */
20  push(text: string): string[] {
21    const out: string[] = []
22    this.line += text
23    let nl: number
24    while ((nl = this.line.indexOf('\n')) >= 0) {
25      const one = this.line.slice(0, nl)
26      this.line = this.line.slice(nl + 1)
27      if (/^\s*(```|~~~)/.test(one)) { this.inFence = !this.inFence; out.push(...this.cut(true)); continue }
28      if (this.inFence || /^\s*\|/.test(one)) continue // code and table rows are not read out
29      this.sentence += one + ' '
30      out.push(...this.cut(true))
31    }
32    // speak whole sentences of a long line before the line ends, once it is clearly prose
33    if (!this.inFence && this.line.length > 3 && !/^\s*(```|~~~|\|)/.test(this.line)) {
34      this.sentence += this.line
35      this.line = ''
36      out.push(...this.cut(false))
37    }
38    return out
39  }
40
41  /** Ends the reply; returns what is left. */
42  end(): string[] {
43    if (!this.inFence && this.line) this.sentence += this.line
44    this.line = ''
45    this.inFence = false
46    return this.cut(true)
47  }
48
49  private cut(atEnd: boolean): string[] {
50    const out: string[] = []
51    let m: RegExpMatchArray | null
52    while ((m = this.sentence.match(/^(.*?[.!?])\s+(.*)$/s))) {
53      const s = clean(m[1] ?? "")
54      if (s) out.push(s)
55      this.sentence = m[2] ?? ""
56    }
57    if (atEnd) {
58      const s = clean(this.sentence)
59      if (s) out.push(s)
60      this.sentence = ''
61    }
62    return out
63  }
64}
65

1declare module 'claude-code' {
2  interface PluginState {
3    'talk-mode': { isOn: boolean; status: string }
4  }
5}
6