speak

1import type { EngineInterface as Engine, Register } from 'claude-code'
2
3const STORE_KEY = 'speak.isOn'
4const LAST_KEY = 'speak.last'
5
6export function prepareSpeech(message: string): string {
7  const text = message
8    .replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, '')
9    .replace(/```[\s\S]*?```/g, ' ')
10    .replace(/`([^`]*)`/g, '$1')
11    .replace(/!?\[([^\]]+)\]\([^)]*\)/g, '$1')
12    .replace(/https?:\/\/\S+/g, 'link')
13    .replace(/^\s*\|.*\|\s*$/gm, '')
14    .replace(/^\s{0,3}#{1,6}\s*/gm, '')
15    .replace(/^\s*[-*+]\s+/gm, '')
16    .replace(/^\s*\d+[.)]\s+/gm, '')
17    .replace(/(^|\s)(\*{1,3}|_{1,3}|~~)(\S(?:[^\n]*?\S)?)\2(?=\s|[.,;:!?)]|$)/g, '$1$3')
18    .replace(/\s+/g, ' ')
19    .trim()
20
21  return text
22}
23
24// Asks macOS what language a text is in and where each of its words starts (NaturalLanguage,
25// so text without spaces such as Chinese is split too), and what the default voice speaks.
26const DETECT_JS = `ObjC.import('NaturalLanguage'); ObjC.import('AppKit');
27function run(argv) {
28  const text = argv[0]
29  const r = $.NLLanguageRecognizer.alloc.init
30  r.processString(text)
31  const h = ObjC.deepUnwrap(r.languageHypothesesWithMaximum(1)) || {}
32  const lang = Object.keys(h)[0] || 'und'
33  const t = $.NLTokenizer.alloc.initWithUnit($.NLTokenUnitWord)
34  t.string = text
35  const tokens = t.tokensForRange({ location: 0, length: $(text).length })
36  const starts = []
37  for (let i = 0; i < tokens.count; i++) starts.push(Number(tokens.objectAtIndex(i).rangeValue.location))
38  const voice = ObjC.deepUnwrap($.NSSpeechSynthesizer.attributesForVoice($.NSSpeechSynthesizer.defaultVoice)) || {}
39  return JSON.stringify({ lang, confidence: h[lang] || 0, defaultLocale: voice.VoiceLocaleIdentifier || '', starts })
40}`
41
42export type Detection = { lang: string; confidence: number; defaultLocale: string }
43
44// Where each word of a text starts, by whitespace: the stand-in when macOS can't be asked.
45export function wordStarts(text: string): number[] {
46  return [...text.matchAll(/\S+/g)].map(m => m.index ?? 0)
47}
48export type Voice = { name: string; locale: string }
49
50// Below this the recognizer is guessing (a two-word reply), so the system voice reads it.
51const MIN_CONFIDENCE = 0.6
52
53// The region a language's voice is taken from when the voices are otherwise equal.
54const MAIN_REGION: Record<string, string> = {
55  en: 'US', es: 'ES', fr: 'FR', de: 'DE', it: 'IT', pt: 'BR', ja: 'JP', ko: 'KR', nl: 'NL', ru: 'RU',
56  'zh-Hans': 'CN', 'zh-Hant': 'TW',
57}
58
59// Reads `say -v '?'`: one voice a line, `Name   xx_YY   # sample`, names may hold spaces.
60export function parseVoices(listing: string): Voice[] {
61  const voices: Voice[] = []
62  for (const line of listing.split('\n')) {
63    const m = /^(.+?)\s+([a-z]{2,3}_[A-Z0-9]{2,3})\s+#/.exec(line)
64    if (m?.[1] && m[2]) voices.push({ name: m[1].trim(), locale: m[2] })
65  }
66  return voices
67}
68
69// The installed voice for the detected language, or undefined to keep the system voice:
70// when unsure, when the system voice already speaks it, or when none is installed.
71// Premium, then Enhanced, then the plain-named voices (Alice, Thomas) before the
72// novelty ones named with their language (Grandpa (Italian (Italy))), then the main region.
73export function pickVoice(voices: Voice[], d: Detection): string | undefined {
74  const base = d.lang.split('-')[0] ?? ''
75  if (d.confidence < MIN_CONFIDENCE || base === '' || base === 'und') return undefined
76  if (d.defaultLocale.split('_')[0] === base) return undefined
77
78  const region = MAIN_REGION[d.lang] ?? MAIN_REGION[base]
79  const rank = (v: Voice) =>
80    (/\(Premium\)/.test(v.name) ? 0 : /\(Enhanced\)/.test(v.name) ? 1 : v.name.includes('(') ? 3 : 2) * 2 +
81    (v.locale.endsWith(`_${region}`) ? 0 : 1)
82  const matching = voices.filter(v => v.locale.startsWith(`${base}_`))
83  matching.sort((a, b) => rank(a) - rank(b))
84  return matching[0]?.name
85}
86
87// Installed voices, read afresh for every reply: a voice removed since would make `say`
88// fall back to the system voice without a word.
89async function listVoices($: Engine): Promise<Voice[]> {
90  let listing = ''
91  try {
92    for await (const piece of $.process.spawn({ argv: ['say', '-v', '?'] })) {
93      if (piece.stream === 'stdout') listing += piece.text
94    }
95  } catch {
96    // say missing: no voices to pick from
97  }
98  return parseVoices(listing)
99}
100
101// What reading a text needs: its voice, chosen by its language (undefined keeps the system
102// voice), and where each word starts, for resuming.
103async function analyze($: Engine, text: string): Promise<{ voice: string | undefined; starts: number[] }> {
104  let out = ''
105  // Read the voices while the language is detected, rather than after.
106  const voices = listVoices($)
107  try {
108    const argv = ['osascript', '-l', 'JavaScript', '-e', DETECT_JS, text]
109    for await (const piece of $.process.spawn({ argv })) {
110      if (piece.stream === 'stdout') out += piece.text
111    }
112    const d = JSON.parse(out) as Detection & { starts?: number[] }
113    const starts = d.starts?.length ? d.starts : wordStarts(text)
114    return { voice: pickVoice(await voices, d), starts }
115  } catch {
116    // osascript missing or unreadable answer: the system voice reads it
117    return { voice: undefined, starts: wordStarts(text) }
118  }
119}
120
121type Speech = { stop: () => void; wordsStarted: () => number; isDone: () => boolean }
122
123// `say --interactive` writes this each time it starts speaking a word. Piped, the text
124// around it is garbage, but the markers still arrive in step with the voice.
125const WORD_MARK = '\x1b[7m'
126
127// Starts `say` on the text and returns a handle to stop it and to count the words begun;
128// `onEnd` runs once it ends, stopped or finished.
129// Text goes in on stdin, so a reply starting with "-" can never be read as a flag.
130function say($: Engine, speech: string, voice: string | undefined, onEnd: () => void): Speech {
131  const argv = ['say', ...(voice ? ['-v', voice] : []), '--interactive', '-f', '-']
132  const child = $.process.spawn({ argv, input: speech })
133  const pieces: AsyncIterator<{ stream: string; text: string }> = child[Symbol.asyncIterator]()
134  let started = 0
135  // The last few characters, so a marker split across two pieces is still counted.
136  let tail = ''
137  let done = false
138
139  void (async () => {
140    try {
141      for (let piece = await pieces.next(); !piece.done; piece = await pieces.next()) {
142        if (piece.value.stream !== 'stdout') continue
143        const joined = tail + piece.value.text
144        started += joined.split(WORD_MARK).length - 1
145        tail = joined.slice(-(WORD_MARK.length - 1))
146      }
147    } catch {
148      // say missing or killed: stay silent rather than interrupt the turn
149    } finally {
150      done = true
151      onEnd()
152    }
153  })()
154
155  return { stop: () => void pieces.return?.(), wordsStarted: () => started, isDone: () => done }
156}
157
158// Words resumed from before the one cut off, so the listener picks the thread back up.
159const RESUME_BACK = 2
160
161// A reply being read: its text, where each word starts, and its voice.
162type Reading = { text: string; starts: number[]; voice: string | undefined }
163// What is being spoken, and the word this `say` began at.
164let current: { speech: Speech; reading: Reading; offset: number } | undefined
165// Where a paused reply picks up again, in the same voice.
166let paused: { reading: Reading; from: number } | undefined
167
168function halt() {
169  current?.speech.stop()
170  current = undefined
171  paused = undefined
172}
173
174// The status line names only the /speak commands that apply right now.
175async function showStatus($: Engine) {
176  const isSpeaking = current !== undefined && !current.speech.isDone()
177  const isOn = (await $.store.get(STORE_KEY)) !== false
178  $.ui.status(
179    paused ? '🔊 paused · /speak resume, stop'
180    : isSpeaking ? '🔊 speaking · /speak pause, stop'
181    : isOn ? '🔊 speak on · /speak last, off'
182    : undefined,
183  )
184}
185
186function speakFrom($: Engine, reading: Reading, from: number) {
187  halt()
188  const onEnd = () => void showStatus($).catch(() => undefined)
189  const rest = reading.text.slice(reading.starts[from] ?? 0)
190  current = { speech: say($, rest, reading.voice, onEnd), reading, offset: from }
191}
192
193export const register: Register = on => {
194  on('session.start', async ($, e, next) => {
195    await $.command.register({
196      name: 'speak',
197      description: 'Toggle spoken replies: /speak, /speak on|off, /speak pause|resume, /speak stop, /speak last',
198    })
199    await showStatus($)
200
201    return next(e)
202  })
203
204  on('command.run', { command: 'speak' }, async ($, e) => {
205    const arg = e.args.trim().toLowerCase()
206
207    if (arg === 'stop') {
208      halt()
209      await showStatus($)
210      return { text: 'Stopped speaking.' }
211    }
212
213    if (arg === 'pause') {
214      if (paused) return { text: 'Already paused.' }
215      if (!current || current.speech.isDone()) return { text: 'Nothing is being spoken.' }
216      // The word cut off is the last one begun; back up from it.
217      // say may split a few more words than the tokenizer (Chinese, Japanese): never past the last.
218      const last = Math.max(0, current.reading.starts.length - 1)
219      const cut = Math.min(last, current.offset + Math.max(0, current.speech.wordsStarted() - 1))
220      const resumeAt = { reading: current.reading, from: Math.max(0, cut - RESUME_BACK) }
221      halt()
222      paused = resumeAt
223      await showStatus($)
224      return { text: 'Paused. /speak resume to continue.' }
225    }
226
227    if (arg === 'resume') {
228      if (!paused) return { text: current && !current.speech.isDone() ? 'Already speaking.' : 'Nothing is paused.' }
229      speakFrom($, paused.reading, paused.from)
230      await showStatus($)
231      return { text: 'Resumed speaking.' }
232    }
233
234    if (arg === 'last') {
235      const last = await $.store.get(LAST_KEY)
236      if (typeof last !== 'string' || last === '') {
237        return { text: 'No reply to speak yet.' }
238      }
239      speakFrom($, { text: last, ...(await analyze($, last)) }, 0)
240      await showStatus($)
241      return { text: 'Speaking the latest reply.' }
242    }
243
244    const wasOn = (await $.store.get(STORE_KEY)) !== false
245    const isOn = arg === 'on' ? true : arg === 'off' ? false : !wasOn
246    await $.store.set(STORE_KEY, isOn)
247    if (!isOn) halt()
248    await showStatus($)
249
250    return { text: `Speaking replies is ${isOn ? 'on' : 'off'}.` }
251  })
252
253  on('turn.complete', async ($, e, next) => {
254    const ran = await next(e)
255
256    if (e.reason !== 'answer' || e.agentId !== undefined) return ran
257
258    const speech = prepareSpeech(e.answer)
259    if (speech === '') return ran
260
261    // Kept even while speaking is off, so `/speak last` can replay it.
262    await $.store.set(LAST_KEY, speech)
263    if ((await $.store.get(STORE_KEY)) === false) return ran
264
265    speakFrom($, { text: speech, ...(await analyze($, speech)) }, 0)
266    await showStatus($)
267
268    return ran
269  })
270}
271