presence

1// Presence: talk with Claude Code through the Avatar app.
2//
3// Avatar (a menu bar app) is the face, the voice and the ears, with its own controls (Play/Pause,
4// Stop, Mic, Man/Woman, Pin, and the speech bubble on a click of the head, with a caret to play
5// from). This mod drives it: replies are spoken as they are written, the face follows the turn,
6// and with the mic on, between turns, what you say goes into Claude Code's prompt box, where you
7// can edit it by keyboard or by voice ("scratch that", "clear") and send it ("send it").
8
9import { atom, read, update } from 'claude-code'
10import type { EngineInterface, Register } from 'claude-code'
11
12import { appendText, earsTrouble, parseUtterance, withoutLast, type EarsEvent } from './ears'
13import { applyEdit, EditHistory, parseEdit } from './editing'
14import { EventLines, requestBody, type AvatarEvent } from './link'
15import { Segmenter } from './segmenter'
16
17const COMMAND = 'presence'
18const COMPOSE_SECTION = 'presence:spoken-replies'
19
20const voiceAtom = atom({ plugin: 'presence', key: 'voice' } as const, true)
21const micAtom = atom({ plugin: 'presence', key: 'mic' } as const, false)
22const earsAtom = atom({ plugin: 'presence', key: 'ears' } as const, 'off')
23const hearingAtom = atom({ plugin: 'presence', key: 'hearing' } as const, null as string | null)
24
25const SPOKEN_REPLIES = [
26  'The user is listening to your replies: everything you write outside code blocks is read aloud by an avatar as you write it.',
27  'Open with a one-sentence answer, then the detail. Write for the ear: short sentences, no tables for prose.',
28  'Code, commands and long output belong in fenced code blocks; they are shown, and announced rather than read.',
29  "You can set the avatar's mood with a cue such as [happy] or [concerned] before a sentence; it is not spoken.",
30  'Do not call any speak tool to say your reply: it is already spoken.',
31].join(' ')
32
33/** The options its manifest's `userConfig` declares. */
34export type Options = { sendPhrase: string; linkPath: string; socket: string }
35
36export function readOptions(raw: Record<string, unknown>): Options {
37  const text = (value: unknown, fallback: string) =>
38    typeof value === 'string' && value.trim().length > 0 ? value.trim() : fallback
39  return {
40    sendPhrase: text(raw.sendPhrase, 'send it'),
41    linkPath: text(raw.linkPath, '/Applications/Avatar.app/Contents/MacOS/avatar-link'),
42    socket: typeof raw.socket === 'string' ? raw.socket.trim() : '',
43  }
44}
45
46// MARK: Session state (module variables: a reload starts afresh and attaches anew)
47
48let options: Options = readOptions({})
49/** Someone is at this session (the terminal's REPL, or the desktop app attached): it attaches to Avatar. */
50let started = false
51let sessionName = ''
52let socketPath = ''
53let attached = false
54let face = 'male'
55let turnRunning = false
56/** The reply id of the running turn (Avatar numbers each reply's segments). */
57let replyId = ''
58/** The text the last dictated phrase appended to the prompt box, for "scratch that". */
59let lastAppended = ''
60/** The prompt box's voice edits, for "undo that" and "redo that". */
61const history = new EditHistory()
62/** Everything sent to Avatar, in order: each request waits for the one before it. */
63let chain: Promise<unknown> = Promise.resolve()
64/** Bumped by Stop: speech queued before it is dropped, and the reply being written goes quiet. */
65let speechGeneration = 0
66/**
67 * The sentences of a reply written before Avatar was attached, said once it is: the desktop app
68 * starts the session with its first prompt, so its first reply is written while the session attaches.
69 */
70let unsaid: { reply: string; segments: string[]; endedAt: number | null } | null = null
71/** How long after its turn ended a reply nobody heard is still said when Avatar attaches. */
72const UNSAID_FOR_MS = 60_000
73
74type Answer = { ok: boolean; status: number; body: Record<string, unknown> }
75
76/** POSTs to one of Avatar's routes; never rejects. */
77async function post($: EngineInterface, route: string, fields: Record<string, unknown> = {}): Promise<Answer> {
78  if (!attached) return { ok: false, status: 0, body: {} }
79  try {
80    const response = await $.http.fetch(`http://avatar${route}`, {
81      method: 'POST',
82      headers: { 'Content-Type': 'application/json' },
83      body: requestBody(sessionName, fields),
84      socketPath,
85    })
86    let body: Record<string, unknown> = {}
87    try {
88      body = JSON.parse(response.text) as Record<string, unknown>
89    } catch {
90      body = {}
91    }
92    return { ok: response.ok, status: response.status, body }
93  } catch {
94    return { ok: false, status: 0, body: {} }
95  }
96}
97
98/** Runs `work` after everything sent before it, so the voice keeps the reply's order. */
99function inOrder(work: () => Promise<unknown>): Promise<unknown> {
100  chain = chain.then(work, work)
101  return chain
102}
103
104/** Replies are spoken: Avatar is running and voice is on for this session. */
105async function speaks($: EngineInterface): Promise<boolean> {
106  return started && attached && (await read($, voiceAtom))
107}
108
109/** Says one sentence of reply `reply`, after what came before. */
110function say($: EngineInterface, text: string, reply: string): Promise<unknown> {
111  const generation = speechGeneration
112  return inOrder(async () => {
113    if (generation === speechGeneration) await post($, '/say', { reply, text })
114  })
115}
116
117/** Says a sentence now, or keeps it for when Avatar is attached. */
118function sayOrKeep($: EngineInterface, text: string, reply: string): void {
119  if (attached) {
120    void say($, text, reply)
121    return
122  }
123  if (unsaid?.reply !== reply) unsaid = { reply, segments: [], endedAt: null }
124  unsaid.segments.push(text)
125}
126
127/** Avatar has just attached: says what was kept of the reply, and ends it if its turn has. */
128async function sayUnsaid($: EngineInterface): Promise<void> {
129  const kept = unsaid
130  unsaid = null
131  if (kept === null) return
132  if (kept.endedAt !== null && (await $.clock.now()) - kept.endedAt > UNSAID_FOR_MS) return
133  for (const segment of kept.segments) void say($, segment, kept.reply)
134  if (kept.endedAt !== null) void inOrder(() => post($, '/reply/end', { reply: kept.reply }))
135}
136
137/** Silence: drops what is queued, stops what is being said. */
138async function stopSpeaking($: EngineInterface): Promise<void> {
139  unsaid = null
140  speechGeneration += 1
141  chain = Promise.resolve()
142  await post($, '/stop')
143}
144
145async function setFace($: EngineInterface, presence: 'listening' | 'thinking' | 'none', turn?: 'running' | 'idle'): Promise<void> {
146  await post($, '/state', { presence, turn })
147}
148
149/** Between turns: listening if the mic is on, else nothing. */
150async function idleFace($: EngineInterface): Promise<void> {
151  await setFace($, (await read($, micAtom)) ? 'listening' : 'none', 'idle')
152}
153
154/** Which face speaks, as Avatar last said. The band above the prompt shows it, so draw it again. */
155function setFaceName($: EngineInterface, name: string | undefined): void {
156  if (name === undefined || name === face) return
157  face = name
158  $.ui.invalidate('ui.render')
159}
160
161async function setMic($: EngineInterface, on: boolean): Promise<void> {
162  await update($, micAtom, () => on)
163  await post($, '/listen', { on })
164  if (!on) {
165    await update($, hearingAtom, () => null)
166    await update($, earsAtom, () => 'off')
167  }
168  if (!turnRunning) await idleFace($)
169}
170
171// MARK: Attaching (the session's `avatar-link attach` child)
172
173/** Keeps a child attached for the session's life: retried while Avatar isn't running. */
174async function attachLoop($: EngineInterface): Promise<void> {
175  for (;;) {
176    let wasAttached = false
177    try {
178      const child = $.process.spawn({ argv: [options.linkPath, 'attach', sessionName] })
179      const lines = new EventLines()
180      for await (const piece of child) {
181        if (piece.stream !== 'stdout') continue
182        for (const event of lines.push(piece.text)) {
183          if (event.type === 'attached') wasAttached = true
184          await received($, event)
185        }
186      }
187    } catch {
188      // No avatar-link (Avatar not installed): try again later.
189    }
190    attached = false
191    // The band above the prompt reads `attached`, which isn't one of its atoms: draw it again.
192    $.ui.invalidate('ui.render')
193    await update($, hearingAtom, () => null)
194    await $.clock.sleep(wasAttached ? 2_000 : 15_000)
195  }
196}
197
198async function received($: EngineInterface, event: AvatarEvent): Promise<void> {
199  switch (event.type) {
200    case 'attached':
201      attached = true
202      $.ui.invalidate('ui.render')
203      setFaceName($, event.face)
204      // A reattach (Avatar relaunched) picks up where the session is.
205      if (await read($, micAtom)) await post($, '/listen', { on: true })
206      if (turnRunning) await setFace($, 'thinking', 'running')
207      else await idleFace($)
208      await sayUnsaid($)
209      return
210    case 'face':
211      setFaceName($, event.face)
212      return
213    case 'mic':
214      // Avatar's own Mic button.
215      await update($, micAtom, () => event.state === 'on')
216      if (event.state !== 'on') {
217        await update($, hearingAtom, () => null)
218        await update($, earsAtom, () => 'off')
219      }
220      if (!turnRunning) await idleFace($)
221      return
222    case 'ears.state': {
223      const trouble = earsTrouble({ type: 'state', state: event.state as EarsEvent['state'], text: event.text })
224      await update($, earsAtom, () => trouble ?? event.state ?? 'off')
225      return
226    }
227    case 'ears.volatile':
228      await update($, hearingAtom, () => event.text ?? null)
229      return
230    case 'ears.final':
231      await update($, hearingAtom, () => null)
232      // Never while Claude is working (Avatar itself drops what it hears while it speaks).
233      if (!turnRunning) await act($, event.text ?? '')
234      return
235  }
236}
237
238/** Puts `text` in the prompt box for a voice edit, remembering what it held for "undo that". */
239async function setBox($: EngineInterface, before: string, text: string): Promise<void> {
240  history.record(before)
241  await $.prompt.fill({ text, mode: 'replace' })
242}
243
244/** A finished phrase: a command, an edit of the prompt box, or words for it. */
245async function act($: EngineInterface, text: string): Promise<void> {
246  const said = parseUtterance(text, options.sendPhrase)
247  switch (said.kind) {
248    case 'dictate': {
249      if (said.text.length === 0) return
250      const box = await $.prompt.read()
251      const edit = parseEdit(said.text)
252      if (edit !== null) {
253        lastAppended = ''
254        if (edit.kind === 'undo' || edit.kind === 'redo') {
255          const previous = edit.kind === 'undo' ? history.undo(box.text) : history.redo(box.text)
256          if (previous === null) $.ui.toast(edit.kind === 'undo' ? 'Nothing to undo' : 'Nothing to redo')
257          else await $.prompt.fill({ text: previous, mode: 'replace' })
258          return
259        }
260        const result = applyEdit(box.text, edit)
261        if ('error' in result) $.ui.toast(result.error)
262        else await setBox($, box.text, result.text)
263        return
264      }
265      // Into an empty (or blank) box, the phrase is the whole text: no stray leading space.
266      const blank = box.text.trim().length === 0
267      const added = blank ? said.text : appendText(box.text, said.text)
268      const filled = await $.prompt.fill({ text: added, mode: blank ? 'replace' : 'append' })
269      if (filled.isFilled) {
270        history.record(box.text)
271        lastAppended = added
272      } else {
273        $.ui.toast(`Heard while the prompt box was busy: “${said.text}”`)
274      }
275      return
276    }
277    case 'send': {
278      const box = await $.prompt.read()
279      const prompt = box.text.trim()
280      if (prompt.length === 0) {
281        $.ui.toast('Nothing to send yet')
282        return
283      }
284      lastAppended = ''
285      history.clear()
286      await $.prompt.fill({ text: '', mode: 'replace' })
287      void $.prompt.submit({ text: prompt, asUser: true })
288      return
289    }
290    case 'scratch': {
291      const box = await $.prompt.read()
292      const rest = withoutLast(box.text, lastAppended)
293      lastAppended = ''
294      if (rest === null) $.ui.toast('Nothing to scratch')
295      else await setBox($, box.text, rest)
296      return
297    }
298    case 'clear': {
299      const box = await $.prompt.read()
300      lastAppended = ''
301      await setBox($, box.text, '')
302      return
303    }
304    case 'stop':
305      await stopSpeaking($)
306      return
307    case 'replay':
308      await post($, '/replay')
309      return
310    case 'micOff':
311      await setMic($, false)
312      return
313  }
314}
315
316// MARK: The command
317
318async function command($: EngineInterface, args: string): Promise<{ text: string }> {
319  const words = args.trim().toLowerCase().replace(/\s+/g, ' ')
320  switch (words) {
321    case 'mic on':
322    case 'mic off':
323      await setMic($, words === 'mic on')
324      return { text: words === 'mic on' ? `The mic is on: between turns, what you say goes into the prompt; say “${options.sendPhrase}” to send it.` : 'The mic is off.' }
325    case 'face male':
326    case 'face female': {
327      const chosen = words.slice(5)
328      await post($, '/face', { face: chosen })
329      setFaceName($, chosen)
330      return { text: `The ${chosen === 'female' ? 'woman' : 'man'} speaks.` }
331    }
332    case 'stop':
333      await stopSpeaking($)
334      return { text: 'Stopped.' }
335    case 'replay':
336      await post($, '/replay')
337      return { text: 'Saying the last reply again.' }
338    case 'on':
339    case 'off':
340      await update($, voiceAtom, () => words === 'on')
341      if (words === 'off') await stopSpeaking($)
342      return { text: words === 'on' ? 'Replies are spoken again.' : 'Replies are not spoken in this session.' }
343    case '':
344    case 'status': {
345      const where = attached ? `Avatar is attached, the ${face === 'female' ? 'woman' : 'man'} speaking` : 'Avatar is not running'
346      const voice = (await read($, voiceAtom)) ? 'replies are spoken' : 'replies are not spoken'
347      const mic = (await read($, micAtom)) ? `the mic is on (${await read($, earsAtom)})` : 'the mic is off'
348      return { text: `${where}; ${voice}; ${mic}.` }
349    }
350    default:
351      return { text: `Usage: /${COMMAND} [status|mic on|mic off|face male|face female|stop|replay|on|off]` }
352  }
353}
354
355// MARK: The hooks
356
357/** Attaches the session to Avatar, once, when someone is at it. */
358async function start($: EngineInterface): Promise<void> {
359  if (started) return
360  started = true
361  const home = (await $.env.get('HOME').catch(() => undefined)) ?? ''
362  socketPath = options.socket.length > 0 ? options.socket : `${home}/Library/Application Support/Avatar/avatar.sock`
363  const id = await $.session.id().catch(() => 'session')
364  sessionName = `${id}-${Math.random().toString(36).slice(2, 8)}`
365  // It ends only with the session (a reload, or the engine going away beneath it).
366  void attachLoop($).catch(() => undefined)
367  await $.command.register({
368    name: COMMAND,
369    description: 'Presence: talk with Claude Code through the Avatar app',
370    argumentHint: '[status|mic on|mic off|face male|face female|stop|replay|on|off]',
371  }).catch(() => undefined)
372}
373
374export const register: Register = (on, rawOptions) => {
375  options = readOptions(rawOptions as Record<string, unknown>)
376
377  // Only a session someone is at attaches: the terminal's REPL at its start, or one the desktop app
378  // (or another surface) attaches to later. A headless run (`claude -p`, a script on the SDK) never
379  // speaks or takes the microphone.
380  on('session.start', async ($, e, next) => {
381    const result = await next(e)
382    if (e.isInteractive || e.surface !== null) await start($)
383    return result
384  })
385
386  on('session.attach', async ($, e, next) => {
387    const result = await next(e)
388    await start($)
389    return result
390  })
391
392  on('command.run', { command: 'presence' }, ($, e) => command($, e.args))
393
394  on('prompt.compose', async ($, e, next) => {
395    const composed = await next(e)
396    if (!(await speaks($))) return composed
397    return { sections: [...composed.sections, { id: COMPOSE_SECTION, text: SPOKEN_REPLIES, scope: 'session' as const }] }
398  })
399
400  on('prompt.submit', async ($, e, next) => {
401    lastAppended = ''
402    history.clear()
403    return next(e)
404  })
405
406  on('turn.start', async ($, e, next) => {
407    turnRunning = true
408    replyId = e.turnId
409    // Every turn, however it was sent (typed, or "send it", which prompt.submit doesn't see): Avatar
410    // keeps the microphone shut until it is over, even in the pauses while the reply is written.
411    if (attached) {
412      await stopSpeaking($)
413      await setFace($, 'thinking', 'running')
414    }
415    return next(e)
416  })
417
418  on('turn.step', async function* ($, e, next) {
419    // Not `speaks()`: a reply begun before Avatar attached is kept and said once it has.
420    const speaking = e.agentId === undefined && started && (await read($, voiceAtom))
421    if (!speaking) return yield* next(e)
422    const reply = replyId || e.turnId
423    const generation = speechGeneration
424    // One sentence at a time, so the bubble's highlight follows the voice.
425    const segmenter = new Segmenter({ batchChars: 1 })
426    const stream = next(e)
427    let step = await stream.next()
428    while (step.done !== true) {
429      const chunk = step.value
430      yield chunk
431      // After Stop, the rest of what is being written isn't said.
432      if (chunk.kind === 'text' && generation === speechGeneration) {
433        for (const segment of segmenter.push(chunk.text)) sayOrKeep($, segment, reply)
434      }
435      step = await stream.next()
436    }
437    if (generation === speechGeneration) for (const segment of segmenter.flush()) sayOrKeep($, segment, reply)
438    return step.value
439  })
440
441  on('turn.complete', async ($, e, next) => {
442    const done = await next(e)
443    if (e.agentId !== undefined) return done
444    turnRunning = false
445    const reply = replyId
446    if (e.isAborted) await stopSpeaking($)
447    if (attached) {
448      void inOrder(() => post($, '/reply/end', { reply }))
449      // Avatar opens the microphone itself once the reply has been said.
450      void inOrder(() => idleFace($))
451    } else if (unsaid?.reply === reply) {
452      unsaid.endedAt = await $.clock.now()
453    }
454    return done
455  })
456
457  // A row above the prompt while Avatar is attached: a button that turns the mic on or off, buttons
458  // for the man's and the woman's face (the one speaking dim), and with the mic on, what is being
459  // heard or that it is listening. Other plugins' rows (and the engine's)
460  // stay, below it.
461  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
462    if (e.props.hasSurvey || !started || !attached) return next(e)
463    const [mic, hearing, ears, beneath] = await Promise.all([
464      read($, micAtom), read($, hearingAtom), read($, earsAtom), next(e),
465    ])
466    const { Box, Button, Text } = $.ui.resolve(e)
467    // The button names what a press does.
468    const toggle = <Button key="presence-mic" plain label={mic ? '🎙  Mic off' : '🎙  Mic on'} onPress={() => setMic($, !mic)} />
469    const faceButton = (name: 'male' | 'female', label: string) => (
470      <Button key={`presence-${name}`} plain dimColor={face === name} label={label}
471        onPress={() => post($, '/face', { face: name }).then(() => setFaceName($, name))} />
472    )
473    const room = Math.max(20, (e.props.bodyColumns ?? 80) - 30)
474    const line = !mic ? ''
475      : hearing !== null ? hearing
476        : e.props.isWorking ? 'waits while Claude works'
477          : ears === 'listening' ? `listening · say “${options.sendPhrase}” to send` : ears
478    const row = line.length === 0
479      ? <Box flexDirection="row" gap={2}>{toggle}{faceButton('male', 'Man')}{faceButton('female', 'Woman')}</Box>
480      : (
481        <Box flexDirection="row" gap={2}>
482          {toggle}
483          {faceButton('male', 'Man')}
484          {faceButton('female', 'Woman')}
485          <Text dimColor={hearing === null} wrap="truncate-end">{line.length > room ? line.slice(0, room - 1) + '…' : line}</Text>
486        </Box>
487      )
488    return (
489      <Box flexDirection="column">
490        {row}
491        {beneath}
492      </Box>
493    )
494  })
495}
496

1// The ears: what the avatar hears, turned into edits of Claude Code's prompt box. The pieces here
2// need no engine, so they are tested on their own; register.tsx does the listening and the editing.
3
4/** The avatar's `ears.state` event. */
5export type EarsEvent = {
6  type: 'state' | 'volatile' | 'final'
7  text?: string
8  state?: 'idle' | 'preparing' | 'listening' | 'denied' | 'unavailable' | 'taken'
9}
10
11/** What a finished phrase means. */
12export type Utterance =
13  | { kind: 'send' }
14  | { kind: 'scratch' }
15  | { kind: 'clear' }
16  | { kind: 'stop' }
17  | { kind: 'replay' }
18  | { kind: 'micOff' }
19  | { kind: 'dictate'; text: string }
20
21/** Lowercase, no punctuation, single spaces: how a spoken command is compared. */
22export function normalize(text: string): string {
23  return text
24    .toLowerCase()
25    .replace(/[“”"'’.,!?;:…]/g, '')
26    .replace(/\s+/g, ' ')
27    .trim()
28}
29
30/**
31 * A command only when it is the whole phrase, so the same words inside a sentence are dictated as
32 * words: "send it" sends, "please send it to Bob" is typed.
33 */
34export function parseUtterance(text: string, sendPhrase: string): Utterance {
35  const said = normalize(text)
36  if (said.length === 0) return { kind: 'dictate', text: '' }
37  if (said === normalize(sendPhrase) || said === 'send prompt') return { kind: 'send' }
38  if (said === 'scratch that' || said === 'delete that') return { kind: 'scratch' }
39  if (['clear', 'clear prompt', 'clear all', 'delete all', 'delete everything'].includes(said)) return { kind: 'clear' }
40  if (said === 'stop' || said === 'stop talking' || said === 'be quiet') return { kind: 'stop' }
41  if (said === 'replay' || said === 'say that again' || said === 'repeat that') return { kind: 'replay' }
42  if (said === 'mic off' || said === 'stop listening') return { kind: 'micOff' }
43  return { kind: 'dictate', text: text.trim() }
44}
45
46/** What to append to the box for a dictated phrase: a space between it and what is there. */
47export function appendText(box: string, phrase: string): string {
48  if (box.length === 0 || /\s$/.test(box)) return phrase
49  return ' ' + phrase
50}
51
52/**
53 * The box without the last dictated phrase, or null when the box no longer ends with it (it was
54 * edited by hand since): "scratch that" never deletes what it didn't write.
55 */
56export function withoutLast(box: string, appended: string): string | null {
57  if (appended.length === 0 || !box.endsWith(appended)) return null
58  return box.slice(0, box.length - appended.length).replace(/\s+$/, '')
59}
60
61/** Why the ears aren't listening, said in a few words for the band; null when listening is fine. */
62export function earsTrouble(event: EarsEvent): string | null {
63  switch (event.state) {
64    case 'taken': return 'another Claude Code session has the microphone'
65    case 'denied': return 'Avatar has no microphone access (System Settings → Privacy & Security → Microphone)'
66    case 'unavailable': return event.text ?? 'Avatar cannot listen'
67    default: return null
68  }
69}
70

1// Editing the prompt box by voice, for a box whose cursor a mod can't place. So every command works
2// on the text itself: at its end ("delete last word"), or on the last place a phrase occurs
3// ("replace cat with dog").
4// Pure functions: the text in, the text out (or why not), so each command is tested on its own.
5
6export type Edit =
7  | { kind: 'replace'; find: string; with: string }
8  | { kind: 'insert'; text: string; where: 'before' | 'after'; anchor: string }
9  | { kind: 'delete'; find: string }
10  | { kind: 'deleteLast'; unit: Unit; count: number }
11  | { kind: 'case'; find: string; to: 'capitalize' | 'upper' | 'lower' }
12  | { kind: 'append'; text: string }
13  | { kind: 'undo' }
14  | { kind: 'redo' }
15
16const COUNTS: Record<string, number> = {
17  a: 1, an: 1, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10,
18  eleven: 11, twelve: 12, thirteen: 13, fourteen: 14, fifteen: 15, sixteen: 16, seventeen: 17,
19  eighteen: 18, nineteen: 19, twenty: 20,
20}
21
22function count(word: string | undefined): number | null {
23  if (word === undefined) return 1
24  if (/^\d+$/.test(word)) return Number(word)
25  return COUNTS[word] ?? null
26}
27
28/** A phrase as spoken, without the recogniser's closing punctuation or quotes. */
29function phrase(text: string): string {
30  return text.trim().replace(/^["“'‘]+|["”'’.,!?;:…]+$/g, '').trim()
31}
32
33/** Lowercase, single spaces, no punctuation at the ends: how a command's words are compared. */
34function words(text: string): string {
35  return text.toLowerCase().replace(/\s+/g, ' ').replace(/^[\s"“'‘]+|[\s"”'’.,!?;:…]+$/g, '')
36}
37
38type Unit = 'character' | 'word' | 'sentence' | 'line'
39
40const UNITS: Record<string, Unit> = {
41  character: 'character', characters: 'character', letter: 'character', letters: 'character',
42  word: 'word', words: 'word', sentence: 'sentence', sentences: 'sentence', line: 'line', lines: 'line',
43}
44
45/**
46 * The edit a finished phrase asks for, or null when it is just words to type. Only a whole phrase is
47 * a command: "please replace the tires" is dictated, "replace tires with wheels" is an edit.
48 */
49export function parseEdit(spoken: string, now: () => Date = () => new Date()): Edit | null {
50  // The recogniser may punctuate after the command's first word ("Type, send it."): that comma is not
51  // part of the command.
52  const original = spoken.trim().replace(/^([A-Za-z]+)[,:;.]\s+/, '$1 ')
53  const said = words(original)
54
55  if (said === 'undo that' || said === 'undo') return { kind: 'undo' }
56  if (said === 'redo that' || said === 'redo') return { kind: 'redo' }
57  if (said === 'new line' || said === 'press return' || said === 'press return key') return { kind: 'append', text: '\n' }
58  if (said === 'new paragraph') return { kind: 'append', text: '\n\n' }
59  if (said === 'insert date' || said === 'insert the date') {
60    return { kind: 'append', text: now().toLocaleDateString(undefined, { year: 'numeric', month: 'long', day: 'numeric' }) }
61  }
62
63  // "type <phrase>": exactly as spoken, never a command.
64  const typed = /^type\s+(.+)$/is.exec(original)
65  if (typed) return { kind: 'append', text: phrase(typed[1]!) }
66
67  // "delete [the] [last|previous] [<count>] <unit>"
68  const last = /^(?:delete|remove)\s+(?:the\s+)?(?:last|previous)?\s*(\w+)?\s*(characters?|letters?|words?|sentences?|lines?)$/.exec(said)
69  if (last) {
70    const n = count(last[1])
71    const unit = UNITS[last[2]!]
72    if (n !== null && n > 0 && unit !== undefined) return { kind: 'deleteLast', unit, count: n }
73  }
74
75  // "replace <phrase> with <phrase>": split on the last " with ".
76  const replace = /^replace\s+(.+)$/is.exec(original)
77  if (replace) {
78    const rest = replace[1]!
79    const split = rest.toLowerCase().lastIndexOf(' with ')
80    if (split > 0) {
81      const find = phrase(rest.slice(0, split))
82      const replacement = phrase(rest.slice(split + 6))
83      if (find.length > 0) return { kind: 'replace', find, with: replacement }
84    }
85  }
86
87  // "insert <phrase> before|after <phrase>": split on the last " before " / " after ".
88  const insert = /^insert\s+(.+)$/is.exec(original)
89  if (insert) {
90    const rest = insert[1]!
91    const lower = rest.toLowerCase()
92    const before = lower.lastIndexOf(' before ')
93    const after = lower.lastIndexOf(' after ')
94    const at = Math.max(before, after)
95    if (at > 0) {
96      const where = at === before ? 'before' : 'after'
97      const text = phrase(rest.slice(0, at))
98      const anchor = phrase(rest.slice(at + where.length + 2))
99      if (text.length > 0 && anchor.length > 0) return { kind: 'insert', text, where, anchor }
100    }
101  }
102
103  // "capitalize|uppercase|lowercase <phrase>"
104  const caseChange = /^(capitali[sz]e|upper\s?case|lower\s?case)\s+(.+)$/is.exec(original)
105  if (caseChange) {
106    const verb = caseChange[1]!.toLowerCase().replace(/\s/g, '')
107    const to = verb.startsWith('capital') ? 'capitalize' : verb === 'uppercase' ? 'upper' : 'lower'
108    const find = phrase(caseChange[2]!)
109    if (find.length > 0) return { kind: 'case', find, to }
110  }
111
112  // "delete|remove <phrase>" (after the unit forms above, and never "delete that"/"delete all").
113  const remove = /^(?:delete|remove)\s+(.+)$/is.exec(original)
114  if (remove && !['that', 'all', 'everything'].includes(words(remove[1]!))) {
115    const find = phrase(remove[1]!)
116    if (find.length > 0) return { kind: 'delete', find }
117  }
118
119  return null
120}
121
122/** Where `find` last occurs in `text`, ignoring case; -1 if it doesn't. */
123function lastIndexOf(text: string, find: string): number {
124  return text.toLowerCase().lastIndexOf(find.toLowerCase())
125}
126
127/** The text with `edit` applied, or a sentence saying why it can't be. Undo and redo are the caller's. */
128export function applyEdit(text: string, edit: Edit): { text: string } | { error: string } {
129  switch (edit.kind) {
130    case 'append': {
131      if (edit.text.startsWith('\n')) return { text: text.replace(/[ \t]+$/, '') + edit.text }
132      const space = text.length === 0 || /\s$/.test(text) ? '' : ' '
133      return { text: text + space + edit.text }
134    }
135    case 'replace': {
136      const at = lastIndexOf(text, edit.find)
137      if (at < 0) return { error: `Couldn't find “${edit.find}”` }
138      return { text: text.slice(0, at) + edit.with + text.slice(at + edit.find.length) }
139    }
140    case 'insert': {
141      const at = lastIndexOf(text, edit.anchor)
142      if (at < 0) return { error: `Couldn't find “${edit.anchor}”` }
143      if (edit.where === 'before') return { text: text.slice(0, at) + edit.text + ' ' + text.slice(at) }
144      const end = at + edit.anchor.length
145      return { text: text.slice(0, end) + ' ' + edit.text + text.slice(end) }
146    }
147    case 'delete': {
148      const at = lastIndexOf(text, edit.find)
149      if (at < 0) return { error: `Couldn't find “${edit.find}”` }
150      const before = text.slice(0, at).replace(/[ \t]+$/, '')
151      const after = text.slice(at + edit.find.length)
152      const joined = before.length > 0 && after.length > 0 && !/^[\s.,!?;:]/.test(after) ? before + ' ' + after.replace(/^[ \t]+/, '') : before + after
153      return { text: joined }
154    }
155    case 'case': {
156      const at = lastIndexOf(text, edit.find)
157      if (at < 0) return { error: `Couldn't find “${edit.find}”` }
158      const found = text.slice(at, at + edit.find.length)
159      const changed = edit.to === 'upper' ? found.toUpperCase()
160        : edit.to === 'lower' ? found.toLowerCase()
161          : found.replace(/(^|\s)(\S)/g, (_, space: string, letter: string) => space + letter.toUpperCase())
162      return { text: text.slice(0, at) + changed + text.slice(at + edit.find.length) }
163    }
164    case 'deleteLast': {
165      let rest = text.replace(/\s+$/, '')
166      if (rest.length === 0) return { error: 'Nothing to delete' }
167      for (let i = 0; i < edit.count && rest.length > 0; i++) {
168        switch (edit.unit) {
169          case 'character':
170            rest = rest.slice(0, -1)
171            break
172          case 'word':
173            rest = rest.replace(/\S+\s*$/, '').replace(/\s+$/, '')
174            break
175          case 'sentence':
176            rest = withoutLastSentence(rest)
177            break
178          case 'line':
179            rest = rest.includes('\n') ? rest.slice(0, rest.lastIndexOf('\n')) : ''
180            break
181        }
182      }
183      return { text: rest }
184    }
185    case 'undo':
186    case 'redo':
187      return { error: 'Undo and redo are kept by the caller' }
188  }
189}
190
191/** `text` without its last sentence: back to the end of the one before, a line break, or the start. */
192function withoutLastSentence(text: string): string {
193  // The last sentence's own closing punctuation isn't a boundary.
194  const body = text.replace(/[.!?…]+["”’']*$/, '')
195  let end = 0
196  for (const match of body.matchAll(/[.!?…]+["”’']*\s+|\n/g)) end = (match.index ?? 0) + match[0].length
197  return text.slice(0, end).replace(/\s+$/, '')
198}
199
200/** The box's voice edits, for "undo that" and "redo that". */
201export class EditHistory {
202  private undos: string[] = []
203  private redos: string[] = []
204
205  /** The box held `before` when a voice edit changed it. */
206  record(before: string): void {
207    this.undos.push(before)
208    if (this.undos.length > 50) this.undos.shift()
209    this.redos = []
210  }
211
212  /** The text to go back to (the box holds `current`), or null. */
213  undo(current: string): string | null {
214    const previous = this.undos.pop()
215    if (previous === undefined) return null
216    this.redos.push(current)
217    return previous
218  }
219
220  redo(current: string): string | null {
221    const next = this.redos.pop()
222    if (next === undefined) return null
223    this.undos.push(current)
224    return next
225  }
226
227  clear(): void {
228    this.undos = []
229    this.redos = []
230  }
231}
232

1// The avatar, as the mod sees it: `avatar-link attach` output split into events, and request bodies.
2// The requests themselves (HTTP over the avatar's Unix socket) are made from register.tsx, where `$`
3// lives. See Avatar/AvatarServers.swift for the routes.
4
5/** One event `avatar-link attach` prints. */
6export type AvatarEvent = {
7  type: string
8  reply?: string
9  segment?: number
10  /** The mod's own name for the segment, as it sent it with `/say`. */
11  key?: string
12  text?: string
13  state?: string
14  face?: string
15}
16
17/** Splits `avatar-link` output (pieces as written, not lines) into events. */
18export class EventLines {
19  private rest = ''
20
21  push(text: string): AvatarEvent[] {
22    this.rest += text
23    const lines = this.rest.split('\n')
24    this.rest = lines.pop() ?? ''
25    const events: AvatarEvent[] = []
26    for (const line of lines) {
27      if (line.trim().length === 0) continue
28      try {
29        const event = JSON.parse(line) as unknown
30        if (typeof event === 'object' && event !== null && typeof (event as AvatarEvent).type === 'string') {
31          events.push(event as AvatarEvent)
32        }
33      } catch {
34        // Not an event: ignore it.
35      }
36    }
37    return events
38  }
39}
40
41/** A request body: the session's name with the route's fields (undefined ones left out). */
42export function requestBody(session: string, fields: Record<string, unknown>): string {
43  const body: Record<string, unknown> = { session }
44  for (const [key, value] of Object.entries(fields)) {
45    if (value !== undefined) body[key] = value
46  }
47  return JSON.stringify(body)
48}
49

1// Turns the model's reply, as it streams in pieces, into segments worth speaking: whole sentences,
2// whole markdown lines, and whole fenced code blocks (which Avatar announces rather than
3// reads). The first sentence goes out on its own so the voice starts quickly; after that sentences
4// are gathered to about `batchChars`, so the speech flows rather than pausing after every one.
5//
6// Lines stay lines inside a segment (Avatar reads markdown line by line), and a blank line
7// ends a paragraph's segment.
8
9/** Words whose trailing period doesn't end a sentence. */
10const ABBREVIATIONS = new Set([
11  'e.g', 'i.e', 'etc', 'vs', 'mr', 'mrs', 'ms', 'dr', 'prof', 'sr', 'jr', 'st', 'no', 'fig', 'approx', 'cf',
12])
13
14export type SegmenterOptions = {
15  /** Characters to gather before a sentence boundary ends a later segment. */
16  batchChars?: number
17}
18
19export class Segmenter {
20  /** Text not yet sorted into sentences: the rest of the current line. */
21  private pending = ''
22  /** Finished sentences waiting to be spoken together. */
23  private gathered = ''
24  /** Whether `gathered` ends in the middle of a line (join with a space, not a newline). */
25  private midLine = false
26  /** The lines of a code block that is still open. */
27  private fence: string[] | null = null
28  private emitted = 0
29  private readonly batchChars: number
30
31  constructor(options: SegmenterOptions = {}) {
32    this.batchChars = options.batchChars ?? 160
33  }
34
35  /** Feeds a piece of the reply; returns the segments it completed. */
36  push(text: string): string[] {
37    const out: string[] = []
38    this.pending += text
39    let newline: number
40    while ((newline = this.pending.indexOf('\n')) >= 0) {
41      const line = this.pending.slice(0, newline)
42      this.pending = this.pending.slice(newline + 1)
43      this.line(line, out)
44    }
45    // Finished sentences of an unfinished line, unless the line might be a fence.
46    if (this.fence === null && !/^\s*[`~]/.test(this.pending)) {
47      this.pending = this.sentences(this.pending, out)
48    }
49    return out
50  }
51
52  /** The reply has ended: whatever is left. */
53  flush(): string[] {
54    const out: string[] = []
55    if (this.pending.length > 0) {
56      const rest = this.pending
57      this.pending = ''
58      this.line(rest, out)
59    }
60    if (this.fence !== null) {
61      // A block the reply never closed is still a block.
62      this.emit(this.fence.join('\n'), out)
63      this.fence = null
64    }
65    this.emitGathered(out)
66    return out
67  }
68
69  /** A complete line (what is left of it, if its first sentences went out already). */
70  private line(line: string, out: string[]): void {
71    const isFence = /^\s*(```|~~~)/.test(line)
72    if (this.fence !== null) {
73      this.fence.push(line)
74      if (isFence) {
75        this.emit(this.fence.join('\n'), out)
76        this.fence = null
77      }
78      return
79    }
80    if (isFence) {
81      // What came before the block is spoken first; the block is one segment of its own.
82      this.emitGathered(out)
83      this.fence = [line]
84      return
85    }
86    if (line.trim().length === 0) {
87      // A paragraph ends.
88      this.emitGathered(out)
89      return
90    }
91    const rest = this.sentences(line, out)
92    if (rest.trim().length > 0) this.add(rest, out)
93    // The line is over: what comes next starts a new one.
94    this.midLine = false
95  }
96
97  /** Adds each finished sentence in `text`; returns what follows the last one. */
98  private sentences(text: string, out: string[]): string {
99    const boundary = /[.!?…]+["')\]]*(?=\s)/g
100    let start = 0
101    let match: RegExpExecArray | null
102    while ((match = boundary.exec(text)) !== null) {
103      const end = match.index + match[0].length
104      const sentence = text.slice(start, end)
105      if (isAbbreviation(sentence)) continue
106      this.add(sentence, out)
107      start = end
108    }
109    return text.slice(start)
110  }
111
112  /** Gathers a sentence (or a line's unterminated rest), emitting when there is enough. */
113  private add(sentence: string, out: string[]): void {
114    const words = this.midLine ? sentence.trim() : sentence.trimEnd()
115    if (words.trim().length === 0) return
116    if (this.gathered.length === 0) this.gathered = words.trimStart()
117    else this.gathered += (this.midLine ? ' ' : '\n') + words
118    this.midLine = true
119    if (this.emitted === 0 || this.gathered.length >= this.batchChars) this.emitGathered(out)
120  }
121
122  private emitGathered(out: string[]): void {
123    this.emit(this.gathered, out)
124    this.gathered = ''
125  }
126
127  private emit(segment: string, out: string[]): void {
128    const text = segment.trim()
129    if (text.length === 0) return
130    out.push(text)
131    this.emitted += 1
132  }
133}
134
135function isAbbreviation(sentence: string): boolean {
136  if (!sentence.endsWith('.')) return false
137  const word = /(\S+)\.$/.exec(sentence.trimEnd())?.[1]?.toLowerCase().replace(/^[("']+/, '')
138  if (word === undefined) return false
139  // "1." of a numbered line, a decimal or a version: never the end of a sentence on its own.
140  if (/^\d+$/.test(word)) return true
141  return ABBREVIATIONS.has(word) || /^[a-z]$/.test(word)
142}
143

1declare module 'claude-code' {
2  interface PluginState {
3    presence: {
4      /** Replies are spoken in this session (`/presence on|off`). */
5      voice: boolean
6      /** The mic is on for this session (`/presence mic on|off`, or Avatar's Mic button). */
7      mic: boolean
8      /** How the ears are: listening, preparing, off, or why not. */
9      ears: string
10      /** What is being heard now (interim), for the band above the prompt. */
11      hearing: string | null
12    }
13  }
14}
15