avatars

1import { atom, read, update } from 'claude-code'
2import type { Engine, PluginOptions, Register, Timer } from 'claude-code'
3
4import { codecCells } from './codec'
5import { withEffects } from './effects'
6import { soloCells, soloRowsFor } from './solo'
7import { courtCells, courtRowsFor } from './court'
8import { buildMode, callPersona, demoHas, isModeName, parseCall, planCall, type Mode } from './modes'
9import type { Setting, VoiceChoice } from '../types'
10
11const DEFAULT_MODE = 'coven'
12const DEFAULT_MODEL = 'eleven_v4_turbo'
13const MODELS = ['eleven_v4_turbo', 'eleven_v4', 'eleven_v3', 'eleven_flash_v2_5', 'eleven_multilingual_v2']
14const MAX_REPLY_CHARS = 12000
15const API = 'https://api.elevenlabs.io'
16const MENU = 'avatars'
17
18const SYSTEM = `You turn a coding assistant's written reply into what it would say aloud to the user who asked.
19
20Rules:
21- Speak as the assistant, first person, to "you". Natural spoken English.
22- One to four sentences, under 80 words. Shorter is better when the reply is short.
23- Lead with the outcome. Keep any question the reply asks the user, and any decision they must make.
24- Never read code, commands, file paths, URLs, IDs or tables aloud: describe them ("I edited the config file").
25- Say numbers the way a person would. No markdown, no lists, no emoji.
26- Output only the words to speak, nothing else.`
27
28// v3/v4 models act on [bracketed] audio tags; older ones would read them out.
29const TAG_RULES = `
30Audio tags: the voice model performs inline directions in square brackets.
31- Use one to three tags, each placed just before the words it colors.
32- Fit the tone to the content: [quietly pleased] for something finished, [apologetic] for a failure, [curious] or [thoughtful] for a question, [dry] for an aside. Free-text directions like [dry amusement] work too.
33- [pause] or [short pause] for a beat before a key point. [sighs] or [laughs softly] only when it genuinely fits.
34- Never sound effects, never shouting, never a tag on every sentence. Restraint sounds better.`
35
36const hasTags = (model: string) => /^eleven_v[34]/.test(model)
37const stripTags = (text: string) => text.replace(/\[[^\]]*\]\s*/g, '').trim()
38const pick = <T,>(list: readonly T[]) => list[Math.floor(Math.random() * list.length)] as T
39
40// ── Modes and options ──
41
42// Module state: a reload reads the modes again and stops any playback.
43let modes = new Map<string, Mode>()
44let modeErrors: string[] = []
45let options: PluginOptions = {}
46let isInteractive = true
47let isLoaded = false
48let isWindows = false
49let userModesDir = ''
50/** The `modes/` folders of installed mode packs, in load order. */
51let packModesDirs: string[] = []
52/** The person's own voices for any mode's characters: `{ mode: { member: voiceId } }`. */
53let voicesFile = ''
54let personal: Record<string, Record<string, string>> = {}
55
56const option = (name: string) => {
57  const value = options[name]
58  return typeof value === 'string' && value.trim() ? value.trim() : undefined
59}
60
61const apiKey = async ($: Engine) => option('api_key') ?? ((await $.env.get('ELEVENLABS_API_KEY'))?.trim() || undefined)
62
63const readMode = async ($: Engine, dir: string, name: string) => {
64  try {
65    const portraits = (await $.fs.exists(`${dir}/portraits.json`)) ? await $.fs.read(`${dir}/portraits.json`) : undefined
66    return buildMode(name, dir, await $.fs.read(`${dir}/mode.json`), portraits)
67  } catch (err) {
68    return `${name}: ${String(err)}`
69  }
70}
71
72const readJson = async ($: Engine, path: string, errors: string[]) => {
73  if (!(await $.fs.exists(path))) return undefined
74  try {
75    return JSON.parse(await $.fs.read(path)) as unknown
76  } catch (err) {
77    errors.push(`${path} does not parse (${String(err)})`)
78    return undefined
79  }
80}
81
82type InstalledPlugins = { plugins?: Record<string, Array<{ scope?: string; installPath?: string }>> }
83
84/**
85 * The `avatars/` folder of every plugin that has one: mode packs. Packs come
86 * from user-scope installs Claude Code lists as installed and not disabled, and
87 * from `CLAUDE_CODE_PLUGIN_DIRS` folders.
88 */
89const packDirs = async ($: Engine, home: string, errors: string[]) => {
90  const claudeDir = ((await $.env.get('CLAUDE_CONFIG_DIR'))?.trim() || `${home}/.claude`).replace(/[\\/]+$/, '')
91  const installed = (await readJson($, `${claudeDir}/plugins/installed_plugins.json`, errors)) as InstalledPlugins | undefined
92  const settings = (await readJson($, `${claudeDir}/settings.json`, errors)) as { enabledPlugins?: Record<string, boolean> } | undefined
93  const roots: string[] = []
94  for (const id of Object.keys(installed?.plugins ?? {}).sort()) {
95    if (settings?.enabledPlugins?.[id] === false) continue
96    for (const install of installed?.plugins?.[id] ?? []) {
97      if (install.scope === 'user' && install.installPath) roots.push(install.installPath)
98    }
99  }
100  const listed = (await $.env.get('CLAUDE_CODE_PLUGIN_DIRS')) ?? ''
101  for (const dir of listed.split(isWindows ? ';' : ':')) {
102    if (dir.trim()) roots.push(dir.trim().replace(/^~(?=[\\/]|$)/, home))
103  }
104  const dirs: string[] = []
105  for (const root of roots) {
106    const dir = `${root.replace(/[\\/]+$/, '')}/avatars`
107    if (root !== $.plugin.root && !dirs.includes(dir) && (await $.fs.exists(`${dir}/modes`))) dirs.push(dir)
108  }
109  return dirs
110}
111
112/** The plugin's modes, then each pack's, then the person's own; later ones win on a name clash. */
113const reloadModes = async ($: Engine) => {
114  isWindows = /^[A-Za-z]:[\\/]/.test($.plugin.root) || $.plugin.root.includes('\\')
115  const home = ((await $.env.get('HOME')) ?? (await $.env.get('USERPROFILE')) ?? '').replace(/[\\/]+$/, '')
116  const avatarsDir = home ? `${home}/.claude/avatars` : ''
117  userModesDir = avatarsDir && `${avatarsDir}/modes`
118  voicesFile = avatarsDir && `${avatarsDir}/voices.json`
119  const found = new Map<string, Mode>()
120  const errors: string[] = []
121  const packs = home ? await packDirs($, home, errors) : []
122  packModesDirs = packs.map(pack => `${pack}/modes`)
123  for (const root of [`${$.plugin.root}/modes`, ...packModesDirs, userModesDir]) {
124    if (!root || !(await $.fs.exists(root))) continue
125    for (const entry of await $.fs.list(root)) {
126      const dir = `${root}/${entry.name}`
127      if (entry.kind !== 'dir' || !isModeName(entry.name) || !(await $.fs.exists(`${dir}/mode.json`))) continue
128      const mode = await readMode($, dir, entry.name)
129      if (typeof mode === 'string') errors.push(mode)
130      else found.set(mode.name, mode)
131    }
132  }
133  const voiceSets: Array<Record<string, Record<string, string>>> = []
134  for (const pack of packs) {
135    const voices = await readJson($, `${pack}/voices.json`, errors)
136    if (voices) voiceSets.push(voices as Record<string, Record<string, string>>)
137  }
138  personal = {}
139  if (voicesFile) {
140    const voices = await readJson($, voicesFile, errors)
141    if (voices) personal = voices as Record<string, Record<string, string>>
142  }
143  voiceSets.push(personal)
144  for (const [name, voices] of voiceSets.flatMap(set => Object.entries(set))) {
145    const mode = found.get(name)
146    if (!mode) continue
147    for (const [who, voice] of Object.entries(voices)) {
148      const member = mode.call?.cast[who]
149      if (member) {
150        member.voice = voice
151        delete member.libraryOwner
152        delete member.pitch
153      } else if (who === 'voice' && !mode.call) {
154        mode.voice = voice
155        delete mode.voiceOwner
156      }
157    }
158  }
159  modes = found
160  modeErrors = errors
161  isLoaded = true
162}
163
164/** Loads the modes on first use when the plugin arrived after the session started. */
165const ensureLoaded = async ($: Engine) => {
166  if (isLoaded) return
167  await reloadModes($)
168  if (isInteractive) await showBand($, await modeOf($))
169}
170
171// ── Settings: this session's, over the plugin's configured defaults ──
172
173const bandMode = atom({ plugin: 'avatars', key: 'bandMode' } as const, false as string | false)
174const enabledSetting = atom({ plugin: 'avatars', key: 'enabled' } as const, null as Setting)
175const modeSetting = atom({ plugin: 'avatars', key: 'mode' } as const, null as Setting)
176const voiceSetting = atom({ plugin: 'avatars', key: 'voice' } as const, null as Setting)
177const modelSetting = atom({ plugin: 'avatars', key: 'model' } as const, null as Setting)
178const lastSaid = atom({ plugin: 'avatars', key: 'last' } as const, null as Setting)
179const libraryVoices = atom({ plugin: 'avatars', key: 'voices' } as const, null as VoiceChoice[] | null)
180const menuNote = atom({ plugin: 'avatars', key: 'note' } as const, '')
181
182const isEnabled = async ($: Engine) => {
183  const session = await read($, enabledSetting)
184  if (session !== null) return session === 'on'
185  return options.speak !== false
186}
187
188const modeOf = async ($: Engine) => {
189  const name = (await read($, modeSetting)) ?? option('mode') ?? DEFAULT_MODE
190  return name === 'off' ? undefined : (modes.get(name) ?? modes.get(DEFAULT_MODE))
191}
192
193const modelOf = async ($: Engine) => (await read($, modelSetting)) ?? option('model') ?? DEFAULT_MODEL
194
195/** The voice of a single-voice mode: the session's pick, the configured one, the mode's own. */
196const voiceOf = async ($: Engine, mode: Mode | undefined) =>
197  (await read($, voiceSetting)) ?? option('voice') ?? mode?.voice ?? ''
198
199const systemFor = (model: string, mode: Mode | undefined) =>
200  SYSTEM +
201  (hasTags(model) ? TAG_RULES : '') +
202  (mode?.call ? callPersona(mode, hasTags(model)) : `${mode?.persona ?? ''}${hasTags(model) ? (mode?.personaTags ?? '') : ''}`)
203
204/** Accepts a bare voice id or any elevenlabs.io URL ending in one. */
205const parseVoice = (arg: string) => arg.match(/([A-Za-z0-9]{20})\/?(?:[?#].*)?$/)?.[1]
206
207// ── The band: the mode's host on the left, whoever talks to them on the right ──
208
209const BAND_KEY = 'band'
210const TICK_MS = 90
211const PREFER_ROWS = 20
212const CHARS_PER_SECOND = 12
213
214type Script = {
215  /** Spoken characters, tags removed; a pause is a run of spaces. */
216  plain: string
217  /** From each character index on, the performed tag in force. */
218  cues: Array<{ at: number; tag: string }>
219}
220
221const scriptOf = (text: string): Script => {
222  let plain = ''
223  const cues: Script['cues'] = []
224  for (const part of text.split(/(\[[^\]]*\])/)) {
225    const tag = part.match(/^\[([^\]]*)\]$/)?.[1]
226    if (tag === undefined) plain += part
227    else if (/pause/i.test(tag)) plain += ' '.repeat(/long/i.test(tag) ? 18 : 9)
228    else cues.push({ at: plain.length, tag })
229  }
230  return { plain, cues }
231}
232
233const captionOf = (text: string) => text.replace(/\[[^\]]*\]\s*/g, '').replace(/\s+/g, ' ').trim()
234
235let speech: { script: Script; startedAt: number } | undefined
236let caption = ''
237let speaker: string | undefined
238let captionBy: string | undefined
239/** Who sits beside the host: the last of the others to speak. */
240let contact: string | undefined
241let level = 0
242let tick = 0
243let faceTimer: Timer | undefined
244let band: { requestId: string; columns: number; rows: number } | undefined
245let want: { columns: number; maxRows: number } | undefined
246let lastFrame: { mode: string; columns: number; rows: number; cells: string } | undefined
247
248/** The speech under way at `now`: the tags so far and whether a sound is voiced. */
249const speechAt = (now: number) => {
250  if (!speech) return undefined
251  const { script } = speech
252  const at = Math.min(script.plain.length - 1, Math.floor(((now - speech.startedAt) / 1000) * CHARS_PER_SECOND))
253  return {
254    tags: script.cues.filter(cue => cue.at <= at).map(cue => cue.tag),
255    isVoiced: at >= 0 && /[\p{L}\p{N}]/u.test(script.plain[at] ?? ' '),
256  }
257}
258
259/** One portrait's animation: blinks, idle looks, talking frames. */
260type Actor = {
261  blinkUntil: number
262  nextBlinkAt: number
263  idle?: { frame: string; until: number }
264  nextIdleAt: number
265  talkFrame: string
266}
267const actors = new Map<string, Actor>()
268const actorOf = (who: string) => {
269  let actor = actors.get(who)
270  if (!actor) {
271    actor = { blinkUntil: 0, nextBlinkAt: 0, nextIdleAt: 0, talkFrame: 'neutral' }
272    actors.set(who, actor)
273  }
274  return actor
275}
276
277/** This tick's frame for one portrait. */
278const actorFrame = (mode: Mode, who: string, now: number, said: ReturnType<typeof speechAt>) => {
279  const actor = actorOf(who)
280  const member = mode.call!.cast[who]
281  if (actor.nextBlinkAt === 0) actor.nextBlinkAt = now + 1000 + Math.random() * 3000
282  if (now >= actor.nextBlinkAt) {
283    actor.blinkUntil = now + 270
284    actor.nextBlinkAt = now + 2500 + Math.random() * 4500
285  }
286  const blink = actor.blinkUntil > now ? 'blink' : undefined
287  if (speaker === who && said) {
288    actor.idle = undefined
289    actor.nextIdleAt = now + 2000
290    if (tick % 2 === 0) {
291      const accent = member?.accent
292      const isAccented = accent !== undefined && said.tags.some(tag => new RegExp(accent.when, 'i').test(tag))
293      const open = isAccented && Math.random() < 0.35 ? accent!.frame : pick(['talk_a', 'talk_b', 'talk_b'])
294      actor.talkFrame = said.isVoiced ? pick([open, open, open, 'neutral']) : 'neutral'
295    }
296    return blink && actor.talkFrame === 'neutral' ? blink : actor.talkFrame
297  }
298  if (blink) return blink
299  if (actor.idle && now >= actor.idle.until) actor.idle = undefined
300  const choices = member?.idle ?? []
301  if (!actor.idle && now >= actor.nextIdleAt && choices.length) {
302    let r = Math.random() * choices.reduce((n, c) => n + c.weight, 0)
303    const choice = choices.find(c => (r -= c.weight) <= 0) ?? choices[0]!
304    actor.idle = { frame: choice.frame, until: now + choice.ms[0] + Math.random() * (choice.ms[1] - choice.ms[0]) }
305    actor.nextIdleAt = actor.idle.until + 1500 + Math.random() * 4000
306  }
307  return actor.idle?.frame ?? 'neutral'
308}
309
310const shownMode = () => (lastShown ? modes.get(lastShown) : undefined)
311/** The mode the band draws, mirrored from `bandMode` for the timer. */
312let lastShown: string | false = false
313
314const othersOf = (mode: Mode) => Object.keys(mode.call!.cast).filter(who => who !== mode.call!.host)
315
316const rowsWanted = () => {
317  const mode = shownMode()
318  if (!mode?.theme || !want) return undefined
319  const rowsFor = mode.theme.layout === 'solo' ? soloRowsFor : courtRowsFor
320  return rowsFor(mode.theme, want.maxRows, want.columns, PREFER_ROWS)
321}
322
323/** A fresh frame at the size last measured, or undefined when the band does not fit. */
324const nextFrame = async ($: Engine) => {
325  const mode = shownMode()
326  const rows = rowsWanted()
327  if (!mode?.theme || !want || !rows) return undefined
328  const now = await $.clock.now()
329  const said = speechAt(now)
330  tick += 1
331  level += ((said?.isVoiced ? 0.55 + Math.random() * 0.45 : said ? 0.15 : 0) - level) * 0.5
332  const others = othersOf(mode)
333  const beside = contact && others.includes(contact) ? contact : (others[0] ?? mode.call!.host)
334  const inCast = (who: string | undefined) => (who && mode.call!.cast[who] ? who : undefined)
335  const cellsFor = mode.theme.layout === 'codec' ? codecCells : mode.theme.layout === 'solo' ? soloCells : courtCells
336  const cells = cellsFor(mode.theme, want.columns, rows, {
337    speaker: inCast(speaker),
338    contact: beside,
339    faces: { host: actorFrame(mode, mode.call!.host, now, said), contact: actorFrame(mode, beside, now, said) },
340    level,
341    caption,
342    captionBy: inCast(captionBy),
343  })
344  lastFrame = { mode: mode.name, columns: want.columns, rows, cells }
345  return lastFrame
346}
347
348/**
349 * The frame a redraw shows: the last one if it still fits, else a fresh one.
350 * Only the timer moves the animation; the engine redraws the band far more often.
351 */
352const currentFrame = async ($: Engine) => {
353  const fits = lastFrame && lastFrame.mode === lastShown && lastFrame.columns === want?.columns && lastFrame.rows === rowsWanted()
354  return fits ? lastFrame : nextFrame($)
355}
356
357const tickBand = async ($: Engine) => {
358  if (!band) return
359  const shown = lastFrame?.cells
360  const frame = await nextFrame($)
361  if (!frame) return
362  if (band.columns !== frame.columns || band.rows !== frame.rows) {
363    $.ui.invalidate('ui.render')
364    return
365  }
366  if (frame.cells === shown) return
367  const blit = await $.ui.blit({ requestId: band.requestId, key: BAND_KEY, cells: frame.cells })
368  if (blit.deny !== undefined) $.ui.invalidate('ui.render')
369}
370
371/** Shows the mode's band, or hides it for a mode without one. */
372const showBand = async ($: Engine, mode: Mode | undefined) => {
373  const name = mode?.theme ? mode.name : false
374  lastShown = name
375  lastFrame = undefined
376  contact = undefined
377  await update($, bandMode, () => name)
378  if (!name) {
379    faceTimer?.cancel()
380    faceTimer = undefined
381    band = undefined
382    return
383  }
384  faceTimer ??= $.clock.every(TICK_MS, () => {
385    void tickBand($).catch(() => {})
386  })
387}
388
389const faceSpeaks = async ($: Engine, text: string, who: string | undefined, delayMs: number) => {
390  caption = captionOf(text)
391  speaker = who
392  captionBy = who
393  const mode = shownMode()
394  if (mode && who && othersOf(mode).includes(who)) contact = who
395  // The first audio arrives a moment after the request goes out.
396  speech = { script: scriptOf(text), startedAt: (await $.clock.now()) + 450 + delayMs }
397}
398
399const clearCaption = () => {
400  caption = ''
401  captionBy = undefined
402}
403
404// ── Speech: ElevenLabs through the mode's effects to the speakers ──
405
406/** A value in a curl config file's double quotes. */
407const curlQuoted = (text: string) =>
408  `"${text.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\r/g, '\\r').replace(/\n/g, '\\n').replace(/\t/g, '\\t')}"`
409
410/** The curl config for one line: handed over stdin, so the key stays out of argv and the environment. */
411const curlConfig = (key: string, voice: string, model: string, text: string) =>
412  [
413    `url = ${curlQuoted(`${API}/v1/text-to-speech/${voice}/stream?output_format=pcm_22050`)}`,
414    'request = "POST"',
415    `header = ${curlQuoted(`xi-api-key: ${key}`)}`,
416    'header = "Content-Type: application/json"',
417    `data-binary = ${curlQuoted(JSON.stringify({ text: hasTags(model) ? text : stripTags(text), model_id: model }))}`,
418    '',
419  ].join('\n')
420
421/** What went wrong, in words the person can act on. */
422const explain = (stderr: string, voice: string) => {
423  const status = stderr.match(/returned error: (\d{3})/)?.[1]
424  if (status === '401') return 'ElevenLabs rejected the API key (401): check it, or that it has text-to-speech access.'
425  if (status === '402') return 'ElevenLabs says the account is out of credits (402).'
426  if (status === '404' || status === '400') return `ElevenLabs could not use voice ${voice} (${status}): it may need adding to your voice library.`
427  if (status === '429') return 'ElevenLabs is rate limiting this key (429): too many requests at once.'
428  if (status) return `ElevenLabs answered HTTP ${status}.`
429  if (/ffmpeg|ffplay/.test(stderr) && /not found|not recognized|No such file/i.test(stderr)) {
430    return 'ffmpeg was not found: install it (brew install ffmpeg, winget install Gyan.FFmpeg, or your package manager) or set its path in the plugin options.'
431  }
432  return stderr.trim().split('\n').pop() ?? 'playback failed'
433}
434
435let generation = 0
436let playing: AsyncGenerator<unknown, unknown> | undefined
437
438const stop = async () => {
439  generation++
440  clearCaption()
441  speaker = undefined
442  const was = playing
443  playing = undefined
444  await was?.return(undefined)
445}
446
447type Line = { voice: string; filter?: string; ring?: string; ringMs?: number; speaker?: string }
448
449/** Speaks one line and resolves when it has played; the error in words, if any. */
450type Played = { error?: string; isVoiceRejected?: boolean }
451
452const playLine = async ($: Engine, text: string, mine: number, line: Line): Promise<Played> => {
453  if (mine !== generation) return {}
454  const key = await apiKey($)
455  if (!key) return { error: 'No ElevenLabs API key: set one in the plugin options (/plugin, Avatars, configure) or ELEVENLABS_API_KEY.' }
456  if (!line.voice) return { error: 'No voice set: pick one with /avatar voice or in the /avatar menu.' }
457  const binDir = `${$.plugin.root}/bin`
458  const child = $.process.spawn({
459    argv: isWindows ? ['cmd.exe', '/d', '/c', 'speak.cmd'] : ['bash', `${binDir}/speak.sh`],
460    cwd: binDir,
461    // Unset rather than empty: cmd's `if defined` counts an empty variable as set.
462    env: Object.fromEntries(
463      Object.entries({ AVATAR_FILTER: line.filter, AVATAR_RING: line.ring, AVATAR_FFMPEG: option('ffmpeg') }).filter(
464        (entry): entry is [string, string] => Boolean(entry[1]),
465      ),
466    ),
467    input: curlConfig(key, line.voice, await modelOf($), text),
468  })
469  playing = child
470  // The ring plays first: the band waits it out before the lips move.
471  if (lastShown) await faceSpeaks($, text, line.speaker, line.ring ? (line.ringMs ?? 0) : 0)
472
473  let stderr = ''
474  try {
475    for await (const { stream, text: piece } of child) {
476      if (stream === 'stderr') stderr += piece
477    }
478  } finally {
479    if (playing === child) {
480      playing = undefined
481      speech = undefined
482      if (!line.speaker) clearCaption()
483    }
484  }
485  if (mine !== generation || !stderr.trim()) return {}
486  return { error: explain(stderr, line.voice), isVoiceRejected: /returned error: 40[04]/.test(stderr) }
487}
488
489/** The mode's filter with a pitch shift in front; speed is kept. */
490const pitched = (filter: string | undefined, pitch: number | undefined) => {
491  if (!pitch || pitch === 1 || !(pitch > 0.5 && pitch < 2)) return filter
492  const shift = `asetrate=22050*${pitch},aresample=22050,atempo=${(1 / pitch).toFixed(4)}`
493  if (!filter) return shift
494  const input = filter.match(/^\s*\[0(?::a)?\]/)?.[0] ?? ''
495  return `${input}${shift},${filter.slice(input.length)}`
496}
497
498const added = new Set<string>()
499
500/**
501 * Adds a public library voice to the person's ElevenLabs library, which some
502 * accounts need before the voice can be used; once per voice per session.
503 */
504const addLibraryVoice = async ($: Engine, voice: string, owner: string | undefined, name: string) => {
505  const key = await apiKey($)
506  if (!owner || !key || added.has(voice)) return false
507  added.add(voice)
508  const r = await $.http.fetch(`${API}/v1/voices/add/${owner}/${voice}`, {
509    method: 'POST',
510    headers: { 'xi-api-key': key, 'Content-Type': 'application/json' },
511    body: JSON.stringify({ new_name: name }),
512  })
513  return r.ok
514}
515
516/** Plays a line; a rejected library voice is added to the person's library and tried once more. */
517const playVoiced = async ($: Engine, text: string, mine: number, line: Line, owner: string | undefined, name: string) => {
518  const played = await playLine($, text, mine, line)
519  if (!played.isVoiceRejected || !(await addLibraryVoice($, line.voice, owner, name))) return played.error
520  return (await playLine($, text, mine, { ...line, ring: undefined })).error
521}
522
523/** Speaks `text` in the mode: one voice, or a call's lines in turn, each in its speaker's voice. */
524const perform = async ($: Engine, text: string, mine: number, mode: Mode | undefined) => {
525  const filter = mode?.audio?.filter
526  const effects = mode?.audio?.effects
527  const ring = mode?.audio?.ring
528  if (!mode?.call) {
529    const voice = await voiceOf($, mode)
530    const owner = voice === mode?.voice ? mode?.voiceOwner : undefined
531    const error = await playVoiced($, text, mine, { voice, filter: withEffects(effects, text, filter) }, owner, `${mode?.title ?? 'Avatars'} narrator`)
532    if (error) $.ui.toast(`avatars: ${error}`)
533    return
534  }
535  const lines = parseCall(mode, text)
536  // The band shows who is calling before they speak.
537  const caller = lines.find(line => line.speaker !== mode.call!.host)?.speaker
538  if (caller) contact = caller
539  for (const [i, line] of lines.entries()) {
540    if (mine !== generation) return
541    const member = mode.call.cast[line.speaker]!
542    const error = await playVoiced(
543      $,
544      line.text,
545      mine,
546      {
547        voice: member.voice,
548        filter: withEffects(effects, line.text, pitched(filter, member.pitch)),
549        speaker: line.speaker,
550        ...(i === 0 && ring ? { ring: `${mode.dir}/${ring.file}`, ringMs: ring.ms } : {}),
551      },
552      member.libraryOwner,
553      `${mode.title}: ${member.name}`,
554    )
555    if (error) {
556      $.ui.toast(`avatars: ${error}`)
557      break
558    }
559    if (i < lines.length - 1) await $.clock.sleep(250)
560  }
561  if (mine === generation) {
562    speaker = undefined
563    clearCaption()
564  }
565}
566
567const say = async ($: Engine, text: string, mode?: Mode) => {
568  await stop()
569  await perform($, text, generation, mode ?? (await modeOf($)))
570}
571
572const speakReply = async ($: Engine, answer: string) => {
573  await stop()
574  const mine = generation
575  const mode = await modeOf($)
576  if (!mode) return
577  const reply = answer.slice(0, MAX_REPLY_CHARS)
578  const r = await $.model.complete({
579    model: 'sonnet',
580    system: systemFor(await modelOf($), mode),
581    prompt: `${mode.call ? `${planCall(mode, reply.length)}\n\n` : ''}<reply>\n${reply}\n</reply>`,
582    maxTokens: 1200,
583    effort: 'low',
584  })
585  if (mine !== generation) return
586  if (!r.isAnswered) {
587    $.ui.toast(`avatars: Sonnet gave no speech (${r.reason})`)
588    return
589  }
590  const text = r.text.trim()
591  await update($, lastSaid, () => text)
592  await perform($, text, mine, mode)
593}
594
595const background = ($: Engine, work: Promise<void>) => {
596  void work.catch(err => $.ui.toast(`avatars: ${String(err)}`))
597}
598
599// ── Actions shared by the commands and the menu ──
600
601const setMode = async ($: Engine, name: string) => {
602  if (name === 'off') {
603    await update($, modeSetting, () => 'off')
604    await stop()
605    await showBand($, undefined)
606    return 'Avatars are off for this session.'
607  }
608  const mode = modes.get(name)
609  if (!mode) return `No mode named ${name}. Modes: ${[...modes.keys()].join(', ')}.`
610  await update($, modeSetting, () => name)
611  await showBand($, mode)
612  background($, say($, mode.sample.join('\n'), mode))
613  return `Mode set to ${mode.title}.`
614}
615
616const setEnabled = async ($: Engine, isOn: boolean) => {
617  await update($, enabledSetting, () => (isOn ? 'on' : 'off'))
618  if (!isOn) await stop()
619  return isOn ? 'Spoken replies are on for this session.' : 'Spoken replies are off for this session.'
620}
621
622const setVoice = async ($: Engine, voice: string) => {
623  await update($, voiceSetting, () => voice)
624  const mode = await modeOf($)
625  if (mode?.call) return `Voice set to ${voice}. It is used by single-voice modes; ${mode.title} has its own cast.`
626  background($, say($, 'Hello. This is the voice I will use for my replies from now on.'))
627  return `Voice set to ${voice}.`
628}
629
630const setModel = async ($: Engine, model: string) => {
631  if (!/^eleven_[a-z0-9_]+$/.test(model)) return `Not an ElevenLabs model id: ${model || '(none)'}. Try ${MODELS.join(', ')}.`
632  await update($, modelSetting, () => model)
633  background($, say($, hasTags(model) ? '[warmly] Switched models. [pause] This is how I sound now.' : 'Switched models. This is how I sound now.'))
634  return `Model set to ${model}${hasTags(model) ? ' (audio tags on)' : ''}.`
635}
636
637/** Gives a character of the current mode the person's own voice, kept in voices.json. */
638const recast = async ($: Engine, who: string, voiceArg: string) => {
639  const mode = await modeOf($)
640  if (!mode) return 'Pick a mode first.'
641  if (!voicesFile) return 'No home folder to keep voices.json in.'
642  const members = mode.call ? Object.keys(mode.call.cast) : ['voice']
643  const member = members.find(m => m === who.toLowerCase())
644  if (!member) return `${mode.title} has no ${who}. Characters: ${members.join(', ')}.`
645  const voice = voiceArg ? parseVoice(voiceArg) : undefined
646  if (voiceArg && !voice) return `Not a voice id or URL: ${voiceArg}`
647  const mine = { ...(personal[mode.name] ?? {}) }
648  if (voice) mine[member] = voice
649  else delete mine[member]
650  const next = { ...personal, [mode.name]: mine }
651  if (!Object.keys(mine).length) delete next[mode.name]
652  await $.fs.write(voicesFile, `${JSON.stringify(next, null, 2)}\n`)
653  await reloadModes($)
654  return voice
655    ? `${member} in ${mode.title} now speaks with ${voice} (kept in ${voicesFile}).`
656    : `${member} in ${mode.title} is back to the mode's own voice.`
657}
658
659const playDemo = async ($: Engine, arg: string) => {
660  const mode = await modeOf($)
661  if (!mode) return 'Avatars are off for this session: pick a mode first.'
662  const demos = mode.demos.length ? mode.demos : [mode.sample]
663  const n = Number(arg)
664  const chosen = !arg ? pick(demos) : Number.isInteger(n) ? demos[n - 1] : demos.find(demo => demoHas(mode, demo, arg))
665  if (!chosen) return `No ${mode.title} demo matches ${arg} (there are ${demos.length}; or name a speaker).`
666  background($, say($, chosen.join('\n'), mode))
667  return `Playing a ${mode.title} demo${arg ? ` (${arg})` : ''}.`
668}
669
670const playScenario = async ($: Engine, name: string) => {
671  const all = [...modes.values()].flatMap(mode => Object.entries(mode.scenarios ?? {}).map(([key, s]) => ({ key, mode, ...s })))
672  const scenario = all.find(s => s.key === name)
673  if (!scenario) return `${name ? `No scenario ${name}. ` : ''}Scenarios: ${all.map(s => `${s.key} (${s.mode.title}: ${s.title})`).join(', ') || 'none'}.`
674  if ((await modeOf($))?.name !== scenario.mode.name) {
675    await update($, modeSetting, () => scenario.mode.name)
676    await showBand($, scenario.mode)
677  }
678  background($, say($, scenario.script.join('\n'), scenario.mode))
679  return `Playing ${scenario.title} (${scenario.script.length} lines; /avatar stop ends it).`
680}
681
682/** Saves this session's setup as the plugin's defaults for new sessions. */
683const saveDefaults = async ($: Engine) => {
684  const mode = await modeOf($)
685  const values: Array<[string, string | boolean]> = [
686    ['mode', mode?.name ?? 'off'],
687    ['model', await modelOf($)],
688    ['speak', await isEnabled($)],
689  ]
690  const voice = await read($, voiceSetting)
691  if (voice) values.push(['voice', voice])
692  const refused: string[] = []
693  for (const [field, value] of values) {
694    const { deny } = await $.config.set({ key: `avatars.${field}`, value })
695    if (deny !== undefined) refused.push(`${field} (${deny})`)
696  }
697  return refused.length ? `Could not save: ${refused.join(', ')}.` : `New sessions will start with ${mode?.title ?? 'avatars off'}.`
698}
699
700const fetchJson = async ($: Engine, path: string) => {
701  const key = await apiKey($)
702  if (!key) throw new Error('No ElevenLabs API key: set one in the plugin options or ELEVENLABS_API_KEY.')
703  const r = await $.http.fetch(`${API}${path}`, { headers: { 'xi-api-key': key } })
704  if (!r.ok) throw new Error(`ElevenLabs answered HTTP ${r.status} for ${path.split('?')[0]}: ${r.text.slice(0, 200)}`)
705  return JSON.parse(r.text) as Record<string, unknown>
706}
707
708type ApiVoice = { voice_id: string; name: string; labels?: Record<string, string>; category?: string; description?: string }
709
710const refreshLibrary = async ($: Engine) => {
711  try {
712    const body = await fetchJson($, '/v1/voices')
713    const voices = ((body.voices as ApiVoice[] | undefined) ?? []).map(v => ({ id: v.voice_id, name: v.name }))
714    await update($, libraryVoices, () => voices)
715  } catch (err) {
716    await update($, menuNote, () => String(err instanceof Error ? err.message : err))
717  }
718}
719
720/** Plain checks of the setup: key, ffmpeg, a player, and whether the key can use each mode's voices. */
721const doctor = async ($: Engine) => {
722  const out: string[] = []
723  const key = await apiKey($)
724  out.push(key ? `API key: set (${option('api_key') ? 'plugin options' : 'ELEVENLABS_API_KEY'}).` : 'API key: missing. Set it in the plugin options (/plugin, Avatars, configure) or export ELEVENLABS_API_KEY.')
725  const ffmpeg = option('ffmpeg') ?? 'ffmpeg'
726  const ran = await $.process.run([ffmpeg, '-hide_banner', '-version'], { timeoutMs: 10_000 }).catch(() => undefined)
727  out.push(ran?.exitCode === 0 ? `ffmpeg: ${ran.stdout.split('\n')[0]}` : `ffmpeg: not found (${ffmpeg}). Needed for effects everywhere, and for playback on macOS and Windows.`)
728  if (isWindows) {
729    const ffplay = await $.process.run(['ffplay', '-hide_banner', '-version'], { timeoutMs: 10_000 }).catch(() => undefined)
730    out.push(ffplay?.exitCode === 0 ? 'ffplay: found.' : 'ffplay: not found. It ships with the usual ffmpeg builds (winget install Gyan.FFmpeg).')
731  }
732  const recastCount = Object.values(personal).reduce((n, voices) => n + Object.keys(voices).length, 0)
733  if (recastCount) out.push(`Your own voices: ${recastCount} character(s), from ${voicesFile}.`)
734  if (modeErrors.length) out.push(`Modes that did not load: ${modeErrors.join(' | ')}`)
735  if (key) {
736    for (const mode of modes.values()) {
737      const voices = mode.call ? Object.entries(mode.call.cast).map(([who, m]) => [who, m.voice] as const) : [['voice', await voiceOf($, mode)] as const]
738      const missing: string[] = []
739      for (const [who, voice] of voices) {
740        const r = await $.http.fetch(`${API}/v1/voices/${voice}`, { headers: { 'xi-api-key': key } })
741        if (!r.ok) missing.push(`${who} (${voice}: HTTP ${r.status})`)
742      }
743      out.push(missing.length ? `${mode.title}: voices this key cannot use: ${missing.join(', ')}.` : `${mode.title}: all voices reachable.`)
744    }
745  }
746  return out.join('\n')
747}
748
749const USAGE = [
750  '/avatar                  open the menu',
751  '/avatar mode <name|off>  switch mode for this session',
752  '/avatar modes            list modes',
753  '/avatar on | off         spoken replies on or off',
754  '/avatar test [n|name]    play a demo of the mode',
755  '/avatar replay | stop',
756  '/avatar voice <id|url>   the voice for single-voice modes',
757  '/avatar recast <who> [id] your own voice for a character (no id: back to the default)',
758  '/avatar model <id>       the ElevenLabs model',
759  '/avatar scenario [name]  play a scripted call',
760  '/avatar default          save this session as the default',
761  '/avatar doctor           check the key, ffmpeg and voices',
762  '/avatar reload           read the mode folders again',
763].join('\n')
764
765const modeList = () =>
766  [...modes.values()].map(mode => `${mode.name}: ${mode.title}. ${mode.description}`).join('\n') +
767  (modeErrors.length ? `\nNot loaded: ${modeErrors.join(' | ')}` : '') +
768  `\nYour own modes go in ${userModesDir} (the create-mode skill walks you through one).`
769
770// ── Tools for the create-mode skill: they use the person's key without showing it ──
771
772const TOOLS = {
773  search: 'search_voices',
774  mine: 'my_voices',
775  audition: 'audition',
776  check: 'check_mode',
777  paths: 'paths',
778} as const
779
780const voiceLine = (v: ApiVoice & { accent?: string; gender?: string; age?: string; use_case?: string; descriptive?: string; language?: string }) => {
781  const traits = [v.gender ?? v.labels?.gender, v.age ?? v.labels?.age, v.accent ?? v.labels?.accent, v.language, v.descriptive ?? v.labels?.descriptive, v.use_case ?? v.labels?.use_case]
782    .filter(Boolean)
783    .join(', ')
784  const owner = (v as { public_owner_id?: string }).public_owner_id
785  return `- ${v.name} (${v.voice_id}${owner ? `, library owner ${owner}` : ''})${traits ? `: ${traits}` : ''}${v.description ? `. ${v.description.slice(0, 160)}` : ''}`
786}
787
788const answer = (text: string, isError = false) => ({ result: text, ...(isError ? { isError: true as const } : {}) })
789
790export const register: Register = (on, opts) => {
791  options = opts
792
793  on('session.start', async ($, e, next) => {
794    isInteractive = e.isInteractive
795    await reloadModes($)
796    if (isInteractive) await showBand($, await modeOf($))
797    await $.command.register({
798      name: 'avatar',
799      description: 'Spoken replies with an avatar: opens the menu, or mode, on, off, test, stop, doctor and more',
800      argumentHint: '[mode <name>|on|off|test|replay|stop|voice <id>|model <id>|scenario|default|doctor|modes|reload|help]',
801      immediate: true,
802    })
803    await $.tool.register({
804      name: TOOLS.search,
805      description:
806        "Searches the ElevenLabs shared voice library with the user's own API key, for picking voices for an avatars mode. Returns names, voice ids and traits.",
807      inputSchema: {
808        type: 'object',
809        properties: {
810          query: { type: 'string', description: 'Free-text search, e.g. "gravelly old pirate" or "french aristocrat woman"' },
811          gender: { type: 'string', enum: ['male', 'female', 'neutral'] },
812          age: { type: 'string', enum: ['young', 'middle_aged', 'old'] },
813          accent: { type: 'string', description: 'e.g. british, american, french, indian' },
814          limit: { type: 'number', description: 'How many voices, at most 30 (default 12)' },
815        },
816        required: ['query'],
817      },
818    })
819    await $.tool.register({
820      name: TOOLS.mine,
821      description: "Lists the voices in the user's own ElevenLabs library (ones they made or added), with ids.",
822    })
823    await $.tool.register({
824      name: TOOLS.audition,
825      description:
826        'Speaks a line aloud in an ElevenLabs voice so the user can judge it, optionally through an avatars mode\'s audio effects. Use it to audition voices and tags while designing a mode. Returns when playback ends.',
827      inputSchema: {
828        type: 'object',
829        properties: {
830          voice: { type: 'string', description: 'The voice id' },
831          text: { type: 'string', description: 'The line, with [audio tags] if the model is v3/v4' },
832          model: { type: 'string', description: `An ElevenLabs model id (default: the session's, e.g. ${DEFAULT_MODEL})` },
833          mode: { type: 'string', description: 'A loaded mode whose audio filter to apply' },
834          filter: { type: 'string', description: 'An ffmpeg -filter_complex graph to try instead of a mode\'s' },
835          pitch: { type: 'number', description: 'Shift the pitch, keeping speed: 0.95 is 5% lower' },
836        },
837        required: ['voice', 'text'],
838      },
839    })
840    await $.tool.register({
841      name: TOOLS.paths,
842      description:
843        "Where the avatars plugin keeps things: the user's modes folder (where new modes go), the built-in example modes, the format reference, and the portrait bake script.",
844    })
845    await $.tool.register({
846      name: TOOLS.check,
847      description:
848        'Checks an avatars mode folder (mode.json, portraits.json, ring sound) and whether the user\'s key can use every voice in it, then reloads the modes so a good one can be switched to with /avatar mode <name>.',
849      inputSchema: {
850        type: 'object',
851        properties: { path: { type: 'string', description: 'The mode folder, absolute' } },
852        required: ['path'],
853      },
854    })
855
856    return next(e)
857  })
858
859  on('turn.complete', async ($, e, next) => {
860    const done = await next(e)
861    await ensureLoaded($)
862    const isMainAnswer = e.agentId === undefined && e.reason === 'answer' && e.answer.trim() !== ''
863    if (isInteractive && isMainAnswer && (await isEnabled($)) && (await modeOf($))) {
864      background($, speakReply($, e.answer))
865    }
866
867    return done
868  })
869
870  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
871    const { Raster } = $.ui.resolve(e)
872    const shown = await read($, bandMode)
873    if (!shown || e.props.hasSurvey || e.surface !== 'terminal' || !Raster) {
874      band = undefined
875      return next(e)
876    }
877    lastShown = shown
878    want = { columns: Math.min(512, e.props.bodyColumns), maxRows: e.props.maxRows }
879    const frame = await currentFrame($)
880    if (!frame) {
881      band = undefined
882      return next(e)
883    }
884    band = { requestId: e.requestId, columns: want.columns, rows: frame.rows }
885
886    return <Raster key={BAND_KEY} columns={want.columns} rows={frame.rows} cells={frame.cells} />
887  })
888
889  on('ui.render', { component: 'Pane', requestId: MENU }, async ($, e) => {
890    const { Box, Text, Button, Select } = $.ui.resolve(e)
891    const mode = await modeOf($)
892    const isOn = await isEnabled($)
893    const model = await modelOf($)
894    const voice = await voiceOf($, mode)
895    const voices = await read($, libraryVoices)
896    const note = await read($, menuNote)
897    const hasKey = (await apiKey($)) !== undefined
898    const scenarios = [...modes.values()].flatMap(m => Object.entries(m.scenarios ?? {}).map(([key, s]) => ({ key, title: `${m.title}: ${s.title}` })))
899    const act = (work: () => Promise<string>) => () => {
900      void work().then(text => update($, menuNote, () => text)).catch(err => update($, menuNote, () => String(err)))
901    }
902
903    const voiceOptions = [
904      ...(mode?.voice ? [{ value: mode.voice, label: `${mode.title} default` }] : []),
905      ...(voices ?? []).filter(v => v.id !== mode?.voice).map(v => ({ value: v.id, label: v.name })),
906    ]
907    if (voice && !voiceOptions.some(o => o.value === voice)) voiceOptions.unshift({ value: voice, label: voice })
908
909    return (
910      <Box flexDirection="column" gap={1}>
911        <Box flexDirection="column">
912          <Text>
913            Spoken replies: <Text bold>{isOn ? 'on' : 'off'}</Text>
914            {'   '}API key: {hasKey ? <Text color="green">set</Text> : <Text color="red">missing</Text>}
915          </Text>
916          {!hasKey && <Text dimColor>Set it with /plugin (Avatars, configure), or export ELEVENLABS_API_KEY and restart.</Text>}
917        </Box>
918        <Box flexDirection="column">
919          <Select
920            key="mode"
921            label="Mode  "
922            autoFocus
923            value={mode?.name ?? 'off'}
924            options={[...[...modes.values()].map(m => ({ value: m.name, label: m.title })), { value: 'off', label: 'Off' }]}
925            onSelect={value => act(() => setMode($, value))()}
926          />
927          {mode && <Text dimColor>{mode.description}</Text>}
928        </Box>
929        {mode && !mode.call && voiceOptions.length > 0 && (
930          <Select key="voice" label="Voice " value={voice} options={voiceOptions} onSelect={value => act(() => setVoice($, value))()} />
931        )}
932        {mode && !mode.call && voices === null && hasKey && (
933          <Button key="load-voices" dimColor onPress={() => void refreshLibrary($)}>
934            Load my ElevenLabs voices
935          </Button>
936        )}
937        <Select
938          key="model"
939          label="Model "
940          value={model}
941          options={[...new Set([model, ...MODELS])].map(m => ({ value: m, label: `${m}${hasTags(m) ? ' (audio tags)' : ''}` }))}
942          onSelect={value => act(() => setModel($, value))()}
943        />
944        {scenarios.length > 0 && (
945          <Select
946            key="scenario"
947            label="Scenario "
948            value=""
949            options={[{ value: '', label: 'Pick one to play' }, ...scenarios.map(s => ({ value: s.key, label: s.title }))]}
950            onSelect={value => value && act(() => playScenario($, value))()}
951          />
952        )}
953        <Box flexDirection="row" gap={1} flexWrap="wrap">
954          <Button key="test" hotkey="t" variant="primary" onPress={act(() => playDemo($, ''))}>Test</Button>
955          <Button key="replay" hotkey="r" onPress={act(async () => {
956            const last = await read($, lastSaid)
957            if (!last) return 'Nothing spoken yet.'
958            background($, say($, last))
959            return 'Replaying.'
960          })}>Replay</Button>
961          <Button key="stop" hotkey="s" onPress={act(async () => (await stop(), 'Stopped.'))}>Stop</Button>
962          <Button key="toggle" hotkey="o" onPress={act(() => setEnabled($, !isOn))}>{isOn ? 'Turn off' : 'Turn on'}</Button>
963          <Button key="default" hotkey="d" onPress={act(() => saveDefaults($))}>Save as default</Button>
964          <Button key="doctor" hotkey="c" onPress={act(() => doctor($))}>Check setup</Button>
965          <Button key="close" role="dismiss" onPress={() => void $.ui.close({ id: MENU })}>Close</Button>
966        </Box>
967        {note !== '' && <Text>{note}</Text>}
968        <Text dimColor>New modes: ask Claude to create one (the create-mode skill), or see {userModesDir}.</Text>
969      </Box>
970    )
971  })
972
973  on('command.run', { command: 'avatar' }, async ($, e) => {
974    await ensureLoaded($)
975    const [verb = '', ...rest] = e.args.trim().split(/\s+/)
976    const arg = rest.join(' ')
977
978    switch (verb.toLowerCase()) {
979      case '':
980      case 'menu': {
981        await update($, menuNote, () => '')
982        const opened = await $.ui.open({ id: MENU, title: 'Avatars', focus: true, closeOnEscape: true })
983        if (!opened.isPlaced) return { text: `${USAGE}\n\n(${opened.reason})` }
984        const mode = await modeOf($)
985        if (mode && !mode.call && (await read($, libraryVoices)) === null && (await apiKey($))) void refreshLibrary($)
986        return { text: 'Opened the avatars menu.' }
987      }
988      case 'help':
989        return { text: USAGE }
990      case 'modes':
991        return { text: modeList() }
992      case 'mode':
993        return { text: arg ? await setMode($, arg.toLowerCase()) : modeList() }
994      case 'on':
995        return { text: await setEnabled($, true) }
996      case 'off':
997        return { text: await setEnabled($, false) }
998      case 'stop':
999        await stop()
1000        return { text: 'Stopped.' }
1001      case 'test':
1002        return { text: await playDemo($, arg) }
1003      case 'replay': {
1004        const last = await read($, lastSaid)
1005        if (!last) return { text: 'Nothing spoken yet.' }
1006        background($, say($, last))
1007        return { text: `Replaying: ${last}` }
1008      }
1009      case 'voice': {
1010        const id = parseVoice(arg)
1011        return { text: id ? await setVoice($, id) : `Not a voice id or URL: ${arg || '(none)'}` }
1012      }
1013      case 'model':
1014        return { text: await setModel($, arg) }
1015      case 'recast': {
1016        const [who = '', voice = ''] = rest
1017        return { text: who ? await recast($, who, voice) : `Usage: /avatar recast <character> [voice id or URL]` }
1018      }
1019      case 'scenario':
1020        return { text: await playScenario($, arg.toLowerCase()) }
1021      case 'default':
1022        return { text: await saveDefaults($) }
1023      case 'doctor':
1024        return { text: await doctor($) }
1025      case 'reload':
1026        await reloadModes($)
1027        await showBand($, await modeOf($))
1028        return { text: modeList() }
1029      default:
1030        return { text: USAGE }
1031    }
1032  })
1033
1034  on('tool.call', { tool: 'mcp__avatars__search_voices' }, async ($, e) => {
1035    const input = e as unknown as { query: string; gender?: string; age?: string; accent?: string; limit?: number }
1036    const params: Array<[string, string]> = [['search', input.query], ['page_size', String(Math.min(30, input.limit ?? 12))]]
1037    for (const field of ['gender', 'age', 'accent'] as const) if (input[field]) params.push([field, input[field]!])
1038    const query = params.map(([k, v]) => `${k}=${encodeURIComponent(v)}`).join('&')
1039    try {
1040      const body = await fetchJson($, `/v1/shared-voices?${query}`)
1041      const voices = (body.voices as Array<ApiVoice & { accent?: string }> | undefined) ?? []
1042      if (!voices.length) return answer('No voices matched; try broader words.')
1043      return answer(`${voices.map(voiceLine).join('\n')}\n\nAudition one with the audition tool before choosing.`)
1044    } catch (err) {
1045      return answer(String(err instanceof Error ? err.message : err), true)
1046    }
1047  })
1048
1049  on('tool.call', { tool: 'mcp__avatars__my_voices' }, async $ => {
1050    try {
1051      const body = await fetchJson($, '/v1/voices')
1052      const voices = (body.voices as ApiVoice[] | undefined) ?? []
1053      return answer(voices.length ? voices.map(voiceLine).join('\n') : 'The library is empty.')
1054    } catch (err) {
1055      return answer(String(err instanceof Error ? err.message : err), true)
1056    }
1057  })
1058
1059  on('tool.call', { tool: 'mcp__avatars__audition' }, async ($, e) => {
1060    const input = e as unknown as { voice: string; text: string; model?: string; mode?: string; filter?: string; pitch?: number }
1061    const voice = parseVoice(input.voice) ?? input.voice
1062    const mode = input.mode ? modes.get(input.mode) : undefined
1063    if (input.model && !/^eleven_[a-z0-9_]+$/.test(input.model)) return answer(`Not an ElevenLabs model id: ${input.model}`, true)
1064    await stop()
1065    const mine = generation
1066    const previous = await read($, modelSetting)
1067    if (input.model) await update($, modelSetting, () => input.model!)
1068    try {
1069      const filter = pitched(input.filter ?? mode?.audio?.filter, input.pitch)
1070      const { error } = await playLine($, input.text, mine, { voice, filter: withEffects(mode?.audio?.effects, input.text, filter) })
1071      return error ? answer(error, true) : answer('Played. Ask the user how it sounded.')
1072    } finally {
1073      if (input.model) await update($, modelSetting, () => previous)
1074    }
1075  })
1076
1077  on('tool.call', { tool: 'mcp__avatars__paths' }, async $ => {
1078    await ensureLoaded($)
1079    const root = $.plugin.root
1080    return answer(
1081      [
1082        `New modes go in: ${userModesDir}/<name>/`,
1083        `Built-in example modes: ${root}/modes/coven (cast with portraits), ${root}/modes/prototype (solo with audio effects), ${root}/modes/narrator (single voice)`,
1084        `Format reference: ${root}/MODES.md`,
1085        `Portrait bake script: ${root}/bin/bake-portraits.py <mode folder> --preview <sheet.png> (needs Python 3 with Pillow and numpy)`,
1086        `Platform: ${isWindows ? 'Windows' : 'macOS or Linux'}`,
1087        `Mode packs: ${packModesDirs.length ? packModesDirs.join(', ') : 'none installed'} (a plugin with an avatars/modes/ folder; see MODES.md)`,
1088        `Loaded modes: ${[...modes.keys()].join(', ')}${modeErrors.length ? `; not loaded: ${modeErrors.join(' | ')}` : ''}`,
1089      ].join('\n'),
1090    )
1091  })
1092
1093  on('tool.call', { tool: 'mcp__avatars__check_mode' }, async ($, e) => {
1094    await ensureLoaded($)
1095    const dir = String((e as unknown as { path: string }).path).replace(/[\\/]+$/, '')
1096    const name = dir.split(/[\\/]/).pop() ?? ''
1097    if (!isModeName(name)) return answer(`The folder name ${name} must be lowercase letters, digits, - or _: it is the mode's name.`, true)
1098    const mode = await readMode($, dir, name)
1099    if (typeof mode === 'string') return answer(mode, true)
1100    const notes: string[] = []
1101    const key = await apiKey($)
1102    const voices = mode.call ? Object.entries(mode.call.cast).map(([who, m]) => [who, m.voice] as const) : mode.voice ? [['voice', mode.voice] as const] : []
1103    for (const [who, voice] of voices) {
1104      if (!key) break
1105      const r = await $.http.fetch(`${API}/v1/voices/${voice}`, { headers: { 'xi-api-key': key } })
1106      if (!r.ok) notes.push(`${who}'s voice ${voice} is not usable with this key (HTTP ${r.status}); add it to the library or pick another.`)
1107    }
1108    if (mode.call && mode.band && mode.theme) {
1109      for (const who of Object.keys(mode.call.cast)) {
1110        const sizes = mode.theme.portraits[who]
1111        if (!sizes) notes.push(`portraits.json has no ${who}: the band shows the host's portrait in their place.`)
1112        else if (!sizes[20]?.frames.talk_a) notes.push(`${who} has no talk_a frame at 20 rows: their mouth will not move.`)
1113      }
1114    }
1115    if (mode.audio?.ring && !(await $.fs.exists(`${dir}/${mode.audio.ring.file}`))) notes.push(`The ring sound ${mode.audio.ring.file} is missing.`)
1116    if (![userModesDir, `${$.plugin.root}/modes`, ...packModesDirs].some(root => dir.startsWith(`${root}/`))) {
1117      notes.push(`It loads only from ${userModesDir}/${name} or an installed mode pack; move it there.`)
1118    }
1119    await reloadModes($)
1120    return answer(
1121      `${mode.title} is valid.${notes.length ? `\n${notes.map(n => `- ${n}`).join('\n')}` : ''}\nSwitch to it with /avatar mode ${name}, then /avatar test.`,
1122    )
1123  })
1124}
1125

1// The codec band, a court layout in the style of a 1998 stealth game's radio:
2// whoever is on the line on the left, the host on the right, and a panel
3// between with signal bars and the frequency on a seven-segment display above
4// the subtitles. Sized exactly like the court. Colours come from the mode's
5// band: `frame` and `frameLit` for the borders, the meter's `lit` and `dim`
6// for the display.
7
8import { cellSafe, clamp01, courtPanel, courtPixels, fade, GAP, packCells, wrapRows, type CourtState, type CourtTheme } from './court'
9
10const DEFAULT_FREQUENCY = '140.85'
11/** The display's box, in pixels from the top: a digit with one pixel above and below, at every band of 12 rows or more. */
12const BOX_TOP = 2
13const BOX_BOTTOM = 14
14/** Below this the display gives its rows to the subtitles and the frequency moves into the top rule. */
15const DISPLAY_MIN_ROWS = 12
16
17/** Seven segments on a 5x9 pixel grid: [x, y] pixels per segment a..g. */
18const SEGMENTS: Array<Array<[number, number]>> = [
19  [[1, 0], [2, 0], [3, 0]],
20  [[4, 1], [4, 2], [4, 3]],
21  [[4, 5], [4, 6], [4, 7]],
22  [[1, 8], [2, 8], [3, 8]],
23  [[0, 5], [0, 6], [0, 7]],
24  [[0, 1], [0, 2], [0, 3]],
25  [[1, 4], [2, 4], [3, 4]],
26]
27const DIGIT_W = 5
28const DIGIT_H = 9
29const DIGIT_SEGMENTS: Record<string, string> = {
30  '0': 'abcdef', '1': 'bc', '2': 'abdeg', '3': 'abcdg', '4': 'bcfg',
31  '5': 'acdfg', '6': 'acdefg', '7': 'abc', '8': 'abcdefg', '9': 'abcdfg',
32}
33
34/** A codec band's cells for one frame, `rows` tall. */
35export const codecCells = (theme: CourtTheme, columns: number, rows: number, state: CourtState) => {
36  const H = rows * 2
37  const pixels: Array<number | undefined> = Array.from({ length: columns * H }, () => undefined)
38  const dot = (x: number, y: number, color: number) => {
39    if (x >= 0 && x < columns && y >= 0 && y < H) pixels[y * columns + x] = color
40  }
41  const text = new Map<number, [string, number]>()
42  const write = (x: number, y: number, s: string, fg: number) =>
43    [...s].forEach((c, i) => {
44      if (x + i >= 0 && x + i < columns && y >= 0 && y < rows) text.set(y * columns + x + i, [c, fg])
45    })
46  const mid = fade(theme.frame, theme.frameLit, 0.5)
47  const { lit, dim } = theme.meter
48  // Every frame is as wide as the host's portrait, so a new caller never moves the layout.
49  const slot = theme.portraits[theme.host]![rows]!.w
50
51  // A portrait centred in its frame: lit while talking, dimmed while the other talks.
52  const portrait = (who: string, x0: number, face: string) => {
53    const { art, rgb } = courtPixels(theme, who, rows, face)
54    const pad = Math.floor((slot - art.w) / 2)
55    const isTalking = state.speaker === who
56    const k = state.speaker && !isTalking ? 0.5 : 1
57    for (let y = 0; y < Math.min(art.h, H); y++) {
58      for (let x = Math.max(0, -pad); x < Math.min(art.w, slot - pad); x++) {
59        const i = (y * art.w + x) * 3
60        dot(x0 + 1 + pad + x, y, (Math.round((rgb[i] ?? 0) * k) << 16) | (Math.round((rgb[i + 1] ?? 0) * k) << 8) | Math.round((rgb[i + 2] ?? 0) * k))
61      }
62    }
63    const border = isTalking ? theme.frameLit : theme.frame
64    for (let y = 0; y < H; y++) {
65      dot(x0, y, border)
66      dot(x0 + slot + 1, y, border)
67    }
68    for (let x = x0; x <= x0 + slot + 1; x++) {
69      dot(x, 0, border)
70      dot(x, H - 1, border)
71    }
72  }
73
74  const frameWidth = slot + 2
75  const panel = courtPanel(theme, rows, columns)
76  const px0 = frameWidth + GAP
77  portrait(state.contact, 0, state.faces.contact)
78  portrait(theme.host, panel ? px0 + panel + GAP : frameWidth + 1, state.faces.host)
79  if (!panel) return packCells(columns, rows, pixels, text)
80
81  const frequency = theme.frequencies?.[state.contact] ?? DEFAULT_FREQUENCY
82  const hasDisplay = rows >= DISPLAY_MIN_ROWS
83  const centred = (y: number, s: string, fg: number) => write(px0 + Math.floor((panel - [...s].length) / 2), y, s, fg)
84  const label = (y: number, name: string) => {
85    write(px0, y, '─'.repeat(panel), theme.frame)
86    centred(y, ` ${name} `, state.speaker ? mid : theme.frame)
87  }
88  label(0, hasDisplay ? theme.title : `${theme.title}   ${frequency}`)
89  label(rows - 1, theme.meter.label)
90  write(px0 - 2, Math.floor(rows / 2), '<', theme.frame)
91  write(px0 + panel + 1, Math.floor(rows / 2), '>', theme.frame)
92
93  if (hasDisplay) {
94    const boxLeft = px0 + 2
95    const boxRight = px0 + panel - 3
96    for (let x = boxLeft; x <= boxRight; x++) {
97      dot(x, BOX_TOP, mid)
98      dot(x, BOX_BOTTOM, mid)
99    }
100    for (let y = BOX_TOP; y <= BOX_BOTTOM; y++) {
101      dot(boxLeft, y, mid)
102      dot(boxRight, y, mid)
103    }
104    const screen = fade(dim, 0, 0.7)
105    for (let y = BOX_TOP + 1; y < BOX_BOTTOM; y++) for (let x = boxLeft + 1; x < boxRight; x++) dot(x, y, screen)
106
107    // Signal bars lengthening downward; the voice lights them from the bottom.
108    const digitTop = (BOX_TOP + BOX_BOTTOM + 1 - DIGIT_H) / 2
109    const bars = (DIGIT_H + 1) / 2
110    const litBars = Math.round(clamp01(state.level) * bars)
111    for (let i = 0; i < bars; i++) {
112      const length = Math.round(2 + 3 * ((i + 1) / bars) ** 0.6)
113      for (let x = 0; x < length; x++) dot(boxLeft + 2 + x, digitTop + i * 2, bars - i <= litBars ? lit : dim)
114    }
115
116    // The caller's frequency, unlit segments ghosted.
117    const digits = [...frequency].filter(c => c !== '.').length
118    let x = boxRight - 1 - (digits * (DIGIT_W + 1) + 2)
119    for (const ch of frequency) {
120      if (ch === '.') {
121        dot(x, digitTop + DIGIT_H - 1, lit)
122        x += 2
123        continue
124      }
125      const on = DIGIT_SEGMENTS[ch] ?? ''
126      SEGMENTS.forEach((segment, s) => {
127        for (const [sx, sy] of segment) dot(x + sx, digitTop + sy, on.includes('abcdefg'[s] ?? '') ? lit : dim)
128      })
129      x += DIGIT_W + 1
130    }
131  }
132
133  // Who speaks and what, centred in the rows the display leaves.
134  const by = state.speaker ?? state.captionBy
135  if (state.caption) {
136    const first = hasDisplay ? Math.ceil((BOX_BOTTOM + 1) / 2) : 1
137    const space = rows - 1 - first
138    const gap = space >= 5 ? 1 : 0
139    const lines = wrapRows(cellSafe(state.caption), panel - 4, Math.max(1, space - (by ? 1 + gap : 0)))
140    const top = first + Math.max(0, Math.floor((space - lines.length - (by ? 1 + gap : 0)) / 2))
141    if (by) {
142      const ink = theme.ink[by] ?? theme.subtitle
143      centred(top, theme.names[by] ?? by.toUpperCase(), state.speaker ? ink : fade(ink, 0, 0.35))
144    }
145    lines.forEach((line, i) => centred(top + (by ? 1 + gap : 0) + i, line, theme.subtitle))
146  }
147
148  return packCells(columns, rows, pixels, text)
149}
150

1// Post effects: `audio.effects` in mode.json, turned into ffmpeg filter graph
2// pieces fresh for every line, so no two lines glitch alike. Effects run on
3// the dry voice, before any pitch shift and the mode's `audio.filter`. Their
4// timing is planned from the line's text, since the audio streams in and its
5// length is not known up front.
6
7export type Effect =
8  | {
9      /** Repeats a short fragment a few times, sometimes reversed: a voice catching on a word. */
10      type: 'stutter'
11      perSecond?: number
12      repeats?: [number, number]
13      ms?: [number, number]
14      reverse?: number
15      /** The share of stutters that speed up into the word and slow back out. */
16      rush?: number
17      /** A rush's top speed: 1.18 is 18% faster and higher. */
18      rushRate?: number
19    }
20  | {
21      /** A failing radio: brief cuts to static or near silence, and a little hiss under it all. */
22      type: 'dropouts'
23      perSecond?: number
24      ms?: [number, number]
25      static?: number
26      hiss?: number
27    }
28
29const SAMPLE_RATE = 22050
30/** Speech is about this many characters a second; a `[pause]` tag adds a beat. */
31const CHARS_PER_SECOND = 15
32const PAUSE_SECONDS = 0.6
33/** No effect starts this close to either end of the line. */
34const MARGIN = 0.3
35
36/** About how long `text` takes to say, in seconds. */
37export const spokenSeconds = (text: string) => {
38  const pauses = (text.match(/\[[^\]]*pause[^\]]*\]/gi) ?? []).length
39  const plain = text.replace(/\[[^\]]*\]/g, '').replace(/\s+/g, ' ').trim()
40  return plain.length / CHARS_PER_SECOND + pauses * PAUSE_SECONDS
41}
42
43const between = (rand: () => number, [lo, hi]: [number, number]) => lo + rand() * (hi - lo)
44
45/** Times in (MARGIN, seconds - MARGIN) at about `perSecond`, sorted, at least `gap` apart. */
46const moments = (rand: () => number, seconds: number, perSecond: number, gap: number) => {
47  const span = seconds - 2 * MARGIN
48  if (span <= 0 || perSecond <= 0) return []
49  let count = Math.floor(span * perSecond)
50  if (rand() < span * perSecond - count) count += 1
51  const times = Array.from({ length: count }, () => MARGIN + rand() * span).sort((a, b) => a - b)
52  return times.filter((t, i) => i === 0 || t - times[i - 1]! >= gap)
53}
54
55const fixed = (n: number) => n.toFixed(3)
56
57/** How long each step of a rush's speed-up or slow-down lasts, in seconds, and how many steps each way. */
58const RUSH_STEP = 0.12
59const RUSH_STEPS = 3
60
61/**
62 * `[from]` stuttered into `[to]`: the voice cut at each moment, a fragment replayed, then on from there.
63 * A rushed stutter speeds up into the fragment and slows back out, faster and higher like tape.
64 */
65const stutter = (e: Extract<Effect, { type: 'stutter' }>, seconds: number, rand: () => number, from: string, to: string) => {
66  const ramp = RUSH_STEP * RUSH_STEPS
67  const events = moments(rand, seconds, e.perSecond ?? 0.5, 2 * ramp + 0.2).map(at => ({
68    at,
69    length: between(rand, e.ms ?? [50, 110]) / 1000,
70    times: Math.round(between(rand, e.repeats ?? [1, 3])),
71    isReversed: rand() < (e.reverse ?? 0.2),
72    isRushed: rand() < (e.rush ?? 0.5),
73  }))
74  if (!events.length) return undefined
75  const peak = e.rushRate ?? 1.18
76  const rate = (step: number) => 1 + ((peak - 1) * step) / RUSH_STEPS
77  const pieces: Array<{ start: number; end?: number; rate: number; isReversed?: boolean }> = []
78  let start = 0
79  for (const event of events) {
80    const rampIn = event.isRushed ? Math.min(ramp, event.at - start) : 0
81    pieces.push({ start, end: event.at - rampIn, rate: 1 })
82    if (rampIn > 0) {
83      for (let step = 1; step <= RUSH_STEPS; step++) {
84        const stepStart = event.at - rampIn + ((step - 1) * rampIn) / RUSH_STEPS
85        pieces.push({ start: stepStart, end: stepStart + rampIn / RUSH_STEPS, rate: rate(step) })
86      }
87    }
88    for (let i = 0; i < event.times; i++) {
89      pieces.push({ start: event.at, end: event.at + event.length, rate: event.isRushed ? peak : 1, isReversed: event.isReversed })
90    }
91    start = event.at
92    if (event.isRushed) {
93      for (let step = RUSH_STEPS; step >= 1; step--) {
94        pieces.push({ start, end: start + RUSH_STEP, rate: rate(step) })
95        start += RUSH_STEP
96      }
97    }
98  }
99  pieces.push({ start, rate: 1 })
100  const labels = pieces.map((_, i) => `[${to}_${i}]`)
101  const chain = (p: (typeof pieces)[number]) =>
102    [
103      `atrim=start=${fixed(p.start)}${p.end === undefined ? '' : `:end=${fixed(p.end)}`}`,
104      ...(p.isReversed ? ['areverse'] : []),
105      'asetpts=PTS-STARTPTS',
106      ...(p.rate === 1 ? [] : [`asetrate=${SAMPLE_RATE}*${p.rate.toFixed(4)}`, `aresample=${SAMPLE_RATE}`]),
107    ].join(',')
108  return [
109    `${from}asplit=${pieces.length}${labels.join('')}`,
110    ...pieces.map((piece, i) => `${labels[i]}${chain(piece)}[${to}_p${i}]`),
111    `${pieces.map((_, i) => `[${to}_p${i}]`).join('')}concat=n=${pieces.length}:v=0:a=1[${to}]`,
112  ].join(';')
113}
114
115/** `[from]` with radio dropouts into `[to]`. */
116const dropouts = (e: Extract<Effect, { type: 'dropouts' }>, seconds: number, rand: () => number, from: string, to: string) => {
117  const cuts = moments(rand, seconds, e.perSecond ?? 0.8, 0.2).map(at => ({
118    at,
119    end: at + between(rand, e.ms ?? [30, 140]) / 1000,
120    isStatic: rand() < (e.static ?? 0.6),
121  }))
122  const during = (list: typeof cuts) => list.map(c => `between(t,${fixed(c.at)},${fixed(c.end)})`).join('+') || '0'
123  const hiss = e.hiss ?? 0.004
124  return [
125    `${from}volume='if(${during(cuts)},0.08,1)':eval=frame[${to}_v]`,
126    `anoisesrc=r=${SAMPLE_RATE}:c=white:a=0.09,volume='if(${during(cuts.filter(c => c.isStatic))},1,${hiss / 0.09})':eval=frame[${to}_n]`,
127    `[${to}_v][${to}_n]amix=inputs=2:normalize=0:duration=first[${to}]`,
128  ].join(';')
129}
130
131/**
132 * `filter` (and any pitch chain in front of it) with the effects ahead of it, for one line of `text`.
133 * A filter may name its input `[0]` or leave it unnamed; both read the effects' output.
134 */
135export const withEffects = (effects: Effect[] | undefined, text: string, filter: string | undefined, rand: () => number = Math.random) => {
136  if (!effects?.length) return filter
137  const seconds = spokenSeconds(text)
138  const graphs: string[] = []
139  let from = '[0]'
140  effects.forEach((effect, i) => {
141    const to = `fx${i}`
142    const graph = effect.type === 'stutter' ? stutter(effect, seconds, rand, from, to) : effect.type === 'dropouts' ? dropouts(effect, seconds, rand, from, to) : undefined
143    if (!graph) return
144    graphs.push(graph)
145    from = `[${to}]`
146  })
147  if (!graphs.length) return filter
148  const body = (filter ?? 'anull').replace(/^\s*\[0(?::a)?\]/, '')
149  return `${graphs.join(';')};${from}${body}`
150}
151

1// The solo band: the host alone, centred, in falling glyph rain that quickens
2// while they speak. The title and a signal meter sit over the left rain, the
3// subtitles over the right. The face's edges fade into the rain, and while it
4// talks, rows of it now and then slip sideways like a bad signal.
5
6import { cellSafe, clamp01, COURT_ROWS, courtPixels, fade, GAP, packCells, steadyRows, wrapRows, type CourtState, type CourtTheme } from './court'
7
8/** Each side needs this many columns for the subtitles to show. */
9const SIDE_MIN = 26
10const CAPTION_MAX = 56
11const METER_GLYPHS = 10
12const DEFAULT_GLYPHS = [...'01:.<>{}#']
13/** How many pixels in from the face's edge the fade into the rain reaches. */
14const FEATHER = 3
15
16const faceWidth = (theme: CourtTheme, rows: number) => theme.portraits[theme.host]?.[rows]?.w
17
18/** The tallest band whose sides fit the subtitles, else the tallest whose face fits. */
19export const soloRowsFor = (theme: CourtTheme, maxRows: number, columns: number, preferRows: number) =>
20  steadyRows(theme.title, spare => {
21    const sizes = COURT_ROWS.filter(r => {
22      const w = faceWidth(theme, r)
23      return r <= Math.min(maxRows, preferRows) && w !== undefined && w <= columns - spare
24    })
25    return sizes.find(r => columns - spare - faceWidth(theme, r)! >= 2 * (SIDE_MIN + GAP)) ?? sizes[0]
26  })
27
28type Drop = { y: number; speed: number; glyphs: string[] }
29const rain = new Map<string, { columns: number; drops: Array<Drop | undefined> }>()
30const pick = <T,>(list: readonly T[]) => list[Math.floor(Math.random() * list.length)]!
31
32/** A solo band's cells for one frame, `rows` tall. Each call advances the rain one tick. */
33export const soloCells = (theme: CourtTheme, columns: number, rows: number, state: CourtState) => {
34  const H = rows * 2
35  const pixels: Array<number | undefined> = Array.from({ length: columns * H }, () => undefined)
36  const text = new Map<number, [string, number]>()
37  const clear = new Set<number>()
38  const write = (x: number, y: number, s: string, fg: number) =>
39    [...s].forEach((c, i) => {
40      if (x + i >= 0 && x + i < columns && y >= 0 && y < rows) text.set(y * columns + x + i, [c, fg])
41    })
42  /** Keeps the rain out of a box of cells, so text over it reads. */
43  const clearBox = (x0: number, y0: number, x1: number, y1: number) => {
44    for (let y = Math.max(0, y0); y <= Math.min(rows - 1, y1); y++) for (let x = Math.max(0, x0); x <= Math.min(columns - 1, x1); x++) clear.add(y * columns + x)
45  }
46
47  const { art, rgb } = courtPixels(theme, theme.host, rows, state.faces.host)
48  const left = Math.floor((columns - art.w) / 2)
49  const right = left + art.w
50  const isTalking = state.speaker === theme.host
51
52  // The face, its edges feathered into the dark, with rows slipping sideways now and then while it talks.
53  const slips = new Map<number, number>()
54  if (Math.random() < (isTalking ? 0.12 : 0.015)) {
55    const top = Math.floor(Math.random() * art.h)
56    const height = 1 + Math.floor(Math.random() * 3)
57    const shift = (Math.random() < 0.5 ? -1 : 1) * (1 + Math.floor(Math.random() * 3))
58    for (let y = top; y < Math.min(art.h, top + height); y++) slips.set(y, shift)
59  }
60  for (let y = 0; y < Math.min(art.h, H); y++) {
61    const shift = slips.get(y) ?? 0
62    for (let x = 0; x < art.w; x++) {
63      const sx = x - shift
64      if (sx < 0 || sx >= art.w) continue
65      const i = (y * art.w + sx) * 3
66      const edge = Math.min(x, y, art.w - 1 - x, art.h - 1 - y)
67      const k = edge >= FEATHER ? 1 : (edge + 1) / (FEATHER + 1)
68      let color = (Math.round((rgb[i] ?? 0) * k) << 16) | (Math.round((rgb[i + 1] ?? 0) * k) << 8) | Math.round((rgb[i + 2] ?? 0) * k)
69      if (shift) color = fade(color, theme.frameLit, 0.35)
70      pixels[y * columns + left + x] = color
71    }
72  }
73
74  // Title and meter over the left rain, the subtitles over the right.
75  const leftWidth = left - GAP
76  const capLeft = right + GAP
77  const capWidth = Math.min(columns - capLeft - 1, CAPTION_MAX)
78  if (leftWidth >= SIDE_MIN) {
79    const title = ` ${theme.title} `
80    const tx = Math.floor((leftWidth - title.length) / 2)
81    clearBox(tx - 1, 0, tx + title.length, 1)
82    write(tx, 0, title, state.speaker ? theme.frameLit : theme.frame)
83    const { meter } = theme
84    const label = `${meter.label} `
85    const mx = Math.floor((leftWidth - label.length - METER_GLYPHS) / 2)
86    const lit = Math.round(clamp01(state.level) * METER_GLYPHS)
87    clearBox(mx - 1, rows - 2, mx + label.length + METER_GLYPHS, rows - 1)
88    write(mx, rows - 1, label, theme.frame)
89    for (let i = 0; i < METER_GLYPHS; i++) write(mx + label.length + i, rows - 1, meter.glyph, i < lit ? meter.lit : meter.dim)
90  }
91  const by = state.speaker ?? state.captionBy
92  if (capWidth >= SIDE_MIN - 2 && state.caption) {
93    const lines = wrapRows(cellSafe(state.caption), capWidth, Math.max(1, rows - 4))
94    const height = lines.length + (by ? 2 : 0)
95    const top = Math.max(0, Math.floor((rows - height) / 2))
96    const widest = Math.max(...lines.map(line => line.length), by ? (theme.names[by] ?? by).length : 0)
97    clearBox(capLeft - 1, top - 1, capLeft + widest, top + height)
98    if (by) {
99      const ink = theme.ink[by] ?? theme.subtitle
100      write(capLeft, top, theme.names[by] ?? by.toUpperCase(), state.speaker ? ink : fade(ink, 0, 0.35))
101    }
102    lines.forEach((line, i) => write(capLeft, top + (by ? 2 : 0) + i, line, theme.subtitle))
103  }
104
105  // The rain: drops down every other column outside the face, bright heads, fading trails.
106  const glyphs = theme.glyphs?.length ? theme.glyphs : DEFAULT_GLYPHS
107  let field = rain.get(theme.title)
108  if (!field || field.columns !== columns) {
109    field = { columns, drops: [] }
110    rain.set(theme.title, field)
111  }
112  const pace = 1 + 2.2 * clamp01(state.level)
113  for (let x = 0; x < columns; x += 2) {
114    if (x >= left - 1 && x <= right) continue
115    let drop = field.drops[x]
116    if (!drop && Math.random() < 0.03 * pace) {
117      const length = 3 + Math.floor(Math.random() * Math.max(3, rows - 2))
118      drop = { y: 0, speed: 0.25 + Math.random() * 0.5, glyphs: Array.from({ length }, () => pick(glyphs)) }
119    }
120    if (!drop) continue
121    drop.y += drop.speed * pace
122    if (Math.random() < 0.2) drop.glyphs[Math.floor(Math.random() * drop.glyphs.length)] = pick(glyphs)
123    const head = Math.floor(drop.y)
124    drop.glyphs.forEach((glyph, k) => {
125      const y = head - k
126      const at = y * columns + x
127      if (y < 0 || y >= rows || clear.has(at) || text.has(at)) return
128      text.set(at, [glyph, k === 0 ? theme.frameLit : fade(theme.frame, theme.meter.dim, k / drop!.glyphs.length)])
129    })
130    field.drops[x] = head - drop.glyphs.length > rows ? undefined : drop
131  }
132
133  return packCells(columns, rows, pixels, text)
134}
135

1// A mode's band above the prompt: its host's portrait on the left, whoever is
2// talking to them on the right, and a panel between with the title, who speaks,
3// the subtitles and a level meter. Portraits are half-block pixels (two per
4// terminal cell); the whole band is one Raster's cells, rebuilt every tick and
5// blitted in place.
6
7/** One baked portrait size: `h` rows of `w` RGB pixels per frame, base64. */
8export type Portrait = { w: number; h: number; frames: Record<string, string> }
9
10/** How one mode's band looks: its cast's portraits, frame colours, title and meter. */
11export type CourtTheme = {
12  /** Per member, per band height in terminal rows. */
13  portraits: Record<string, Record<number, Portrait>>
14  /** Who is always in the left frame. */
15  host: string
16  names: Record<string, string>
17  ink: Record<string, number>
18  frame: number
19  /** The frame of whoever is talking. */
20  frameLit: number
21  corner: number
22  /** Centred in the panel's top rule. */
23  title: string
24  subtitle: number
25  /** The level meter centred in the bottom rule: `label` then `glyph`s lit by the voice. */
26  meter: { label: string; glyph: string; lit: number; dim: number }
27  /** `codec`: a radio call with a frequency panel. `solo`: the host alone, centred in glyph rain. */
28  layout?: 'codec' | 'solo'
29  /** Court layout: an image over the subtitles, per band height, that brightens with the voice. */
30  emblem?: Record<number, Portrait>
31  /** Solo layout: the glyphs the rain is made of. */
32  glyphs?: string[]
33  /** Codec layout: the frequency on the panel while each member is on the line. */
34  frequencies?: Record<string, string>
35}
36
37export type CourtState = {
38  /** Who is talking, or undefined between lines. */
39  speaker?: string
40  /** Who is in the right frame: the last member other than the host to speak. */
41  contact: string
42  /** Each frame's portrait this tick. */
43  faces: { host: string; contact: string }
44  /** 0..1, how much of the meter is lit. */
45  level: number
46  caption: string
47  /** Who said the caption (it stays after the call ends). */
48  captionBy?: string
49}
50
51const DEFAULT_COLOR = 0x01000000
52const HALF_BLOCK = 0x2580
53export const GAP = 3
54/** Characters a cell may hold beyond printable ASCII (each width 1). */
55const EXTRA_GLYPHS = new Set([...'…—–‘’“”'])
56
57export const clamp01 = (v: number) => (v < 0 ? 0 : v > 1 ? 1 : v)
58
59// ── base64 (decoded once per art; encoded every frame) ──
60
61const B64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
62const B64_INDEX = new Map([...B64].map((c, i) => [c, i]))
63
64const fromBase64 = (text: string) => {
65  const clean = text.replace(/=+$/, '')
66  const out = new Uint8Array(Math.floor((clean.length * 3) / 4))
67  let bits = 0
68  let value = 0
69  let o = 0
70  for (const c of clean) {
71    value = (value << 6) | (B64_INDEX.get(c) ?? 0)
72    bits += 6
73    if (bits >= 8) {
74      bits -= 8
75      out[o++] = (value >> bits) & 0xff
76    }
77  }
78  return out
79}
80
81const toBase64 = (bytes: Uint8Array) => {
82  let out = ''
83  for (let i = 0; i < bytes.length; i += 3) {
84    const a = bytes[i] ?? 0
85    const b = bytes[i + 1] ?? 0
86    const c = bytes[i + 2] ?? 0
87    const n = (a << 16) | (b << 8) | c
88    out += B64[(n >> 18) & 63]! + B64[(n >> 12) & 63]!
89    out += i + 1 < bytes.length ? B64[(n >> 6) & 63]! : '='
90    out += i + 2 < bytes.length ? B64[n & 63]! : '='
91  }
92  return out
93}
94
95
96export const fade = (from: number, to: number, t: number) => {
97  let out = 0
98  for (const shift of [16, 8, 0]) {
99    const a = (from >> shift) & 0xff
100    const b = (to >> shift) & 0xff
101    out |= Math.round(a + (b - a) * t) << shift
102  }
103  return out
104}
105
106/** Words of `text` in `count` rows of `width`, the last cut with an ellipsis. */
107export const wrapRows = (text: string, width: number, count: number) => {
108  const rows: string[] = []
109  let line = ''
110  for (const word of text.split(' ')) {
111    const next = line ? `${line} ${word}` : word
112    if (next.length <= width) {
113      line = next
114      continue
115    }
116    if (rows.length === count - 1) {
117      rows.push(next.slice(0, Math.max(0, width - 1)) + '…')
118      return rows
119    }
120    rows.push(line)
121    line = word.slice(0, width)
122  }
123  if (line) rows.push(line)
124  return rows.slice(0, count)
125}
126
127// Accented Latin letters (chérie, señora) are one cell wide too; NFC folds a combining accent into its letter.
128export const cellSafe = (text: string) =>
129  [...text.normalize('NFC')].map(c => (/[\x20-\x7e\u00a0-\u017f]/.test(c) || EXTRA_GLYPHS.has(c) ? c : '?')).join('')
130
131
132/** Pixels and text into cells: half blocks, the terminal's background where empty. */
133export const packCells = (
134  columns: number,
135  rows: number,
136  pixels: Array<number | undefined>,
137  text: Map<number, [string, number]>,
138) => {
139  const words = new Uint32Array(columns * rows * 3)
140  for (let y = 0; y < rows; y++) {
141    for (let cx = 0; cx < columns; cx++) {
142      const i = (y * columns + cx) * 3
143      const t = text.get(y * columns + cx)
144      const top = pixels[2 * y * columns + cx]
145      const bottom = pixels[(2 * y + 1) * columns + cx]
146      if (t) {
147        words[i] = t[0].codePointAt(0) ?? 0x20
148        words[i + 1] = t[1]
149        words[i + 2] = DEFAULT_COLOR
150      } else if (top === undefined && bottom === undefined) {
151        words[i] = 0x20
152        words[i + 1] = DEFAULT_COLOR
153        words[i + 2] = DEFAULT_COLOR
154      } else if (top === undefined) {
155        words[i] = 0x2584
156        words[i + 1] = bottom ?? DEFAULT_COLOR
157        words[i + 2] = DEFAULT_COLOR
158      } else {
159        words[i] = HALF_BLOCK
160        words[i + 1] = top
161        words[i + 2] = bottom ?? DEFAULT_COLOR
162      }
163    }
164  }
165  return toBase64(new Uint8Array(words.buffer))
166}
167
168export const COURT_ROWS = [20, 16, 12, 10, 8]
169const METER_GLYPHS = 10
170const COURT_PANEL = 46
171const COURT_SPARE = 4
172
173const courtDecoded = new Map<string, Uint8Array>()
174/** The emblem's top, in pixels: just under the title's rule. */
175const EMBLEM_TOP = 3
176const courtEmblemPixels = (theme: CourtTheme, rows: number, emblem: Portrait) => {
177  const key = `${theme.title}$emblem${rows}`
178  let rgb = courtDecoded.get(key)
179  if (!rgb) {
180    rgb = fromBase64(emblem.frames.neutral ?? '')
181    courtDecoded.set(key, rgb)
182  }
183  return rgb
184}
185export const courtPixels = (theme: CourtTheme, who: string, rows: number, frame: string) => {
186  const art = (theme.portraits[who] ?? theme.portraits[theme.host]!)[rows] as Portrait
187  // Blinks are a single frame: a half-blink shows it too.
188  const name = frame in art.frames ? frame : frame.startsWith('blink') ? 'blink' : 'neutral'
189  const key = `${theme.title}${who}${rows}${name}`
190  let rgb = courtDecoded.get(key)
191  if (!rgb) {
192    rgb = fromBase64(art.frames[name] ?? art.frames.neutral ?? '')
193    courtDecoded.set(key, rgb)
194  }
195  return { art, rgb }
196}
197
198/** Both frames must fit; the panel between them shows only when there is room. */
199const courtFits = (theme: CourtTheme, rows: number, columns: number) => {
200  const art = theme.portraits[theme.host]![rows]
201  return art !== undefined && 2 * (art.w + 3) - 1 <= columns
202}
203
204/**
205 * The panel's width between the two frames at this size, or 0 when it won't fit.
206 * Fixed: the band's width wobbles a few columns as the UI
207 * around the prompt changes, and a panel that followed it would flicker.
208 */
209export const courtPanel = (theme: CourtTheme, rows: number, columns: number) => {
210  const art = theme.portraits[theme.host]![rows]
211  return art && 2 * (art.w + 2) + 2 * GAP + COURT_PANEL <= columns ? COURT_PANEL : 0
212}
213
214const lastRows = new Map<string, number | undefined>()
215
216/**
217 * The band height for `key`, from `best(spare)`: the height that fits with `spare` columns held back.
218 * Keeps the size in use unless it stops fitting, and grows only with room to spare, so a width
219 * hovering at a threshold doesn't flip the band between sizes.
220 */
221export const steadyRows = (key: string, best: (spare: number) => number | undefined) => {
222  const last = lastRows.get(key)
223  const now = best(0)
224  const roomy = best(COURT_SPARE)
225  const keep = last !== undefined && now !== undefined && last <= now && (roomy ?? 0) <= last
226  const rows = keep ? last : (roomy ?? now)
227  lastRows.set(key, rows)
228  return rows
229}
230
231/** The tallest band that fits with its panel, else the tallest whose frames fit. */
232export const courtRowsFor = (theme: CourtTheme, maxRows: number, columns: number, preferRows: number) =>
233  steadyRows(theme.title, spare => {
234    const sizes = COURT_ROWS.filter(r => r <= Math.min(maxRows, preferRows) && courtFits(theme, r, columns - spare))
235    return sizes.find(r => courtPanel(theme, r, columns - spare) > 0) ?? sizes[0]
236  })
237
238/** A court band's cells for one frame, `rows` tall. */
239export const courtCells = (theme: CourtTheme, columns: number, rows: number, state: CourtState) => {
240  const H = rows * 2
241  const pixels: Array<number | undefined> = Array.from({ length: columns * H }, () => undefined)
242  const dot = (x: number, y: number, color: number) => {
243    if (x >= 0 && x < columns && y >= 0 && y < H) pixels[y * columns + x] = color
244  }
245  const text = new Map<number, [string, number]>()
246  const write = (x: number, y: number, s: string, fg: number) =>
247    [...s].forEach((c, i) => {
248      if (x + i >= 0 && x + i < columns && y >= 0 && y < rows) text.set(y * columns + x + i, [c, fg])
249    })
250
251  // A portrait in its frame, lit while talking. Never dimmed: painted portraits
252  // lose their detail darkened.
253  const frame = (who: string, x0: number, face: string) => {
254    const { art, rgb } = courtPixels(theme, who, rows, face)
255    for (let y = 0; y < art.h; y++) {
256      for (let x = 0; x < art.w; x++) {
257        const i = (y * art.w + x) * 3
258        dot(x0 + 1 + x, y, ((rgb[i] ?? 0) << 16) | ((rgb[i + 1] ?? 0) << 8) | (rgb[i + 2] ?? 0))
259      }
260    }
261    const border = state.speaker === who ? theme.frameLit : theme.frame
262    for (let y = 0; y < H; y++) {
263      dot(x0, y, border)
264      dot(x0 + art.w + 1, y, border)
265    }
266    for (let x = x0; x <= x0 + art.w + 1; x++) {
267      dot(x, 0, border)
268      dot(x, H - 1, border)
269    }
270    for (const [cx, cy] of [[x0, 0], [x0 + art.w + 1, 0], [x0, H - 1], [x0 + art.w + 1, H - 1]] as const) dot(cx, cy, theme.corner)
271  }
272  const frameWidth = theme.portraits[theme.host]![rows]!.w + 2
273  const panel = courtPanel(theme, rows, columns)
274  frame(theme.host, 0, state.faces.host)
275  const px0 = frameWidth + GAP
276  frame(state.contact, panel ? px0 + panel + GAP : frameWidth + 1, state.faces.contact)
277  if (!panel) return packCells(columns, rows, pixels, text)
278
279  // Rules top and bottom: the title above, the meter below; who speaks and what between.
280  const centred = (y: number, s: string, fg: number) => write(px0 + Math.floor((panel - [...s].length) / 2), y, s, fg)
281  const rule = '─'.repeat(panel)
282  write(px0, 0, rule, theme.frame)
283  centred(0, ` ${theme.title} `, theme.frameLit)
284  write(px0, rows - 1, rule, theme.frame)
285  // An empty glyph leaves only the label.
286  const { meter } = theme
287  const glyphs = meter.glyph ? METER_GLYPHS : 0
288  const lit = Math.round(clamp01(state.level) * glyphs)
289  const label = ` ${meter.label} `
290  const meterLeft = px0 + Math.floor((panel - label.length - glyphs - (glyphs ? 1 : 0)) / 2)
291  write(meterLeft, rows - 1, label, theme.frame)
292  for (let i = 0; i < glyphs; i++) write(meterLeft + label.length + i, rows - 1, meter.glyph, i < lit ? meter.lit : meter.dim)
293  if (glyphs) write(meterLeft + label.length + glyphs, rows - 1, ' ', theme.frame)
294
295  // The emblem under the title, flickering a little and brightening while someone speaks.
296  let first = 2
297  const emblem = theme.emblem?.[rows]
298  if (emblem) {
299    const rgb = courtEmblemPixels(theme, rows, emblem)
300    const k = 0.6 + 0.4 * clamp01(state.level) + (Math.random() - 0.5) * 0.08
301    const ex = px0 + Math.floor((panel - emblem.w) / 2)
302    for (let y = 0; y < emblem.h; y++) {
303      for (let x = 0; x < emblem.w; x++) {
304        const i = (y * emblem.w + x) * 3
305        const r = rgb[i] ?? 0
306        const g = rgb[i + 1] ?? 0
307        const b = rgb[i + 2] ?? 0
308        if (r + g + b < 24) continue
309        const c = (v: number) => Math.min(255, Math.round(v * k))
310        dot(ex + x, EMBLEM_TOP + y, (c(r) << 16) | (c(g) << 8) | c(b))
311      }
312    }
313    first = Math.ceil((EMBLEM_TOP + emblem.h) / 2)
314  }
315
316  const by = state.speaker ?? state.captionBy
317  if (state.caption) {
318    const space = rows - 1 - first
319    const head = by ? (space >= 5 ? 2 : 1) : 0
320    const lines = wrapRows(cellSafe(state.caption), panel - 4, Math.max(1, space - head))
321    const top = first + Math.max(0, Math.floor((space - lines.length - head) / 2))
322    const ink = by ? (theme.ink[by] ?? theme.subtitle) : theme.subtitle
323    if (by) centred(top, theme.names[by] ?? by.toUpperCase(), state.speaker ? ink : fade(ink, 0, 0.35))
324    lines.forEach((line, i) => centred(top + head + i, line, theme.subtitle))
325  }
326  return packCells(columns, rows, pixels, text)
327}
328

1// Modes are folders of data: `mode.json` (who speaks, how, and how it sounds),
2// and for a cast mode `portraits.json` (baked by bin/bake-portraits.py) and an
3// optional ring sound. The built-in modes ship in the plugin's `modes/`, mode
4// packs (other plugins) in their `avatars/modes/`, and the person's own in
5// `~/.claude/avatars/modes/`; later ones win on a name clash.
6// MODES.md documents the format.
7
8import type { CourtTheme, Portrait } from './court'
9import type { Effect } from './effects'
10
11type Weighted = { weight: number; text: string }
12
13/** One speaking member of a cast mode, as `mode.json` holds them. */
14export type CastMember = {
15  /** How Sonnet writes them before a line, in capitals. */
16  name: string
17  /** Other names Sonnet might write for them ("queen", "boss"). */
18  aliases?: string[]
19  /** Their ElevenLabs voice id. */
20  voice: string
21  /** A public library voice's owner id, so it can be added to an account that needs that first. */
22  libraryOwner?: string
23  /** Shifts their voice's pitch, keeping its speed: 0.95 is 5% lower. */
24  pitch?: number
25  /** Their name and subtitle colour in the band, `#rrggbb`. */
26  ink?: string
27  persona: string
28  /** What brings them into a call. */
29  speaksAbout: string
30  /** Audio tags that suit them, for v3/v4 voice models. */
31  tags?: string
32  /** Portrait frames shown while someone else talks. */
33  idle?: Array<{ frame: string; weight: number; ms: [number, number] }>
34  /** A frame that punctuates talking when a line's tags match `when` (a regex, case-insensitive). */
35  accent?: { frame: string; when: string }
36  /** Codec layout: the frequency the panel shows while they are on the line. */
37  frequency?: string
38}
39
40export type ModeFile = {
41  title: string
42  description: string
43  /** A single-voice mode's default voice; the person may pick another. */
44  voice?: string
45  /** The default voice's library owner id, as for a cast member. */
46  voiceOwner?: string
47  /** A single-voice mode's persona, added to Sonnet's instructions. */
48  persona?: string
49  /** Extra persona guidance used only when the voice model performs tags. */
50  personaTags?: string
51  /** A cast mode: Sonnet writes `NAME:` lines, each spoken in its member's voice. */
52  call?: {
53    /** What the call is, who did the work, who is always present. */
54    premise: string
55    /** Always in the band's left frame. */
56    host: string
57    /** Who says a reply that came back without any `NAME:` lines. */
58    fallback: string
59    cast: Record<string, CastMember>
60    /** Extra cast notes (a pet that never speaks). */
61    extras?: string[]
62    /** A paragraph on tone. */
63    style?: string
64    /** Who trades lines with whom. */
65    turns: string
66    /** What must survive the delivery, beyond the outcome and any question. */
67    accuracy?: string
68    /** How the voices differ, completing "Keep each voice distinct: ". */
69    distinct?: string
70    /** Picked per call: who reports. */
71    reporters?: Weighted[]
72    /** Picked per call: what kind of call it is, with an optional note on who joins. */
73    kinds: Array<Weighted & { note?: string }>
74  }
75  /** The band above the prompt (cast modes with portraits). Colours are `#rrggbb`. */
76  band?: {
77    title: string
78    frame: string
79    frameLit: string
80    corner: string
81    subtitle: string
82    meter: { label: string; glyph: string; lit: string; dim: string }
83    /** `codec`: a radio call with a frequency panel. `solo`: the host alone, centred in glyph rain. */
84    layout?: 'codec' | 'solo'
85    /** Solo layout: the characters the rain falls in, each one cell wide. */
86    glyphs?: string
87    /** Codec layout: the frequency shown for a member without their own. */
88    frequency?: string
89  }
90  audio?: {
91    /** An ffmpeg -filter_complex graph over mono 22050 Hz audio. */
92    filter?: string
93    /** Post effects on the dry voice, planned fresh for every line, ahead of `filter`. */
94    effects?: Effect[]
95    /** Raw s16le mono 22050 Hz, played before a call's first line, and its length. */
96    ring?: { file: string; ms: number }
97  }
98  /** Played when the mode is switched on: lines (`NAME: text` for a cast mode). */
99  sample: string[]
100  /** Played by `/avatar test`. */
101  demos: string[][]
102  /** Longer scripted calls for `/avatar scenario`. */
103  scenarios?: Record<string, { title: string; script: string[] }>
104}
105
106export type Mode = ModeFile & {
107  name: string
108  /** The folder it was read from. */
109  dir: string
110  /** The band's look, when the mode has portraits and a band. */
111  theme?: CourtTheme
112}
113
114const color = (hex: string | undefined, fallback: number) => {
115  const n = hex ? Number.parseInt(hex.replace(/^#/, ''), 16) : Number.NaN
116  return Number.isFinite(n) ? n : fallback
117}
118
119/** What is wrong with a mode file, if anything, for the person to fix. */
120const problemsOf = (file: ModeFile, portraits: Record<string, Record<number, Portrait>> | undefined) => {
121  const problems: string[] = []
122  if (typeof file.title !== 'string') problems.push('no title')
123  if (!Array.isArray(file.sample) || !Array.isArray(file.demos)) problems.push('sample and demos must be arrays')
124  const call = file.call
125  if (call) {
126    const members = Object.keys(call.cast ?? {})
127    if (!members.length) problems.push('call.cast is empty')
128    if (!members.includes(call.host)) problems.push(`call.host ${call.host} is not in the cast`)
129    if (!members.includes(call.fallback)) problems.push(`call.fallback ${call.fallback} is not in the cast`)
130    if (!call.kinds?.length) problems.push('call.kinds is empty')
131    for (const [who, m] of Object.entries(call.cast ?? {})) {
132      if (!m.voice || !m.name) problems.push(`${who} needs a name and a voice`)
133    }
134    if (file.band && !portraits) problems.push('band needs portraits.json')
135    if (file.band && portraits && !portraits[call.host]) problems.push(`portraits.json has no ${call.host}`)
136    if (file.band?.layout !== undefined && file.band.layout !== 'codec' && file.band.layout !== 'solo') {
137      problems.push(`band.layout ${file.band.layout} is not a layout (codec, solo, or leave it out)`)
138    }
139  } else if (file.band) {
140    problems.push('band needs a call (a cast)')
141  }
142  return problems
143}
144
145const themeOf = (file: ModeFile, portraits: Record<string, Record<number, Portrait>>): CourtTheme | undefined => {
146  const { call, band } = file
147  if (!call || !band) return undefined
148  const cast = Object.entries(call.cast)
149  return {
150    portraits,
151    host: call.host,
152    names: Object.fromEntries(cast.map(([who, m]) => [who, m.name])),
153    ink: Object.fromEntries(cast.map(([who, m]) => [who, color(m.ink, 0xffffff)])),
154    frame: color(band.frame, 0x444444),
155    frameLit: color(band.frameLit, 0xaaaaaa),
156    corner: color(band.corner, 0xcccccc),
157    title: band.title,
158    subtitle: color(band.subtitle, 0xffffff),
159    meter: {
160      label: band.meter.label,
161      glyph: band.meter.glyph,
162      lit: color(band.meter.lit, 0xffffff),
163      dim: color(band.meter.dim, 0x333333),
164    },
165    layout: band.layout,
166    emblem: portraits.$emblem,
167    glyphs: band.glyphs ? [...band.glyphs] : undefined,
168    frequencies: Object.fromEntries(
169      cast.flatMap(([who, m]) => {
170        const frequency = m.frequency ?? band.frequency
171        return frequency ? [[who, frequency] as const] : []
172      }),
173    ),
174  }
175}
176
177/** A mode from its files' text, or what is wrong with it. */
178export const buildMode = (name: string, dir: string, modeText: string, portraitsText: string | undefined): Mode | string => {
179  let file: ModeFile
180  try {
181    file = JSON.parse(modeText) as ModeFile
182  } catch (err) {
183    return `${name}: mode.json does not parse (${String(err)})`
184  }
185  let portraits: Record<string, Record<number, Portrait>> | undefined
186  if (portraitsText !== undefined) {
187    try {
188      portraits = JSON.parse(portraitsText)
189    } catch (err) {
190      return `${name}: portraits.json does not parse (${String(err)})`
191    }
192  }
193  const problems = problemsOf(file, portraits)
194  if (problems.length) return `${name}: ${problems.join('; ')}`
195  return { ...file, name, dir, theme: portraits ? themeOf(file, portraits) : undefined }
196}
197
198/** A folder name that can be a mode's name. */
199export const isModeName = (name: string) => /^[a-z0-9][a-z0-9_-]*$/.test(name)
200
201const pickWeighted = <T extends { weight: number }>(list: readonly T[]) => {
202  let r = Math.random() * list.reduce((n, item) => n + item.weight, 0)
203  return list.find(item => (r -= item.weight) <= 0) ?? list[0]!
204}
205
206/** The call format and every member's persona, for Sonnet's system prompt. */
207export const callPersona = (mode: Mode, hasTags: boolean) => {
208  const call = mode.call!
209  const cast = Object.values(call.cast)
210    .map(m => `- ${m.name}: ${m.persona}\n  Speaks about: ${m.speaksAbout}${hasTags && m.tags ? `\n  Tags that suit them: ${m.tags}` : ''}`)
211    .concat((call.extras ?? []).map(extra => `- ${extra}`))
212    .join('\n')
213  const rules = [
214    `One speaker per line, each line on its own line starting "NAME: ". ${call.turns}`,
215    `The call is delivery, not content: the outcome, the numbers that matter, and any question or decision for the user must come through accurately.${call.accuracy ? ` ${call.accuracy}` : ''}`,
216    'Match the length the <call> block asks for.',
217    ...(call.distinct ? [`Keep each voice distinct: ${call.distinct}`] : []),
218    ...(hasTags ? ['Start each line with one tag that fits the speaker and the moment.'] : []),
219    'No narration, no sound effects, nothing outside the lines.',
220  ]
221  return `
222Format: this replaces the single-speaker rules above. ${call.premise}
223
224The cast (each line starts with the speaker's name in capitals, then a colon):
225${cast}
226${call.style ? `\n${call.style}\n` : ''}
227Rules:
228${rules.map(rule => `- ${rule}`).join('\n')}`
229}
230
231/** The <call> block for one reply: who reports, what kind of call, how long. */
232export const planCall = (mode: Mode, replyChars: number) => {
233  const call = mode.call!
234  const kind = pickWeighted(call.kinds)
235  const lines =
236    replyChars < 500 ? '2 to 3 lines, under 50 words' :
237    replyChars < 1500 ? '3 to 5 lines, under 100 words' :
238    replyChars < 4000 ? '4 to 7 lines, under 160 words' :
239    '6 to 10 lines, under 230 words'
240  return [
241    '<call>',
242    ...(call.reporters?.length ? [`Reporting: ${pickWeighted(call.reporters).text}`] : []),
243    `Kind of call: ${kind.text}`,
244    ...(kind.note ? [kind.note] : []),
245    `Length: ${lines}.`,
246    '</call>',
247  ].join('\n')
248}
249
250/** Who a `NAME` Sonnet wrote means, by key, name or alias. */
251export const speakerOf = (mode: Mode, written: string) => {
252  const said = written.trim().toLowerCase()
253  for (const [who, m] of Object.entries(mode.call?.cast ?? {})) {
254    const names = [who, m.name, ...(m.aliases ?? [])].map(n => n.toLowerCase())
255    if (names.includes(said)) return who
256  }
257  return undefined
258}
259
260/** `NAME: ...` lines for the mode's cast; a line without a known name continues the last. */
261export const parseCall = (mode: Mode, text: string) => {
262  const lines: Array<{ speaker: string; text: string }> = []
263  for (const raw of text.split('\n')) {
264    const m = raw.match(/^\s*\**\s*([A-Za-z][A-Za-z .'-]{0,24}?)\s*\**\s*:\s*(.+)$/)
265    const speaker = m ? speakerOf(mode, m[1] ?? '') : undefined
266    if (m && speaker) lines.push({ speaker, text: (m[2] ?? '').trim() })
267    else if (raw.trim() && lines.length) lines[lines.length - 1]!.text += ` ${raw.trim()}`
268  }
269  return lines.length ? lines : [{ speaker: mode.call!.fallback, text: text.trim() }]
270}
271
272/** Whether a demo has this speaker in it, by key, name or alias. */
273export const demoHas = (mode: Mode, demo: string[], who: string) => {
274  const wanted = speakerOf(mode, who)
275  return wanted !== undefined && demo.some(line => speakerOf(mode, line.match(/^\s*([^:]{1,24}):/)?.[1] ?? '') === wanted)
276}
277

1/** A per-session setting; null falls back to the plugin's configured default. */
2export type Setting = string | null
3
4/** A voice the menu offers. */
5export type VoiceChoice = { id: string; name: string }
6
7declare module 'claude-code' {
8  interface PluginState {
9    avatars: {
10      /** The mode whose band shows above the prompt, or false for none. */
11      bandMode: string | false
12      enabled: Setting
13      mode: Setting
14      voice: Setting
15      model: Setting
16      last: Setting
17      /** The person's ElevenLabs voices, once the menu has fetched them. */
18      voices: VoiceChoice[] | null
19      /** A line the menu shows under its controls. */
20      note: string
21    }
22  }
23}
24