avatar7

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface as Engine, Register } from 'claude-code'
3
4import type { Announce, Line, ModelUse, Say, Station } from '../types'
5import { ambientCells, ambientPixel, QUAD, type AmbientLayer, type Field } from './ambient'
6import {
7  aliveArgv,
8  CHECK_FRAMES,
9  claimArgv,
10  DRAIN_FRAMES,
11  drainArgv,
12  follows,
13  forgetArgv,
14  givesOnEnd,
15  heldArgv,
16  latestArgv,
17  MIRROR_FRAMES,
18  mirrorArgv,
19  newRelay,
20  parseRemote,
21  playArgv,
22  promptedArgv,
23  releaseArgv,
24  startArgv,
25  takeArgv,
26  type Mirror,
27  type Relay,
28  type RelayHost,
29  wantedArgv,
30  wantedMove,
31} from './relay'
32import {
33  contextBody,
34  EPISODE_TTL_S,
35  episodeKey,
36  episodeOf,
37  journalFrom,
38  journalKey,
39  journalPrompt,
40  memoryAt,
41  memoryNote,
42  parseContext,
43  parseRecall,
44  queryFor,
45  recallBody,
46  recalls,
47  RECALL_EPISODES,
48  RECALL_JOURNALS,
49  RECALL_OLD_JOURNALS,
50  journalsFor,
51  rpcArgv,
52  storeBody,
53  unsummed,
54  visitOf,
55  CONSOLIDATE_MAX,
56  type MemoryAt,
57} from './memory'
58import {
59  CHAT_LINES,
60  CONSULT_CHARS_ASKED,
61  DUO_TURNS,
62  EVENT_MIN_FRAMES,
63  EVENT_QUIET_FRAMES,
64  EVENT_SPAN_FRAMES,
65  enqueue,
66  fresh,
67  isOpinion,
68  lineFrom,
69  personalize,
70  pickEvent,
71  pickGuest,
72  promptFor,
73  QUESTION_FRAMES,
74  reads,
75  RECENT_LINES,
76  STYLE,
77  type Ask,
78  type Duo,
79  type Persona,
80  type Queued,
81  type Story,
82} from './speech'
83import { ASKED_CHARS, commandEvent, heard, heardSay, landed, nextStreak, streakNote, type Streak } from './hearing'
84import { detachedArgv, PLAY_START_MS, SAPI_PLAY, synthArgv } from './voice'
85import { faceCells, H, noise, TINT, W, type Faces, type View } from './draw'
86import { DEFAULT_GRAIN, GLITCH_STEPS, GRAINS, hdFrame, hdKey, hdSize, hdStamp, isSettled, pickHdFace, type Hd, type HdView, type Settle } from './hd'
87import { begin, end, isHeard, restored, silent, start, typeOn, voiced, type Typing } from './line'
88import { answered, ask as askFace, calm, hold, isWaiting, react, release, stage as stageFace, tick, type Face, type Mood } from './mood'
89
90const PANE = 'avatar7'
91const FACE = 'face'
92const FRAME_MS = 66
93// The weather around the face moves at a third of the face's pace.
94const AMBIENT_FRAMES = 3
95// The HD weather: a loop of HD_LOOP pictures, one per ambient step.
96const HD_LOOP = 12
97const HD_STEP_S = (AMBIENT_FRAMES * FRAME_MS) / 1000
98const AMB_LEFT = 'amb-left'
99const AMB_RIGHT = 'amb-right'
100const AMB_BAND = 'amb-band'
101// The scene rises behind the face to its middle and runs this many rows past
102// the text.
103const SCENE_OVERFLOW_ROWS = 6
104const DEFAULT = 'shodan'
105const AVATARS = ['shodan', 'hal', 'glados', 'ada', 'duck7', 'pod042', 'kaneda', 'commis', 'fox', 'adjutant', 'morte', 'pda', 'lain', 'tachikoma', 'nova']
106
107const MIN_SIZE = 16
108// A refusal acted in a scene shakes the portrait this much of a real one.
109const SCENE_GLITCH = 0.4
110
111// Rows kept under the face for the line, which may wrap once, the pending
112// approval, the buttons and the two rules between them.
113const TEXT_ROWS = 6
114
115// The engine redraws on a change of width, never on a change of height alone:
116// the clock asks for a render this often so the face follows both.
117const REFIT_FRAMES = 15
118
119// The face's side in pixels for a pane body: as wide as the body, as tall as
120// the body leaves (two pixels per row, a few rows kept for the text), even,
121// never past the baked portrait.
122const fit = (columns: number, rows: number): number => {
123  const side = Math.min(W, columns, Math.max(MIN_SIZE / 2, rows - TEXT_ROWS) * 2)
124  return Math.max(MIN_SIZE, side - (side % 2))
125}
126
127const line = atom({ plugin: 'avatar7', key: 'line' } as const, { text: '', at: 0 })
128const isMuted = atom({ plugin: 'avatar7', key: 'isMuted' } as const, false)
129// SAPI volume, 0 to 100, kept across sessions in $.store.
130const volume = atom({ plugin: 'avatar7', key: 'volume' } as const, 100)
131const VOLUME_STEP = 10
132// The avatar on duty, by its id: other mods read it (jukebox7 picks its music).
133const onDuty = atom({ plugin: 'avatar7', key: 'avatar' } as const, '')
134// The on-duty persona's color, read by jukebox7 to light its pane alike.
135const tint = atom({ plugin: 'avatar7', key: 'color' } as const, '')
136// The on-duty persona's station, read by jukebox7 for its avatar's pick.
137const station = atom({ plugin: 'avatar7', key: 'station' } as const, { name: '', artists: [] } as Station)
138// The mods that asked for a voice, by plugin name: each publishes its own
139// `announce` key, and the avatar hears the write.
140// True while a line is heard: jukebox7 lowers its music meanwhile.
141const isVoicing = atom({ plugin: 'avatar7', key: 'isVoicing' } as const, false)
142const announcers = atom({ plugin: 'avatar7', key: 'announcers' } as const, {} as Record<string, Announce>)
143
144// An ask the mode may settle alone (auto mode): the face waits at once, the
145// avatar speaks only if the permission prompt is still up after ~2 s.
146const ASK_FRAMES = 30
147// The longest a wait keeps the face: 15 minutes.
148const WAIT_CAP_FRAMES = Math.round((15 * 60_000) / FRAME_MS)
149
150// How long a line about a call through mesh7 waits for mesh7-pane's verdict:
151// its poll runs every 1.5 s.
152const SAY_WAIT_FRAMES = 27
153
154
155// The remote voice's engine calls (relay.ts holds the rest): declared here,
156// in this file, as the engine follows $ into nothing imported.
157async function relayOpen($: Engine, r: Relay): Promise<void> {
158  r.session = await $.session.id()
159  r.isSsh = (await $.process.run(['sh', '-c', 'printf %s "$SSH_CONNECTION"'])).stdout.trim() !== ''
160}
161
162async function relayGive($: Engine, r: Relay): Promise<void> {
163  await $.process.run(releaseArgv(r.session))
164  r.isHeld = false
165  r.isByWanted = false
166}
167
168// Each frame: who holds the relay, now and then; the face, when held and
169// changed; the page's presses.
170function relayTick($: Engine, r: Relay, frame: number, host: RelayHost): void {
171  if (r.session === '') return
172  if (frame % CHECK_FRAMES === 0) {
173    void (async () => {
174      // After a /clear the process goes on under a new id, with no
175      // session.start: a relay held by force follows it there.
176      const id = await $.session.id()
177      if (id !== r.session) {
178        r.session = id
179        if (r.isForced) await $.process.run(takeArgv(r.session))
180      }
181      r.isHeld = (await $.process.run(heldArgv(r.session))).stdout.trim() === 'up'
182      const isWanted = (await $.process.run(wantedArgv())).stdout.trim() === 'up'
183      const latest = isWanted && !r.isHeld ? (await $.process.run(latestArgv())).stdout.trim() : ''
184      const move = wantedMove(r, isWanted, latest)
185      if (move === 'claim') {
186        await $.process.run(claimArgv(r.session))
187        r.isHeld = (await $.process.run(heldArgv(r.session))).stdout.trim() === 'up'
188        if (r.isHeld) r.isByWanted = true
189      } else if (move === 'give') {
190        await relayGive($, r)
191      }
192      if (!r.isHeld) r.mirrored = ''
193    })().catch(err => $.ui.log(`avatar7: the relay check failed: ${String(err)}`, { to: 'debug' }))
194  }
195  if (r.isHeld && !r.isMirroring && frame % MIRROR_FRAMES === 0) {
196    const face = host.face()
197    if (face !== undefined) {
198      r.isMirroring = true
199      void (async () => {
200        const json = JSON.stringify(await face)
201        if (json !== r.mirrored) await $.process.run(mirrorArgv(r.session), { stdin: json })
202        r.mirrored = json
203      })()
204        .catch(err => $.ui.log(`avatar7: the relay's mirror failed: ${String(err)}`, { to: 'debug' }))
205        .finally(() => {
206          r.isMirroring = false
207        })
208    }
209  }
210  if (r.isHeld && !r.isDraining && frame % DRAIN_FRAMES === 0) {
211    r.isDraining = true
212    void (async () => {
213      const out = await $.process.run(drainArgv())
214      for (const each of out.stdout.split('\n')) {
215        const press = parseRemote(each)
216        if (press !== undefined) await host.press(press)
217      }
218    })()
219      .catch(err => $.ui.log(`avatar7: the page's commands failed: ${String(err)}`, { to: 'debug' }))
220      .finally(() => {
221        r.isDraining = false
222      })
223  }
224}
225
226// /avatar remote on: start the relay when no service runs it, and hold it.
227async function relayOn($: Engine, r: Relay): Promise<string> {
228  if ((await $.process.run(aliveArgv())).stdout.trim() !== 'up') {
229    await $.process.run(startArgv($.plugin.root))
230    await $.clock.sleep(1500)
231  }
232  r.isForced = true
233  r.isByWanted = false
234  await $.process.run(takeArgv(r.session))
235  const ip = await $.process.run(['sh', '-c', 'tailscale ip -4 | head -1'])
236  return `The voice leaves this machine: open http://${ip.stdout.trim()}:8797/ on the other one and click listen.`
237}
238
239// /avatar remote off: give the relay back; the relay itself keeps running.
240async function relayOff($: Engine, r: Relay): Promise<string> {
241  r.isForced = false
242  await relayGive($, r)
243  return 'The voice comes back to this machine.'
244}
245
246async function relayFollow($: Engine, r: Relay, origin: string): Promise<void> {
247  if (r.session !== '') await $.process.run(promptedArgv(r.session))
248  const isWanted = r.session !== '' && (await $.process.run(wantedArgv())).stdout.trim() === 'up'
249  const move = follows(r, origin, isWanted)
250  if (move === 'take') {
251    await $.process.run(takeArgv(r.session))
252    // Typed here and taken only because the phone listens: it goes back when
253    // the phone stops. Remote Control or ssh keep it as before.
254    r.isByWanted = origin === 'composer' && !r.isSsh
255  } else if (move === 'give') await relayGive($, r)
256  // Written whole again at the next mirror: a release left `{}` behind.
257  if (r.session !== '') r.mirrored = ''
258}
259
260async function relayEnd($: Engine, r: Relay, reason: string): Promise<void> {
261  if (r.session !== '') await $.process.run(forgetArgv(r.session))
262  if (givesOnEnd(r, reason)) await relayGive($, r)
263}
264
265// A persona's faces (draw.ts): the portrait, and the optional frames baked
266// beside it (tools/bake.py --frame): mouth open for speaking, a frown for a
267// refusal.
268// Whether this terminal draws pictures for the Image element: kitty and
269// Ghostty implement the Unicode placeholders it needs, WezTerm and Windows
270// Terminal do not. AVATAR7_HD=0 keeps the half blocks anyway. Set at
271// session start, before the faces load.
272let isHdTerminal = false
273export const drawsPictures = (env: string): boolean => {
274  const [term = '', program = '', flag = ''] = env.split('|')
275  if (flag === '0') return false
276  return term === 'xterm-kitty' || term === 'xterm-ghostty' || program === 'ghostty'
277}
278
279// The faces and scene tools/bake_hd.py baked for this persona, null when
280// they are not there (a fresh clone: they are derived, kept out of git).
281async function loadHd($: Engine, dir: string): Promise<Hd | null> {
282  const bytes = async (name: string): Promise<Uint8Array | null> => {
283    try {
284      return Uint8Array.fromBase64((await $.fs.read(`${dir}/${name}`, { as: 'bytes' })).base64)
285    } catch {
286      return null
287    }
288  }
289  let meta: { size?: number; sceneWidth?: number; sceneHeight?: number }
290  try {
291    meta = JSON.parse(String(await $.fs.read(`${dir}/hd.json`))) as typeof meta
292  } catch {
293    return null
294  }
295  const side = meta.size ?? 0
296  const base = await bytes('face-hd.rgb')
297  if (base === null || side === 0 || base.length !== side * side * 3) return null
298  const pixels = await bytes('scene-hd.rgb')
299  const width = meta.sceneWidth ?? 0
300  const height = meta.sceneHeight ?? 0
301  const scene = pixels !== null && pixels.length === width * height * 3 ? { pixels, width, height } : null
302  const talk = await bytes('face-talk-hd.rgb')
303  const deny = await bytes('face-deny-hd.rgb')
304  return { side, base, talk, deny, scene, stamp: hdStamp(base, talk, deny, pixels) }
305}
306
307// Where the HD pictures go, one file per picture key: kitty reads them
308// itself, so a change of picture sends a path, not a megabyte through the
309// terminal, and the face no longer blinks out while one arrives. Memory, on
310// Linux; shared by sessions, a key always meaning the same pixels.
311const HD_DIR = '/dev/shm/avatar7-hd'
312const hdPath = (key: string): string => `${HD_DIR}/${key.replace(/[^A-Za-z0-9.-]/g, '_')}.rgba`
313
314// $.fs.write takes text: the pixels go as base64, decoded beside.
315async function writeHd($: Engine, key: string, rgba: Uint8Array): Promise<void> {
316  const path = hdPath(key)
317  await $.fs.write(`${path}.b64`, rgba.toBase64())
318  const done = await $.process.run(['sh', '-c', 'base64 -d "$1.b64" > "$1.part" && mv "$1.part" "$1" && rm -f "$1.b64"', 'sh', path])
319  if (done.exitCode !== 0) throw new Error(`${path}: ${done.stderr}`)
320}
321
322// The pictures already in HD_DIR, written by an earlier load or another
323// session: ready, by path, so a reload makes none of them again.
324async function hdOnDisk($: Engine, files: Map<string, 'pending' | 'ready'>): Promise<void> {
325  const ls = await $.process.run(['sh', '-c', 'ls "$1" 2>/dev/null; true', 'sh', HD_DIR])
326  for (const name of ls.stdout.split('\n')) if (name.endsWith('.rgba')) files.set(`${HD_DIR}/${name}`, 'ready')
327}
328
329// Starts writing a picture's file unless it is there or on its way; `files`,
330// by path, marks it pending, then ready. `rgba`, when the render just made the
331// picture, spares making it twice.
332function hdEnsure($: Engine, files: Map<string, 'pending' | 'ready'>, hv: HdView, rgba?: Uint8Array): void {
333  const key = hdKey(hv)
334  const path = hdPath(key)
335  if (files.has(path)) return
336  files.set(path, 'pending')
337  void writeHd($, key, rgba ?? hdFrame(hv)).then(
338    () => files.set(path, 'ready'),
339    err => {
340      files.delete(path)
341      $.ui.log(`avatar7: HD picture not written: ${String(err)}`)
342    },
343  )
344}
345
346async function loadFaces($: Engine, dir: string): Promise<Faces> {
347  const read = async (name: string): Promise<Uint8Array | null> => {
348    try {
349      return Uint8Array.fromBase64((await $.fs.read(`${dir}/${name}`, { as: 'bytes' })).base64)
350    } catch {
351      return null
352    }
353  }
354  const base = await read('face.rgb')
355  if (base === null) throw new Error(`${dir}/face.rgb unreadable`)
356  return { base, talk: await read('face-talk.rgb'), deny: await read('face-deny.rgb'), hd: isHdTerminal ? await loadHd($, dir) : null }
357}
358
359
360// A scene layer's backdrop, read from the persona's folder beside its faces.
361async function loadScenes($: Engine, dir: string, persona: Persona): Promise<Persona> {
362  for (const layer of persona.ambient ?? []) {
363    if (layer.kind !== 'scene' || layer.file === undefined) continue
364    try {
365      layer.pixels = Uint8Array.fromBase64((await $.fs.read(`${dir}/${layer.file}`, { as: 'bytes' })).base64)
366    } catch {
367      $.ui.log(`avatar7: ${dir}/${layer.file} unreadable, run tools/bake_scene.py`)
368    }
369  }
370  return persona
371}
372
373// A private complement, kept out of the repository: ~/.config/avatar7/personas/<id>.json
374// adds scenes (events), extends the character (persona, asks), may rename the
375// user (nobody). Read with cat, as the engine's reads stay in the plugin folder.
376export type Private = { persona?: string; events?: Story[]; asks?: string; nobody?: string }
377export const withPrivate = (pub: Persona, priv: Private | null): Persona => {
378  if (priv === null) return pub
379  return {
380    ...pub,
381    persona: priv.persona ? `${pub.persona} ${priv.persona}` : pub.persona,
382    events: [...(pub.events ?? []), ...(priv.events ?? []).filter(e => typeof e.story === 'string')],
383    asks: priv.asks ? (pub.asks ? `${pub.asks}; ${priv.asks}` : priv.asks) : pub.asks,
384    nobody: priv.nobody ?? pub.nobody,
385  }
386}
387const privateArgv = (id: string): string[] => ['sh', '-c', 'cat "$HOME/.config/avatar7/personas/$1.json" 2>/dev/null; true', 'avatar7-private', id]
388
389// The persona as written in the repository, completed by the private file if any.
390async function readPersona($: Engine, id: string): Promise<Persona> {
391  const dir = `${$.plugin.root}/personas/${id}`
392  const pub = JSON.parse(String(await $.fs.read(`${dir}/persona.json`))) as Persona
393  let priv: Private | null = null
394  const out = (await $.process.run(privateArgv(id))).stdout.trim()
395  if (out !== '') {
396    try {
397      priv = JSON.parse(out) as Private
398    } catch {
399      $.ui.log(`avatar7: ~/.config/avatar7/personas/${id}.json is not valid JSON, ignored`)
400    }
401  }
402  return loadScenes($, dir, withPrivate(pub, priv))
403}
404
405type Guest = { id: string; persona: Persona; faces: Faces }
406
407// What speaking changes, in one object so speak() (a file-level function, as
408// the engine requires for $) can: the line under the face, the queue, a
409// visiting persona and whether its face shows, the question put to the user,
410// each persona's chat with the user, and the avatar's last lines.
411type Stage = {
412  typing: Typing
413  queue: Queued[]
414  guest: Guest | null
415  isGuestShown: boolean
416  openQuestion: { text: string; at: number } | null
417  chats: Map<string, string[]>
418  recent: string[]
419}
420
421async function loadGuest($: Engine, id: string): Promise<Guest | null> {
422  try {
423    const dir = `${$.plugin.root}/personas/${id}`
424    const persona = await readPersona($, id)
425    return { id, persona, faces: await loadFaces($, dir) }
426  } catch {
427    return null
428  }
429}
430
431
432
433// One JSON-RPC call to the personas' mem7; '' when it does not answer.
434async function mem7($: Engine, at: MemoryAt, body: string): Promise<string> {
435  try {
436    return (await $.process.run(rpcArgv(at), { stdin: body, timeoutMs: 6_000 })).stdout
437  } catch {
438    return ''
439  }
440}
441
442// What the persona remembers for a line: its last journals, and the
443// exchanges that match what was said.
444async function remembered($: Engine, at: MemoryAt, agent: string, query: string): Promise<string> {
445  const latest = parseRecall(await mem7($, at, recallBody(agent, ['journal'], RECALL_JOURNALS)))
446  if (query.trim() === '') return memoryNote(latest, [])
447  const matched = parseContext(await mem7($, at, contextBody(agent, query, ['journal'], RECALL_JOURNALS + RECALL_OLD_JOURNALS)))
448  const episodes = parseContext(await mem7($, at, contextBody(agent, query, ['episode'], RECALL_EPISODES)))
449  return memoryNote(journalsFor(latest, matched), episodes)
450}
451
452async function keep($: Engine, at: MemoryAt, agent: string, value: string, tags: string[]): Promise<void> {
453  await mem7($, at, storeBody(agent, episodeKey(agent, new Date()), value, ['episode', ...tags], EPISODE_TTL_S))
454}
455
456// Each model call's usage, published for usage-bell, which tallies the mods'
457// share of the plan.
458async function used($: Engine, model: string, r: { usage?: { input_tokens: number; output_tokens: number; cache_read_input_tokens?: number } }): Promise<void> {
459  if (r.usage === undefined) return
460  await $.state.set({ plugin: 'avatar7', key: 'modelUse' }, {
461    model, input: r.usage.input_tokens, output: r.usage.output_tokens, cacheRead: r.usage.cache_read_input_tokens ?? 0, at: Date.now(),
462  } satisfies ModelUse)
463}
464
465// The persona sums up, in its own voice, the exchanges since its last journal.
466async function summarize($: Engine, at: MemoryAt, agent: string, voice: Persona, userName: string): Promise<void> {
467  const journals = parseRecall(await mem7($, at, recallBody(agent, ['journal'], 1)))
468  const episodes = unsummed(parseRecall(await mem7($, at, recallBody(agent, ['episode'], CONSOLIDATE_MAX))), journals)
469  if (episodes.length === 0) return
470  const r = await $.model.complete({
471    model: 'haiku',
472    system: personalize(voice.persona, userName, voice.nobody) + STYLE,
473    prompt: journalPrompt(episodes, userName !== '' ? userName : 'the user'),
474    maxTokens: 200,
475    timeoutMs: 30_000,
476  })
477  await used($, 'haiku', r)
478  const journal = journalFrom(r)
479  // Nothing worth keeping is a journal too: the same exchanges are not read again.
480  await mem7($, at, storeBody(agent, journalKey(agent, new Date()), journal ?? 'Nothing worth keeping.', ['journal']))
481}
482
483// What a line needs from the session around it, read when the line is picked.
484type Speaking = {
485  voice: Persona
486  voiceId: string
487  isGuestTurn: boolean
488  // The other in a dialogue, by id: the visit goes to both memories.
489  otherId: string
490  memory: MemoryAt | null
491  // The persona on duty, by name: whom a guest's turn speaks to.
492  host: string
493  asked: string
494  userName: string
495  relay: Relay
496  // The frame now: it moves on while the line is written and voiced.
497  now: () => number
498  say: (ask: Ask) => void
499}
500
501// One line, from the model (or as written) to the voice: written, typed under
502// the face, synthesized and played; the speaking slot is given back at the end.
503async function speak($: Engine, stage: Stage, ask: Ask, c: Speaking): Promise<void> {
504  try {
505    // The line: a greeting as written, anything else from the model.
506    // A consultation and a chat keep all their sentences.
507    const isConsult = isOpinion(ask)
508    const isChat = typeof ask === 'object' && 'chat' in ask
509    const write = async (): Promise<string> => {
510      const need = reads(ask)
511      const conversation =
512        need === null
513          ? ''
514          : (await $.session.messages())
515              .filter(m => m.text.trim() !== '')
516              .slice(-need.count)
517              .map(m => `${m.role}: ${m.text.replace(/\s+/g, ' ').trim().slice(0, need.chars)}`)
518              .join('\n')
519      const other = c.isGuestTurn ? c.host : (stage.guest?.persona.name ?? 'a visitor')
520      const prompt = promptFor(ask, { voice: c.voice, other, conversation, chatPast: stage.chats.get(c.voiceId) ?? [], asked: c.asked, recent: stage.recent })
521      if (prompt === null) return typeof ask === 'object' && 'greet' in ask ? ask.greet : ''
522      const past = c.memory !== null && recalls(ask) ? await remembered($, c.memory, c.voiceId, queryFor(ask, c.asked, other)) : ''
523      const r = await $.model.complete({
524        model: 'haiku',
525        system: personalize(c.voice.persona, c.userName, c.voice.nobody) + STYLE,
526        prompt: prompt + past,
527        maxTokens: isOpinion(ask) ? 160 : 80,
528        timeoutMs: 15_000,
529      })
530      await used($, 'haiku', r)
531      return lineFrom(r, ask, c.voice, c.now(), c.userName)
532    }
533    const text = typeof ask === 'object' && 'greet' in ask ? ask.greet : await write()
534    if (text === '') return
535    // An opinion that ends on a question opens the answer field too.
536    // A chat's question is answered by the next chat.
537    if (typeof ask === 'object' && 'chat' in ask) {
538      const past = [...(stage.chats.get(c.voiceId) ?? []), `User: ${ask.chat}`, `${c.voice.name}: ${text}`]
539      stage.chats.set(c.voiceId, past.slice(-CHAT_LINES))
540    }
541    if (ask === 'question' || (isConsult && !isChat && text.endsWith('?'))) {
542      stage.openQuestion = { text, at: c.now() }
543      $.ui.invalidate('ui.render')
544    }
545    stage.recent.push(text)
546    if (stage.recent.length > RECENT_LINES) stage.recent.shift()
547    if (c.memory !== null) {
548      const at = c.memory
549      const episode = episodeOf(ask, c.voice.name, text)
550      if (episode !== null) void keep($, at, c.voiceId, episode.value, [episode.kind])
551      if (typeof ask === 'object' && 'duo' in ask && ask.turn === DUO_TURNS - 1 && c.otherId !== '') {
552        const last = `${c.voice.name}: ${text}`
553        const otherName = c.isGuestTurn ? c.host : (stage.guest?.persona.name ?? c.otherId)
554        void keep($, at, c.voiceId, visitOf(ask.history, last, otherName), ['visit'])
555        void keep($, at, c.otherId, visitOf(ask.history, last, c.voice.name), ['visit'])
556      }
557    }
558    const isDuo = typeof ask === 'object' && 'duo' in ask
559    const shown = isDuo ? `${c.voice.name}: ${text}` : text
560    const isQuiet = await read($, isMuted)
561    stage.typing = start(stage.typing, shown, isQuiet, c.now())
562    const seq = stage.typing.seq
563    await update($, line, () => ({ text: shown, at: c.now() }) satisfies Line)
564    if (!isQuiet) {
565      const made = await $.process.run(synthArgv(c.voiceId, c.voice, await read($, volume)), {
566        stdin: text,
567        timeoutMs: 30_000,
568      })
569      const heard = voiced(stage.typing, seq, made.stdout, c.now(), FRAME_MS)
570      stage.typing = heard.t
571      const { wav, ms } = heard
572      if (wav !== '') {
573        // Detached, so a reload of this module no longer cuts the line;
574        // the voice is held for the WAV's length plus PowerShell's start.
575        await update($, isVoicing, () => true)
576        await $.process.run(detachedArgv(playArgv(wav, c.relay.session, SAPI_PLAY, seq)))
577        await $.clock.sleep(ms + PLAY_START_MS)
578      }
579    }
580    if (typeof ask === 'object' && 'duo' in ask && ask.turn < DUO_TURNS - 1) {
581      c.say({ ...ask, turn: ask.turn + 1, history: [...ask.history, `${c.voice.name}: ${text}`] })
582    }
583  } catch (err) {
584    // A synthesis past its timeout, a model call that failed: the line
585    // is lost, the reason kept.
586    $.ui.log(`avatar7: a line was lost: ${String(err)}`, { to: 'debug' })
587  } finally {
588    // A dialogue ends on its last turn, or on a turn that said nothing.
589    if (typeof ask === 'object' && 'duo' in ask && !stage.queue.some(q => typeof q.ask === 'object' && 'duo' in q.ask)) {
590      stage.isGuestShown = false
591      stage.guest = null
592    }
593    stage.typing = end(stage.typing)
594    if (await read($, isVoicing)) await update($, isVoicing, () => false)
595  }
596}
597
598export const register: Register = (on, options) => {
599  const userName = typeof options.user_name === 'string' ? options.user_name.trim() : ''
600  // The personas' own mem7, or null: then they forget between sessions.
601  const memory = memoryAt(options)
602  let frame = 0
603  // The face: its mood and the waits that hold it (mood.ts).
604  let face: Face = calm
605  // What speaking changes: the line, the queue, the guest, the open question,
606  // the chats and the last lines (Stage); speak() takes it whole.
607  const stage: Stage = { typing: silent, queue: [], guest: null, isGuestShown: false, openQuestion: null, chats: new Map(), recent: [] }
608  // The remote voice, its own state (relay.ts); relay.session is '' in a
609  // background session.
610  const relay = newRelay()
611  let sessionDir = ''
612  let lastPrompt = ''
613  let streak: Streak = { mood: 'watch', count: 0 }
614  // The persona's own events: on unless /avatar events off; the next one's frame.
615  let eventsOn = true
616  // Visits from other personas, a share of the events, switched apart.
617  let visitsOn = true
618  let nextEventAt = Infinity
619  const nextGap = (): number => EVENT_MIN_FRAMES + Math.random() * EVENT_SPAN_FRAMES
620  // A visit: the guest read (loadGuest), then the host opens.
621  const startDuo = (g: Guest, topic: Duo['topic']): boolean => {
622    // One visit at a time: a second would take over the first's turns.
623    if (stage.guest !== null) return false
624    stage.guest = g
625    face = react(face, 'watch', frame, 30)
626    speakLater({ duo: g.id, turn: 0, topic, history: [] })
627    return true
628  }
629  const startEvent = (ask: Ask): void => {
630    if (typeof ask === 'object' && 'story' in ask) {
631      face = stageFace(face, ask.mood, frame, 60)
632    }
633    speakLater(ask)
634  }
635  let faces: Faces | null = null
636  let who: Persona | null = null
637  // The avatar `who` was read from, so a line keeps its voice through a switch.
638  let whoId = ''
639  let size = W
640  // The ambient's geometry, set at each render: the margins either side of
641  // the face, the band under the text, in cells; and its own clock, which
642  // runs faster on a refusal or an error and slower while a human decides.
643  let ambLeft = 0
644  let ambRight = 0
645  let ambBand = 0
646  let ambField: Field = { width: 0, height: 0, sceneTop: 0, sceneBottom: 0 }
647  let ambT = 0
648  let asked = ''
649  const speakLater = (ask: Ask): void => {
650    stage.queue = enqueue(stage.queue, ask, frame)
651  }
652  // The last `say` of each mod, for /avatar voices.
653  const saidHere = new Map<string, Say>()
654  // An avatar picked in the pane or by /avatar; the clock swaps it in.
655  let pendingAvatar: string | null = null
656  let avatarPick = 0
657  let isPicking = false
658  // The field where the user asks the avatar's opinion, open or not.
659  let isConsulting = false
660  // The field where the user talks to the avatar personally.
661  let isChatting = false
662  // Display names for the picker, read from each persona.json.
663  const names: Record<string, string> = {}
664  let lastModel = ''
665
666  // The face's frame, as draw.ts takes it: whoever is shown (the guest
667  // during a visit), its mood and weather, read once per frame.
668  const view = (): View => {
669    const isGuest = stage.isGuestShown && stage.guest !== null
670    return {
671      frame,
672      frameMs: FRAME_MS,
673      mood: face.mood,
674      faces: isGuest && stage.guest !== null ? stage.guest.faces : faces,
675      persona: isGuest && stage.guest !== null ? stage.guest.persona : who,
676      isHeard: isHeard(stage.typing, frame),
677      glitch: face.isStaged ? SCENE_GLITCH : 1,
678      size,
679      layers: ambientLayers(),
680      field: ambField,
681      ambLeft,
682      ambT,
683    }
684  }
685  const cells = (): string => faceCells(view())
686  // The face last blitted, to skip the frames that change nothing.
687  let lastFace = ''
688
689  // The face as a real image (hd.ts), where the terminal draws pictures and
690  // the persona has its HD bake: the band's width in columns, set at each
691  // render; whether the last render mounted the Image; the key of the
692  // picture it shows, so the clock blits only when the picture changes; and
693  // the last pictures made, since the same few come back (moods, mouth).
694  let hdColumns = 0
695  // The art pixel's side (hd.ts GRAINS): /avatar pixel <n>, kept in $.store.
696  let grain: number = DEFAULT_GRAIN
697  type HdSource = { file: string; format: 'rgba'; width: number; height: number } | { rgba: string; width: number; height: number }
698  type HdBand = HdView['band']
699  // The picture files written, or being written, by path.
700  const hdFiles = new Map<string, 'pending' | 'ready'>()
701  // Each band's source as shown, kept as one object, so a redraw that
702  // changes nothing hands the engine the very same source and nothing is
703  // sent; and its key.
704  const hdShown: Record<HdBand, { source: HdSource; key: string } | null> = { face: null, under: null }
705  const isHdShown = (): boolean => hdShown.face !== null
706  // The pane's size and the frame it last changed (hd.ts isSettled).
707  const hdResize: Settle = { size: '', since: 0 }
708  let isHdSettled = false
709  // The scene layers with their HD bake in place of the 512 px one, made
710  // once per layer list so the fitted scene's cache holds.
711  const hdLayersOf = new WeakMap<AmbientLayer[], AmbientLayer[]>()
712  const hdLayers = (layers: AmbientLayer[], hd: Hd | null | undefined): AmbientLayer[] => {
713    const scene = hd?.scene
714    if (scene === null || scene === undefined) return layers
715    let swapped = hdLayersOf.get(layers)
716    if (swapped === undefined) {
717      swapped = layers.map(l => (l.kind === 'scene' ? { ...l, pixels: scene.pixels, width: scene.width, height: scene.height } : l))
718      hdLayersOf.set(layers, swapped)
719    }
720    return swapped
721  }
722  const hdView = (band: HdBand): HdView | null => {
723    const v = view()
724    const hd = v.faces?.hd
725    if (!isHdTerminal || hd === undefined || hd === null || v.persona === null || hdColumns === 0) return null
726    if (band === 'under' && ambBand === 0) return null
727    const isGuest = stage.isGuestShown && stage.guest !== null
728    // The weather loops over HD_LOOP pictures, each made once.
729    const step = Math.floor(ambT / HD_STEP_S) % HD_LOOP
730    const isFace = band === 'face'
731    return {
732      who: isGuest && stage.guest !== null ? stage.guest.id : whoId,
733      band,
734      hd: isFace ? hd : null,
735      mood: isFace ? v.mood : v.mood === 'deny' ? 'deny' : 'idle',
736      face: pickHdFace(hd, v.mood, v.isHeard, noise(v.frame >> 2, 7)),
737      glitchStep: isFace && v.mood === 'deny' ? 1 + ((v.frame >> 2) % GLITCH_STEPS) : 0,
738      glitch: v.glitch,
739      grain,
740      color: v.persona.color ?? '#00ff9c',
741      cutout: v.persona.cutout ?? 12,
742      columns: hdColumns,
743      rows: isFace ? size / 2 : ambBand,
744      size: isFace ? size : 0,
745      layers: hdLayers(v.layers, hd),
746      field: v.field,
747      top: isFace ? 0 : size + TEXT_ROWS * 2,
748      t: step * HD_STEP_S,
749      step,
750      isStorm: v.mood === 'deny',
751      stamp: hd.stamp,
752    }
753  }
754  const fileSource = (hv: HdView): HdSource => ({ file: hdPath(hdKey(hv)), format: 'rgba', ...hdSize(hv.columns, hv.rows) })
755  // The source a band shows at a render: its picture if written, else the
756  // last one until the clock swaps the new one in, else (the first) bytes;
757  // and the view whose file is still to write.
758  // The engine takes at most 2 MiB of Image source a tree: a band whose
759  // bytes would pass what is left waits for its file, and the clock renders
760  // again once it is written.
761  const HD_INLINE_MAX = 2_000_000
762  const hdRender = (band: HdBand, budget: { left: number }): { source: HdSource | null; toWrite: HdView | null; rgba?: Uint8Array } => {
763    const hv = hdView(band)
764    if (hv === null) {
765      hdShown[band] = null
766      return { source: null, toWrite: null }
767    }
768    const key = hdKey(hv)
769    const was = hdShown[band]
770    const isReady = hdFiles.get(hdPath(key)) === 'ready'
771    let rgba: Uint8Array | undefined
772    if (isReady) {
773      if (was?.key !== key) hdShown[band] = { source: fileSource(hv), key }
774    } else {
775      const { width, height } = hdSize(hv.columns, hv.rows)
776      if (was === null || was.source.width !== width || was.source.height !== height) {
777        // Mid-resize: the last picture, stretched, and nothing made.
778        if (was !== null && !isHdSettled) return { source: was.source, toWrite: null }
779        const inline = Math.ceil((width * height * 4) / 3) * 4
780        if (inline > budget.left) return { source: was?.source ?? null, toWrite: hv }
781        budget.left -= inline
782        rgba = hdFrame(hv)
783        hdShown[band] = { source: { rgba: rgba.toBase64(), width, height }, key }
784      }
785    }
786    return { source: hdShown[band]?.source ?? null, toWrite: isReady ? null : hv, rgba }
787  }
788  // At a frame: the band's new picture to swap in once written, or the view
789  // whose file is still to write.
790  const hdTick = (band: HdBand): { swap: HdSource | null; toWrite: HdView | null; isRefit: boolean } => {
791    const hv = hdView(band)
792    const shown = hdShown[band]
793    if (hv === null) return { swap: null, toWrite: null, isRefit: false }
794    const key = hdKey(hv)
795    // Not shown yet: its file, once written, mounts it at a render.
796    if (shown === null) return { swap: null, toWrite: null, isRefit: hdFiles.get(hdPath(key)) === 'ready' }
797    if (key === shown.key) return { swap: null, toWrite: null, isRefit: false }
798    // A new size: once it holds, a render makes its picture.
799    const { width, height } = hdSize(hv.columns, hv.rows)
800    if (shown.source.width !== width || shown.source.height !== height) return { swap: null, toWrite: null, isRefit: isSettled(hdResize, hdResize.size, frame) }
801    if (hdFiles.get(hdPath(key)) !== 'ready') return { swap: null, toWrite: hv, isRefit: false }
802    const source = fileSource(hv)
803    hdShown[band] = { source, key }
804    return { swap: source, toWrite: null, isRefit: false }
805  }
806
807  // The weather of whoever is shown: the guest's during a visit.
808  const ambientLayers = (): AmbientLayer[] =>
809    (stage.isGuestShown && stage.guest !== null ? stage.guest.persona.ambient : who?.ambient) ?? []
810
811  // The repaints of the ambient's Rasters, the ones the last render mounted.
812  const ambientBlits = (): { requestId: string; key: string; columns: number; rows: number; cells: string }[] => {
813    const layers = ambientLayers()
814    if (layers.length === 0) return []
815    const isStorm = face.mood === 'deny'
816    const rows = size / 2
817    const paint = (key: string, x0: number, y0: number, columns: number, height: number) => ({
818      requestId: PANE,
819      key,
820      columns,
821      rows: height,
822      cells: ambientCells(layers, ambField, x0, y0, columns, height, ambT, isStorm),
823    })
824    return [
825      // Beside the face only in half blocks: the HD picture holds its own scene.
826      ...(ambLeft > 0 && !isHdShown() ? [paint(AMB_LEFT, 0, 0, ambLeft, rows)] : []),
827      ...(ambRight > 0 && !isHdShown() ? [paint(AMB_RIGHT, ambLeft + size, 0, ambRight, rows)] : []),
828      ...(ambBand > 0 && hdShown.under === null ? [paint(AMB_BAND, 0, size + TEXT_ROWS * 2, ambField.width / QUAD, ambBand)] : []),
829    ]
830  }
831
832  const describe = (e: Record<string, unknown>): string => {
833    const hint = e.command ?? e.file_path ?? e.pattern ?? e.url ?? ''
834    return `${String(e.tool)} ${String(hint).slice(0, 80)}`.trim()
835  }
836
837  on('session.start', async ($, e, next) => {
838    // A daemon's background session (a spare kept warm, a `claude --bg`)
839    // inherits the plugin dirs but nobody watches it: no face, no voice.
840    const kind = await $.process.run(['sh', '-c', 'printf %s "$CLAUDE_CODE_SESSION_KIND"'])
841    if (kind.stdout === 'bg') return next(e)
842    const env = await $.process.run(['sh', '-c', 'printf %s "$TERM|$TERM_PROGRAM|$AVATAR7_HD"'])
843    isHdTerminal = drawsPictures(env.stdout)
844    // Pictures a day old belong to panes long gone.
845    if (isHdTerminal) await $.process.run(['sh', '-c', `mkdir -p ${HD_DIR} && find ${HD_DIR} -type f -mmin +1440 -delete`])
846    // A reload mid-line kills the timer that would lower it: jukebox7 would
847    // stay ducked.
848    if (await read($, isVoicing)) await update($, isVoicing, () => false)
849    await relayOpen($, relay)
850    sessionDir = e.cwd.split('/').filter(Boolean).at(-1) ?? '/'
851
852    await $.command.register({
853      name: 'avatar',
854      description: `Open the avatar pane, or switch: /avatar ${AVATARS.join('|')}; /avatar event, /avatar duo [id], /avatar events on|off, /avatar visits on|off, /avatar remote on|off, /avatar pixel ${GRAINS.join('|')}`,
855    })
856    await $.command.register({ name: 'avatar-mute', description: 'Toggle the avatar voice' })
857    await $.command.register({ name: 'avatar-talk', description: 'Ask the avatar what it thinks of the conversation' })
858    await $.command.register({
859      name: 'avatar-chat',
860      description: 'Talk to the avatar personally, not about the session: /avatar-chat <what you say>',
861    })
862    await $.command.register({
863      name: 'avatar-ask',
864      description: 'Ask the avatar its opinion on the session: /avatar-ask <question>',
865    })
866
867    const storedVolume = await $.store.get('volume')
868    if (typeof storedVolume === 'number') await update($, volume, () => storedVolume)
869
870    const stored = await $.store.get('avatar')
871    const id = typeof stored === 'string' && AVATARS.includes(stored) ? stored : DEFAULT
872    await update($, onDuty, () => id)
873    try {
874      const dir = `${$.plugin.root}/personas/${id}`
875      who = await readPersona($, id)
876      whoId = id
877      await update($, tint, () => who?.color ?? '')
878      await update($, station, () => ({ name: who?.name ?? '', artists: who?.station ?? [] }))
879      faces = await loadFaces($, dir)
880    } catch {
881      $.ui.log(`avatar7: personas/${id} unreadable, run tools/bake.py ${id}`)
882    }
883    if (isHdTerminal) await hdOnDisk($, hdFiles)
884    if (memory !== null && who !== null) {
885      const voice = who
886      void summarize($, memory, id, voice, userName).catch(err => $.ui.log(`avatar7: no journal for ${id}: ${String(err)}`, { to: 'debug' }))
887    }
888    for (const each of AVATARS) {
889      try {
890        names[each] = (JSON.parse(String(await $.fs.read(`${$.plugin.root}/personas/${each}/persona.json`))) as Persona).name
891      } catch {
892        names[each] = each
893      }
894    }
895
896    const last = await read($, line)
897    stage.typing = restored(stage.typing, last.text)
898    eventsOn = (await $.store.get('events')) !== false
899    const storedGrain = await $.store.get('pixel')
900    if (typeof storedGrain === 'number' && (GRAINS as readonly number[]).includes(storedGrain)) grain = storedGrain
901    visitsOn = (await $.store.get('visits')) !== false
902    nextEventAt = frame + nextGap()
903
904    $.clock.every(FRAME_MS, () => {
905      frame += 1
906      if (pendingAvatar !== null) {
907        const id = pendingAvatar
908        pendingAvatar = null
909        const pick = ++avatarPick
910        void (async () => {
911          const dir = `${$.plugin.root}/personas/${id}`
912          // Loaded whole before anything switches: a persona that cannot be
913          // read leaves the one on duty, and of two quick picks the last wins.
914          const chosen = await readPersona($, id)
915          const chosenFaces = await loadFaces($, dir)
916          if (pick !== avatarPick) return
917          who = chosen
918          whoId = id
919          faces = chosenFaces
920          await update($, tint, () => who?.color ?? '')
921          await update($, station, () => ({ name: who?.name ?? '', artists: who?.station ?? [] }))
922          await $.store.set('avatar', id)
923          await update($, onDuty, () => id)
924          await $.ui.open({ id: PANE, title: who.name })
925  
926          // Through the queue like any line: never over another voice, and
927          // jukebox7 hears isVoicing for it too.
928          speakLater({ greet: personalize(who.greeting, userName, who.nobody) })
929          if (memory !== null) await summarize($, memory, id, chosen, userName)
930        })().catch(err => $.ui.log(`avatar7: personas/${id} could not take over: ${String(err)}`))
931      }
932      face = tick(face, frame, WAIT_CAP_FRAMES)
933      relayTick($, relay, frame, {
934        face: () => {
935          if (who === null) return undefined
936          const shownWho = stage.isGuestShown && stage.guest !== null ? stage.guest.persona : who
937          const base = {
938            persona: stage.isGuestShown && stage.guest !== null ? stage.guest.id : whoId,
939            name: shownWho.name,
940            color: shownWho.color ?? '',
941            mood: face.mood,
942            line: stage.typing.text,
943            seq: stage.typing.seq,
944            speakMs: Math.max(0, stage.typing.speakUntil - stage.typing.from) * FRAME_MS,
945            isSpeaking: isHeard(stage.typing, frame),
946            question: stage.openQuestion?.text ?? '',
947            avatars: AVATARS.map(id => ({ id, name: names[id] ?? id })),
948            eventsOn,
949            visitsOn,
950            session: [relay.session.slice(0, 8), sessionDir, lastPrompt === '' ? '' : `"${lastPrompt.slice(0, 40)}"`].filter(Boolean).join(' / '),
951          }
952          return (async (): Promise<Mirror> => ({ ...base, isMuted: await read($, isMuted), volume: await read($, volume) }))()
953        },
954        press: async r => {
955          if (r.cmd === 'talk') speakLater('talk')
956          else if (r.cmd === 'ask') {
957            const question = r.text.replace(/\s+/g, ' ').trim().slice(0, CONSULT_CHARS_ASKED)
958            if (question !== '') speakLater({ consult: question })
959          } else if (r.cmd === 'chat') {
960            const said = r.text.replace(/\s+/g, ' ').trim().slice(0, CONSULT_CHARS_ASKED)
961            if (said !== '') speakLater({ chat: said })
962          } else if (r.cmd === 'answer') {
963            const answer = r.text.replace(/\s+/g, ' ').trim().slice(0, ASKED_CHARS)
964            const q = stage.openQuestion
965            if (answer !== '' && q !== null) {
966              stage.openQuestion = null
967              speakLater({ question: q.text, answer })
968            }
969          } else if (r.cmd === 'avatar') {
970            if (AVATARS.includes(r.id) && r.id !== whoId) pendingAvatar = r.id
971          } else if (r.cmd === 'duo') {
972            // A visit now, as /avatar duo with no name: a guest picked at random among the friends.
973            const gid = pickGuest(AVATARS, whoId, Math.random(), who?.friends)
974            const g = gid === undefined || gid === whoId ? null : await loadGuest($, gid)
975            if (g !== null) startDuo(g, Math.random() < 0.5 ? 'session' : 'stories')
976          } else if (r.cmd === 'mute') await update($, isMuted, was => !was)
977          else if (r.cmd === 'events') {
978            eventsOn = !eventsOn
979            await $.store.set('events', eventsOn)
980          } else if (r.cmd === 'visits') {
981            visitsOn = !visitsOn
982            await $.store.set('visits', visitsOn)
983          } else if (r.cmd === 'volume') {
984            const step = r.step * VOLUME_STEP
985            const v = await update($, volume, was => Math.min(100, Math.max(0, was + step)))
986            await $.store.set('volume', v)
987          }
988          $.ui.invalidate('ui.render')
989        },
990      })
991      let isRefit = false
992      for (const [band, key] of [['face', FACE], ['under', AMB_BAND]] as const) {
993        const tick = hdTick(band)
994        if (tick.toWrite !== null) hdEnsure($, hdFiles, tick.toWrite)
995        if (tick.swap !== null) void $.ui.blit({ requestId: PANE, key, source: tick.swap })
996        isRefit ||= tick.isRefit
997      }
998      if (isRefit) $.ui.invalidate('ui.render')
999      // Sent only when it changed: each blit redraws the screen, and in a
1000      // terminal without synchronized output (WezTerm) the cursor jumped at
1001      // 15 i/s. A full render draws the face afresh, so a skip never leaves
1002      // a stale one.
1003      if (!isHdShown()) {
1004        const drawn = cells()
1005        if (drawn !== lastFace) {
1006          lastFace = drawn
1007          void $.ui.blit({ requestId: PANE, key: FACE, columns: size, rows: size / 2, cells: drawn })
1008        }
1009      }
1010      if (frame % AMBIENT_FRAMES === 0) {
1011        ambT += ((AMBIENT_FRAMES * FRAME_MS) / 1000) * (face.mood === 'deny' || face.mood === 'error' ? 2 : face.mood === 'wait' ? 0.5 : 1)
1012        for (const each of ambientBlits()) void $.ui.blit(each)
1013      }
1014      const typedOn = typeOn(stage.typing, frame)
1015      if (typedOn !== stage.typing) {
1016        stage.typing = typedOn
1017        $.ui.invalidate('ui.render')
1018      } else if (frame % REFIT_FRAMES === 0) {
1019        $.ui.invalidate('ui.render')
1020      }
1021
1022      if (face.askSince !== null && frame - face.askSince === ASK_FRAMES) {
1023        speakLater({ mood: 'wait', event: `call waiting for the user's permission: ${face.askCall}` })
1024      }
1025
1026      // Now and then, when nothing else happens, a moment of the persona's own.
1027      if (
1028        eventsOn &&
1029        who !== null &&
1030        frame >= nextEventAt &&
1031        stage.queue.length === 0 &&
1032        !stage.typing.isSpeaking &&
1033        !isWaiting(face) &&
1034        stage.openQuestion === null &&
1035        frame - stage.typing.lastSpoke > EVENT_QUIET_FRAMES
1036      ) {
1037        nextEventAt = frame + nextGap()
1038        // One event in three is a visit from another persona.
1039        const gid = visitsOn && Math.random() < 1 / 3 ? pickGuest(AVATARS, whoId, Math.random(), who.friends) : undefined
1040        if (gid !== undefined) {
1041          void loadGuest($, gid).then(g => {
1042            if (g !== null) startDuo(g, Math.random() < 0.5 ? 'session' : 'stories')
1043          })
1044        } else {
1045          const ev = pickEvent(who.events ?? [], who.asks !== undefined, Math.random(), Math.random())
1046          if (ev !== undefined) startEvent(ev)
1047        }
1048      }
1049      if (stage.openQuestion !== null && frame - stage.openQuestion.at > QUESTION_FRAMES) {
1050        stage.openQuestion = null
1051        $.ui.invalidate('ui.render')
1052      }
1053
1054      stage.queue = fresh(stage.queue, frame)
1055      // A line may wait a moment for a mod to say better (a mesh7 verdict).
1056      const ready = stage.queue.findIndex(q => typeof q.ask !== 'object' || !('after' in q.ask) || (q.ask.after ?? 0) <= frame)
1057      const first = stage.queue[ready]
1058      if (first === undefined || stage.typing.isSpeaking || who === null) return
1059      stage.queue = stage.queue.filter((_, i) => i !== ready)
1060      const ask = first.ask
1061      stage.typing = begin(stage.typing, frame)
1062      // In a dialogue the guest speaks the odd turns, with its own voice and face.
1063      const isGuestTurn = typeof ask === 'object' && 'duo' in ask && ask.turn % 2 === 1 && stage.guest !== null
1064      const voice = isGuestTurn && stage.guest !== null ? stage.guest.persona : who
1065      const voiceId = isGuestTurn && stage.guest !== null ? stage.guest.id : whoId
1066      const otherId = isGuestTurn ? whoId : (stage.guest?.id ?? '')
1067      stage.isGuestShown = isGuestTurn
1068      const host = who.name
1069      $.clock.after(1, () =>
1070        speak($, stage, ask, { voice, voiceId, isGuestTurn, otherId, memory, host, asked, userName, relay, now: () => frame, say: speakLater }),
1071      )
1072    })
1073
1074    // Opened after the plugins beneath have started: the pane opened last is
1075    // the one shown, and mesh7-pane or jukebox7 would take that place. Once
1076    // more a moment later, should one of them open late.
1077    const started = await next(e)
1078    void $.ui.open({ id: PANE, title: who?.name ?? 'avatar7' })
1079    $.clock.after(1500, () => void $.ui.open({ id: PANE, title: who?.name ?? 'avatar7' }))
1080    return started
1081  })
1082
1083  on('command.run', { command: 'avatar' }, async ($, e) => {
1084    const id = e.args.trim().toLowerCase()
1085
1086    if (id === '') {
1087      const opened = await $.ui.open({ id: PANE, title: who?.name ?? 'avatar7' })
1088      return { text: opened.isPlaced ? `${who?.name ?? 'avatar7'} is watching.` : 'The pane needs a wider terminal.' }
1089    }
1090    // The mods heard asking for a voice, and what they asked.
1091    // The persona's own events: one now, or switched on or off for good.
1092    if (id === 'event') {
1093      const ev = who === null ? undefined : pickEvent(who.events ?? [], who.asks !== undefined, Math.random(), Math.random())
1094      if (ev === undefined) return { text: `${who?.name ?? 'avatar7'} has no events of its own yet.` }
1095      startEvent(ev)
1096      return { text: ev === 'question' ? `${who?.name} has a question.` : `Something happens to ${who?.name}.` }
1097    }
1098    // A visit now: /avatar duo <id>, or a guest picked at random.
1099    if (id === 'duo' || id.startsWith('duo ')) {
1100      const want = id.slice(3).trim()
1101      const gid = want !== '' ? want : pickGuest(AVATARS, whoId, Math.random(), who?.friends)
1102      if (gid === undefined || !AVATARS.includes(gid) || gid === whoId) {
1103        return { text: `Pick another avatar: ${AVATARS.filter(a => a !== whoId).join(', ')}.` }
1104      }
1105      const g = await loadGuest($, gid)
1106      if (g === null) return { text: `personas/${gid} is unreadable.` }
1107      if (!startDuo(g, Math.random() < 0.5 ? 'session' : 'stories')) return { text: `${stage.guest?.persona.name ?? 'A guest'} is visiting already.` }
1108      return { text: `${g.persona.name} visits ${who?.name ?? 'avatar7'}.` }
1109    }
1110    if (id === 'visits on' || id === 'visits off') {
1111      visitsOn = id === 'visits on'
1112      await $.store.set('visits', visitsOn)
1113      $.ui.invalidate('ui.render')
1114      return { text: visitsOn ? 'The avatars visit each other again.' : 'No more visits.' }
1115    }
1116    if (id === 'events on' || id === 'events off') {
1117      eventsOn = id === 'events on'
1118      await $.store.set('events', eventsOn)
1119      $.ui.invalidate('ui.render')
1120      return { text: eventsOn ? 'The avatars live their own stories again.' : 'No more events of their own.' }
1121    }
1122    if (id === 'voices') {
1123      const all = [
1124        ...Object.entries(await read($, announcers)).map(([plugin, a]) => `${plugin} (toasts): ${a.mood}, ${a.event}`),
1125        ...[...saidHere].map(([plugin, a]) => `${plugin} (say): ${a.mood}, ${a.event}`),
1126      ]
1127      return { text: all.length === 0 ? 'No mod has asked for a voice in this session.' : all.join('\n') }
1128    }
1129    if (id.startsWith('pixel')) {
1130      const n = Number(id.slice('pixel'.length).trim())
1131      if (!(GRAINS as readonly number[]).includes(n)) return { text: `The pixel's size, in kitty or Ghostty: /avatar pixel ${GRAINS.join('|')} (1 smooth, 6 the half blocks' grid); now ${grain}.` }
1132      grain = n
1133      await $.store.set('pixel', n)
1134      $.ui.invalidate('ui.render')
1135      return { text: isHdTerminal ? `Pixels of ${n}.` : `Pixels of ${n}, seen in kitty or Ghostty; this terminal keeps the half blocks.` }
1136    }
1137    if (id === 'remote on') return { text: await relayOn($, relay) }
1138    if (id === 'remote off') return { text: await relayOff($, relay) }
1139    if (!AVATARS.includes(id)) return { text: `Unknown avatar. Choose one of: ${AVATARS.join(', ')}.` }
1140
1141    pendingAvatar = id
1142    return { text: `${names[id] ?? id} takes over.` }
1143  })
1144
1145  on('command.run', { command: 'avatar-talk' }, async () => {
1146    speakLater('talk')
1147    return { text: `${who?.name ?? 'avatar7'} reads the conversation.` }
1148  })
1149
1150  on('command.run', { command: 'avatar-chat' }, async ($, e) => {
1151    const said = e.args.replace(/\s+/g, ' ').trim().slice(0, CONSULT_CHARS_ASKED)
1152    if (said === '') return { text: 'Usage: /avatar-chat <what you say>' }
1153    speakLater({ chat: said })
1154    return { text: `${who?.name ?? 'avatar7'} listens.` }
1155  })
1156
1157  on('command.run', { command: 'avatar-ask' }, async ($, e) => {
1158    const question = e.args.replace(/\s+/g, ' ').trim().slice(0, CONSULT_CHARS_ASKED)
1159    if (question === '') return { text: 'Usage: /avatar-ask <question>' }
1160    speakLater({ consult: question })
1161    return { text: `${who?.name ?? 'avatar7'} reads the session and thinks it over.` }
1162  })
1163
1164  // Every other slash command, native or custom: the listed ones earn a line,
1165  // the rest run untouched.
1166  on('command.run', async ($, e, next) => {
1167    const heard = commandEvent(e.command, e.args)
1168    if (heard !== undefined) speakLater(heard)
1169    return next(e)
1170  })
1171
1172  // An automatic compaction, the main conversation's only: the manual one
1173  // came through /compact.
1174  on('session.compact', async ($, e, next) => {
1175    if (e.trigger === 'auto' && e.agentId === undefined) {
1176      speakLater({ mood: 'error', event: 'the context window filled up and the conversation was compacted on its own' })
1177    }
1178    return next(e)
1179  })
1180
1181  on('command.run', { command: 'avatar-mute' }, async $ => {
1182    const muted = await update($, isMuted, was => !was)
1183    return { text: muted ? 'The avatar falls silent.' : 'The avatar speaks again.' }
1184  })
1185
1186  // A session that ends gives the relay back; a /clear keeps a forced one.
1187  on('session.end', async ($, e, next) => {
1188    await relayEnd($, relay, e.reason)
1189    return next(e)
1190  })
1191
1192  // A model switch is no command the mod sees (the /model picker settles it
1193  // after the command has run, and Remote Control or /config switch without
1194  // one): the model in use is compared at each turn's end.
1195  on('turn.complete', async ($, e, next) => {
1196    asked = ''
1197    if (relay.session !== '') {
1198      const model = await $.session.model()
1199      if (lastModel !== '' && model !== lastModel && who !== null) {
1200        speakLater({ mood: 'watch', event: `the assistant now runs on ${model}, no longer on ${lastModel}` })

1// Pixel weather in the black around the face: one or two layers per persona,
2// read from its persona.json `ambient`. Every pixel is a function of its place
3// and the ambient's own time through hashed noise, so no drop is kept; only
4// the fitted scene and the current bolt are cached.
5
6export type AmbientKind = 'rain' | 'rise' | 'wind' | 'stars' | 'bolt' | 'pulse' | 'scene' | 'grid'
7
8// density: the share of columns, rows or pixels that carry something (0-1);
9// speed: pixels a second for what moves, the twinkle's pace for stars.
10// scene: a backdrop baked by tools/bake_scene.py, `file` of `width` x `height`
11// pixels, its bytes attached as `pixels` when the persona loads.
12export type AmbientLayer = {
13  kind: AmbientKind
14  color: string
15  density?: number
16  speed?: number
17  file?: string
18  width?: number
19  height?: number
20  pixels?: Uint8Array
21  // What moves in a scene: blinking red beacons, flickering neon signs,
22  // windows going dark; none when absent, the backdrop then stays still.
23  animate?: ('beacons' | 'neon' | 'windows')[]
24  // grid: where its horizon sits, as a share of the scene's bottom row.
25  horizon?: number
26}
27
28// The field the layers draw on: the pane's width, two pixels per column, and
29// the face's height plus the band under the text, two pixels per row. A cell
30// is drawn as a quadrant block (2x2), so its pixels are half as wide as tall:
31// QUAD of them across make one square. A scene covers the field from
32// `sceneTop` down to `sceneBottom`: from the face's middle, then a few rows
33// past the text.
34export type Field = { width: number; height: number; sceneTop: number; sceneBottom: number }
35export const QUAD = 2
36
37// How bright the weather may get against the face: it stays behind it.
38const DIM = 0.5
39
40const hash = (a: number, b: number, c = 0): number => {
41  let n = (a * 374761393 + b * 668265263 + c * 2147483647) | 0
42  n = Math.imul(n ^ (n >>> 13), 1274126177)
43  return ((n ^ (n >>> 16)) >>> 0) / 4294967296
44}
45
46// Parsed once per color, not per pixel. Black stays black; `#abc` is
47// `#aabbcc`; anything else unreadable is white.
48const colors = new Map<string, number>()
49const rgb = (hex: string): number => {
50  let c = colors.get(hex)
51  if (c === undefined) {
52    let h = hex.replace('#', '')
53    if (h.length === 3) h = [...h].map(d => d + d).join('')
54    c = /^[0-9a-f]{6}$/i.test(h) ? parseInt(h, 16) : 0xffffff
55    colors.set(hex, c)
56  }
57  return c
58}
59
60const scale = (c: number, k: number): number =>
61  (Math.min(255, Math.round(((c >> 16) & 0xff) * k)) << 16) |
62  (Math.min(255, Math.round(((c >> 8) & 0xff) * k)) << 8) |
63  Math.min(255, Math.round((c & 0xff) * k))
64
65// A streak running along one line, its head brightest: rain down a column,
66// wind along a row. Returns the light at `at` on that line, 0 when dark.
67const streak = (line: number, at: number, length: number, t: number, speed: number, trail: number, density: number, salt: number): number => {
68  if (hash(line, salt) >= density) return 0
69  const period = length + trail + Math.floor(hash(line, salt + 1) * length)
70  const head = (t * speed * (0.7 + 0.6 * hash(line, salt + 2)) + hash(line, salt + 3) * period) % period
71  const d = head - at
72  return d >= 0 && d < trail ? 1 - d / trail : 0
73}
74
75// A scene fitted to the field's width, anchored to its bottom, its top fading
76// into the black; each pixel sorted once so the right ones move: red lights
77// on the antennas blink, neon signs flicker, windows go dark and light again.
78const STILL = 0
79const BEACON = 1
80const NEON_SIGN = 2
81const WINDOW = 3
82// The backdrop stays a little under full light, behind the face.
83const SCENE = 0.8
84// A scene without animate: one shared empty list, not one per pixel.
85const NO_MOVES: NonNullable<AmbientLayer['animate']> = []
86type Scene = { width: number; height: number; rgb: Uint32Array; sort: Uint8Array }
87let sceneOf: Uint8Array | null = null
88let sceneWidth = 0
89let scene: Scene = { width: 0, height: 0, rgb: new Uint32Array(0), sort: new Uint8Array(0) }
90
91const fitScene = (layer: AmbientLayer, f: Field): Scene | null => {
92  const px = layer.pixels
93  const sw = layer.width ?? 0
94  const sh = layer.height ?? 0
95  const h = f.sceneBottom - f.sceneTop
96  if (px === undefined || sw === 0 || sh === 0 || f.width === 0 || h <= 0) return null
97  if (px === sceneOf && f.width === sceneWidth && h === scene.height) return scene
98  // Cover the region, as a CSS background does: scaled until both sides fill
99  // it, centered across, the ground kept and the sky cropped.
100  const w = f.width
101  const k = Math.max(w / QUAD / sw, h / sh)
102  const left = (sw * k - w / QUAD) / 2
103  const top = sh * k - h
104  const rgbOut = new Uint32Array(w * h)
105  const sort = new Uint8Array(w * h)
106  const lums = new Float32Array(w * h)
107  const lumAt = (at: number): number => lums[at] ?? 0
108  for (let y = 0; y < h; y++) {
109    const y0 = Math.min(sh - 1, Math.floor((y + top) / k))
110    const y1 = Math.max(y0 + 1, Math.min(sh, Math.floor((y + 1 + top) / k)))
111    // Faded into the black at the top, and at the bottom where it runs out
112    // under the text.
113    const fade = Math.min(1, y / (h * 0.2), (h - 1 - y) / (h * 0.12))
114    for (let x = 0; x < w; x++) {
115      const x0 = Math.min(sw - 1, Math.floor((x / QUAD + left) / k))
116      const x1 = Math.max(x0 + 1, Math.min(sw, Math.floor(((x + 1) / QUAD + left) / k)))
117      // The block's mean, but a beacon or a sign keeps its strongest pixel:
118      // averaged, a light one pixel wide would sink into the night.
119      let r = 0
120      let g = 0
121      let b = 0
122      let best = -1
123      let bestColor = 0
124      let bestSort = STILL
125      for (let sy = y0; sy < y1; sy++) {
126        for (let sx = x0; sx < x1; sx++) {
127          const i = (sy * sw + sx) * 3
128          const pr = px[i] ?? 0
129          const pg = px[i + 1] ?? 0
130          const pb = px[i + 2] ?? 0
131          r += pr
132          g += pg
133          b += pb
134          const max = Math.max(pr, pg, pb)
135          const sat = max === 0 ? 0 : (max - Math.min(pr, pg, pb)) / max
136          const isBeacon = pr > 140 && pg < pr * 0.5 && pb < pr * 0.5
137          const isSign = !isBeacon && sat > 0.45 && max > 150
138          if ((isBeacon || isSign) && max > best) {
139            best = max
140            bestColor = (pr << 16) | (pg << 8) | pb
141            bestSort = isBeacon ? BEACON : NEON_SIGN
142          }
143        }
144      }
145      const n = (x1 - x0) * (y1 - y0)
146      const mean = ((Math.round(r / n) << 16) | (Math.round(g / n) << 8) | Math.round(b / n)) >>> 0
147      const lum = (0.3 * r + 0.59 * g + 0.11 * b) / n
148      const at = y * w + x
149      rgbOut[at] = scale(best >= 0 ? bestColor : mean, fade)
150      sort[at] = best >= 0 ? bestSort : STILL
151      lums[at] = lum
152    }
153  }
154  // A window is a point of light: brighter than the pixels around it, not a
155  // lit wall or a patch of sky, which would go dark in specks.
156  // A sign stands out from what is beside it; a saturated sky does not, and
157  // would flicker in bars. Signs are upright strips: compare left and right.
158  const SIDE = 4
159  for (let y = 2; y < h - 2; y++) {
160    for (let x = SIDE; x < w - SIDE; x++) {
161      const at = y * w + x
162      if (sort[at] !== NEON_SIGN) continue
163      let beside = 0
164      for (const dy of [-2, 0, 2]) beside += lumAt(at + dy * w - SIDE) + lumAt(at + dy * w + SIDE)
165      if (lumAt(at) < 1.3 * (beside / 6)) sort[at] = STILL
166    }
167  }
168  for (let y = 1; y < h - 1; y++) {
169    for (let x = 1; x < w - 1; x++) {
170      const at = y * w + x
171      if (sort[at] !== STILL || lumAt(at) < 60) continue
172      let around = 0
173      for (let dy = -1; dy <= 1; dy++) for (let dx = -1; dx <= 1; dx++) around += lumAt(at + dy * w + dx)
174      if (lumAt(at) > 1.35 * ((around - lumAt(at)) / 8)) sort[at] = WINDOW
175    }
176  }
177  sceneOf = px
178  sceneWidth = w
179  scene = { width: w, height: h, rgb: rgbOut, sort }
180  return scene
181}
182
183// The scene's color at a field pixel, or -1 where it does not reach.
184const scenePixel = (layer: AmbientLayer, f: Field, x: number, y: number, t: number): number => {
185  const s = fitScene(layer, f)
186  if (s === null) return -1
187  const sy = y - f.sceneTop
188  if (sy < 0 || sy >= s.height || x >= s.width) return -1
189  const at = sy * s.width + x
190  let k = SCENE
191  const moves = layer.animate ?? NO_MOVES
192  const sort = s.sort[at]
193  if (!(sort === BEACON ? moves.includes('beacons') : sort === NEON_SIGN ? moves.includes('neon') : sort === WINDOW && moves.includes('windows'))) {
194    return scale(s.rgb[at] ?? 0, k)
195  }
196  switch (sort) {
197    case BEACON:
198      k *= (t * 0.8 + hash(x >> 1, sy >> 1, 77)) % 1 < 0.6 ? 1 : 0.2
199      break
200    case NEON_SIGN:
201      k *= hash(x >> 2, Math.floor(t * 6), 78) < 0.03 ? 0.3 : 1
202      break
203    case WINDOW:
204      k *= hash(x, sy, Math.floor(t / 4 + hash(x, sy, 79) * 7)) < 0.15 ? 0.45 : 1
205      break
206  }
207  return scale(s.rgb[at] ?? 0, k)
208}
209
210// A bolt strikes in a slot of 0.4 s: rarely on its own, every slot while
211// `isStorm`. Its path is cached for the slot.
212let boltFor = ''
213let boltPath: number[] = []
214const bolt = (f: Field, t: number, isStorm: boolean): number[] | null => {
215  const slot = Math.floor(t / 0.4)
216  if (!isStorm && hash(slot, 51) >= 0.03) return null
217  if (isStorm && hash(slot, 52) >= 0.5) return null
218  const id = `${slot}:${f.width}x${f.height}`
219  if (id !== boltFor) {
220    boltPath = []
221    let x = Math.floor(hash(slot, 53) * f.width)
222    for (let y = 0; y < f.height; y++) {
223      x += Math.round((hash(slot, y, 54) - 0.5) * 3)
224      boltPath.push(x)
225    }
226    boltFor = id
227  }
228  return boltPath
229}
230
231// One layer's light at a pixel, 0 to 1.
232const light = (layer: AmbientLayer, f: Field, x: number, y: number, t: number, isStorm: boolean): number => {
233  const density = layer.density
234  const speed = layer.speed
235  switch (layer.kind) {
236    case 'rain':
237      return streak(x, y, f.height, t, speed ?? 24, 6, density ?? 0.25, 1)
238    case 'wind':
239      return streak(y, x, f.width, t, (speed ?? 40) * QUAD, 12 * QUAD, density ?? 0.12, 11)
240    case 'rise': {
241      // Up a column, swaying a pixel either side, fading as it climbs.
242      for (let c = x - 1; c <= x + 1; c++) {
243        if (hash(c, 21) >= (density ?? 0.15)) continue
244        const period = f.height + Math.floor(hash(c, 22) * f.height)
245        const climbed = (t * (speed ?? 8) * (0.6 + 0.8 * hash(c, 23)) + hash(c, 24) * period) % period
246        const py = Math.round(f.height - climbed)
247        const px = c + Math.round(Math.sin(t * 1.7 + c) * 1)
248        if (px === x && (py === y || py === y + 1)) return Math.max(0.15, py / f.height) * (py === y ? 1 : 0.5)
249      }
250      return 0
251    }
252    case 'stars': {
253      if (hash(x, y, 31) >= (density ?? 0.015) / QUAD) return 0
254      return 0.25 + 0.75 * (0.5 + 0.5 * Math.sin(t * (speed ?? 1) * (0.5 + hash(x, y, 32)) + hash(x, y, 33) * 6.28))
255    }
256    case 'bolt': {
257      const path = bolt(f, t, isStorm)
258      if (path === null) return 0
259      const d = Math.abs((path[y] ?? -9) - x)
260      return d === 0 ? 1 : d === 1 ? 0.25 : 0
261    }
262    case 'scene':
263      // Drawn in color by scenePixel.
264      return 0
265    case 'grid': {
266      // An outrun floor: rows closer together toward the horizon, scrolling
267      // toward the viewer; rays fanning out from its middle.
268      // Its horizon where the scene fades out under the text, so the floor
269      // and its vanishing point fill the band below.
270      const horizon = Math.round(f.sceneBottom * (layer.horizon ?? 0.92))
271      if (y < horizon) return 0
272      if (y === horizon) return 1
273      // Perspective: a floor point at distance Z shows at depth 1/Z below the
274      // horizon, and its offset across shrinks by the same depth, so the rays
275      // meet at the middle of the horizon.
276      const span = f.height - horizon
277      const depth = (y - horizon) / span
278      const next = (y - horizon + 1) / span
279      // A row line lands on the pixel row whose span of Z holds it: one pixel
280      // thin. Rows every ROW of Z, coming toward the viewer; toward the horizon
281      // a pixel row spans more of Z, and they fade before they would merge.
282      const ROW = 0.2
283      const z = 1 / depth
284      const zNext = 1 / next
285      const shift = t * (speed ?? 1.5)
286      const rowFade = Math.min(1, Math.max(0, (ROW / 2 - (z - zNext)) / (ROW / 4)))
287      const row = Math.floor(z / ROW + shift) !== Math.floor(zNext / ROW + shift) ? rowFade : 0
288      // Rays every SPACING squares across, as seen at the bottom row, drawn
289      // soft (by distance to the line) so a slanted one does not stair-step,
290      // and faded where they crowd toward the vanishing point.
291      const SPACING = 8
292      const gap = SPACING * depth * QUAD
293      const across = (x + 0.5 - f.width / 2) / QUAD / depth / SPACING
294      const off = Math.abs(across - Math.round(across)) * gap
295      const ray = Math.max(0, 1 - off) * Math.min(1, Math.max(0, (gap - 3) / 6))
296      const k = Math.max(row, ray)
297      return k > 0 ? k * (0.35 + 0.65 * depth) : 0
298    }
299    case 'pulse': {
300      // Wires every 9 rows, a bright pulse running along each.
301      if (y % 9 !== 4) return 0
302      const wire = Math.floor(y / 9)
303      const at = (t * (speed ?? 30) * QUAD + hash(wire, 61) * (f.width + 20)) % (f.width + 20)
304      const d = Math.abs(at - x) / QUAD
305      return d < 3 ? 1 - d / 3 : 0.12
306    }
307  }
308}
309
310// Light added to light, each channel capped: a faint drop over a bright sky
311// brightens it a little instead of punching a dark hole in it.
312const add = (a: number, b: number): number =>
313  (Math.min(255, ((a >> 16) & 0xff) + ((b >> 16) & 0xff)) << 16) |
314  (Math.min(255, ((a >> 8) & 0xff) + ((b >> 8) & 0xff)) << 8) |
315  Math.min(255, (a & 0xff) + (b & 0xff))
316
317// The color of a field pixel: a scene lays its backdrop, the layers above
318// add their light to it; black when nothing does. `isStorm` raises bolts on
319// a refusal.
320export const ambientPixel = (layers: AmbientLayer[], f: Field, x: number, y: number, t: number, isStorm: boolean): number => {
321  let out = 0
322  for (const layer of layers) {
323    if (layer.kind === 'scene') {
324      const c = scenePixel(layer, f, x, y, t)
325      if (c >= 0) out = c
326      continue
327    }
328    const k = light(layer, f, x, y, t, isStorm)
329    if (k > 0) out = add(out, scale(rgb(layer.color), k * DIM))
330  }
331  return out
332}
333
334// Quadrant blocks by the mask of their lit quarters: upper left 1, upper
335// right 2, lower left 4, lower right 8.
336const QUADRANTS = [0x20, 0x2598, 0x259d, 0x2580, 0x2596, 0x258c, 0x259e, 0x259b, 0x2597, 0x259a, 0x2590, 0x259c, 0x2584, 0x2599, 0x259f, 0x2588]
337
338const distance = (a: number, b: number): number =>
339  Math.abs(((a >> 16) & 0xff) - ((b >> 16) & 0xff)) + Math.abs(((a >> 8) & 0xff) - ((b >> 8) & 0xff)) + Math.abs((a & 0xff) - (b & 0xff))
340
341// The mean color of the quarters in `mask` (bit m: quarter m), 0 for none.
342const meanOf = (q: readonly number[], mask: number): number => {
343  let r = 0
344  let g = 0
345  let b = 0
346  let n = 0
347  for (let m = 0; m < 4; m++) {
348    if ((mask & (1 << m)) === 0) continue
349    const c = q[m] ?? 0
350    r += (c >> 16) & 0xff
351    g += (c >> 8) & 0xff
352    b += c & 0xff
353    n += 1
354  }
355  return n === 0 ? 0 : (Math.round(r / n) << 16) | (Math.round(g / n) << 8) | Math.round(b / n)
356}
357
358// A Raster's cells for the field's rectangle at column x0 and pixel row y0,
359// `columns` wide and `rows` tall. Each cell holds four pixels and shows two
360// colors, as chafa does: the two most different quarters lead, each other
361// quarter joins the nearer one, and the glyph draws the first group.
362export const ambientCells = (layers: AmbientLayer[], f: Field, x0: number, y0: number, columns: number, rows: number, t: number, isStorm: boolean): string => {
363  const words = new Uint32Array(columns * rows * 3)
364  const q = [0, 0, 0, 0]
365  for (let row = 0; row < rows; row++) {
366    for (let col = 0; col < columns; col++) {
367      const i = (row * columns + col) * 3
368      const x = (x0 + col) * QUAD
369      const y = y0 + row * 2
370      q[0] = ambientPixel(layers, f, x, y, t, isStorm)
371      q[1] = ambientPixel(layers, f, x + 1, y, t, isStorm)
372      q[2] = ambientPixel(layers, f, x, y + 1, t, isStorm)
373      q[3] = ambientPixel(layers, f, x + 1, y + 1, t, isStorm)
374      let a = 0
375      let b = 1
376      let far = -1
377      for (let m = 0; m < 4; m++) {
378        for (let n = m + 1; n < 4; n++) {
379          const d = distance(q[m] ?? 0, q[n] ?? 0)
380          if (d > far) {
381            far = d
382            a = m
383            b = n
384          }
385        }
386      }
387      // Each quarter joins the nearer lead; no array per cell.
388      const qa = q[a] ?? 0
389      const qb = q[b] ?? 0
390      let mask = 0
391      for (let m = 0; m < 4; m++) {
392        const c = q[m] ?? 0
393        if (distance(c, qa) <= distance(c, qb)) mask |= 1 << m
394      }
395      words[i] = QUADRANTS[mask] ?? 0
396      words[i + 1] = meanOf(q, mask)
397      words[i + 2] = meanOf(q, ~mask & 0xf)
398    }
399  }
400  return new Uint8Array(words.buffer).toBase64()
401}
402

1// The remote voice: while tools/relay.py runs, the session that holds it sends
2// its lines, its face and the page's presses through a spool, for a browser
3// tab on another machine of the tailnet; every other session, and this one
4// when it does not hold it, speaks on the host. Here: the protocol, the shell
5// that touches the spool, the state and the decisions. The engine calls ($)
6// stay in register.tsx, which the engine requires: $ is never followed across
7// an import.
8
9export const RELAY_SPOOL = '${XDG_CACHE_HOME:-$HOME/.cache}/avatar7/relay'
10
11// The face as the relay's page draws it: who, in which mood, saying what, for
12// how long; written whole to the spool when it changes, while the relay runs.
13// The pane's controls ride along, so the page can draw them as they stand.
14export type Mirror = {
15  persona: string
16  name: string
17  color: string
18  mood: string
19  line: string
20  seq: number
21  speakMs: number
22  isSpeaking: boolean
23  question: string
24  avatars: { id: string; name: string }[]
25  isMuted: boolean
26  eventsOn: boolean
27  visitsOn: boolean
28  volume: number
29  // Which session holds the relay: short id, folder, last prompt typed.
30  session: string
31}
32
33// What the page may ask: the pane's gestures, never mesh7's approvals. talk,
34// ask and chat do reach the session: Haiku reads its last messages and the
35// answer goes back to the page, which also shows 40 characters of the last
36// prompt. The relay binds to the tailnet only, and takes presses only while a
37// session holds it.
38export type Remote =
39  | { cmd: 'talk' | 'mute' | 'events' | 'visits' | 'duo' }
40  | { cmd: 'ask' | 'answer' | 'chat'; text: string }
41  | { cmd: 'avatar'; id: string }
42  | { cmd: 'volume'; step: 1 | -1 }
43export const parseRemote = (line: string): Remote | undefined => {
44  let r: unknown
45  try {
46    r = JSON.parse(line)
47  } catch {
48    return undefined
49  }
50  if (typeof r !== 'object' || r === null) return undefined
51  const o = r as Record<string, unknown>
52  if (o.cmd === 'talk' || o.cmd === 'mute' || o.cmd === 'events' || o.cmd === 'visits' || o.cmd === 'duo') return { cmd: o.cmd }
53  if ((o.cmd === 'ask' || o.cmd === 'answer' || o.cmd === 'chat') && typeof o.text === 'string') return { cmd: o.cmd, text: o.text }
54  if (o.cmd === 'avatar' && typeof o.id === 'string') return { cmd: 'avatar', id: o.id }
55  if (o.cmd === 'volume' && (o.step === 1 || o.step === -1)) return { cmd: 'volume', step: o.step }
56  return undefined
57}
58
59// How often, in frames, the clock looks for the relay's spool, mirrors, and
60// reads the page.
61export const CHECK_FRAMES = 30
62export const MIRROR_FRAMES = 3
63export const DRAIN_FRAMES = 8
64
65// The relay belongs to the session that ran /avatar remote on, named in its
66// spool's `owner`: every other session keeps its voice and face on this machine.
67// A relay that died without cleaning up (killed, crashed at start) leaves its
68// spool behind: the owner holds it only while the process in relay.pid lives.
69const ownsRelay = `[ "$(cat "${RELAY_SPOOL}/owner" 2>/dev/null)" = "$2" ] && kill -0 "$(cat "${RELAY_SPOOL}/relay.pid" 2>/dev/null)" 2>/dev/null`
70const owned = (name: string, script: string, session: string): string[] => ['sh', '-c', script, name, '', session]
71// Prints `up` while this session holds a live relay.
72export const heldArgv = (session: string): string[] => owned('avatar7-relay', `${ownsRelay} && echo up`, session)
73// Prints `up` while a relay process lives, whoever holds it.
74export const aliveArgv = (): string[] => ['sh', '-c', `p="${RELAY_SPOOL}/relay.pid"; [ -f "$p" ] && kill -0 "$(cat "$p")" 2>/dev/null && echo up`]
75// No service runs the relay: started from the plugin, detached so a reload
76// does not kill it, on a clean spool.
77export const startArgv = (root: string): string[] => [
78  'bash',
79  '-c',
80  `rm -rf "${RELAY_SPOOL}"; setsid python3 "$0/tools/relay.py" </dev/null >/dev/null 2>&1 &`,
81  root,
82]
83// Prints `up` while a page listens to the live relay: relay.py keeps `wanted`
84// in the spool while one is connected, and a grace after the last one goes.
85export const wantedArgv = (): string[] => [
86  'sh',
87  '-c',
88  `[ -f "${RELAY_SPOOL}/wanted" ] && kill -0 "$(cat "${RELAY_SPOOL}/relay.pid" 2>/dev/null)" 2>/dev/null && echo up`,
89  'avatar7-wanted',
90]
91// Take the relay only when nobody holds it: noclobber makes the owner file
92// the lock, so two sessions claiming at once leave one owner.
93export const claimArgv = (session: string): string[] =>
94  owned('avatar7-claim', `[ -d "${RELAY_SPOOL}" ] && (set -C; printf %s "$2" > "${RELAY_SPOOL}/owner") 2>/dev/null; true`, session)
95// Each prompt, from any session that could hold the relay, leaves its mark
96// under active/: the newest is where the user is. A session that ends takes
97// its mark away.
98export const promptedArgv = (session: string): string[] =>
99  owned('avatar7-prompted', `[ -d "${RELAY_SPOOL}" ] && mkdir -p "${RELAY_SPOOL}/active" && touch "${RELAY_SPOOL}/active/$2"; true`, session)
100export const forgetArgv = (session: string): string[] =>
101  owned('avatar7-forget', `rm -f "${RELAY_SPOOL}/active/$2"; true`, session)
102// The session prompted last, or nothing when none has been since the relay
103// started.
104export const latestArgv = (): string[] => ['sh', '-c', `ls -t "${RELAY_SPOOL}/active" 2>/dev/null | head -1`, 'avatar7-latest']
105// Take the relay (when it runs), or give it back (when this session has it).
106export const takeArgv = (session: string): string[] =>
107  owned('avatar7-take', `[ -d "${RELAY_SPOOL}" ] && printf %s "$2" > "${RELAY_SPOOL}/owner"`, session)
108// Given back, the page hears that nobody holds the relay rather than keeping
109// the last face it saw.
110export const releaseArgv = (session: string): string[] =>
111  owned(
112    'avatar7-release',
113    `${ownsRelay} && rm -f "${RELAY_SPOOL}/owner" && printf '{}' > "${RELAY_SPOOL}/state.part" && mv "${RELAY_SPOOL}/state.part" "${RELAY_SPOOL}/state.json"; true`,
114    session,
115  )
116export const mirrorArgv = (session: string): string[] =>
117  owned('avatar7-mirror', `d="${RELAY_SPOOL}"; ${ownsRelay} && cat > "$d/state.part" && mv "$d/state.part" "$d/state.json"`, session)
118// The page's buttons, queued by the relay as one JSON file each under cmd/:
119// read in order and removed, one per line.
120export const drainArgv = (): string[] => [
121  'sh',
122  '-c',
123  `for f in "${RELAY_SPOOL}"/cmd/*.json; do [ -f "$f" ] && cat "$f" && echo && rm -f "$f"; done; true`,
124]
125
126// A line's WAV: to the spool when this session holds the relay (Opus for the
127// trip, renamed in place so never served half written), otherwise played on
128// the host by `local`, a shell command reading the WAV as $1. The WAV goes
129// either way. Its name ends with the line's seq (`<ns>-<seq>.ogg`): the page
130// pairs each voice with its line exactly, whatever reaches it first.
131export const playArgv = (wav: string, session: string, local: string, seq = 0): string[] => [
132  'bash',
133  '-c',
134  `d="${RELAY_SPOOL}"; if ${ownsRelay}; then n="$d/$(date +%s%N)-$3"; ` +
135    // A 16 s line: 732 KB of WAV, 67 KB of Opus, 0.3 s to encode.
136    `if ffmpeg -loglevel error -nostdin -i "$1" -c:a libopus -b:a 32k -f ogg "$n.part"; then mv "$n.part" "$n.ogg"; ` +
137    `else cp "$1" "$n.part" && mv "$n.part" "$n.wav"; fi; ` +
138    `else ${local}; fi; rm -f "$1"`,
139  'avatar7-play',
140  wav,
141  session,
142  String(Math.max(0, Math.floor(seq))),
143]
144
145// The relay as one session sees it. `session` is this session's id, '' for a
146// background session, which never holds the relay.
147export type Relay = {
148  session: string
149  isHeld: boolean
150  // /avatar remote on: held whatever the next prompt's origin, until off.
151  isForced: boolean
152  // A session typed into over ssh: its user is elsewhere, like Remote Control's.
153  isSsh: boolean
154  // Taken because a page listens, not asked: given back when none does.
155  isByWanted: boolean
156  isMirroring: boolean
157  isDraining: boolean
158  mirrored: string
159}
160export const newRelay = (): Relay => ({ session: '', isHeld: false, isForced: false, isSsh: false, isByWanted: false, isMirroring: false, isDraining: false, mirrored: '' })
161
162// What the relay asks of avatar7: the face to show (undefined while there is
163// none), and what to do with a press from the page.
164export type RelayHost = {
165  face: () => Promise<Mirror> | undefined
166  press: (r: Remote) => Promise<void>
167}
168
169// Where a prompt sends the voice: away through the relay for one sent by
170// Remote Control or typed over ssh, or typed here while a page listens (the
171// phone asks for it by listening); back here for one typed at this terminal
172// otherwise (unless held by force).
173export const follows = (r: Relay, origin: string, isWanted = false): 'take' | 'give' | 'stay' =>
174  r.session === ''
175    ? 'stay'
176    : origin === 'bridge' || (origin === 'composer' && (r.isSsh || isWanted))
177      ? 'take'
178      : origin === 'composer' && !r.isForced
179        ? 'give'
180        : 'stay'
181
182// Between prompts, at each check: a page listening and nobody holding the
183// relay, this session claims it if it was prompted last; the last page gone (its grace over), a relay
184// this session took for it goes back. Forced, or taken by Remote Control or
185// ssh, it stays.
186// Of several sessions, only the one prompted last claims (`latest` empty:
187// none was since the relay started, the first to look takes it).
188export const wantedMove = (r: Relay, isWanted: boolean, latest = ''): 'claim' | 'give' | 'none' =>
189  r.session === ''
190    ? 'none'
191    : isWanted && !r.isHeld && (latest === '' || latest === r.session)
192      ? 'claim'
193      : !isWanted && r.isHeld && r.isByWanted && !r.isForced
194        ? 'give'
195        : 'none'
196
197// A session that ends gives the relay back: an owner gone for good would hold
198// the page on its last face, and keep the others' voices off it. A /clear goes
199// on in this process: a relay held by force stays held, and the next check
200// hands it to the new session id.
201export const givesOnEnd = (r: Relay, reason: string): boolean => r.session !== '' && !(reason === 'clear' && r.isForced)
202

1// A persona's memory across sessions, kept by a mem7 of its own (the
2// memory_url option; empty, the personas forget as before). Each persona is a
3// mem7 agent: it writes its exchanges with the user and its visits, reads its
4// own memories and the shared `world`, never another persona's (the scopes of
5// that mem7). At the start of a session, the persona on duty sums up what
6// happened since its last journal; the journal stays, the exchanges word for
7// word go after 30 days. Here: what is kept, the JSON-RPC bodies and the
8// parsing. The engine calls ($) stay in register.tsx, which the engine
9// requires: $ is never followed across an import.
10
11import type { Ask } from './speech'
12
13// The exchanges word for word, then only the journals.
14export const EPISODE_TTL_S = 30 * 24 * 3600
15// What a line recalls: a few exchanges matching what was said, the last
16// journals whatever was said.
17export const RECALL_EPISODES = 3
18export const RECALL_JOURNALS = 2
19// And older journals that match what was said: past two sessions, an
20// exchange is gone in 30 days, and only its journal still holds it.
21export const RECALL_OLD_JOURNALS = 2
22// An exchange recalled is cut to this many characters, a journal to more:
23// it is already the short form.
24export const RECALL_CHARS = 300
25export const JOURNAL_CHARS = 700
26// At most this many exchanges summed up in one journal.
27export const CONSOLIDATE_MAX = 40
28
29// The identity mem7 honours on a request that carries its token.
30export const META_AGENT = 'art.flux7/agent'
31
32export type Memory = { key: string; value: string; updated: string }
33
34// Where the persona's memory lives: the mem7 URL and the file holding its
35// token (MEM7_TOKEN=...). The token goes to curl through a file descriptor,
36// never on the command line, where ps would show it.
37export type MemoryAt = { url: string; envFile: string }
38
39export const memoryAt = (options: Record<string, unknown>): MemoryAt | null => {
40  const url = typeof options.memory_url === 'string' ? options.memory_url.trim().replace(/\/+$/, '') : ''
41  const env = typeof options.memory_env === 'string' ? options.memory_env.trim() : ''
42  return url === '' ? null : { url, envFile: env }
43}
44
45// The argv that posts one JSON-RPC body (on stdin) to mem7's /rpc.
46export const rpcArgv = (at: MemoryAt): string[] => [
47  'bash',
48  '-c',
49  'f=$1; case $f in "~/"*) f=$HOME/${f#"~/"};; esac; tok=$( [ -r "$f" ] && sed -n "s/^MEM7_TOKEN=//p" "$f" ); ' +
50    'curl -sS -m 5 -H @<(printf "Authorization: Bearer %s\\n" "$tok") -H "Content-Type: application/json" --data-binary @- "$2/rpc"',
51  'avatar7-mem7',
52  at.envFile,
53  at.url,
54]
55
56const call = (agent: string, name: string, args: Record<string, unknown>): string =>
57  JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name, arguments: args, _meta: { [META_AGENT]: agent } } })
58
59export const storeBody = (agent: string, key: string, value: string, tags: string[], ttl = 0): string =>
60  call(agent, 'memory_store', { key, value, tags, ...(ttl > 0 ? { ttl } : {}) })
61
62// Search in plain words over the persona's memories (mem7 scopes the reads).
63export const contextBody = (agent: string, query: string, tags: string[], limit: number): string =>
64  call(agent, 'memory_context', { query, mode: 'natural', tags, limit })
65
66// The most recent memories with these tags, whatever their words.
67export const recallBody = (agent: string, tags: string[], limit: number): string =>
68  call(agent, 'memory_recall', { tags, limit })
69
70// The text of a tools/call answer, or '' for an error or anything else.
71const resultText = (stdout: string): string => {
72  try {
73    const r = JSON.parse(stdout) as { result?: { content?: { text?: string }[]; isError?: boolean } }
74    if (r.result === undefined || r.result.isError === true) return ''
75    return r.result.content?.[0]?.text ?? ''
76  } catch {
77    return ''
78  }
79}
80
81// memory_context answers a JSON array inside the text.
82export const parseContext = (stdout: string): Memory[] => {
83  try {
84    const rows = JSON.parse(resultText(stdout)) as { key?: unknown; value?: unknown; updated?: unknown }[]
85    if (!Array.isArray(rows)) return []
86    return rows.flatMap(r =>
87      typeof r.key === 'string' && typeof r.value === 'string' ? [{ key: r.key, value: r.value, updated: typeof r.updated === 'string' ? r.updated : '' }] : [],
88    )
89  } catch {
90    return []
91  }
92}
93
94// memory_recall answers markdown: "## key", the value (lines and all), then
95// "Tags:", "Agent:", "Updated:" lines. Most recent first.
96export const parseRecall = (stdout: string): Memory[] => {
97  const text = resultText(stdout)
98  if (!text.startsWith('## ')) return []
99  return text
100    .split(/^## /m)
101    .filter(Boolean)
102    .flatMap(block => {
103      const nl = block.indexOf('\n')
104      const key = block.slice(0, nl).trim()
105      const rest = block.slice(nl + 1)
106      const tags = rest.search(/\nTags: [^\n]*\n(Agent: [^\n]*\n)?Updated: /)
107      if (nl < 0 || tags < 0) return []
108      const updated = /\nUpdated: (\S+)/.exec(rest.slice(tags))?.[1] ?? ''
109      return [{ key, value: rest.slice(0, tags), updated }]
110    })
111}
112
113// The lines that recall: those that answer the user or speak of the session,
114// and the opening of a visit. A tool's verdict stays immediate.
115export const recalls = (ask: Ask): boolean => {
116  if (ask === 'talk' || ask === 'question') return true
117  if ('duo' in ask) return ask.turn <= 1
118  return 'chat' in ask || 'answer' in ask || 'consult' in ask
119}
120
121// What the recall searches for: what the user said, else the visitor, else
122// the last prompt typed.
123export const queryFor = (ask: Ask, asked: string, other: string): string => {
124  if (typeof ask === 'object') {
125    if ('chat' in ask) return ask.chat
126    if ('consult' in ask) return ask.consult
127    if ('answer' in ask) return `${ask.question} ${ask.answer}`
128    if ('duo' in ask) return other
129  }
130  return asked
131}
132
133// What the model reads of the past, after the prompt; '' when nothing came back.
134// The journals a line reads: the last ones and the older ones that matched,
135// each once, most recent first.
136export const journalsFor = (latest: Memory[], matched: Memory[]): Memory[] => {
137  const seen = new Set(latest.map(j => j.key))
138  return [...latest, ...matched.filter(j => !seen.has(j.key)).slice(0, RECALL_OLD_JOURNALS)].sort((a, b) => (a.updated < b.updated ? 1 : -1))
139}
140
141export const memoryNote = (journals: Memory[], episodes: Memory[]): string => {
142  if (journals.length === 0 && episodes.length === 0) return ''
143  const cut = (s: string, max: number): string => {
144    const one = s.replace(/\s+/g, ' ').trim()
145    return one.length > max ? `${one.slice(0, max)}...` : one
146  }
147  const parts = ['\nWhat you remember from earlier sessions (use it only if it fits; never invent more):']
148  for (const j of [...journals].reverse()) parts.push(`- (${j.updated.slice(0, 10)}, your journal) ${cut(j.value, JOURNAL_CHARS)}`)
149  for (const e of episodes) parts.push(`- (${e.updated.slice(0, 10)}) ${cut(e.value, RECALL_CHARS)}`)
150  return parts.join('\n')
151}
152
153// The exchange kept for a line, or null when the line keeps none: the user's
154// words and the persona's answer, as they were said.
155export const episodeOf = (ask: Ask, name: string, text: string): { value: string; kind: string } | null => {
156  if (typeof ask !== 'object' || text === '') return null
157  if ('chat' in ask) return { value: `User: ${ask.chat}\n${name}: ${text}`, kind: 'chat' }
158  if ('consult' in ask) return { value: `User asked your opinion: ${ask.consult}\n${name}: ${text}`, kind: 'consult' }
159  if ('answer' in ask) return { value: `${name} asked: ${ask.question}\nUser: ${ask.answer}\n${name}: ${text}`, kind: 'answer' }
160  return null
161}
162
163// A visit, kept whole by each of the two at its last line, each in its own memory.
164export const visitOf = (history: string[], last: string, other: string): string => `Visit with ${other}:\n${[...history, last].join('\n')}`
165
166// Keys sort by time; the milliseconds keep two lines of one second apart.
167const stamp = (at: Date): string => at.toISOString().replace(/[-:]/g, '').replace('.', '')
168export const episodeKey = (agent: string, at: Date): string => `${agent}.ep.${stamp(at)}`
169export const journalKey = (agent: string, at: Date): string => `${agent}.journal.${stamp(at)}`
170
171// The exchanges not yet summed up: newer than the last journal, oldest first.
172export const unsummed = (episodes: Memory[], journals: Memory[]): Memory[] => {
173  const since = journals.reduce((m, j) => (j.updated > m ? j.updated : m), '')
174  return episodes
175    .filter(e => e.updated > since)
176    .sort((a, b) => (a.updated < b.updated ? -1 : 1))
177    .slice(-CONSOLIDATE_MAX)
178}
179
180// The model's answer when there is nothing to keep.
181export const NOTHING = 'NOTHING'
182
183export const journalPrompt = (episodes: Memory[], user: string): string =>
184  `Here are your exchanges since your last journal, oldest first:\n` +
185  episodes.map(e => `(${e.updated.slice(0, 10)}) ${e.value}`).join('\n\n') +
186  `\n\nWrite your journal entry: two or three sentences, first person, in English, on what you learned about ${user} ` +
187  `and what happened between you and the others. Keep what would matter next time; drop the small talk. ` +
188  `Never write about money, family or health, even if they came up. If nothing is worth keeping, answer only ${NOTHING}.`
189
190export const journalFrom = (r: { isAnswered: true; text: string } | { isAnswered: false }): string | null => {
191  if (!r.isAnswered) return null
192  const text = r.text.replace(/\s+/g, ' ').trim()
193  return text === '' || text.startsWith(NOTHING) ? null : text
194}
195

1// What the avatar says and when: the kinds of line (Ask), the queue that
2// orders them, the persona's text, and how a line is asked of the model and
3// kept. Pure: register.tsx reads the session, calls the model and plays the
4// voice, since the engine follows $ into nothing imported.
5
6import type { AmbientLayer } from './ambient'
7import type { Mood } from './mood'
8
9// Rules every persona keeps, whatever its character: added to each prompt.
10export const STYLE = ' The user may write in French; you always answer in English. Never flattering. No quotes, no emoji, no em dash.'
11
12// When poked, the avatar reads the last messages of the conversation, each cut
13// to this many characters.
14export const TALK_MESSAGES = 6
15export const TALK_CHARS = 300
16// Asked for an opinion, it reads further back and may say more.
17export const CONSULT_MESSAGES = 12
18export const CONSULT_CHARS = 600
19export const CONSULT_CHARS_ASKED = 400
20// A chat remembers its last six exchanges, per persona.
21export const CHAT_LINES = 12
22
23// The avatar's last lines, given back to the model so it does not repeat
24// its own wording over a long session.
25export const RECENT_LINES = 3
26
27export const recentNote = (lines: string[]): string =>
28  lines.length === 0 ? '' : `\nYour last lines, do not reuse their wording or openings:\n${lines.map(l => `- ${l}`).join('\n')}`
29
30// A line to speak: an event in a mood, or the user's poke, which reads the
31// conversation first; or, now and then, a moment of the persona's own story,
32// a question it puts to the user, and its reaction to the user's answer.
33export type Ask =
34  | { mood: Mood; event: string; tool?: string; after?: number }
35  | 'talk'
36  | { story: string; mood: Mood }
37  | 'question'
38  | { question: string; answer: string }
39  | { consult: string }
40  | { chat: string }
41  | Duo
42  | { greet: string }
43
44// One turn of a dialogue with a visiting persona: even turns are the host's,
45// odd ones the guest's; `history` holds the lines said so far, by name.
46export type Duo = { duo: string; turn: number; topic: 'session' | 'stories'; history: string[] }
47export const DUO_TURNS = 6
48
49// A guest for the persona on duty: any other avatar, its `friends` three
50// times as likely as the rest, picked by `roll` in [0, 1).
51export const FRIEND_WEIGHT = 3
52
53export const pickGuest = (avatars: string[], current: string, roll: number, friends: string[] = []): string | undefined => {
54  const others = avatars.filter(a => a !== current)
55  const weights = others.map(a => (friends.includes(a) ? FRIEND_WEIGHT : 1))
56  let left = roll * weights.reduce((sum, w) => sum + w, 0)
57  for (const [i, a] of others.entries()) {
58    left -= weights[i] ?? 1
59    if (left < 0) return a
60  }
61  return others.at(-1)
62}
63
64export type Story = { story: string; mood: Exclude<Mood, 'idle'> }
65
66// The persona's own lines for when the model gives none. A question or a
67// dialogue turn has none: it is simply not said. The string asks are tested
68// first, since `in` on a string throws (it once silenced every poke).
69export const fallbackPool = (ask: Ask, fallback: Persona['fallback']): string[] => {
70  if (ask === 'talk') return fallback.idle
71  if (ask === 'question') return []
72  if ('duo' in ask) return []
73  if ('story' in ask || 'answer' in ask || 'greet' in ask) return fallback.idle
74  // A stock line is no answer to a question: better silent.
75  if ('consult' in ask || 'chat' in ask) return []
76  return ask.mood === 'wait' ? (fallback.wait ?? fallback.watch) : fallback[ask.mood as Exclude<Mood, 'idle' | 'wait'>]
77}
78
79// One of the persona's own events, or undefined when it has none: a story
80// or a question, even odds when it has both. `roll` and `pick` are in [0, 1).
81export const pickEvent = (stories: Story[], canAsk: boolean, roll: number, pick: number): Ask | undefined => {
82  const story = stories[Math.floor(pick * stories.length)]
83  if (story !== undefined && (!canAsk || roll < 0.5)) return { story: story.story, mood: story.mood }
84  return canAsk ? 'question' : undefined
85}
86
87// Rare: one event every 20 to 40 minutes, and only after a minute of quiet.
88export const EVENT_MIN_FRAMES = (20 * 60_000) / 66
89export const EVENT_SPAN_FRAMES = (20 * 60_000) / 66
90export const EVENT_QUIET_FRAMES = 60_000 / 66
91// A question unanswered for five minutes goes away.
92export const QUESTION_FRAMES = (5 * 60_000) / 66
93
94// The lines waiting for the voice, most urgent first: the user's poke, a
95// refusal, a failure or a wait, then calm news; among equals the oldest. Full,
96// the least urgent goes; a line that waited too long is dropped unspoken.
97export type Queued = { ask: Ask; rank: number; at: number }
98export const QUEUE_MAX = 4
99// ~20 s of frames: past that, a line speaks of something already gone.
100export const STALE_FRAMES = 300
101
102// The persona's own events come last; the user's poke and answer first.
103export const rankOf = (ask: Ask): number => {
104  if (ask === 'talk') return 4
105  if (ask === 'question') return 0
106  if ('duo' in ask) return 0
107  if ('answer' in ask || 'greet' in ask || 'consult' in ask || 'chat' in ask) return 4
108  if ('story' in ask) return 0
109  return ask.mood === 'deny' ? 3 : ask.mood === 'error' || ask.mood === 'wait' ? 2 : 1
110}
111
112export const enqueue = (queue: Queued[], ask: Ask, at: number): Queued[] =>
113  [...queue, { ask, rank: rankOf(ask), at }].sort((a, b) => b.rank - a.rank || a.at - b.at).slice(0, QUEUE_MAX)
114
115// A dialogue's turn never goes stale either: dropped while it waited (behind
116// the user's chats, which rank higher), it left the guest on stage for good,
117// since only a turn said ends the visit, and no duo could start again.
118export const fresh = (queue: Queued[], now: number): Queued[] =>
119  queue.filter(
120    q =>
121      q.ask === 'talk' ||
122      (typeof q.ask === 'object' && ('answer' in q.ask || 'greet' in q.ask || 'consult' in q.ask || 'chat' in q.ask || 'duo' in q.ask)) ||
123      now - q.at <= STALE_FRAMES,
124  )
125
126export type Persona = {
127  name: string
128  voice: string
129  rate: number
130  pitch?: number
131  // Piper voice name and pace (length-scale, under 1 is faster).
132  // fx: an ffmpeg audio filter run on the WAV, from aresample=22050 so pitch
133  // shifts by asetrate hold whatever the voice's own rate.
134  // speaker: for a multi-speaker model, the speaker's id (speaker_id_map).
135  piper?: { voice: string; lengthScale?: number; fx?: string; speaker?: number }
136  color: string
137  eyes: { x: number; y: number; rx: number; ry: number }[]
138  mouth: { x: number; y: number; half: number } | null
139  greeting: string
140  persona: string
141  fallback: Record<Exclude<Mood, 'wait'>, string[]> & { wait?: string[] }
142  nobody?: string
143  // The artists this persona would put on; jukebox7 plays them.
144  station?: string[]
145  // Moments of its own story it lives now and then, and the bent of the
146  // questions it asks the user; either may be absent.
147  events?: Story[]
148  asks?: string
149  // The avatars it gets on with, or against: they visit it more often.
150  friends?: string[]
151  // Pixel weather in the black around the face (hooks/ambient.ts).
152  ambient?: AmbientLayer[]
153  // The portrait's luminance (0-255) under which the scene shows through it;
154  // lower for a face with dark hair, which would otherwise turn see-through.
155  cutout?: number
156}
157
158// `{, user}` in a persona's text becomes ", <name>": the user_name option,
159// else the persona's `nobody`, else nothing (the braces and their text drop).
160export const personalize = (text: string, name: string, nobody: string | undefined): string =>
161  text.replace(/\{([^{}]*)user([^{}]*)\}/g, (_, before: string, after: string) => {
162    const who = name !== '' ? name : (nobody ?? '')
163    return who === '' ? '' : `${before}${who}${after}`
164  })
165
166
167// What a line reads before it is written: how many of the session's last
168// messages, each cut to how many characters; null when it reads none.
169export const reads = (ask: Ask): { count: number; chars: number } | null => {
170  if (ask === 'talk' || ask === 'question') return { count: TALK_MESSAGES, chars: TALK_CHARS }
171  if ('consult' in ask) return { count: CONSULT_MESSAGES, chars: CONSULT_CHARS }
172  if ('duo' in ask && ask.turn === 0 && ask.topic === 'session') return { count: TALK_MESSAGES, chars: TALK_CHARS }
173  return null
174}
175
176// An opinion (a consult, a chat) keeps all its sentences and gets more tokens.
177export const isOpinion = (ask: Ask): boolean => typeof ask === 'object' && ('consult' in ask || 'chat' in ask)
178
179export type Asking = {
180  voice: Persona
181  // In a dialogue, the one spoken to.
182  other: string
183  // The session's last messages, as reads() asked; '' when none.
184  conversation: string
185  // This persona's chat with the user so far, for a chat.
186  chatPast: string[]
187  // What the user asked in this turn, judged against.
188  asked: string
189  // The avatar's own last lines, not to repeat.
190  recent: string[]
191}
192
193// The prompt for a line, or null for a greeting, spoken as written.
194export const promptFor = (ask: Ask, c: Asking): string | null => {
195  let prompt: string
196  if (ask === 'talk') {
197    prompt = `The user pokes you and wants your take on where the conversation stands.\nLast messages, oldest first:\n${c.conversation}`
198  } else if (ask === 'question') {
199    prompt =
200      `Ask the user ONE short question, then stop: philosophical, from your own story, or technical, ` +
201      `about the work in the conversation below. Your bent: ${c.voice.asks ?? 'what your character would wonder'}.\n` +
202      `Last messages, oldest first:\n${c.conversation}`
203  } else if ('story' in ask) {
204    prompt = `A moment of your own story happens now, unrelated to the tool calls: ${ask.story}. Say what you live or feel in it.`
205  } else if ('duo' in ask) {
206    const about =
207      ask.topic === 'session' ? `the work going on in this terminal session. Last messages, oldest first:\n${c.conversation}` : 'where your two stories cross'
208    prompt =
209      ask.turn === 0
210        ? `${c.other} visits your terminal. Open a short exchange with them, speaking to them directly, about ${about}`
211        : `You are talking with ${c.other}. The exchange so far:\n${ask.history.join('\n')}\n` +
212          (ask.turn === DUO_TURNS - 1 ? 'Close the exchange in one sentence, to them.' : 'Answer them in one sentence.')
213  } else if ('consult' in ask) {
214    prompt =
215      `The user is working with an AI assistant in this terminal session and asks your opinion on it.\n` +
216      `Last messages, oldest first:\n${c.conversation}\n` +
217      `The user asks you: ${ask.consult}\n` +
218      `Answer in character, in two or three sentences: take a position, name what you would change or keep. ` +
219      `You may end on one question back.`
220  } else if ('chat' in ask) {
221    prompt =
222      `The user talks to you personally, about whatever they like, not about the terminal session. ` +
223      `This time you may use two or three sentences instead of one.\n` +
224      (c.chatPast.length > 0 ? `Your conversation so far, oldest first:\n${c.chatPast.join('\n')}\n` : '') +
225      `The user says: ${ask.chat}\n` +
226      `Answer in character, from your own world and what you know of the user; you may ask one question back.`
227  } else if ('answer' in ask) {
228    prompt = `You asked the user: ${ask.question}\nThe user answered: ${ask.answer}\nReact in character: challenge it, approve it your way, or ask one follow-up.`
229  } else if ('greet' in ask) {
230    return null
231  } else {
232    prompt = c.asked === '' ? `Event: ${ask.event}` : `The user asked: ${c.asked}\nEvent: ${ask.event}`
233  }
234  return prompt + recentNote(c.recent)
235}
236
237// The line kept from the model's answer: an opinion whole, any other line its
238// first; with no answer, one of the persona's stock lines (`roll` picks it),
239// or '' where none fits.
240export const lineFrom = (r: { isAnswered: true; text: string } | { isAnswered: false }, ask: Ask, voice: Persona, roll: number, userName: string): string => {
241  if (r.isAnswered) return isOpinion(ask) ? r.text.replace(/\s+/g, ' ').trim() : (r.text.trim().split('\n')[0] ?? '')
242  const pool = fallbackPool(ask, voice.fallback)
243  return personalize(pool[roll % pool.length] ?? '', userName, voice.nobody)
244}
245

1// What the avatar hears besides tool calls: the slash commands that change
2// the session, the other mods' announce and say, and the run of like
3// outcomes the calls make. Pure.
4
5import type { Announce, Say } from '../types'
6import type { Mood } from './mood'
7
8// How much of the user's last prompt the avatar reads, so it judges a call
9// against what was asked rather than the bare gesture.
10export const ASKED_CHARS = 200
11
12// The slash commands the avatar reacts to, each with what it means: those
13// that change the session or mark a moment. Any other (a look at /context,
14// /mcp, a mod's own pane) passes in silence. A manual /compact is heard here;
15// an automatic one through session.compact.
16export const COMMANDS: Record<string, { mood: Exclude<Mood, 'idle'>; means: string }> = {
17  clear: { mood: 'watch', means: 'wipes the whole conversation and starts over; say farewell to what is gone' },
18  compact: { mood: 'watch', means: 'compacts the conversation: the assistant keeps a summary and forgets the rest' },
19  fast: { mood: 'watch', means: 'toggles fast mode for the assistant' },
20  rewind: { mood: 'error', means: 'rewinds the conversation to undo what went wrong' },
21  resume: { mood: 'watch', means: 'resumes an older session' },
22  brief: { mood: 'watch', means: 'asks for the morning brief: weather, mail, news; the day starts' },
23  veille: { mood: 'watch', means: 'publishes the AI and streaming watch to the team Discord' },
24  document: { mood: 'watch', means: 'publishes a working document as a page' },
25  galerie: { mood: 'watch', means: 'publishes the latest image renders as a gallery' },
26  'mesh-approve': { mood: 'wait', means: 'decides the approvals mesh7 holds for a human' },
27  'code-review': { mood: 'watch', means: 'puts the code under review' },
28  'security-review': { mood: 'watch', means: 'puts the code under a security review' },
29  code7: { mood: 'watch', means: 'takes the keyboard back: the user writes the code, the assistant only teaches' },
30}
31
32// The line a command earns, or undefined when it passes in silence.
33export const commandEvent = (command: string, args: string): { mood: Exclude<Mood, 'idle'>; event: string } | undefined => {
34  const known = COMMANDS[command]
35  if (known === undefined) return undefined
36  const typed = args.replace(/\s+/g, ' ').trim().slice(0, ASKED_CHARS)
37  return { mood: known.mood, event: `the user runs /${command}${typed === '' ? '' : ` ${typed}`}, which ${known.means}` }
38}
39
40// Whether a write landed. The host answers a plain write without `isSet`
41// (2.1.288, despite StateSetResult): only a write another one beat says false.
42export const landed = (done: unknown): boolean => (done as { isSet?: boolean } | undefined)?.isSet !== false
43
44// A write another mod made, as an announce the avatar keeps, or undefined:
45// its own key `announce`, a calm or amber face, and the event in words.
46export const heard = (w: { plugin: string; key: string; value: unknown }): Announce | undefined => {
47  if (w.key !== 'announce' || w.plugin === 'avatar7') return undefined
48  const a = w.value as Partial<Announce> | undefined
49  return a !== undefined && (a.mood === 'watch' || a.mood === 'error') && typeof a.event === 'string'
50    ? { mood: a.mood, event: a.event }
51    : undefined
52}
53
54// A `say` another mod published: speak it now, no toast needed.
55export const heardSay = (w: { plugin: string; key: string; value: unknown }): Say | undefined => {
56  if (w.key !== 'say' || w.plugin === 'avatar7') return undefined
57  const a = w.value as Partial<Say> | undefined
58  return a !== undefined &&
59    (a.mood === 'watch' || a.mood === 'error' || a.mood === 'deny' || a.mood === 'wait') &&
60    typeof a.event === 'string' &&
61    typeof a.at === 'number'
62    ? {
63        mood: a.mood,
64        event: a.event,
65        at: a.at,
66        ...(typeof a.tool === 'string' ? { tool: a.tool } : {}),
67        ...(a.hold === true ? { hold: true } : {}),
68        ...(a.release === true ? { release: true } : {}),
69      }
70    : undefined
71}
72
73// The run of like outcomes the last calls made: a third denial in a row, or
74// a success after a string of failures, is news the line should carry.
75export type Streak = { mood: 'watch' | 'deny' | 'error'; count: number }
76
77export const nextStreak = (s: Streak, now: Streak['mood']): Streak =>
78  s.mood === now ? { mood: now, count: s.count + 1 } : { mood: now, count: 1 }
79
80export const ordinal = (n: number): string =>
81  `${n}${n % 100 >= 11 && n % 100 <= 13 ? 'th' : (['th', 'st', 'nd', 'rd'][n % 10] ?? 'th')}`
82
83export const streakNote = (before: Streak, after: Streak): string => {
84  const what = (m: Streak['mood'], n: number) => (m === 'deny' ? 'denial' : 'failure') + (n > 1 ? 's' : '')
85  if (after.mood !== 'watch' && after.count >= 2) return ` (${ordinal(after.count)} ${what(after.mood, 1)} in a row)`
86  if (after.mood === 'watch' && before.mood !== 'watch' && before.count >= 3) {
87    return ` (first success after ${before.count} ${what(before.mood, before.count)} in a row)`
88  }
89  return ''
90}
91

1// The voice: a line synthesized by Piper (or spoken by SAPI when Piper is
2// missing), and a WAV played on the host. Pure: argv only; register.tsx runs
3// them, since the engine follows $ into nothing imported.
4
5import type { Persona } from './speech'
6
7export const POWERSHELL = '/mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe'
8// A persona with a `pitch` (-10 to 10) speaks through the SAPI COM voice,
9// which takes the pitch as XML; the text is escaped for it.
10export const speakScript = (who: { voice: string; rate: number; pitch?: number }, vol: number) =>
11  '[Console]::InputEncoding=[Text.Encoding]::UTF8; $t=[Console]::In.ReadToEnd(); ' +
12  (who.pitch === undefined
13    ? 'Add-Type -AssemblyName System.Speech; $s=New-Object System.Speech.Synthesis.SpeechSynthesizer; ' +
14      `$s.SelectVoice("${who.voice}"); $s.Rate=${who.rate}; $s.Volume=${vol}; $s.Speak($t)`
15    : '$q=[char]34; $v=New-Object -ComObject SAPI.SpVoice; ' +
16      `$v.Voice=($v.GetVoices() | ? { $_.GetDescription() -like "${who.voice}*" } | select -First 1); ` +
17      `$v.Rate=${who.rate}; $v.Volume=${vol}; ` +
18      `[void]$v.Speak("<pitch absmiddle=" + $q + "${who.pitch}" + $q + ">" + [Security.SecurityElement]::Escape($t) + "</pitch>", 8)`)
19
20// A persona with a `piper` voice speaks through Piper (local neural TTS, on
21// the CPU, in WSL), its WAV played by Windows through SAPI; SAPI stays the voice
22// when Piper or the model is missing. Piper lives in ~/.local/share/piper:
23// .venv with piper-tts, voices/<name>.onnx. A private voice at
24// custom/<avatar id>.onnx wins over the persona's (its own pace, no filter
25// unless custom/<id>.fx holds one): voices one keeps out of the repo.
26// Synthesis and playback are two runs so the line can be typed with the voice:
27// this one prints the WAV's path and its length in seconds, or speaks through
28// SAPI itself and prints nothing when Piper is missing.
29export const PIPER = '$HOME/.local/share/piper'
30export const synthArgv = (id: string, who: Persona, vol: number): string[] => [
31  'bash',
32  '-c',
33  [
34    't=$(cat)',
35    `m="${PIPER}/voices/$1.onnx"`,
36    `c="${PIPER}/custom/$6"`,
37    'if [ -n "$6" ] && [ -f "$c.onnx" ]; then m="$c.onnx"; set -- "$c" "$2" 1 "$4" "$(cat "$c.fx" 2>/dev/null)" "$6" ""; fi',
38    `if [ -n "$1" ] && [ -f "$m" ] && [ -x "${PIPER}/.venv/bin/python" ]; then`,
39    '  w=$(mktemp --suffix=.wav); f=""; keep=""',
40    // Every temporary goes on exit, but the WAV handed to the player.
41    '  trap \'[ -n "$keep" ] || rm -f "$w"; rm -f "$f"\' EXIT',
42    // With ffmpeg the user's volume is applied after the leveling below.
43    '  v="$2"; command -v ffmpeg >/dev/null && v=1',
44    // $7: a speaker of a multi-speaker model (vctk, libritts_r), by its id.
45    // A Piper that fails speaks through SAPI, as a missing one does.
46    `  printf %s "$t" | "${PIPER}/.venv/bin/python" -m piper -m "$m" \${7:+-s "$7"} -f "$w" --volume "$v" --length-scale "$3" 2>/dev/null || { printf %s "$t" | "${POWERSHELL}" -NoProfile -Command "$4"; exit 0; }`,
47    // The persona's ffmpeg filter (pitch, metal, glitch), skipped without ffmpeg.
48    '  if [ -n "$5" ] && command -v ffmpeg >/dev/null; then',
49    '    f=$(mktemp --suffix=.wav)',
50    '    ffmpeg -loglevel error -y -i "$w" -af "$5" "$f" && mv "$f" "$w"',
51    '  fi',
52    // Every voice leveled to the same loudness, then the user's volume.
53    '  if command -v ffmpeg >/dev/null; then',
54    '    f=$(mktemp --suffix=.wav)',
55    '    ffmpeg -loglevel error -y -i "$w" -af "loudnorm=I=-18:TP=-2:LRA=11,aresample=22050,volume=$2" "$f" && mv "$f" "$w"',
56    '  fi',
57    `  "${PIPER}/.venv/bin/python" -c 'import sys, wave; w = wave.open(sys.argv[1]); print(sys.argv[1]); print(w.getnframes() / w.getframerate())' "$w" && keep=1`,
58    'else',
59    `  printf %s "$t" | "${POWERSHELL}" -NoProfile -Command "$4"`,
60    'fi',
61  ].join('\n'),
62  'avatar7-speak',
63  who.piper?.voice ?? '',
64  String(Math.max(0, Math.min(100, vol)) / 100),
65  String(who.piper?.lengthScale ?? 1),
66  speakScript(who, vol),
67  who.piper?.fx ?? '',
68  id,
69  who.piper?.speaker === undefined ? '' : String(who.piper.speaker),
70]
71
72// SAPI plays the WAV on the host: SoundPlayer on a \\wsl.localhost path can
73// fall silent (returns at once, no error) while SAPI still reads it. The
74// relay, when this session holds it, takes the WAV instead (relay.ts).
75export const SAPI_PLAY = `"${POWERSHELL}" -NoProfile -Command "\\$v=New-Object -ComObject SAPI.SpVoice; \\$s=New-Object -ComObject SAPI.SpFileStream; \\$s.Open('$(wslpath -w "$1")'); [void]\\$v.SpeakStream(\\$s); \\$s.Close()"`
76
77// How long PowerShell takes to start a detached playback, at most (measured
78// ~0.3 s warm, more cold): the voice is held this much past the WAV's length.
79export const PLAY_START_MS = 900
80
81// A command run in its own session: the engine kills a module's children on
82// reload, and a line half spoken was cut with them.
83export const detachedArgv = (argv: string[]): string[] => ['bash', '-c', 'setsid "$0" "$@" </dev/null >/dev/null 2>&1 &', ...argv]
84

1// The face as the pane draws it: the portrait tinted by the mood, glitched on
2// a refusal, scanlined, framed like a comm window and cut out over the scene
3// behind it. Pure: register.tsx hands it a View, once per frame, and blits
4// the cells.
5
6import { ambientPixel, QUAD, type AmbientLayer, type Field } from './ambient'
7import type { Mood } from './mood'
8import type { Hd } from './hd'
9
10// Portraits are baked by tools/bake.py into W x H raw RGB pixels; each cell
11// is an upper half block, so two pixel rows per cell row.
12export const W = 64
13export const H = 64
14
15// The comm window's corner brackets, in face pixels.
16const BRACKET = 7
17
18export const TINT: Record<Mood, number> = {
19  idle: 0x000000,
20  watch: 0x00e5ff,
21  deny: 0xff2a6d,
22  error: 0xffb000,
23  wait: 0x7a5cff,
24}
25
26// Cheap deterministic noise for the glitch.
27export const noise = (a: number, b: number): number => {
28  let n = (a * 374761393 + b * 668265263) | 0
29  n = Math.imul(n ^ (n >>> 13), 1274126177)
30  return ((n ^ (n >>> 16)) >>> 0) / 4294967296
31}
32
33// `hd`: the same faces as real images (hd.ts), where the terminal draws them
34// and tools/bake_hd.py has baked them; null otherwise.
35export type Faces = { base: Uint8Array; talk: Uint8Array | null; deny: Uint8Array | null; hd?: Hd | null }
36
37// Which face to draw now: the frown on a refusal or a failure, the mouth
38// flapping at an uneven pace while the voice is heard, else the portrait.
39export const pickFace = <T,>(faces: { base: T; talk: T | null; deny: T | null }, mood: string, isSpeaking: boolean, flap: number): T =>
40  (mood === 'deny' || mood === 'error') && faces.deny !== null
41    ? faces.deny
42    : isSpeaking && faces.talk !== null && flap < 0.55
43      ? faces.talk
44      : faces.base
45
46// Everything a frame of the face depends on.
47export type View = {
48  frame: number
49  frameMs: number
50  mood: Mood
51  // The faces of whoever is shown (the guest's during a visit), and that
52  // persona's frame color, cutout and weather; null while none is loaded.
53  faces: Faces | null
54  persona: { color?: string; cutout?: number } | null
55  isHeard: boolean
56  // How hard a refusal glitches the portrait: 1 for a call denied, less for
57  // a refusal the persona acts in a scene of its own.
58  glitch: number
59  // The face's side in output pixels.
60  size: number
61  layers: AmbientLayer[]
62  field: Field
63  ambLeft: number
64  ambT: number
65}
66
67// A portrait pixel, x and y in W x H.
68export const facePixel = (v: View, x: number, y: number): number => {
69  const t = v.frame * (v.frameMs / 1000)
70  const isGlitch = v.mood === 'deny' && noise(v.frame, y >> 2) < 0.35 * v.glitch
71  const gx = isGlitch ? Math.min(W - 1, Math.max(0, x + Math.round((noise(y, v.frame) - 0.5) * 10 * v.glitch))) : x
72
73  const img = v.faces === null ? null : pickFace(v.faces, v.mood, v.isHeard, noise(v.frame >> 2, 7))
74  if (img === null || v.persona === null) return noise(x * 7 + v.frame, y) < 0.3 ? 0x1a2a22 : 0x020806
75
76  const i = (y * W + gx) * 3
77  let r = img[i] ?? 0
78  let g = img[i + 1] ?? 0
79  let b = img[i + 2] ?? 0
80  // No eye glow, blink or pulse for now: on several portraits the ellipses
81  // missed the eyes and read as smudges. `eyes` and `mouth` stay in each
82  // persona.json for a better effect.
83  let k = 1
84
85  // Mood: pull the portrait toward the mood's color by its luminance.
86  if (v.mood !== 'idle') {
87    const lum = (0.3 * r + 0.59 * g + 0.11 * b) / 255
88    const tint = TINT[v.mood]
89    const mix = v.mood === 'watch' ? 0.3 : v.mood === 'wait' ? 0.45 : 0.65
90    r = r * (1 - mix) + ((tint >> 16) & 0xff) * lum * 1.3 * mix
91    g = g * (1 - mix) + ((tint >> 8) & 0xff) * lum * 1.3 * mix
92    b = b * (1 - mix) + (tint & 0xff) * lum * 1.3 * mix
93  }
94
95  // Holding its breath while a human decides.
96  if (v.mood === 'wait') k *= 0.8 + 0.2 * Math.sin(t * 2.5)
97
98  // Snow when glitching; the scanlines are drawn at the output size.
99  if (isGlitch && noise(x, y + v.frame) < 0.04 * v.glitch) return 0xffffff
100
101  const c = (n: number) => Math.min(255, Math.round(n * k))
102  return (c(r) << 16) | (c(g) << 8) | c(b)
103}
104
105// One output pixel averages the block of portrait pixels it covers, so the
106// face shrinks with the pane and keeps its features.
107export const faceSample = (v: View, ox: number, oy: number): number => {
108  const { size } = v
109  const x0 = Math.floor((ox * W) / size)
110  const x1 = Math.max(x0 + 1, Math.floor(((ox + 1) * W) / size))
111  const y0 = Math.floor((oy * H) / size)
112  const y1 = Math.max(y0 + 1, Math.floor(((oy + 1) * H) / size))
113  let r = 0
114  let g = 0
115  let b = 0
116  for (let y = y0; y < y1; y++) {
117    for (let x = x0; x < x1; x++) {
118      const p = facePixel(v, x, y)
119      r += (p >> 16) & 0xff
120      g += (p >> 8) & 0xff
121      b += p & 0xff
122    }
123  }
124  const n = (x1 - x0) * (y1 - y0)
125  let k = oy % 2 === 1 ? 0.7 : 1
126  if (oy === Math.floor((v.frame * 0.8 * size) / H) % size) k *= 1.35
127  const c = (s: number) => Math.min(255, Math.round((s / n) * k))
128  const drawn = (c(r) << 16) | (c(g) << 8) | c(b)
129  // A comm window's frame: bright brackets at the corners, a faint line
130  // along the edges, in the persona's color or the mood's.
131  const edge = Math.min(ox, oy, size - 1 - ox, size - 1 - oy)
132  if (edge === 0 && v.layers.length > 0) {
133    const isCorner = Math.min(ox, size - 1 - ox) < BRACKET && Math.min(oy, size - 1 - oy) < BRACKET
134    const tone = v.mood === 'idle' ? parseInt((v.persona?.color ?? '#00ff9c').slice(1), 16) : TINT[v.mood]
135    const kk = isCorner ? 1 : 0.3
136    return (Math.round(((tone >> 16) & 0xff) * kk) << 16) | (Math.round(((tone >> 8) & 0xff) * kk) << 8) | Math.round((tone & 0xff) * kk)
137  }
138  // The portrait's dark background lets the scene behind it through, by
139  // degrees so its edge does not ring.
140  if (v.layers.length === 0) return drawn
141  const cutout = v.persona?.cutout ?? 12
142  const alpha = Math.min(1, Math.max(0, ((0.3 * r + 0.59 * g + 0.11 * b) / n - cutout) / (cutout * 2 + 4)))
143  if (alpha === 1) return drawn
144  const back = ambientPixel(v.layers, v.field, (v.ambLeft + ox) * QUAD, oy, v.ambT, v.mood === 'deny')
145  const mix = (shift: number) => Math.round(((drawn >> shift) & 0xff) * alpha + ((back >> shift) & 0xff) * (1 - alpha)) << shift
146  return mix(16) | mix(8) | mix(0)
147}
148
149// The Raster's cells for the face: one upper half block per cell, its two
150// pixels as foreground and background, base64 as the Raster takes them.
151export const faceCells = (v: View): string => {
152  const rows = v.size / 2
153  const words = new Uint32Array(v.size * rows * 3)
154  for (let row = 0; row < rows; row++) {
155    for (let x = 0; x < v.size; x++) {
156      const i = (row * v.size + x) * 3
157      words[i] = 0x2580
158      words[i + 1] = faceSample(v, x, row * 2)
159      words[i + 2] = faceSample(v, x, row * 2 + 1)
160    }
161  }
162  return new Uint8Array(words.buffer).toBase64()
163}
164

1// The pane as real images, where the terminal draws pictures (kitty,
2// Ghostty): the band of the face and the band under the text, each the pane
3// wide, in art pixels of a chosen grain. The weather (ambient.ts) is drawn on
4// its own field as the half blocks draw it; the scene on that field made finer
5// by the grain; the portrait at its baked resolution cut out over them,
6// tinted by the mood, torn on a refusal, scanlined and framed like a comm
7// window. Pure, as draw.ts is: register.tsx hands it an HdView, writes each
8// picture once and swaps them as the weather loops.
9
10import { ambientPixel, QUAD, type AmbientLayer, type Field } from './ambient'
11import { noise, pickFace, TINT } from './draw'
12import type { Mood } from './mood'
13
14// Output pixels per terminal column; a row is twice as tall. Kitty scales
15// the picture to the cells, so this is the detail kept, not a size on screen.
16export const PX = 6
17
18// The tears a refusal cycles through: a finite set, so each is made once.
19export const GLITCH_STEPS = 6
20
21// The pictures tools/bake_hd.py writes beside a persona's PNGs: the portrait
22// and its frames, `side` x `side` RGB, and the scene, RGB too.
23export type Hd = {
24  side: number
25  base: Uint8Array
26  talk: Uint8Array | null
27  deny: Uint8Array | null
28  scene: { pixels: Uint8Array; width: number; height: number } | null
29  // A print of the bytes above, so a picture baked again never meets a file
30  // made from the old one.
31  stamp?: string
32}
33
34export type HdView = {
35  // Whose pane, and which band of it: two personas may share a color, never
36  // a picture.
37  who: string
38  band: 'face' | 'under'
39  // The faces; null for the band under the text.
40  hd: Hd | null
41  mood: Mood
42  // Which face: the frown, the open mouth or the portrait (draw.ts pickFace).
43  face: 'base' | 'talk' | 'deny'
44  // The glitch's draw: one of GLITCH_STEPS tears while a refusal shows, 0 else.
45  glitchStep: number
46  glitch: number
47  // The art pixel's side in output pixels (GRAINS).
48  grain: number
49  color: string
50  cutout: number
51  // The band in cells; the face is `size` columns wide (size / 2 rows),
52  // centered, 0 for none.
53  columns: number
54  rows: number
55  size: number
56  // The weather and its field (ambient.ts), the band's first field row, the
57  // weather's time and its step in the loop (the key's share of it).
58  layers: AmbientLayer[]
59  field: Field
60  top: number
61  t: number
62  step: number
63  isStorm: boolean
64  stamp?: string
65}
66
67// How bright the scene stays behind the face, and the scanlines.
68const SCANLINE = 0.8
69
70const hex = (s: string): number => {
71  const h = s.replace('#', '')
72  return /^[0-9a-f]{6}$/i.test(h) ? parseInt(h, 16) : 0x00ff9c
73}
74
75// The face to draw now, by the same rule as the half-block pane.
76export const pickHdFace = (hd: Hd, mood: string, isSpeaking: boolean, flap: number): HdView['face'] =>
77  pickFace({ base: 'base', talk: hd.talk === null ? null : 'talk', deny: hd.deny === null ? null : 'deny' } as const, mood, isSpeaking, flap)
78
79// What decides the picture: equal keys, equal pixels.
80export const hdKey = (v: HdView): string =>
81  [v.who, v.band, v.hd === null ? '-' : v.face, v.mood, v.glitchStep, v.grain, v.columns, v.rows, v.size, v.top, v.step, v.isStorm ? 's' : '', v.color, v.cutout, v.hd?.side ?? 0, v.stamp ?? ''].join('|')
82
83// While the pane is being resized, each column brings a new size and the
84// pictures for it would be made at every render; the band keeps its last
85// picture, stretched by the terminal, until the size has held SETTLE_FRAMES.
86export const SETTLE_FRAMES = 6
87export type Settle = { size: string; since: number }
88export const isSettled = (s: Settle, size: string, frame: number): boolean => {
89  if (s.size !== size) {
90    s.size = size
91    s.since = frame
92  }
93  return frame - s.since >= SETTLE_FRAMES
94}
95
96// FNV-1a over a sample of the bytes: enough to tell two bakes apart.
97export const hdStamp = (...parts: (Uint8Array | null)[]): string => {
98  let h = 0x811c9dc5
99  for (const p of parts) {
100    if (p === null) continue
101    for (let i = 0; i < p.length; i += 97) h = Math.imul(h ^ (p[i] ?? 0), 0x01000193)
102  }
103  return (h >>> 0).toString(36)
104}
105
106export const hdSize = (columns: number, rows: number): { width: number; height: number } => ({ width: columns * PX, height: rows * 2 * PX })
107
108// One art pixel is `grain` output pixels a side: 6 is the half blocks' own
109// grid (one pixel a column), 3 twice as fine, 1 the bake's full detail. Each
110// art pixel averages what it covers, as the half blocks do, and the
111// scanlines darken every other art row.
112export const GRAINS = [1, 2, 3, 6] as const
113export const DEFAULT_GRAIN = 3
114
115// A field pixel is half a column wide and half a row tall: 3 x 6 output
116// pixels. The scene's field is made `6 / grain` times finer.
117const FIELD_X = PX / QUAD
118const FIELD_Y = PX
119
120const addRgb = (a: number, b: number): number =>
121  (Math.min(255, ((a >> 16) & 0xff) + ((b >> 16) & 0xff)) << 16) |
122  (Math.min(255, ((a >> 8) & 0xff) + ((b >> 8) & 0xff)) << 8) |
123  Math.min(255, (a & 0xff) + (b & 0xff))
124
125export const hdFrame = (v: HdView): Uint8Array => {
126  const { width, height } = hdSize(v.columns, v.rows)
127  const out = new Uint8Array(width * height * 4)
128  const hd = v.hd
129  const img = hd === null ? null : (hd[v.face] ?? hd.base)
130  const S = hd?.side ?? 0
131  const side = hd === null ? 0 : v.size * PX
132  const left = Math.floor((v.columns - v.size) / 2) * PX
133  const isDeny = v.mood === 'deny'
134  const tint = TINT[v.mood]
135  const mix = v.mood === 'watch' ? 0.3 : v.mood === 'wait' ? 0.45 : 0.65
136  const tone = v.mood === 'idle' ? hex(v.color) : tint
137  const g = Math.max(1, Math.round(v.grain))
138  const k = Math.max(1, Math.round(PX / g))
139  const fine: Field = { width: v.field.width * k, height: v.field.height * k, sceneTop: v.field.sceneTop * k, sceneBottom: v.field.sceneBottom * k }
140  const scenes = v.layers.filter(l => l.kind === 'scene')
141  const weather = v.layers.filter(l => l.kind !== 'scene')
142  const bracket = Math.max(6, Math.round(side * 0.11))
143  // The frame's line: two output pixels, or one art pixel when coarser.
144  const rim = Math.max(2, g)
145  // The glitch tears the face into bands, a few of them shifted sideways.
146  const bands = 16
147  const shiftOf = (y: number): number => {
148    if (!isDeny || v.glitch === 0 || side === 0) return 0
149    const band = Math.floor((y * bands) / side)
150    return noise(v.glitchStep, band) < 0.35 * v.glitch ? Math.round((noise(band, v.glitchStep) - 0.5) * side * 0.08 * v.glitch) : 0
151  }
152
153  // The color at an output point, before scanlines: the scene and the
154  // weather, the face over them.
155  const rgb = [0, 0, 0]
156  const at = (x: number, y: number): void => {
157    let back = 0
158    if (scenes.length > 0) back = ambientPixel(scenes, fine, Math.floor((x * k) / FIELD_X), v.top * k + Math.floor((y * k) / FIELD_Y), v.t, v.isStorm)
159    if (weather.length > 0) back = addRgb(back, ambientPixel(weather, v.field, Math.floor(x / FIELD_X), v.top + Math.floor(y / FIELD_Y), v.t, v.isStorm))
160    let r = (back >> 16) & 0xff
161    let gg = (back >> 8) & 0xff
162    let b = back & 0xff
163    const fx = x - left
164    if (img !== null && fx >= 0 && fx < side && y < side) {
165      const gx = Math.min(side - 1, Math.max(0, Math.floor(fx + shiftOf(y))))
166      const i = (Math.floor((y * S) / side) * S + Math.floor((gx * S) / side)) * 3
167      let fr = img[i] ?? 0
168      let fg = img[i + 1] ?? 0
169      let fb = img[i + 2] ?? 0
170      const lum = 0.3 * fr + 0.59 * fg + 0.11 * fb
171      if (v.mood !== 'idle') {
172        const l = (lum / 255) * 1.3 * mix
173        fr = fr * (1 - mix) + ((tint >> 16) & 0xff) * l
174        fg = fg * (1 - mix) + ((tint >> 8) & 0xff) * l
175        fb = fb * (1 - mix) + (tint & 0xff) * l
176      }
177      if (v.mood === 'wait') {
178        fr *= 0.9
179        fg *= 0.9
180        fb *= 0.9
181      }
182      // The portrait's dark background lets what is behind it through, by degrees.
183      const alpha = Math.min(1, Math.max(0, (lum - v.cutout) / (v.cutout * 2 + 4)))
184      r = fr * alpha + r * (1 - alpha)
185      gg = fg * alpha + gg * (1 - alpha)
186      b = fb * alpha + b * (1 - alpha)
187    }
188    rgb[0] = r
189    rgb[1] = gg
190    rgb[2] = b
191  }
192
193  // Up to 3 x 3 samples an art pixel: enough to average a coarse grain.
194  const n = Math.min(3, g)
195  for (let y0 = 0; y0 < height; y0 += g) {
196    const line = (g === 1 ? y0 % 2 : (y0 / g) % 2) === 1 ? SCANLINE : 1
197    for (let x0 = 0; x0 < width; x0 += g) {
198      let r = 0
199      let gg = 0
200      let b = 0
201      for (let j = 0; j < n; j++) {
202        for (let i = 0; i < n; i++) {
203          at(Math.min(width - 1, x0 + ((i + 0.5) * g) / n), Math.min(height - 1, y0 + ((j + 0.5) * g) / n))
204          r += rgb[0] ?? 0
205          gg += rgb[1] ?? 0
206          b += rgb[2] ?? 0
207        }
208      }
209      r /= n * n
210      gg /= n * n
211      b /= n * n
212      // The comm window: bright brackets at the corners, a faint line between.
213      const fx = x0 - left
214      if (side > 0 && fx >= 0 && fx < side && y0 < side) {
215        const edge = Math.min(fx, y0, side - g - fx, side - g - y0)
216        if (edge < rim) {
217          const isCorner = Math.min(fx, side - 1 - fx) < bracket && Math.min(y0, side - 1 - y0) < bracket
218          const kk = isCorner ? 1 : edge < Math.max(1, g) ? 0.3 : -1
219          if (kk > 0) {
220            r = ((tone >> 16) & 0xff) * kk
221            gg = ((tone >> 8) & 0xff) * kk
222            b = (tone & 0xff) * kk
223          }
224        } else if (shiftOf(y0) !== 0 && noise(x0, y0 + v.glitchStep) < 0.02 * v.glitch) r = gg = b = 255
225      }
226      const cr = Math.min(255, Math.round(r * line))
227      const cg = Math.min(255, Math.round(gg * line))
228      const cb = Math.min(255, Math.round(b * line))
229      for (let y = y0; y < Math.min(height, y0 + g); y++) {
230        for (let x = x0; x < Math.min(width, x0 + g); x++) {
231          const o = (y * width + x) * 4
232          out[o] = cr
233          out[o + 1] = cg
234          out[o + 2] = cb
235          out[o + 3] = 255
236        }
237      }
238    }
239  }
240  return out
241}
242

1// The line under the face: its text, how far it is typed, at what pace, and
2// the voice that carries it. Pure: the frame clock and the speaking timer in
3// register.tsx move it only through the transitions below.
4
5export type Typing = {
6  text: string
7  // Characters shown, fractional: the pace adds a share of one per frame.
8  typed: number
9  // Characters per frame, and the frame typing starts at.
10  rate: number
11  from: number
12  // Which line this is: a voice that comes back late types only its own.
13  seq: number
14  // Until when the voice is heard, in frames: the mouth moves meanwhile.
15  speakUntil: number
16  // One line at a time: set while a line is written, voiced and heard.
17  isSpeaking: boolean
18  lastSpoke: number
19}
20
21export const silent: Typing = { text: '', typed: 0, rate: 2, from: 0, seq: 0, speakUntil: 0, isSpeaking: false, lastSpoke: -Infinity }
22
23// The line waits for its voice: typed from when SAPI starts the WAV (PowerShell
24// takes ~0.3 s to get there), over the audio's length. Without a WAV within
25// HOLD_FRAMES (Piper missing, SAPI speaking itself) it is typed anyway.
26export const PLAY_LEAD_FRAMES = 5
27export const HOLD_FRAMES = 75
28
29// A line kept from before a reload: shown whole, nothing to type.
30export const restored = (t: Typing, text: string): Typing => ({ ...t, text, typed: text.length })
31
32// The speaking slot: taken when a line is picked from the queue, given back
33// when it was heard.
34export const begin = (t: Typing, now: number): Typing => ({ ...t, isSpeaking: true, lastSpoke: now })
35export const end = (t: Typing): Typing => ({ ...t, isSpeaking: false })
36
37// A new line: held until its voice is ready, unless muted.
38export const start = (t: Typing, text: string, isQuiet: boolean, now: number): Typing => ({
39  ...t,
40  text,
41  typed: 0,
42  rate: 2,
43  from: isQuiet ? now : now + HOLD_FRAMES,
44  seq: t.seq + 1,
45})
46
47// The WAV synthArgv made (its path, then its length in seconds); the line is
48// typed over the audio's length while it plays, if it is still the line on
49// screen. wav is '' when SAPI already spoke it.
50export const voiced = (t: Typing, seq: number, stdout: string, now: number, frameMs: number): { t: Typing; wav: string; ms: number } => {
51  const [wav = '', seconds = ''] = stdout.trim().split('\n')
52  const ms = Number(seconds) * 1000
53  const frames = Math.round(ms / frameMs)
54  if (wav === '' || !(frames > 0)) return { t, wav: '', ms: 0 }
55  if (seq !== t.seq) return { t, wav, ms }
56  const from = now + PLAY_LEAD_FRAMES
57  return { t: { ...t, rate: Math.max(t.text.length / frames, 0.2), from, speakUntil: from + frames }, wav, ms }
58}
59
60// One frame of typing: the same object when nothing moved, so the clock
61// redraws only on a change.
62export const typeOn = (t: Typing, now: number): Typing =>
63  t.typed < t.text.length && now >= t.from ? { ...t, typed: Math.min(t.text.length, t.typed + t.rate) } : t
64
65export const isHeard = (t: Typing, now: number): boolean => now < t.speakUntil
66

1// The face's state: which mood it shows, until which frame, and the two waits
2// that hold it (a call mesh7 holds for a human, a call at the permission
3// prompt). Pure: every change is one of the transitions below, so the rule
4// that a wait keeps the face lives here once instead of in each hook.
5
6export type Mood = 'idle' | 'watch' | 'deny' | 'error' | 'wait'
7
8export type Face = {
9  mood: Mood
10  // The frame after which the mood fades back to idle; Infinity while waiting.
11  until: number
12  // A call mesh7 holds for a human, as mesh7-pane said it; null when none.
13  held: string | null
14  heldSince: number
15  // A call put to the permission prompt (a mesh7 hook's `ask`, a settings rule).
16  askSince: number | null
17  askCall: string
18  // The mood comes from a moment of the persona's own story, not from a
19  // call: the face shows it, softer (a smaller glitch on a staged refusal).
20  isStaged: boolean
21}
22
23export const calm: Face = { mood: 'idle', until: 0, held: null, heldSince: 0, askSince: null, askCall: '', isStaged: false }
24
25export const isWaiting = (f: Face): boolean => f.held !== null || f.askSince !== null
26
27// How long a mood shows, in frames: a calm look passes faster than an alarm.
28export const span = (m: Mood): number => (m === 'watch' ? 12 : 30)
29
30// A mood for a while; a wait keeps the face as it is.
31export const react = (f: Face, mood: Mood, now: number, frames = span(mood)): Face =>
32  isWaiting(f) ? f : { ...f, mood, until: now + frames, isStaged: false }
33
34// A scene of the persona's own: its mood, acted rather than suffered.
35export const stage = (f: Face, mood: Mood, now: number, frames: number): Face =>
36  isWaiting(f) ? f : { ...f, mood, until: now + frames, isStaged: true }
37
38// mesh7 holds a call for a human: the face waits until the release.
39export const hold = (f: Face, call: string, now: number): Face => ({ ...f, held: call, heldSince: now, mood: 'wait', until: Infinity, isStaged: false })
40
41// The human decided: the verdict's mood, for a while.
42export const release = (f: Face, mood: Mood, now: number): Face => ({ ...f, held: null, mood, until: now + 30, isStaged: false })
43
44// A call goes to the permission prompt: the face waits until it ran.
45export const ask = (f: Face, call: string, now: number): Face => ({ ...f, askSince: now, askCall: call, mood: 'wait', until: Infinity, isStaged: false })
46
47// The call ran (or was interrupted): the prompt is gone. The mood stays until
48// the call's own outcome sets it.
49export const answered = (f: Face): Face => (f.askSince === null ? f : { ...f, askSince: null })
50
51// Each frame: a wait whose end never came (mesh7-pane reloaded before its
52// release, a prompt that vanished) lets go after `cap` frames; a mood past
53// its time fades to idle.
54export const tick = (f: Face, now: number, cap: number): Face => {
55  let g = f
56  if ((g.askSince !== null && now - g.askSince > cap) || (g.held !== null && now - g.heldSince > cap)) {
57    g = { ...g, askSince: null, held: null, until: now }
58  }
59  if (now > g.until && g.mood !== 'idle') g = { ...g, mood: 'idle', isStaged: false }
60  return g
61}
62

1export type Line = { text: string; at: number }
2
3// What a mod asks of the avatar when it toasts, published by that mod under
4// its own `announce` key: the face to wear (calm, or amber for a warning) and
5// what happened, for the line the avatar speaks.
6// A persona's station: its name and the artists it would put on, none for a
7// persona without one.
8export type Station = { name: string; artists: string[] }
9
10export type Announce = { mood: 'watch' | 'error'; event: string }
11
12// What a mod asks the avatar to say now, with no toast: published under the
13// mod's own `say` key, the face to wear, what happened, and when (ms), so the
14// same event twice is still two writes. The avatar queues it by urgency.
15// tool: the call it is about (Claude Code's name), whose own line the avatar
16// then drops; hold: a call held for a human, the face waits; release: decided.
17export type Say = {
18  mood: 'watch' | 'error' | 'deny' | 'wait'
19  event: string
20  at: number
21  tool?: string
22  hold?: boolean
23  release?: boolean
24}
25
26// One model call of this mod, as it was billed to the plan: usage-bell counts
27// them (a hook on model.complete does not see another mod's calls).
28export type ModelUse = { model: string; input: number; output: number; cacheRead: number; at: number }
29
30declare module 'claude-code' {
31  interface PluginState {
32    avatar7: {
33      line: Line
34      isMuted: boolean
35      volume: number
36      avatar: string
37      color: string
38      // Each mod's announce, by plugin name, as heard; kept across a reload.
39      announcers: Record<string, Announce>
40      // True while a line is heard (the WAV playing), read by jukebox7 to duck.
41      isVoicing: boolean
42      // The on-duty persona's music, from its persona.json: jukebox7 plays it.
43      station: Station
44      // The last model call, for usage-bell's tally.
45      modelUse: ModelUse
46    }
47  }
48}
49