route-ledger

1import type { EngineInterface, Register } from 'claude-code'
2
3import type { LedgerEntry } from '../types'
4
5const PREFIX = 'ledger:'
6const CAP = 2000
7const DAY = 86_400_000
8
9const MODELS = ['haiku', 'sonnet', 'opus', 'fable']
10const EFFORTS = ['low', 'medium', 'high', 'xhigh', 'max']
11
12// "claude-opus-5-5[1m]" -> "opus". `best` is Opus (as effort-gate reads it); anything unknown is kept as typed.
13const family = (m: string) => {
14  const s = m.toLowerCase()
15  return MODELS.find(f => s.includes(f)) ?? (s === 'best' ? 'opus' : s)
16}
17
18// First `key: value` of the file's frontmatter, unquoted.
19function front(text: string, key: string) {
20  const block = /^---\r?\n([\s\S]*?)\r?\n---/.exec(text)?.[1] ?? ''
21  const v = new RegExp(`^${key}` + String.raw`:\s*(.+?)\s*$`, 'm').exec(block)?.[1]
22  return v?.replace(/^["']|["']$/g, '')
23}
24
25// Project agents shadow user agents. A missing file is just "no frontmatter".
26async function agentFile($: EngineInterface, agent: string): Promise<{ model?: string; effort?: string }> {
27  const home = (await $.env.get('HOME')) || (await $.env.get('USERPROFILE')) || ''
28  const cwd = await $.session.cwd()
29  for (const dir of [`${cwd}/.claude/agents`, `${home}/.claude/agents`]) {
30    try {
31      const text = await $.fs.read(`${dir}/${agent}.md`)
32      if (typeof text === 'string' && text) return { model: front(text, 'model'), effort: front(text, 'effort') }
33    } catch {}
34  }
35  return {}
36}
37
38// ---- reading `claude ... --bg` out of a shell command ----
39
40type Tok = { text: string; quoted: boolean } // quoted: the word began inside quotes (a prompt, never a flag)
41
42// Splits a command into segments (at unquoted ; | & newline) of words (at unquoted whitespace).
43// ponytail: no heredocs, backticks, $(...) or backslash escapes outside quotes; an odd apostrophe swallows the rest, which only hides flags.
44export function split(cmd: string): Tok[][] {
45  const segs: Tok[][] = [[]]
46  let cur = ''
47  let has = false
48  let startQ = false
49  let q: string | null = null
50  const word = () => {
51    if (has) segs[segs.length - 1]!.push({ text: cur, quoted: startQ })
52    cur = ''
53    has = false
54    startQ = false
55  }
56  for (let i = 0; i < cmd.length; i++) {
57    const c = cmd[i]!
58    if (q) {
59      if (c === q) q = null
60      else if (c === '\\' && q === '"' && cmd[i + 1] === '"') cur += cmd[++i]
61      else cur += c
62    } else if (c === '"' || c === "'") {
63      if (!has) startQ = true
64      has = true
65      q = c
66    } else if (c === '\n' || c === ';' || c === '|' || c === '&') {
67      word()
68      if (segs[segs.length - 1]!.length) segs.push([])
69    } else if (/\s/.test(c)) word()
70    else {
71      cur += c
72      has = true
73    }
74  }
75  word()
76  return segs.filter(s => s.length)
77}
78
79const VALUE_FLAGS: Record<string, 'model' | 'effort' | 'agent' | 'advisor' | 'name'> = {
80  '--model': 'model',
81  '--effort': 'effort',
82  '--agent': 'agent',
83  '--advisor': 'advisor',
84  '--name': 'name',
85  '-n': 'name',
86}
87export type BgLaunch = Partial<Record<(typeof VALUE_FLAGS)[string], string>>
88
89const SHELL = /^(?:(?:ba|z|da|k|c)?sh|pwsh|powershell|cmd)(?:\.exe)?$/i
90
91// Every `claude ... --bg/--background` segment. `claude` must be the segment's command word (after FOO=bar
92// assignments; a path ending in claude or claude.exe is fine); flags are read only from unquoted words, and a
93// quoted word counts only as the value right after a flag that takes one. `sh -c "..."` is read inside.
94export function bgLaunches(cmd: string): BgLaunch[] {
95  const out: BgLaunch[] = []
96  for (const seg of split(cmd)) {
97    let i = 0
98    while (i < seg.length && !seg[i]!.quoted && /^[A-Za-z_]\w*=/.test(seg[i]!.text)) i++
99    const word = seg[i]?.text.split(/[\\/]/).pop() ?? ''
100    if (SHELL.test(word)) {
101      const k = seg.findIndex((t, n) => n > i && !t.quoted && /^[-/](?:c|command)$/i.test(t.text))
102      if (k >= 0) {
103        const rest = seg.slice(k + 1)
104        out.push(...bgLaunches(rest.length === 1 ? rest[0]!.text : rest.map(t => (t.quoted ? JSON.stringify(t.text) : t.text)).join(' ')))
105      }
106      continue
107    }
108    if (!/^claude(?:\.exe)?$/i.test(word)) continue
109    const found: BgLaunch = {}
110    let bg = false
111    for (let j = i + 1; j < seg.length; j++) {
112      const t = seg[j]!
113      if (t.quoted) continue
114      if (t.text === '--') break
115      const m = /^(--[a-z-]+|-n)(?:=(.*))?$/i.exec(t.text)
116      if (!m) continue
117      if (m[1] === '--bg' || m[1] === '--background') {
118        bg = true
119        continue
120      }
121      const key = VALUE_FLAGS[m[1]!.toLowerCase()]
122      if (!key) continue
123      let v = m[2]
124      const next = seg[j + 1]
125      if (v === undefined && next && (next.quoted || !next.text.startsWith('-'))) {
126        v = next.text
127        j++
128      }
129      if (v !== undefined) found[key] = v
130    }
131    if (bg) out.push(found)
132  }
133  return out
134}
135
136// ---- ledger ----
137
138async function mainModel($: EngineInterface) {
139  try {
140    return await $.session.model()
141  } catch {
142    return 'opus' // not exposed: treat the inherited model as Opus
143  }
144}
145
146async function repoName($: EngineInterface) {
147  let dir: string | undefined
148  try {
149    dir = (await $.session.repo())?.root
150  } catch {}
151  dir ??= await $.session.cwd()
152  return dir.replace(/[\\/]+$/, '').split(/[\\/]/).pop() || dir
153}
154
155const norm = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()
156const clip = (s: string) => s.slice(0, 80)
157const tag = (model: string, effort?: string) => `${model}@${effort ?? '?'}`
158
159// Higher on the ladder: model first, then effort within the same model (compared only when both are known).
160function higher(a: LedgerEntry, b: LedgerEntry) {
161  const ma = MODELS.indexOf(a.model)
162  const mb = MODELS.indexOf(b.model)
163  if (ma !== mb) return mb > ma
164  const ea = EFFORTS.indexOf(a.effort ?? '')
165  const eb = EFFORTS.indexOf(b.effort ?? '')
166  return ea >= 0 && eb >= 0 && eb > ea
167}
168
169// The store is one file shared by every Claude Code process, so each session writes only its own key.
170// Writes in this process run one after another, so parallel launches do not overwrite each other.
171// ponytail: a session's whole entry list is rewritten per change (<= 2000 small entries).
172let queue: Promise<unknown> = Promise.resolve()
173const edit = (fn: () => Promise<void>) => {
174  queue = queue.then(fn, fn).catch(() => {}) // never let a ledger problem reach the tool call
175  return queue
176}
177
178const load = async ($: EngineInterface, key: string) => ((await $.store.get(key)) as LedgerEntry[] | undefined) ?? []
179
180const change = ($: EngineInterface, key: string, fn: (all: LedgerEntry[]) => void) =>
181  edit(async () => {
182    const all = await load($, key)
183    fn(all)
184    await $.store.set(key, all.slice(-CAP))
185  })
186
187async function loadAll($: EngineInterface) {
188  const keys = ((await $.store.keys()) as string[]).filter(k => k.startsWith(PREFIX))
189  return (await Promise.all(keys.map(k => load($, k)))).flat().sort((a, b) => a.ts - b.ts)
190}
191
192// A retry is a later launch in the same session with the same label, once the earlier one has finished
193// (a bg launch has no outcome to wait for). Escalated: the retry runs higher on the ladder.
194function link(all: LedgerEntry[], entry: LedgerEntry) {
195  const prev = all.find(x => x.id === entry.prev)
196  delete entry.prev
197  if (!prev) return
198  prev.retried = true
199  if (higher(prev, entry)) prev.escalatedTo = tag(entry.model, entry.effort)
200}
201
202type Launch = Pick<LedgerEntry, 'kind' | 'agent' | 'model' | 'effort' | 'advisor' | 'label' | 'running'>
203
204// Appends the launch and returns once it is written. `now` links retries at once (bg); a subagent links when
205// its outcome (and so its real model) is known.
206async function launch($: EngineInterface, l: Launch, now: boolean) {
207  const sid = await $.session.id()
208  const key = PREFIX + sid
209  const entry: LedgerEntry = {
210    ...l,
211    id: Math.random().toString(36).slice(2),
212    ts: await $.clock.now(),
213    sid,
214    repo: await repoName($),
215    label: clip(l.label),
216  }
217  await change($, key, all => {
218    const k = norm(entry.label)
219    if (k) {
220      const prev = [...all].reverse().find(x => x.sid === sid && norm(x.label) === k)
221      if (prev && !prev.running) entry.prev = prev.id
222    }
223    all.push(entry)
224    if (now) link(all, entry)
225  })
226  return { key, id: entry.id }
227}
228
229type Ref = { key: string; id: string }
230
231const finish = ($: EngineInterface, ref: Ref, patch: Partial<LedgerEntry>) =>
232  change($, ref.key, all => {
233    const entry = all.find(x => x.id === ref.id)
234    if (!entry) return // already dropped by the cap
235    Object.assign(entry, patch)
236    if (!patch.agentId) delete entry.running // a background subagent runs on until its turn ends
237    link(all, entry)
238  })
239
240const drop = ($: EngineInterface, refs: Ref[]) =>
241  Promise.all(
242    [...new Set(refs.map(r => r.key))].map(key =>
243      change($, key, all => {
244        const gone = new Set(refs.filter(r => r.key === key).map(r => r.id))
245        all.splice(0, all.length, ...all.filter(x => !gone.has(x.id)))
246      }),
247    ),
248  )
249
250// Background subagents end later, in their own turns. In-process: agent ids whose outcome is awaited.
251const watching = new Set<string>()
252
253// Keeps the total near CAP: sessions whose newest entry is older than the CAPth newest entry overall go.
254async function prune($: EngineInterface) {
255  const keys = ((await $.store.keys()) as string[]).filter(k => k.startsWith(PREFIX))
256  const lists = await Promise.all(keys.map(async k => [k, await load($, k)] as const))
257  const ts = lists.flatMap(([, l]) => l.map(x => x.ts)).sort((a, b) => b - a)
258  if (ts.length <= CAP) return
259  const cutoff = ts[CAP - 1]!
260  for (const [k, l] of lists) if (!l.length || l[l.length - 1]!.ts < cutoff) await $.store.delete(k)
261}
262
263// ---- review ----
264
265const median = (xs: number[]) => {
266  if (!xs.length) return undefined
267  const s = [...xs].sort((a, b) => a - b)
268  const m = s.length >> 1
269  return s.length % 2 ? s[m]! : Math.round((s[m - 1]! + s[m]!) / 2)
270}
271const fmtTokens = (n?: number) => (n === undefined ? '-' : n >= 1000 ? `${(n / 1000).toFixed(n >= 10_000 ? 0 : 1)}k` : String(n))
272const fmtMs = (n?: number) => {
273  if (n === undefined) return '-'
274  const s = Math.round(n / 1000)
275  return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m${String(s % 60).padStart(2, '0')}s`
276}
277
278export function review(all: LedgerEntry[], days: number, now: number) {
279  const entries = all.filter(x => x.ts >= now - days * DAY)
280  if (!entries.length) return `routing-review: no launches in the last ${days}d`
281  const groups = new Map<string, LedgerEntry[]>()
282  for (const x of entries) {
283    const k = `${tag(x.model, x.effort)}\t${x.agent ?? '-'}`
284    groups.set(k, [...(groups.get(k) ?? []), x])
285  }
286  const rows = [['model@effort', 'agent', 'n', 'ok', 'err', 'retry', 'esc', 'tok~', 'time~']]
287  for (const [k, g] of [...groups].sort((a, b) => b[1].length - a[1].length)) {
288    const [route, agent] = k.split('\t')
289    rows.push([
290      route!,
291      agent!,
292      String(g.length),
293      String(g.filter(x => x.ok === true).length),
294      String(g.filter(x => x.ok === false).length),
295      String(g.filter(x => x.retried).length),
296      String(g.filter(x => x.escalatedTo).length),
297      fmtTokens(median(g.flatMap(x => (x.tokens === undefined ? [] : [x.tokens])))),
298      fmtMs(median(g.flatMap(x => (x.ms === undefined ? [] : [x.ms])))),
299    ])
300  }
301  const w = rows[0]!.map((_, i) => Math.max(...rows.map(r => r[i]!.length)))
302  const out = [
303    `routing-review: last ${days}d, ${entries.length} launches (ok/err/tok/time only where an outcome was seen: subagents; bg sessions have none, so ok+err < n)`,
304    ...rows.map(r => r.map((c, i) => c.padEnd(w[i]!)).join('  ').trimEnd()),
305  ]
306  const esc = entries.filter(x => x.escalatedTo).slice(-5).reverse()
307  if (esc.length) {
308    out.push('', 'Recent escalations:')
309    for (const x of esc) out.push(`- ${x.label}: ${tag(x.model, x.effort)} -> ${x.escalatedTo}`)
310  }
311  return out.join('\n')
312}
313
314// ---- hooks ----
315
316export const register: Register = on => {
317  on('session.start', async ($, e, next) => {
318    const result = await next(e)
319    await $.command.register({
320      name: 'routing-review',
321      description: 'Per model@effort launches, outcomes, retries and escalations of delegated workers',
322      argumentHint: '[days]',
323    })
324    try {
325      await prune($)
326    } catch {}
327    return result
328  })
329
330  on('command.run', { command: 'routing-review' }, async ($, e) => {
331    const days = Math.max(Number.parseFloat(e.args.trim()), 0) || 14
332    return { text: review(await loadAll($), days, await $.clock.now()) }
333  })
334
335  on('tool.call', { tool: 'Agent' }, async ($, e, next) => {
336    if (e.subagent_type === 'fork') return next(e) // forks inherit the parent; nothing to tune
337    let ref: Ref | undefined
338    try {
339      const def = e.subagent_type ? await agentFile($, e.subagent_type) : {}
340      let model = e.model ?? def.model
341      if (!model || model === 'inherit') model = await mainModel($)
342      ref = await launch(
343        $,
344        {
345          kind: 'subagent',
346          agent: e.subagent_type,
347          model: family(model),
348          effort: e.effort ?? def.effort,
349          label: e.description ?? '',
350          running: true,
351        },
352        false,
353      )
354    } catch {}
355    const ran = await next(e)
356    if (!ref) return ran
357    try {
358      if (ran.deny !== undefined) {
359        await drop($, [ref]) // refused: nothing ran
360        return ran
361      }
362      // Agent calls usually run in the background here: the result then says so and carries no usage.
363      const r = (ran.result ?? {}) as {
364        status?: string
365        agentId?: string
366        resolvedModel?: string
367        totalDurationMs?: number
368        totalTokens?: number
369      }
370      const patch: Partial<LedgerEntry> = r.resolvedModel ? { model: family(r.resolvedModel) } : {}
371      if (ran.isError === true) patch.ok = false
372      else if (r.status === 'completed') Object.assign(patch, { ok: true, ms: r.totalDurationMs, tokens: r.totalTokens })
373      else if (r.status === 'async_launched' && r.agentId) {
374        patch.agentId = r.agentId
375        watching.add(r.agentId)
376      }
377      await finish($, ref, patch)
378    } catch {}
379    return ran
380  })
381
382  // The outcome of a background subagent: its own turns end here, carrying its agent id.
383  on('turn.complete', async ($, e, next) => {
384    const result = await next(e)
385    if (!e.agentId || !watching.has(e.agentId)) return result
386    try {
387      const agentId = e.agentId
388      const key = PREFIX + (await $.session.id())
389      const u = e.usage
390      await change($, key, all => {
391        const entry = all.find(x => x.agentId === agentId)
392        if (!entry) return
393        delete entry.running
394        entry.ok = e.reason === 'answer'
395        entry.ms = (entry.ms ?? 0) + e.durationMs
396        if (u) entry.tokens = (entry.tokens ?? 0) + u.input_tokens + u.output_tokens
397        link(all, entry)
398      })
399    } catch {}
400    return result
401  })
402
403  // bg outcomes: the parent cannot observe a background session and the worker's own mod cannot learn its `-n` name
404  // or effort from the engine API, so background launches are recorded without ok/duration/tokens.
405  for (const tool of ['Bash', 'PowerShell'] as const) {
406    on('tool.call', { tool }, async ($, e, next) => {
407      const refs: Ref[] = []
408      let single = false
409      try {
410        const cmd = String(e.command ?? '')
411        const launches = bgLaunches(cmd)
412        single = launches.length > 0 && split(cmd).length === 1
413        for (const f of launches) {
414          const def = f.agent ? await agentFile($, f.agent) : {}
415          refs.push(
416            await launch(
417              $,
418              {
419                kind: 'bg',
420                agent: f.agent,
421                model: family(f.model ?? def.model ?? 'opus'), // no --model: the default, treated as Opus
422                effort: f.effort ?? def.effort,
423                advisor: f.advisor,
424                label: f.name ?? '',
425              },
426              true,
427            ),
428          )
429        }
430      } catch {}
431      const ran = await next(e)
432      // Denied: nothing ran. Errored: only when the claude segment was the whole command do we know it failed,
433      // otherwise a later `&& failing-cmd` would wrongly erase a session that did start.
434      if (refs.length && (ran.deny !== undefined || (single && ran.isError === true))) await drop($, refs)
435      return ran
436    })
437  }
438}
439

1export type LedgerEntry = {
2  id: string
3  /** ms, $.clock.now() */
4  ts: number
5  /** parent session id */
6  sid: string
7  repo: string
8  kind: 'subagent' | 'bg'
9  agent?: string
10  /** model family (haiku|sonnet|opus|fable) or the raw string */
11  model: string
12  effort?: string
13  advisor?: string
14  /** first 80 chars of the subagent description or the bg --name; never the prompt */
15  label: string
16  /** subagents only: bg outcomes are not observable from the parent */
17  ok?: boolean
18  ms?: number
19  tokens?: number
20  /** background subagent: its agent id, to fill the outcome from its own turn.complete */
21  agentId?: string
22  /** a subagent that has not finished yet (never counts as the earlier run of a retry) */
23  running?: true
24  /** transient: the finished earlier entry this launch retries, until the link is made */
25  prev?: string
26  /** a later launch in the same session reused the label */
27  retried?: boolean
28  /** the retry ran higher on the ladder: "model@effort" of that retry */
29  escalatedTo?: string
30}
31