mod-faults

1import type { EngineInterface, EventName, Next, On, Register } from 'claude-code'
2
3// <metrics: generated from shared/metrics.ts by shared/sync.sh>
4// Shared mod metrics (schema v2). Source of truth: ~/.claude/mods/shared/metrics.ts.
5// shared/sync.sh inlines it between the metrics markers in each mod's register.tsx (the engine only lets $
6// into functions declared in the same file). Edit here, run sync.sh, never edit an inlined copy.
7//
8// Rows appended to ~/.claude/mods/metrics.jsonl, all with { v: 2, t, mod, event, ver, session, project }:
9//   loaded   once per load (logLoaded, or count() in the mod's session.start hook), at session.start: proves which version of the mod is running
10//   turn     once per turn with activity (logTurns, or count() in the mod's turn.complete hook): { hooks: { <event>: calls } }
11//   <other>  outcome rows from track(), e.g. merge-gate 'verdict'
12// Privacy: never put prompt, command, tool-input or matched-secret text in a row; lengths, kinds and counts only.
13
14const APPEND = 'mkdir -p "$HOME/.claude/mods" && printf "%s\\n" "$1" >> "$HOME/.claude/mods/metrics.jsonl"'
15
16
17const ctx = { ver: '', session: '', project: '' }
18const stats = new Map<string, number>()
19let writeFailed = false
20
21const message = (err: unknown) => (err instanceof Error ? err.message : String(err)).slice(0, 300)
22
23function write($: EngineInterface, mod: string, event: string, data: Record<string, unknown>): void {
24  void (async () => {
25    const t = new Date(await $.clock.now()).toISOString()
26    if (!ctx.project) ctx.project = (await $.session.cwd()).split('/').filter(Boolean).pop() ?? ''
27    const line = JSON.stringify({ v: 2, t, mod, event, ver: ctx.ver, session: ctx.session, project: ctx.project, ...data })
28    const { exitCode, stderr } = await $.process.run(['/bin/sh', '-c', APPEND, 'mod-metrics', line], { timeoutMs: 3000 })
29    if (exitCode !== 0) throw new Error(stderr.trim() || `exit ${exitCode}`)
30  })().catch((err: unknown) => {
31    if (writeFailed) return
32    writeFailed = true
33    $.ui.toast(`${mod}: metrics write failed (${message(err)})`)
34  })
35}
36
37// An outcome row: something the mod decided or did. Fire-and-forget.
38function track($: EngineInterface, mod: string, event: string, data: Record<string, unknown> = {}): void {
39  write($, mod, event, data)
40}
41
42async function loaded($: EngineInterface): Promise<void> {
43  const mod = $.plugin.name
44  ctx.session = await $.session.id().catch(() => '')
45  ctx.project = ''
46  try {
47    const manifest = JSON.parse(await $.fs.read(`${$.plugin.root}/.claude-plugin/plugin.json`)) as { version?: string }
48    ctx.ver = manifest.version ?? ''
49  } catch {
50    ctx.ver = '?'
51  }
52  write($, mod, 'loaded', {})
53}
54
55function flush($: EngineInterface): void {
56  const mod = $.plugin.name
57  if (!stats.size) return
58  const hooks = Object.fromEntries(stats)
59  stats.clear()
60  write($, mod, 'turn', { hooks })
61}
62
63// The engine allows one hook per event per plugin, and counts them by spelling. A mod that hooks
64// session.start or turn.complete itself calls count() in that hook, which writes the row; otherwise
65// it calls the matching helper once in register.
66function logLoaded(on: On): void {
67  on('session.start', async ($, e, next) => {
68    await loaded($)
69
70    return next(e)
71  })
72}
73
74function logTurns(on: On): void {
75  on('turn.complete', async ($, e, next) => {
76    flush($)
77
78    return next(e)
79  })
80}
81
82// First statement of every hook: counts the call into this turn's row. The engine only accepts
83// function literals as hooks, so calls are counted from inside rather than by wrapping.
84// In a session.start hook it also writes the loaded row; in a turn.complete hook it writes the turn row.
85function count($: EngineInterface, event: string): void {
86  stats.set(event, (stats.get(event) ?? 0) + 1)
87  if (event === 'session.start') void loaded($)
88  if (event === 'turn.complete') flush($)
89}
90// </metrics>
91
92// <modlist: generated from shared/modlist.ts by shared/sync.sh>
93async function godModList($: EngineInterface): Promise<{ names: string[]; file: string } | undefined> {
94  try {
95    const home = await $.env.get('HOME')
96    const known = JSON.parse(await $.fs.read(`${home}/.claude/plugins/known_marketplaces.json`))
97    const root = known['god-mods']?.installLocation ?? known['god-mods']?.source?.path
98    if (!root) return undefined
99    const file = `${root}/.claude-plugin/marketplace.json`
100    const names = (JSON.parse(await $.fs.read(file)).plugins as { name: string }[]).map(p => p.name)
101
102    return names.length ? { names, file } : undefined
103  } catch {
104    return undefined
105  }
106}
107// </modlist>
108
109// Without the marketplace manifest, only plugins installed as <name>@god-mods count
110let GOD_MODS = new Set<string>()
111let listSource = 'unavailable (matching @god-mods installs only)'
112
113async function loadModList($: EngineInterface) {
114  const list = await godModList($)
115  if (!list) return
116  GOD_MODS = new Set(list.names.filter(n => n !== 'mod-faults'))
117  listSource = list.file
118}
119
120const isGodMod = (plugin: string) => GOD_MODS.has(plugin.split('@')[0]!) || plugin.endsWith('@god-mods')
121const FAULTS = new Set(['skipped', 'kept', 'expired', 'caught', 'rejected'])
122// The third fault from one mod in a session toasts, once
123const ALERT_AT = 3
124
125const seen = new Set<string>()
126const counts = new Map<string, number>()
127
128// One short lowercase leading word of the reason at most (e.g. "timeout"): reasons can quote tool input or
129// error text, which metrics rows never carry
130const reasonKind = (reason: string | undefined) => (reason ? (reason.match(/^[a-z][a-z-]{0,23}(?=[:\s(]|$)/)?.[0] ?? 'other') : 'none')
131
132function record($: EngineInterface, plugin: string, hookEvent: string, outcome: string, reason: string | undefined) {
133  const n = (counts.get(plugin) ?? 0) + 1
134  counts.set(plugin, n)
135  track($, 'mod-faults', 'fault', { plugin, hookEvent, outcome, reasonKind: reasonKind(reason) })
136  if (n === ALERT_AT) $.ui.toast(`mod-faults: ${plugin} has faulted ${n} times this session (/mod-faults)`)
137}
138
139async function watch<N extends EventName>($: EngineInterface, e: Parameters<Next<N>>[0], next: Next<N>) {
140  try {
141    return await next(e)
142  } finally {
143    try {
144      for (const t of next.trace) {
145        if (!isGodMod(t.plugin)) continue
146        seen.add(t.plugin)
147        if (FAULTS.has(t.outcome) && !t.reason?.startsWith('bypassed')) record($, t.plugin, t.event, t.outcome, t.reason)
148      }
149    } catch {
150      // Never let fault bookkeeping break the event
151    }
152  }
153}
154
155// Every god-mod hooks session.start, so once it settles, a listed mod missing from the trace is
156// above this one in the chain (or not enabled): its faults are invisible here.
157const blind = () => [...GOD_MODS].filter(m => !seen.has(m)).sort()
158const FIX = 'Move "mod-faults@god-mods" to the top of enabledPlugins in ~/.claude/settings.json: plugins nest in that order, first outermost.'
159
160async function start($: EngineInterface, e: Parameters<Next<'session.start'>>[0], next: Next<'session.start'>) {
161  count($, 'session.start')
162  await loadModList($)
163  await $.command.register({ name: 'mod-faults', description: 'Show god-mods faults this session and which mods are watched' })
164  const started = await watch($, e, next)
165  const unseen = blind()
166  if (unseen.length) {
167    track($, 'mod-faults', 'blind', { mods: unseen, of: GOD_MODS.size })
168    $.ui.toast(`mod-faults: can't see faults in ${unseen.length} of ${GOD_MODS.size} god-mods above it in the chain (/mod-faults)`)
169  }
170
171  return started
172}
173
174async function turnEnd($: EngineInterface, e: Parameters<Next<'turn.complete'>>[0], next: Next<'turn.complete'>) {
175  count($, 'turn.complete')
176
177  return watch($, e, next)
178}
179
180async function report() {
181  const faults = [...counts].map(([p, n]) => `  ${p}: ${n}`).join('\n') || '  none'
182  const watched = [...seen].sort().join(', ') || 'none yet'
183  const unseen = blind()
184  const blindText = unseen.length ? `\nNot visible (above this mod, or not enabled): ${unseen.join(', ')}\n${FIX}` : ''
185
186  return {
187    text: `god-mods faults this session:\n${faults}\nWatched beneath this mod: ${watched}${blindText}\nMod list (${GOD_MODS.size}): ${listSource}\nRecorded in ~/.claude/mods/metrics.jsonl; /mod-stats shows faults across sessions`,
188  }
189}
190
191export const register: Register = (on: On) => {
192  on('session.start', start)
193  on('turn.complete', turnEnd)
194  on('command.run', { command: 'mod-faults' }, report)
195  // coverage: observer — one watch() hook on every event; tests/faults.test.ts drives it through tool.call
196  on('session.end', watch)
197  on('session.measure', watch)
198  on('session.append', watch)
199  on('turn.start', watch)
200  on('prompt.submit', watch)
201  on('prompt.compose', watch)
202  on('tool.call', watch)
203  on('tool.describe', watch)
204  on('agent.spawn', watch)
205  on('command.run', watch)
206  on('ui.render', watch)
207  on('ui.close', watch)
208}
209