asclexis-rsi

1import type { Register } from 'claude-code'
2import { checkAfter, checkBefore, classifyError, scoreboard, tracker, type Row } from './logic'
3
4const LOG_DIR = '.claude/logs/rsi'
5const DAY = 86_400_000
6
7// Metadata only. Rows never hold command text, file contents, tool output, prompts or paths
8// (see README). Every logging step is caught: telemetry must never block or fail a tool call.
9let sid = ''
10let calls = 0
11let fails = 0
12let guards = 0
13const track = tracker()
14// Serialised read-append-write (`$.fs` has no append). Fire-and-forget for tool calls.
15let queue: Promise<unknown> = Promise.resolve()
16
17function append($: any, ts: string, row: object) {
18  const path = `${LOG_DIR}/${ts.slice(0, 10)}.jsonl`
19  queue = queue
20    .then(async () => {
21      const cur = (await $.fs.exists(path)) ? ((await $.fs.read(path)) as string) : ''
22      await $.fs.write(path, `${cur}${JSON.stringify(row)}\n`)
23    })
24    .catch(() => {})
25  return queue
26}
27async function sessionId($: any): Promise<string> {
28  try { return (sid ||= await $.session.id()) } catch { return 'unknown' }
29}
30async function iso($: any): Promise<string> {
31  return new Date(await $.clock.now()).toISOString()
32}
33function refresh($: any) {
34  $.ui.status(`rsi: ${calls} calls · ${fails} fail · ${guards} guard`)
35}
36
37export const register: Register = on => {
38  on('session.start', async ($, e) => {
39    try {
40      await $.command.register({ name: 'rsi', description: 'RSI scoreboard: tool failures, retries, guards (this session + 7 days)' })
41      await append($, await iso($), { v: 1, ts: await iso($), session: await sessionId($), kind: 'session', event: 'start' })
42      refresh($)
43    } catch {}
44    return e as any
45  })
46
47  on('session.end', async ($, e) => {
48    try {
49      const ts = await iso($)
50      await append($, ts, { v: 1, ts, session: await sessionId($), kind: 'session', event: 'end' })
51    } catch {}
52    return e as any
53  })
54
55  // One tool.call hook (the engine allows one unmatched hook per event). Two failure modes:
56  // - Guard (deny rules, the code before `next`): FAILS CLOSED. If it throws, the .catch denies
57  //   the call, as the skill's guard pattern does.
58  // - Observer + hints (everything after `next`): FAILS OPEN. Wrapped in its own try/catch, so
59  //   a logging or hint error can never fail or alter the call.
60  on('tool.call', async ($, e, next) => {
61    const { tool, tool_use_id, agentId, requestMeta, ...input } = e as any
62    const guardRule = checkBefore(tool, input)
63    let t0 = 0
64    try { t0 = await $.clock.now() } catch {}
65    const ran: any = guardRule
66      ? { deny: `asclexis-rsi/${guardRule.id}: ${guardRule.message}` }
67      : await next(e)
68    try {
69      const denied = ran.deny !== undefined
70      const failed = denied || ran.isError === true
71      const text: string = denied ? '' : (ran.text ?? '')
72      const hintRule = denied ? undefined : checkAfter(tool, input, text, failed)
73      const rule = guardRule ?? hintRule
74      const { retry, edit_test_fail } = track.note(tool, input, !failed)
75      const ts = await iso($)
76      const row: Row = {
77        v: 1, ts, session: await sessionId($), kind: 'tool', tool,
78        ok: !failed,
79        duration_ms: t0 ? Math.max(0, (await $.clock.now()) - t0) : null,
80        denied,
81        guard: guardRule?.id ?? null,
82        hint: hintRule?.id ?? null,
83        error_class: rule?.errorClass ?? (failed ? classifyError(text) : null),
84        retry, edit_test_fail,
85      }
86      calls++
87      if (failed) fails++
88      if (rule) guards++
89      append($, ts, row)
90      refresh($)
91      if (hintRule) return { ...ran, context: [...(ran.context ?? []), `[asclexis-rsi:${hintRule.id}] ${hintRule.message}`] }
92    } catch {}
93    return ran
94  }).catch(($, e, next) =>
95    next.called ? next(e) : { deny: 'asclexis-rsi: its guard failed, so the call is held. Retry, or disable asclexis-rsi.' },
96  )
97
98  on('command.run', { command: 'rsi' }, async $ => {
99    try {
100      await queue
101      const now = await $.clock.now()
102      const cutoff = new Date(now - 6 * DAY).toISOString().slice(0, 10)
103      const names = (await $.fs.list(LOG_DIR)).map((f: any) => f.name as string).filter((n: string) => /^\d{4}-\d{2}-\d{2}\.jsonl$/.test(n) && n.slice(0, 10) >= cutoff)
104      const rows: Row[] = []
105      for (const n of names)
106        for (const line of ((await $.fs.read(`${LOG_DIR}/${n}`)) as string).split('\n'))
107          try { if (line) rows.push(JSON.parse(line)) } catch {}
108      return { text: scoreboard(await sessionId($), rows) }
109    } catch {
110      return { text: 'RSI scoreboard: no log yet (.claude/logs/rsi is empty or unreadable).' }
111    }
112  })
113}
114

1import { RULES, type Rule } from './rules'
2
3export type Row = {
4  v: 1
5  ts: string
6  session: string
7  kind: 'tool'
8  tool: string
9  ok: boolean
10  duration_ms: number | null
11  denied: boolean
12  guard: string | null
13  hint: string | null
14  error_class: string | null
15  retry: boolean
16  edit_test_fail: boolean
17}
18
19const EDIT_TOOLS = ['Edit', 'Write', 'MultiEdit', 'NotebookEdit']
20const TEST_CMD = /\b(?:pytest|vitest|playwright\s+test|npm\s+(?:run\s+)?test|npm\s+t)\b/
21
22/** Non-reversible 32-bit FNV-1a of the input. Held in memory only, never logged. */
23export function fingerprint(tool: string, input: unknown): string {
24  const s = tool + JSON.stringify(input ?? null)
25  let h = 0x811c9dc5
26  for (let i = 0; i < s.length; i++) h = Math.imul(h ^ s.charCodeAt(i), 0x01000193)
27  return (h >>> 0).toString(16)
28}
29
30const cmdOf = (tool: string, input: unknown): string =>
31  tool === 'Bash' && input && typeof (input as { command?: unknown }).command === 'string'
32    ? (input as { command: string }).command
33    : ''
34
35/** First `before`/deny rule that matches this call, if any. */
36export function checkBefore(tool: string, input: unknown, rules: readonly Rule[] = RULES): Rule | undefined {
37  const cmd = cmdOf(tool, input)
38  return rules.find(r => r.when === 'before' && r.action === 'deny' && r.tool === tool && new RegExp(r.match).test(cmd))
39}
40
41/** First `after` rule that matches the finished call, if any. */
42export function checkAfter(
43  tool: string,
44  input: unknown,
45  text: string,
46  failed: boolean,
47  rules: readonly Rule[] = RULES,
48): Rule | undefined {
49  const cmd = cmdOf(tool, input)
50  return rules.find(
51    r =>
52      r.when === 'after' &&
53      r.tool === tool &&
54      (failed || r.onSuccess === true) &&
55      (r.command === undefined || new RegExp(r.command).test(cmd)) &&
56      new RegExp(r.match).test(text) &&
57      (r.absent === undefined || !new RegExp(r.absent).test(text)),
58  )
59}
60
61/**
62 * Fixed error category for a failure no rule claimed. Never raw text.
63 * Order matters: the first four checks win, then common Bash failure shapes,
64 * then a bounded `exit-<n>` (1-255) from the `Exit code` line, else 'other'.
65 * Returns only fixed literals or `exit-` plus captured digits, never input text.
66 */
67export function classifyError(text: string): string {
68  if (/Exit code 144\b/.test(text)) return 'exit-144-self-kill'
69  if (/timed out|timeout/i.test(text)) return 'timeout'
70  if (/did not match|not found in file|String to replace not found/i.test(text)) return 'edit-no-match'
71  if (/permission denied|EACCES/i.test(text)) return 'permission-denied'
72  if (/ModuleNotFoundError|No module named/.test(text)) return 'module-not-found'
73  if (/command not found|Exit code 127\b/.test(text)) return 'command-not-found'
74  if (/No such file or directory|ENOENT/.test(text)) return 'no-such-file'
75  if (/\b\d+ failed\b/.test(text)) return 'tests-failed'
76  if (/^fatal: /m.test(text)) return 'git-fatal'
77  if (/SyntaxError|syntax error/i.test(text)) return 'syntax-error'
78  const m = /^Exit code (\d{1,3})\b/m.exec(text)
79  if (m) {
80    const n = Number(m[1])
81    if (n >= 1 && n <= 255) return `exit-${n}`
82  }
83  return 'other'
84}
85
86export const isTestCommand = (tool: string, input: unknown): boolean => TEST_CMD.test(cmdOf(tool, input))
87export const isEditTool = (tool: string): boolean => EDIT_TOOLS.includes(tool)
88
89/** In-memory per-session state for retry and edit-then-test-fail detection. */
90export function tracker() {
91  const recent: { key: string; ok: boolean }[] = []
92  let editedSinceTest = false
93  return {
94    /** Call once per finished (or denied) call, in order. */
95    note(tool: string, input: unknown, ok: boolean): { retry: boolean; edit_test_fail: boolean } {
96      const key = fingerprint(tool, input)
97      const retry = recent.some(r => r.key === key && !r.ok)
98      recent.push({ key, ok })
99      if (recent.length > 5) recent.shift()
100      let edit_test_fail = false
101      if (isTestCommand(tool, input)) {
102        edit_test_fail = !ok && editedSinceTest
103        editedSinceTest = false
104      } else if (isEditTool(tool) && ok) editedSinceTest = true
105      return { retry, edit_test_fail }
106    },
107  }
108}
109
110export type Summary = {
111  calls: number
112  fail: number
113  retries: number
114  denials: number
115  hints: Record<string, number>
116  editTestFail: number
117  tools: Record<string, { calls: number; fail: number; retries: number }>
118}
119
120export function summarize(rows: readonly Row[]): Summary {
121  const s: Summary = { calls: 0, fail: 0, retries: 0, denials: 0, hints: {}, editTestFail: 0, tools: {} }
122  for (const r of rows) {
123    if (r.kind !== 'tool') continue
124    const t = (s.tools[r.tool] ??= { calls: 0, fail: 0, retries: 0 })
125    s.calls++
126    t.calls++
127    if (!r.ok) (s.fail++, t.fail++)
128    if (r.retry) (s.retries++, t.retries++)
129    if (r.denied) s.denials++
130    if (r.hint) s.hints[r.hint] = (s.hints[r.hint] ?? 0) + 1
131    if (r.edit_test_fail) s.editTestFail++
132  }
133  return s
134}
135
136const pct = (a: number, b: number) => (b === 0 ? '0%' : `${Math.round((100 * a) / b)}%`)
137
138function block(title: string, s: Summary): string[] {
139  const hints = Object.entries(s.hints).map(([k, n]) => `${k}=${n}`).join(' ') || 'none'
140  const lines = [
141    `${title}: ${s.calls} calls, ${s.fail} fail (${pct(s.fail, s.calls)}), ${s.retries} retries, ${s.denials} denials, ${s.editTestFail} edit->test-fail`,
142    `  hints: ${hints}`,
143  ]
144  for (const [tool, t] of Object.entries(s.tools).sort((a, b) => b[1].calls - a[1].calls))
145    lines.push(`  ${tool.padEnd(14)} calls ${String(t.calls).padStart(4)}  fail ${String(t.fail).padStart(3)}  rate ${pct(t.fail, t.calls).padStart(4)}  retries ${t.retries}`)
146  return lines
147}
148
149export function scoreboard(session: string, rows: readonly Row[]): string {
150  const week = summarize(rows)
151  const mine = summarize(rows.filter(r => r.session === session))
152  return ['RSI scoreboard', ...block('This session', mine), ...block('Last 7 days', week)].join('\n')
153}
154

1// Data-only guard/hint table (RSI contract). `rsi-loop` proposals may edit ONLY this file,
2// replay/cases.ts and docs/agentic/recurring-failures.md.
3// Fields: id, tool, when (before|after), match (regex source; before: over the command,
4// after: over the error/output text), action (deny|hint), message, source, errorClass.
5// Optional after-only refinements: `command` (regex the command must also match),
6// `onSuccess` (also fire when the call did not fail), `absent` (regex that must NOT occur in output).
7export type Rule = {
8  id: string
9  tool: string
10  when: 'before' | 'after'
11  match: string
12  action: 'deny' | 'hint'
13  message: string
14  source: string
15  errorClass: string
16  command?: string
17  onSuccess?: boolean
18  absent?: string
19}
20
21export const RULES: readonly Rule[] = [
22  {
23    id: 'pkill-self',
24    tool: 'Bash',
25    when: 'before',
26    match: String.raw`\bpkill\s+(?:-[A-Za-z0-9]+\s+)*(?:-[A-Za-z]*f[A-Za-z]*|--full)\b`,
27    action: 'deny',
28    message:
29      '`pkill -f` can match this shell itself and kill it (exit 144). Stop by port instead: `lsof -ti:PORT -sTCP:LISTEN | xargs kill`.',
30    source: 'handoff 2026-10-17 s4: pkill -f killed the agent shell (exit 144)',
31    errorClass: 'exit-144-self-kill',
32  },
33  {
34    id: 'playwright-exec-path',
35    tool: 'Bash',
36    when: 'after',
37    match: String.raw`Executable doesn't exist at \S*ms-playwright`,
38    action: 'hint',
39    message:
40      'Playwright bundled browser revision differs. Launch with executablePath=~/.cache/ms-playwright/chromium-1200/chrome-linux64/chrome.',
41    source: 'handoff 2026-10-17 s4: Playwright browser path mismatch',
42    errorClass: 'playwright-exec-missing',
43  },
44  {
45    id: 'gh-pr-edit-projects-classic',
46    tool: 'Bash',
47    when: 'after',
48    match: String.raw`Projects \(classic\)|projectCards`,
49    command: String.raw`\bgh\s+pr\s+edit\b`,
50    action: 'hint',
51    message:
52      '`gh pr edit` fails on Projects (classic). Use: gh api -X PATCH repos/{owner}/{repo}/pulls/N -f title=... -f body=...',
53    source: 'handoff 2026-10-17 s2.2: gh pr edit failing on Projects classic',
54    errorClass: 'gh-projects-classic',
55  },
56  {
57    id: 'venv-husk',
58    tool: 'Bash',
59    when: 'after',
60    match: String.raw`^`,
61    command: String.raw`\.?venv/bin/(?:pytest|python3?\s+-m\s+pytest)\b`,
62    onSuccess: true,
63    absent: String.raw`\d+ (?:passed|failed|skipped|errors?)|no tests ran|collected \d+`,
64    action: 'hint',
65    message:
66      'Exit 0 with no pytest summary: a checked-in venv bin file is a husk that silently no-ops. Use ~/venvs/asclexis-311/bin/pytest.',
67    source: 'memory wsl-pytest-venv-husk-gotcha: checked-in venv bin files exit 0 without running',
68    errorClass: 'venv-husk',
69  },
70  {
71    id: 'llama-cpp-no-compiler',
72    tool: 'Bash',
73    when: 'after',
74    match: String.raw`No CMAKE_C_COMPILER|CMAKE_CXX_COMPILER (?:not set|could not be found)|gcc: command not found`,
75    action: 'hint',
76    message: 'llama-cpp-python needs a C compiler. Install one first: sudo apt-get install -y build-essential cmake.',
77    source: 'handoff 2026-10-17 s2.2: llama-cpp-python with no C compiler',
78    errorClass: 'no-c-compiler',
79  },
80  {
81    id: 'embedding-bundle-missing',
82    tool: 'Bash',
83    when: 'after',
84    match: String.raw`test_hc_emb_002_fetched_model_loads_offline|test_api_rag_index_002[bc]?|test_search_vectors_async_calculates_similarity`,
85    action: 'hint',
86    message:
87      'These failures are environmental. Fetch the model with python scripts/download_models.py embedding. Do not lower the 0.7 threshold. Judge on the collected count (recurring-failures #4).',
88    source: 'pr-review leads 223,218',
89    errorClass: 'embedding-bundle-missing',
90  },
91  {
92    id: 'no-such-file-path',
93    tool: 'Bash',
94    when: 'after',
95    match: String.raw`No such file or directory`,
96    action: 'hint',
97    message:
98      'A path was not found ("No such file or directory"). Resolve and verify the path (ls its parent directory) before retrying the command unchanged.',
99    source: 'rsi-2026-10-24-gpubox-1: Bash/no-such-file cluster, 35 failed calls since 2026-10-21, no path recorded',
100    errorClass: 'no-such-file',
101  },
102]
103