Exam Gate

1// Exam Gate: when Claude finishes a turn, run the project's checklist.
2// Fail -> send Claude back with the failures. Too many fails in a row -> stop and ask the human.
3
4const CONFIG_FILE = '.exam-gate.json'
5const DEFAULT_MAX_ROUNDS = 3
6
7// Shared by the hooks below
8let touched = false // did Claude change anything this turn?
9let rounds = 0 // consecutive automatic send-backs since the human last spoke
10
11async function sha256(text) {
12  const bytes = new TextEncoder().encode(text)
13  const digest = await crypto.subtle.digest('SHA-256', bytes)
14  return Array.from(new Uint8Array(digest)).map((b) => b.toString(16).padStart(2, '0')).join('')
15}
16
17async function loadConfig($) {
18  if (!(await $.fs.exists(CONFIG_FILE))) return null
19  try {
20    const cfg = JSON.parse(await $.fs.read(CONFIG_FILE))
21    if (!cfg || !Array.isArray(cfg.checks)) return { error: CONFIG_FILE + ' needs a "checks" array' }
22    return cfg
23  } catch (err) {
24    return { error: CONFIG_FILE + ' is not valid JSON: ' + String(err) }
25  }
26}
27
28// A command check runs code from the project folder, so the human approves each distinct command once.
29async function commandApproved($, argv) {
30  const key = 'approved:' + (await sha256(JSON.stringify(argv)))
31  if (await $.store.get(key)) return true
32  let answer = 'Skip this check'
33  try {
34    answer = await $.ui.ask('exam-gate wants to run this command as a check: ' + argv.join(' '), ['Allow and remember', 'Skip this check'])
35  } catch {
36    // dismissed, or no one to ask (claude -p): stay unapproved
37  }
38  if (answer !== 'Allow and remember') return false
39  await $.store.set(key, true)
40  return true
41}
42
43// One check -> { ok, detail }
44async function runCheck($, check) {
45  try {
46    if (check.type === 'exists') {
47      const ok = await $.fs.exists(check.path)
48      return { ok, detail: ok ? '' : 'file missing: ' + check.path }
49    }
50    if (check.type === 'min_chars') {
51      if (!(await $.fs.exists(check.path))) return { ok: false, detail: 'file missing: ' + check.path }
52      const text = await $.fs.read(check.path)
53      const ok = text.length >= check.min
54      return { ok, detail: ok ? '' : check.path + ' has ' + text.length + ' characters, needs at least ' + check.min }
55    }
56    if (check.type === 'contains' || check.type === 'not_contains') {
57      if (!(await $.fs.exists(check.path))) return { ok: false, detail: 'file missing: ' + check.path }
58      const text = await $.fs.read(check.path)
59      const found = new RegExp(check.pattern, check.flags || 'i').test(text)
60      const ok = check.type === 'contains' ? found : !found
61      const verb = check.type === 'contains' ? 'does not contain' : 'still contains'
62      return { ok, detail: ok ? '' : check.path + ' ' + verb + ' /' + check.pattern + '/' }
63    }
64    if (check.type === 'command') {
65      if (!Array.isArray(check.argv) || check.argv.length === 0) return { ok: false, detail: 'command check needs an "argv" list' }
66      if (!(await commandApproved($, check.argv))) return { ok: false, detail: 'command not approved by the user: ' + check.argv.join(' ') }
67      const r = await $.process.run(check.argv, { timeoutMs: (check.timeout_s || 60) * 1000 })
68      const tail = (r.stderr || r.stdout || '').trim().slice(-500)
69      return { ok: r.exitCode === 0, detail: r.exitCode === 0 ? '' : check.argv.join(' ') + ' exited ' + r.exitCode + (tail ? ': ' + tail : '') }
70    }
71    return { ok: false, detail: 'unknown check type: ' + check.type }
72  } catch (err) {
73    return { ok: false, detail: 'check could not run (' + check.type + '): ' + String(err) }
74  }
75}
76
77async function runAll($, cfg) {
78  const failures = []
79  for (const check of cfg.checks) {
80    const r = await runCheck($, check)
81    if (!r.ok) failures.push(r.detail)
82  }
83  return failures
84}
85
86export function register(on) {
87  on('session.start', async ($, e, next) => {
88    await $.command.register({ name: 'exam', description: 'Run the .exam-gate.json checklist now' })
89    return next(e)
90  })
91
92  // Remember whether Claude changed anything this turn
93  on('tool.call', { tool: ['Edit', 'Write', 'NotebookEdit'] }, async ($, e, next) => {
94    touched = true
95    return next(e)
96  })
97
98  // The human speaking resets the breaker; our own send-backs do not come from the composer
99  on('prompt.submit', async ($, e, next) => {
100    if (e.origin && e.origin.kind === 'composer') rounds = 0
101    return next(e)
102  })
103
104  on('command.run', { command: 'exam' }, async ($) => {
105    const cfg = await loadConfig($)
106    if (!cfg) return { text: 'No ' + CONFIG_FILE + ' in this folder.' }
107    if (cfg.error) return { text: cfg.error }
108    const failures = await runAll($, cfg)
109    if (failures.length === 0) return { text: 'All ' + cfg.checks.length + ' checks passed.' }
110    return { text: failures.length + ' of ' + cfg.checks.length + ' checks failed:\n- ' + failures.join('\n- ') }
111  })
112
113  on('turn.complete', async ($, e, next) => {
114    const result = await next(e)
115    // Only a finished main-conversation answer counts
116    if (e.agentId || e.reason !== 'answer') return result
117
118    const cfg = await loadConfig($)
119    if (!cfg) return result
120    if (cfg.error) {
121      $.ui.toast('exam-gate: ' + cfg.error)
122      return result
123    }
124    if (!touched && !cfg.always) return result
125    touched = false
126
127    const failures = await runAll($, cfg)
128    if (failures.length === 0) {
129      rounds = 0
130      $.ui.log('exam-gate: all ' + cfg.checks.length + ' checks passed')
131      return result
132    }
133
134    const maxRounds = cfg.max_rounds || DEFAULT_MAX_ROUNDS
135    rounds += 1
136    if (rounds > maxRounds) {
137      // Breaker tripped: stop automating, hand control back to the human
138      $.ui.toast('exam-gate: still failing after ' + maxRounds + ' tries. Stopped, your call.', { timeoutMs: 15000 })
139      $.ui.log('exam-gate: breaker tripped. Remaining failures:\n- ' + failures.join('\n- '))
140      return result
141    }
142
143    $.ui.log('exam-gate: ' + failures.length + ' check(s) failed, sending Claude back (try ' + rounds + ' of ' + maxRounds + ')')
144    // Not awaited: it waits for the session to go idle, which happens after this hook returns
145    $.prompt.submit({
146      text: 'The checklist for this task did not pass. Fix only these failures, then stop:\n- ' + failures.join('\n- '),
147    }).catch(() => {})
148    return result
149  })
150}
151