evidence-check

1import { update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4const TURN = { plugin: 'evidence-check', key: 'turn' } as const
5
6const JEV_TIMEOUT_MS = 8000
7// Jev's probability that the reply asserts verified success; below this the phrase was a hedge, a recap or an edit note.
8const MIN_CLAIM = 0.65
9
10// Bash commands that test, build, lint or probe something.
11const CHECK_COMMAND = new RegExp(
12  [
13    String.raw`\bgo\s+(?:test|vet|build)\b`,
14    String.raw`\b(?:pytest|tox|nox|phpunit|rspec|ctest)\b`,
15    String.raw`\bpython3?\s+-m\s+(?:pytest|unittest|mypy|ruff)\b`,
16    String.raw`\b(?:npm|pnpm|yarn|bun|deno)\s+(?:run\s+)?(?:test|tests|build|lint|typecheck|type-check|check|e2e|verify)\b`,
17    String.raw`\b(?:vitest|jest|mocha|tsc|eslint|biome|playwright|cypress)\b`,
18    String.raw`\bprettier\s+(?:--check|-c)\b`,
19    String.raw`\bcargo\s+(?:test|check|build|clippy|nextest)\b`,
20    String.raw`\b(?:ruff|mypy|pyright|flake8|pylint|golangci-lint|staticcheck|shellcheck|hadolint|tflint)\b`,
21    String.raw`\b(?:make|just|task)\s+(?:test|tests|check|lint|build|verify|e2e|ci)\b`,
22    String.raw`\b(?:mvn|mvnw|gradle|gradlew)\b[^\n;&|]*\b(?:test|verify|check|build)\b`,
23    String.raw`\b(?:dotnet|swift|zig)\s+(?:test|build)\b`,
24    String.raw`\bnode\s+--test\b`,
25    String.raw`\b(?:curl|wget|grpcurl|httpie|k6)\b`,
26    String.raw`\b(?:psql|mysql|sqlite3|mongosh|redis-cli)\b`,
27  ].join('|'),
28  'i',
29)
30
31// MCP tools that query a database, drive a browser or read live telemetry.
32const CHECK_MCP = /playwright|puppeteer|browser|chrome|sql|postgres|mysql|mongo|redis|database|bigquery|query|sentry|logging|monitoring|test/i
33
34export function checkOf(tool: string, input: Record<string, unknown>): string | undefined {
35  if (tool === 'Bash') return CHECK_COMMAND.exec(String(input.command ?? ''))?.[0]
36  if (tool.startsWith('mcp__') && CHECK_MCP.test(tool)) return tool
37  return undefined
38}
39
40const CLAIM = new RegExp(
41  [
42    String.raw`\b(?:all\s+)?(?:the\s+)?(?:unit\s+|integration\s+|e2e\s+)?tests?\s+(?:now\s+|all\s+|still\s+)?(?:pass(?:es|ed)?|are\s+(?:passing|green)|succeed(?:s|ed)?)\b`,
43    String.raw`\ball\s+(?:green|passing|checks\s+pass(?:ed)?)\b`,
44    String.raw`\b(?:it|they|build|suite|ci|checks?|lint|everything)\s+(?:now\s+)?pass(?:es|ed)?\b`,
45    String.raw`\bpassed\b(?!\s+(?:to|in|into|as|through|by|the|a|an|it|them|along|down|on|over)\b)`,
46    String.raw`(?<!\b(?:a|the)\s)\bfixed\b(?![-\w])(?!\s+(?:size|width|height|point|cost|number|rate|length|position|at|by)\b)`,
47    String.raw`\bverified\b`,
48    String.raw`\b(?:works|working)\s+now\b|\bnow\s+works\b|\bit\s+works\b`,
49    String.raw`\bno\s+breaking\s+changes?\b`,
50    String.raw`\bbuilds?\s+(?:succeeds|succeeded|is\s+green|passes|cleanly|successfully)\b`,
51    String.raw`\b(?:sudah|udah)\s+(?:jalan|berjalan|beres|fix|aman|bisa|works?)\b`,
52    String.raw`\bberhasil\b`,
53    String.raw`\b(?:test|tes)\s+(?:sudah\s+)?lulus\b`,
54  ].join('|'),
55  'gi',
56)
57
58// Words earlier in the same sentence that turn a claim into a hedge, a negation or a plan.
59const HEDGE =
60  /\b(?:not|never|cannot|unable|without|unverified|untested|should|will|would|might|may|could|if|once|until|unless|to|ensure|expect|expected|hope|whether|belum|tidak|tak|gak|nggak|bukan|kalau|jika|harusnya|seharusnya|semoga|supaya)\b|n't\b/i
61
62/** The reply with code, inline code, block quotes and quoted strings removed. */
63export function stripQuoted(text: string): string {
64  return text
65    .replace(/```[\s\S]*?(?:```|$)/g, ' ')
66    .replace(/~~~[\s\S]*?(?:~~~|$)/g, ' ')
67    .replace(/`[^`\n]*`/g, ' ')
68    .replace(/^[ \t]*>.*$/gm, ' ')
69    .replace(/"[^"\n]*"/g, ' ')
70    .replace(/“[^”\n]*”/g, ' ')
71}
72
73/** The first success claim the reply states as fact, if any. */
74export function findClaim(answer: string): string | undefined {
75  const text = stripQuoted(answer)
76  for (const m of text.matchAll(CLAIM)) {
77    const sentence = text.slice(Math.max(0, m.index - 80), m.index).split(/[.!?;:\n]/).at(-1) ?? ''
78    if (!HEDGE.test(sentence)) return m[0].replace(/\s+/g, ' ').trim()
79  }
80  return undefined
81}
82
83const SECRETS: readonly [RegExp, string][] = [
84  [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(-----END [A-Z ]*PRIVATE KEY-----|$)/g, '[REDACTED KEY]'],
85  [/\beyJ[\w-]+\.[\w-]+\.[\w-]*/g, '[REDACTED JWT]'],
86  [/\b(Bearer|Basic|Token)\s+[\w.~+/=-]{8,}/gi, '$1 [REDACTED]'],
87  [/\b([a-z][a-z0-9+.-]*:\/\/)[^\s:/@]+:[^\s@/]+@/gi, '$1[REDACTED]@'],
88  [/\b((?:set-)?cookie)\s*:\s*[^\n]+/gi, '$1: [REDACTED]'],
89  [/\b(password|passwd|pwd|pass|secret|token|api[_-]?key|access[_-]?key|private[_-]?key|client[_-]?secret|auth)(["']?\s*[:=]\s*)("[^"]*"|'[^']*'|\S+)/gi, '$1$2[REDACTED]'],
90  [/\bsshpass\s+-p\s*\S+/g, 'sshpass -p [REDACTED]'],
91  [/\b(sk-[\w-]{16,}|gh[pousr]_\w{20,}|github_pat_\w{20,}|xox[abprs]-[\w-]{10,}|glpat-[\w-]{16,}|AKIA[0-9A-Z]{16}|AIza[\w-]{35})/g, '[REDACTED KEY]'],
92  [/\b(?=[A-Za-z0-9+/_-]*\d)(?=[A-Za-z0-9+/_-]*[A-Za-z])[A-Za-z0-9+/_-]{40,}={0,2}/g, '[REDACTED]'],
93]
94
95export function redact(text: string): string {
96  return SECRETS.reduce((t, [pattern, replacement]) => t.replace(pattern, replacement), text)
97}
98
99const excerpt = (text: string) => (text.length > 3000 ? `${text.slice(0, 1000)}\n…\n${text.slice(-2000)}` : text)
100
101type Confirmed = 'yes' | 'no' | 'unavailable'
102
103// Jev drops hedges, recaps of an earlier turn's run and plain edit notes that the phrase list cannot tell apart.
104async function confirm($: EngineInterface, answer: string, claim: string, tools: number): Promise<Confirmed> {
105  const request = {
106    state: {
107      reply: excerpt(redact(stripQuoted(answer))),
108      matched_phrase: redact(claim),
109      tool_calls_this_turn: tools,
110      note: 'No test, build, lint, curl or DB check ran this turn.',
111    },
112    questions: {
113      claims: {
114        type: 'noul',
115        instructions:
116          'Does `reply` assert, as a fact, that work was verified to succeed or now works (tests pass, bug fixed and working, build green, verified, no breaking change, nothing else affected)? False if it only hedges (should/might/once you run), only describes edits made, or reports a result from an earlier turn.',
117      },
118    },
119  }
120  try {
121    const home = (await $.env.get('HOME')) ?? ''
122    const ran = await $.process.run([home ? `${home}/.local/bin/jev` : 'jev'], { stdin: JSON.stringify(request), timeoutMs: JEV_TIMEOUT_MS })
123    if (ran.exitCode !== 0) return 'unavailable'
124    const noul = (JSON.parse(ran.stdout) as { claims?: { noul?: unknown } }).claims?.noul
125    if (typeof noul !== 'number') return 'unavailable'
126    return noul >= MIN_CLAIM ? 'yes' : 'no'
127  } catch {
128    return 'unavailable'
129  }
130}
131
132export const register: Register = on => {
133  on('turn.start', async ($, e, next) => {
134    await $.state.set(TURN, { tools: 0, checks: [] })
135    return next(e)
136  })
137
138  on('tool.call', async ($, e, next) => {
139    const result = await next(e)
140    if (result.deny === undefined) {
141      const check = checkOf(String(e.tool), e as Record<string, unknown>)
142      await update($, TURN, t => ({ tools: (t?.tools ?? 0) + 1, checks: check ? [...(t?.checks ?? []), check] : (t?.checks ?? []) }))
143    }
144    return result
145  }).catch(($, e, next) => next(e))
146
147  on('turn.complete', async ($, e, next) => {
148    const done = await next(e)
149    if (e.agentId !== undefined || e.reason !== 'answer') return done
150
151    const { value: turn } = await $.state.get(TURN)
152    if (turn && turn.checks.length > 0) return done
153    const claim = findClaim(e.answer)
154    if (!claim) return done
155
156    const confirmed = await confirm($, e.answer, claim, turn?.tools ?? 0)
157    if (confirmed === 'no') return done
158
159    const shown = redact(claim).slice(0, 80)
160    const text =
161      `evidence-check: the reply claims "${shown}" but no test or check ran this turn` +
162      (confirmed === 'unavailable' ? ' (Jev unavailable: phrase match only)' : '')
163    $.ui.toast(text, { timeoutMs: 8000 })
164    await $.session.append({ message: { type: 'system', content: [{ type: 'text', text }] } })
165    return done
166  }).catch(($, e, next) => next(e))
167}
168

1/** What ran during the current main-loop turn (subagents' calls included). */
2export type EvidenceCheckTurn = {
3  tools: number
4  checks: string[]
5}
6
7declare module 'claude-code' {
8  interface PluginState {
9    'evidence-check': {
10      turn: EvidenceCheckTurn
11    }
12  }
13}
14