qa-gate

1import type { EngineInterface, PluginOptions, Register } from 'claude-code'
2
3import { asBool, asList, asNumber, asString, parseConfigText, pick } from './config'
4import type { RawConfig } from './config'
5import { bashWrites, changedFiles, porcelainPaths } from './bash-writes'
6import { CHECKS, LOCKFILES, failureMessage, planScripts, scopeArgs, scriptArgv } from './scripts'
7import type { Check, Failure, PackageManager } from './scripts'
8
9type Settings = {
10  enabled: boolean
11  packageJson: string
12  packageManager: PackageManager | 'auto'
13  checks: Check[]
14  scriptNames: Partial<Record<Check, string[]>>
15  codeExtensions: string[]
16  maxAttempts: number
17  timeoutMs: number
18  scope: 'all' | 'changed'
19  testArgs: string[] | undefined
20  watchBash: boolean
21}
22
23const DEFAULT_CODE = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.mts', '.cts', '.vue', '.svelte', '.astro', '.css', '.scss', '.html', '.json', '.md', '.mdx']
24
25/** Reads a JSON config file of the project; a missing or broken one is empty. */
26async function readConfigFile($: EngineInterface, path: string): Promise<RawConfig> {
27  if (path.trim() === '' || !(await $.fs.exists(path))) return {}
28  const parsed = parseConfigText(path, await $.fs.read(path))
29  if (typeof parsed === 'string') {
30    $.ui.log(parsed)
31    return {}
32  }
33  return parsed
34}
35
36async function load($: EngineInterface, options: PluginOptions): Promise<Settings> {
37  const configFile = asString(options.configFile) ?? '.claude/guardrail-mods/qa-gate.json'
38  const file = await readConfigFile($, configFile)
39  const manager = pick(file, options, 'packageManager', 'auto', asString)
40  const names: Partial<Record<Check, string[]>> = {}
41  const scripts = typeof file.scripts === 'object' && file.scripts !== null ? (file.scripts as RawConfig) : {}
42  for (const check of CHECKS) {
43    const list = asList(scripts[check])
44    if (list !== undefined) names[check] = list
45  }
46  return {
47    enabled: pick(file, options, 'enabled', true, asBool),
48    packageJson: pick(file, options, 'packageJson', 'package.json', asString),
49    packageManager: (['npm', 'pnpm', 'yarn', 'bun'] as const).find(m => m === manager) ?? 'auto',
50    checks: pick(file, options, 'checks', [...CHECKS], asList).filter((c): c is Check => (CHECKS as readonly string[]).includes(c)),
51    scriptNames: names,
52    codeExtensions: pick(file, options, 'codeExtensions', DEFAULT_CODE, asList).map(x => x.toLowerCase()),
53    maxAttempts: pick(file, options, 'maxAttempts', 3, asNumber),
54    // timeoutSeconds is the /config option; timeoutMs is kept for project files.
55    timeoutMs: asNumber(file.timeoutMs) ?? pick(file, options, 'timeoutSeconds', 300, asNumber) * 1000,
56    scope: pick(file, options, 'scope', 'all', asString) === 'changed' ? 'changed' : 'all',
57    testArgs: asList(file.testArgs) ?? asList(options.testArgs),
58    watchBash: pick(file, options, 'watchBash', true, asBool),
59  }
60}
61
62async function detectManager($: EngineInterface, dir: string): Promise<PackageManager> {
63  for (const [lockfile, manager] of LOCKFILES) {
64    if (await $.fs.exists(`${dir}${lockfile}`)) return manager
65  }
66  return 'npm'
67}
68
69// The code files Claude changed since the gate last passed (the gate runs
70// only when there are some), and how many times in a row it has held the turn.
71const changed = new Set<string>()
72let attempts = 0
73
74// The repository root, once found; null outside a git repository.
75let gitRoot: string | null | undefined
76
77/** Files with uncommitted changes that pass `keep`, by absolute path, stamped with mtime and size; undefined outside git. */
78async function dirtyFiles($: EngineInterface, keep: (path: string) => boolean): Promise<Map<string, string> | undefined> {
79  if (gitRoot === undefined) {
80    const top = await $.process.run(['git', 'rev-parse', '--show-toplevel'], { timeoutMs: 10_000 }).catch(() => undefined)
81    gitRoot = top !== undefined && top.exitCode === 0 ? top.stdout.trim() : null
82  }
83  if (gitRoot === null) return undefined
84  const status = await $.process.run(['git', 'status', '--porcelain', '-z', '--untracked-files=all'], { timeoutMs: 10_000 }).catch(() => undefined)
85  if (status === undefined || status.exitCode !== 0) return undefined
86  const files = new Map<string, string>()
87  for (const rel of porcelainPaths(status.stdout).filter(keep).slice(0, 500)) {
88    const path = `${gitRoot}/${rel}`
89    const stat = await $.fs.stat(path).catch(() => undefined)
90    if (stat !== undefined && stat.kind === 'file') files.set(path, `${stat.mtimeMs}:${stat.size}`)
91  }
92  return files
93}
94
95/** Runs the gate: undefined when every script passes or none applies, else what Claude must fix. */
96async function runGate($: EngineInterface, settings: Settings): Promise<string | undefined> {
97  if (!(await $.fs.exists(settings.packageJson))) {
98    $.ui.status(`🧪 qa-gate: no ${settings.packageJson}, nothing to run`)
99    return undefined
100  }
101  const pkg = parseConfigText(settings.packageJson, await $.fs.read(settings.packageJson))
102  if (typeof pkg === 'string') {
103    $.ui.log(pkg)
104    return undefined
105  }
106  const scripts = typeof pkg.scripts === 'object' && pkg.scripts !== null ? (pkg.scripts as RawConfig) : {}
107  const plan = planScripts(scripts, settings.scriptNames, settings.checks)
108  if (plan.length === 0) {
109    $.ui.status(`🧪 qa-gate: no ${settings.checks.join(', ')} scripts in ${settings.packageJson}`)
110    return undefined
111  }
112
113  const cut = settings.packageJson.lastIndexOf('/')
114  const dir = cut < 0 ? '' : settings.packageJson.slice(0, cut + 1)
115  const manager = settings.packageManager === 'auto' ? await detectManager($, dir) : settings.packageManager
116
117  // In `changed` scope, lint, spell check and (with testArgs) tests get only the changed files that still exist.
118  const files: string[] = []
119  if (settings.scope === 'changed') {
120    for (const file of changed) if (await $.fs.exists(file)) files.push(file)
121  }
122
123  const failures: Failure[] = []
124  const commands = new Map<string, string>()
125  for (const { check, script } of plan) {
126    const argv = scriptArgv(manager, script, scopeArgs(check, files, settings.testArgs))
127    commands.set(script, argv.join(' '))
128    $.ui.status(`🧪 qa-gate: running ${script}…`)
129    const started = Date.now()
130    try {
131      const ran = await $.process.run(argv, {
132        cwd: dir === '' ? undefined : dir,
133        env: { CI: 'true', FORCE_COLOR: '0' },
134        timeoutMs: settings.timeoutMs,
135      })
136      if (ran.exitCode !== 0) {
137        failures.push({ check, script, exitCode: ran.exitCode, output: `${ran.stdout}\n${ran.stderr}` })
138      }
139    } catch (error) {
140      const reason = error instanceof Error ? error.message : String(error)
141      failures.push({
142        check,
143        script,
144        exitCode: -1,
145        // The engine rejects when the command is still running at the timeout.
146        output: /tim(ed|e) ?out|still running/i.test(reason) || Date.now() - started >= settings.timeoutMs - 1000
147          ? `${script} did not finish within ${Math.round(settings.timeoutMs / 1000)}s (raise timeoutSeconds, or set scope to "changed"): ${reason}`
148          : reason,
149      })
150    }
151  }
152
153  // Say which checks had no script, so a missing one never reads as a pass.
154  const missing = settings.checks.filter(check => !plan.some(p => p.check === check))
155  const none = missing.length > 0 ? ` · no ${missing.join(', ')} script` : ''
156  const passed = plan.length - failures.length
157  $.ui.status(
158    failures.length === 0
159      ? `✅ qa-gate: ${passed}/${plan.length} passed${none}`
160      : `❌ qa-gate: ${failures.map(f => f.script).join(', ')} failed${none}`,
161  )
162  return failures.length === 0 ? undefined : failureMessage(failures, script => commands.get(script) ?? script)
163}
164
165export const register: Register = (on, options) => {
166  on('tool.call', { tool: ['Write', 'Edit', 'NotebookEdit'] }, async ($, e, next) => {
167    const ran = await next(e)
168    if (ran.deny !== undefined || ran.isError === true) return ran
169    const path = e.tool === 'NotebookEdit' ? e.notebook_path : e.tool === 'Write' || e.tool === 'Edit' ? e.file_path : ''
170    const settings = await load($, options)
171    if (settings.codeExtensions.some(ext => path.toLowerCase().endsWith(ext))) changed.add(path)
172    return ran
173  })
174
175  // Code changed through the shell counts too: what the command names as
176  // written, and what git shows changed while it ran.
177  on('tool.call', { tool: 'Bash' }, async ($, e, next) => {
178    const settings = await load($, options)
179    if (!settings.enabled || !settings.watchBash) return next(e)
180    const isCode = (path: string) => settings.codeExtensions.some(ext => path.toLowerCase().endsWith(ext))
181
182    const before = await dirtyFiles($, isCode)
183    const ran = await next(e)
184    if (ran.deny !== undefined) return ran
185    const after = before === undefined ? undefined : await dirtyFiles($, isCode)
186    if (before !== undefined && after !== undefined) for (const file of changedFiles(before, after)) changed.add(file)
187    for (const write of bashWrites(e.command)) if (isCode(write.path)) changed.add(write.path)
188    return ran
189  })
190
191  on('classic.Stop', async ($, e, next) => {
192    const settings = await load($, options)
193    if (!settings.enabled || changed.size === 0) return next(e)
194
195    const problems = await runGate($, settings)
196    if (problems === undefined) {
197      changed.clear()
198      attempts = 0
199      return next(e)
200    }
201    attempts += 1
202    if (attempts > settings.maxAttempts) {
203      $.ui.toast(`qa-gate: still failing after ${settings.maxAttempts} tries; letting the turn end`)
204      // Start over on the next code change rather than holding every later turn.
205      changed.clear()
206      attempts = 0
207      return next(e)
208    }
209    return { block: problems }
210  })
211}
212

1// Shared by every guardrail mod (each plugin carries its own copy, since a
2// plugin may only import its own files). Settings are layered:
3//   1. the mod's built-in defaults,
4//   2. the plugin's userConfig options (/config, or pluginConfigs in settings),
5//   3. the project's JSON config file, when it exists.
6// Reading the file needs `$`, which never crosses an import, so each
7// register.ts reads it and hands the text to `parseConfigText`.
8import type { PluginOptions } from 'claude-code'
9
10export type RawConfig = Record<string, unknown>
11
12const isRecord = (value: unknown): value is RawConfig =>
13  typeof value === 'object' && value !== null && !Array.isArray(value)
14
15/** Parses a config file's text: an object, or an error message to log. */
16export function parseConfigText(path: string, text: string): RawConfig | string {
17  try {
18    const parsed: unknown = JSON.parse(text)
19    return isRecord(parsed) ? parsed : `${path} is not a JSON object; using the defaults`
20  } catch (error) {
21    return `could not parse ${path}: ${error instanceof Error ? error.message : String(error)}`
22  }
23}
24
25/** The first defined value among the file's, the option's and the default. */
26export function pick<T>(
27  file: RawConfig,
28  options: PluginOptions,
29  key: string,
30  fallback: T,
31  read: (value: unknown) => T | undefined,
32): T {
33  return read(file[key]) ?? read(options[key]) ?? fallback
34}
35
36export const asBool = (value: unknown): boolean | undefined =>
37  typeof value === 'boolean' ? value : value === 'true' ? true : value === 'false' ? false : undefined
38
39export const asString = (value: unknown): string | undefined =>
40  typeof value === 'string' ? value : undefined
41
42export const asNumber = (value: unknown): number | undefined =>
43  typeof value === 'number' && Number.isFinite(value) ? value : undefined
44
45/** A list from a JSON array, a `multiple` option, or a comma-separated string. */
46export const asList = (value: unknown): string[] | undefined => {
47  if (Array.isArray(value)) return value.filter((item): item is string => typeof item === 'string')
48  if (typeof value === 'string' && value.trim() !== '') {
49    return value.split(',').map(item => item.trim()).filter(item => item !== '')
50  }
51  return undefined
52}
53

1// The files a shell command writes, read from the command itself: redirects,
2// tee, in-place sed/perl, cp/mv/install destinations, dd of=, curl -o and
3// wget -O. A best-effort reading (a script the command runs can write
4// anything), so each mod also compares the working tree before and after.
5// Shared by the guardrail mods; each plugin carries its own copy.
6
7export type BashWrite = {
8  path: string
9  /** How the command writes it: new text (`redirect`, `tee`, `in-place`, `download`) or a copy (`copy`). */
10  via: 'redirect' | 'tee' | 'in-place' | 'copy' | 'download'
11}
12
13/** Splits a command into words, honoring quotes and backslashes; operators are their own words. */
14export function shellWords(command: string): string[] {
15  const words: string[] = []
16  let word = ''
17  let hasWord = false
18  let quote: '"' | "'" | undefined
19  const push = () => {
20    if (hasWord) words.push(word)
21    word = ''
22    hasWord = false
23  }
24  for (let i = 0; i < command.length; i++) {
25    const c = command[i]!
26    if (quote !== undefined) {
27      if (c === quote) quote = undefined
28      else if (c === '\\' && quote === '"' && i + 1 < command.length) word += command[++i]
29      else word += c
30      continue
31    }
32    if (c === '"' || c === "'") {
33      quote = c
34      hasWord = true
35    } else if (c === '\\' && i + 1 < command.length) {
36      word += command[++i]
37      hasWord = true
38    } else if (/\s/.test(c)) {
39      push()
40      if (c === '\n') words.push(';')
41    } else if (c === ';' || c === '|' || c === '&' || c === '(' || c === ')') {
42      push()
43      const pair = c + (command[i + 1] ?? '')
44      if (pair === '&&' || pair === '||' || pair === '>|') {
45        words.push(pair)
46        i++
47      } else if (c === '&' && command[i + 1] === '>') {
48        words.push('&>')
49        i++
50      } else words.push(c)
51    } else if (c === '>' || c === '<') {
52      // A leading fd number (2>) belongs to the operator, not the word before it.
53      const fd = /^\d$/.test(word) ? word : ''
54      if (fd === '') push()
55      else {
56        word = ''
57        hasWord = false
58      }
59      let op = fd + c
60      while (command[i + 1] === '>' || command[i + 1] === '&' || (c === '<' && command[i + 1] === '<')) op += command[++i]
61      words.push(op)
62    } else {
63      word += c
64      hasWord = true
65    }
66  }
67  push()
68  return words
69}
70
71const SEPARATORS = new Set([';', '&&', '||', '|', '&', '(', ')'])
72const NOT_A_FILE = /^(\/dev\/|&\d|-$)/
73
74const isRedirect = (word: string) => /^(\d|&)?>{1,2}\|?$/.test(word)
75
76/** The files the command writes, by how it writes them. */
77export function bashWrites(command: string): BashWrite[] {
78  // A heredoc's body is data, not shell: keep its opening line, drop the body.
79  const shell = command.replace(/(<<-?\s*(['"]?)([\w-]+)\2[^\n]*)\n[\s\S]*?\n\s*\3\s*(?=\n|$)/g, '$1')
80  const words = shellWords(shell)
81  const writes: BashWrite[] = []
82  const add = (path: string | undefined, via: BashWrite['via']) => {
83    if (path !== undefined && path !== '' && !NOT_A_FILE.test(path) && !SEPARATORS.has(path) && !isRedirect(path)) writes.push({ path, via })
84  }
85
86  // Redirects anywhere in the command
87  words.forEach((word, i) => {
88    if (isRedirect(word)) add(words[i + 1], 'redirect')
89  })
90
91  // Each simple command: its program and arguments, redirects left out
92  const segments: string[][] = [[]]
93  for (let i = 0; i < words.length; i++) {
94    const word = words[i]!
95    if (SEPARATORS.has(word)) segments.push([])
96    else if (isRedirect(word) || /^\d?<{1,3}$/.test(word)) i++
97    else segments.at(-1)!.push(word)
98  }
99
100  for (const segment of segments) {
101    // Skip env assignments and wrappers: FOO=1 sudo sed -i ...
102    let start = 0
103    while (start < segment.length && (/^[A-Za-z_][A-Za-z0-9_]*=/.test(segment[start]!) || ['sudo', 'command', 'env', 'xargs', 'time', 'nohup'].includes(segment[start]!))) start++
104    const [program = '', ...args] = segment.slice(start)
105    const name = program.split('/').at(-1) ?? program
106    const operands = args.filter(a => !a.startsWith('-'))
107
108    if (name === 'tee') {
109      for (const file of operands) add(file, 'tee')
110    } else if (name === 'sed' || name === 'gsed') {
111      if (args.some(a => /^-[a-zA-Z]*i/.test(a) || a.startsWith('--in-place'))) {
112        // The first operand is the script, unless one was given with -e/-f.
113        const hasScriptOption = args.some(a => a === '-e' || a === '-f' || a.startsWith('--expression') || a.startsWith('--file'))
114        const scriptArgs = new Set<string>()
115        args.forEach((a, i) => {
116          if ((a === '-e' || a === '-f') && args[i + 1] !== undefined) scriptArgs.add(args[i + 1]!)
117        })
118        // macOS spells no backup suffix as a separate '' after -i
119        const files = operands.filter(a => a !== '' && !scriptArgs.has(a))
120        for (const file of hasScriptOption ? files : files.slice(1)) add(file, 'in-place')
121      }
122    } else if (name === 'perl') {
123      if (args.some(a => /^-[a-zA-Z]*i/.test(a))) {
124        const scriptArgs = new Set<string>()
125        args.forEach((a, i) => {
126          if (/^-[a-zA-Z]*e$/.test(a) && args[i + 1] !== undefined) scriptArgs.add(args[i + 1]!)
127        })
128        for (const file of operands.filter(a => !scriptArgs.has(a))) add(file, 'in-place')
129      }
130    } else if (name === 'cp' || name === 'mv' || name === 'install' || name === 'rsync') {
131      const target = args.findIndex(a => a === '-t' || a === '--target-directory')
132      add(target >= 0 ? args[target + 1] : operands.at(-1), 'copy')
133    } else if (name === 'dd') {
134      for (const a of args) if (a.startsWith('of=')) add(a.slice(3), 'redirect')
135    } else if (name === 'curl' || name === 'wget') {
136      args.forEach((a, i) => {
137        if ((name === 'curl' && (a === '-o' || a === '--output')) || (name === 'wget' && (a === '-O' || a === '--output-document'))) {
138          add(args[i + 1], 'download')
139        }
140      })
141    }
142  }
143
144  const seen = new Set<string>()
145  return writes.filter(w => {
146    const key = `${w.via}:${w.path}`
147    if (seen.has(key)) return false
148    seen.add(key)
149    return true
150  })
151}
152
153/** Parses `git status --porcelain -z`: the paths with changes or untracked, relative to the repository root. */
154export function porcelainPaths(output: string): string[] {
155  const entries = output.split('\0')
156  const paths: string[] = []
157  for (let i = 0; i < entries.length; i++) {
158    const entry = entries[i]!
159    if (entry.length < 4) continue
160    const status = entry.slice(0, 2)
161    paths.push(entry.slice(3))
162    // A rename or copy is followed by its old path, which is gone.
163    if (status.includes('R') || status.includes('C')) i++
164  }
165  return paths
166}
167
168/** The files whose change time moved between two snapshots, or that only the second has. */
169export function changedFiles(before: ReadonlyMap<string, string>, after: ReadonlyMap<string, string>): string[] {
170  return [...after].filter(([path, mtime]) => before.get(path) !== mtime).map(([path]) => path)
171}
172

1// Which package.json scripts the gate runs, how, and how a failure is told.
2
3export type Check = 'lint' | 'typecheck' | 'spellcheck' | 'test'
4
5export const CHECKS: readonly Check[] = ['lint', 'typecheck', 'spellcheck', 'test']
6
7/** The script names each check looks for, first match wins. */
8export const DEFAULT_SCRIPTS: Record<Check, string[]> = {
9  lint: ['lint'],
10  typecheck: ['typecheck', 'type-check', 'check-types', 'types', 'tsc'],
11  spellcheck: ['spellcheck', 'spell-check', 'spell', 'cspell'],
12  test: ['test', 'test:unit'],
13}
14
15export type PackageManager = 'npm' | 'pnpm' | 'yarn' | 'bun'
16
17export const LOCKFILES: readonly [string, PackageManager][] = [
18  ['pnpm-lock.yaml', 'pnpm'],
19  ['yarn.lock', 'yarn'],
20  ['bun.lockb', 'bun'],
21  ['bun.lock', 'bun'],
22  ['package-lock.json', 'npm'],
23]
24
25// npm init's placeholder test script, which always fails.
26const PLACEHOLDER_TEST = /no test specified/
27
28/** The scripts to run, in order: the first defined name for each check, skipping missing ones. */
29export function planScripts(
30  scripts: Record<string, unknown>,
31  names: Partial<Record<Check, readonly string[]>>,
32  checks: readonly Check[] = CHECKS,
33): { check: Check; script: string }[] {
34  const plan: { check: Check; script: string }[] = []
35  for (const check of checks) {
36    const script = (names[check] ?? DEFAULT_SCRIPTS[check]).find(name => {
37      const body = scripts[name]
38      return typeof body === 'string' && body.trim() !== '' && !(check === 'test' && PLACEHOLDER_TEST.test(body))
39    })
40    if (script !== undefined) plan.push({ check, script })
41  }
42  return plan
43}
44
45/** The argv that runs a script under the package manager, with extra arguments passed through to it. */
46export function scriptArgv(manager: PackageManager, script: string, extra: readonly string[] = []): string[] {
47  switch (manager) {
48    case 'yarn':
49    case 'pnpm':
50    case 'bun':
51      return [manager, 'run', script, ...extra]
52    default:
53      return ['npm', 'run', '--silent', script, ...(extra.length > 0 ? ['--', ...extra] : [])]
54  }
55}
56
57/**
58 * The arguments that scope a check to the files Claude changed: the files
59 * themselves for lint and spell check, `testArgs` then the files for tests
60 * (only when `testArgs` is set, since runners spell it differently), and
61 * none for the type check, which needs the whole project.
62 */
63export function scopeArgs(check: Check, files: readonly string[], testArgs: readonly string[] | undefined): string[] | undefined {
64  if (files.length === 0) return undefined
65  if (check === 'lint' || check === 'spellcheck') return [...files]
66  if (check === 'test' && testArgs !== undefined) return [...testArgs, ...files]
67  return undefined
68}
69
70/** The tail of a script's output: enough to act on, not the whole log. */
71export function tail(output: string, maxLines = 60, maxChars = 4000): string {
72  const lines = output.replace(/\x1b\[[0-9;]*m/g, '').trimEnd().split('\n')
73  let text = lines.slice(-maxLines).join('\n')
74  if (text.length > maxChars) text = '…' + text.slice(-maxChars)
75  return lines.length > maxLines ? `…(${lines.length - maxLines} earlier lines cut)\n${text}` : text
76}
77
78export type Failure = { check: Check; script: string; exitCode: number; output: string }
79
80/** What Claude reads when the gate holds the turn. */
81export function failureMessage(failures: readonly Failure[], command: (script: string) => string): string {
82  return [
83    `qa-gate: ${failures.length} check${failures.length === 1 ? '' : 's'} failed after your code changes. Fix the errors below, then finish:`,
84    ...failures.map(f =>
85      [`### ${f.check} (\`${command(f.script)}\` exited ${f.exitCode})`, '```', tail(f.output), '```'].join('\n'),
86    ),
87  ].join('\n\n')
88}
89