olo-evidence

1import type { EngineInterface, Register } from 'claude-code'
2
3import type { EvidenceLedger, EvidenceRun, EvidenceVisual } from '../types'
4import {
5  APPROVAL_MARKER,
6  backgroundOutputPath,
7  carriesResult,
8  emptyLedger,
9  judge,
10  type Judged,
11  LEDGER_MARKER,
12  markersOf,
13  outcomeOf,
14  parseInvocations,
15  prBodySource,
16  prProblems,
17  prStatement,
18  redirectTarget,
19  summaryMarkdown,
20  unwaived,
21  visualFrom,
22} from './ledger'
23
24const KEEP_RUNS = 200
25const KEEP_VISUALS = 200
26const FS_READ_LIMIT = 4 * 1024 * 1024
27const TAIL_BYTES = 3_000_000
28const IDENTITY_TTL_MS = 60_000
29
30// A command that can move HEAD or the checkout: the cached branch identity is dropped after it.
31const MOVES_HEAD = /\bgit\b[^\n]*\b(?:switch|checkout|worktree|rebase|merge|reset|pull|branch)\b|(?:^|[;&|]\s*)(?:cd|pushd|Set-Location)\s/
32
33// Writes to one branch's ledger are serialised: parallel tool calls must not lose entries.
34let writes: Promise<unknown> = Promise.resolve()
35let identityCache: { at: number; key: string; branch: string; top: string } | undefined
36
37export const register: Register = on => {
38  on('session.start', async ($, e, next) => {
39    await $.command.register({ name: 'evidence', description: "This branch's recorded builds, test runs and visual checks, and what still stands before a PR" })
40    return next(e)
41  })
42
43  on('command.run', { command: 'evidence' }, async $ => {
44    const id = await identity($)
45    if (id === undefined) {
46      return { text: 'Not in a git checkout: no evidence ledger.' }
47    }
48    const ledger = await load($, id.key)
49    const judged = await judgeAll($, ledger)
50    const changed = await changedFiles($)
51    const problems = prProblems(judged, ledger.visuals, changed, '')
52    return { text: summaryMarkdown(judged, ledger.visuals, problems, id.branch, await $.clock.now()) }
53  })
54
55  // Record runs and visual checks; gate PR creation on the record.
56  on('tool.call', async ($, e, next) => {
57    const command = e.tool === 'Bash' || e.tool === 'PowerShell' ? e.command : ''
58
59    if (command.length > 0) {
60      const refusal = await prGate($, command)
61      if (refusal !== undefined) {
62        $.ui.toast('olo-evidence: the PR is not backed by the record yet', { timeoutMs: 8000 })
63        return { deny: refusal }
64      }
65    }
66
67    const invocations = command.length > 0 ? parseInvocations(command) : []
68    const redirect = invocations.length > 0 ? redirectTarget(command) : null
69    if (redirect !== null) {
70      // The new run is about to overwrite this log: earlier runs judged from it keep the
71      // outcome it holds NOW, or they would be judged by the new run's output.
72      await settle($, () => freezeRunsOn($, redirect))
73    }
74
75    const ran = await next(e)
76    if (ran.deny !== undefined) {
77      return ran
78    }
79
80    // Recording is best effort and must never fail the call it records: the tool has run.
81    await settle($, async () => {
82      if (MOVES_HEAD.test(command)) {
83        identityCache = undefined
84      }
85      const visual = visualFrom(e.tool, { file_path: e.tool === 'Read' ? e.file_path : undefined, command })
86      if (visual !== null && ran.isError !== true) {
87        await record($, ledger => ({
88          ...ledger,
89          visuals: [...ledger.visuals, { at: 0, kind: visual, what: e.tool === 'Read' ? e.file_path : command.slice(0, 200) }].slice(-KEEP_VISUALS),
90        }))
91      }
92      if (invocations.length > 0) {
93        const text = typeof ran.text === 'string' ? ran.text : ''
94        const logPath = backgroundOutputPath(text) ?? redirect
95        await record($, ledger => ({
96          ...ledger,
97          runs: [
98            ...ledger.runs,
99            ...invocations.map(
100              (inv, i): EvidenceRun => ({
101                id: `${e.tool_use_id}-${i}`,
102                at: 0,
103                kind: inv.kind,
104                command: command.slice(0, 300),
105                config: inv.config,
106                scope: inv.scope,
107                logPath,
108                captured: carriesResult(inv.kind, text) ? outcomeOf(inv.kind, text) : null,
109              }),
110            ),
111          ].slice(-KEEP_RUNS),
112        }))
113      }
114    })
115    return ran
116  })
117    // Only the gate runs before next(e); if it throws, the call goes ahead (fail open), and
118    // where next was already called, next(e) replays its result without running anything twice.
119    .catch(($, e, next) => next(e))
120}
121
122/** Runs best-effort work; a failure is swallowed so the tool call it rides on is unaffected. */
123async function settle($: EngineInterface, work: () => Promise<void>): Promise<void> {
124  try {
125    await work()
126  } catch {
127    // The record misses one entry; the session goes on.
128  }
129}
130
131async function prGate($: EngineInterface, command: string): Promise<string | undefined> {
132  const statement = prStatement(command)
133  if (statement === undefined) {
134    return undefined
135  }
136  const markers = markersOf(statement)
137  if (markers.includes(APPROVAL_MARKER)) {
138    return undefined
139  }
140  const id = await identity($)
141  if (id === undefined) {
142    return undefined
143  }
144  const ledger = await load($, id.key)
145  const judged = await judgeAll($, ledger)
146  const changed = await changedFiles($)
147  const source = prBodySource(statement)
148  const body = source.file !== undefined ? await readSmall($, source.file) ?? '' : source.inline ?? ''
149  const problems = unwaived(prProblems(judged, ledger.visuals, changed, body), markers)
150  if (problems.length === 0) {
151    return undefined
152  }
153  const waivers = [...new Set(problems.flatMap(p => p.waivedBy))]
154  return [
155    `olo-evidence refused this: the record for ${id.branch} does not back the PR yet.`,
156    ...problems.map(p => `- ${p.message}`),
157    `Run /evidence to see the record.${waivers.includes(LEDGER_MARKER) ? ` If those runs happened outside this record (an earlier session), end the gh command with a comment carrying ${LEDGER_MARKER} and say so in the PR body.` : ''}`,
158  ].join('\n')
159}
160
161/** Earlier runs that read their outcome from `path` take what it says now, and stop reading it. */
162async function freezeRunsOn($: EngineInterface, path: string): Promise<void> {
163  const id = await identity($)
164  if (id === undefined) {
165    return
166  }
167  const same = (p: string | null) => p !== null && p.replace(/\\/g, '/').toLowerCase() === path.replace(/\\/g, '/').toLowerCase()
168  const ledger = await load($, id.key)
169  if (!ledger.runs.some(r => same(r.logPath))) {
170    return
171  }
172  const log = await readTail($, path)
173  await record($, current => ({
174    ...current,
175    runs: current.runs.map(run => {
176      if (!same(run.logPath)) {
177        return run
178      }
179      const outcome = log === undefined ? undefined : outcomeOf(run.kind, log)
180      return { ...run, logPath: null, captured: outcome !== undefined && outcome.state !== 'unknown' ? outcome : run.captured }
181    }),
182  }))
183}
184
185async function judgeAll($: EngineInterface, ledger: EvidenceLedger): Promise<Judged[]> {
186  return Promise.all(ledger.runs.map(async run => judge(run, run.logPath === null ? undefined : await readTail($, run.logPath))))
187}
188
189async function changedFiles($: EngineInterface): Promise<string[]> {
190  try {
191    const r = await $.process.run(['git', 'diff', '--name-only', 'origin/master...HEAD'], { timeoutMs: 20_000 })
192    return r.exitCode === 0 ? r.stdout.split('\n').map(l => l.trim()).filter(Boolean) : []
193  } catch {
194    return []
195  }
196}
197
198async function identity($: EngineInterface): Promise<{ key: string; branch: string; top: string } | undefined> {
199  const now = await $.clock.now()
200  if (identityCache !== undefined && now - identityCache.at < IDENTITY_TTL_MS) {
201    return identityCache
202  }
203  try {
204    const [top, branch] = await Promise.all([
205      $.process.run(['git', 'rev-parse', '--show-toplevel'], { timeoutMs: 5000 }),
206      $.process.run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], { timeoutMs: 5000 }),
207    ])
208    if (top.exitCode !== 0 || branch.exitCode !== 0) {
209      return undefined
210    }
211    const t = top.stdout.trim()
212    const b = branch.stdout.trim()
213    identityCache = { at: now, key: `ledger:${t.toLowerCase()}|${b}`, branch: b, top: t }
214    return identityCache
215  } catch {
216    return undefined
217  }
218}
219
220async function load($: EngineInterface, key: string): Promise<EvidenceLedger> {
221  const stored = (await $.store.get(key)) as EvidenceLedger | undefined
222  return stored !== undefined && Array.isArray(stored.runs) && Array.isArray(stored.visuals) ? stored : emptyLedger()
223}
224
225/** Applies `change` to this branch's ledger, stamping new entries with the time. */
226async function record($: EngineInterface, change: (ledger: EvidenceLedger) => EvidenceLedger): Promise<void> {
227  const id = await identity($)
228  if (id === undefined) {
229    return
230  }
231  const now = await $.clock.now()
232  const step = writes.then(async () => {
233    const before = await load($, id.key)
234    const after = change(before)
235    const stamp = <T extends { at: number }>(xs: T[]) => xs.map(x => (x.at === 0 ? { ...x, at: now } : x))
236    await $.store.set(id.key, { runs: stamp(after.runs), visuals: stamp(after.visuals) as EvidenceVisual[] })
237  })
238  writes = step.catch(() => undefined)
239  await step
240}
241
242async function readSmall($: EngineInterface, path: string): Promise<string | undefined> {
243  try {
244    const stat = await $.fs.stat(path)
245    return stat.kind === 'file' && stat.size <= FS_READ_LIMIT ? await $.fs.read(path) : undefined
246  } catch {
247    return undefined
248  }
249}
250
251/**
252 * A log's text: whole when `$.fs.read` takes it (up to 4 MiB), else its last few MB through
253 * this mod's scripts/tail.ps1 (pwsh is always present here), else `tail` where one exists.
254 */
255async function readTail($: EngineInterface, path: string): Promise<string | undefined> {
256  try {
257    const stat = await $.fs.stat(path)
258    if (stat.kind !== 'file') {
259      return undefined
260    }
261    if (stat.size <= FS_READ_LIMIT) {
262      return await $.fs.read(path)
263    }
264  } catch {
265    return undefined
266  }
267  const attempts: string[][] = [
268    ['pwsh', '-NoProfile', '-NonInteractive', '-File', `${$.plugin.root}/scripts/tail.ps1`, '-Path', path, '-Bytes', String(TAIL_BYTES)],
269    ['tail', '-c', String(TAIL_BYTES), path],
270  ]
271  for (const argv of attempts) {
272    try {
273      const r = await $.process.run(argv, { timeoutMs: 15_000 })
274      if (r.exitCode === 0) {
275        return r.stdout
276      }
277    } catch {
278      // Try the next way.
279    }
280  }
281  return undefined
282}
283

1// Pure core of the evidence ledger: recognising build and test invocations, reading their
2// outcome from a log, and judging a branch's evidence against what a PR claims.
3
4import type { EvidenceLedger, EvidenceOutcome, EvidenceRun, EvidenceRunKind, EvidenceVisual } from '../types'
5import { commandWords, commentOf, exeName, innerCommand, splitStatements, type Statement } from './shell'
6
7// ------------------------------------------------------------ invocations ---
8
9export type Invocation = { kind: EvidenceRunKind; config: string; scope: string }
10
11const CONFIGS = /^(?:Debug|Release|RelWithDebInfo|MinSizeRel|Dist)$/i
12
13function cap(s: string): string {
14  return s.length === 0 ? s : s[0]!.toUpperCase() + s.slice(1).toLowerCase()
15}
16
17/** The word after `flag`, or the value of `flag=value`. */
18function flagValue(words: string[], ...flags: string[]): string | undefined {
19  for (let i = 0; i < words.length; i += 1) {
20    const w = words[i] ?? ''
21    for (const f of flags) {
22      if (w === f) {
23        return words[i + 1]
24      }
25      if (w.startsWith(`${f}=`)) {
26        return w.slice(f.length + 1)
27      }
28    }
29  }
30  return undefined
31}
32
33function config(value: string | undefined): string {
34  return value !== undefined && CONFIGS.test(value) ? cap(value) : '?'
35}
36
37/**
38 * Every build and test run a shell command starts (a `build && test` line is two). Only a
39 * statement whose COMMAND is the build or test counts: `rg OloEngine-Tests`, `git log --grep
40 * ctest` or a PR body naming the exe start nothing, and `--target OloEngine-Tests` is a target.
41 * A shell wrapper (`pwsh -Command`, build-lock.ps1's form, `bash -c`, `cmd /c`) is looked into.
42 */
43export function parseInvocations(command: string): Invocation[] {
44  const out: Invocation[] = []
45  for (const statement of splitStatements(command)) {
46    const words = commandWords(statement.words)
47    const name = exeName(words[0])
48    const inner = innerCommand(words)
49
50    if (inner !== undefined) {
51      out.push(...parseInvocations(inner))
52    } else if (name === 'cmake' && words[1] === '--build') {
53      const tree = (words[2] ?? '').replace(/\\/g, '/').replace(/\/$/, '').split('/').pop() ?? ''
54      const at = words.indexOf('--target')
55      const targets: string[] = []
56      for (let i = at + 1; at >= 0 && i < words.length && !(words[i] ?? '-').startsWith('-'); i += 1) {
57        targets.push(words[i] ?? '')
58      }
59      out.push({ kind: 'build', config: config(flagValue(words, '--config')), scope: `${tree}${targets.length > 0 ? ` ${targets.join(' ')}` : ''}` })
60    } else if (name === 'oloengine-tests') {
61      if (words.includes('--gtest_list_tests')) {
62        continue
63      }
64      // A path segment names the config (build-cached/OloEngine/tests/Debug/OloEngine-Tests.exe).
65      const fromPath = /[\\/](Debug|Release|RelWithDebInfo|MinSizeRel|Dist)[\\/][^\\/]*$/i.exec(words[0] ?? '')?.[1]
66      out.push({ kind: 'test', config: config(fromPath), scope: flagValue(words, '--gtest_filter') ?? '(all)' })
67    } else if (name === 'ctest') {
68      if (words.includes('-N') || words.includes('--show-only')) {
69        continue
70      }
71      const regex = flagValue(words, '-R', '--tests-regex')
72      out.push({ kind: 'test', config: config(flagValue(words, '-C', '--build-config')), scope: regex ? `ctest -R ${regex}` : 'ctest (all)' })
73    }
74  }
75  return out
76}
77
78/** Where a command sends its output: the last file redirect or tee target. */
79export function redirectTarget(command: string): string | null {
80  const targets: string[] = []
81  for (const m of command.matchAll(/(?:^|[^0-9&>])(?:\*|[12])?>>?\s*("?)([^\s"|;&<>]+)\1/g)) {
82    targets.push(m[2] ?? '')
83  }
84  for (const m of command.matchAll(/\|\s*(?:tee(?:\s+-a)?|Tee-Object(?:\s+-(?:FilePath|Append))*|Out-File(?:\s+-(?:FilePath|Append|Encoding\s+\S+))*)\s+("?)([^\s"|;&<>]+)\1/gi)) {
85    targets.push(m[2] ?? '')
86  }
87  const real = targets.filter(t => t.length > 0 && !/^(?:&\d|\/dev\/null|\$null|nul)$/i.test(t))
88  return real.length === 0 ? null : real[real.length - 1]!
89}
90
91/** A backgrounded shell command's output file, from the tool's own reply. */
92export function backgroundOutputPath(toolText: string): string | null {
93  return /Output is being written to:\s*(\S+\.output)/.exec(toolText)?.[1] ?? null
94}
95
96// ---------------------------------------------------------------- outcome ---
97
98const NINJA_STOPPED = /^ninja: build stopped: (.*)$/m
99const NINJA_FAILED = /^FAILED: /m
100const MSBUILD_FAILED = /^\s*Build FAILED\.\s*$|^\s*[1-9]\d* Error\(s\)\s*$/m
101const BUILD_DONE = /^\[build-lock\] released|^ninja: no work to do\.|^\s*Build succeeded\.|^\[(\d+)\/\1\] Linking/m
102const GTEST_HEADER = /^\[==========\] Running (\d+) tests? from \d+ test suites?\./gm
103const GTEST_DONE = /^\[==========\] \d+ tests? from \d+ test suites? ran\./m
104
105function unique(items: string[]): string[] {
106  return [...new Set(items)]
107}
108
109export function buildOutcome(log: string): EvidenceOutcome {
110  const stopped = NINJA_STOPPED.exec(log)
111  if (stopped || NINJA_FAILED.test(log) || MSBUILD_FAILED.test(log)) {
112    const first = log.split('\n').find(l => /: (?:fatal )?error[: ]|error (?:C|LNK)\d{4}/.test(l))?.trim()
113    return { state: 'failed', detail: first ? `failed: ${first.slice(0, 160)}` : `failed${stopped ? `: ${stopped[1]}` : ''}`, failed: [], ran: 0, skipped: 0 }
114  }
115  if (BUILD_DONE.test(log)) {
116    return { state: 'passed', detail: 'built', failed: [], ran: 0, skipped: 0 }
117  }
118  return { state: /\[build-lock\] (?:acquired|waiting|queued)|^\[\d+\/\d+\]/m.test(log) ? 'running' : 'unknown', detail: 'no result in the log yet', failed: [], ran: 0, skipped: 0 }
119}
120
121const CTEST_SUMMARY = /^\s*(\d+)% tests passed, (\d+) tests? failed out of (\d+)/m
122
123/** ctest's own summary, which it prints without -V where no gtest header appears. */
124function ctestOutcome(log: string): EvidenceOutcome | undefined {
125  const m = CTEST_SUMMARY.exec(log)
126  if (m === null) {
127    return undefined
128  }
129  const failedCount = Number(m[2])
130  const total = Number(m[3])
131  const list = log.slice(log.indexOf('The following tests FAILED:'))
132  const names = failedCount > 0 ? unique([...list.matchAll(/^\s*\d+ - (\S+) \(/gm)].map(x => x[1] ?? '')) : []
133  if (total === 0) {
134    return { state: 'failed', detail: 'ctest ran no tests', failed: [], ran: 0, skipped: 0 }
135  }
136  return failedCount > 0
137    ? { state: 'failed', detail: `${failedCount} failed of ${total} (ctest)`, failed: names, ran: total, skipped: 0 }
138    : { state: 'passed', detail: `${total} passed (ctest)`, failed: [], ran: total, skipped: 0 }
139}
140
141export function testOutcome(log: string): EvidenceOutcome {
142  if (/^No tests were found!!!\s*$/m.test(log)) {
143    return { state: 'failed', detail: 'ctest found no tests', failed: [], ran: 0, skipped: 0 }
144  }
145  const ctest = ctestOutcome(log)
146  if (ctest !== undefined) {
147    return ctest
148  }
149  const starts = [...log.matchAll(GTEST_HEADER)]
150  if (starts.length === 0) {
151    return { state: 'unknown', detail: 'no gtest output in the log', failed: [], ran: 0, skipped: 0 }
152  }
153  let ran = 0
154  let skipped = 0
155  let zero = 0
156  let open = false
157  const failed: string[] = []
158  starts.forEach((m, i) => {
159    const seg = log.slice(m.index ?? 0, starts[i + 1]?.index ?? log.length)
160    if (Number(m[1]) === 0) {
161      zero += 1
162      return
163    }
164    const runs = unique([...seg.matchAll(/^\[ RUN      \] (\S+)/gm)].map(x => x[1] ?? ''))
165    ran += runs.length
166    skipped += unique([...seg.matchAll(/^\[  SKIPPED \] (\S+)/gm)].map(x => x[1] ?? '')).filter(n => runs.includes(n)).length
167    failed.push(...[...seg.matchAll(/^\[  FAILED  \] ([^\s,]+\.[^\s,]+)/gm)].map(x => x[1] ?? ''))
168    if (!GTEST_DONE.test(seg)) {
169      open = true
170    }
171  })
172  const names = unique(failed)
173  if (zero > 0 && ran === 0) {
174    return { state: 'failed', detail: 'the filter selected no tests', failed: [], ran: 0, skipped: 0 }
175  }
176  if (names.length > 0) {
177    return { state: 'failed', detail: `${names.length} failed of ${ran}`, failed: names, ran, skipped }
178  }
179  if (open) {
180    return { state: 'running', detail: `ended inside a test after ${ran} (still running, or it died)`, failed: [], ran, skipped }
181  }
182  if (ran > 0 && skipped === ran) {
183    return { state: 'failed', detail: `all ${ran} skipped: nothing verified`, failed: [], ran, skipped }
184  }
185  return { state: 'passed', detail: `${ran - skipped} passed${skipped > 0 ? `, ${skipped} skipped` : ''}`, failed: [], ran, skipped }
186}
187
188/** NULs (the engine logger), CRs and ANSI colour sequences (gtest with --gtest_color) out. */
189export function cleanLog(text: string): string {
190  return text
191    .replace(/\u0000/g, '')
192    .replace(/\u001b\[[0-9;]*[A-Za-z]/g, '')
193    .replace(/\r\n?/g, '\n')
194}
195
196export function outcomeOf(kind: EvidenceRunKind, log: string): EvidenceOutcome {
197  const clean = cleanLog(log)
198  return kind === 'build' ? buildOutcome(clean) : testOutcome(clean)
199}
200
201/** Does this output carry a result at all (so it is worth keeping as the captured outcome)? */
202export function carriesResult(kind: EvidenceRunKind, text: string): boolean {
203  return outcomeOf(kind, text).state !== 'unknown'
204}
205
206// ---------------------------------------------------------------- visuals ---
207
208// No leading \b: an MCP tool name puts `__` (word characters) right before `olo_`.
209const LIVE_CAPTURE = /olo_(?:screenshot|render_capture_target|render_probe_pixel|probe_pixel|render_frame_breakdown)\b/
210const LIVE_CHECK = /olo_(?:shader_errors|log_tail)\b/
211
212/** What a tool call contributes to the visual-evidence record, if anything. */
213export function visualFrom(tool: string, input: { file_path?: string; command?: string }): EvidenceVisual['kind'] | null {
214  if (tool === 'Read' && typeof input.file_path === 'string' && /\.png$/i.test(input.file_path)) {
215    return 'png-viewed'
216  }
217  const text = `${tool} ${input.command ?? ''}`
218  if (LIVE_CAPTURE.test(text)) {
219    return 'live-capture'
220  }
221  if (LIVE_CHECK.test(text)) {
222    return 'live-check'
223  }
224  return null
225}
226
227// --------------------------------------------------------------- judgement ---
228
229export type Judged = { run: EvidenceRun; outcome: EvidenceOutcome }
230
231/** The outcome a run stands at: its log when that can be read, else what was captured. */
232export function judge(run: EvidenceRun, log: string | undefined): Judged {
233  const fromLog = log === undefined ? undefined : outcomeOf(run.kind, log)
234  const outcome =
235    fromLog !== undefined && fromLog.state !== 'unknown'
236      ? fromLog
237      : run.captured ?? { state: 'unknown' as const, detail: log === undefined ? 'no output recorded' : 'no result in its output', failed: [], ran: 0, skipped: 0 }
238  return { run, outcome }
239}
240
241export function runKey(run: EvidenceRun): string {
242  return `${run.kind}|${run.config}|${run.scope}`
243}
244
245/** The latest run of each distinct (kind, config, scope): earlier ones are superseded. */
246export function latestPerKey(judged: readonly Judged[]): Judged[] {
247  const latest = new Map<string, Judged>()
248  for (const j of [...judged].sort((a, b) => a.run.at - b.run.at)) {
249    latest.set(runKey(j.run), j)
250  }
251  return [...latest.values()]
252}
253
254const RENDER_PATH = /^(?:OloEngine\/src\/OloEngine\/(?:Renderer|RHI)\/|OloEngine\/src\/Platform\/(?:OpenGL|Vulkan)\/|OloEditor\/assets\/shaders\/)|\.(?:glsl|vert|frag|comp|geom|tesc|tese|mesh|task|rgen|rchit|rahit|rmiss)$/
255const CODE_PATH = /\.(?:cpp|h|hpp|inl|cs|glsl|vert|frag|comp|cmake)$|CMakeLists\.txt$/
256const SAYS_NO_FRAME = /could ?not inspect a frame|cannot inspect a frame|no frame could be inspected|not visually verified|could not run the editor/i
257
258export const LEDGER_MARKER = 'OLO_LEDGER_INCOMPLETE'
259export const APPROVAL_MARKER = 'OLO_USER_APPROVED'
260
261export type PrProblem = { rule: string; message: string; waivedBy: string[] }
262
263/**
264 * What stands between this branch's evidence and the PR. `body` is the PR text ('' when
265 * unknown), `changed` the files the branch changes against master.
266 */
267export function prProblems(judged: readonly Judged[], visuals: readonly EvidenceVisual[], changed: readonly string[], body: string): PrProblem[] {
268  const problems: PrProblem[] = []
269  const latest = latestPerKey(judged)
270  const codeChanged = changed.some(f => CODE_PATH.test(f))
271
272  // A run counts as evidence only once its outcome is known: one whose log is gone, or that
273  // printed no result, proves nothing either way.
274  const known = judged.filter(j => j.outcome.state !== 'unknown')
275  if (codeChanged && known.length === 0) {
276    problems.push({
277      rule: 'no-runs',
278      message:
279        judged.length === 0
280          ? 'The branch changes code, and no local build or test run is recorded for it. Build and run the relevant tests first (CLAUDE.md, Definition of done).'
281          : 'The branch changes code, and no recorded build or test run has a known outcome (their logs are gone or printed no result). Run them again so the result is read.',
282      waivedBy: [LEDGER_MARKER, APPROVAL_MARKER],
283    })
284  }
285
286  for (const { run, outcome } of latest) {
287    if (outcome.state === 'unknown' && known.length > 0) {
288      problems.push({
289        rule: 'unproven-run',
290        message: `The latest ${run.kind} ${run.config} ${run.scope} has no known outcome (${outcome.detail}). Re-run it, or say in the PR body where its result is.`,
291        waivedBy: [LEDGER_MARKER, APPROVAL_MARKER],
292      })
293    }
294    if (outcome.state === 'running') {
295      problems.push({
296        rule: 'still-running',
297        message: `${run.kind} ${run.config} ${run.scope} has not finished (${outcome.detail}). Push and open the PR only after every local check has finished.`,
298        waivedBy: [APPROVAL_MARKER],
299      })
300    }
301    if (outcome.state === 'failed') {
302      const unexplained = outcome.failed.filter(name => !body.includes(name))
303      const explained = outcome.failed.length > 0 && unexplained.length === 0
304      if (!explained) {
305        problems.push({
306          rule: 'failed-run',
307          message:
308            `The latest ${run.kind} ${run.config} ${run.scope} ${outcome.detail}` +
309            `${unexplained.length > 0 ? ` (${unexplained.slice(0, 6).join(', ')})` : ''}. Re-run it green, or, if a failure is pre-existing and unrelated, name each failing test in the PR body with its evidence.`,
310          waivedBy: [APPROVAL_MARKER],
311        })
312      }
313    }
314  }
315
316  for (const config of ['Debug', 'Release'] as const) {
317    if (claims(body, config) && !latest.some(j => j.run.config === config && j.outcome.state === 'passed')) {
318      problems.push({
319        rule: `claims-${config.toLowerCase()}`,
320        message: `The PR body claims a ${config} result, but no ${config} build or test run passed in the record. A claim needs its run (memory: no-pr-before-local-checks-finish).`,
321        waivedBy: [LEDGER_MARKER, APPROVAL_MARKER],
322      })
323    }
324  }
325
326  if (changed.some(f => RENDER_PATH.test(f)) && !SAYS_NO_FRAME.test(body)) {
327    const missing = [
328      visuals.some(v => v.kind === 'png-viewed') ? '' : 'no evidence PNG was looked at (Read it)',
329      visuals.some(v => v.kind === 'live-capture') ? '' : 'no live editor capture (olo_screenshot / olo_render_capture_target)',
330    ].filter(Boolean)
331    if (missing.length > 0) {
332      problems.push({
333        rule: 'visual-evidence',
334        message:
335          `The branch changes renderer or shader code, and ${missing.join(' and ')}. CLAUDE.md: a rendering change is not done on unit tests alone. ` +
336          'If you cannot inspect a frame, say so in the PR body ("could not inspect a frame").',
337        waivedBy: [APPROVAL_MARKER],
338      })
339    }
340  }
341  return problems
342}
343
344// A claim is a line that names the configuration AND a verdict; naming Debug in prose
345// ("a Debug-only failure would cost a CI round") claims nothing.
346const VERDICT = /\b(?:pass(?:ed|es|ing)?|green|verified|ran|built|succeeded)\b|✅|\b0 fail/i
347
348export function claims(body: string, config: 'Debug' | 'Release'): boolean {
349  const named = new RegExp(`\\b${config}\\b`)
350  return body.split('\n').some(line => named.test(line) && VERDICT.test(line))
351}
352
353/**
354 * The problems no marker waives. `markers` is the trailing comment of the `gh pr` statement
355 * (see prStatement): a marker inside a quoted PR body or another statement is data.
356 */
357export function unwaived(problems: readonly PrProblem[], markers: string): PrProblem[] {
358  return problems.filter(p => !p.waivedBy.some(marker => markers.includes(marker)))
359}
360
361/**
362 * The `gh pr create` (or `gh pr edit` that sets the body) statement of a command, if any:
363 * through prefixes (`GH_TOKEN=x`, `env`) and shell wrappers (`pwsh -Command`, `bash -c`,
364 * `cmd /c`). Its words are the gh command's own; its comment is the gh statement's plus the
365 * wrapper's, so a marker on either counts and a quoted one on neither.
366 */
367export function prStatement(command: string): Statement | undefined {
368  for (const s of splitStatements(command)) {
369    const words = commandWords(s.words)
370    const inner = innerCommand(words)
371    if (inner !== undefined) {
372      const found = prStatement(inner)
373      if (found !== undefined) {
374        return { ...found, text: `${found.text} # ${commentOf(s.text)}` }
375      }
376      continue
377    }
378    if (exeName(words[0]) !== 'gh' || words[1] !== 'pr') {
379      continue
380    }
381    if (words[2] === 'create' || (words[2] === 'edit' && words.some(x => /^(?:--body|--body-file|-b|-F)(?:=|$)/.test(x)))) {
382      return { ...s, words }
383    }
384  }
385  return undefined
386}
387
388/** Where that statement's PR body comes from: a file, inline text, or neither. */
389export function prBodySource(statement: Statement): { file?: string; inline?: string } {
390  const file = flagValue(statement.words, '--body-file', '-F')
391  if (file !== undefined) {
392    return { file }
393  }
394  const inline = flagValue(statement.words, '--body', '-b')
395  return inline !== undefined ? { inline } : {}
396}
397
398export function markersOf(statement: Statement): string {
399  return commentOf(statement.text)
400}
401
402// ---------------------------------------------------------------- display ---
403
404function ago(at: number, now: number): string {
405  const min = Math.max(0, Math.round((now - at) / 60_000))
406  return min < 60 ? `${min} min ago` : `${Math.round(min / 60)} h ago`
407}
408
409const MARK: Record<EvidenceOutcome['state'], string> = { passed: 'PASS', failed: 'FAIL', running: 'RUNNING', unknown: '?' }
410
411export function summaryMarkdown(judged: readonly Judged[], visuals: readonly EvidenceVisual[], problems: readonly PrProblem[], branch: string, now: number): string {
412  const lines = [`**Evidence for \`${branch}\`**`, '']
413  const latest = new Set(latestPerKey(judged))
414  const recent = [...judged].sort((a, b) => b.run.at - a.run.at).slice(0, 25)
415  if (recent.length === 0) {
416    lines.push('- no build or test run recorded yet')
417  }
418  for (const j of recent) {
419    const superseded = latest.has(j) ? '' : ' (superseded)'
420    lines.push(`- ${MARK[j.outcome.state]} ${j.run.kind} ${j.run.config} ${j.run.scope}: ${j.outcome.detail}, ${ago(j.run.at, now)}${superseded}`)
421  }
422  const count = (k: EvidenceVisual['kind']) => visuals.filter(v => v.kind === k).length
423  lines.push('', `**Visual**: ${count('png-viewed')} PNG(s) looked at, ${count('live-capture')} live capture(s), ${count('live-check')} live log/shader check(s)`, '')
424  lines.push(problems.length === 0 ? '**Ready for a PR** as far as the record shows.' : '**Before a PR:**')
425  for (const p of problems) {
426    lines.push(`- ${p.message}`)
427  }
428  return lines.join('\n')
429}
430
431export function emptyLedger(): EvidenceLedger {
432  return { runs: [], visuals: [] }
433}
434

1// Quote-aware shell statement splitting, the same as olo-command-guards' (mods are
2// independent plugins, so each carries its own copy; keep the two in step).
3
4export type Separator = 'start' | '&&' | '||' | ';' | '|' | '\n' | '&'
5
6export type Statement = { separator: Separator; text: string; words: string[] }
7
8/**
9 * Splits a shell command into statements, remembering which separator led into each,
10 * and tokenises each one. Quote-aware (single, double, and PowerShell backtick escapes
11 * are left as literal characters), so a `;` inside a commit message is not a split.
12 */
13export function splitStatements(command: string): Statement[] {
14  const statements: Statement[] = []
15  let current = ''
16  let separator: Separator = 'start'
17  let quote: '"' | "'" | undefined
18  let i = 0
19
20  const flush = (next: Separator) => {
21    if (current.trim().length > 0) {
22      statements.push({ separator, text: current.trim(), words: tokenize(current) })
23    }
24    current = ''
25    separator = next
26  }
27
28  while (i < command.length) {
29    const c = command[i] ?? ''
30    const two = command.slice(i, i + 2)
31    if (quote !== undefined) {
32      if (c === '\\' && quote === '"' && i + 1 < command.length) {
33        current += two
34        i += 2
35        continue
36      }
37      if (c === quote) {
38        quote = undefined
39      }
40      current += c
41      i += 1
42      continue
43    }
44    if (c === '"' || c === "'") {
45      quote = c
46      current += c
47      i += 1
48      continue
49    }
50    if (c === '#' && (current.length === 0 || /\s$/.test(current))) {
51      // A comment runs to the end of the line; keep it in the text (markers live there)
52      // but out of the words.
53      const end = command.indexOf('\n', i)
54      const stop = end === -1 ? command.length : end
55      current += command.slice(i, stop)
56      i = stop
57      continue
58    }
59    if (two === '&&' || two === '||') {
60      flush(two)
61      i += 2
62      continue
63    }
64    if (c === ';' || c === '\n' || c === '|' || (c === '&' && command[i + 1] !== '>' && command[i - 1] !== '>')) {
65      flush(c as Separator)
66      i += 1
67      continue
68    }
69    current += c
70    i += 1
71  }
72  flush('start')
73  return statements
74}
75
76/** Words of one statement, quotes removed, a trailing `#` comment dropped. */
77export function tokenize(text: string): string[] {
78  const words: string[] = []
79  let word = ''
80  let quote: '"' | "'" | undefined
81  let started = false
82  for (let i = 0; i < text.length; i += 1) {
83    const c = text[i] ?? ''
84    if (quote !== undefined) {
85      if (c === quote) {
86        quote = undefined
87      } else {
88        word += c
89      }
90      continue
91    }
92    if (c === '"' || c === "'") {
93      quote = c
94      started = true
95      continue
96    }
97    if (c === '#' && !started) {
98      break
99    }
100    if (/\s/.test(c)) {
101      if (started) {
102        words.push(word)
103      }
104      word = ''
105      started = false
106      continue
107    }
108    word += c
109    started = true
110  }
111  if (started) {
112    words.push(word)
113  }
114  return words
115}
116
117export function commentOf(text: string): string {
118  let quote: '"' | "'" | undefined
119  for (let i = 0; i < text.length; i += 1) {
120    const c = text[i] ?? ''
121    if (quote !== undefined) {
122      if (c === quote) {
123        quote = undefined
124      }
125      continue
126    }
127    if (c === '"' || c === "'") {
128      quote = c
129      continue
130    }
131    if (c === '#' && (i === 0 || /\s/.test(text[i - 1] ?? ''))) {
132      return text.slice(i + 1)
133    }
134  }
135  return ''
136}
137
138export function exeName(word: string | undefined): string {
139  return (word ?? '').replace(/^.*[\\/]/, '').replace(/\.exe$/i, '').toLowerCase()
140}
141
142// Launchers that run the command after them, with the options of theirs that take a value.
143// `env -S '<cmd>'` splits its value into the command it runs.
144const LAUNCHERS: Record<string, { withValue: string[]; positional?: number }> = {
145  env: { withValue: ['-u', '--unset', '-C', '--chdir', '-P'] },
146  timeout: { withValue: ['-s', '--signal', '-k', '--kill-after'], positional: 1 },
147  nice: { withValue: ['-n', '--adjustment'] },
148  time: { withValue: ['-o', '--output', '-f', '--format'] },
149  call: { withValue: [] },
150  command: { withValue: [] },
151  builtin: { withValue: [] },
152  exec: { withValue: ['-a'] },
153}
154
155/**
156 * A statement's words from the command it runs. Drops `&`, `VAR=value` assignments and the
157 * launchers above with their own options (`env -i -u X`, `timeout -s KILL 60`, `nice -n 5`),
158 * so `env -i GH_TOKEN=x gh pr merge` is still `gh pr merge`. Normalising only exposes the
159 * command that runs: it never removes a word the rules would have refused.
160 */
161export function commandWords(words: readonly string[]): string[] {
162  let rest = [...words]
163  for (;;) {
164    const w = rest[0] ?? ''
165    if (w === '&' || /^\w+=/.test(w)) {
166      rest = rest.slice(1)
167      continue
168    }
169    const launcher = LAUNCHERS[exeName(w)]
170    if (launcher === undefined) {
171      return rest
172    }
173    let i = 1
174    while (i < rest.length && (rest[i] ?? '').startsWith('-') && rest[i] !== '-') {
175      const flag = rest[i] ?? ''
176      // env -S '<string>' [more]: the string is split into env's own arguments, and may
177      // itself start with env options (`env -S '-i gh pr merge 1'` runs `gh pr merge 1`), so
178      // the expanded words go back through env, not straight to the command.
179      const split =
180        exeName(w) !== 'env'
181          ? undefined
182          : flag === '-S' || flag === '--split-string'
183            ? { value: rest[i + 1] ?? '', after: i + 2 }
184            : flag.startsWith('--split-string=')
185              ? { value: flag.slice('--split-string='.length), after: i + 1 }
186              : /^-S./.test(flag)
187                ? { value: flag.slice(2), after: i + 1 }
188                : undefined
189      if (split !== undefined) {
190        rest = [w, ...rest.slice(1, i), ...tokenize(split.value), ...rest.slice(split.after)]
191        i = -1
192        break
193      }
194      i += launcher.withValue.includes(flag) ? 2 : 1
195    }
196    if (i === -1) {
197      continue
198    }
199    if (rest[i] === '-') {
200      i += 1
201    }
202    rest = rest.slice(i + (launcher.positional ?? 0))
203  }
204}
205
206// PowerShell options that take a value (the rest are switches).
207const PWSH_WITH_VALUE = new Set([
208  '-executionpolicy', '-ep', '-ex', '-workingdirectory', '-wd', '-windowstyle', '-w', '-configurationname', '-config',
209  '-outputformat', '-o', '-of', '-inputformat', '-if', '-in', '-version', '-v', '-settingsfile', '-custompipename', '-psconsolefile',
210])
211// bash/sh options that take a value.
212const SH_WITH_VALUE = new Set(['-o', '+o', '-O', '+O', '--rcfile', '--init-file'])
213// A script that runs its own -Command parameter: build-lock.ps1 runs the build it is handed.
214const RUNS_ITS_COMMAND = /^build-lock\.ps1$/i
215
216/**
217 * The command a shell wrapper runs, read from the wrapper's OWN options only (words after a
218 * script operand are the script's arguments, not the shell's): `pwsh/powershell -Command|-c
219 * <cmd>`; `bash/sh/zsh/dash` with a single-dash flag group containing `c` (`-c`, `-lc`, `-ec`)
220 * then `<cmd>`; `cmd /c|/k <cmd...>`. The one script whose `-Command` IS what runs is
221 * build-lock.ps1. Undefined when the statement is no such wrapper.
222 */
223export function innerCommand(words: readonly string[]): string | undefined {
224  const name = exeName(words[0])
225  if (name === 'pwsh' || name === 'powershell') {
226    for (let i = 1; i < words.length; i += 1) {
227      const w = words[i] ?? ''
228      const lower = w.toLowerCase()
229      if (lower === '-command' || lower === '-c') {
230        return words.slice(i + 1).join(' ')
231      }
232      if (lower === '-file' || lower === '-f') {
233        const script = words[i + 1] ?? ''
234        if (!RUNS_ITS_COMMAND.test(script.replace(/^.*[\\/]/, ''))) {
235          return undefined
236        }
237        const at = words.findIndex((x, j) => j > i + 1 && /^-command$/i.test(x))
238        return at >= 0 ? words[at + 1] : undefined
239      }
240      if (lower === '-encodedcommand' || lower === '-e' || lower === '-ec') {
241        return undefined
242      }
243      if (!w.startsWith('-')) {
244        // A bare operand: pwsh runs it as a script file, Windows PowerShell as a command.
245        return name === 'powershell' ? words.slice(i).join(' ') : undefined
246      }
247      if (PWSH_WITH_VALUE.has(lower)) {
248        i += 1
249      }
250    }
251    return undefined
252  }
253  if (name === 'bash' || name === 'sh' || name === 'zsh' || name === 'dash') {
254    for (let i = 1; i < words.length; i += 1) {
255      const w = words[i] ?? ''
256      if (w === '--' || w === '-') {
257        return undefined
258      }
259      if (SH_WITH_VALUE.has(w)) {
260        i += 1
261        continue
262      }
263      if (/^-[A-Za-z]*c[A-Za-z]*$/.test(w)) {
264        return words[i + 1]
265      }
266      if (!/^[-+]/.test(w)) {
267        // The script operand: every word after it is the script's.
268        return undefined
269      }
270    }
271    return undefined
272  }
273  if (name === 'cmd') {
274    for (let i = 1; i < words.length; i += 1) {
275      const w = words[i] ?? ''
276      if (/^\/[ck]$/i.test(w)) {
277        return words.slice(i + 1).join(' ')
278      }
279      if (!w.startsWith('/')) {
280        return undefined
281      }
282    }
283    return undefined
284  }
285  return undefined
286}
287

1export type EvidenceRunKind = 'build' | 'test'
2
3export type EvidenceOutcome = {
4  state: 'passed' | 'failed' | 'running' | 'unknown'
5  /** One line: what the log says. */
6  detail: string
7  failed: string[]
8  ran: number
9  skipped: number
10}
11
12export type EvidenceRun = {
13  id: string
14  at: number
15  kind: EvidenceRunKind
16  command: string
17  config: string
18  /** Builds: the tree and targets. Tests: the gtest filter or ctest regex. */
19  scope: string
20  /** Where the output went (a redirect, or a background task's output file). */
21  logPath: string | null
22  /** The outcome read from the tool's own output, when it carried one. */
23  captured: EvidenceOutcome | null
24}
25
26export type EvidenceVisual = {
27  at: number
28  kind: 'png-viewed' | 'live-capture' | 'live-check'
29  what: string
30}
31
32export type EvidenceLedger = {
33  runs: EvidenceRun[]
34  visuals: EvidenceVisual[]
35}
36