test-grader

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register, Timer } from 'claude-code'
3
4import type { Before, Confidence, Coverage, ExistingRun, ExistingTest, TrackedTest, Verdict } from '../types'
5
6import { attr, byDirOf, coverageAnswer, coverageNote, kindAt, mergeParts, packageViews, pct, testedLine } from './coverage'
7import type { CoverCommand, PackageTests, PackageView } from './coverage'
8import { TEST_FILE, among, byCase, caseOf, ignoredBy, uniqueRows, withoutTemplates, caseLine, caseNames, casesAround, casesIn, changedCases, fits, isTemplate, kindOf, shortPath, suitesOf } from './discovery'
9import { MAX_REPLY, asAsked, caseTextOf, caseTextsOf, foldCases, othersOf, clamp, excerptOf, loopsOf, parseVerdicts, unratedWhy } from './excerpt'
10import type { Graded } from './excerpt'
11import { MAX_PROPOSED, changeOf, measuredLine, mutate, overlayOf, pickToMeasure, throughGrade } from './measure'
12import type { Measured, Proposed } from './measure'
13import { goProfileOf, isGenerated, isHelperName, mergeProfile, moduleOf, roleOf } from './gocover'
14import type { PackageRole } from './gocover'
15import { gradesKey, keep, unkeep } from './kept'
16import type { KeptGrades, SavedGrades } from './kept'
17import { costOf } from './prices'
18import { EVIDENCE_DESCRIPTION, EVIDENCE_HINT, EVIDENCE_MAX, EVIDENCE_SCHEMA, EVIDENCE_TOOL, CONTEXT_DESCRIPTION, COVERAGE_DESCRIPTION, COVERAGE_SCHEMA, COVERAGE_TOOL, CONTEXT_MAX, VERIFY_SIBLINGS, CONTEXT_SCHEMA, CONTEXT_TOOL, FOLLOW_UP, GRADE_DESCRIPTION, GRADE_SCHEMA, GRADE_TOOL, GRADES_DESCRIPTION, GRADES_LIMIT, GRADES_SCHEMA, GRADES_TOOL, GRADING_SECTION, LANGUAGE_NAMES, LANGUAGE_ORDER, MAX_ROUNDS, RUBRIC, SPENT_FOLLOW_UP, VERIFY_DESCRIPTION, VERIFY_SCHEMA, VERIFY_TOOL, guideOf } from './prompts'
19import { PROJECT_MARKS, goTagsOf, isBuildFailure, isSetupFailure, isNoneRun, runArgv, shown, tailOf } from './runner'
20import type { RunTarget, Runners } from './runner'
21import { LAYERS, LAYERS_FILE, LAYER_NAMES, layerOf, layerRulesOf } from './layers'
22import type { Layer, LayerRules } from './layers'
23import { goRanOf, jsRanOf, mergeRan, ranStateOf, tagsOfArgv } from './ran'
24import type { RanRecord, RanState } from './ran'
25import { CHAR_BUDGET, NODE_BUDGET, charCount, drawable, nodeCount, printable, problemOf } from './tree'
26import { DEFAULT_MODEL, modelOf, workersOf } from './settings'
27import { FLAGGED, LISTED, isFlagged, verdictOf } from './verdicts'
28import type { State } from './verdicts'
29
30const PANE = 'test-grader'
31const tests = atom({ plugin: 'test-grader', key: 'tests' } as const, [])
32const coverage = atom({ plugin: 'test-grader', key: 'coverage' } as const, null)
33const run = atom({ plugin: 'test-grader', key: 'run' } as const, { state: 'idle' })
34const existingMeta = atom({ plugin: 'test-grader', key: 'existing' } as const, { state: 'idle', done: 0, total: 0, results: [] })
35// The grades of the project's tests: one value of the session's state is held to 4 MiB, and a
36// project of thousands of tests, each with its reasons, is more. The results go in chunks of a
37// family, in order; the run's own value counts them and keeps its results field empty
38const CHUNK = 1_000_000
39const readChunk = async ($: EngineInterface, id: number): Promise<ExistingTest[]> => (await $.state.get({ plugin: 'test-grader', key: 'results', id: String(id) })).value ?? []
40const writeChunk = async ($: EngineInterface, id: number, chunk: ExistingTest[]): Promise<void> => void (await $.state.set({ plugin: 'test-grader', key: 'results', id: String(id) }, chunk))
41const readRun = async ($: EngineInterface): Promise<ExistingRun> => {
42  const meta = await read($, existingMeta)
43  if (!meta.chunks) return meta
44  const parts = await Promise.all(Array.from({ length: meta.chunks }, (_, i) => readChunk($, i)))
45  return { ...meta, results: parts.flat() }
46}
47// each write of the run waits for the one before: a write reads the whole and puts it back
48let runWrites: Promise<unknown> = Promise.resolve()
49const chunksWritten = new Map<number, string>()
50const updateRun = ($: EngineInterface, fn: (r: ExistingRun) => ExistingRun): Promise<void> => {
51  const done = runWrites.then(async () => {
52    const before = await readRun($)
53    const { results, ...meta } = fn(before)
54    const chunks: ExistingTest[][] = []
55    let size = CHUNK
56    for (const t of results) {
57      const length = JSON.stringify(t).length + 1
58      if (size + length > CHUNK) {
59        chunks.push([])
60        size = 0
61      }
62      chunks.at(-1)!.push(t)
63      size += length
64    }
65    // a chunk as it was last written is not written again
66    for (const [i, chunk] of chunks.entries()) {
67      const text = JSON.stringify(chunk)
68      if (chunksWritten.get(i) === text) continue
69      await writeChunk($, i, chunk)
70      chunksWritten.set(i, text)
71    }
72    // fewer than before: the chunks past the end are emptied
73    for (let i = chunks.length; i < (before.chunks ?? 0); i += 1) {
74      await writeChunk($, i, [])
75      chunksWritten.delete(i)
76    }
77    await update($, existingMeta, () => ({ ...meta, results: [], chunks: chunks.length }))
78  })
79  runWrites = done.catch(() => undefined)
80  return done
81}
82const noteError = atom({ plugin: 'test-grader', key: 'noteError' } as const, null)
83const opened = atom({ plugin: 'test-grader', key: 'open' } as const, [])
84const fileOpen = atom({ plugin: 'test-grader', key: 'fileOpen' } as const, {})
85const openError = atom({ plugin: 'test-grader', key: 'openError' } as const, null)
86const seen = atom({ plugin: 'test-grader', key: 'seen' } as const, {})
87const openFor = atom({ plugin: 'test-grader', key: 'openFor' } as const, null)
88const root = atom({ plugin: 'test-grader', key: 'root' } as const, null)
89const rounds = atom({ plugin: 'test-grader', key: 'rounds' } as const, {})
90const outbox = atom({ plugin: 'test-grader', key: 'outbox' } as const, { accepted: [], going: [], spent: [] })
91const coverWith = atom({ plugin: 'test-grader', key: 'coverWith' } as const, null)
92const saveError = atom({ plugin: 'test-grader', key: 'saveError' } as const, null)
93const graderError = atom({ plugin: 'test-grader', key: 'graderError' } as const, null)
94const unrated = atom({ plugin: 'test-grader', key: 'unrated' } as const, {})
95const testRuns = atom({ plugin: 'test-grader', key: 'testRuns' } as const, {})
96const basesFound = atom({ plugin: 'test-grader', key: 'basesFound' } as const, 0)
97const survived = atom({ plugin: 'test-grader', key: 'survived' } as const, {})
98const modified = atom({ plugin: 'test-grader', key: 'modified' } as const, [])
99const layersFound = atom({ plugin: 'test-grader', key: 'layersFound' } as const, 0)
100const ranRecord = atom({ plugin: 'test-grader', key: 'ranRecord' } as const, null)
101const proposed = atom({ plugin: 'test-grader', key: 'proposed' } as const, {})
102const measuredStrong = atom({ plugin: 'test-grader', key: 'measuredStrong' } as const, {})
103
104const GREEN = '#4ade80'
105const AMBER = '#fbbf24'
106const RED = '#f87171'
107const MUTED = '#8b90a0'
108const TRACK = '#343848'
109const VIOLET = '#a78bfa'
110const BLUE = '#60a5fa'
111// the rows kept pressed open, and the tests written this session the list keeps: the oldest go
112const MAX_OPEN = 60
113const MAX_TESTS = 5_000
114const CELLS = 12
115// Go's package bars: how many show, and how wide their labels may be
116const PACKAGE_BARS = 8
117const PACKAGE_LABEL = 28
118// where the pane keeps whether every package is shown
119const ALL_PACKAGES = 'cov:packages'
120// Grade all tests: cases per grader call
121const BATCH = 10
122// grader calls in flight at once, from the graderWorkers setting (1 to 20), 10 by default
123let parallel = 10
124// the model that grades, from the graderModel setting; set as the module loads, and again
125// when the person changes it in /config
126let graderModel: string = DEFAULT_MODEL
127// grades again what the first grade flagged (a regrade, evidence, a last round), when set
128let escalateModel: string | null = null
129// which form of the grader request the host took last: an older host takes only a plainer one
130let shapeTaken = 0
131
132const ORANGE = '#fb923c'
133const PINK = '#f472b6'
134const VERDICT_COLOR: Record<Verdict, string> = { strong: GREEN, shallow: AMBER, brittle: ORANGE, hollow: RED, duplicate: PINK }
135const verdictColor = (v: Verdict | undefined): string => (v === undefined ? MUTED : VERDICT_COLOR[v])
136
137// a count of tokens as the pane shows it: 950, 310k, 1.2M
138const tokens = (n: number): string => (n >= 1e6 ? `${(n / 1e6).toFixed(1)}M` : n >= 1e3 ? `${Math.round(n / 1e3)}k` : `${n}`)
139
140const pctColor = (p: number): string => (p >= 80 ? GREEN : p >= 50 ? AMBER : RED)
141
142// the lines a text fills at this width, broken between words; a word wider than a line, in pieces
143const wrapWords = (s: string, width: number): string[] => {
144  // a width that is no number, or less than one, would never end the loop below
145  const n = Number.isFinite(width) ? Math.max(1, Math.floor(width)) : 60
146  const lines: string[] = []
147  let line = ''
148  for (const word of s.split(' ')) {
149    for (let rest = word; ; ) {
150      const room = line ? n - line.length - 1 : n
151      if (rest.length <= room) {
152        line = line ? `${line} ${rest}` : rest
153        break
154      }
155      if (line) lines.push(line), (line = '')
156      else lines.push(rest.slice(0, n)), (rest = rest.slice(n))
157    }
158  }
159  return [...lines, line]
160}
161
162// A note to Claude: a user-role row added to the conversation, read in the turn under way or
163// the next one, never a prompt, so no turn is started for it. A refusal or a failure is kept
164// for the pane to show, until a note goes through. The debug log has every note, added or not
165// (a test cannot see a row a mod appends)
166const share = async ($: EngineInterface, text: string): Promise<void> => {
167  let error: string | null = null
168  try {
169    const row = await $.session.append({ message: { type: 'user', content: [{ type: 'text', text }] } })
170    if (row.deny !== undefined) error = row.deny
171  } catch (err) {
172    error = err instanceof Error ? err.message : String(err)
173  }
174  $.ui.log(`test-grader: note to Claude (${error === null ? 'appended' : `not appended: ${error}`}): ${text}`, { to: 'debug' })
175  await update($, noteError, () => error)
176}
177
178// the flagged tests of a list, worst grade first, one line each
179const flaggedLines = (list: { file: string; name: string; verdict?: Verdict; reason?: string; confidence?: Confidence; before?: Before }[], cwd: string): string[] =>
180  FLAGGED.flatMap(v =>
181    list.filter(t => t.verdict === v).map(t => `- ${v}${unsure(t)} · ${shortPath(t.file, cwd)} · ${t.name} — ${t.reason ?? ''}${beforeLine(t)}`),
182  )
183// the grade a test had before it was graded again: was the earlier concern met, or is this a new one
184// (one the same as the new grade, word for word, tells nothing)
185const beforeLine = (t: { verdict?: Verdict; reason?: string; before?: Before }): string =>
186  t.before && !(t.before.verdict === t.verdict && t.before.reason === t.reason) ? `\n  Before: ${t.before.verdict}${t.before.reason ? ` — ${t.before.reason}` : ''}` : ''
187// a test's grade, kept as its grade before when it is graded again
188const beforeOf = (t: { verdict?: Verdict; reason?: string }): { before: Before } | Record<string, never> =>
189  t.verdict ? { before: { verdict: t.verdict, ...(t.reason ? { reason: t.reason } : {}) } } : {}
190// a grade the grader was not sure of, marked where it is told: it may read otherwise on a regrade
191const unsure = (t: { confidence?: Confidence }): string => (t.confidence === 'low' || t.confidence === 'medium' ? ` (${t.confidence} confidence)` : '')
192
193// What grader calls cost, summed: tokens in (of them read from the prompt cache) and out
194export type Spent = { input: number; cached: number; output: number; cost: number; unpriced: number }
195const addUsage = (spent: Spent | undefined, model: string, usage: { input_tokens?: number; output_tokens?: number; cache_read_input_tokens?: number; cache_creation_input_tokens?: number } | undefined): void => {
196  if (!spent || !usage) return
197  spent.input += (usage.input_tokens ?? 0) + (usage.cache_read_input_tokens ?? 0) + (usage.cache_creation_input_tokens ?? 0)
198  spent.cached += usage.cache_read_input_tokens ?? 0
199  spent.output += usage.output_tokens ?? 0
200  // priced call by call: Haiku 5.5's price depends on each prompt's length
201  const cost = costOf(model, usage)
202  if (cost === null) spent.unpriced += 1
203  else spent.cost += cost
204}
205
206// a run's cost as the pane shows it: dollars to the cent, or finer below one; a call whose
207// model has no known price says so
208const dollars = (spent: { cost?: number; unpriced?: number }): string => {
209  const cost = spent.cost ?? 0
210  const shown = cost === 0 ? '' : cost >= 1 ? `$${cost.toFixed(2)}` : `$${cost.toPrecision(2)}`
211  const unpriced = spent.unpriced ? `${spent.unpriced} call${spent.unpriced === 1 ? '' : 's'} unpriced` : ''
212  return [shown && `about ${shown}`, unpriced].filter(Boolean).join(', ')
213}
214
215// the project's own rules for its tests, from .test-grader.md at its root: read at a session's
216// start and each turn's end; the grader is told them after the rubric
217const RUBRIC_FILE = '.test-grader.md'
218const MAX_RULES = 4_000
219let projectRules = ''
220const readRules = async ($: EngineInterface): Promise<void> => {
221  const cwd = await projectDir($)
222  projectRules = cwd ? ((await $.fs.read(`${cwd}/${RUBRIC_FILE}`).catch(() => '')) ?? '').trim().slice(0, MAX_RULES) : ''
223  const layers = cwd ? ((await $.fs.read(`${cwd}/${LAYERS_FILE}`).catch(() => '')) ?? '') : ''
224  if (layers !== layersText) {
225    layersText = layers
226    layerRules = layerRulesOf(layers)
227    // every file read again for its layer by the rules now
228    layerCache.clear()
229    readAt.clear()
230    await update($, layersFound, n => n + 1)
231  }
232}
233
234// Each test file's layer, read from its path and, once its text is read, its imports and Go
235// build tags; the project's .test-grader-layers rules first
236let layersText = ''
237let layerRules: LayerRules = []
238const layerCache = new Map<string, Layer>()
239// a Go file's build tags, read with its layer: a coverage run built without one of them did not
240// compile the file's tests
241const tagsCache = new Map<string, string[]>()
242const tagsAt = (file: string): string[] => tagsCache.get(file) ?? []
243const keepTags = (file: string, text: string): void => {
244  if (file.endsWith('.go')) tagsCache.set(file, goTagsOf(text))
245}
246const layerAt = (cwd: string, file: string): Layer => layerCache.get(file) ?? layerOf(shortPath(file, cwd), null, layerRules)
247
248// an API error worth trying again: too many requests, overloaded, the server's own, or no answer
249const RETRIES = 3
250const isPassing = (r: { reason: string; status?: number | null; error?: string }): boolean =>
251  r.reason === 'api-error' && (r.status === null || r.status === 429 || r.status === 529 || (r.status ?? 0) >= 500 || r.error === 'rate_limit' || r.error === 'overloaded' || r.error === 'server_error')
252const sleep = ($: EngineInterface, ms: number): Promise<void> => new Promise(done => void $.clock.after(ms, () => done()))
253// how long one grader call may take
254const CALL_TIMEOUT = 120_000
255
256// prior: the grades these cases had for this same text, which a regrade keeps unless it finds them wrong
257// edited: the grades they had before their text changed, for the grader to say whether the change met that concern
258type GradeOptions = { evidence?: string; isMeasured?: boolean; model?: string; signal?: AbortSignal; spent?: Spent; confirming?: Graded[]; prior?: { name: string; verdict: Verdict; reason?: string }[]; edited?: { name: string; verdict: Verdict; reason?: string }[] }
259
260// The code a test file tests, as the grader reads it beside the test: the project files it
261// imports by a relative path (JS and TS, Python, Ruby), its package's other files (Go), or the
262// class it is named for (Kotlin, Java), each cut to its share of MAX_UNDER_TEST. Kept per file
263// as it last read, until the file changes
264const MAX_UNDER_TEST = 16_000
265const MAX_UNDER_TEST_FILES = 4
266const underTestCache = new Map<string, { hash: string; text: string }>()
267const codeUnderTest = async ($: EngineInterface, file: string, text: string): Promise<string> => {
268  const hash = fingerprint(text)
269  const kept = underTestCache.get(file)
270  if (kept?.hash === hash) return kept.text
271  const cwd = await projectDir($)
272  const paths = await candidatesFor($, file, text, cwd)
273  const found: { path: string; text: string }[] = []
274  for (const path of paths) {
275    if (found.length >= MAX_UNDER_TEST_FILES || found.some(f => f.path === path) || path === file || TEST_FILE.test(path)) continue
276    const body = await $.fs.read(path).catch(() => null)
277    if (body !== null && body.trim() !== '') found.push({ path, text: body })
278  }
279  const share = Math.floor(MAX_UNDER_TEST / Math.max(1, found.length))
280  const out =
281    found.length === 0
282      ? ''
283      : ['The code under test, as the test file reaches it:', ...found.map(f => [`--- ${shortPath(f.path, cwd)} ---`, '```', clamp(f.text, share), '```'].join('\n'))].join('\n')
284  underTestCache.set(file, { hash, text: out })
285  return out
286}
287
288// where a test file's code under test may be, most likely first; a path that is not there is
289// passed over when read
290const candidatesFor = async ($: EngineInterface, file: string, text: string, cwd: string): Promise<string[]> => {
291  const dir = file.slice(0, file.lastIndexOf('/'))
292  const join = (base: string, rel: string): string => {
293    const parts = `${base}/${rel}`.split('/')
294    const out: string[] = []
295    for (const part of parts) part === '..' ? out.pop() : part !== '.' && out.push(part)
296    return out.join('/')
297  }
298  const kind = kindOf(file)
299  if (kind === 'js') {
300    const specs = [...text.matchAll(/(?:\bfrom\s+|\brequire\(\s*|\bimport\(\s*|^import\s+)['"](\.{1,2}\/[^'"]+)['"]/gm)].map(m => m[1]!)
301    return specs.flatMap(spec => {
302      const base = join(dir, spec.replace(/\.[cm]?js$/, ''))
303      return [spec.match(/\.[cm]?[jt]sx?$/) ? join(dir, spec) : null, ...['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '/index.ts', '/index.js'].map(ext => base + ext)].filter((p): p is string => p !== null)
304    })
305  }
306  if (kind === 'py') {
307    return [...text.matchAll(/^\s*from\s+(\.*)([\w.]*)\s+import\b/gm)].flatMap(m => {
308      const rel = m[2]!.replace(/\./g, '/')
309      if (m[1]) {
310        const up = '../'.repeat(m[1].length - 1)
311        return rel ? [join(dir, `${up}${rel}.py`), join(dir, `${up}${rel}/__init__.py`)] : []
312      }
313      return rel ? [`${cwd}/${rel}.py`, `${cwd}/src/${rel}.py`, `${dir}/${rel}.py`] : []
314    })
315  }
316  if (kind === 'rb') return [...text.matchAll(/^\s*require_relative\s+['"]([^'"]+)['"]/gm)].map(m => join(dir, m[1]!.endsWith('.rb') ? m[1]! : `${m[1]}.rb`))
317  if (kind === 'go') {
318    const entries = await $.fs.list(dir).catch(() => [])
319    return entries.filter(e => e.kind === 'file' && e.name.endsWith('.go') && !e.name.endsWith('_test.go')).map(e => `${dir}/${e.name}`)
320  }
321  if (kind === 'jvm') {
322    const name = file.slice(file.lastIndexOf('/') + 1).replace(/(Tests?|Spec|IT)\.(kt|java)$/, '.$2')
323    const mainDir = dir.replace(/\/src\/test\//, '/src/main/')
324    return mainDir === dir ? [] : [`${mainDir}/${name}`]
325  }
326  return []
327}
328
329// These cases of this file, judged; null when the grader gave no answer. A plain grade that
330// flags a test is checked by a second, more careful call before the flag stands: where it
331// disagrees, its grade is the one given. A test a measured mutation made fail is not hollow
332const grade = async ($: EngineInterface, file: string, text: string, names: string[], options: GradeOptions = {}): Promise<Graded[] | null> => {
333  const first = await gradeCall($, file, text, names, options)
334  if (first === null) return null
335  const measured = options.isMeasured ? first.map(v => (v.verdict === 'hollow' ? { ...v, verdict: 'strong' as const, reason: `${v.reason} (A measured mutation made it fail, so it is not hollow.)` } : v)) : first
336  // evidence and a second look are already the careful call
337  if (options.evidence || options.model !== undefined) return measured
338  const flagged = measured.filter(v => isFlagged(v.verdict))
339  if (flagged.length === 0) return measured
340  const second = await gradeCall($, file, text, flagged.map(v => v.name), { ...options, model: escalateModel ?? graderModel, confirming: flagged })
341  if (second === null) return measured
342  return measured.map(v => (isFlagged(v.verdict) ? (second.find(s => s.name === v.name) ?? v) : v))
343}
344
345// The mutations test_verify ran that a test under review let through: a test that misses a
346// measured change to the code is not strong, unless the change alters nothing it should catch
347// or the test has changed since to catch it
348const MAX_SURVIVED = 3
349const MAX_SURVIVED_TESTS = 500
350const survivedOf = (all: Record<string, { change: string; textOf: string }[]>, file: string, text: string, names: string[]): string[] => {
351  const own = ownTexts(text, file)
352  const cases = [...new Set(caseNames(text, file))]
353  const lines = Object.entries(all).flatMap(([key, ms]) => {
354    const at = key.indexOf('::')
355    const [of, caseName] = [key.slice(0, at), key.slice(at + 2)]
356    if (of !== file) return []
357    const name = names.find(n => caseOf(cases, n) === caseName)
358    if (name === undefined) return []
359    return ms.map(m => `"${name}" still passed with ${m.change}${m.textOf === own(caseName) ? '' : ' (measured before its text last changed)'}`)
360  })
361  if (lines.length === 0) return []
362  return [
363    `test_verify ran these tests against a mutated copy of the code, and they let the change through: ${lines.join('; ')}.`,
364    'A test that lets a measured change through is not strong, unless the change alters nothing the test should catch, or the test changed since and its assertions now catch it. Say which in reason; where it is a gap, name it in missed.',
365  ]
366}
367
368// What a grader call reads besides the rubric and the project's rules: the test file (whole or
369// an excerpt), the code under test, and what it is asked, with any last grades, evidence or flags
370type AskOptions = Pick<GradeOptions, 'evidence' | 'isMeasured' | 'confirming' | 'prior' | 'edited'>
371const askOf = async ($: EngineInterface, file: string, text: string, names: string[], { evidence, isMeasured, confirming, prior, edited }: AskOptions = {}): Promise<{ source: string; underTest: string; ask: string }> => {
372  const source = excerptOf(text, names, file)
373  const underTest = await codeUnderTest($, file, text).catch(() => '')
374  const ask = [
375    ...(source !== text
376      ? [
377          "The file is long, so the source above is an excerpt: the cases under review whole, the file's head, and the declarations they use from elsewhere in it. Other tests are left out.",
378          'Judge each case by what it does. Do not mark one down for code the excerpt leaves out.',
379          ...othersOf(text, names, file),
380        ]
381      : []),
382    ...(prior && prior.length > 0
383      ? [
384          `These were graded before, on this same text: ${JSON.stringify(prior.map(p => ({ name: p.name, verdict: p.verdict, reason: p.reason ?? '' })))}`,
385          'A grade belongs to the test, not to the run: keep each unless you find it wrong. Where you change one, say in reason what the last grade got wrong.',
386        ]
387      : []),
388    ...(edited && edited.length > 0
389      ? [
390          `These were graded before their text last changed: ${JSON.stringify(edited.map(p => ({ name: p.name, verdict: p.verdict, reason: p.reason ?? '' })))}`,
391          'Grade each as it is now. Where a change met the earlier concern, say so in reason. Flag one again only for a gap a plausible bug slips through, not for wording or style, and where it is a different concern from the earlier one, say that it is.',
392        ]
393      : []),
394    ...survivedOf(await read($, survived), file, text, names),
395    ...(evidence
396      ? [
397          `The developer's session sent evidence about this test: ${JSON.stringify(evidence)}`,
398          'Weigh it, but check each claim against the source above: you cannot run code. Evidence cannot add an assertion the source does not contain.',
399          'Evidence should name a concrete mutation (where, before, after), the command run, and the test\'s output before and after; evidence "measured by test-grader" was run by the tool itself, not claimed. Accept it only when the mutation changes behaviour the test\'s assertions in the source would detect; reject evidence that only reports the test passing, coverage, or claims about code not shown.',
400          'In reason, say which part of the evidence changed your verdict, or why it did not.',
401          ...(isMeasured ? ['The mutation was measured: the test failed with it, so it is not "hollow".'] : []),
402        ]
403      : []),
404    ...(confirming
405      ? [
406          `A first, quick pass flagged these: ${JSON.stringify(confirming.map(v => ({ name: v.name, verdict: v.verdict, reason: v.reason })))}`,
407          'Check each yourself against the source. Keep a flag only where you agree; for "shallow", name the bug in missed. Where the first pass was wrong, or you are unsure, answer "strong".',
408        ]
409      : []),
410    `Review ONLY these test cases: ${JSON.stringify(names)}`,
411    'Give one verdict per name, under that name exactly. A test whose cases run inside it (t.Run subtests, table rows, subTest) gets one verdict for all its cases together, under its own name.',
412    ...loopsOf(text, names, file),
413  ].join('\n')
414  return { source, underTest, ask }
415}
416
417// One grader call: these cases of this file, judged; null when the grader gave no answer. An
418// API error that may pass is tried again, waiting longer each time; a stopped run is not
419const gradeCall = async ($: EngineInterface, file: string, text: string, names: string[], { evidence, isMeasured, model, signal, spent, confirming, prior, edited }: GradeOptions = {}): Promise<Graded[] | null> => {
420  // why each test asked about got no verdict, for its row; a verdict clears it. A second look
421  // that gives none leaves the first grade standing, so it says nothing
422  const noteWhy = async (why: (name: string) => string | null): Promise<void> => {
423    if (confirming) return
424    await update($, unrated, all => {
425      const next = { ...all }
426      for (const name of names) {
427        const reason = why(name)
428        if (reason === null) delete next[roundKey(file, name)]
429        else next[roundKey(file, name)] = reason
430      }
431      return next
432    })
433  }
434  const { source, underTest, ask } = await askOf($, file, text, names, { evidence, isMeasured, confirming, prior, edited })
435  const request = {
436    model: model ?? graderModel,
437    maxTokens: MAX_REPLY,
438    // the plain grade asks for little thought; a second look, or evidence, the model's own
439    ...(model === undefined && !evidence ? { effort: 'low' as const } : {}),
440    timeoutMs: CALL_TIMEOUT,
441    system: [{ text: RUBRIC, cache: true as const }, ...(projectRules ? [{ text: `The project's own rules for its tests (${RUBRIC_FILE}):\n${projectRules}` }] : [])],
442    // the file first and marked, so the next batch of the same file reads it from the cache
443    prompt: [{ text: [`Test file: ${file}`, '```', source, '```', ...(underTest ? ['', underTest] : [])].join('\n'), cache: true as const }, { text: `\n${ask}` }],
444  }
445  // an older host takes less: the same request with plain texts and no effort, then the model
446  // and one prompt alone; the first it takes is kept for the session's later calls
447  const joined = (blocks: { text: string }[], by: string): string => blocks.map(b => b.text).join(by)
448  const shapes: Parameters<typeof $.model.complete>[0][] = [
449    request,
450    { model: request.model, maxTokens: request.maxTokens, timeoutMs: request.timeoutMs, system: joined(request.system, '\n\n'), prompt: joined(request.prompt, '') },
451    { model: request.model, prompt: `${joined(request.system, '\n\n')}\n\n${joined(request.prompt, '')}` },
452  ]
453  for (let attempt = 0; ; attempt++) {
454    if (signal?.aborted) return null
455    let reply: Awaited<ReturnType<typeof $.model.complete>> | undefined
456    let refused = ''
457    for (let s = shapeTaken; s < shapes.length && reply === undefined; s++) {
458      try {
459        reply = await $.model.complete(shapes[s]!, signal ? { signal } : undefined)
460        shapeTaken = s
461      } catch (err) {
462        // a request the host will not send rejects at once: a blocked model, or a shape it does not take
463        if (signal?.aborted) return null
464        refused = err instanceof Error ? err.message : String(err)
465        $.ui.log(`test-grader: the grader call failed for ${file} (request ${s + 1} of ${shapes.length}: ${refused})`, { to: 'debug' })
466      }
467    }
468    if (reply === undefined) {
469      await update($, graderError, () => `The grader (${request.model}) call failed: ${refused}`)
470      await noteWhy(() => `The grader (${request.model}) call failed: ${refused}`)
471      return null
472    }
473    addUsage(spent, request.model, reply.usage)
474    if (reply.isAnswered) {
475      const parsed = parseVerdicts(reply.text)
476      const verdicts = foldCases(names, asAsked(names, parsed.verdicts))
477      const { isCut } = parsed
478      if (isCut) {
479        $.ui.log(`test-grader: a grader reply was cut off (${reply.usage?.output_tokens ?? '?'} of ${MAX_REPLY} tokens) for ${file}: kept ${verdicts.length} verdicts of ${JSON.stringify(names)}`, { to: 'debug' })
480      }
481      // an answer with no verdict for any case asked about leaves them unrated: the pane says
482      // what came back, so a model that will not answer in the format can be told apart
483      if (names.length > 0 && !verdicts.some(v => among(names, v.name))) {
484        const said = reply.text.replace(/\s+/g, ' ').trim()
485        $.ui.log(`test-grader: the grader answered with no verdict for ${file}: ${said.slice(0, 2_000)}`, { to: 'debug' })
486        await update($, graderError, () => `The grader (${request.model}) answered with no verdict it could read: "${said.length > 160 ? `${said.slice(0, 160)}…` : said}".`)
487      } else await update($, graderError, () => null)
488      await noteWhy(name => unratedWhy(reply.text, verdicts, isCut, names, name, request.model))
489      await recordProposals($, file, text, verdicts).catch(() => undefined)
490      return verdicts
491    }
492    if (attempt >= RETRIES || !isPassing(reply as never)) {
493      const why = `${reply.reason}${'status' in reply ? ` ${reply.status ?? ''} ${reply.error}` : ''}`
494      $.ui.log(`test-grader: the grader gave no answer for ${file} (${why})`, { to: 'debug' })
495      // shown in the pane: a setting or an account that cannot reach the model says so there
496      await update($, graderError, () => `The grader (${request.model}) gave no answer: ${why}.`)
497      await noteWhy(() => `The grader (${request.model}) gave no answer: ${why}.`)
498      return null
499    }
500    // 2s, 4s, 8s, each with up to a second more, so parallel calls do not retry together
501    await sleep($, 2_000 * 2 ** attempt + Math.floor(Math.random() * 1_000))
502  }
503}
504
505// A test Claude wrote or edited, given a flagged grade, is told to Claude in a note, never a
506// prompt: the system prompt has told it to look up its tests' grades with test_grades once it is
507// done writing them, and follow up. A test graded strong after that is told as accepted.
508// Each test gets MAX_ROUNDS such rounds; past them the note says test-grader stops on it
509const roundKey = (file: string, name: string): string => `${file}::${name}`
510
511// the reasons these tests of a file have no verdict, as their last grader call left them, by name
512const unratedOf = async ($: EngineInterface, file: string): Promise<(name: string) => { reason?: string }> => {
513  const all = await read($, unrated)
514  return name => (all[roundKey(file, name)] ? { reason: all[roundKey(file, name)] } : {})
515}
516// a grading that failed outright: each of its tests says how
517const noteFailed = async ($: EngineInterface, file: string, names: string[], err: unknown): Promise<void> => {
518  const why = `Grading failed: ${err instanceof Error ? err.message : String(err)}`
519  await update($, unrated, all => ({ ...all, ...Object.fromEntries(names.map(name => [roundKey(file, name), why])) }))
520}
521
522// the round a test is on now: one more for a flagged grade, none once it is strong
523const countRound = async ($: EngineInterface, file: string, name: string, verdict: Verdict | undefined): Promise<{ round: number; wasRetried: boolean }> => {
524  const key = roundKey(file, name)
525  const before = (await read($, rounds))[key] ?? 0
526  const round = isFlagged(verdict) ? before + 1 : 0
527  if (round !== before) await update($, rounds, all => (({ [key]: _, ...rest }) => (round > 0 ? { ...rest, [key]: round } : rest))(all))
528  return { round, wasRetried: before > 0 }
529}
530
531// Grades wait in the outbox while grading is under way, then go as one note: several flagged
532// tests, or several files graded, are one round, not one note each.
533// A test graded again before then is listed once, at its latest grade
534type Report = { file: string; name: string; verdict?: Verdict; reason?: string; before?: Before }
535const reportGrades = async ($: EngineInterface, graded: Report[]): Promise<void> => {
536  for (const t of graded) {
537    // a test waiting in the outbox is in a round already counted (one edit graded on both
538    // lists): its entry takes this grade's words and keeps its place
539    const isSame = (o: Report): boolean => roundKey(o.file, o.name) === roundKey(t.file, t.name)
540    const box = await read($, outbox)
541    const waiting = [...box.accepted, ...box.going, ...box.spent].find(isSame)
542    // the same kind of grade: its words and verdict replace the waiting one's, in its place
543    if (waiting && isFlagged(waiting.verdict ?? (box.accepted.includes(waiting) ? 'strong' : undefined)) === isFlagged(t.verdict)) {
544      const take = (list: Report[]) => list.map(o => (isSame(o) ? { ...o, verdict: t.verdict ?? o.verdict, reason: t.reason ?? o.reason, before: t.before ?? o.before } : o))
545      await update($, outbox, b => ({ accepted: take(b.accepted), going: take(b.going), spent: take(b.spent) }))
546      continue
547    }
548    // another kind (a flagged test now strong): the waiting entry's round is undone, and the new
549    // grade counted in its stead
550    if (waiting && isFlagged(waiting.verdict)) {
551      const key = roundKey(t.file, t.name)
552      await update($, rounds, all => {
553        const left = (all[key] ?? 1) - 1
554        const { [key]: _, ...rest } = all
555        return left > 0 ? { ...rest, [key]: left } : rest
556      })
557    }
558    const { round, wasRetried } = await countRound($, t.file, t.name, t.verdict)
559    const kind = round === 0 && wasRetried ? 'accepted' : round > 0 && round <= MAX_ROUNDS ? 'going' : round === MAX_ROUNDS + 1 ? 'spent' : null
560    await update($, outbox, box => {
561      const key = roundKey(t.file, t.name)
562      const others = (list: Report[]) => list.filter(o => roundKey(o.file, o.name) !== key)
563      const next = { accepted: others(box.accepted), going: others(box.going), spent: others(box.spent) }
564      return kind === null ? next : { ...next, [kind]: [...next[kind], t] }
565    })
566  }
567  await flush($)
568}
569
570// While a subagent is still at work its files keep changing: their grades wait for it to finish
571// (its turn's end sends them), for HOLD_MAX at most
572const HOLD_MAX = 10 * 60_000
573let heldSince: number | null = null
574const isSubagentWorking = async ($: EngineInterface): Promise<boolean> => (await $.agent.list().catch(() => [])).some(a => a.status === 'running' && a.type !== 'teammate')
575const flush = async ($: EngineInterface): Promise<void> => {
576  if (working > 0) return
577  const box = await read($, outbox)
578  const { accepted, going, spent } = box
579  if (accepted.length + going.length + spent.length === 0) return
580  const now = await $.clock.now()
581  if (await isSubagentWorking($)) {
582    heldSince ??= now
583    if (now - heldSince < HOLD_MAX) return
584  }
585  heldSince = null
586  await update($, outbox, () => ({ accepted: [], going: [], spent: [] }))
587  const cwd = await projectDir($)
588  const lines = [
589    ...(accepted.length > 0 ? ['Now graded strong (test-grader):', ...accepted.map(t => `- strong · ${shortPath(t.file, cwd)} · ${t.name}`)] : []),
590    ...(going.length > 0 ? ['Tests that need work (test-grader):', ...flaggedLines(going, cwd), FOLLOW_UP] : []),
591    ...(spent.length > 0 ? [`Still flagged after ${MAX_ROUNDS} rounds (test-grader stops on these):`, ...flaggedLines(spent, cwd), SPENT_FOLLOW_UP] : []),
592  ]
593  // a note, added to the conversation: no turn is started for it
594  await share($, lines.join('\n'))
595}
596
597// Grading under way in this load of the module: a Grade all run, a regrade, a new test's
598// grading. The host keeps their marks (a run running, rows reviewing, tests pending) across a
599// reload of this mod, which drops the work itself; at a session's start with none under way
600// here, resume takes the marks left behind for work to do again
601let working = 0
602// the last grading under way done: the grades it left go to Claude
603const finishWork = async ($: EngineInterface): Promise<void> => {
604  working -= 1
605  if (working === 0) await flush($).catch(() => undefined)
606}
607const busy = async <T,>($: EngineInterface, work: () => Promise<T>): Promise<T> => {
608  working += 1
609  try {
610    return await work()
611  } finally {
612    await finishWork($)
613  }
614}
615
616// work started on the next tick, counted as under way from now
617const soon = ($: EngineInterface, work: () => Promise<void>): void => {
618  working += 1
619  void $.clock.after(1, () => void work().catch(() => undefined).finally(() => finishWork($)))
620}
621
622// A grade is for the text it read: the file as it is now when any of these cases' own text has
623// changed since (an outside edit while the grader ran), else null. Graded again at most so often
624const MAX_STALE = 2
625const staleOf = async ($: EngineInterface, file: string, graded: string, names: string[]): Promise<string | null> => {
626  const now = await $.fs.read(file).catch(() => null)
627  if (now === null || now === graded) return null
628  return names.some(name => caseTextOf(graded, name, file) !== caseTextOf(now, name, file)) ? now : null
629}
630
631// model: a second look, for a test graded again after a flagged grade
632const evaluate = ($: EngineInterface, file: string, ids: Map<string, string>, model?: string): Promise<void> => busy($, () => evaluateNow($, file, ids, model))
633const evaluateNow = async ($: EngineInterface, file: string, ids: Map<string, string>, model?: string, tries = 0): Promise<void> => {
634  const fail = async (): Promise<void> => {
635    const why = await unratedOf($, file)
636    await update($, tests, list => list.map(t => (ids.has(t.id) && t.status === 'pending' ? { ...t, status: 'failed' as const, ...why(t.name) } : t)))
637  }
638  try {
639    const text = await $.fs.read(file)
640    const edited = (await read($, tests)).flatMap(t => (ids.has(t.id) && t.before ? [{ name: t.name, ...t.before }] : []))
641    const verdicts = await grade($, file, text, [...ids.values()], { model, ...(edited.length > 0 ? { edited } : {}) })
642    if (verdicts === null) return fail()
643    // the file changed under the grade: graded again on its new text, not kept for the old
644    if (tries < MAX_STALE && (await staleOf($, file, text, [...ids.values()])) !== null) return evaluateNow($, file, ids, model, tries + 1)
645    const why = await unratedOf($, file)
646    const claimed = byCase([...new Set(ids.values())], verdicts)
647    await update($, tests, list =>
648      // a looped test becomes one entry per case it generates
649      list.flatMap((t): TrackedTest[] => {
650        if (!ids.has(t.id)) return [t]
651        const found = claimed.get(t.name) ?? []
652        if (found.length === 0) return [{ ...t, status: 'failed', ...why(t.name) }]
653        return found.map((v, k) => ({
654          ...t,
655          id: k === 0 ? t.id : `${t.id}-${k}`,
656          name: v.name,
657          status: 'done',
658          summary: v.summary,
659          verdict: v.verdict,
660          reason: v.reason,
661          confidence: v.confidence,
662        }))
663      }),
664    )
665    const mine = (await read($, tests)).filter(t => [...ids.keys()].some(id => t.id === id || t.id.startsWith(`${id}-`)))
666    await keepTracked($, file, text, mine).catch(() => undefined)
667    await reportGrades($, mine)
668  } catch (err) {
669    await noteFailed($, file, [...ids.values()], err)
670    await fail()
671  }
672}
673
674// A grade given to a test Claude wrote goes into the saved grades too, for the text it read, so
675// it outlives the session: the session's own list does not, and a later one would list the test
676// ungraded and grade it again as new
677const keepTracked = async ($: EngineInterface, file: string, text: string, graded: TrackedTest[]): Promise<void> => {
678  // one outside the project is graded and told, not listed: nor kept with the project's grades
679  const cwd = await projectDir($)
680  if (cwd === '' || !file.startsWith(`${cwd}/`)) return
681  const own = ownTexts(text, file)
682  const suites = suitesOf(text, file)
683  const rows: ExistingTest[] = graded.flatMap(t =>
684    t.status === 'done' && t.verdict
685      ? [{ file, name: t.name, verdict: t.verdict, summary: t.summary, reason: t.reason, ...(t.confidence ? { confidence: t.confidence } : {}), ...(suites.has(t.name) ? { suite: suites.get(t.name) } : {}), ...(t.before ? { before: t.before } : {}), textOf: own(t.name) }]
686      : [],
687  )
688  if (rows.length === 0) return
689  const names = new Set(rows.map(r => r.name))
690  await updateRun($, r => ({ ...r, results: [...r.results.filter(t => !(t.file === file && names.has(t.name))), ...rows] }))
691  await saveGrades($)
692}
693
694const mtime = async ($: EngineInterface, path: string): Promise<number | null> => {
695  try {
696    return (await $.fs.stat(path)).mtimeMs
697  } catch {
698    return null
699  }
700}
701
702// a Go file none of whose code ran, as read: whether it is generated, and its package's role
703const filesRead = new Map<string, { isGenerated: boolean; role: PackageRole | undefined }>()
704const GENERATED_READS = 300
705
706// the report a coverage run left in this folder (the project's, or a part's), its paths relative to it
707const readCoverageAt = async ($: EngineInterface, cwd: string): Promise<Coverage | null> => {
708  // a file the project's ignore list names does not count (scratch code in backend/tmp/)
709  // so does one git ignores, by the root's .gitignore or the part's, and one the part's own
710  // .test-grader-ignore names, read from the part's folder
711  const root = await projectDir($)
712  const isIgnored = await ignoreOf($, root)
713  const listOf = async (path: string): Promise<string> => (await $.fs.read(path).catch(() => '')).split('\n').filter(l => !l.trim().startsWith('!')).join('\n')
714  const isGitIgnored = ignoredBy(await listOf(`${root}/.gitignore`))
715  const isPartIgnored = cwd === root ? () => false : ignoredBy([await listOf(`${cwd}/.gitignore`), await listOf(`${cwd}/${IGNORE}`)].join('\n'))
716  const isKept = (file: string): boolean => {
717    const rel = shortPath(file, root)
718    return !isIgnored(rel) && !isGitIgnored(rel) && !(file.startsWith(`${cwd}/`) && isPartIgnored(file.slice(cwd.length + 1)))
719  }
720  const summaryPath = `${cwd}/coverage/coverage-summary.json`
721  const lcovPath = `${cwd}/coverage/lcov.info`
722  const xmlPath = `${cwd}/coverage.xml`
723  const goPath = `${cwd}/.test-grader-go-coverage.txt`
724
725  const at = await mtime($, summaryPath)
726  if (at !== null) {
727    try {
728      const report = JSON.parse(await $.fs.read(summaryPath)) as Record<string, Record<string, { pct?: unknown; total?: unknown; covered?: unknown }>>
729      const total = report.total!
730      const byFile = Object.entries(report)
731        .filter(([file]) => file !== 'total')
732        .map(([file, m]) => ({ file, total: Number(m.lines?.total ?? 0), covered: Number(m.lines?.covered ?? 0) }))
733      return {
734        byDir: byDirOf(byFile.filter(f => isKept(f.file.startsWith('/') ? f.file : `${cwd}/${f.file}`)), cwd),
735        lines: pct(total.lines?.pct),
736        statements: pct(total.statements?.pct),
737        branches: pct(total.branches?.pct),
738        functions: pct(total.functions?.pct),
739        ...(Number(total.statements?.total) > 0 ? { statementCount: { total: Number(total.statements!.total), covered: Number(total.statements!.covered ?? 0) } } : {}),
740        source: 'coverage-summary.json',
741        updatedAt: at,
742      }
743    } catch {
744      /* fall through to the next format */
745    }
746  }
747  const lcovAt = await mtime($, lcovPath)
748  if (lcovAt !== null) {
749    const text = await $.fs.read(lcovPath)
750    const sum = (key: string): number => [...text.matchAll(new RegExp(`^${key}:(\\d+)`, 'gm'))].reduce((s, m) => s + Number(m[1]), 0)
751    const ratio = (hit: string, found: string): number | null => (sum(found) > 0 ? pct((sum(hit) / sum(found)) * 100) : null)
752    // per file: each record from its SF: line to its end_of_record
753    const byFile = text.split('end_of_record').flatMap(record => {
754      const file = record.match(/^SF:(.+)$/m)?.[1]?.trim()
755      const count = (key: string): number => Number(record.match(new RegExp(`^${key}:(\\d+)`, 'm'))?.[1] ?? 0)
756      return file ? [{ file: file.startsWith('/') ? file : `${cwd}/${file}`, total: count('LF'), covered: count('LH') }] : []
757    })
758    return {
759      byDir: byDirOf(byFile.filter(f => isKept(f.file)), cwd),
760      lines: ratio('LH', 'LF'),
761      statements: null,
762      branches: ratio('BRH', 'BRF'),
763      functions: ratio('FNH', 'FNF'),
764      source: 'lcov.info',
765      updatedAt: lcovAt,
766    }
767  }
768  const xmlAt = await mtime($, xmlPath)
769  if (xmlAt !== null) {
770    const head = (await $.fs.read(xmlPath)).slice(0, 2000)
771    return { lines: attr(head, 'line-rate'), statements: null, branches: attr(head, 'branch-rate'), functions: null, source: 'coverage.xml', updatedAt: xmlAt }
772  }
773  const profileAt = await mtime($, `${cwd}/${GO_PROFILE}`)
774  if (profileAt !== null) {
775    const goMod = await $.fs.read(`${cwd}/go.mod`).catch(() => '')
776    const profile = await $.fs.read(`${cwd}/${GO_PROFILE}`)
777    // generated code (mockery's mocks, protobuf) is left out: a file none of whose code ran is
778    // read for Go's generated-code header, at most GENERATED_READS of them, each once a session
779    const first = goProfileOf(profile, moduleOf(goMod), cwd, isKept)
780    const unrun = first.byFile.filter(f => f.covered === 0 && f.file.startsWith('/')).slice(0, GENERATED_READS)
781    for (const f of unrun) if (!filesRead.has(f.file)) filesRead.set(f.file, await $.fs.read(f.file).then(text => ({ isGenerated: isGenerated(text), role: roleOf(text) }), () => ({ isGenerated: false, role: undefined })))
782    // parsed again only where generated files are to be left out: a large profile is slow to parse
783    const isAnyGenerated = first.byFile.some(f => filesRead.get(f.file)?.isGenerated === true)
784    const { statements, byFile, byPackage } = isAnyGenerated ? goProfileOf(profile, moduleOf(goMod), cwd, file => isKept(file) && filesRead.get(file)?.isGenerated !== true) : first
785    if (statements !== null) {
786      // each package's role, read from its first file: a command or test helper is not code its
787      // tests are missing. Only a package none of whose code ran is read (Go counts a package's
788      // own tests alone, so one with no tests is at 0%), at most ROLE_READS of them, each file
789      // read once a session
790      const firstFile = new Map<string, string>()
791      for (const f of byFile) {
792        const dir = f.file.slice(0, f.file.lastIndexOf('/'))
793        const name = dir === cwd ? './' : `${dir.startsWith(`${cwd}/`) ? dir.slice(cwd.length + 1) : dir}/`
794        if (!firstFile.has(name) && f.file.startsWith('/')) firstFile.set(name, f.file)
795      }
796      // a package is a helper by its folder's name, read or not; else by its first file read
797      const roled = byPackage.map(p => {
798        const folder = p.name.replace(/\/$/, '').split('/').pop() ?? ''
799        const file = firstFile.get(p.name)
800        const role = isHelperName(folder) ? 'helper' : file ? filesRead.get(file)?.role : undefined
801        return role ? { ...p, role } : p
802      })
803      const statementCount = { total: byFile.reduce((n, f) => n + f.total, 0), covered: byFile.reduce((n, f) => n + f.covered, 0) }
804      // the build tags the last run here was given, named with the figures: a total that counts the
805      // integration tests says so
806      const tagsBy = (await read($, ranRecord))?.tagsBy ?? {}
807      const tags = [...new Set(Object.entries(tagsBy).filter(([dir]) => dir === cwd || dir.startsWith(`${cwd}/`)).flatMap(([, t]) => t))].sort()
808      const source = tags.length > 0 ? `go test -coverprofile -tags=${tags.join(',')}` : 'go test -coverprofile'
809      return { byDir: byDirOf(byFile, cwd), byPackage: roled, statementCount, lines: null, statements: pct(statements), branches: null, functions: null, source, updatedAt: profileAt }
810    }
811  }
812  const goAt = await mtime($, goPath)
813  if (goAt !== null) {
814    const text = await $.fs.read(goPath)
815    const values = [...text.matchAll(/coverage:\s+([0-9.]+)% of statements/g)].map(m => Number(m[1]))
816    if (values.length > 0) {
817      const mean = values.reduce((s, v) => s + v, 0) / values.length
818      return { lines: null, statements: pct(mean), branches: null, functions: null, source: 'go test -cover', updatedAt: goAt }
819    }
820  }
821  return null
822}
823
824// The project's coverage: its own report, or where the project is made of parts each with its own
825// way to measure (backend/ in Go, mobile/ with jest), every part's, its folders by their path in
826// the project
827const readCoverage = async ($: EngineInterface): Promise<Coverage | null> => {
828  const cwd = await projectDir($)
829  const dirs = coverParts.map(p => p.dir)
830  if (dirs.length === 0 || (dirs.length === 1 && dirs[0] === '')) return readCoverageAt($, cwd)
831  const found = (await Promise.all(dirs.map(async dir => ({ dir, cov: await readCoverageAt($, `${cwd}/${dir}`).catch(() => null) })))).flatMap(p => (p.cov ? [{ dir: p.dir, cov: p.cov }] : []))
832  return mergeParts(found)
833}
834
835const refreshCoverage = async ($: EngineInterface): Promise<void> => {
836  try {
837    const next = await readCoverage($)
838    await update($, coverage, () => next)
839  } catch {
840    /* no coverage yet is a normal state */
841  }
842}
843
844// The coverage report as the watch last saw it, by each report's modification time: a report
845// a run outside the pane writes shows in the pane within one watch period
846const GO_PROFILE = '.test-grader-go-cover.out'
847const REPORTS = ['coverage/coverage-summary.json', 'coverage/lcov.info', 'coverage.xml', GO_PROFILE, '.test-grader-go-coverage.txt']
848let reportsAt = ''
849// not while a run of test-grader's own is writing them: Go writes its profile package by package,
850// and a half-written one would be parsed every period and drawn as the figures; the run reads them
851// once it ends
852const refreshCoverageIfChanged = async ($: EngineInterface): Promise<void> => {
853  if (isMeasuring) return
854  const cwd = await projectDir($)
855  const bases = coverParts.length > 0 ? coverParts.map(p => (p.dir ? `${cwd}/${p.dir}` : cwd)) : [cwd]
856  const at = (await Promise.all(bases.flatMap(base => REPORTS.map(r => mtime($, `${base}/${r}`))))).join(',')
857  if (at === reportsAt) return
858  reportsAt = at
859  await refreshCoverage($)
860}
861
862// the project's coverage run: its command, and how a note to Claude names it
863const detectCommand = async ($: EngineInterface, cwd: string): Promise<CoverCommand | undefined> => {
864  const exists = async (name: string): Promise<boolean> => (await mtime($, `${cwd}/${name}`)) !== null
865  if (await exists('package.json')) {
866    const pkg = await $.fs.read(`${cwd}/package.json`)
867    // the project's own coverage script first: it knows how the project measures it
868    const scripts = (() => {
869      try {
870        return (JSON.parse(pkg) as { scripts?: Record<string, unknown> }).scripts ?? {}
871      } catch {
872        return {}
873      }
874    })()
875    if (typeof scripts.coverage === 'string') return { argv: ['npm', 'run', '--silent', 'coverage'], label: 'npm run coverage' }
876    if (/"vitest"/.test(pkg)) return { argv: ['npx', 'vitest', 'run', '--coverage', '--coverage.reporter=json-summary', '--coverage.reporter=lcov'], label: 'npx vitest run --coverage' }
877    if (/"jest"/.test(pkg)) return { argv: ['npx', 'jest', '--coverage', '--coverageReporters=json-summary', '--coverageReporters=lcov'], label: 'npx jest --coverage' }
878  }
879  if ((await exists('pytest.ini')) || (await exists('pyproject.toml')) || (await exists('setup.cfg'))) {
880    return { argv: ['python3', '-m', 'pytest', '--cov', '--cov-report=xml'], label: 'pytest --cov' }
881  }
882  // the profile gives each file's statements, so the total is weighted by package size and
883  // the folders can be told apart; the printed lines are kept for a run that wrote no profile
884  if (await exists('go.mod')) return { argv: ['go', 'test', './...', '-cover', `-coverprofile=${GO_PROFILE}`], label: 'go test ./... -coverprofile', goOutput: '.test-grader-go-coverage.txt' }
885  return undefined
886}
887
888// A coverage run, the pane showing it under way: the project's, or (rel, a folder's path in the
889// project) in a Go project that folder's packages alone, merged into the module's last profile.
890// What it ran and how it ended, or why it could not run
891const NO_COVERAGE = 'No coverage script, jest, vitest, pytest or Go project found here.'
892const measure = async ($: EngineInterface, rel = ''): Promise<{ command: CoverCommand; exitCode: number; output: string } | string> => {
893  const cwd = await projectDir($)
894  const setRun = (state: 'idle' | 'running' | 'failed', message?: string) => update($, run, () => ({ state, message }))
895  isMeasuring = true
896  try {
897    coverParts = await detectParts($, cwd)
898    // a project that lost its way to measure keeps the pane's coverage, which says so
899    if (coverParts.length === 0) return (await setRun('failed', NO_COVERAGE), NO_COVERAGE)
900    await update($, coverWith, () => labelOfParts(coverParts))
901    // the parts the folder is in, or that are in it
902    const chosen = coverParts.filter(p => rel === '' || p.dir === '' || rel === p.dir || rel.startsWith(`${p.dir}/`) || p.dir.startsWith(`${rel}/`))
903    if (chosen.length === 0) return `No part of the project measures ${rel}/: coverage is measured in ${coverParts.map(p => `${p.dir}/`).join(', ')}.`
904    await setRun('running')
905    const runs = await Promise.all(chosen.map(p => measurePart($, cwd, p, rel === p.dir || p.dir.startsWith(`${rel}/`) || rel === '' ? '' : p.dir === '' ? rel : rel.slice(p.dir.length + 1))))
906    await refreshCoverage($)
907    const failed = runs.find(r => r.exitCode !== 0)
908    await setRun(failed ? 'failed' : 'idle', failed ? `Tests exited with ${failed.exitCode}${(await read($, coverage)) ? ': the figures are from the tests that ran' : ''}.` : undefined)
909    if (runs.length === 1) return runs[0]!
910    return {
911      command: { argv: [], label: runs.map(r => r.command.label).join(' · ') },
912      exitCode: failed?.exitCode ?? 0,
913      output: runs.filter(r => r.exitCode !== 0).map(r => r.output).join('\n'),
914    }
915  } catch (err) {
916    const message = err instanceof Error ? err.message : String(err)
917    await setRun('failed', message)
918    return message
919  } finally {
920    isMeasuring = false
921  }
922}
923
924// one part's run, in its folder; sub: a folder in it, which a Go part measures alone, merged into
925// its module's last profile
926const measurePart = async ($: EngineInterface, cwd: string, part: Part, sub: string): Promise<{ command: CoverCommand; exitCode: number; output: string }> => {
927  const base = part.dir ? `${cwd}/${part.dir}` : cwd
928  const where = (label: string): string => (part.dir ? `${label} in ${part.dir}/` : label)
929  const isGoFolder = sub !== '' && part.command.goOutput !== undefined
930  const command: CoverCommand = isGoFolder ? { argv: ['go', 'test', `./${sub}/...`, '-cover', `-coverprofile=${GO_PROFILE}`], label: `go test ./${sub}/... -coverprofile` } : part.command
931  const whole = isGoFolder ? await $.fs.read(`${base}/${GO_PROFILE}`).catch(() => null) : null
932  // which tests the run ran: Go's -v lines, Jest's or Vitest's JSON results in a file of
933  // test-grader's own; a project's own coverage script is run as it is
934  const isGo = command.goOutput !== undefined || isGoFolder
935  const runner = isGo ? 'go' : command.argv[1] === 'jest' ? 'jest' : command.argv[1] === 'vitest' ? 'vitest' : null
936  const ranDir = runner ? await keptDir($, cwd, 'ran').catch(() => null) : null
937  const results = ranDir && runner !== 'go' ? `${ranDir}/${part.dir.replace(/[^A-Za-z0-9._-]+/g, '-') || 'root'}.results.json` : null
938  if (results) await $.fs.write(results, '').catch(() => undefined)
939  const extra = isGo ? ['-v'] : !results ? [] : runner === 'jest' ? ['--json', `--outputFile=${results}`] : ['--reporter=default', '--reporter=json', `--outputFile.json=${results}`]
940  const runEnv = await runEnvIn($, cwd)
941  const result = await $.process.run([...command.argv, ...extra], { cwd: base, timeoutMs: 600_000, ...runEnv })
942  // the run's own lines, without -v's line for each test that ran
943  const stdout = isGo ? result.stdout.split('\n').filter(l => !/^(=== (RUN|PAUSE|CONT|NAME)\s|\s*--- (PASS|SKIP): )/.test(l)).join('\n') : result.stdout
944  await recordRan($, cwd, base, sub, runner, result.stdout, results, [...command.argv, ...(runEnv.env?.GOFLAGS ?? '').split(/\s+/).filter(Boolean)]).catch(error => $.ui.log(`test-grader: the tests the coverage run ran could not be read: ${error instanceof Error ? error.message : String(error)}`, { to: 'debug' }))
945  if (command.goOutput) await $.fs.write(`${base}/${command.goOutput}`, stdout)
946  const part2 = whole === null ? null : await $.fs.read(`${base}/${GO_PROFILE}`).catch(() => null)
947  if (whole !== null && part2 !== null) await $.fs.write(`${base}/${GO_PROFILE}`, mergeProfile(whole, part2, moduleOf(await $.fs.read(`${base}/go.mod`).catch(() => '')), sub))
948  return { command: { ...command, label: where(command.label) }, exitCode: result.exitCode, output: [stdout, result.stderr].join('\n') }
949}
950
951// The tests a coverage run ran, over what the runs before left: kept for the session, and in a
952// file of test-grader's own for the next
953const RAN_FILE = 'ran.json'
954const recordRan = async ($: EngineInterface, cwd: string, base: string, sub: string, runner: 'go' | 'jest' | 'vitest' | null, stdout: string, results: string | null, argv: string[]): Promise<void> => {
955  if (runner === null) return
956  const by = runner === 'go' ? goRanOf(stdout, base, moduleOf(await $.fs.read(`${base}/go.mod`).catch(() => '')) ?? '') : results ? jsRanOf(await $.fs.read(results)) : null
957  if (by === null) return
958  const dir = sub ? `${base}/${sub}` : base
959  const next: RanRecord = { at: await $.clock.now(), measured: [dir], by, ...(runner === 'go' ? { tagsBy: { [dir]: tagsOfArgv(argv) } } : {}) }
960  const merged = mergeRan(await read($, ranRecord), next)
961  await update($, ranRecord, () => merged)
962  const kept = await keptDir($, cwd, 'ran')
963  const text = JSON.stringify(merged)
964  if (kept && text.length <= PART) await $.fs.write(`${kept}/${RAN_FILE}`, text)
965}
966const loadRan = async ($: EngineInterface): Promise<void> => {
967  if ((await read($, ranRecord)) !== null) return
968  const dir = await keptDir($, await projectDir($), 'ran')
969  const text = dir ? await $.fs.read(`${dir}/${RAN_FILE}`).catch(() => null) : null
970  if (text) await update($, ranRecord, () => JSON.parse(text) as RanRecord)
971}
972
973// A run a reload cut off: its state says running, but nothing in this load of the module runs it
974let isMeasuring = false
975const endCutOff = async ($: EngineInterface): Promise<void> => {
976  if (!isMeasuring && (await read($, run)).state === 'running') await update($, run, () => ({ state: 'failed' as const, message: 'The coverage run was cut off by a reload of test-grader: run it again.' }))
977  const runs = await read($, testRuns)
978  const ended = Object.fromEntries(Object.entries(runs).map(([k, r]) => [k, r.state === 'running' && !runningTests.has(k) ? { state: 'failed' as const, tail: 'Cut off by a reload of test-grader: run it again.' } : r]))
979  if (Object.entries(ended).some(([k, r]) => r !== runs[k])) await update($, testRuns, () => ended)
980}
981const runningTests = new Set<string>()
982
983// what the pane last drew, for /test-grader pane-info: a pane the host shows blank can be told from
984// one test-grader never drew
985type Draw = { at: number; ms: number; columns: number; surface: string; nodes: number; chars: number; bytes: number; problem?: string }
986let lastDraw: Draw | null = null
987const paneInfo = async ($: EngineInterface): Promise<string> => {
988  if (!lastDraw) return 'test-grader has not drawn the pane since it last loaded: open it with /test-grader.'
989  const d = lastDraw
990  const ago = Math.max(0, Math.round(((await $.clock.now()) - d.at) / 1000))
991  return (
992    `The pane was last drawn ${ago}s ago, on the ${d.surface} surface at ${d.columns} columns, in ${d.ms}ms: ${d.nodes} elements and texts (the engine takes 20000), ${d.chars} characters of text (it blanks past 100000), ${d.bytes} characters as sent. ` +
993    (d.problem ? `It was not sent: ${d.problem}.` : 'It passed every rule test-grader knows the engine holds it to; a pane still blank after that drawing was refused for a reason test-grader does not check, so please report these figures.')
994  )
995}
996
997// /test-grader reset-view: every row and group closed, the runs shown cleared, a coverage run
998// left running by a reload ended
999const resetView = async ($: EngineInterface): Promise<string> => {
1000  await update($, opened, () => [])
1001  await update($, fileOpen, () => ({}))
1002  await update($, testRuns, () => ({}))
1003  await endCutOff($)
1004  return 'Test pane reset: every row and folder closed, and the test runs it showed cleared.'
1005}
1006
1007const runCoverage = async ($: EngineInterface): Promise<void> => {
1008  const ran = await measure($)
1009  if (typeof ran === 'string') return
1010  const cov = await read($, coverage)
1011  await share($, coverageNote(ran.command, ran.exitCode, ran.output, cov, await viewsOf($, cov)) + (await notRunNote($)))
1012}
1013
1014// Go's packages as the coverage figure should read them, by the listed tests in each package's
1015// folder and how many of them the last run did not build
1016const packageTestsOf = (entries: { file: string; name: string }[], stateOf: (t: { file: string; name: string }) => RanState | undefined, cwd: string): Record<string, PackageTests> => {
1017  const by: Record<string, PackageTests> = {}
1018  for (const t of entries) {
1019    if (!t.file.endsWith('.go')) continue
1020    const dir = t.file.slice(0, t.file.lastIndexOf('/'))
1021    const tally = (by[dir === cwd ? './' : `${shortPath(dir, cwd)}/`] ??= { tests: 0, unbuilt: 0 })
1022    tally.tests++
1023    if (stateOf(t) === 'not built') tally.unbuilt++
1024  }
1025  return by
1026}
1027const viewsOf = async ($: EngineInterface, cov: Coverage | null): Promise<PackageView[]> => {
1028  if (!cov?.byPackage?.length) return []
1029  const cwd = await projectDir($)
1030  const record = await read($, ranRecord)
1031  const entries = entriesOf(await readRun($), (await read($, tests)).filter(t => t.file.startsWith(`${cwd}/`)), await read($, modified))
1032  return packageViews(cov.byPackage, packageTestsOf(entries, t => ranStateOf(record, t.file, t.name, tagsAt(t.file)), cwd))
1033}
1034
1035// what a coverage run tells Claude of the listed tests it reached but did not run: how many, and
1036// the files most of them are in
1037const notRunNote = async ($: EngineInterface): Promise<string> => {
1038  const record = await read($, ranRecord)
1039  if (!record) return ''
1040  const cwd = await projectDir($)
1041  const entries = entriesOf(await readRun($), (await read($, tests)).filter(t => t.file.startsWith(`${cwd}/`)), await read($, modified))
1042  const states = entries.map(t => ({ ...t, state: ranStateOf(record, t.file, t.name, tagsAt(t.file)) }))
1043  const missed = states.filter(t => t.state === 'never ran' || t.state === 'skipped')
1044  const unbuilt = states.filter(t => t.state === 'not built')
1045  const tags = unbuiltTags(unbuilt.map(t => t.file))
1046  const unbuiltNote =
1047    unbuilt.length === 0
1048      ? ''
1049      : `\n${unbuilt.length === 1 ? '1 graded test is' : `${unbuilt.length} graded tests are`} in files with a build tag the run was not given (${tags.join(', ')}), so ${unbuilt.length === 1 ? 'it was' : 'they were'} not compiled: the coverage command leaves ${tags.length === 1 ? 'that tag' : 'those tags'} out, which says nothing of the tests. To measure them too, put GOFLAGS="-tags=${tags.join(',')} -p=1" in ${ENV_FILE}, with what they need to run (-p=1 runs one package at a time, for tests that share one database or emulator); it applies to every run test-grader starts, test_verify's and Run test's too. test_grades with ran: "not built" lists them.`
1050  if (missed.length === 0) return unbuiltNote
1051  const byFile = new Map<string, number>()
1052  for (const t of missed) byFile.set(t.file, (byFile.get(t.file) ?? 0) + 1)
1053  const files = [...byFile.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
1054  const never = missed.filter(t => t.state === 'never ran').length
1055  const counts = [...(never > 0 ? [`${never} never ran`] : []), ...(missed.length - never > 0 ? [`${missed.length - never} skipped`] : [])].join(' and ')
1056  return `\nOf the graded tests this run reached, ${counts}: their grades say nothing of whether they pass. In ${files.slice(0, MAX_NAMED_FILES).map(([f, n]) => `${shortPath(f, cwd)} (${n})`).join(', ')}${files.length > MAX_NAMED_FILES ? ` and ${files.length - MAX_NAMED_FILES} more files` : ''}. test_grades with ran: "never ran" lists them.${unbuiltNote}`
1057}
1058
1059// Claude's coverage tool: the project's run, or a folder's, waited for and answered
1060const answerCoverage = async ($: EngineInterface, input: { path?: unknown }): Promise<string> => {
1061  if ((await read($, run)).state === 'running') return 'A coverage run is already under way; wait for it to finish, then ask again.'
1062  const cwd = await projectDir($)
1063  const given = typeof input.path === 'string' ? input.path.trim().replace(/\/+$/, '').replace(/^\.\//, '') : ''
1064  const abs = given === '' || given === '.' ? cwd : given.startsWith('/') ? given : `${cwd}/${given}`
1065  if (abs !== cwd && !abs.startsWith(`${cwd}/`)) return `${given} is outside the project (${cwd}).`
1066  const rel = abs === cwd ? '' : abs.slice(cwd.length + 1)
1067  const before = await read($, coverage)
1068  const ran = await measure($, rel)
1069  if (typeof ran === 'string') return `Coverage could not be measured: ${ran}`
1070  const after = await read($, coverage)
1071  return coverageAnswer(rel, ran.command, ran.exitCode, ran.output, before, after, await viewsOf($, after)) + (await notRunNote($))
1072}
1073
1074// what a finished Grade all tests run tells Claude: the counts, then every flagged and
1075// unrated test (the strong are counted, not listed)
1076// changed: the files graded again because they changed since their last grading; added: the
1077// files graded for the first time; each by its path in the project
1078type RunFiles = { changed: string[]; added: string[] }
1079const MAX_NAMED_FILES = 20
1080const namedFiles = (label: string, files: string[]): string[] =>
1081  files.length === 0 ? [] : [`${label}: ${files.slice(0, MAX_NAMED_FILES).join(', ')}${files.length > MAX_NAMED_FILES ? ` and ${files.length - MAX_NAMED_FILES} more` : ''}.`]
1082const existingNote = (results: ExistingTest[], cwd: string, scope?: string, files?: RunFiles): string => {
1083  const count = (v: Verdict): number => results.filter(t => t.verdict === v).length
1084  const unrated = results.filter(t => !t.verdict)
1085  const counts = [`${results.length} graded`, `${count('strong')} strong`, ...FLAGGED.filter(v => count(v) > 0).map(v => `${count(v)} ${v}`)]
1086  if (unrated.length > 0) counts.push(`${unrated.length} unrated`)
1087  const lines = [`Test grading (test-grader) finished${scope ? ` for ${scope}` : ''}: ${counts.join(' · ')}.`]
1088  if (files) lines.push(...namedFiles('Changed since their last grading', files.changed), ...namedFiles('Graded for the first time', files.added))
1089  const flagged = flaggedLines(results, cwd)
1090  if (flagged.length > 0) lines.push(`Need work, worst first (${FLAGGED.join(', then ')}):`, ...flagged.slice(0, MAX_NOTED), EVIDENCE_HINT)
1091  if (unrated.length > 0) lines.push('Unrated (the grader gave no verdict):', ...unrated.slice(0, MAX_NOTED).map(t => `- ${shortPath(t.file, cwd)} · ${t.name}`))
1092  // a big run's lists cut short: the rest counted by file, the files with the most first
1093  if (flagged.length > MAX_NOTED || unrated.length > MAX_NOTED) {
1094    const left = [...results.filter(t => isFlagged(t.verdict)).slice(MAX_NOTED), ...unrated.slice(MAX_NOTED)]
1095    const byFile = new Map<string, Map<string, number>>()
1096    for (const t of left) {
1097      const tally = byFile.get(t.file) ?? new Map<string, number>()
1098      const state = t.verdict ?? 'unrated'
1099      tally.set(state, (tally.get(state) ?? 0) + 1)
1100      byFile.set(t.file, tally)
1101    }
1102    const files = [...byFile].map(([file, tally]) => ({ file, tally, n: [...tally.values()].reduce((a, b) => a + b, 0) })).sort((a, b) => b.n - a.n || a.file.localeCompare(b.file))
1103    lines.push(
1104      `${left.length} more not listed, by file: ${files.slice(0, MAX_NAMED_FILES).map(f => `${shortPath(f.file, cwd)} (${[...f.tally].map(([s, n]) => `${n} ${s}`).join(', ')})`).join('; ')}${files.length > MAX_NAMED_FILES ? `; and ${files.length - MAX_NAMED_FILES} more files` : ''}.`,
1105      'test_grades with path lists a file\'s or a folder\'s in full.',
1106    )
1107  }
1108  return lines.join('\n')
1109}
1110// how many flagged, and how many unrated, tests a note lists by name: a big run's lists
1111// would fill Claude's context, so the rest are counted by file
1112const MAX_NOTED = 40
1113
1114// Grade all tests: every case of every test file git tracks, BATCH cases a call and
1115// parallel calls at once; the results keep file order. A batch the grader fails leaves
1116// its cases unrated, and the run goes on
1117// a file's contents, fingerprinted (FNV-1a), with its length
1118export const fingerprint = (text: string): string => {
1119  let hash = 0x811c9dc5
1120  for (let i = 0; i < text.length; i++) hash = Math.imul(hash ^ text.charCodeAt(i), 0x01000193)
1121  return `${(hash >>> 0).toString(16)}-${text.length}`
1122}
1123
1124// a test's own text, fingerprinted: what a verdict given on evidence was given for
1125const ownText = (text: string, name: string, file: string): string => fingerprint(caseTextOf(text, name, file) ?? '')
1126// the same for many tests of one file, its cases found once
1127// (the last file's kept: a run's batches of one file ask for it in turn)
1128let ownLast: { text: string; file: string; of: (name: string) => string } | null = null
1129const ownTexts = (text: string, file: string): ((name: string) => string) => {
1130  if (ownLast?.text === text && ownLast.file === file) return ownLast.of
1131  const cases = caseTextsOf(text, file)
1132  const of = (name: string): string => fingerprint(cases(name) ?? '')
1133  ownLast = { text, file, of }
1134  return of
1135}
1136// a verdict given on evidence holds, and is not graded again, while the test's own text is as it was
1137const isHeld = (t: { name: string; evidence?: string; evidenceOf?: string }, text: string, file: string): boolean =>
1138  Boolean(t.evidence && t.evidenceOf && t.evidenceOf === ownText(text, t.name, file))
1139
1140// The store holds 4 MiB for every project together: a project's grades past this go to files
1141// of their own, under the Claude configuration folder, in parts a file holds (a file is read
1142// and written to 4 MiB, and a character is up to 3 bytes)
1143const STORE_ROOM = 1_000_000
1144const PART = 1_000_000
1145type GradesOnDisk = { v: 2; onDisk: string; parts: number }
1146const gradesDir = ($: EngineInterface, cwd: string): Promise<string | null> => keptDir($, cwd, 'grades')
1147// where test-grader keeps a project's files of one kind, outside the project
1148const keptDir = async ($: EngineInterface, cwd: string, kind: 'grades' | 'ran'): Promise<string | null> => {
1149  const home = await $.env.get('HOME')
1150  const config = (await $.env.get('CLAUDE_CONFIG_DIR')) || (home ? `${home}/.claude` : '')
1151  return config ? `${config}/test-grader/${kind}/${cwd.replace(/[^A-Za-z0-9._-]+/g, '-')}` : null
1152}
1153// cut where no character's two halves are parted
1154const partsOf = (text: string): string[] => {
1155  const parts: string[] = []
1156  for (let at = 0; at < text.length; ) {
1157    let end = Math.min(at + PART, text.length)
1158    const last = text.charCodeAt(end - 1)
1159    if (end < text.length && last >= 0xd800 && last <= 0xdbff) end -= 1
1160    parts.push(text.slice(at, end))
1161    at = end
1162  }
1163  return parts
1164}
1165// the grades a pointer names, or none where a part is missing or they do not read whole
1166const readOnDisk = async ($: EngineInterface, pointer: GradesOnDisk): Promise<KeptGrades | undefined> => {
1167  try {
1168    const parts = await Promise.all(Array.from({ length: pointer.parts }, (_, i) => $.fs.read(`${pointer.onDisk}/${i}.part`)))
1169    return JSON.parse(parts.join('')) as KeptGrades
1170  } catch (error) {
1171    $.ui.log(`test-grader: the grades in ${pointer.onDisk} could not be read: ${error instanceof Error ? error.message : String(error)}`, { to: 'debug' })
1172    return undefined
1173  }
1174}
1175const saveGrades = async ($: EngineInterface): Promise<void> => {
1176  const cwd = await projectDir($)
1177  if (!cwd) return
1178  const run = await readRun($)
1179  const saved: SavedGrades = {
1180    results: run.results.filter(t => !t.isUngraded).map(({ isPending: _, ...t }) => t),
1181    hashes: run.hashes ?? {},
1182    ...(run.finishedAt === undefined ? {} : { finishedAt: run.finishedAt }),
1183  }
1184  const why = (error: unknown): string => (error instanceof Error ? error.message : String(error))
1185  // many: in files of their own, the store naming them
1186  const text = JSON.stringify(keep(saved, false))
1187  const dir = text.length > STORE_ROOM ? await gradesDir($, cwd) : null
1188  if (dir !== null) {
1189    try {
1190      const parts = partsOf(text)
1191      for (const [i, part] of parts.entries()) await $.fs.write(`${dir}/${i}.part`, part)
1192      const pointer: GradesOnDisk = { v: 2, onDisk: dir, parts: parts.length }
1193      await $.store.set(gradesKey(cwd), pointer)
1194      // the parts of a larger save before, past the new end: emptied, as the engine deletes no file
1195      const stale = (await $.fs.list(dir).catch(() => [])).filter(e => e.kind === 'file' && /^\d+\.part$/.test(e.name) && Number.parseInt(e.name, 10) >= parts.length && (e.size ?? 1) > 0)
1196      for (const e of stale) await $.fs.write(`${dir}/${e.name}`, '').catch(() => undefined)
1197      await update($, saveError, () => null)
1198      return
1199    } catch (error) {
1200      $.ui.log(`test-grader: the grades could not be written to ${dir}: ${why(error)}`, { to: 'debug' })

1// Coverage figures worked out from a report: by folder, the least covered, and the note a run
2// sends Claude. Pure: reading the report stays in register.tsx
3import type { Coverage, CoveragePart } from '../types'
4
5import { shortPath } from './discovery'
6
7export const pct = (v: unknown): number | null => (typeof v === 'number' && Number.isFinite(v) ? Math.round(v * 10) / 10 : null)
8
9export const attr = (xml: string, name: string): number | null => {
10  const m = xml.match(new RegExp(`${name}="([0-9.]+)"`))
11  return m ? pct(Number(m[1]) * 100) : null
12}
13
14// Line coverage by folder, every folder holding the lines of all beneath it: by its path in
15// the project ('' the project itself)
16export const byDirOf = (files: { file: string; total: number; covered: number }[], cwd: string): Record<string, { total: number; covered: number }> => {
17  const dirs: Record<string, { total: number; covered: number }> = {}
18  for (const f of files) {
19    if (f.total <= 0) continue
20    const rel = shortPath(f.file, cwd)
21    const parts = rel.split('/').slice(0, -1)
22    for (let i = 0; i <= parts.length; i++) {
23      const dir = parts.slice(0, i).join('/')
24      const d = (dirs[dir] ??= { total: 0, covered: 0 })
25      d.total += f.total
26      d.covered += f.covered
27    }
28  }
29  return dirs
30}
31
32// The parts' reports as one: each part's figures kept apart (Go's statements and jest's lines do
33// not add up), its folders and packages by their path in the project. A project of one part is
34// that part's report, its paths under the part's folder
35export const mergeParts = (found: { dir: string; cov: Coverage }[]): Coverage | null => {
36  if (found.length === 0) return null
37  const inPart = (dir: string, path: string): string => (path === '' ? dir : `${dir}/${path}`)
38  const byDir: Record<string, { total: number; covered: number }> = {}
39  for (const { dir, cov } of found) for (const [path, d] of Object.entries(cov.byDir ?? {})) byDir[inPart(dir, path)] = d
40  const byPackage = found.flatMap(({ dir, cov }) => (cov.byPackage ?? []).map(p => ({ ...p, name: p.name === './' ? `${dir}/` : `${dir}/${p.name}` })))
41  const parts: CoveragePart[] = found.map(({ dir, cov }) => ({ dir, lines: cov.lines, statements: cov.statements, branches: cov.branches, functions: cov.functions, source: cov.source }))
42  const only = found.length === 1 ? found[0]!.cov : null
43  // statements add up across parts, Go's and jest's alike: the project's figure where every part
44  // counted them
45  const counts = found.map(f => f.cov.statementCount)
46  const statementCount = counts.every(c => c !== undefined) ? counts.reduce((a, c) => ({ total: a.total + c!.total, covered: a.covered + c!.covered }), { total: 0, covered: 0 }) : undefined
47  return {
48    lines: only?.lines ?? null,
49    statements: only?.statements ?? (statementCount && statementCount.total > 0 ? pct((statementCount.covered / statementCount.total) * 100) : null),
50    branches: only?.branches ?? null,
51    functions: only?.functions ?? null,
52    source: parts.map(p => `${p.dir}/: ${p.source}`).join(' · '),
53    updatedAt: Math.max(...found.map(f => f.cov.updatedAt ?? 0)) || null,
54    byDir,
55    ...(byPackage.length > 0 ? { byPackage } : {}),
56    ...(statementCount ? { statementCount } : {}),
57    parts,
58  }
59}
60
61// what a report measures: Go's statements, else lines; a project of parts names each kind it has
62const kindOfFigures = (c: { lines: number | null; statements: number | null }): string => (c.lines === null && c.statements !== null ? 'statements' : 'lines')
63export const kindOf = (cov: Coverage | null): string => (cov?.parts && cov.parts.length > 1 ? [...new Set(cov.parts.map(kindOfFigures))].join(' or ') : cov ? kindOfFigures(cov) : 'lines')
64// the kind of the part a folder is in, by its path in the project
65export const kindAt = (cov: Coverage | null, path: string): string => {
66  const part = cov?.parts?.find(p => path === p.dir || path.startsWith(`${p.dir}/`))
67  return part ? kindOfFigures(part) : kindOf(cov)
68}
69// a report's figures as a note says them
70const figuresOf = (c: { lines: number | null; statements: number | null; branches: number | null; functions: number | null }): string =>
71  ([['lines', c.lines], ['statements', c.statements], ['branches', c.branches], ['functions', c.functions]] as const)
72    .filter(([, v]) => v !== null)
73    .map(([name, v]) => `${name} ${v}%`)
74    .join(' · ')
75
76// Go's packages as the figure should read them: tested (tests of it were built and run, some of
77// its tests maybe not built), not built (every test of it has a build tag the run was not given:
78// unmeasured, not low), no tests, a command with no tests, or a test helper. tests: by package
79// name, how many listed tests it has and how many of them were not built
80export type PackageTests = { tests: number; unbuilt: number }
81export type PackageState = 'tested' | 'not built' | 'no tests' | 'command' | 'helper'
82export type PackageView = { name: string; total: number; covered: number; pct: number; state: PackageState; unbuilt: number }
83// where no package has a listed test the list is not known yet, and a package is taken as tested
84// unless its role says otherwise
85export const packageViews = (packages: { name: string; total: number; covered: number; role?: 'command' | 'helper' }[], tests: Record<string, PackageTests>): PackageView[] => {
86  const isListed = packages.some(p => (tests[p.name]?.tests ?? 0) > 0)
87  return packages
88    .filter(p => p.total > 0)
89    .map(p => {
90      const t = tests[p.name] ?? { tests: 0, unbuilt: 0 }
91      const state: PackageState =
92        p.role === 'helper' ? 'helper' : t.tests > t.unbuilt ? 'tested' : t.unbuilt > 0 ? 'not built' : p.role === 'command' ? 'command' : isListed ? 'no tests' : 'tested'
93      return { name: p.name, total: p.total, covered: p.covered, pct: pct((p.covered / p.total) * 100)!, state, unbuilt: t.unbuilt }
94    })
95}
96
97// what a statements total over every package hides: the figure over the tested packages, their
98// median, and what the total counts that no test is meant for or no test was built for; null
99// where every package is tested
100const plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? '' : 's'}`
101export const testedLine = (views: PackageView[]): string | null => {
102  const tested = views.filter(v => v.state === 'tested')
103  if (tested.length === views.length || tested.length === 0) return null
104  const total = tested.reduce((s, v) => s + v.total, 0)
105  const covered = tested.reduce((s, v) => s + v.covered, 0)
106  const sorted = tested.map(v => v.pct).sort((a, b) => a - b)
107  const mid = Math.floor(sorted.length / 2)
108  const median = sorted.length % 2 === 1 ? sorted[mid]! : pct((sorted[mid - 1]! + sorted[mid]!) / 2)!
109  const count = (state: PackageState): number => views.filter(v => v.state === state).length
110  const unbuilt = views.filter(v => v.state === 'not built')
111  const left = [
112    ...(count('command') > 0 ? [`${plural(count('command'), 'command')} (package main) with no tests`] : []),
113    ...(count('no tests') > 0 ? [`${plural(count('no tests'), 'other package')} with no tests`] : []),
114    ...(count('helper') > 0 ? [`${plural(count('helper'), 'test helper')}`] : []),
115    ...(unbuilt.length > 0 ? (n => [`${plural(unbuilt.length, 'package')} whose ${plural(n, 'test')} ${n === 1 ? 'was' : 'were'} not built (unmeasured, not low)`])(unbuilt.reduce((s, v) => s + v.unbuilt, 0)) : []),
116  ]
117  return `${pct((covered / total) * 100)}% over the ${plural(tested.length, 'package')} whose tests ran (median package ${median}%); the total also counts ${left.join(', ')}.`
118}
119
120// the least covered folders, a few lines each at least, lowest first: where more tests would pay;
121// a folder of Go packages none of which is tested is left out: more tests are not what it lacks
122const LEAST_COVERED = 5
123const MIN_LINES = 20
124// whether a folder holds packages and none of them tested
125export const isUntestedDir = (views: PackageView[], dir: string): boolean => {
126  const under = views.filter(v => v.name === `${dir}/` || v.name.startsWith(`${dir}/`))
127  return under.length > 0 && under.every(v => v.state !== 'tested')
128}
129// a folder's figure less its packages that are not tested, which would rank it low for code no
130// test is meant for
131const testedOf = (views: PackageView[], dir: string, d: { total: number; covered: number }): { total: number; covered: number } =>
132  views
133    .filter(v => v.state !== 'tested' && (v.name === `${dir}/` || v.name.startsWith(`${dir}/`)))
134    .reduce((t, v) => ({ total: t.total - v.total, covered: t.covered - v.covered }), d)
135const leastCovered = (cov: Coverage | null, views: PackageView[]): string[] =>
136  Object.entries(cov?.byDir ?? {})
137    .map(([dir, d]) => [dir, testedOf(views, dir, d)] as const)
138    .filter(([dir, d]) => dir !== '' && d.total >= MIN_LINES && !isUntestedDir(views, dir))
139    .map(([dir, d]) => ({ dir, p: (d.covered / d.total) * 100 }))
140    .filter(d => d.p < 80)
141    .sort((a, b) => a.p - b.p)
142    .slice(0, LEAST_COVERED)
143    .map(d => `${d.dir}/ ${Math.round(d.p)}%`)
144
145// a coverage run: its command, how a note to Claude names it, and where Go prints its figures
146export type CoverCommand = { argv: string[]; label: string; goOutput?: string }
147
148// what a finished coverage run tells Claude: the figures it left, or how it failed and
149// the end of what it printed
150const COVER_TAIL = 20
151// views: Go's packages, as packageViews reads them
152// a Go total's tested line, under the part it is of where there are parts
153const testedNote = (cov: Coverage | null, views: PackageView[]): string => {
154  const parts = cov?.parts && cov.parts.length > 1 ? cov.parts : null
155  const lines = parts
156    ? parts.flatMap(p => {
157        const line = testedLine(views.filter(v => v.name.startsWith(`${p.dir}/`)))
158        return line ? [`${p.dir}/: ${line}`] : []
159      })
160    : [testedLine(views)].filter((l): l is string => l !== null)
161  return lines.map(l => `\n${l}`).join('')
162}
163export const coverageNote = (command: CoverCommand, exitCode: number, output: string, cov: Coverage | null, views: PackageView[] = []): string => {
164  const figures = !cov
165    ? ''
166    : cov.parts && cov.parts.length > 1
167      ? (cov.statements !== null ? `the whole project ${cov.statements}% of statements (its parts' added up); ` : '') + cov.parts.map(p => `${p.dir}/ ${figuresOf(p)} (${p.source})`).join('; ')
168      : figuresOf(cov) && `${figuresOf(cov)} (${cov.source})`
169  if (exitCode === 0) {
170    const least = leastCovered(cov, views)
171    return figures
172      ? `Coverage run (test-grader) finished: ${figures}.${testedNote(cov, views)}${least.length > 0 ? `\nLeast covered folders (${kindOf(cov)}): ${least.join(', ')}.` : ''}`
173      : `Coverage run (test-grader) finished, but ${command.label} wrote no report test-grader reads.`
174  }
175  const lines = output.split('\n').filter(l => l.trim() !== '').slice(-COVER_TAIL)
176  // tests failed, but the run left figures: they are given, with what failed (Go's FAIL lines)
177  if (figures) {
178    const failed = [...new Set(output.split('\n').flatMap(l => l.match(/^FAIL\s+(\S+)/)?.[1] ?? []).filter(p => p !== 'FAIL'))]
179    return [
180      `Coverage run (test-grader) finished with failing tests (${command.label} exited with ${exitCode}): ${figures}. The figures are from the tests that ran.`,
181      ...(failed.length > 0 ? [`Failed: ${failed.slice(0, 10).join(', ')}${failed.length > 10 ? ` and ${failed.length - 10} more` : ''}.`] : [`The last ${lines.length} lines it printed:`, ...lines]),
182    ].join('\n')
183  }
184  return [`Coverage run (test-grader) failed: ${command.label} exited with ${exitCode}. The last ${lines.length} lines it printed:`, ...lines].join('\n')
185}
186
187// what Claude's coverage tool answers: a folder's figure (the project's, rel '') now and before
188// the run, the project's beside it, and the least covered folders under it, lowest first
189const UNDER = 8
190export const coverageAnswer = (rel: string, command: CoverCommand, exitCode: number, output: string, before: Coverage | null, after: Coverage | null, views: PackageView[] = []): string => {
191  const of = (cov: Coverage | null, dir: string): number | null => {
192    const d = cov?.byDir?.[dir]
193    return d && d.total > 0 ? pct((d.covered / d.total) * 100) : null
194  }
195  const figure = (where: string, dir: string): string => {
196    const now = of(after, dir)
197    const was = of(before, dir)
198    if (now === null) return `${where} has no figure in the report: none of its code was measured.`
199    return `${where}: ${now}% ${kindAt(after, dir)}${was === null ? '' : was === now ? ', unchanged' : `, was ${was}%`}.`
200  }
201  const where = rel === '' ? 'The project' : `${rel}/`
202  const lines = [`Coverage (test-grader), by ${command.label}:`]
203  if (exitCode !== 0) lines.push(`It exited with ${exitCode}: the figures are from the tests that ran.`)
204  // a project of parts has no one figure: each part's, or the part the folder is in
205  const parts = after?.parts && after.parts.length > 1 ? after.parts : null
206  if (!(parts && rel === '')) lines.push(figure(where, rel))
207  else if (after?.statements != null) lines.push(`The whole project: ${after.statements}% of statements, its parts' added up${before?.statements != null && before.statements !== after.statements ? `, was ${before.statements}%` : ''}.`)
208  if (parts) lines.push(...parts.filter(p => rel === '' || rel.startsWith(`${p.dir}/`)).map(p => figure(`${p.dir}/`, p.dir)))
209  else if (rel !== '') lines.push(figure('The project', ''))
210  if (rel === '') lines.push(...testedNote(after, views).split('\n').filter(Boolean))
211  const under = Object.entries(after?.byDir ?? {})
212    .map(([dir, d]) => [dir, testedOf(views, dir, d)] as const)
213    .filter(([dir, d]) => d.total > 0 && dir !== rel && (rel === '' ? dir !== '' : dir.startsWith(`${rel}/`)) && !isUntestedDir(views, dir))
214    .map(([dir, d]) => ({ dir, p: (d.covered / d.total) * 100, d }))
215    .sort((a, b) => a.p - b.p || a.dir.localeCompare(b.dir))
216    .slice(0, UNDER)
217  if (under.length > 0) lines.push(`Least covered folders${rel === '' ? '' : ` in ${rel}/`} (${rel === '' ? kindOf(after) : kindAt(after, rel)}): ${under.map(u => `${u.dir}/ ${Math.round(u.p)}% (${u.d.covered} of ${u.d.total})`).join(', ')}.`)
218  if (!after) lines.splice(1, lines.length - 1, `${command.label} wrote no report test-grader reads.`)
219  if (exitCode !== 0) {
220    const tail = output.split('\n').filter(l => l.trim() !== '').slice(-COVER_TAIL)
221    lines.push(`The last ${tail.length} lines it printed:`, ...tail)
222  }
223  return lines.join('\n')
224}
225

1// Test discovery: which files hold tests, and which cases each declares, read as code
2
3// the files that hold tests, by name: JS and TS (*.test.*, *.spec.*, __tests__/, test/,
4// tests/), Go, Python, Ruby (minitest and RSpec), Swift, Kotlin and Java, C#, PHP, Rust
5export const TEST_FILE = new RegExp(
6  [
7    /(\.|_)(test|spec)\.[cm]?[jt]sx?$/,
8    /(^|\/)(__tests__|tests?)\/[^/]+\.[cm]?[jt]sx?$/,
9    /_test\.(go|py|rb)$/,
10    /(^|\/)test_[^/]*\.(py|rb)$/,
11    /_spec\.rb$/,
12    /(Tests?|Spec|IT)\.(swift|kt|java)$/,
13    /(^|\/)src\/test\/.+\.(kt|java)$/,
14    /Tests?\.(cs|php)$/,
15    /(^|\/)tests\/.+\.rs$/,
16    /(^|\/|_)tests?\.rs$/,
17  ]
18    .map(r => r.source)
19    .join('|'),
20)
21
22// the language a test file is written in: what its cases and groups look like
23export type Kind = 'js' | 'go' | 'py' | 'rb' | 'swift' | 'jvm' | 'cs' | 'php' | 'rs'
24export const kindOf = (file: string): Kind =>
25  /\.[cm]?[jt]sx?$/.test(file)
26    ? 'js'
27    : file.endsWith('.go')
28      ? 'go'
29      : file.endsWith('.py')
30        ? 'py'
31        : file.endsWith('.rb')
32          ? 'rb'
33          : file.endsWith('.swift')
34            ? 'swift'
35            : /\.(kt|java)$/.test(file)
36              ? 'jvm'
37              : file.endsWith('.cs')
38                ? 'cs'
39                : file.endsWith('.php')
40                  ? 'php'
41                  : 'rs'
42
43// A case pattern, and where its match must stand in code for it to count: at its keyword
44// (open; a JS case's name is a string), or at the name it captures (a PHP @test docblock is a
45// comment, the method under it code)
46type Pattern = { re: RegExp; at: 'open' | 'name' }
47const open = (re: RegExp): Pattern => ({ re, at: 'open' })
48// a call's arguments, one level of parentheses deep inside: it.each([f(1), 2])
49const ARGS = String.raw`\((?:[^()]|\((?:[^()]|\([^()]*\))*\))*\)`
50// a JS case opens its own line, so one quoted inside a fixture string is not one; its
51// name runs to the closing quote, past any escaped one
52const QUOTED = String.raw`(['"\x60])((?:\\.|(?!\1)[^\\\n])+)\1`
53const JS_CASE = new RegExp(String.raw`^[ \t]*(?:it|test|Deno\.test)(?:\.(?:only|skip|concurrent|sequential|todo|fails|failing|each(?:${ARGS}|\x60[^\x60]*\x60)))*\s*\(\s*${QUOTED}`, 'gm')
54const ATTRS = String.raw`(?:\s*(?:@\w+(?:${ARGS})?|\[[^\]\n]*\]|#\[[^\]\n]*\]))*`
55export const CASE_PATTERNS: Record<Kind, Pattern[]> = {
56  js: [open(JS_CASE)],
57  go: [
58    // TestMain(m *testing.M) sets the package's tests up: it is not one
59    open(/\bfunc\s+(Test(?!Main\b)\w+)\s*\(/g),
60    // a Go suite's test: a Test method of the suite type (testify)
61    open(/\bfunc\s+\(\s*\w+\s+\*?(\w+)\s*\)\s+(Test\w+)\s*\(/g),
62  ],
63  py: [open(/^[ \t]*(?:async\s+)?def\s+(test_\w+)/gm)],
64  rb: [
65    open(/^[ \t]*def\s+(test_\w+)/gm),
66    // RSpec's it, specify, example, scenario; Rails' and minitest/spec's test "…" do
67    open(new RegExp(String.raw`^[ \t]*(?:it|specify|example|scenario|test)\s*\(?\s*(['"])((?:\\.|(?!\1)[^\\\n])+)\1`, 'gm')),
68  ],
69  swift: [open(/\bfunc\s+(test\w+)\s*\(/g), open(new RegExp(String.raw`@Test\b(?:${ARGS})?${ATTRS}\s*(?:(?:public|private|internal|static|mutating)\s+)*func\s+(\w+)\s*\(`, 'g'))],
70  jvm: [
71    open(
72      new RegExp(
73        String.raw`@(?:Test|ParameterizedTest|RepeatedTest|TestFactory|TestTemplate)\b(?:${ARGS})?${ATTRS}\s*(?:(?:public|protected|private|internal|open|override|suspend|static|final)\s+)*(?:void\s+|fun\s+)(\x60[^\x60\n]+\x60|\w+)\s*\(`,
74        'g',
75      ),
76    ),
77  ],
78  cs: [
79    open(
80      new RegExp(
81        String.raw`\[\s*(?:Fact|Theory|Test|TestMethod|DataTestMethod|TestCase|TestCaseSource)\b[^\]\n]*\]${ATTRS}\s*(?:(?:public|private|internal|protected|static|async|virtual|override)\s+)*(?:async\s+)?(?:Task|ValueTask|void)\s+(\w+)\s*\(`,
82        'g',
83      ),
84    ),
85  ],
86  php: [
87    open(/^\s*(?:(?:public|protected|private|static|final)\s+)*function\s+(test\w+)\s*\(/gm),
88    open(new RegExp(String.raw`#\[Test\]${ATTRS}\s*(?:(?:public|protected|private|static|final)\s+)*function\s+(\w+)\s*\(`, 'g')),
89    { re: /@test\b[^\n]*\n(?:[^\n]*\n)*?\s*(?:(?:public|protected|private|static|final)\s+)*function\s+(\w+)\s*\(/g, at: 'name' },
90    // Pest: it('…') and test('…'), as in JS
91    open(JS_CASE),
92  ],
93  rs: [
94    open(
95      /^\s*#\[(?:\w+::)*(?:test|rstest|test_case|quickcheck)\b[^\]\n]*\]\s*\n(?:\s*#\[[^\n]*\]\s*\n)*\s*(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?(?:unsafe\s+)?fn\s+(\w+)/gm,
96    ),
97  ],
98}
99
100// what groups a language's cases: describe blocks, classes, modules, suites
101const GROUP_PATTERNS: Record<Kind, RegExp> = {
102  js: new RegExp(String.raw`^[ \t]*(?:describe|context|suite|test\.describe)(?:\.(?:only|skip|serial|parallel|concurrent|each(?:${ARGS}|\x60[^\x60]*\x60)))*\s*\(\s*${QUOTED}`, 'gm'),
103  go: /(?!)/g,
104  py: /^[ \t]*class\s+(\w+)/gm,
105  rb: new RegExp(String.raw`^[ \t]*(?:(?:RSpec\.)?(?:describe|context|feature)\s*\(?\s*(?:(['"])((?:\\.|(?!\1)[^\\\n])+)\1|([A-Z][\w:]*))|class\s+(\w+))`, 'gm'),
106  swift: /^[ \t]*(?:@Suite\b[^\n]*\n\s*)?(?:(?:final|public|private|internal)\s+)*(?:class|struct|extension)\s+(\w+)/gm,
107  jvm: /^[ \t]*(?:@Nested\s+)?(?:(?:public|private|protected|internal|open|abstract|final|static|inner)\s+)*class\s+(\w+)/gm,
108  cs: /^[ \t]*(?:(?:public|private|protected|internal|static|sealed|abstract|partial)\s+)*class\s+(\w+)/gm,
109  php: /^[ \t]*(?:(?:final|abstract)\s+)*class\s+(\w+)/gm,
110  rs: /^[ \t]*(?:pub(?:\([^)]*\))?\s+)?mod\s+(\w+)/gm,
111}
112
113// a Go suite's test, its suite the receiver's type; and a Go test that only runs a suite
114export const GO_SUITE_CASE = /\bfunc\s+\(\s*\w+\s+\*?(\w+)\s*\)\s+(Test\w+)\s*\(/g
115export const GO_SUITE_RUNNER = /\bfunc\s+(Test\w+)\s*\(\s*\w+\s+\*testing\.T\s*\)\s*\{\s*suite\.Run\([^)]*\)\)?\s*\}/g
116
117// a match's name: a JS or Ruby case's quoted text, else the identifier; a Kotlin `name in
118// backticks` without them
119export const nameOf = (m: RegExpMatchArray): string => {
120  const raw = m[2] ?? m[3] ?? m[4] ?? (m[1] as string)
121  return raw.startsWith('`') && raw.endsWith('`') ? raw.slice(1, -1) : raw.replace(/\\(.)/g, '$1')
122}
123
124// the families of syntax a test file's strings and comments follow: JS and TS; Go; Python
125// and Ruby; PHP; and the C-like rest (Rust, Swift, Kotlin, Java, C#)
126export type Lang = 'js' | 'go' | 'py' | 'php' | 'c'
127export const langOf = (file: string): Lang => {
128  const kind = kindOf(file)
129  return kind === 'js' || kind === 'go' || kind === 'py' || kind === 'php' ? kind : kind === 'rb' ? 'py' : 'c'
130}
131
132// Which characters of a source sit inside a string literal or a comment (1) rather than in
133// code (0): a test written out as text, a fixture, is not one of the file's tests
134export const quotedMask = (text: string, lang: Lang): Uint8Array => {
135  const n = text.length
136  const mask = new Uint8Array(n)
137  const fill = (from: number, to: number): number => (mask.fill(1, from, to), to)
138  // past a string's opening quote at `from`: where it ends, past its closing quote; one that
139  // may not span lines ends at its line's end
140  const close = (from: number, quote: string, { escapes = true, lines = false } = {}): number => {
141    for (let j = from; j < n; j++) {
142      if (escapes && text[j] === '\\') j++
143      else if (text.startsWith(quote, j)) return j + quote.length
144      else if (text[j] === '\n' && !lines) return j
145    }
146    return n
147  }
148  // JS: the ${…} holes open in templates, innermost last, each with the braces opened in it
149  const holes: number[] = []
150  // a JS template's text from `from` to its close or its next hole
151  const template = (from: number, scan: number): number => {
152    for (let j = scan; j < n; j++) {
153      if (text[j] === '\\') j++
154      else if (text[j] === '`') return fill(from, j + 1)
155      else if (text[j] === '$' && text[j + 1] === '{') return holes.push(0), fill(from, j + 2)
156    }
157    return fill(from, n)
158  }
159  // a JS slash opens a regex where a value is due, not after one
160  const isRegexAt = (at: number): boolean => {
161    let k = at - 1
162    while (k >= 0 && /\s/.test(text[k]!)) k--
163    if (k < 0 || '(,=:[!&|?{};+-*%~^'.includes(text[k]!)) return true
164    return /\b(?:return|typeof|case|in|of|delete|void|throw|new|else|do|yield|await)$/.test(text.slice(Math.max(0, k - 9), k + 1))
165  }
166  const regexEnd = (at: number): number => {
167    let isClass = false
168    for (let j = at + 1; j < n; j++) {
169      const t = text[j]
170      if (t === '\\') j++
171      else if (t === '\n') return j
172      else if (isClass) isClass = t !== ']'
173      else if (t === '[') isClass = true
174      else if (t === '/') return j + 1
175    }
176    return n
177  }
178  const RUST_RAW = /r(#*)"/y
179  let i = 0
180  while (i < n) {
181    const c = text[i]!
182    const d = text[i + 1]
183    if (lang === 'py' ? c === '#' : (c === '/' && d === '/') || (lang === 'php' && c === '#' && d !== '[')) {
184      const end = text.indexOf('\n', i)
185      i = fill(i, end < 0 ? n : end)
186    } else if (lang !== 'py' && c === '/' && d === '*') {
187      const end = text.indexOf('*/', i + 2)
188      i = fill(i, end < 0 ? n : end + 2)
189    } else if (lang === 'js' && c === '`') i = template(i, i + 1)
190    else if (lang === 'js' && holes.length > 0 && (c === '{' || c === '}')) {
191      const top = holes.length - 1
192      if (c === '{') holes[top]! += 1
193      else if (holes[top]! > 0) holes[top]! -= 1
194      else {
195        holes.pop()
196        i = template(i, i + 1)
197        continue
198      }
199      i++
200    } else if (lang === 'js' && c === '/' && isRegexAt(i)) i = fill(i, regexEnd(i))
201    else if (lang === 'go' && c === '`') i = fill(i, close(i + 1, '`', { escapes: false, lines: true }))
202    else if (lang !== 'js' && lang !== 'go' && (text.startsWith('"""', i) || (lang === 'py' && text.startsWith("'''", i)))) {
203      i = fill(i, close(i + 3, text.slice(i, i + 3), { lines: true }))
204    } else if (lang === 'c' && c === 'r' && !/\w/.test(text[i - 1] ?? '') && ((RUST_RAW.lastIndex = i), RUST_RAW.test(text))) {
205      i = fill(i, close(RUST_RAW.lastIndex, `"${text.slice(i + 1, RUST_RAW.lastIndex - 1)}`, { escapes: false, lines: true }))
206    } else if (c === '"' || (c === "'" && lang !== 'c')) i = fill(i, close(i + 1, c))
207    // C-like: a quote opens a char literal ('a', '\n'), not a Rust lifetime ('a)
208    else if (c === "'" && d === '\\') i = fill(i, close(i + 1, "'"))
209    else if (c === "'" && text[i + 2] === "'") i = fill(i, i + 3)
210    else i++
211  }
212  return mask
213}
214
215// where a match's own keyword stands, past the indent a line-anchored pattern takes in
216export const opensOf = (m: RegExpMatchArray): number => (m.index ?? 0) + m[0].length - m[0].trimStart().length
217
218// Where a block that opens at `from` ends: past the brace that closes the first brace opened
219// after it, in code (JS, Go, Swift, Kotlin, Java, C#, PHP, Rust); in Python and Ruby, before
220// the next line in code indented no deeper than the block's first
221export const blockEnd = (text: string, quoted: Uint8Array, kind: Kind, from: number): number => {
222  if (kind === 'py' || kind === 'rb') {
223    const lineStart = text.lastIndexOf('\n', from - 1) + 1
224    const indent = (text.slice(lineStart).match(/^[ \t]*/)?.[0] ?? '').length
225    const lines = /\n([ \t]*)(\S)/g
226    lines.lastIndex = text.indexOf('\n', from)
227    if (lines.lastIndex < 0) return text.length
228    for (let m = lines.exec(text); m; m = lines.exec(text)) {
229      const at = m.index + 1 + m[1]!.length
230      if (quoted[at] === 1) continue
231      // Ruby's closing end sits at the block's own indent, and belongs to it
232      if (m[1]!.length < indent || (m[1]!.length === indent && !(kind === 'rb' && text.startsWith('end', at)))) return m.index
233      if (m[1]!.length === indent) return text.indexOf('\n', at) < 0 ? text.length : text.indexOf('\n', at)
234    }
235    return text.length
236  }
237  let depth = 0
238  for (let i = from; i < text.length; i++) {
239    if (quoted[i] === 1) continue
240    if (text[i] === '{') depth++
241    else if (text[i] === '}' && depth > 0 && --depth === 0) return i + 1
242  }
243  return text.length
244}
245
246export type Case = { name: string; at: number; opens: number; isRunner: boolean; plain: string; groups: string[] }
247
248// every case a test file declares in its code, in file order: where its match starts (at:
249// for a JS case, its line's start) and where its keyword stands (opens). A Go function that
250// only runs a suite is marked a runner. Two cases of one name are told apart by the groups
251// they sit in (describe › name), and failing that by their order (name (2)); plain is the
252// name as written, groups the blocks around it
253export const casesIn = (text: string, file: string): Case[] => {
254  const kind = kindOf(file)
255  const quoted = quotedMask(text, langOf(file))
256  const isOpenCode = (m: RegExpMatchArray): boolean => quoted[opensOf(m)] !== 1
257  const runners = kind === 'go' ? new Set([...text.matchAll(GO_SUITE_RUNNER)].filter(isOpenCode).map(m => m[1]!)) : new Set<string>()
258  const found = CASE_PATTERNS[kind].flatMap(({ re, at }) =>
259    [...text.matchAll(re)]
260      .filter(m => (at === 'open' ? isOpenCode(m) : quoted[(m.index ?? 0) + m[0].lastIndexOf(m[1]!)] !== 1))
261      .map(m => ({ name: nameOf(m), at: m.index ?? 0, opens: opensOf(m), isRunner: runners.has(nameOf(m)) })),
262  )
263  // one place matched by two patterns (PHP's test… method under #[Test]) is one case
264  const cases = [...new Map(found.map(c => [c.opens, c])).values()].sort((a, b) => a.at - b.at)
265  const counts = new Map<string, number>()
266  for (const c of cases) if (!c.isRunner) counts.set(c.name, (counts.get(c.name) ?? 0) + 1)
267  const groups = [...text.matchAll(GROUP_PATTERNS[kind])]
268    .filter(isOpenCode)
269    .map(m => ({ name: nameOf(m), from: opensOf(m), to: blockEnd(text, quoted, kind, opensOf(m)) }))
270  const pathOf = (at: number): string[] => groups.filter(g => g.from < at && at < g.to).map(g => g.name)
271  const qualified = cases.map(c => {
272    const path = pathOf(c.opens)
273    const name = (counts.get(c.name) ?? 0) > 1 && path.length > 0 ? `${path.join(' › ')} › ${c.name}` : c.name
274    return { ...c, name, plain: c.name, groups: path }
275  })
276  // still alike (no groups, or the same ones): by their order, the first keeping its name
277  const seen = new Map<string, number>()
278  return qualified.map(c => {
279    if (c.isRunner) return c
280    const n = (seen.get(c.name) ?? 0) + 1
281    seen.set(c.name, n)
282    return n === 1 ? c : { ...c, name: `${c.name} (${n})` }
283  })
284}
285
286export const caseNames = (text: string, file: string): string[] => casesIn(text, file).flatMap(c => (c.isRunner ? [] : [c.name]))
287
288// each Go suite test's suite, by its name
289export const suitesOf = (text: string, file: string): Map<string, string> => {
290  const quoted = quotedMask(text, langOf(file))
291  return new Map([...text.matchAll(GO_SUITE_CASE)].filter(m => quoted[opensOf(m)] !== 1).map(m => [m[2]!, m[1]!]))
292}
293
294// where each case starts in a file, in file order: right after the previous case closes
295// (in JS a line opening with "})"; elsewhere its block's end), so what sits between two cases (a comment, the data a
296// loop runs over, the loop itself) goes with the case below it; failing a close, on
297// the line after the previous case's first
298export const caseStarts = (text: string, file: string): { name: string; at: number; opens: number }[] => {
299  const found = casesIn(text, file)
300  const kind = kindOf(file)
301  const quoted = kind === 'js' ? null : quotedMask(text, langOf(file))
302  return found.map((start, i) => {
303    const prev = found[i - 1]
304    const at = (): number => {
305      if (!prev) return start.at
306      // other languages: on the line after the previous case's block ends
307      if (quoted) {
308        const end = blockEnd(text, quoted, kind, prev.opens)
309        const next = text.indexOf('\n', end)
310        if (end <= start.at && next >= 0 && next < start.at) return next + 1
311      }
312      const between = text.slice(prev.at, start.at)
313      // the previous case's own close, at its indent: a helper declared after it keeps its head
314      const indent = text.slice(text.lastIndexOf('\n', prev.opens - 1) + 1, prev.opens)
315      const own = /^[ \t]*$/.test(indent) ? between.match(new RegExp(`\\n${indent}\\}\\)[^\\n]*\\n`)) : null
316      const closes = [...between.matchAll(/\n[ \t]*\}\)[^\n]*\n/g)]
317      const last = own ?? closes[closes.length - 1]
318      const after = last ? last.index! + last[0].length : between.indexOf('\n') + 1
319      return after > 0 ? prev.at + after : start.at
320    }
321    return { name: start.name, at: at(), opens: start.opens }
322  })
323}
324
325// a top-level declaration a test can use: a constant, a helper, a type, a fixture
326export const DECLARATION = /^(?:export\s+)?(?:declare\s+)?(?:(?:const|let|var|function\*?|async\s+function\*?|class|type|interface|enum|func|def|fn|struct)\s+(\w+)|(\w+)\s*=(?!=))/gm
327
328// A name with a hole in it is a template: the cases a loop or a table generates. A hole is a
329// ${…}, or as it.each and test.each fill one, a printf mark (%s, %p, %i, %d, %j, %o, %#) or a
330// $field of the row. The grader names each case as it expands, and a returned name belongs to
331// the template it fits
332const HOLE = /\$\{[^}]*\}|%[sdifjoOpP#]|\$[A-Za-z_][\w.]*/
333export const isTemplate = (name: string): boolean => HOLE.test(name)
334// a template as a regular expression's source, unanchored: its holes match any text
335export const templateSource = (template: string): string =>
336  template.split(new RegExp(HOLE.source, 'g')).map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('.+?')
337export const fits = (template: string, name: string): boolean => {
338  if (!isTemplate(template)) return template === name
339  const parts = template.split(new RegExp(HOLE.source, 'g')).map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
340  return new RegExp(`^${parts.join('[\\s\\S]+?')}$`).test(name)
341}
342
343// whether a name the lists hold is still among a file's cases: itself, or a case of a loop
344export const among = (names: string[], name: string): boolean => names.some(n => fits(n, name))
345
346// Each row to one case of its file: its own name's, else the first template it fits. A loop
347// named only by its row, it.each(...)('%s'), fits every name in the file: matched by fit alone,
348// every other test's row would be counted again under it
349export const byCase = <T extends { name: string }>(names: string[], rows: T[]): Map<string, T[]> => {
350  const owned = new Map<string, T[]>(names.map(n => [n, []]))
351  const taken = new Set<T>()
352  for (const t of rows) if (owned.has(t.name)) (owned.get(t.name)!.push(t), taken.add(t))
353  for (const n of names) if (isTemplate(n)) for (const t of rows) if (!taken.has(t) && fits(n, t.name)) (owned.get(n)!.push(t), taken.add(t))
354  return owned
355}
356// the case a name is: its own, else the first loop it fits
357export const caseOf = (names: string[], name: string): string | undefined => names.find(n => n === name) ?? names.find(n => fits(n, name))
358// one row per test: a row a fault listed twice counts once
359export const uniqueRows = <T extends { file: string; name: string }>(rows: T[]): T[] => {
360  const seen = new Set<string>()
361  return rows.filter(t => {
362    const key = `${t.file}\u0000${t.name}`
363    return seen.has(key) ? false : (seen.add(key), true)
364  })
365}
366
367// A looped test's row under its template name, kept from before its cases were graded one by
368// one, gives way to those cases' rows (its own list's, or alongside, the other list's): the test
369// is counted once, not also as unrated
370export const withoutTemplates = <T extends { file: string; name: string }>(rows: T[], alongside: { file: string; name: string }[] = rows): T[] => {
371  const cases = new Map<string, string[]>()
372  for (const t of alongside) if (!isTemplate(t.name)) cases.set(t.file, [...(cases.get(t.file) ?? []), t.name])
373  return rows.filter(t => !isTemplate(t.name) || !(cases.get(t.file) ?? []).some(n => fits(t.name, n)))
374}
375
376// the line a case opens on: its own it( or test(, a looped case's the loop's; else the top
377export const caseLine = (text: string, name: string, file: string): number => {
378  const found = casesIn(text, file).find(c => fits(c.name, name))
379  return found ? text.slice(0, found.opens).split('\n').length : 1
380}
381
382// the cases whose text a span of the file (from, to) falls in: each case from its start to the
383// next one's, so an edit inside a test's body names that test
384export const casesAround = (text: string, file: string, from: number, to: number): string[] => {
385  const starts = caseStarts(text, file)
386  return starts.filter((c, i) => c.at < Math.max(to, from + 1) && from < (starts[i + 1]?.at ?? text.length)).map(c => c.name)
387}
388
389// The cases a change to a file touched: those whose own text (from their start to the next
390// case's) differs between the file before and after, and those it added
391export const changedCases = (before: string, after: string, file: string): string[] => {
392  const textsOf = (text: string): Map<string, string> => {
393    const starts = caseStarts(text, file)
394    return new Map(starts.map((c, i) => [c.name, text.slice(c.at, starts[i + 1]?.at ?? text.length).trim()]))
395  }
396  const was = textsOf(before)
397  return [...textsOf(after)].filter(([name, text]) => was.get(name) !== text).map(([name]) => name)
398}
399
400// a file by its path in the project, or as it is when it lies outside
401export const shortPath = (file: string, cwd: string): string => (cwd && file.startsWith(`${cwd}/`) ? file.slice(cwd.length + 1) : file)
402
403// A project's ignore list (.test-grader-ignore), as gitignore reads one: a pattern per line, # a
404// comment; * any run within a name, ** any run of folders, ? one character; a pattern with a /
405// before its end is from the project's root, one without matches a name at any depth; a
406// trailing / names a folder alone. Whether a path in the project is ignored
407export const ignoredBy = (list: string): ((path: string) => boolean) => {
408  const rules = list
409    .split('\n')
410    .map(line => line.trim())
411    .filter(line => line !== '' && !line.startsWith('#'))
412    .map(line => {
413      const isDir = line.endsWith('/')
414      const pattern = line.replace(/\/+$/, '')
415      const isRooted = pattern.includes('/')
416      const body = pattern
417        .replace(/^\//, '')
418        .split(/(\*\*\/?|\*|\?)/)
419        .map(part => (part === '**/' ? '(?:.*/)?' : part === '**' ? '.*' : part === '*' ? '[^/]*' : part === '?' ? '[^/]' : part.replace(/[.+^${}()|[\]\\]/g, '\\$&')))
420        .join('')
421      return new RegExp(`${isRooted ? '^' : '(?:^|/)'}${body}${isDir ? '/' : '(?:/|$)'}`)
422    })
423  return path => rules.some(r => r.test(path))
424}
425

1import type { Confidence, Verdict } from '../types'
2import { verdictOf } from './verdicts'
3import { DECLARATION, among, caseNames, caseStarts, fits, isTemplate, langOf } from './discovery'
4
5// what the grader reads, and what its reply holds: pure text work, no engine calls
6
7export const MAX_SOURCE = 40_000
8// of a file too long to send whole: at most this much of its head (imports, helpers), and of
9// any one case under review
10export const MAX_HEAD = 12_000
11export const MAX_BODY = 20_000
12
13export const clamp = (s: string, n: number): string => (s.length > n ? `${s.slice(0, n - 1)}…` : s)
14// What the grader reads: the whole file when it fits. Else, in file order: its head, the
15// cases under review whole, each from its start to the next case's, and of what sits between
16// the other cases, each piece that declares a name the shown code uses
17export const excerptOf = (source: string, names: string[], file: string): string => {
18  if (source.length <= MAX_SOURCE) return source
19  const starts = caseStarts(source, file)
20  const head = clamp(source.slice(0, starts[0]?.at ?? source.length), MAX_HEAD)
21  const pieces = starts.map((start, i) => ({
22    // a looped case is asked about by its expanded name: the loop that generates it is shown
23    isChosen: names.some(name => fits(start.name, name)),
24    whole: source.slice(start.at, starts[i + 1]?.at ?? source.length).trimEnd(),
25    // what sits above the case's own line: comments, data, helpers
26    declares: [...source.slice(start.at, start.opens).matchAll(DECLARATION)].map(m => (m[1] ?? m[2])!),
27    gap: source.slice(start.at, start.opens).trimEnd(),
28  }))
29  const extra = new Set<number>()
30  let shown = pieces.filter(p => p.isChosen).map(p => p.whole).join('\n')
31  // a helper the shown code uses can use another, so until nothing more is named
32  for (let isGrowing = true; isGrowing; ) {
33    isGrowing = false
34    pieces.forEach((p, i) => {
35      if (p.isChosen || extra.has(i) || !p.declares.some(name => new RegExp(`\\b${name}\\b`).test(shown))) return
36      extra.add(i)
37      shown += `\n${p.gap}`
38      isGrowing = true
39    })
40  }
41  const note = langOf(file) === 'py' ? '#' : '//'
42  const LEFT_OUT = `${note} … other tests left out …`
43  const out = [head.trimEnd()]
44  pieces.forEach((p, i) => {
45    const piece = p.isChosen
46      ? p.whole.length > MAX_BODY
47        ? `${p.whole.slice(0, MAX_BODY)}\n${note} … the rest of this test is left out: it is too long to send …`
48        : p.whole
49      : extra.has(i)
50        ? p.gap
51        : null
52    if (piece !== null) out.push(piece)
53    else if (out[out.length - 1] !== LEFT_OUT) out.push(LEFT_OUT)
54  })
55  return out.join('\n\n')
56}
57
58// Of a file sent as an excerpt, the tests it leaves out, by name: the grader reads what the
59// siblings cover (TestX_ThresholdBoundary) before it calls a case missing
60const MAX_OTHERS = 80
61export const othersOf = (source: string, names: string[], file: string): string[] => {
62  const others = [...new Set(caseNames(source, file))].filter(n => !names.some(name => fits(n, name)))
63  if (others.length === 0) return []
64  const shown = others.slice(0, MAX_OTHERS).map(n => JSON.stringify(n)).join(', ')
65  return [`The tests left out, by name: ${shown}${others.length > MAX_OTHERS ? ` and ${others.length - MAX_OTHERS} more` : ''}. A case one of them covers by its name is not missing.`]
66}
67
68// a case's own text, from its start to the next case's (a looped case's: its loop's); null
69// when the file no longer has it
70export const caseTextOf = (source: string, name: string, file: string): string | null => caseTextsOf(source, file)(name)
71// the same for many names of one file, its cases found once
72export const caseTextsOf = (source: string, file: string): ((name: string) => string | null) => {
73  const starts = caseStarts(source, file)
74  return name => {
75    const i = starts.findIndex(s => fits(s.name, name))
76    return i < 0 ? null : source.slice(starts[i]!.at, starts[i + 1]?.at ?? source.length).trimEnd()
77  }
78}
79
80// The asked names that are cases a loop generates, each set under the loop's own name in the
81// file: the grader is told so, to judge each by that loop's body with its variable bound
82export const loopsOf = (source: string, names: string[], file: string): string[] => {
83  const templates = [...new Set(caseNames(source, file))].filter(isTemplate)
84  const byLoop = new Map<string, string[]>()
85  for (const name of names) {
86    const loop = isTemplate(name) ? undefined : templates.find(t => fits(t, name))
87    if (loop) byLoop.set(loop, [...(byLoop.get(loop) ?? []), name])
88  }
89  return [...byLoop].map(([loop, cases]) => `${cases.map(c => JSON.stringify(c)).join(', ')} ${cases.length === 1 ? 'is a case' : 'are cases'} of the loop that declares the test ${JSON.stringify(loop)}: judge ${cases.length === 1 ? 'it' : 'each'} by that loop's body, its variable bound to the case's value.`)
90}
91
92// the verdicts in a grader reply; of one cut off before its closing ], each object that
93// arrived whole (isCut)
94const CONFIDENCES: readonly unknown[] = ['high', 'medium', 'low']
95export const parseVerdicts = (text: string): { verdicts: Graded[]; isCut: boolean } => {
96  const start = text.indexOf('[')
97  if (start < 0) return { verdicts: [], isCut: false }
98  const objects: unknown[] = []
99  let depth = 0
100  let from = -1
101  let inString = false
102  let isClosed = false
103  for (let i = start + 1; i < text.length && !isClosed; i++) {
104    const c = text[i]
105    if (inString) {
106      if (c === '\\') i++
107      else if (c === '"') inString = false
108    } else if (c === '"') inString = true
109    else if (c === '{') {
110      if (depth === 0) from = i
111      depth++
112    } else if (c === '}') {
113      depth--
114      if (depth === 0) {
115        try {
116          objects.push(JSON.parse(text.slice(from, i + 1)))
117        } catch {
118          // a malformed one is skipped; the rest still count
119        }
120      }
121    } else if (c === ']' && depth === 0) isClosed = true
122  }
123  const verdicts = objects.flatMap((r): Graded[] => {
124    const o = r as Record<string, unknown>
125    const verdict = verdictOf(o.verdict)
126    if (typeof o.name !== 'string' || !verdict) return []
127    const reason = String(o.reason ?? '')
128    const missed = typeof o.missed === 'string' ? o.missed.trim() : ''
129    const sure = typeof o.confidence === 'string' && CONFIDENCES.includes(o.confidence.trim().toLowerCase()) ? { confidence: o.confidence.trim().toLowerCase() as Confidence } : {}
130    // shallow only with a bug the grader can name; told whoever fixes it, as the case to add
131    if (verdict === 'shallow' && missed === '') return [{ name: o.name, summary: String(o.summary ?? ''), verdict: 'strong', reason: `${reason} (Graded strong: no bug it would miss was named.)`.trim(), ...sure }]
132    if (verdict === 'strong') {
133      // strong only with a bug the grader can name; with none, another reviewer could fairly disagree
134      const catches = catchesOf(o.catches)
135      if (catches === null) return [{ name: o.name, summary: String(o.summary ?? ''), verdict, reason: `${reason} (It named no bug the test would catch.)`.trim(), confidence: 'low' }]
136      return [{ name: o.name, summary: String(o.summary ?? ''), verdict, reason: `${reason} It catches: ${catches.bug}`.trim(), ...sure, catches }]
137    }
138    return [{ name: o.name, summary: String(o.summary ?? ''), verdict, reason: verdict === 'shallow' ? `${reason} It would miss: ${missed}` : reason, ...sure }]
139  })
140  return { verdicts, isCut: !isClosed }
141}
142
143export type Graded = { name: string; summary: string; verdict: Verdict; reason: string; confidence?: Confidence; catches?: Catches }
144
145// The bug a strong test would catch, as its grader named it: in a sentence, and, where the grader
146// saw the code under test, the change to it that makes the bug, for a background run to measure
147export type Catches = { bug: string; file?: string; find?: string; replace?: string }
148export const catchesOf = (given: unknown): Catches | null => {
149  if (typeof given === 'string') return given.trim() === '' ? null : { bug: given.trim() }
150  if (given === null || typeof given !== 'object') return null
151  const o = given as Record<string, unknown>
152  const text = (v: unknown): string => (typeof v === 'string' ? v : '')
153  const bug = text(o.bug).trim()
154  if (bug === '') return null
155  const [file, find, replace] = [text(o.file).trim(), text(o.find), text(o.replace)]
156  // a change only where it names a file and alters something
157  return file !== '' && find !== '' && find !== replace ? { bug, file, find, replace } : { bug }
158}
159
160// A test the grader graded case by case (Test › xdr role, Test/xdr role, Test > xdr role) when
161// asked for the test: one verdict for it, the worst of its cases', its reason saying which case
162// earned it. A verdict under the test's own name stands, its cases' set aside
163const CASE_MARK = /^\s*(?:›|>|\/|::|-)\s*/
164const WORST: Verdict[] = ['hollow', 'duplicate', 'shallow', 'brittle', 'strong']
165export const foldCases = <V extends { name: string; verdict: Verdict; reason: string; summary: string }>(names: string[], verdicts: V[]): V[] => {
166  const caseOf = (v: V): { test: string; label: string } | null => {
167    if (among(names, v.name)) return null
168    // the longest asked name it extends, so Test_A is not taken for Test_AB's case
169    const test = names.filter(n => !isTemplate(n) && v.name.startsWith(n) && CASE_MARK.test(v.name.slice(n.length))).sort((a, b) => b.length - a.length)[0]
170    return test === undefined ? null : { test, label: v.name.slice(test.length).replace(CASE_MARK, '') }
171  }
172  const cases = new Map<string, { label: string; v: V }[]>()
173  const rest: V[] = []
174  for (const v of verdicts) {
175    const c = caseOf(v)
176    if (c === null) rest.push(v)
177    else cases.set(c.test, [...(cases.get(c.test) ?? []), { label: c.label, v }])
178  }
179  const folded = [...cases.entries()]
180    .filter(([test]) => !rest.some(v => v.name === test))
181    .map(([test, of]): V => {
182      const worst = [...of].sort((a, b) => WORST.indexOf(a.v.verdict) - WORST.indexOf(b.v.verdict))[0]!
183      const labels = of.map(c => JSON.stringify(c.label)).join(', ')
184      const reason = worst.v.verdict === 'strong' ? `Graded case by case (${labels}), each strong. ${worst.v.reason}` : `Graded case by case (${labels}); the case ${JSON.stringify(worst.label)} is ${worst.v.verdict}: ${worst.v.reason}`
185      return { ...worst.v, name: test, reason: reason.trim() }
186    })
187  return [...rest, ...folded]
188}
189
190// a name as a model may echo it back: curly quotes straight, dashes plain, an escape's
191// backslash dropped, each run of space one
192const loose = (name: string): string =>
193  name
194    .replace(/[‘’‚‛′]/g, "'")
195    .replace(/[“”„‟″]/g, '"')
196    .replace(/[‐‑‒–—]/g, '-')
197    .replace(/…/g, '...')
198    .replace(/\\(['"`\\])/g, '$1')
199    .replace(/\s+/g, ' ')
200    .trim()
201
202// each verdict under the name it was asked by: a verdict whose name differs from one asked
203// name alone only in its quotes, dashes, escapes or spacing answers for that test
204// and one named with the groups around it ("ApiClient › appends every file") answers for the
205// test asked by the end of that name, where one test alone has it; a name that extends an
206// asked one by a case mark is a case of that test, not a group's
207const GROUP_MARK = /\s+(?:›|>)\s+/
208export const asAsked = <V extends { name: string }>(names: string[], verdicts: V[]): V[] => {
209  const asked = [...new Set(names)]
210  const alikeTo = (name: string): string[] => asked.filter(n => !isTemplate(n) && loose(n) === loose(name))
211  return verdicts.map(v => {
212    if (among(names, v.name)) return v
213    const alike = alikeTo(v.name)
214    if (alike.length === 1) return { ...v, name: alike[0]! }
215    if (asked.some(n => !isTemplate(n) && v.name.startsWith(n) && CASE_MARK.test(v.name.slice(n.length)))) return v
216    const parts = v.name.split(GROUP_MARK)
217    for (let k = 1; k < parts.length; k++) {
218      const tail = parts.slice(k).join(' › ')
219      const fitting = asked.filter(n => isTemplate(n) && fits(n, tail))
220      const matched = [...new Set([...alikeTo(tail), ...(among(asked.filter(n => !isTemplate(n)), tail) ? [tail] : [])])]
221      if (matched.length + fitting.length === 1) return { ...v, name: matched[0] ?? tail }
222      if (matched.length + fitting.length > 1) return v
223    }
224    // the group's name joined by a space or a colon alone ("parseDebugId reads the debug ID"):
225    // the longest test asked the name ends with, where a space or a colon comes before it (one
226    // ending part way into a word is no group's: no shorter test is taken in its stead)
227    const said = loose(v.name)
228    const ending = asked.filter(n => !isTemplate(n) && said.length > loose(n).length && said.endsWith(loose(n))).sort((a, b) => b.length - a.length)[0]
229    return ending !== undefined && /[\s:]$/.test(said.slice(0, said.length - loose(ending).length)) ? { ...v, name: ending } : v
230  })
231}
232
233// the grader's reply limit, in tokens: a reply cut off there loses the verdicts it had not reached
234export const MAX_REPLY = 8000
235
236// Why a test asked about got no verdict from this reply, for its row to say; null when it got
237// one. The likely causes in turn: the reply cut off before it, no verdict read at all, its own
238// verdict unreadable (an unknown grade, a broken object), a verdict under another name, left out
239export const unratedWhy = (text: string, verdicts: Graded[], isCut: boolean, names: string[], name: string, model: string): string | null => {
240  if (verdicts.some(v => fits(name, v.name))) return null
241  // the tests answered, not the verdicts: a loop's or table's cases are many verdicts for one test
242  const answered = names.filter(n => verdicts.some(v => fits(n, v.name))).length
243  const count = `it gave ${answered} of the ${names.length} verdicts asked for`
244  if (isCut) return `The grader's (${model}) reply was cut off at its ${MAX_REPLY}-token limit before it reached this test: ${count}.`
245  if (verdicts.length === 0) {
246    const said = text.replace(/\s+/g, ' ').trim()
247    return `The grader (${model}) answered with no verdict it could read: "${said.length > 160 ? `${said.slice(0, 160)}…` : said}".`
248  }
249  // the test's own object, as it came back: its name as JSON writes it, and the braces around it
250  const at = isTemplate(name) ? -1 : text.indexOf(JSON.stringify(name))
251  if (at >= 0) {
252    const from = text.lastIndexOf('{', at)
253    const to = text.indexOf('}', at)
254    const own = text.slice(from < 0 ? at : from, to < 0 ? undefined : to + 1).replace(/\s+/g, ' ')
255    return `The grader (${model}) answered for this test, but its verdict could not be read: ${own.length > 240 ? `${own.slice(0, 240)}…` : own}`
256  }
257  const strays = verdicts.filter(v => !among(names, v.name)).map(v => JSON.stringify(v.name))
258  if (strays.length > 0) return `The grader (${model}) gave no verdict under this test's name; it answered for ${strays.slice(0, 3).join(', ')}${strays.length > 3 ? ` and ${strays.length - 3} more` : ''}, which no test asked about is named.`
259  return `The grader (${model}) left this test out of its answer: ${count}.`
260}
261

1// Strong grades measured: each names a bug its test would catch, and where the grader saw the
2// code under test, the change that makes it. A few at a time, while the session is idle, the
3// change is made and the test run: one that still passes let its bug through. Pure: no engine
4// calls
5
6// a strong grade's change to measure, by file::name; textOf: the test's own text when graded
7export type Proposed = { bug: string; file: string; find: string; replace: string; textOf: string }
8// held: the test failed with the change; through: it passed; unmeasured: the run showed nothing
9// (the test failed unchanged, or the change did not build), why saying which
10export type Measured = { state: 'held' | 'through' | 'unmeasured'; change: string; textOf: string; why?: string }
11
12// how many proposals are kept: the most recent, by when they were graded
13export const MAX_PROPOSED = 3000
14
15// The next tests to measure: strong now, with a change proposed for their text as it stands, and
16// not measured at that text; a shuffled pick, so a big project's sample is spread over it
17export const pickToMeasure = (
18  strong: { key: string; textOf: string | undefined }[],
19  proposed: Record<string, Proposed>,
20  measured: Record<string, Measured>,
21  count: number,
22  random: () => number = Math.random,
23): string[] => {
24  const open = strong.filter(t => {
25    const p = proposed[t.key]
26    if (!p || (t.textOf !== undefined && p.textOf !== t.textOf)) return false
27    return measured[t.key]?.textOf !== p.textOf
28  })
29  for (let i = open.length - 1; i > 0; i--) {
30    const j = Math.floor(random() * (i + 1))
31    ;[open[i], open[j]] = [open[j]!, open[i]!]
32  }
33  return open.slice(0, Math.max(0, count)).map(t => t.key)
34}
35
36// the code with the change made, or why it cannot be: the text to find must be there exactly once
37export const mutate = (code: string, find: string, replace: string): { code: string } | { why: string } => {
38  const count = code.split(find).length - 1
39  if (count !== 1) return { why: count === 0 ? 'the text to change is not in the file' : `the text to change is in the file ${count} times, not once` }
40  return { code: code.replace(find, () => replace) }
41}
42
43// how a change is named, in notes and reasons
44export const changeOf = (find: string, replace: string, file: string): string => `${JSON.stringify(find)} replaced by ${JSON.stringify(replace)} in ${file}`
45
46// a Go build's overlay: the file built from another, the source left as it is
47export const overlayOf = (file: string, replacement: string): string => JSON.stringify({ Replace: { [file]: replacement } })
48
49// the grade a test that let its named bug through is given instead of strong
50export const throughGrade = (bug: string, change: string): { verdict: 'shallow'; reason: string } => ({
51  verdict: 'shallow',
52  reason: `Measured: it still passes with ${change}, the bug its strong grade named. It would miss: ${bug}`,
53})
54
55// the pane's line: of the strong tests listed, how many a measured change made fail, and how
56// many tests let theirs through (graded shallow since); null before any is measured
57export const measuredLine = (strongKeys: string[], measured: Record<string, Measured>): string | null => {
58  const results = Object.values(measured)
59  const through = results.filter(m => m.state === 'through').length
60  if (results.length === 0) return null
61  const held = strongKeys.filter(k => measured[k]?.state === 'held').length
62  return `measured: ${held} of ${strongKeys.length} strong${through > 0 ? ` · ${through} let their named bug through (now shallow)` : ''}`
63}
64

1// Go's coverage profile (go test -coverprofile) read: pure, so the engine calls stay in register.tsx
2//
3// Each line after the mode is one block: `<import path>/<file>.go:<from>,<to> <statements> <count>`.
4// A block can appear more than once (a package tested by several test binaries): it counts once,
5// covered when any run covered it. Go measures statements alone: no lines, branches or functions
6
7// the module path go.mod declares, which every import path in the profile starts with
8export const moduleOf = (goMod: string): string | null => goMod.match(/^module\s+(\S+)/m)?.[1]?.replace(/^"|"$/g, '') ?? null
9
10type Tally = { total: number; covered: number }
11
12// isKept: whether a file counts (not one the project's ignore list names)
13export const goProfileOf = (profile: string, module: string | null, cwd: string, isKept: (file: string) => boolean = () => true): { statements: number | null; byFile: ({ file: string } & Tally)[]; byPackage: ({ name: string } & Tally)[] } => {
14  const blocks = new Map<string, { file: string; statements: number; isCovered: boolean }>()
15  for (const line of profile.split('\n')) {
16    const m = line.trim().match(/^(.+\.go):(\d+\.\d+,\d+\.\d+) (\d+) (\d+)$/)
17    if (!m) continue
18    const [, path, span, statements, count] = m
19    const key = `${path}:${span}`
20    const was = blocks.get(key)
21    blocks.set(key, { file: path!, statements: Number(statements), isCovered: (was?.isCovered ?? false) || Number(count) > 0 })
22  }
23  // an import path inside the module is a file in the project; one outside it keeps its path
24  const local = (path: string): string => (module && path.startsWith(`${module}/`) ? `${cwd}/${path.slice(module.length + 1)}` : path)
25  const files = new Map<string, Tally>()
26  for (const b of blocks.values()) {
27    const f = files.get(b.file) ?? { total: 0, covered: 0 }
28    f.total += b.statements
29    if (b.isCovered) f.covered += b.statements
30    files.set(b.file, f)
31  }
32  const byFile = [...files].map(([path, f]) => ({ file: local(path), ...f })).filter(f => isKept(f.file))
33  const total = byFile.reduce((s, f) => s + f.total, 0)
34  const covered = byFile.reduce((s, f) => s + f.covered, 0)
35  // a package is its files' folder: by its path in the project ('./' the module's root), one
36  // outside the module by its import path
37  const packages = new Map<string, Tally>()
38  for (const f of byFile) {
39    const dir = f.file.slice(0, f.file.lastIndexOf('/'))
40    const name = dir === cwd ? './' : `${dir.startsWith(`${cwd}/`) ? dir.slice(cwd.length + 1) : dir}/`
41    const p = packages.get(name) ?? { total: 0, covered: 0 }
42    p.total += f.total
43    p.covered += f.covered
44    packages.set(name, p)
45  }
46  const byPackage = [...packages].filter(([, p]) => p.total > 0).map(([name, p]) => ({ name, ...p }))
47  return { statements: total > 0 ? (covered / total) * 100 : null, byFile, byPackage }
48}
49
50// A folder's run (go test ./<folder>/...) merged into the module's last profile: the blocks of
51// the folder's files, and of every folder under it, are the new run's; the rest are kept
52export const mergeProfile = (whole: string, part: string, module: string | null, rel: string): string => {
53  const isInFolder = (line: string): boolean => module !== null && (line.match(/^(.+\.go):/)?.[1] ?? '').startsWith(`${module}/${rel}/`)
54  const blocks = (profile: string): string[] => profile.split('\n').filter(l => l.trim() !== '' && !l.startsWith('mode:'))
55  const mode = part.match(/^mode: .+$/m)?.[0] ?? whole.match(/^mode: .+$/m)?.[0] ?? 'mode: set'
56  return [mode, ...blocks(whole).filter(l => !isInFolder(l)), ...blocks(part), ''].join('\n')
57}
58
59// what a package is for, read from one of its files: a command (package main), or a test helper
60// (mocks, or code that imports testing or a mocking library), which coverage tells apart from the
61// code tests are written for; undefined for anything else
62export type PackageRole = 'command' | 'helper'
63const HELPER_IMPORT = /^\s*(?:import\s+)?(?:[\w.]+\s+)?"(?:testing|github\.com\/stretchr\/testify\/mock|go\.uber\.org\/mock\/gomock|github\.com\/golang\/mock\/gomock)"/m
64export const roleOf = (source: string): PackageRole | undefined => {
65  const name = source.match(/^package\s+(\w+)/m)?.[1]
66  if (name === undefined) return undefined
67  if (name === 'main') return 'command'
68  return isHelperName(name) || HELPER_IMPORT.test(source) ? 'helper' : undefined
69}
70
71// a folder or package named as test code: mocks, fakes, testutil, fixtures, or Go's xxxtest
72// convention (httptest, receipttest), not an English word that happens to end in test
73const NOT_HELPERS = new Set(['latest', 'contest', 'protest', 'attest', 'detest', 'greatest', 'smallest', 'fastest', 'shortest', 'longest', 'biggest', 'smartest'])
74export const isHelperName = (name: string): boolean =>
75  /^(?:\w*mocks?|fakes?|testutils?|testhelpers?|testing|fixtures|testdata|testkit|testsupport)$/.test(name) || (/^\w+test$/.test(name) && !NOT_HELPERS.has(name))
76
77// Go's standard header for generated code, "Code generated … DO NOT EDIT.", before the package
78// clause: code no one writes tests for
79export const isGenerated = (source: string): boolean => {
80  const at = source.search(/^package\s/m)
81  const head = at === -1 ? source : source.slice(0, at)
82  return /^\/\/ Code generated .* DO NOT EDIT\.$/m.test(head)
83}
84

1// The project's grades as the store keeps them: short codes, no summaries when room runs out. Pure
2import type { Confidence, ExistingTest, Verdict } from '../types'
3
4import { verdictOf } from './verdicts'
5
6// The project's grades outlive the session: kept in the store under the project's folder,
7// with each graded file's fingerprint, so a later session lists them and Grade all tests
8// grades again only the files changed since. Listed-but-ungraded rows are not kept
9export type SavedGrades = { results: ExistingTest[]; hashes: Record<string, string>; finishedAt?: number }
10// as kept: by file, each file's fingerprint and its tests as [name, verdict, summary, reason,
11// suite, evidence, evidenceOf, confidence (m or l; high left out), textOf], verdicts as g (strong), w (shallow), b (brittle), u (hollow), d (duplicate)
12// (none: unrated), the first three as the grades before these were kept; a file's path is
13// written once
14type KeptTest = [string, string, string?, string?, string?, string?, string?, string?, string?]
15export type KeptGrades = { v: 2; files: Record<string, { hash?: string; tests: KeptTest[] }>; finishedAt?: number }
16export const gradesKey = (cwd: string): string => `grades:${cwd}`
17const VERDICT_CODE: Record<Verdict, string> = { strong: 'g', shallow: 'w', brittle: 'b', hollow: 'u', duplicate: 'd' }
18const CODE_VERDICT: Record<string, Verdict> = Object.fromEntries(Object.entries(VERDICT_CODE).map(([v, c]) => [c, v as Verdict]))
19
20// lean: the summaries left out, for a project whose grades are too many to keep whole
21export const keep = (saved: SavedGrades, isLean: boolean): KeptGrades => {
22  const files: KeptGrades['files'] = {}
23  for (const [file, hash] of Object.entries(saved.hashes)) files[file] = { hash, tests: [] }
24  for (const t of saved.results) {
25    const row: KeptTest = [t.name, t.verdict ? VERDICT_CODE[t.verdict] : '', isLean ? '' : (t.summary ?? ''), t.reason ?? '', t.suite ?? '', t.evidence ?? '', t.evidence ? (t.evidenceOf ?? '') : '', t.confidence === 'medium' ? 'm' : t.confidence === 'low' ? 'l' : '', t.verdict ? (t.textOf ?? '') : '']
26    while (row.length > 2 && !row[row.length - 1]) row.pop()
27    ;(files[t.file] ??= { tests: [] }).tests.push(row)
28  }
29  return { v: 2, files, ...(saved.finishedAt === undefined ? {} : { finishedAt: saved.finishedAt }) }
30}
31export const unkeep = (kept: KeptGrades | SavedGrades): SavedGrades => {
32  // the oldest form, verdicts in words: the old words read as the nearest grade
33  if (!('v' in kept)) return { ...kept, results: kept.results.map(({ verdict, ...t }) => (verdictOf(verdict) ? { ...t, verdict: verdictOf(verdict)! } : t)) }
34  const results: ExistingTest[] = []
35  const hashes: Record<string, string> = {}
36  for (const [file, { hash, tests }] of Object.entries(kept.files)) {
37    if (hash) hashes[file] = hash
38    for (const [name, code, summary, reason, suite, evidence, evidenceOf, sure, textOf] of tests) {
39      const confidence: Confidence | undefined = sure === 'm' ? 'medium' : sure === 'l' ? 'low' : undefined
40      results.push({ file, name, ...(CODE_VERDICT[code] ? { verdict: CODE_VERDICT[code] } : {}), ...(summary ? { summary } : {}), ...(reason ? { reason } : {}), ...(suite ? { suite } : {}), ...(evidence ? { evidence } : {}), ...(evidence && evidenceOf ? { evidenceOf } : {}), ...(confidence ? { confidence } : {}), ...(textOf ? { textOf } : {}) })
41    }
42  }
43  return { results, hashes, ...(kept.finishedAt === undefined ? {} : { finishedAt: kept.finishedAt }) }
44}
45

1// What a grader call costs, at the Claude API's list prices (USD per million tokens, from
2// platform.claude.com/docs/en/about-claude/pricing): pure, so the engine calls stay in register.tsx
3
4type Price = { input: number; write: number; read: number; output: number }
5type Usage = { input_tokens?: number; output_tokens?: number; cache_read_input_tokens?: number; cache_creation_input_tokens?: number }
6
7// Haiku 5.5 is priced by the prompt's length: one of over 100,000 tokens pays the higher prices
8const HAIKU_5_5: Price = { input: 0.1, write: 0.125, read: 0.01, output: 0.5 }
9const HAIKU_5_5_LONG: Price = { input: 0.5, write: 0.625, read: 0.05, output: 2.5 }
10export const LONG_PROMPT = 100_000
11
12// by family and version; a family's alias (haiku, sonnet, opus) is its latest, as Claude Code maps it
13const PRICES: Record<string, Price> = {
14  'haiku-5-5': HAIKU_5_5,
15  'haiku-4-5': { input: 1, write: 1.25, read: 0.1, output: 5 },
16  'sonnet-5-5': { input: 2, write: 2.5, read: 0.1, output: 10 },
17  'sonnet-5': { input: 2, write: 2.5, read: 0.2, output: 10 },
18  'sonnet-4-6': { input: 3, write: 3.75, read: 0.3, output: 15 },
19  'sonnet-4-5': { input: 3, write: 3.75, read: 0.3, output: 15 },
20  'opus-5-5': { input: 4, write: 5, read: 0.2, output: 20 },
21  'opus-5': { input: 5, write: 6.25, read: 0.5, output: 25 },
22  'opus-4-8': { input: 5, write: 6.25, read: 0.5, output: 25 },
23  'opus-4-7': { input: 5, write: 6.25, read: 0.5, output: 25 },
24  'opus-4-6': { input: 5, write: 6.25, read: 0.5, output: 25 },
25  'opus-4-5': { input: 5, write: 6.25, read: 0.5, output: 25 },
26  'fable-5-1': { input: 10, write: 12.5, read: 0.25, output: 50 },
27  'fable-5': { input: 10, write: 12.5, read: 1, output: 50 },
28}
29const ALIASES: Record<string, string> = { haiku: 'haiku-5-5', sonnet: 'sonnet-5-5', opus: 'opus-5-5', fable: 'fable-5-1' }
30
31// the price key a model name comes to: an alias, or an id with a provider's prefix and a date or
32// version after it (us.anthropic.claude-haiku-5-5-20260101-v1:0); null for a model not listed
33export const priceKeyOf = (model: string): string | null => {
34  const id = model.trim().toLowerCase()
35  if (ALIASES[id]) return ALIASES[id]!
36  const m = id.match(/claude-(haiku|sonnet|opus|fable)-(\d+)(?:-(\d))?(?!\d)/)
37  if (!m) return null
38  const key = m[3] ? `${m[1]}-${m[2]}-${m[3]}` : `${m[1]}-${m[2]}`
39  return PRICES[key] ? key : null
40}
41
42// one call's cost in USD, or null when the model's price is not known
43export const costOf = (model: string, usage: Usage): number | null => {
44  const key = priceKeyOf(model)
45  if (key === null) return null
46  const input = usage.input_tokens ?? 0
47  const write = usage.cache_creation_input_tokens ?? 0
48  const read = usage.cache_read_input_tokens ?? 0
49  const price = key === 'haiku-5-5' && input + write + read > LONG_PROMPT ? HAIKU_5_5_LONG : PRICES[key]!
50  return (input * price.input + write * price.write + read * price.read + (usage.output_tokens ?? 0) * price.output) / 1e6
51}
52

1// What the models read: the grader's rubric, the system prompt's section on tests, the follow-ups
2// a note ends with, and the descriptions and schemas of Claude's three tools. Pure: no engine calls
3import type { Kind } from './discovery'
4import { FIX, FLAGGED, LISTED } from './verdicts'
5
6// the rubric every grader call opens with, fixed so the prompt cache keeps it
7export const RUBRIC = [
8  'You are a careful, concise reviewer of automated tests. Answer with JSON only.',
9  'For each test case you are asked about, say in one plain sentence what it verifies (summary) and judge whether it is a decent test.',
10  'verdict, one of five, each naming what is wrong:',
11  '"strong" = a plausible bug in the code under test would make it fail, and a correct change to how the code works would not;',
12  '"shallow" = it can fail, but misses the likely bugs: happy path only, checks that a value is defined or truthy, loose matchers, one easy case where the edges matter;',
13  '"brittle" = it checks real behaviour but would also fail on a correct change: large snapshots, exact mock call order or counts, private state or implementation details, real time, timing, network or order between tests;',
14  '"hollow" = no real bug could make it fail: no assertion, a tautology, asserts only on its own mock, a snapshot of nothing, would pass with the code under test deleted;',
15  '"duplicate" = another test in the file already catches the same bugs. Start the reason with: Repeats "<that test>", keep "<the one to keep>". Keep the clearer or stronger of the two; of two tests that repeat each other, mark only the one to delete duplicate, never both.',
16  'Where more than one fits, give the first of: hollow, duplicate, shallow, brittle.',
17  'Default to "strong". Flag a test only when you can point at what in it a reviewer would change; where you are unsure between "strong" and a flagged grade, answer "strong".',
18  'Judge "shallow" against the whole file: a test that checks one case is strong when other tests in the file cover the edges and errors, or when that one case is all its name promises. Matching a prefix, a subset or one key field is not shallow when that is the contract under test.',
19  'Give "shallow" only with a concrete bug it would let through: in missed, an input and the wrong result the code could give that the test would still pass. A "shallow" with no missed counts as "strong".',
20  'Never mark a test down for code you cannot see, or for not covering what another test covers.',
21  'reason: one short sentence justifying the verdict.',
22  'A name with ${...} in it is a template for cases generated in a loop: grade each case the loop generates separately, named as the loop expands it.',
23  "A loop's variables belong only to the tests inside that loop: do not fault a test outside it for not using them.",
24  'A name with › in it is the groups the test sits in (describe blocks, classes), then its own name; a name ending in (2) is the second test of that name. Answer with each name exactly as given.',
25  'When the code under test is shown, judge each assertion against what that code really does.',
26  'Ask only for behaviour the code under test has: a missed case must be one the code shown could get wrong. Where a test\'s name promises behaviour the code does not have (an allowlist the code never checks, a cache it never keeps), the fault is the name: say so in reason and suggest a name for what it checks, and judge the test by what it checks.',
27  'Do not rest a verdict on how the language, runtime or build treats the code (strict mode, a transform, module loading) unless the source shows it: take the test to run as written, and judge what its assertions would catch.',
28  'Examples. it("parses a date", () => expect(parse("2024-01-02")).toEqual(new Date(2024, 0, 2))) beside tests of invalid and empty input: strong, its siblings cover the edges; catches: {"bug": "parse(\\"2024-01-02\\") giving February 2 when the month is not made zero-based", "file": "src/date.ts", "find": "Number(m) - 1", "replace": "Number(m)"}.',
29  'expect(error.message.startsWith("Invalid amount")) where the message prefix is what callers rely on: strong.',
30  'expect(total(items)).toBeDefined(): shallow, missed: "total([{price: 2}, {price: 3}]) returning 4 would pass".',
31  'expect(fn).toHaveBeenCalledTimes(3) on an internal helper: brittle. expect(true).toBe(true): hollow.',
32  'Strict mocks assert by themselves: a gomock controller fails the test on any call it was not told to expect, and an EXPECT() with no Times means exactly once; mockery, Mockito strict stubs and the like work the same way. A test built on them checks its calls even with no assertion after them: never call it hollow or shallow for "passing if the mock is never called".',
33  'Exact names, order or shapes are behaviour, not implementation details, when they are the contract callers rely on: which steps a plan holds, the keys of a payload, the order of a public list. Asserting them is not brittle; brittle is pinning what could change without any caller noticing.',
34  'For "strong", name in catches one bug the test would catch: bug, one sentence naming an input and the wrong result it would fail on. Where the code under test is shown, also give the change to it that makes that bug: file (its path as shown), find (exact text found once in that file: a whole expression or line), replace (that text with the bug). Pick a change that still builds and alters only the behaviour the test checks. catches is {} unless the verdict is "strong"; a "strong" that names no bug counts as low confidence.',
35  'confidence: "high" when the source shows the verdict plainly; "medium" when it rests on code you can only partly see, or on a judgement call; "low" when another careful reviewer could fairly give a different verdict.',
36  'Return a JSON array: [{"name": string, "summary": string, "verdict": "strong"|"shallow"|"brittle"|"hollow"|"duplicate", "reason": string, "missed": string, "confidence": "high"|"medium"|"low", "catches": {"bug": string, "file": string, "find": string, "replace": string}}], missed empty unless the verdict is "shallow"',
37].join('\n')
38
39// the rounds Claude gets to fix a flagged test, as the notes and the system prompt promise
40export const MAX_ROUNDS = 3
41export const FOLLOW_UP = `Once you are done writing tests, fix each of these as its grade asks (${FLAGGED.map(v => `${v}: ${FIX[v]}`).join('; ')}), or, where one is better than rated, send your evidence with the test_evidence tool. Each test gets ${MAX_ROUNDS} rounds.`
42export const SPENT_FOLLOW_UP = 'Tell the person which of these still need work and why.'
43
44// told to Claude in the system prompt, ahead of any test it writes: how to write a test the
45// grader rates strong, and how to follow up on the grades
46export const GRADING_SECTION = [
47  '# Test grading (test-grader)',
48  'Every test you write or edit is graded in the background by a reviewer model, with a grade that names what is wrong: strong (a plausible bug makes it fail, a correct refactor does not), shallow (misses the likely bugs: happy path only, defined or truthy checks), brittle (fails on correct changes: big snapshots, exact mock calls, implementation details, timing), hollow (cannot fail: no real assertion, a tautology, tests the mock) or duplicate (another test catches the same bugs). Write tests that grade strong:',
49  '- Assert on behaviour: the return value, the thrown error, the state or output the code produces. Never assert only that a mock was called, that a value is defined or truthy, or that a thing equals itself.',
50  '- Before you keep a test, ask which plausible bug in the code would make it fail. If none, rewrite it. If it would pass with the function body deleted, it is hollow.',
51  '- One behaviour per test, named for the behaviour and the case ("rejects a negative amount"), not the function.',
52  '- Cover edges and errors, not only the happy path: empty, boundary, invalid input, failure paths. Prefer several small tests to one long one.',
53  "- Use real code where you can; mock only I/O, time and randomness, and assert on what the code did with the mock's answer, not on the mock.",
54  '- Make it deterministic: fixed clocks, seeds and data; no sleeps, no order dependence, no shared mutable state.',
55  '- No snapshots unless the snapshot is small and reviewed; assert on what the code does, not on how it does it.',
56  'Grades arrive as notes; nothing waits on them. When you have finished writing or editing tests for the task, call test_grades with written: true. Fix each flagged test as its grade asks, worst first: rewrite a hollow one, delete or merge a duplicate, add the missing case to a shallow one, and loosen a brittle one to assert on behaviour.',
57  'Where one is better than rated, prove it: test_verify runs it, applies a mutation to the code under test, runs it again and puts the file back, and sends what it measured as evidence; with siblings: true it also says which other tests in the file catch the change. Or send test_evidence: run the test unchanged (it must pass), apply the mutation, run again (it must fail), revert, and quote both results. Where a grade looks wrong, test_context shows what the grader read for that test.',
58  `Each change is graded again; call test_grades again to see the new grades. Tests listed as being graded: wait a moment and ask again. Tests listed as unrated or never graded: test_grade grades them and answers with the result. When you write tests to raise coverage, call test_coverage with the folder once you are done, for its new figure. After ${MAX_ROUNDS} rounds on one test, tell the person what is left instead.`,
59].join('\n')
60
61// A guide for each language, in the mod's guides folder (guides/js.md, guides/go.md, ...): the
62// section names only the ones for the languages the project's tests are written in, for Claude
63// to read before it writes tests
64export const LANGUAGE_NAMES: Record<Kind, string> = {
65  js: 'JavaScript and TypeScript',
66  go: 'Go',
67  py: 'Python',
68  rb: 'Ruby',
69  swift: 'Swift',
70  jvm: 'Java and Kotlin',
71  cs: 'C#',
72  php: 'PHP',
73  rs: 'Rust',
74}
75export const guideOf = (root: string, kind: Kind): string => `${root}/guides/${kind}.md`
76export const LANGUAGE_ORDER: Kind[] = ['js', 'go', 'py', 'rb', 'swift', 'jvm', 'cs', 'php', 'rs']
77
78// the session's tool for evidence that a test is better (or worse) than its verdict
79export const EVIDENCE_TOOL = 'test_evidence'
80export const EVIDENCE_MAX = 4_000
81export const EVIDENCE_HINT =
82  'If one of these is better than rated, send your evidence (a mutation that makes it fail, what it alone catches) with the test_evidence tool to have it regraded.'
83export const EVIDENCE_DESCRIPTION =
84  'Ask test-grader to regrade one test on evidence that it deserves a different verdict. The grader cannot run code, so give it facts it can check against the source: the exact mutation you made (file, line, before and after), the command you ran, and the test\'s output before and after. ' +
85  'Strong evidence: a mutation that changes behaviour and makes only this test fail. Weak evidence: that the test passes, that it has coverage, or that other tests cover the same code. Send one test per call; test_verify measures a mutation for you.'
86export const EVIDENCE_SCHEMA = {
87  type: 'object',
88  properties: {
89    file: { type: 'string', description: 'The test file, absolute or relative to the project' },
90    test: { type: 'string', description: 'The test name as written (it(...)/test(...)), or as its loop generates it' },
91    evidence: { type: 'string', description: 'What shows the test is better or worse than rated' },
92  },
93  required: ['file', 'test', 'evidence'],
94}
95
96// The session's tool for evidence test-grader measures itself: the test is run as it is (it
97// must pass), then with one change made to the code under test (it should fail), and the file
98// put back. What was run and what came of it goes to the grader as evidence. It changes files
99// and runs commands, so the person is asked before it runs
100export const VERIFY_TOOL = 'test_verify'
101export const VERIFY_SIBLINGS = 20
102export const VERIFY_DESCRIPTION =
103  'Have test-grader measure whether a test catches a bug: it runs the test unchanged (it must pass), applies your mutation to the code under test (replace one exact piece of text in one file), runs the test again (it should fail), and puts the file back. ' +
104  'What it measured is sent to the grader as evidence, and the test regraded. Use it for a test you believe is better than its grade: pick a mutation that breaks the behaviour the test asserts and still compiles; a mutation that does not build measures nothing, and is refused.'
105export const VERIFY_SCHEMA = {
106  type: 'object',
107  properties: {
108    file: { type: 'string', description: 'The test file, absolute or relative to the project' },
109    test: { type: 'string', description: 'The test name, as test_grades lists it' },
110    mutate: { type: 'string', description: 'The file of code under test to change for the second run (not a test file)' },
111    find: { type: 'string', description: 'Exact text in the mutate file (the code under test, never the test file), found there exactly once, to replace' },
112    replace: { type: 'string', description: 'What to put in its place: a plausible bug' },
113    siblings: { type: 'boolean', description: `Also run the file's other tests with the mutation (up to ${VERIFY_SIBLINGS}) and report which of them fail too: whether this test alone catches the change` },
114    env: { type: 'object', additionalProperties: { type: 'string' }, description: "Variables for the runs, over the project's .test-grader-env: an emulator's address a test skips without, say" },
115  },
116  required: ['file', 'test', 'mutate', 'find', 'replace'],
117}
118
119// the session's tool for the grades as they stand: the flagged tests by default,
120// worst first, each at its line, so Claude can find them without the pane
121export const GRADES_TOOL = 'test_grades'
122export const GRADES_LIMIT = 50
123export const GRADES_DESCRIPTION =
124  'List the tests test-grader has graded, with each grade, what the test checks and why. A grade names what is wrong, and so the fix: hollow (cannot fail: rewrite it to assert on what the code does), duplicate (delete it or merge it), shallow (add the case it misses), brittle (assert on behaviour, not how the code does it), strong (keep it). ' +
125  'By default the flagged ones (hollow, duplicate, shallow, brittle) and the unrated, worst first, each with its file and line. ' +
126  'Call it with written: true once you have finished writing or editing tests, and again after each fix. Use path to narrow to a file or folder.'
127export const GRADES_SCHEMA = {
128  type: 'object',
129  properties: {
130    verdicts: {
131      type: 'array',
132      items: { type: 'string', enum: [...LISTED] },
133      description: 'Which tests to list, by state; default ["hollow", "duplicate", "shallow", "brittle", "unrated"]. unrated: the grader gave no verdict; reviewing: being graded; ungraded: never graded',
134    },
135    path: { type: 'string', description: 'Only tests in this file or folder, absolute or relative to the project' },
136    written: { type: 'boolean', description: 'Only the tests written or edited this session' },
137    layer: { type: 'string', enum: ['unit', 'integration', 'e2e'], description: "Only tests of this layer, as their file's path, build tag or imports, or the project's .test-grader-layers, say" },
138    ran: { type: 'string', enum: ['never ran', 'skipped', 'not built'], description: 'List the tests the last coverage run reached but never ran, skipped, or did not build (a Go file with a build tag the run was not given), whatever their grade' },
139    limit: { type: 'number', description: `How many tests to list at most; default ${GRADES_LIMIT}` },
140  },
141}
142
143// the session's tool to grade tests now: the ones not rated yet, or with again every one, in
144// the project or a file or folder of it. It waits for the run, and answers with what it found
145export const GRADE_TOOL = 'test_grade'
146export const GRADE_DESCRIPTION =
147  'Grade tests now, as the pane\'s Grade all tests does, and wait for the result: the counts, then every flagged and unrated test. ' +
148  'By default it grades the tests not rated yet (never graded, unrated, or in a file changed since its last grading) and keeps the grades that stand. With again: true it grades every test in scope again. ' +
149  'Use path to narrow it to a file or folder; a whole project can take minutes.'
150export const GRADE_SCHEMA = {
151  type: 'object',
152  properties: {
153    path: { type: 'string', description: 'Only the test files in this file or folder, absolute or relative to the project; default the whole project' },
154    again: { type: 'boolean', description: 'Grade every test in scope again, the rated ones too' },
155  },
156}
157
158// the session's tool to measure coverage now, of the project or a folder of it, and wait for the figures
159export const COVERAGE_TOOL = 'test_coverage'
160export const COVERAGE_DESCRIPTION =
161  "Run the project's coverage now and wait for it: a folder's figure (with path) before and after the run, the project's, and the least covered folders under it. " +
162  'In a Go project a folder\'s run measures only its packages (go test ./<folder>/...), far faster than the whole module, and the rest keep their last figures; elsewhere the whole run is made and the folder\'s figure read from it. ' +
163  'Call it when you have finished writing tests to raise coverage, with the folder you worked on.'
164export const COVERAGE_SCHEMA = {
165  type: 'object',
166  properties: {
167    path: { type: 'string', description: 'A folder, absolute or relative to the project; default the whole project' },
168  },
169}
170
171// the session's tool to see what the grader reads for a test: to tell a misjudged test from a
172// grader that could not see what it needed
173export const CONTEXT_TOOL = 'test_context'
174export const CONTEXT_MAX = 60_000
175export const CONTEXT_DESCRIPTION =
176  'Show exactly what test-grader\'s grader reads for one test: the test file as sent (whole, or the excerpt and the tests it leaves out), the code under test it was given, the project\'s rules, and what it is asked, with the test\'s last grade. ' +
177  'Use it when a grade looks wrong, to see whether the grader could see the helper, the sibling test or the code it needed.'
178export const CONTEXT_SCHEMA = {
179  type: 'object',
180  properties: {
181    file: { type: 'string', description: 'The test file, absolute or relative to the project' },
182    test: { type: 'string', description: 'The test name, as test_grades lists it' },
183  },
184  required: ['file', 'test'],
185}
186
187

1// The command that runs one test, by its language and the project's runner: pure, so the
2// engine calls stay in register.tsx
3import { isTemplate, templateSource } from './discovery'
4import type { Kind } from './discovery'
5
6// what the project runs its tests with, as found at its root
7export type Runners = {
8  js?: 'vitest' | 'jest' | 'node' | 'playwright'
9  /** node: its script loads TypeScript with tsx */
10  isTsx?: boolean
11  /** node: the package has Playwright too, for its .spec files */
12  hasPlaywright?: boolean
13  jvm?: 'gradle' | 'maven'
14  isBundled?: boolean
15  isPest?: boolean
16}
17
18// a test as its runner names it: its file (in the project), its own name, the groups around
19// it, its line, and a Go suite test's suite
20export type RunTarget = { rel: string; kind: Kind; plain: string; groups: string[]; line: number; suite?: string; tags?: string[] }
21
22// a Go file's build tags its tests need: the names its //go:build line asks for, not those it
23// rules out (//go:build integration && !short needs integration)
24export const goTagsOf = (text: string): string[] => {
25  const line = /^\/\/go:build (.+)$/m.exec(text.slice(0, text.search(/^package /m) >>> 0))?.[1] ?? ''
26  return [...new Set([...line.matchAll(/(!?)\b([A-Za-z_][\w.]*)/g)].filter(m => m[1] === '').map(m => m[2]!))]
27}
28
29const SPEC = /\.spec\.[cm]?[jt]sx?$/
30const escapeRegex = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
31// a name as a runner's pattern: a loop's template matches each of its cases
32const patternOf = (name: string): string => (isTemplate(name) ? templateSource(name) : escapeRegex(name))
33// a test's class: its innermost group, else the file's name less its extension
34const classOf = (t: RunTarget): string => t.groups[t.groups.length - 1] ?? t.rel.slice(t.rel.lastIndexOf('/') + 1).replace(/\.\w+$/, '')
35
36// the argv that runs this one test, or null where test-grader knows no runner for it
37export const runArgv = (t: RunTarget, runners: Runners): string[] | null => {
38  const dir = t.rel.includes('/') ? t.rel.slice(0, t.rel.lastIndexOf('/')) : '.'
39  switch (t.kind) {
40    case 'js': {
41      const full = [...t.groups.map(escapeRegex), patternOf(t.plain)].join(' ')
42      if (runners.js === 'vitest') return ['npx', 'vitest', 'run', t.rel, '-t', `^${full}$`]
43      if (runners.js === 'jest') return ['npx', 'jest', t.rel, '-t', `^${full}$`]
44      // node's own runner, for a package whose scripts run node --test; its Playwright, if it has
45      // one, runs the .spec files
46      if (runners.js === 'node' && !(runners.hasPlaywright && SPEC.test(t.rel)))
47        return ['node', ...(runners.isTsx ? ['--import', 'tsx'] : []), '--test', '--test-name-pattern', `${patternOf(t.plain)}$`, t.rel]
48      if (runners.js === 'playwright' || runners.js === 'node') return ['npx', 'playwright', 'test', `${t.rel}:${t.line}`]
49      return null
50    }
51    case 'py':
52      return ['python3', '-m', 'pytest', '-q', [t.rel, ...t.groups, t.plain].join('::')]
53    case 'go':
54      return ['go', 'test', `./${dir}`, '-count=1', '-v', ...(t.tags?.length ? ['-tags', t.tags.join(',')] : []), '-run', t.suite ? `/^${t.plain}$` : `^${t.plain}$`]
55    case 'rb':
56      if (t.rel.endsWith('_spec.rb')) return [...(runners.isBundled ? ['bundle', 'exec'] : []), 'rspec', `${t.rel}:${t.line}`]
57      return ['ruby', '-Itest', t.rel, '-n', `/^${escapeRegex(t.plain.replace(/ /g, '_'))}$|^test_${escapeRegex(t.plain.replace(/ /g, '_'))}$/`]
58    case 'rs':
59      return ['cargo', 'test', t.plain]
60    case 'jvm':
61      if (runners.jvm === 'gradle') return ['./gradlew', 'test', '--tests', `*${classOf(t)}.${t.plain}`]
62      if (runners.jvm === 'maven') return ['mvn', '-q', 'test', `-Dtest=${classOf(t)}#${t.plain}`]
63      return null
64    case 'cs':
65      return ['dotnet', 'test', '--filter', `FullyQualifiedName~${classOf(t)}.${t.plain}`]
66    case 'php':
67      return runners.isPest ? ['vendor/bin/pest', t.rel, '--filter', t.plain] : ['vendor/bin/phpunit', '--filter', `/::${escapeRegex(t.plain)}$/`, t.rel]
68    case 'swift':
69      return ['swift', 'test', '--filter', `${classOf(t)}/${t.plain}`]
70  }
71}
72
73// a command as a person would type it
74export const shown = (argv: readonly string[]): string => argv.map(a => (/^[\w./:=@%^+-]+$/.test(a) ? a : `'${a.replace(/'/g, `'\\''`)}'`)).join(' ')
75
76// the last lines a run printed, the empty ones left out
77export const tailOf = (output: string, lines: number): string => output.split('\n').filter(l => l.trim() !== '').slice(-lines).join('\n')
78
79// a run's output that says the code did not build or load, not that a test failed: Go's
80// [build failed] and compiler lines, TypeScript's error TS, a SyntaxError, Rust's error[E…],
81// javac's and Kotlin's compilation errors, Swift's and C#'s compiler errors
82// the files that mark where a language's project starts, its tests run from there: a Go module
83// in backend/, a jest app in mobile/
84export const PROJECT_MARKS: Partial<Record<Kind, string[]>> = {
85  go: ['go.mod'],
86  js: ['package.json'],
87  py: ['pyproject.toml', 'pytest.ini', 'setup.cfg', 'setup.py'],
88  rs: ['Cargo.toml'],
89  jvm: ['build.gradle', 'build.gradle.kts', 'pom.xml'],
90  rb: ['Gemfile'],
91  php: ['composer.json'],
92  swift: ['Package.swift'],
93}
94
95// a run's output that says the test could not be run at all: no module, no runner, no test found
96export const isSetupFailure = (tail: string): boolean =>
97  /cannot find main module|go\.mod file not found|no Go files in|command not found|No tests found|no tests ran|no tests to run|no test files|ENOENT|Cannot find module|could not be found|not recognized as an internal or external command/i.test(tail)
98
99// a run's output that says it ran no test at all, though it may have exited 0: every test
100// skipped or none matched. Go's [no tests to run], or -v lines with no test passed or failed (a
101// t.Skip); Jest's and Vitest's Tests line, and pytest's summary, with nothing passed or failed;
102// Node's runner's pass 0 and fail 0
103const NOTHING_PASSED = '(?![^\\n]*\\b\\d+ (?:passed|failed)\\b)'
104export const isNoneRun = (output: string): boolean =>
105  /\[no tests to run\]|testing: warning: no tests to run|^ok\s.*\[no test files\]/m.test(output) ||
106  (/^=== RUN /m.test(output) && !/^\s*--- (?:PASS|FAIL):/m.test(output)) ||
107  new RegExp(`^Tests:\\s+${NOTHING_PASSED}[^\\n]*\\btotal\\b`, 'm').test(output) ||
108  new RegExp(`^\\s*Tests\\s+${NOTHING_PASSED}[^\\n]*\\(\\d+\\)\\s*$`, 'm').test(output) ||
109  new RegExp(`^=+ ${NOTHING_PASSED}[^\\n]*\\bin [\\d.]+s\\b[^\\n]*=+$`, 'm').test(output) ||
110  (/^[#ℹ] pass 0$/m.test(output) && /^[#ℹ] fail 0$/m.test(output))
111
112export const isBuildFailure = (tail: string): boolean =>
113  /\[build failed\]|\[setup failed\]|^# \S+\n\S+\.go:\d+:\d+: |\berror TS\d+:|\bSyntaxError\b|\bIndentationError\b|\berror\[E\d+\]|COMPILATION ERROR|Compilation failed|\berror CS\d+:|\berror: cannot find symbol|^e: .*\.kt:/m.test(tail)
114

1// A test's layer: unit, integration or end-to-end, read from its file's path and text; pure,
2// so the engine calls stay in register.tsx
3import { ignoredBy } from './discovery'
4import { goTagsOf } from './runner'
5
6export type Layer = 'unit' | 'integration' | 'e2e'
7export const LAYERS: readonly Layer[] = ['unit', 'integration', 'e2e']
8export const LAYER_NAMES: Record<Layer, string> = { unit: 'unit', integration: 'integration', e2e: 'end-to-end' }
9
10// a project's own rules, one a line, the first that matches a file's path winning:
11//   integration: **/*.sqlite.test.ts
12//   e2e: maestro/
13export const LAYERS_FILE = '.test-grader-layers'
14export type LayerRules = { layer: Layer; matches: (path: string) => boolean }[]
15export const layerRulesOf = (text: string): LayerRules =>
16  text.split('\n').flatMap(line => {
17    const m = /^\s*(unit|integration|e2e|end-to-end)\s*:\s*(\S.*?)\s*$/.exec(line)
18    return m ? [{ layer: (m[1] === 'end-to-end' ? 'e2e' : m[1]) as Layer, matches: ignoredBy(m[2]!) }] : []
19  })
20
21const E2E_PATH = /(^|\/)(e2e|end-to-end|acceptance|playwright|cypress)(\/|$)|[._-]e2e([._-]|$)|\.cy\.[cm]?[jt]sx?$/i
22const INTEGRATION_PATH = /(^|\/)(integration|integration[-_]tests?|it)(\/|$)|[._-]integration([._-]|$)|\.(int|sqlite|db|pg|postgres|mysql)\.(test|spec)\.|IT\.(java|kt)$/i
23// a browser or device driven from the test: Playwright, Cypress, Detox, WebdriverIO, Selenium
24const E2E_TEXT = /\bfrom\s+['"](@playwright\/test|cypress|detox|webdriverio|selenium-webdriver)['"]|\brequire\(\s*['"](@playwright\/test|cypress|detox|webdriverio|selenium-webdriver)['"]\s*\)/
25// a real database or service started for the test (a database driver imported into it), or a
26// marker that says so
27const INTEGRATION_TEXT =
28  /@pytest\.mark\.integration\b|\btestcontainers\b|@Tag\(\s*"integration"\s*\)|@SpringBootTest\b|\b(?:from\s+|require\(\s*)['"](?:better-sqlite3|sqlite3|pg|mysql2|mongodb-memory-server|ioredis|redis-memory-server)['"]/
29
30// rel: the file's path in the project; text: its source, when read
31export const layerOf = (rel: string, text: string | null, rules: LayerRules = []): Layer => {
32  const own = rules.find(r => r.matches(rel))
33  if (own) return own.layer
34  const tags = text !== null && rel.endsWith('.go') ? goTagsOf(text) : []
35  if (E2E_PATH.test(rel) || tags.includes('e2e') || (text !== null && E2E_TEXT.test(text))) return 'e2e'
36  if (INTEGRATION_PATH.test(rel) || tags.includes('integration') || (text !== null && INTEGRATION_TEXT.test(text))) return 'integration'
37  return 'unit'
38}
39

1// Which tests the last coverage run ran, read from what its runner reported: Go's -v lines, and
2// Jest's or Vitest's JSON results; pure, so the engine calls stay in register.tsx
3import { fits } from './discovery'
4
5export type Outcome = 'passed' | 'failed' | 'skipped'
6// a graded test as the last run left it: run, skipped, or not run at all though its file was in
7// the run's reach; not built: a Go file whose build tag the run was not given, so it was never
8// compiled, as the project chose and no fault of the test
9export type RanState = 'ran' | 'skipped' | 'never ran' | 'not built'
10
11// the run's tests by where they live, a JS file's path or a Go package's folder, each name with
12// how it ended; measured: the folders the run reached, in which a test not listed did not run
13// tagsBy: the build tags each Go folder was run with
14export type RanRecord = { at: number; measured: string[]; by: Record<string, Record<string, Outcome>>; tagsBy?: Record<string, string[]> }
15
16// the build tags a go test argv gives: -tags a,b, -tags=a,b, or the older space-separated list
17export const tagsOfArgv = (argv: string[]): string[] => {
18  const at = argv.findIndex(a => a === '-tags' || a === '--tags' || a.startsWith('-tags=') || a.startsWith('--tags='))
19  if (at === -1) return []
20  const value = argv[at]!.includes('=') ? argv[at]!.slice(argv[at]!.indexOf('=') + 1) : (argv[at + 1] ?? '')
21  return value.split(/[,\s]+/).filter(Boolean)
22}
23
24const better = (a: Outcome | undefined, b: Outcome): Outcome => (a === undefined || a === 'skipped' ? b : a)
25
26// go test -v: each test's --- PASS, FAIL or SKIP line, under the package line that ends its
27// output (ok, FAIL or ? and the package's import path); a package is told by its folder
28export const goRanOf = (output: string, moduleDir: string, module: string): Record<string, Record<string, Outcome>> => {
29  const by: Record<string, Record<string, Outcome>> = {}
30  let pending: [string, Outcome][] = []
31  for (const line of output.split('\n')) {
32    const result = /^\s*--- (PASS|FAIL|SKIP): (\S+)/.exec(line)
33    if (result) {
34      pending.push([result[2]!, result[1] === 'PASS' ? 'passed' : result[1] === 'FAIL' ? 'failed' : 'skipped'])
35      continue
36    }
37    const pkg = /^(?:ok|FAIL|\?)\s+(\S+)(?:\s|$)/.exec(line)?.[1]
38    if (!pkg) continue
39    // the lines above are this package's, kept only for a package of the module
40    const ended = pending
41    pending = []
42    if (pkg !== module && !pkg.startsWith(`${module}/`)) continue
43    const tests = (by[`${moduleDir}${pkg.slice(module.length)}`] ??= {})
44    for (const [name, outcome] of ended) tests[name] = better(tests[name], outcome)
45  }
46  return by
47}
48
49// Jest's --json, and Vitest's json reporter, which writes the same shape: each file's path,
50// each test's title and status
51export const jsRanOf = (json: string): Record<string, Record<string, Outcome>> => {
52  const report = JSON.parse(json) as { testResults?: { name?: string; assertionResults?: { title?: string; status?: string }[] }[] }
53  const by: Record<string, Record<string, Outcome>> = {}
54  for (const file of report.testResults ?? []) {
55    if (typeof file.name !== 'string') continue
56    const tests = (by[file.name] ??= {})
57    for (const t of file.assertionResults ?? []) {
58      if (typeof t.title !== 'string') continue
59      const outcome: Outcome = t.status === 'passed' ? 'passed' : t.status === 'failed' ? 'failed' : 'skipped'
60      tests[t.title] = better(tests[t.title], outcome)
61    }
62  }
63  return by
64}
65
66// a later run of some folders over the record before: what it reached is replaced, the rest kept
67export const mergeRan = (before: RanRecord | null, next: RanRecord): RanRecord => {
68  if (!before) return next
69  const isUnder = (key: string): boolean => next.measured.some(d => key === d || key.startsWith(`${d}/`))
70  const kept = Object.fromEntries(Object.entries(before.by).filter(([key]) => !isUnder(key)))
71  const tagsBy = Object.fromEntries([...Object.entries(before.tagsBy ?? {}).filter(([d]) => !isUnder(d)), ...Object.entries(next.tagsBy ?? {})])
72  return { at: next.at, measured: [...before.measured.filter(d => !isUnder(d)), ...next.measured], by: { ...kept, ...next.by }, ...(Object.keys(tagsBy).length > 0 ? { tagsBy } : {}) }
73}
74
75// a graded test's state in the record: undefined where the run did not reach its file; tags: a Go
76// file's build tags
77export const ranStateOf = (record: RanRecord | null, file: string, name: string, tags: string[] = []): RanState | undefined => {
78  if (!record) return undefined
79  const isGo = file.endsWith('.go')
80  const key = isGo ? file.slice(0, file.lastIndexOf('/')) : file
81  const reach = record.measured.find(d => key === d || key.startsWith(`${d}/`))
82  if (reach === undefined) return undefined
83  // Go: the test, its subtests, or a suite's method under its suite; JS: the title, or a
84  // template's cases
85  const outcomes = Object.entries(record.by[key] ?? {})
86    .filter(([test]) => (isGo ? test === name || test.startsWith(`${name}/`) || test.endsWith(`/${name}`) : test === name || fits(name, test)))
87    .map(([, outcome]) => outcome)
88  if (outcomes.some(o => o !== 'skipped')) return 'ran'
89  if (outcomes.length > 0) return 'skipped'
90  const given = record.tagsBy?.[reach] ?? []
91  return isGo && tags.some(t => !given.includes(t)) ? 'not built' : 'never ran'
92}
93