ln-maintainer

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register, ToolCallInput, ToolCallResult } from 'claude-code'
3
4import type { SkillVerdict, ValidatorOutcome } from '../types'
5import type { EvalCase, EvalResult } from './evals'
6import { describeRow, isClaudeDocument, mergeLatest, needsAttention, parseJson, readResults, RESULTS_ROOT, SKIPPED_DIRECTORIES, summarize, toRows } from './evals'
7import { inRepository, resultToken, skillOf, toRepositoryPath, VALIDATE_COMMAND, validatorDetail, vocabularyTokens, WATCHED } from './repo'
8
9const PANE = 'ln-evals'
10const COMMAND = 'ln-evals'
11const STORE_KEY = 'verdicts'
12const validatorState = atom({ plugin: 'ln-maintainer', key: 'validator' } as const, null)
13const verdictsState = atom({ plugin: 'ln-maintainer', key: 'verdicts' } as const, [])
14const rowsState = atom({ plugin: 'ln-maintainer', key: 'evals' } as const, [])
15
16// Module state: a hot reload runs register again and starts these over, which is what a fresh load should do.
17let isValidating = true
18let pwsh = 'pwsh'
19let currentRun: Promise<ValidatorOutcome> | undefined
20let queuedRun: Promise<ValidatorOutcome> | undefined
21let isUnavailable = false
22let treeDigest: string | undefined
23let activeSkill: string | undefined
24let vocabulary: Set<string> | undefined
25
26async function runValidator($: EngineInterface): Promise<ValidatorOutcome> {
27  try {
28    const run = await $.process.run([pwsh, '-NoProfile', '-NonInteractive', '-Command', VALIDATE_COMMAND], { cwd: await $.session.root(), timeoutMs: 120_000 })
29    return { state: run.exitCode === 0 ? 'pass' : 'fail', detail: validatorDetail(run.exitCode, run.stdout, run.stderr) }
30  } catch (error) {
31    return { state: 'unavailable', detail: error instanceof Error ? error.message : String(error) }
32  }
33}
34
35// One run at a time; edits made while it runs share a single follow-up run.
36function validate($: EngineInterface): Promise<ValidatorOutcome> {
37  if (!currentRun) {
38    currentRun = runValidator($).finally(() => {
39      currentRun = undefined
40    })
41    return currentRun
42  }
43  queuedRun ??= currentRun.then(() => {
44    queuedRun = undefined
45    return validate($)
46  })
47  return queuedRun
48}
49
50async function report($: EngineInterface, outcome: ValidatorOutcome) {
51  await update($, validatorState, () => outcome)
52  if (outcome.state === 'unavailable') {
53    isUnavailable = true
54    $.ui.status(undefined)
55    $.ui.toast(`ln-maintainer: validator unavailable (${outcome.detail.slice(0, 80)}); set pwshPath in /config`)
56    return
57  }
58  $.ui.status(outcome.state === 'pass' ? 'validator ✓' : `validator ✗ ${outcome.detail.slice(0, 90)}`)
59}
60
61async function treeStatus($: EngineInterface): Promise<string> {
62  const status = await $.process.run(['git', 'status', '--porcelain'], { cwd: await $.session.root() })
63  return status.exitCode === 0 ? status.stdout : ''
64}
65
66async function revision($: EngineInterface): Promise<string> {
67  const head = await $.process.run(['git', 'rev-parse', '--short', 'HEAD'], { cwd: await $.session.root() })
68  const short = head.exitCode === 0 ? head.stdout.trim() : 'no-git'
69  return (await treeStatus($)).trim() === '' ? short : `${short}*`
70}
71
72async function recordVerdict($: EngineInterface, skill: string, answer: string) {
73  if (!vocabulary) {
74    vocabulary = await $.fs.read(inRepository(await $.session.root(), 'SKILL_TEMPLATE.md')).then(vocabularyTokens, () => new Set<string>())
75  }
76  const token = resultToken(answer)
77  const verdict: SkillVerdict = { skill, token, isVocabulary: token !== null && vocabulary.has(token), revision: await revision($) }
78  $.ui.status(`${skill} ${token ?? '?'} · ${verdict.revision}`)
79  if (!verdict.isVocabulary && vocabulary.size > 0) $.ui.toast(`${skill}: the Result field does not start with a shared vocabulary verdict token`)
80  const history = await update($, verdictsState, list => [...list, verdict].slice(-50))
81  await $.store.set(STORE_KEY, history)
82}
83
84async function listCases($: EngineInterface, root: string): Promise<EvalCase[]> {
85  const cases: EvalCase[] = []
86  for (const plugin of await $.fs.list(inRepository(root, 'plugins'))) {
87    if (plugin.kind !== 'dir' || !(await $.fs.exists(inRepository(root, `plugins/${plugin.name}/evals`)))) continue
88    for (const entry of await $.fs.list(inRepository(root, `plugins/${plugin.name}/evals`))) {
89      if (entry.kind !== 'dir' || entry.name === 'results') continue
90      const fired = await $.fs.read(inRepository(root, `plugins/${plugin.name}/evals/${entry.name}/graders/skill-fired.md`)).catch(() => '')
91      cases.push({ plugin: plugin.name, name: entry.name, skill: skillOf(fired) ?? '?' })
92    }
93  }
94  return cases.sort((a, b) => a.plugin.localeCompare(b.plugin) || a.name.localeCompare(b.name))
95}
96
97async function listJsonFiles($: EngineInterface, root: string, directory: string, depth: number): Promise<string[]> {
98  if (depth < 0) return []
99  const files: string[] = []
100  for (const entry of await $.fs.list(inRepository(root, directory))) {
101    const path = `${directory}/${entry.name}`
102    if (entry.kind === 'dir' && !SKIPPED_DIRECTORIES.has(entry.name)) files.push(...(await listJsonFiles($, root, path, depth - 1)))
103    if (entry.kind === 'file' && entry.name.endsWith('.json') && entry.name !== 'provenance.json') files.push(path)
104  }
105  return files
106}
107
108async function collectRows($: EngineInterface) {
109  const root = await $.session.root()
110  const cases = await listCases($, root)
111  const results = new Map<string, EvalResult>()
112  if (await $.fs.exists(inRepository(root, RESULTS_ROOT))) {
113    for (const path of await listJsonFiles($, root, RESULTS_ROOT, 5)) {
114      const document = parseJson(await $.fs.read(inRepository(root, path)))
115      const provenancePath = inRepository(root, path.replace(/[^/]+$/, 'provenance.json'))
116      const sibling = isClaudeDocument(document) ? parseJson(await $.fs.read(provenancePath).catch(() => '')) : undefined
117      mergeLatest(results, readResults(document, sibling, path, (await $.fs.stat(inRepository(root, path))).mtimeMs))
118    }
119  }
120  const paths: string[] = []
121  for (const result of results.values()) {
122    for (const path of Object.keys(result.provenance ?? {})) if (!paths.includes(path) && (await $.fs.exists(inRepository(root, path)))) paths.push(path)
123  }
124  const hashes = new Map<string, string>()
125  if (paths.length > 0) {
126    const run = await $.process.run(['git', 'hash-object', '--stdin-paths'], { cwd: root, stdin: paths.join('\n') + '\n' })
127    if (run.exitCode === 0) run.stdout.trim().split(/\r?\n/).forEach((hash, index) => hashes.set(paths[index] as string, hash.trim()))
128  }
129  return toRows(cases, results, hashes)
130}
131
132/** After an edit to a contract-owning file, validate and tell the model when the repository no longer passes. */
133async function afterEdit($: EngineInterface, e: ToolCallInput, next: (e: ToolCallInput) => Promise<ToolCallResult>): Promise<ToolCallResult> {
134  const ran = await next(e)
135  if (!isValidating || isUnavailable || ran.deny !== undefined || ran.isError) return ran
136  const fields = e as { file_path?: unknown; notebook_path?: unknown }
137  const path = typeof fields.file_path === 'string' ? fields.file_path : fields.notebook_path
138  const relative = typeof path === 'string' ? toRepositoryPath(await $.session.root(), path) : undefined
139  if (relative === undefined || !WATCHED.test(relative)) return ran
140  const outcome = await validate($)
141  await report($, outcome)
142  if (outcome.state !== 'fail') return ran
143  const note = `Repository validator fails after editing ${relative}: ${outcome.detail.replace(/\.$/, '')}. This is expected while a multi-file change is in progress; resolve it before finishing.`
144  return { ...ran, context: [...(ran.context ?? []), note] }
145}
146
147/**
148 * Maintainer tooling for this repository only, never distributed with the skill plugins.
149 * It observes and reports: it spawns no agents, submits no prompts, calls no models and registers no tools.
150 */
151export const register: Register = (on, options) => {
152  isValidating = options.validateOnEdit !== false
153  pwsh = typeof options.pwshPath === 'string' && options.pwshPath !== '' ? options.pwshPath : 'pwsh'
154  currentRun = queuedRun = treeDigest = activeSkill = vocabulary = undefined
155  isUnavailable = false
156
157  on('session.start', async ($, e, next) => {
158    const result = await next(e)
159    await $.command.register({ name: COMMAND, description: 'Show every eval case with its latest result and whether it still binds to the current skill' })
160    const stored = await $.store.get(STORE_KEY)
161    if (Array.isArray(stored)) await update($, verdictsState, () => stored as SkillVerdict[])
162    if (isValidating) treeDigest = await treeStatus($)
163    return result
164  })
165
166  on('turn.start', ($, e, next) => {
167    activeSkill = undefined
168    return next(e)
169  })
170
171  on('skill.prompt', ($, e, next) => {
172    activeSkill = skillOf(e.skill) ?? activeSkill
173    return next(e)
174  })
175
176  on('tool.call', { tool: 'Edit' }, afterEdit)
177  on('tool.call', { tool: 'Write' }, afterEdit)
178  on('tool.call', { tool: 'NotebookEdit' }, afterEdit)
179
180  on('turn.complete', async ($, e, next) => {
181    const result = await next(e)
182    if (e.agentId !== undefined || e.isAborted) return result
183    const skill = activeSkill
184    activeSkill = undefined
185    if (skill) await recordVerdict($, skill, e.answer)
186    // Shell edits (sed, scripts) reach no Edit hook: validate the final tree once per main-loop turn.
187    if (isValidating && !isUnavailable) {
188      const now = await treeStatus($)
189      if (now !== treeDigest) {
190        treeDigest = now
191        const outcome = await validate($)
192        await report($, outcome)
193        if (outcome.state === 'fail') $.ui.toast(`Repository validator fails: ${outcome.detail.slice(0, 120)}`)
194      }
195    }
196    return result
197  })
198
199  on('command.run', { command: COMMAND }, async $ => {
200    const rows = await collectRows($)
201    await update($, rowsState, () => rows)
202    await $.ui.open({ id: PANE, title: 'Eval freshness' })
203    return { text: [summarize(rows), ...rows.filter(needsAttention).slice(0, 40).map(describeRow)].join('\n') }
204  })
205
206  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
207    const { Box, Button, Text } = $.ui.resolve(e)
208    const rows = await read($, rowsState)
209    const verdicts = await read($, verdictsState)
210    const room = Math.max(3, (e.viewport?.rows ?? 30) - 10)
211    const ordered = [...rows].sort((a, b) => Number(needsAttention(b)) - Number(needsAttention(a)))
212    return (
213      <Box flexDirection="column">
214        <Box>
215          <Text>{rows.length ? summarize(rows) : 'No eval cases collected yet.'} </Text>
216          <Button
217            key="refresh"
218            label="Refresh"
219            onPress={async () => {
220              const fresh = await collectRows($)
221              await update($, rowsState, () => fresh)
222            }}
223          />
224        </Box>
225        {ordered.slice(0, room).map(row => (
226          <Text key={`case-${row.name}`} dimColor={!needsAttention(row)}>
227            {describeRow(row)}
228          </Text>
229        ))}
230        {rows.length > room && <Text dimColor>{`… ${rows.length - room} more; enlarge the pane to see them`}</Text>}
231        <Text bold>
232          Recent ln verdicts
233        </Text>
234        {verdicts.length === 0 && <Text dimColor>None recorded yet.</Text>}
235        {verdicts
236          .slice(-5)
237          .reverse()
238          .map((verdict, index) => (
239            <Text key={`verdict-${index}`} dimColor={verdict.isVocabulary}>
240              {`${verdict.skill} ${verdict.token ?? '?'} · ${verdict.revision}${verdict.isVocabulary ? '' : '  (not a vocabulary token)'}`}
241            </Text>
242          ))}
243      </Box>
244    )
245  })
246}
247

1/** Pure eval bookkeeping: parse result documents and decide each case's freshness. */
2import type { EvalRow } from '../types'
3
4export type EvalCase = { plugin: string; name: string; skill: string }
5export type EvalResult = {
6  score: number | null
7  isFailing: boolean
8  failedGraders: string[]
9  provenance: Record<string, string> | undefined
10  source: string
11  hostModel: string
12  modifiedMs: number
13}
14type Grader = { name?: unknown; scored?: unknown; passed?: unknown }
15type Run = { error?: unknown; graders?: Grader[] }
16
17export const RESULTS_ROOT = '.eval-results'
18export const SKIPPED_DIRECTORIES = new Set(['workspace', '.git', 'node_modules', 'home'])
19
20function failedGraders(runs: Run[]): string[] {
21  const names = new Set<string>()
22  for (const run of runs) {
23    for (const grader of run.graders ?? []) {
24      if (grader.scored !== false && grader.passed === false && typeof grader.name === 'string') names.add(grader.name)
25    }
26  }
27  return [...names]
28}
29
30export function parseJson(text: string): any {
31  try {
32    return JSON.parse(text.replace(/^/, ''))
33  } catch {
34    return undefined
35  }
36}
37
38export function isResultDocument(document: any): boolean {
39  return document?.schemaVersion === 1 && Array.isArray(document.cases)
40}
41
42export function isClaudeDocument(document: any): boolean {
43  return isResultDocument(document) && document.cases.some((one: any) => one?.arms)
44}
45
46/**
47 * Results of the Codex runner (`host: codex`, provenance per case) and of `claude plugin eval`
48 * (cases with `arms`; provenance from the sibling provenance.json the CI workflow writes).
49 */
50export function readResults(document: any, provenanceDocument: any, source: string, modifiedMs: number): Map<string, EvalResult> {
51  const results = new Map<string, EvalResult>()
52  if (!isResultDocument(document)) return results
53  const isClaude = isClaudeDocument(document)
54  const threshold = typeof document.threshold === 'number' ? document.threshold : 1
55  for (const one of document.cases) {
56    if (typeof one?.name !== 'string') continue
57    const runs: Run[] = isClaude ? one.arms?.with ?? [] : one.runs ?? []
58    const score = typeof one.score === 'number' ? one.score : typeof one.aggregates?.score === 'number' ? one.aggregates.score : null
59    results.set(one.name, {
60      score,
61      isFailing: score === null || score < threshold || runs.some(run => typeof run.error === 'string' && run.error !== ''),
62      failedGraders: failedGraders(runs),
63      provenance: (isClaude ? provenanceDocument?.cases?.[one.name]?.files : one.provenance?.files) ?? undefined,
64      source,
65      hostModel: `${isClaude ? 'claude' : document.host ?? '?'} ${document.model || 'default'}`,
66      modifiedMs,
67    })
68  }
69  return results
70}
71
72/** Keeps the newest result of each case. */
73export function mergeLatest(into: Map<string, EvalResult>, from: Map<string, EvalResult>) {
74  for (const [name, result] of from) {
75    const known = into.get(name)
76    if (!known || known.modifiedMs < result.modifiedMs) into.set(name, result)
77  }
78}
79
80export function toRows(cases: EvalCase[], results: Map<string, EvalResult>, hashes: Map<string, string>): EvalRow[] {
81  return cases.map(evalCase => {
82    const result = results.get(evalCase.name)
83    if (!result) return { ...evalCase, freshness: 'not-run', score: null, isFailing: false, failedGraders: [], source: '', hostModel: '' }
84    const provenance = result.provenance
85    const freshness = !provenance ? 'unbound' : Object.entries(provenance).every(([path, hash]) => hashes.get(path) === hash) ? 'current' : 'stale'
86    return { ...evalCase, freshness, score: result.score, isFailing: result.isFailing, failedGraders: result.failedGraders, source: result.source, hostModel: result.hostModel }
87  })
88}
89
90export function needsAttention(row: EvalRow): boolean {
91  return row.freshness !== 'current' || row.isFailing
92}
93
94export function summarize(rows: EvalRow[]): string {
95  const count = (freshness: EvalRow['freshness']) => rows.filter(row => row.freshness === freshness).length
96  const passing = rows.filter(row => row.freshness === 'current' && !row.isFailing).length
97  return `${rows.length} cases: ${passing} current and passing, ${count('current') - passing} current and failing, ${count('stale')} stale, ${count('unbound')} unbound, ${count('not-run')} NOT RUN`
98}
99
100const MARK: Record<EvalRow['freshness'], string> = { current: 'current', stale: 'STALE', unbound: 'unbound', 'not-run': 'NOT RUN' }
101
102export function describeRow(row: EvalRow): string {
103  const outcome = row.freshness === 'not-run' ? '    ' : row.isFailing ? 'FAIL' : 'pass'
104  const score = row.score === null ? '-' : row.score.toFixed(2)
105  const failed = row.failedGraders.length ? `  failed: ${row.failedGraders.join(', ')}` : ''
106  return `${MARK[row.freshness].padEnd(8)} ${outcome} ${score.padStart(4)}  ${row.name}  ${row.hostModel}${failed}`
107}
108

1/** Pure helpers: no engine access, so tests and the hooks module share them. */
2
3/** Files whose edits can break a repository contract the validator owns. */
4export const WATCHED = /^(plugins\/|docs\/|\.claude-plugin\/|\.agents\/|\.github\/repository-metadata\.json$|scripts\/validate-repository\.ps1$|(SKILL_TEMPLATE|AGENTS|CLAUDE|README)\.md$)/
5
6export const EDIT_TOOLS = ['Edit', 'Write', 'NotebookEdit'] as const
7
8/** Runs the validator and prints its own message instead of PowerShell's decorated error record. */
9export const VALIDATE_COMMAND = "& { try { & ./scripts/validate-repository.ps1 } catch { [Console]::Error.WriteLine($_.Exception.Message); exit 1 } }"
10
11/** Repository-relative path with forward slashes, or undefined outside the repository. */
12export function toRepositoryPath(root: string, path: string): string | undefined {
13  const normalizedRoot = root.replace(/\\/g, '/').replace(/\/+$/, '')
14  const normalizedPath = path.replace(/\\/g, '/')
15  const isWindows = /^[A-Za-z]:\//.test(normalizedRoot)
16  const isAbsolute = /^([A-Za-z]:)?\//.test(normalizedPath)
17  if (!isAbsolute) return normalizedPath.replace(/^\.\//, '')
18  const prefix = normalizedRoot + '/'
19  const isInside = isWindows ? normalizedPath.toLowerCase().startsWith(prefix.toLowerCase()) : normalizedPath.startsWith(prefix)
20  return isInside ? normalizedPath.slice(prefix.length) : undefined
21}
22
23/** The verdict tokens of SKILL_TEMPLATE.md's vocabulary, their single owner. */
24export function vocabularyTokens(template: string): Set<string> {
25  const section = template.match(/\n## Verdict vocabulary\n([\s\S]*?)\n## /)?.[1] ?? ''
26  return new Set([...section.matchAll(/`([A-Z][A-Z_]+)[ `]/g)].map(match => match[1] as string))
27}
28
29/** The first token of a report's Result field, read the way the eval verdict graders read it. */
30export function resultToken(answer: string): string | null {
31  return answer.match(/Result:\**\s*[*`]*([A-Z][A-Z_]+)\b/)?.[1] ?? null
32}
33
34/** The first meaningful line of a failure, or the last line of a success. */
35export function validatorDetail(exitCode: number, stdout: string, stderr: string): string {
36  const isPass = exitCode === 0
37  const lines = (isPass ? stdout : stderr || stdout).split(/\r?\n/).map(line => line.trim()).filter(Boolean)
38  return (isPass ? lines.at(-1) : lines[0]) ?? ''
39}
40
41export function skillOf(skillName: string): string | undefined {
42  return skillName.match(/ln-\d{2}-[a-z0-9-]+/)?.[0]
43}
44
45/** An absolute path under the repository root: the engine resolves a relative path against the plugin's own folder. */
46export function inRepository(root: string, path: string): string {
47  return `${root.replace(/[\\/]+$/, '')}/${path}`
48}
49

1export type ValidatorOutcome = {
2  state: 'pass' | 'fail' | 'unavailable'
3  detail: string
4}
5
6export type SkillVerdict = {
7  skill: string
8  token: string | null
9  isVocabulary: boolean
10  revision: string
11}
12
13export type EvalFreshness = 'current' | 'stale' | 'unbound' | 'not-run'
14
15export type EvalRow = {
16  plugin: string
17  name: string
18  skill: string
19  freshness: EvalFreshness
20  score: number | null
21  isFailing: boolean
22  failedGraders: string[]
23  source: string
24  hostModel: string
25}
26
27declare module 'claude-code' {
28  interface PluginState {
29    'ln-maintainer': {
30      validator: ValidatorOutcome | null
31      verdicts: SkillVerdict[]
32      evals: EvalRow[]
33    }
34  }
35}
36