perf-delta

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { PerfReport, PerfSample } from '../types'
5import { baselineKey, commandStderr, isSafeRef, isTimeout, MIN_RUNS, parseArgs, parseTime, reportLines, statsOf, verdictOf } from './delta'
6
7const PANE = 'perf-delta'
8const TITLE = '性能差'
9const BASELINES = 'baselines'
10const KEEP_BASELINES = 30
11
12const report = atom({ plugin: 'perf-delta', key: 'report' } as const, null as PerfReport | null)
13
14type Settings = { runs: number; timeoutMs: number }
15
16// Whether /usr/bin/time takes -l (BSD, macOS) or -v (GNU), learned once per load.
17let timeFlag: '-l' | '-v' | null | undefined
18let isBusy = false
19
20async function flagOf($: EngineInterface): Promise<'-l' | '-v' | null> {
21  if (timeFlag !== undefined) return timeFlag
22  const exists = await $.fs.exists('/usr/bin/time').catch(() => false)
23  const os = exists ? await $.process.run(['uname', '-s']).catch(() => undefined) : undefined
24  timeFlag = !exists ? null : os?.stdout.trim() === 'Darwin' ? '-l' : '-v'
25  return timeFlag
26}
27
28/** One warm-up run, then `runs` measured ones, in `cwd`. Throws on a failing run. */
29async function measure($: EngineInterface, cwd: string, command: string, label: string, s: Settings): Promise<PerfSample> {
30  const flag = await flagOf($)
31  const argv = flag === null ? ['/bin/sh', '-c', command] : ['/usr/bin/time', flag, '/bin/sh', '-c', command]
32  const timesMs: number[] = []
33  const rssBytes: number[] = []
34  for (let i = 0; i <= s.runs; i += 1) {
35    await update($, report, old => (old === null ? null : { ...old, note: `${label}: ${i === 0 ? 'ウォームアップ' : `${i}/${s.runs} 回目`}` }))
36    const started = await $.clock.now()
37    let failure: unknown
38    const ran = await $.process.run(argv, { cwd, timeoutMs: s.timeoutMs }).catch(err => {
39      failure = err
40      return undefined
41    })
42    const elapsed = (await $.clock.now()) - started
43    // The run rejects both when it outlasts the timeout and when it cannot start (a missing folder).
44    if (ran === undefined && !isTimeout(elapsed, s.timeoutMs)) {
45      throw new Error(`${label}: 起動できませんでした — ${failure instanceof Error ? failure.message : String(failure)}`.slice(0, 300))
46    }
47    if (ran === undefined) throw new Error(`${label}: ${s.timeoutMs / 1000}秒以内に終わりませんでした`)
48    if (ran.exitCode !== 0) throw new Error(`${label}: 終了コード ${ran.exitCode}${commandStderr(ran.stderr) === '' ? '' : ` — ${commandStderr(ran.stderr).slice(-300)}`}`)
49    if (i === 0) continue
50    const parsed = parseTime(ran.stderr)
51    // time(1) reports to 10 ms: under a second the host clock's own measure is the finer one.
52    timesMs.push(parsed.wallMs !== undefined && (parsed.wallMs >= 1000 || elapsed <= 0) ? parsed.wallMs : elapsed)
53    if (parsed.rssBytes !== undefined) rssBytes.push(parsed.rssBytes)
54  }
55  return { label, command, at: await $.clock.now(), timesMs, rssBytes }
56}
57
58type Baselines = Record<string, PerfSample>
59const baselinesOf = async ($: EngineInterface): Promise<Baselines> => ((await $.store.get(BASELINES)) as Baselines | undefined) ?? {}
60
61async function saveBaseline($: EngineInterface, key: string, sample: PerfSample): Promise<void> {
62  const all = { ...(await baselinesOf($)), [key]: { ...sample, label: '基準' } }
63  const kept = Object.entries(all)
64    .sort(([, a], [, b]) => b.at - a.at)
65    .slice(0, KEEP_BASELINES)
66  await $.store.set(BASELINES, Object.fromEntries(kept))
67}
68
69async function git($: EngineInterface, cwd: string, argv: string[]) {
70  return $.process.run(['git', ...argv], { cwd, timeoutMs: 120_000 }).catch(() => undefined)
71}
72
73const tmpOf = async ($: EngineInterface): Promise<string> => ((await $.env.get('TMPDIR')) || '/tmp').replace(/\/+$/, '')
74// Its folder's name carries when it was made, then a random part mktemp chose.
75const WORKTREE = /\/perf-delta-(\d+)-[\w]+$/
76// The longest a run can take is about 8.5 hours (51 runs of at most 10 minutes): past this a
77// worktree of ours is left over from a run cut short (a reload, a crash). Ours are locked while
78// they run, so a locked one waits longer: a laptop asleep mid-run may pass the wall-clock bound.
79const STALE_MS = 12 * 60 * 60_000
80const LOCKED_STALE_MS = 48 * 60 * 60_000
81
82/** The folder below the repository's top that cwd is, as git reads it: right in a linked worktree too. */
83async function folderOf($: EngineInterface, cwd: string): Promise<string> {
84  const ran = await git($, cwd, ['rev-parse', '--show-prefix'])
85  return ran?.exitCode === 0 ? ran.stdout.trim().replace(/\/+$/, '') : ''
86}
87
88/**
89 * Removes a worktree of ours, locked or not. When git cannot (a process still writing in it), its
90 * folder goes and then its registration alone: never a prune, which would drop the person's own
91 * worktrees whose folders are away (an unmounted disk).
92 */
93async function dropWorktree($: EngineInterface, root: string, dir: string): Promise<void> {
94  const remove = () => git($, root, ['worktree', 'remove', '--force', '--force', '--', dir])
95  if ((await remove())?.exitCode === 0) return
96  if (WORKTREE.test(dir)) await $.process.run(['rm', '-rf', '--', dir], { timeoutMs: 120_000 }).catch(() => undefined)
97  await remove()
98}
99
100/** Worktrees of ours a run cut short left registered in the repository, by when each was made. */
101async function dropStale($: EngineInterface, root: string): Promise<void> {
102  const now = await $.clock.now()
103  const tmp = await tmpOf($)
104  // git lists a worktree by its real path: TMPDIR on macOS is under /var, a link to /private/var.
105  const real = (await $.fs.stat(tmp, { resolve: true }).catch(() => undefined))?.realPath ?? tmp
106  const listed = await git($, root, ['worktree', 'list', '--porcelain'])
107  for (const block of listed?.exitCode === 0 ? listed.stdout.split('\n\n') : []) {
108    const dir = block.match(/^worktree (.+)$/m)?.[1] ?? ''
109    const made = Number(dir.match(WORKTREE)?.[1] ?? NaN)
110    const isOurs = dir.startsWith(`${tmp}/`) || dir.startsWith(`${real}/`)
111    const age = /^locked\b/m.test(block) ? LOCKED_STALE_MS : STALE_MS
112    if (isOurs && now - made > age) await dropWorktree($, root, dir)
113  }
114}
115
116/** Runs `command` in a temporary worktree of `ref`, in the same folder below the top. */
117async function measureRef($: EngineInterface, root: string, below: string, ref: string, command: string, s: Settings): Promise<PerfSample> {
118  // A folder only this user can have made: a name another could guess and fill first is none.
119  const made = await $.process.run(['mktemp', '-d', `${await tmpOf($)}/perf-delta-${await $.clock.now()}-XXXXXX`]).catch(() => undefined)
120  const dir = made?.exitCode === 0 ? made.stdout.trim() : ''
121  if (!WORKTREE.test(dir)) throw new Error('一時フォルダを作れませんでした（TMPDIR を確認してください）')
122  // Locked while it runs, so another session's start never takes it for one left behind.
123  const added = await git($, root, ['worktree', 'add', '--lock', '--detach', '--', dir, ref])
124  if (added?.exitCode !== 0) {
125    await dropWorktree($, root, dir)
126    throw new Error(`git worktree add ${ref} に失敗: ${added?.stderr.trim().slice(-200) ?? ''}`)
127  }
128  try {
129    const inside = below === '' ? dir : `${dir}/${below}`
130    if (below !== '' && !(await $.fs.exists(inside).catch(() => false))) throw new Error(`${ref} には ${below} がありません（このフォルダはその時点のコミットにありません）`)
131    return await measure($, inside, command, ref, s)
132  } finally {
133    await dropWorktree($, root, dir)
134  }
135}
136
137function toastOf(r: PerfReport): string {
138  const t = verdictOf(r.before === undefined ? undefined : statsOf(r.before.timesMs), r.after === undefined ? undefined : statsOf(r.after.timesMs))
139  if (r.before === undefined) return '⏱ 基準を保存しました'
140  if (t.deltaPct === undefined) return '⏱ 計測しました'
141  const word = t.verdict === 'faster' ? '速くなった' : t.verdict === 'slower' ? '遅くなった' : t.verdict === 'few' ? '回数が少なく判定できません' : 'ばらつきの範囲内'
142  return `⏱ ${t.deltaPct > 0 ? '+' : ''}${t.deltaPct.toFixed(1)}% ${word} — /perf`
143}
144
145export const register: Register = (on, options) => {
146  const settings: Settings = {
147    // At least MIN_RUNS: with fewer no difference is ever called.
148    runs: Math.max(MIN_RUNS, Math.min(50, Math.round(Number(options.runs ?? 5)) || 5)),
149    timeoutMs: Math.max(5, Math.min(600, Number(options.timeoutSec ?? 300) || 300)) * 1000,
150  }
151
152  on('session.start', async ($, e, next) => {
153    await $.command.register({
154      name: 'perf',
155      description: 'ベンチマークの変更前後を比較（/perf <cmd> · /perf base <cmd> · /perf ref <git-ref> <cmd> · /perf show）',
156      argumentHint: '<cmd> | base <cmd> | ref <git-ref> <cmd> | show | clear [cmd]',
157    })
158    const started = await next(e)
159    // Worktrees of ours a run cut short left: cleaned after the start, not holding it up.
160    const root = (await $.session.repo())?.root
161    if (root !== undefined) $.clock.after(0, () => void dropStale($, root).catch(() => undefined))
162    return started
163  })
164
165  on('command.run', { command: 'perf' }, async ($, e) => {
166    const { mode, ref, command } = parseArgs(e.args)
167    const root = (await $.session.repo())?.root ?? (await $.session.cwd())
168    const cwd = await $.session.cwd()
169    const below = await folderOf($, cwd)
170
171    if (mode === 'show') {
172      const shown = await read($, report)
173      if (shown === null) return { text: '⏱ まだ計測していません。/perf <コマンド> で計測します。' }
174      await $.ui.open({ id: PANE, title: TITLE })
175      return { text: [`⏱ ${shown.command}`, ...reportLines(shown.before, shown.after)].join('\n') }
176    }
177    if (mode === 'clear') {
178      const all = await baselinesOf($)
179      const kept = command === '' ? {} : Object.fromEntries(Object.entries(all).filter(([key]) => key !== baselineKey(root, command, below)))
180      await $.store.set(BASELINES, kept)
181      return { text: `⏱ 基準を${command === '' ? 'すべて' : ''}削除しました。` }
182    }
183    if (command === '') return { text: '⏱ 計測するコマンドを指定してください。' }
184    if (mode === 'ref' && (ref === undefined || !isSafeRef(ref))) return { text: `⏱ git ref として使えない値です: ${ref ?? ''}` }
185    if (isBusy) return { text: '⏱ 計測中です。終わるまでお待ちください。' }
186
187    const key = baselineKey(root, command, below)
188    const saved = (await baselinesOf($))[key]
189    isBusy = true
190    await update($, report, (): PerfReport => ({ status: 'running', command, note: '準備中' }))
191    $.ui.status('⏱ 計測中')
192    $.clock.after(0, () => {
193      void (async () => {
194        try {
195          if (mode === 'base') {
196            const after = await measure($, cwd, command, '基準', settings)
197            await saveBaseline($, key, after)
198            await update($, report, (): PerfReport => ({ status: 'done', command, after }))
199          } else if (mode === 'ref') {
200            const before = await measureRef($, root, below, ref as string, command, settings)
201            const after = await measure($, cwd, command, '現在', settings)
202            await update($, report, (): PerfReport => ({ status: 'done', command, before, after }))
203          } else {
204            const after = await measure($, cwd, command, saved === undefined ? '基準' : '現在', settings)
205            if (saved === undefined) await saveBaseline($, key, after)
206            await update($, report, (): PerfReport => ({ status: 'done', command, ...(saved === undefined ? {} : { before: saved }), after }))
207          }
208          const done = await read($, report)
209          if (done !== null) $.ui.toast(toastOf(done))
210        } catch (err) {
211          await update($, report, (): PerfReport => ({ status: 'failed', command, note: err instanceof Error ? err.message : String(err) }))
212          $.ui.toast('⏱ 計測に失敗しました — /perf show')
213        } finally {
214          isBusy = false
215          $.ui.status(undefined)
216        }
217      })()
218    })
219    await $.ui.open({ id: PANE, title: TITLE })
220    const plan =
221      mode === 'ref' ? `${ref} を一時 worktree で計測してから現在を計測` : mode === 'base' || saved === undefined ? '基準として計測（次回の /perf と比較）' : `保存済みの基準（${new Date(saved.at).toLocaleString()}）と比較`
222    return { text: `⏱ ${command}: ${plan}。ウォームアップ1回 + ${settings.runs}回。` }
223  })
224
225  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
226    const { Box, Text } = $.ui.resolve(e)
227    const shown = await read($, report)
228    if (shown === null) return <Text dimColor>まだ計測していません。</Text>
229    const lines = reportLines(shown.before, shown.after)
230    const color = (line: string) => (line.includes('悪化') ? 'red' : line.includes('改善') ? 'green' : line.includes('⚠') ? 'yellow' : undefined)
231    return (
232      <Box flexDirection="column">
233        <Text bold wrap="truncate-end">{`⏱ ${shown.command}`}</Text>
234        {shown.status === 'running' && <Text color="cyan">{`計測中… ${shown.note ?? ''}`}</Text>}
235        {shown.status === 'failed' && <Text color="red">{`失敗: ${shown.note ?? ''}`}</Text>}
236        {shown.status === 'done' && shown.before === undefined && <Text dimColor>基準として保存しました。変更後にもう一度 /perf を実行すると比較します。</Text>}
237        {lines.map((line, index) => (
238          <Text key={`l-${index}`} color={color(line)} bold={!line.startsWith(' ')} wrap="truncate-end">
239            {line}
240          </Text>
241        ))}
242      </Box>
243    )
244  })
245}
246

1// The comparison, pure: what /usr/bin/time said of a run, the statistics of a side, and
2// whether two sides differ by more than their noise.
3
4import type { PerfSample } from '../types'
5
6/** Wall time (ms) and peak memory (bytes) from `/usr/bin/time -l` (macOS) or `-v` (GNU). */
7export function parseTime(stderr: string): { wallMs?: number; rssBytes?: number } {
8  const out: { wallMs?: number; rssBytes?: number } = {}
9  const bsdReal = stderr.match(/^\s*([\d.]+)\s+real\b/m)
10  if (bsdReal?.[1] !== undefined) out.wallMs = Number(bsdReal[1]) * 1000
11  const bsdRss = stderr.match(/^\s*(\d+)\s+maximum resident set size/m)
12  if (bsdRss?.[1] !== undefined) out.rssBytes = Number(bsdRss[1])
13  const gnuWall = stderr.match(/Elapsed \(wall clock\) time \([^)]*\):\s*(?:(\d+):)?(\d+):([\d.]+)/)
14  if (gnuWall !== null) out.wallMs = (Number(gnuWall[1] ?? 0) * 3600 + Number(gnuWall[2]) * 60 + Number(gnuWall[3])) * 1000
15  const gnuRss = stderr.match(/Maximum resident set size \(kbytes\):\s*(\d+)/)
16  if (gnuRss?.[1] !== undefined) out.rssBytes = Number(gnuRss[1]) * 1024
17  return out
18}
19
20/** The lines /usr/bin/time added, taken off what the command itself wrote to stderr. */
21export function commandStderr(stderr: string): string {
22  return stderr
23    .split('\n')
24    .filter(line => !/^\s*[\d.]+\s+(?:real|user|sys)\b|^\s*\d+\s+[a-z][a-z ()-]+$|^\s*(?:Command being timed|User time|System time|Percent of CPU|Elapsed|Average|Maximum resident|Major|Minor|Voluntary|Involuntary|Swaps|File system|Socket|Signals|Page size|Exit status)\b/.test(line))
25    .join('\n')
26    .trim()
27}
28
29export type Stats = { n: number; median: number; mean: number; stdev: number; cv: number; min: number; max: number }
30
31export function statsOf(values: readonly number[]): Stats | undefined {
32  if (values.length === 0) return undefined
33  const sorted = [...values].sort((a, b) => a - b)
34  const n = sorted.length
35  const mid = Math.floor(n / 2)
36  const median = n % 2 === 1 ? (sorted[mid] as number) : ((sorted[mid - 1] as number) + (sorted[mid] as number)) / 2
37  const mean = sorted.reduce((sum, v) => sum + v, 0) / n
38  const stdev = n < 2 ? 0 : Math.sqrt(sorted.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (n - 1))
39  return { n, median, mean, stdev, cv: mean === 0 ? 0 : (stdev / mean) * 100, min: sorted[0] as number, max: sorted[n - 1] as number }
40}
41
42export type Verdict = 'faster' | 'slower' | 'noise' | 'few' | 'unknown'
43
44/** Runs a side needs before a difference is called: with fewer there is no spread to weigh it by. */
45export const MIN_RUNS = 3
46
47/**
48 * Whether the medians differ by more than the noise: more than twice the larger standard
49 * deviation and more than 1% of the baseline, each side measured at least MIN_RUNS times. A
50 * lower value is better (time, memory).
51 */
52export function verdictOf(before: Stats | undefined, after: Stats | undefined): { verdict: Verdict; deltaPct?: number } {
53  if (before === undefined || after === undefined || before.median === 0) return { verdict: 'unknown' }
54  const delta = after.median - before.median
55  const deltaPct = (delta / before.median) * 100
56  if (before.n < MIN_RUNS || after.n < MIN_RUNS) return { verdict: 'few', deltaPct }
57  const noise = 2 * Math.max(before.stdev, after.stdev)
58  if (Math.abs(delta) <= noise || Math.abs(deltaPct) < 1) return { verdict: 'noise', deltaPct }
59  return { verdict: delta < 0 ? 'faster' : 'slower', deltaPct }
60}
61
62/**
63 * Whether a run that rejected did so at its time limit (else it could not start: a folder gone).
64 * A second's slack: a timer fires a little early, and the wall clock can step.
65 */
66export const isTimeout = (elapsedMs: number, timeoutMs: number): boolean => elapsedMs >= timeoutMs - 1000
67
68export const ms = (v: number): string => (v >= 10_000 ? `${(v / 1000).toFixed(1)}s` : v >= 1000 ? `${(v / 1000).toFixed(2)}s` : `${v.toFixed(v < 10 ? 1 : 0)}ms`)
69export const mb = (v: number): string => `${(v / 1024 / 1024).toFixed(v >= 100 * 1024 * 1024 ? 0 : 1)}MB`
70export const pct = (v: number): string => `${v > 0 ? '+' : ''}${v.toFixed(1)}%`
71
72const WORD: Record<Verdict, string> = { faster: '改善', slower: '悪化', noise: 'ばらつきの範囲内', few: `回数が少なく判定できません（各 ${MIN_RUNS} 回以上。古い基準なら /perf base で取り直してください）`, unknown: '比較できません' }
73
74/** The comparison as lines: one for time, one for memory, and a warning per unsteady side. */
75export function reportLines(before: PerfSample | undefined, after: PerfSample | undefined): string[] {
76  const lines: string[] = []
77  const side = (label: string, s: Stats | undefined, fmt: (v: number) => string) =>
78    s === undefined ? `${label} —` : `${label} 中央値 ${fmt(s.median)} (±${fmt(s.stdev)}, CV ${s.cv.toFixed(1)}%, ${fmt(s.min)}〜${fmt(s.max)}, n=${s.n})`
79  for (const [name, pick, fmt] of [
80    ['時間', (x: PerfSample) => x.timesMs, ms],
81    ['メモリ', (x: PerfSample) => x.rssBytes, mb],
82  ] as const) {
83    const b = before === undefined ? undefined : statsOf(pick(before))
84    const a = after === undefined ? undefined : statsOf(pick(after))
85    if (b === undefined && a === undefined) continue
86    const { verdict, deltaPct } = verdictOf(b, a)
87    lines.push(`${name}: ${deltaPct === undefined ? '' : `${pct(deltaPct)} `}${WORD[verdict]}`)
88    if (before !== undefined) lines.push(`  ${side(before.label, b, fmt)}`)
89    if (after !== undefined) lines.push(`  ${side(after.label, a, fmt)}`)
90    for (const [label, s] of [[before?.label, b], [after?.label, a]] as const) {
91      if (s !== undefined && s.cv > 10) lines.push(`  ⚠ ${label} のばらつきが大きい（CV ${s.cv.toFixed(0)}%）: 回数を増やすか、他の負荷を止めて測り直してください`)
92    }
93  }
94  return lines
95}
96
97/** `/perf` arguments: `base <cmd>`, `ref <git-ref> <cmd>`, `clear`, `show`, or a command. */
98export function parseArgs(args: string): { mode: 'run' | 'base' | 'ref' | 'show' | 'clear'; ref?: string; command: string } {
99  const text = args.trim()
100  const [first = '', ...rest] = text.split(/\s+/)
101  if (first === 'show' || first === '') return { mode: 'show', command: '' }
102  if (first === 'clear') return { mode: 'clear', command: rest.join(' ') }
103  if (first === 'base') return { mode: 'base', command: text.slice(4).trim() }
104  if (first === 'ref' && rest.length >= 2) return { mode: 'ref', ref: rest[0], command: text.slice(4).trim().slice((rest[0] as string).length).trim() }
105  // `ref` with no command, or nothing at all, asks for what is missing: it is no command to run.
106  if (first === 'ref') return rest[0] === undefined ? { mode: 'ref', command: '' } : { mode: 'ref', ref: rest[0], command: '' }
107  return { mode: 'run', command: text }
108}
109
110/** A baseline is the command's in the folder it ran in (below the top, as git names it): at the top the key it always was. */
111export const baselineKey = (root: string, command: string, below = ''): string =>
112  `${root}\u0000${below === '' ? '' : `${below}\u0000`}${command.trim().replace(/\s+/g, ' ')}`
113
114/** A git ref safe to hand to git: no option, no whitespace, no shell or revision-range tricks. */
115export const isSafeRef = (ref: string): boolean => /^[\w][\w./~^@{}-]*$/.test(ref) && !ref.includes('..')
116

1/** One side of a comparison: the measured runs of a command in one tree. */
2export type PerfSample = {
3  /** Where it ran: `HEAD~1` (a worktree), `基準` (a saved baseline) or `現在`. */
4  label: string
5  command: string
6  at: number
7  /** Wall time of each measured run, in milliseconds. */
8  timesMs: number[]
9  /** Peak resident memory of each run, in bytes, where /usr/bin/time reported it. */
10  rssBytes: number[]
11}
12
13/** A /perf comparison, as the pane draws it. */
14export type PerfReport = {
15  status: 'running' | 'done' | 'failed'
16  command: string
17  before?: PerfSample
18  after?: PerfSample
19  /** What is running now, or why it failed. */
20  note?: string
21}
22
23declare module 'claude-code' {
24  interface PluginState {
25    'perf-delta': {
26      report: PerfReport | null
27    }
28  }
29}
30