cache-stats

1import { atom, read, update } from 'claude-code'
2import type { Register } from 'claude-code'
3
4import { addRequest, bar, emptySession, fmtDuration, fmtLeft, fmtPercent, fmtTokens, report, summarize } from './stats'
5import type { Ttl } from './stats'
6
7const session = atom({ plugin: 'cache-stats', key: 'session' } as const, emptySession())
8const tick = atom({ plugin: 'cache-stats', key: 'tick' } as const, 0)
9const isHidden = atom({ plugin: 'cache-stats', key: 'isHidden' } as const, false)
10
11// How often the countdown redraws. Fine enough for minutes, cheap enough to leave on.
12const TICK_MS = 15_000
13const USAGE = 'Usage: /cache [hide|show]'
14
15type Segment = { text: string; color?: string }
16
17export const register: Register = (on, options) => {
18  const ttl: Ttl = options.ttl === '5m' ? '5m' : '1h'
19
20  on('session.start', async ($, e, next) => {
21    await $.command
22      .register({
23        name: 'cache',
24        description: 'Prompt cache: time left, hit rate, misses and why, session time',
25        argumentHint: '[hide|show]',
26      })
27      .catch(() => {}) // a name Claude Code already has is refused: the band still works
28    const now = await $.clock.now()
29    await update($, session, s => ({ ...s, startedAt: s.startedAt ?? now }))
30    $.clock.every(TICK_MS, () => void update($, tick, n => n + 1).catch(() => {}))
31    return next(e)
32  })
33
34  // One stop chunk per request to the model, with that request's own usage.
35  // Subagents run on caches of their own, so only the main loop is counted.
36  on('turn.step', async function* ($, e, next) {
37    const stream = next(e)
38    for await (const chunk of stream) {
39      if (chunk.kind === 'stop' && chunk.usage && !e.agentId) {
40        const u = chunk.usage
41        const now = await $.clock.now()
42        await update($, session, s =>
43          addRequest(
44            s,
45            { model: u.model, input: u.input_tokens, read: u.cache_read_input_tokens, created: u.cache_creation_input_tokens, output: u.output_tokens },
46            now,
47            ttl,
48          ),
49        ).catch(() => {})
50      }
51      yield chunk
52    }
53    return await stream.result
54  })
55
56  on('turn.complete', async ($, e, next) => {
57    if (!e.agentId) {
58      await update($, session, s => ({ ...s, turns: s.turns + 1, workMs: s.workMs + e.durationMs })).catch(() => {})
59    }
60    return next(e)
61  })
62
63  on('command.run', { command: 'cache' }, async ($, e) => {
64    const word = e.args.trim().toLowerCase()
65    if (word === 'hide') {
66      await update($, isHidden, () => true)
67      return { text: 'Cache line hidden. /cache show brings it back.' }
68    }
69    if (word === 'show') {
70      await update($, isHidden, () => false)
71      return { text: 'Cache line shown.' }
72    }
73    if (word !== '') return { text: USAGE }
74    return { text: report(await read($, session), await $.clock.now(), ttl) }
75  })
76
77  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
78    const data = await read($, session)
79    await read($, tick) // subscribes the band to the timer
80    if (e.props.hasSurvey || data.total.requests === 0 || (await read($, isHidden))) return next(e)
81
82    const { Box, Text } = $.ui.resolve(e)
83    const now = await $.clock.now()
84    const s = summarize(data, now, ttl)
85    const tone = (hit: number | null) => (hit === null ? 'inactive' : hit >= 0.8 ? 'success' : hit >= 0.5 ? 'warning' : 'error')
86
87    const segments: Segment[] = [
88      { text: `${bar(s.lastHit ?? 0)} ${fmtPercent(s.lastHit)} hit`, color: tone(s.lastHit) },
89      s.isWarm
90        ? { text: `warm ${fmtLeft(s.leftMs ?? 0)}`, color: (s.leftMs ?? 0) < 60_000 ? 'warning' : 'success' }
91        : { text: `cold, next message re-caches ${fmtTokens(s.cached)}`, color: 'error' },
92      { text: `session ${fmtDuration(s.sessionMs)} (${fmtDuration(data.workMs)} working)` },
93      { text: `avg ${fmtPercent(s.sessionHit)}`, color: 'inactive' },
94      { text: `${data.total.requests} requests`, color: 'inactive' },
95    ]
96    if (s.isWarm) segments.splice(2, 0, { text: `${fmtTokens(s.cached)} cached`, color: 'inactive' })
97
98    // Drop the least important segments from the end until the line fits.
99    const room = Math.max(20, (e.props.bodyColumns ?? 100) - 10)
100    const shown = [...segments]
101    const width = () => shown.reduce((sum, seg) => sum + seg.text.length + 3, 0)
102    while (shown.length > 2 && width() > room) shown.pop()
103
104    return (
105      <Box>
106        <Text color="claude">◆ Cache </Text>
107        {shown.map((seg, i) => (
108          <Text color={seg.color}>
109            {i > 0 ? ' · ' : ''}
110            {seg.text}
111          </Text>
112        ))}
113      </Box>
114    )
115  })
116}
117

1import type { CacheMiss, CacheRequest, CacheSession } from '../types'
2
3// The numbers behind the band and /cache. Pure: no engine, so it is tested alone.
4
5/** A prompt smaller than this is not worth calling a miss over. */
6const MISS_FLOOR_TOKENS = 5000
7/** Below this share of the prompt served from cache, a request counts as a miss. */
8const MISS_BELOW = 0.5
9const KEEP_RECENT = 20
10const KEEP_MISSES = 50
11
12export const TTL_MS = { '5m': 5 * 60_000, '1h': 60 * 60_000 } as const
13export type Ttl = keyof typeof TTL_MS
14
15export const emptySession = (): CacheSession => ({
16  startedAt: null,
17  workMs: 0,
18  turns: 0,
19  total: { requests: 0, input: 0, read: 0, created: 0, output: 0 },
20  recent: [],
21  misses: [],
22  lastAt: null,
23})
24
25/** The prompt a request sent: everything it read, wrote or sent uncached. */
26export const promptTokens = (r: Pick<CacheRequest, 'input' | 'read' | 'created'>) => r.input + r.read + r.created
27
28/** The share of the prompt the cache served, 0 to 1; null for an empty prompt. */
29export function hitRate(r: Pick<CacheRequest, 'input' | 'read' | 'created'>): number | null {
30  const prompt = promptTokens(r)
31  return prompt === 0 ? null : r.read / prompt
32}
33
34type Usage = { model: string; input: number; read: number; created: number; output: number }
35
36/** The session after one more request. A miss is a big prompt the cache mostly did not serve. */
37export function addRequest(session: CacheSession, usage: Usage, now: number, ttl: Ttl): CacheSession {
38  const gapMs = session.lastAt === null ? null : Math.max(0, now - session.lastAt)
39  const request: CacheRequest = { at: now, gapMs, ...usage }
40  const hit = hitRate(request)
41  const isMiss = gapMs !== null && hit !== null && hit < MISS_BELOW && promptTokens(request) >= MISS_FLOOR_TOKENS
42  const miss: CacheMiss[] = isMiss
43    ? [{ at: now, hit: hit ?? 0, created: usage.created, gapMs, reason: gapMs > TTL_MS[ttl] ? 'expired' : 'changed' }]
44    : []
45  return {
46    ...session,
47    startedAt: session.startedAt ?? now,
48    lastAt: now,
49    total: {
50      requests: session.total.requests + 1,
51      input: session.total.input + usage.input,
52      read: session.total.read + usage.read,
53      created: session.total.created + usage.created,
54      output: session.total.output + usage.output,
55    },
56    recent: [...session.recent, request].slice(-KEEP_RECENT),
57    misses: [...session.misses, ...miss].slice(-KEEP_MISSES),
58  }
59}
60
61export type Summary = {
62  /** The last request's hit rate and the session's. */
63  lastHit: number | null
64  sessionHit: number | null
65  /** Tokens in the cache now: what the last request's prompt held. */
66  cached: number
67  /** Milliseconds left before the cache lapses; 0 once it has. null before the first request. */
68  leftMs: number | null
69  isWarm: boolean
70  sessionMs: number
71}
72
73export function summarize(session: CacheSession, now: number, ttl: Ttl): Summary {
74  const last = session.recent[session.recent.length - 1]
75  const all = session.total.input + session.total.read + session.total.created
76  const leftMs = session.lastAt === null ? null : Math.max(0, session.lastAt + TTL_MS[ttl] - now)
77  return {
78    lastHit: last ? hitRate(last) : null,
79    sessionHit: all === 0 ? null : session.total.read / all,
80    cached: last ? promptTokens(last) : 0,
81    leftMs,
82    isWarm: leftMs !== null && leftMs > 0,
83    sessionMs: session.startedAt === null ? 0 : Math.max(0, now - session.startedAt),
84  }
85}
86
87export function fmtTokens(n: number): string {
88  if (n < 1000) return String(Math.round(n))
89  if (n < 1_000_000) return `${Math.round(n / 1000)}k`
90  return `${(n / 1_000_000).toFixed(1)}M`
91}
92
93export function fmtDuration(ms: number): string {
94  const s = Math.round(ms / 1000)
95  if (s < 60) return `${s}s`
96  const m = Math.floor(s / 60)
97  if (m < 60) return `${m}m`
98  const h = Math.floor(m / 60)
99  return `${h}h ${String(m % 60).padStart(2, '0')}m`
100}
101
102/** The time left on a warm cache: seconds under a minute, minutes after. */
103export const fmtLeft = (ms: number) => (ms < 60_000 ? `${Math.ceil(ms / 1000)}s` : `${Math.ceil(ms / 60_000)}m`)
104
105export const fmtPercent = (share: number | null) => (share === null ? '-' : `${Math.round(share * 100)}%`)
106
107export function fmtClock(at: number): string {
108  const d = new Date(at)
109  return `${String(d.getHours()).padStart(2, '0')}:${String(d.getMinutes()).padStart(2, '0')}`
110}
111
112/** A bar of `width` cells filled to `share`. */
113export function bar(share: number, width = 10): string {
114  const filled = Math.max(0, Math.min(width, Math.round(share * width)))
115  return '█'.repeat(filled) + '░'.repeat(width - filled)
116}
117
118/** The /cache text: what the cache is doing now, how the session went, and every miss with its cause. */
119export function report(session: CacheSession, now: number, ttl: Ttl): string {
120  if (session.total.requests === 0) return 'No requests yet: the numbers start with the next reply.'
121  const s = summarize(session, now, ttl)
122  const t = session.total
123  const lines: string[] = []
124
125  lines.push(
126    s.isWarm
127      ? `Cache is warm: about ${fmtLeft(s.leftMs ?? 0)} left (${ttl} lifetime), holding ${fmtTokens(s.cached)} tokens.`
128      : `Cache is cold (${ttl} lifetime): the next message re-caches about ${fmtTokens(s.cached)} tokens.`,
129  )
130  lines.push(
131    `Session: ${fmtDuration(s.sessionMs)}, ${fmtDuration(session.workMs)} of it the model working, ${session.turns} turns, ${t.requests} requests.`,
132  )
133  lines.push(
134    `Hit rate: ${fmtPercent(s.sessionHit)} of the session (last request ${fmtPercent(s.lastHit)}). ` +
135      `${fmtTokens(t.read)} tokens read from cache, ${fmtTokens(t.created)} written, ${fmtTokens(t.input)} sent uncached, ${fmtTokens(t.output)} generated.`,
136  )
137
138  if (session.misses.length === 0) {
139    lines.push('Misses: none. Every large request was served from the cache.')
140  } else {
141    lines.push(`Misses (${session.misses.length}), a large prompt that mostly missed the cache:`)
142    for (const m of session.misses.slice(-8)) {
143      const why =
144        m.reason === 'expired'
145          ? `idle ${fmtDuration(m.gapMs ?? 0)}, longer than the ${ttl} the cache lives`
146          : `only ${fmtDuration(m.gapMs ?? 0)} after the last request, so the prompt or model changed (/model, /compact, an edited CLAUDE.md or a new MCP tool)`
147      lines.push(`  ${fmtClock(m.at)}  hit ${fmtPercent(m.hit)}, re-cached ${fmtTokens(m.created)}: ${why}`)
148    }
149  }
150
151  lines.push('Last requests:')
152  for (const r of session.recent.slice(-8)) {
153    lines.push(
154      `  ${fmtClock(r.at)}  ${fmtPercent(hitRate(r)).padStart(4)} hit  read ${fmtTokens(r.read).padStart(5)}  wrote ${fmtTokens(r.created).padStart(5)}  new ${fmtTokens(r.input).padStart(5)}  out ${fmtTokens(r.output).padStart(5)}  ${r.model}`,
155    )
156  }
157  lines.push('Subagents have caches of their own and are not counted here.')
158  return lines.join('\n')
159}
160

1/** One request to the model, as the API reported its usage. */
2export type CacheRequest = {
3  at: number
4  model: string
5  /** Prompt tokens neither read from nor written to the cache. */
6  input: number
7  /** Prompt tokens the cache served. */
8  read: number
9  /** Prompt tokens written to the cache. */
10  created: number
11  output: number
12  /** Time since the previous request, null for the first. */
13  gapMs: number | null
14}
15
16/** A request that should have hit the cache and mostly did not. */
17export type CacheMiss = {
18  at: number
19  /** Share of the prompt the cache served, 0 to 1. */
20  hit: number
21  /** Tokens written to the cache again. */
22  created: number
23  gapMs: number | null
24  /** `expired`: idle longer than the cache lives. `changed`: the prompt or model changed under it. */
25  reason: 'expired' | 'changed'
26}
27
28export type CacheSession = {
29  startedAt: number | null
30  /** Time the model spent on turns, summed. */
31  workMs: number
32  turns: number
33  total: { requests: number; input: number; read: number; created: number; output: number }
34  recent: CacheRequest[]
35  misses: CacheMiss[]
36  lastAt: number | null
37}
38
39declare module 'claude-code' {
40  interface PluginState {
41    'cache-stats': {
42      session: CacheSession
43      tick: number
44      isHidden: boolean
45    }
46  }
47}
48