context-cache-meter

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register, TurnUsage } from 'claude-code'
3
4import type { ContextFill, StepCache } from '../types'
5import {
6  cacheRatio,
7  cacheStatus,
8  contextLevel,
9  formatDuration,
10  formatTokens,
11  gaugePercent,
12  isColdStep,
13  meterBar,
14  suspectsShortTtl,
15} from './meter'
16
17const TICK_MS = 30_000
18const CONFIRM_MS = 8_000
19const DEFAULT_TTL_MINUTES = 60
20
21const lastResponseAt = atom({ plugin: 'context-cache-meter', key: 'lastResponseAt' } as const, null)
22const turnStart = atom({ plugin: 'context-cache-meter', key: 'turnStart' } as const, null)
23const coldStep = atom({ plugin: 'context-cache-meter', key: 'coldStep' } as const, null)
24const ttlWarned = atom({ plugin: 'context-cache-meter', key: 'ttlWarned' } as const, false)
25const context = atom({ plugin: 'context-cache-meter', key: 'context' } as const, null)
26const notified = atom({ plugin: 'context-cache-meter', key: 'notified' } as const, 0)
27const compactArmed = atom({ plugin: 'context-cache-meter', key: 'compactArmed' } as const, false)
28const isCompacting = atom({ plugin: 'context-cache-meter', key: 'isCompacting' } as const, false)
29const tick = atom({ plugin: 'context-cache-meter', key: 'tick' } as const, 0)
30
31// The token count at which auto-compaction runs, or undefined when it is off or
32// cannot be read (a fresh session has no breakdown yet): the hints fall back to
33// the model's window then. The summary breakdown is estimated locally, free.
34async function autoCompactMark($: EngineInterface): Promise<number | undefined> {
35  try {
36    const breakdown = (await $.session.usage({ breakdown: 'summary' })).context.breakdown
37    return breakdown?.isAutoCompactEnabled ? breakdown.autoCompactThreshold : undefined
38  } catch {
39    return undefined
40  }
41}
42
43// The fill moved: keep it for the band and announce a threshold once on the
44// way up. A drop (compact, /clear) re-arms the thresholds below it.
45async function takeFill($: EngineInterface, reading: ContextFill) {
46  // Named fields only: a reading with a breakdown must not land in the state.
47  const fill: ContextFill = {
48    tokens: reading.tokens,
49    window: reading.window,
50    percent: reading.percent,
51    compactAt: await autoCompactMark($),
52  }
53  await update($, context, () => fill)
54
55  const gauge = gaugePercent(fill)
56  const level = contextLevel(gauge)
57  const announced = await read($, notified)
58
59  if (level === announced) return
60
61  await update($, notified, () => level)
62
63  if (level > announced) {
64    const percent = Math.round(gauge ?? 0)
65    const reference = fill.compactAt === undefined ? '' : ' of auto-compact'
66    if (level === 90) {
67      $.ui.toast(`Context at ${percent} %${reference} – run /compact now`, {
68        timeoutMs: 10_000,
69      })
70    } else {
71      $.ui.toast(`Context at ${percent} %${reference} – plan /compact soon`, {
72        timeoutMs: 8_000,
73      })
74    }
75  }
76}
77
78// One main-thread request answered: renew the cache clock and note how the
79// cache served it. The first request of a turn is what shows a cold start; the
80// turn's later requests read what it just wrote and would hide it.
81async function recordStep(
82  $: EngineInterface,
83  step: { index: number; usage: TurnUsage | null; sentAt: number },
84  ttlMs: number,
85) {
86  // No response arrived (interrupted, failed): the cache saw nothing.
87  if (step.usage === null) return
88
89  const previous = await read($, lastResponseAt)
90  const finished = await $.clock.now()
91  await update($, lastResponseAt, () => finished)
92
93  const entry: StepCache = {
94    index: step.index,
95    read: step.usage.cache_read_input_tokens,
96    created: step.usage.cache_creation_input_tokens,
97    uncached: step.usage.input_tokens,
98    gapMs: previous === null ? null : Math.max(0, step.sentAt - previous),
99  }
100
101  if (entry.index === 0) await update($, turnStart, () => entry)
102
103  // Cold with no earlier response is just a session's first request: expected.
104  if (entry.gapMs !== null && isColdStep(entry)) {
105    await update($, coldStep, () => entry)
106  } else if (entry.index === 0) {
107    await update($, coldStep, () => null)
108  }
109
110  if (suspectsShortTtl(entry, ttlMs) && !(await read($, ttlWarned))) {
111    await update($, ttlWarned, () => true)
112    $.ui.toast(
113      `Cache cold after ${formatDuration(entry.gapMs ?? 0, 'floor')} – possibly only 5 min lifetime (setting cacheTtlMinutes)`,
114      { timeoutMs: 12_000 },
115    )
116  }
117}
118
119// The compact button: a first press arms it, a second within a few seconds
120// compacts. A compaction cannot be undone, so one stray click must not do it.
121async function pressCompact($: EngineInterface) {
122  if (await read($, isCompacting)) return
123
124  if (!(await read($, compactArmed))) {
125    await update($, compactArmed, () => true)
126    $.clock.after(CONFIRM_MS, () => {
127      void update($, compactArmed, () => false)
128    })
129    return
130  }
131
132  await update($, compactArmed, () => false)
133  await update($, isCompacting, () => true)
134
135  try {
136    const result = await $.session.compact()
137    if ('skip' in result) {
138      $.ui.toast(`Compaction skipped: ${result.skip}`, { timeoutMs: 8_000 })
139    }
140  } catch {
141    $.ui.toast('Compaction failed', { timeoutMs: 8_000 })
142  } finally {
143    await update($, isCompacting, () => false)
144  }
145}
146
147export const register: Register = (on, options) => {
148  const ttlMinutes = Number(options.cacheTtlMinutes)
149  const ttlMs = (ttlMinutes > 0 ? ttlMinutes : DEFAULT_TTL_MINUTES) * 60_000
150
151  let timer: { cancel: () => void } | null = null
152
153  on('session.start', async ($, e, next) => {
154    timer?.cancel()
155    timer = $.clock.every(TICK_MS, () => {
156      void update($, tick, n => n + 1)
157    })
158
159    try {
160      const usage = await $.session.usage()
161      await takeFill($, usage.context)
162    } catch {
163      // No reading yet (fresh session): the first measurement fills it in.
164    }
165
166    return next(e)
167  })
168
169  on('session.end', ($, e, next) => {
170    timer?.cancel()
171    timer = null
172
173    return next(e)
174  })
175
176  on('session.measure', async ($, e, next) => {
177    if (e.changed.includes('context')) {
178      await takeFill($, e.context)
179    }
180
181    return next(e)
182  })
183
184  // Every request of the main thread, not the turn's sum: the sum would show a
185  // turn that started cold and then read its own writes as a cache success.
186  on('turn.step', async function* ($, e, next) {
187    // A subagent's requests have a cache of their own.
188    if (e.agentId !== undefined) return yield* next(e)
189
190    const sentAt = await $.clock.now()
191    const response = yield* next(e)
192
193    try {
194      await recordStep($, { index: e.index, usage: response.usage, sentAt }, ttlMs)
195    } catch {
196      // The band is a convenience; a failed note must not disturb the turn.
197    }
198
199    return response
200  })
201
202  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
203    await read($, tick)
204    const last = await read($, lastResponseAt)
205    const start = await read($, turnStart)
206    const cold = await read($, coldStep)
207    const fill = await read($, context)
208    const armed = await read($, compactArmed)
209    const compacting = await read($, isCompacting)
210
211    if (e.props.hasSurvey) {
212      return next(e)
213    }
214
215    const { Box, Button, Text } = $.ui.resolve(e)
216    const now = await $.clock.now()
217    const cache = cacheStatus(now, last, ttlMs)
218    const percent = gaugePercent(fill)
219    const level = contextLevel(percent)
220    const isWide = e.props.bodyColumns >= 115
221    const compactAt = fill?.compactAt
222
223    const contextColor = level === 90 ? 'error' : level === 70 ? 'warning' : 'success'
224    const cacheColor =
225      e.props.isWorking || cache.state === 'warm'
226        ? 'success'
227        : cache.state === 'expiring'
228          ? 'warning'
229          : 'suggestion'
230
231    const rewrite =
232      fill?.tokens === undefined ? '' : `, ~${formatTokens(fill.tokens)} rewritten`
233    const cacheLabel = e.props.isWorking
234      ? '● active'
235      : cache.state === 'cold'
236        ? last === null
237          ? `○ cold${rewrite}`
238          : `○ cold (since ${formatDuration(cache.idleMs - ttlMs, 'floor')})${rewrite}`
239        : `● warm, ${formatDuration(cache.remainingMs, 'ceil')} left${cache.state === 'expiring' ? ' ⚠' : ''}`
240
241    const startNote =
242      start === null
243        ? null
244        : `Turn start ${cacheRatio(start)} % from cache${
245            start.gapMs === null ? '' : ` (after ${formatDuration(start.gapMs, 'floor')} pause)`
246          }`
247
248    const reference = compactAt === undefined ? '' : ' of auto-compact'
249    const hint =
250      level === 90
251        ? `▲ Context at ${Math.round(percent ?? 0)} %${reference} – run /compact now (or /clear when switching topics).`
252        : level === 70
253          ? `▲ Context at ${Math.round(percent ?? 0)} %${reference} – plan /compact soon, ideally at a task boundary.`
254          : null
255
256    const coldNote =
257      cold === null
258        ? null
259        : `○ Cold cache in last turn${cold.index > 0 ? ` (from step ${cold.index + 1})` : ''}: ~${formatTokens(cold.created)} tokens rewritten${
260            cold.gapMs === null ? '' : `, after ${formatDuration(cold.gapMs, 'floor')} pause`
261          }.`
262    const isTtlSuspected = cold !== null && suspectsShortTtl(cold, ttlMs)
263
264    return (
265      <Box flexDirection="column">
266        <Text>
267          <Text bold>Context </Text>
268          {percent === undefined ? (
269            <Text dimColor>–</Text>
270          ) : (
271            <Text color={contextColor}>
272              {meterBar(percent)} {Math.round(percent)} %{reference}
273            </Text>
274          )}
275          {fill?.tokens !== undefined && (
276            <Text dimColor>
277              {' '}
278              ({formatTokens(fill.tokens)}/{formatTokens(compactAt ?? fill.window)})
279            </Text>
280          )}
281          <Text dimColor>{'  ·  '}</Text>
282          <Text bold>Cache </Text>
283          <Text color={cacheColor}>{cacheLabel}</Text>
284          {isWide && startNote !== null && <Text dimColor>{`  ·  ${startNote}`}</Text>}
285        </Text>
286        {hint !== null && (
287          <Text bold color={contextColor}>
288            {hint}
289          </Text>
290        )}
291        {level === 90 && compacting && <Text dimColor>compacting …</Text>}
292        {level === 90 && !compacting && !e.props.isWorking && (
293          <Box>
294            <Button
295              key="compact"
296              label={armed ? 'Sure? Press again to compact' : 'Compact now'}
297              onPress={() => pressCompact($)}
298            />
299          </Box>
300        )}
301        {coldNote !== null && (
302          <Text>
303            <Text color="suggestion">{coldNote}</Text>
304            {isTtlSuspected && (
305              <Text color="warning">
306                {' '}
307                Already cold after {formatDuration(cold?.gapMs ?? 0, 'floor')} – possibly only 5 min
308                lifetime (setting cacheTtlMinutes).
309              </Text>
310            )}
311          </Text>
312        )}
313      </Box>
314    )
315  })
316}
317

1import type { ContextFill, StepCache } from '../types'
2
3export const WARN_PERCENT = 70
4export const CRITICAL_PERCENT = 90
5
6export type Level = 0 | 70 | 90
7
8export const contextLevel = (percent: number | undefined): Level => {
9  if (percent === undefined) return 0
10  if (percent >= CRITICAL_PERCENT) return 90
11  if (percent >= WARN_PERCENT) return 70
12  return 0
13}
14
15/**
16 * How full the context is for the hints: tokens over the auto-compact mark when
17 * there is one (the point that matters), else the engine's percent of the window.
18 */
19export const gaugePercent = (fill: ContextFill | null): number | undefined => {
20  if (fill === null) return undefined
21  if (fill.compactAt !== undefined && fill.compactAt > 0 && fill.tokens !== undefined) {
22    return (fill.tokens / fill.compactAt) * 100
23  }
24  return fill.percent
25}
26
27export const meterBar = (percent: number, cells = 10): string => {
28  const clamped = Math.min(100, Math.max(0, percent))
29  const filled = Math.round((clamped / 100) * cells)
30  return '█'.repeat(filled) + '░'.repeat(cells - filled)
31}
32
33export const formatTokens = (tokens: number): string => {
34  if (tokens < 1000) return String(tokens)
35  if (tokens < 1_000_000) return `${Math.round(tokens / 1000)}k`
36  return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '').replace('.', ',')}M`
37}
38
39export const formatDuration = (ms: number, round: 'floor' | 'ceil'): string => {
40  const minutes = Math[round](ms / 60_000)
41  if (minutes < 1) return '<1 min'
42  if (minutes < 60) return `${minutes} min`
43  const hours = Math.floor(minutes / 60)
44  const rest = minutes % 60
45  return rest === 0 ? `${hours} hr` : `${hours} hr ${rest} min`
46}
47
48export type CacheState = 'cold' | 'warm' | 'expiring'
49
50export type CacheStatus = {
51  state: CacheState
52  remainingMs: number
53  idleMs: number
54}
55
56/**
57 * Warm while the last API response is younger than the cache's lifetime; every
58 * response that reads the cache renews it, so the clock restarts per response.
59 * `expiring`: the last fifth of the lifetime (at least one minute).
60 */
61export const cacheStatus = (
62  now: number,
63  lastResponseAt: number | null,
64  ttlMs: number,
65): CacheStatus => {
66  if (lastResponseAt === null) return { state: 'cold', remainingMs: 0, idleMs: 0 }
67
68  const idleMs = Math.max(0, now - lastResponseAt)
69  const remainingMs = ttlMs - idleMs
70
71  if (remainingMs <= 0) return { state: 'cold', remainingMs: 0, idleMs }
72
73  const isEnding = remainingMs <= Math.max(60_000, ttlMs * 0.2)
74  return { state: isEnding ? 'expiring' : 'warm', remainingMs, idleMs }
75}
76
77/** The API's default cache lifetime: a cache gone sooner than this is not the lifetime's doing. */
78export const SHORT_TTL_MS = 5 * 60_000
79
80/** Share of a request's input that the cache served, as a whole percentage. */
81export const cacheRatio = (step: StepCache): number =>
82  Math.round((step.read / Math.max(1, step.read + step.created + step.uncached)) * 100)
83
84/** A request that wrote much and read little: the cache was not there. */
85export const isColdStep = (step: StepCache): boolean =>
86  step.created >= 1000 && step.read < 0.5 * (step.read + step.created + step.uncached)
87
88/**
89 * The cache was cold after a pause the assumed lifetime says it should have
90 * survived, yet longer than the shortest lifetime there is: the lifetime may
91 * really be shorter. Other causes (model switch, /compact, changed tools) look
92 * the same, so this is a suspicion, never a measurement.
93 */
94export const suspectsShortTtl = (step: StepCache, ttlMs: number): boolean =>
95  step.gapMs !== null && step.gapMs >= SHORT_TTL_MS && step.gapMs < ttlMs && isColdStep(step)
96

1/**
2 * What one main-thread model request did to the prompt cache, as the API
3 * reported it. `gapMs`: how long the cache had sat unused, from the previous
4 * response's end to this request's start; null when no earlier response is known.
5 */
6export type StepCache = {
7  index: number
8  read: number
9  created: number
10  uncached: number
11  gapMs: number | null
12}
13
14/**
15 * `compactAt`: the token count at which auto-compaction runs, absent when it is
16 * off (or not readable); the hints measure against it, else against `window`.
17 */
18export type ContextFill = {
19  tokens?: number
20  window: number
21  percent?: number
22  compactAt?: number
23}
24
25declare module 'claude-code' {
26  interface PluginState {
27    'context-cache-meter': {
28      /** ms since the epoch of the last main-thread response that reached the API. */
29      lastResponseAt: number | null
30      /** The first request of the latest turn: how the turn started against the cache. */
31      turnStart: StepCache | null
32      /** The latest request that found the cache cold after a known pause; cleared when a turn starts warm. */
33      coldStep: StepCache | null
34      /** Whether the "cache died earlier than the lifetime assumes" toast was shown this session. */
35      ttlWarned: boolean
36      /** The context fill as `session.measure` last reported it, with the auto-compact mark. */
37      context: ContextFill | null
38      /** Highest threshold (0, 70 or 90) already announced for the current fill. */
39      notified: number
40      /** The compact button was pressed once and waits for the confirming press. */
41      compactArmed: boolean
42      /** A compaction started from the button is running. */
43      isCompacting: boolean
44      /** Counts the clock's ticks so the band redraws and the countdown moves. */
45      tick: number
46    }
47  }
48}
49