cache-warmer

1import type { EngineInterface, ModelUsage, Register, Timer } from 'claude-code'
2
3import type { CacheTtl, CacheTtlOption, StopReason, WarmerConfig, WarmerState } from '../types'
4
5const TTL_MS: Record<CacheTtl, number> = { '1h': 60 * 60_000, '5m': 5 * 60_000 }
6
7// The cache clock starts when a request starts, but the mod only sees the turn end.
8// The lead covers the last request's generation time plus one tick.
9const PING_LEAD_MS: Record<CacheTtl, number> = { '1h': 5 * 60_000, '5m': 90_000 }
10
11// Prices as multiples of the model's base input price, from
12// https://platform.claude.com/docs/en/about-claude/pricing (2026-10-15).
13const CACHE_WRITE_RATE: Record<CacheTtl, number> = { '1h': 2, '5m': 1.25 }
14// Claude Code caches a fork at the 5m TTL.
15const FORK_WRITE_RATE = 1.25
16const OUTPUT_RATE = 5
17const READ_RATE_BY_MODEL_PREFIX: readonly (readonly [string, number])[] = [
18  ['claude-fable-5-1', 0.025],
19  ['claude-mythos-5-1', 0.025],
20  ['claude-opus-5-5', 0.05],
21  ['claude-sonnet-5-5', 0.05],
22]
23// Haiku 5.5 and older models. An unknown model gets it too, which stops warming early, not late.
24const DEFAULT_READ_RATE = 0.1
25
26// A ping this close to expiry may land after it and pay a full rewrite.
27const TOO_LATE_MS = 15_000
28const TICK_MS = 15_000
29const PING_PROMPT = 'Reply with one dot.'
30
31const CACHE_TTL_OPTIONS: readonly CacheTtlOption[] = ['auto', '1h', '5m']
32
33export function parseConfig(options: Readonly<Record<string, unknown>>): WarmerConfig {
34  const cacheTtl = options.cacheTtl ?? 'auto'
35  if (!CACHE_TTL_OPTIONS.includes(cacheTtl as CacheTtlOption)) {
36    throw new Error(`cache-warmer: cacheTtl is ${String(cacheTtl)}; valid options are ${CACHE_TTL_OPTIONS.join(', ')}`)
37  }
38  const maxPings = Number(options.maxPings ?? 0)
39  const minContextTokens = Number(options.minContextTokens ?? 20_000)
40  if (!Number.isInteger(maxPings) || maxPings < 0) {
41    throw new Error(`cache-warmer: maxPings is ${String(options.maxPings)}; use a whole number, 0 for the break-even count`)
42  }
43  if (!Number.isFinite(minContextTokens) || minContextTokens < 0) {
44    throw new Error(`cache-warmer: minContextTokens is ${String(options.minContextTokens)}; use a token count of 0 or more`)
45  }
46  return { cacheTtl: cacheTtl as CacheTtlOption, maxPings, minContextTokens }
47}
48
49export function parseTtl(text: string | undefined): CacheTtl | null {
50  const trimmed = text?.trim().toLowerCase()
51  if (trimmed === '1h' || trimmed === '5m') {
52    return trimmed
53  }
54  return null
55}
56
57export function readRateFor(model: string): number {
58  return READ_RATE_BY_MODEL_PREFIX.find(([prefix]) => model.startsWith(prefix))?.[1] ?? DEFAULT_READ_RATE
59}
60
61// In input-token equivalents: what letting the cache lapse costs over keeping it warm.
62export function rewriteCost(state: WarmerState): number {
63  return state.contextTokens * (CACHE_WRITE_RATE[state.ttl] - state.readRate)
64}
65
66export function pingCost(usage: ModelUsage, readRate: number): number {
67  return (
68    usage.cache_read_input_tokens * readRate +
69    usage.cache_creation_input_tokens * FORK_WRITE_RATE +
70    usage.input_tokens +
71    usage.output_tokens * OUTPUT_RATE
72  )
73}
74
75// Pinging past the rewrite cost loses even if the person comes back, so the spend
76// never exceeds one rewrite: at worst twice the cost of a perfect guess.
77function isBudgetSpent(state: WarmerState, config: WarmerConfig): boolean {
78  if (config.maxPings > 0) {
79    return state.pingCount >= config.maxPings
80  }
81  const nextPingCost = state.contextTokens * state.readRate
82  return state.pingSpendInputTokens + nextPingCost > rewriteCost(state)
83}
84
85// A cold cache rewrites the transcript, so it writes more than it reads.
86export function isColdPing(usage: ModelUsage): boolean {
87  return usage.cache_creation_input_tokens > usage.cache_read_input_tokens
88}
89
90export type NextAction = { kind: 'wait' } | { kind: 'ping' } | { kind: 'stop'; reason: StopReason }
91
92// Decided on the clock alone, so a cache that already lapsed (a laptop asleep past expiry)
93// costs nothing to detect.
94export function nextAction(state: WarmerState, config: WarmerConfig, nowMs: number): NextAction {
95  if (!state.isEnabled || state.stopReason !== null || state.isTurnRunning || state.isPinging) {
96    return { kind: 'wait' }
97  }
98  if (state.lastCacheTouchMs === null) {
99    return { kind: 'wait' }
100  }
101  const expiresAtMs = state.lastCacheTouchMs + TTL_MS[state.ttl]
102  if (nowMs >= expiresAtMs - TOO_LATE_MS) {
103    return { kind: 'stop', reason: 'expired-before-ping' }
104  }
105  if (nowMs < expiresAtMs - PING_LEAD_MS[state.ttl]) {
106    return { kind: 'wait' }
107  }
108  if (state.contextTokens < config.minContextTokens) {
109    return { kind: 'stop', reason: 'small-context' }
110  }
111  if (isBudgetSpent(state, config)) {
112    return { kind: 'stop', reason: 'budget-spent' }
113  }
114  return { kind: 'ping' }
115}
116
117export function describeStop(reason: StopReason): string {
118  switch (reason) {
119    case 'budget-spent':
120      return 'pings cost as much as one cache rewrite'
121    case 'expired-before-ping':
122      return 'cache expired before a ping could run'
123    case 'cold-ping':
124      return 'a ping found the cache cold'
125    case 'small-context':
126      return 'context too small to be worth warming'
127    case 'nothing-to-fork':
128      return 'nothing to fork yet'
129    case 'api-error':
130      return 'a ping hit an API error'
131    case 'empty-reply':
132      return 'a ping got an empty reply'
133    case 'aborted':
134      return 'a ping was aborted'
135    case 'turned-off':
136      return 'turned off'
137  }
138}
139
140export function statusText(state: WarmerState, nowMs: number): string | undefined {
141  if (state.stopReason !== null) {
142    return `cache warmer stopped: ${describeStop(state.stopReason)}`
143  }
144  if (!state.isEnabled || state.lastCacheTouchMs === null) {
145    return undefined
146  }
147  const minutesLeft = Math.max(0, Math.round((state.lastCacheTouchMs + TTL_MS[state.ttl] - nowMs) / 60_000))
148  const budget = rewriteCost(state)
149  const spentPercent = budget > 0 ? Math.round((state.pingSpendInputTokens / budget) * 100) : 0
150  return `cache warm ${minutesLeft}m (${state.ttl}) · ${state.pingCount} pings, ${spentPercent}% of a rewrite`
151}
152
153async function resolveTtl($: EngineInterface, option: CacheTtlOption): Promise<CacheTtl> {
154  switch (option) {
155    case '1h':
156    case '5m':
157      return option
158    case 'auto':
159      return detectTtl($)
160  }
161}
162
163// Mirrors Claude Code's own precedence for the main thread's TTL.
164async function detectTtl($: EngineInterface): Promise<CacheTtl> {
165  if (await $.env.get('FORCE_PROMPT_CACHING_5M')) {
166    return '5m'
167  }
168  const envTtl = parseTtl(await $.env.get('CLAUDE_CODE_PROMPT_CACHE_TTL'))
169  if (envTtl !== null) {
170    return envTtl
171  }
172  const settings = await $.settings.read()
173  const settingsTtl = parseTtl(typeof settings.promptCacheTtl === 'string' ? settings.promptCacheTtl : undefined)
174  if (settingsTtl !== null) {
175    return settingsTtl
176  }
177  const identity = await $.session.identity()
178  if (identity.kind === 'claude.ai') {
179    // Past plan usage the session runs on usage credits, which get 5m.
180    const { rateLimits } = await $.session.usage()
181    return rateLimits.some((limit) => limit.percentUsed >= 100) ? '5m' : '1h'
182  }
183  if (await $.env.get('ENABLE_PROMPT_CACHING_1H')) {
184    return '1h'
185  }
186  return '5m'
187}
188
189type Warmer = { config: WarmerConfig; state: WarmerState }
190
191async function showStatus($: EngineInterface, warmer: Warmer) {
192  const nowMs = await $.clock.now()
193  $.ui.status(statusText(warmer.state, nowMs))
194}
195
196async function stop($: EngineInterface, warmer: Warmer, reason: StopReason) {
197  warmer.state.stopReason = reason
198  await $.ui.log(`cache-warmer: ${describeStop(reason)}`)
199}
200
201async function ping($: EngineInterface, warmer: Warmer) {
202  const { state } = warmer
203  state.isPinging = true
204  try {
205    const pingStartMs = await $.clock.now()
206    const fork = await $.model.fork({ prompt: PING_PROMPT })
207    if (!fork.isAnswered) {
208      await stop($, warmer, fork.reason)
209      return
210    }
211    state.pingSpendInputTokens += pingCost(fork.usage, state.readRate)
212    await $.ui.log(
213      `cache-warmer: ping ${state.pingCount + 1} read ${fork.usage.cache_read_input_tokens} wrote ${fork.usage.cache_creation_input_tokens} of ${state.contextTokens}`,
214    )
215    if (isColdPing(fork.usage)) {
216      await stop($, warmer, 'cold-ping')
217      return
218    }
219    state.pingCount += 1
220    state.lastCacheTouchMs = pingStartMs
221  } finally {
222    state.isPinging = false
223  }
224}
225
226async function onTick($: EngineInterface, warmer: Warmer) {
227  const nowMs = await $.clock.now()
228  const action = nextAction(warmer.state, warmer.config, nowMs)
229  switch (action.kind) {
230    case 'wait':
231      break
232    case 'ping':
233      await ping($, warmer)
234      break
235    case 'stop':
236      await stop($, warmer, action.reason)
237      break
238  }
239  await showStatus($, warmer)
240}
241
242export const register: Register = (on, options) => {
243  const config = parseConfig(options)
244  const state: WarmerState = {
245    isEnabled: true,
246    isTurnRunning: false,
247    isPinging: false,
248    lastCacheTouchMs: null,
249    pingCount: 0,
250    pingSpendInputTokens: 0,
251    contextTokens: 0,
252    readRate: DEFAULT_READ_RATE,
253    stopReason: null,
254    ttl: '5m',
255  }
256  const warmer: Warmer = { config, state }
257  let tick: Timer | null = null
258
259  on('session.start', async ($, e, next) => {
260    await $.command.register({
261      name: 'cache-warmer',
262      description: 'Shows the prompt cache warmer, or turns it on or off: /cache-warmer [status|on|off]',
263    })
264    tick?.cancel()
265    // A -p run exits after its turn, so there is no idle cache to keep.
266    if (!e.isInteractive) {
267      return next(e)
268    }
269    state.ttl = await resolveTtl($, config.cacheTtl)
270    tick = $.clock.every(TICK_MS, () => {
271      void onTick($, warmer)
272    })
273    return next(e)
274  })
275
276  on('turn.start', async ($, e, next) => {
277    if (e.agentId === undefined) {
278      state.isTurnRunning = true
279    }
280    return next(e)
281  })
282
283  on('turn.complete', async ($, e, next) => {
284    if (e.agentId !== undefined) {
285      return next(e)
286    }
287    state.isTurnRunning = false
288    // An aborted or failed turn may have sent no request, so it leaves the old touch time.
289    if (e.usage !== undefined) {
290      state.lastCacheTouchMs = await $.clock.now()
291      state.pingCount = 0
292      state.pingSpendInputTokens = 0
293      state.readRate = readRateFor(e.usage.model)
294      // The turn's usage sums its requests; the session's figure is the last request's window.
295      state.contextTokens = (await $.session.usage()).context.tokens ?? 0
296      state.ttl = await resolveTtl($, config.cacheTtl)
297      if (state.stopReason !== 'turned-off') {
298        state.stopReason = null
299      }
300    }
301    await showStatus($, warmer)
302    return next(e)
303  })
304
305  on('command.run', { command: 'cache-warmer' }, async ($, e) => {
306    const verb = e.args.trim() || 'status'
307    if (verb === 'on') {
308      state.isEnabled = true
309      state.stopReason = null
310    } else if (verb === 'off') {
311      state.isEnabled = false
312      state.stopReason = 'turned-off'
313    } else if (verb !== 'status') {
314      return { text: `Unknown argument "${verb}". Valid arguments: status, on, off.` }
315    }
316    await showStatus($, warmer)
317    const nowMs = await $.clock.now()
318    return { text: statusText(state, nowMs) ?? `cache warmer on (${state.ttl}); waiting for the first turn` }
319  })
320}
321

1export type CacheTtl = '1h' | '5m'
2
3export type CacheTtlOption = 'auto' | CacheTtl
4
5export type WarmerConfig = {
6  cacheTtl: CacheTtlOption
7  // 0 means stop once pings cost as much as one cache rewrite.
8  maxPings: number
9  minContextTokens: number
10}
11
12export type StopReason =
13  | 'budget-spent'
14  | 'expired-before-ping'
15  | 'cold-ping'
16  | 'small-context'
17  | 'nothing-to-fork'
18  | 'api-error'
19  | 'empty-reply'
20  | 'aborted'
21  | 'turned-off'
22
23export type WarmerState = {
24  isEnabled: boolean
25  isTurnRunning: boolean
26  isPinging: boolean
27  // When the main thread's cache was last read or written, by a turn or a ping.
28  lastCacheTouchMs: number | null
29  pingCount: number
30  // Input-token equivalents spent on pings since the last turn.
31  pingSpendInputTokens: number
32  // The last main-thread request's window, from the session's usage.
33  contextTokens: number
34  // Cache read price as a multiple of the session model's input price.
35  readRate: number
36  stopReason: StopReason | null
37  ttl: CacheTtl
38}
39