cache-keepalive

1import type { EngineInterface, Register } from 'claude-code'
2import type { KeepaliveTiming } from '../types'
3
4// Keeps the main conversation's prompt cache warm while the session sits open and idle: a short
5// prompt LEAD_MS before the cache would expire. The cache lifetime runs from the start of the
6// main thread's last answered request, and each such request resets it; a request with no
7// response (interrupted before its first token, or dead on an API error) leaves it as it was.
8// Bumps stop at the model's cap; the last one asks the agent to close the session (a closing skill, a handoff skill, or a handoff it writes) while the cache is still warm.
9// The person's own next prompt resets the count. A bump that finds the cache already cold has
10// just rebuilt it, so the bumps go on; COLD_LIMIT cold bumps in a row (the TTL dropped to 5m
11// past plan usage, so every bump lands after expiry) stop them until the person is back. So
12// does a change of account (`claude auth status`, polled each tick while the cache may be warm;
13// a read that a prompt or a turn overlapped is discarded, and a failed read changes nothing):
14// the prompt cache is the account's own, so the next request under another account rebuilds it
15// whole. A bump that got no response left the cache as it was; it is not sent again, and the
16// status line says so. `/keepalive off` stops them for the conversation, a typed prompt
17// included, until `/keepalive on`.
18//
19// A compaction replaces the conversation, so the cache holds none of what the next request sends:
20// the timing starts over and no bump comes until a request writes the new conversation; a request
21// still running when the conversation was replaced (a compaction, /clear) times nothing. The bump
22// after a conversation's first request is never counted cold: Claude Code sends that request's
23// marked system message as a block list and, unmarked on the next request, as a plain string, so
24// the next request misses the cache past the system prompt however warm it is.
25//
26// Every bump is submitted asUser, so the model reads the bare text without the engine's
27// "The cache-keepalive plugin sent a message" frame. A bump is still not a rewind point: the
28// rewind list holds the person's own prompts, and asUser keeps the plugin origin.
29//
30// Caps sit below each model's break-even (pings that cost as much as one cold rebuild):
31// Sonnet 5.5 19, Opus 5.5 39, Fable 5.1 79.
32
33export const CAPS: ReadonlyArray<readonly [string, number]> = [
34  ['fable', 12],
35  ['opus', 12],
36  ['sonnet', 6],
37  ['haiku', 6],
38]
39const DEFAULT_CAP = 6
40export const LEAD_MS = 2 * 60_000
41export const HEADROOM_MS = 15_000 // the latest a bump may go: request latency before expiry
42export const TICK_MS = 30_000
43export const COLD_LIMIT = 2
44export const COMMAND = 'keepalive'
45const TIMING = { plugin: 'cache-keepalive', key: 'timing' } as const
46export const AUTH_STATUS = ['claude', 'auth', 'status', '--json'] as const
47const PERSON_ORIGINS: ReadonlySet<string> = new Set(['composer', 'bridge', 'slack-ping'])
48
49export function capFor(model: string): number {
50  const m = model.toLowerCase()
51  return CAPS.find(([family]) => m.includes(family))?.[1] ?? DEFAULT_CAP
52}
53
54export function bumpText(n: number, cap: number): string {
55  return n < cap
56    ? 'keepalive ping. Reply only: ok'
57    : 'Last keepalive ping: the prompt cache expires after this. ' +
58        'Close the session now, so a fresh session can pick up from it: run a session-closing ' +
59        'skill with the Skill tool if one is installed, else a handoff skill, else write a ' +
60        'handoff yourself. Do nothing else.'
61}
62
63// `/keepalive off|on`: the off flag after the args, and the line the command prints.
64export function keepaliveCommand(args: string, isOff: boolean): { isOff: boolean; text: string } {
65  const arg = args.trim().toLowerCase()
66  if (arg === 'off') return { isOff: true, text: 'keepalive off: no pings in this conversation until /keepalive on' }
67  if (arg === 'on') return { isOff: false, text: 'keepalive on: pings resume from your next request' }
68  return { isOff, text: `keepalive is ${isOff ? 'off' : 'on'}; /keepalive off or /keepalive on` }
69}
70
71// Whether a bump is due: idle, not stopped or off, and the last main request's cache inside the
72// window from LEAD_MS to HEADROOM_MS before it expires (past that the bump would rebuild a cold
73// cache), with no bump sent yet for that request (a bump's own request moves lastSend on).
74export function isDue(s: Pick<KeepaliveTiming, 'lastSend' | 'bumpedFor' | 'isStopped' | 'isOff'> & { now: number; ttl: number; isBusy: boolean }): boolean {
75  const expiresAt = s.lastSend + s.ttl
76  return (
77    !s.isBusy &&
78    !s.isStopped &&
79    !s.isOff &&
80    s.lastSend > 0 &&
81    s.bumpedFor !== s.lastSend &&
82    s.now >= expiresAt - LEAD_MS &&
83    s.now < expiresAt - HEADROOM_MS
84  )
85}
86
87// A bump that wrote more of the prompt to the cache than it read found the cache cold.
88export function wasCold(usage: { cache_read_input_tokens: number; cache_creation_input_tokens: number }): boolean {
89  return usage.cache_read_input_tokens < usage.cache_creation_input_tokens
90}
91
92// Whether the latest bump answered the conversation's first request, whose cache entry the bump
93// cannot read whatever its age.
94export function followsFirstRequest(t: Pick<KeepaliveTiming, 'bumpedFor' | 'firstSend'>): boolean {
95  return t.firstSend > 0 && t.bumpedFor === t.firstSend
96}
97
98// Whether a compaction replaced the main conversation: not a subagent's own, not a skipped one,
99// and not a `precompute`, which only keeps its summary for later.
100export function isConversationReplaced(e: { trigger: string; agentId?: string }, result: { skip?: string }): boolean {
101  return e.agentId === undefined && e.trigger !== 'precompute' && result.skip === undefined
102}
103
104// Whether the account the session would send under now is another than the one the cache was
105// written under. An unknown account on either side (logged out, a failed read) is no change.
106export function isAccountChanged(cachedUnder: string, current: string): boolean {
107  return cachedUnder !== '' && current !== '' && cachedUnder !== current
108}
109
110export function accountChangedText(from: string, to: string): string {
111  return `keepalive: account changed (${from} → ${to}), cache cold; stopped until your next prompt`
112}
113
114const fresh = (): KeepaliveTiming => ({ lastSend: 0, bumpedFor: 0, firstSend: 0, bumps: 0, coldBumps: 0, model: '', account: '', isStopped: false, isOff: false })
115
116// The logged-in account's email from `claude auth status --json`; '' when it says logged out,
117// undefined when it says neither (not JSON, or logged in with no email).
118export function accountOf(authStatus: string): string | undefined {
119  try {
120    const { loggedIn, email } = JSON.parse(authStatus) as { loggedIn?: unknown; email?: unknown }
121    if (loggedIn === false) return ''
122    return loggedIn === true && typeof email === 'string' && email !== '' ? email : undefined
123  } catch {
124    return undefined
125  }
126}
127
128// The account this session sends under, '' when logged out; undefined when the read failed or named no account.
129async function currentAccount($: EngineInterface): Promise<string | undefined> {
130  try {
131    const { exitCode, stdout } = await $.process.run(AUTH_STATUS, { timeoutMs: 5_000 })
132    return exitCode === 0 ? accountOf(stdout) : undefined
133  } catch {
134    return undefined
135  }
136}
137
138// One load's timing writes: how many went out, and the newest of them that landed.
139type Writes = { issued: number; newestLanded: number }
140
141// Stores the timing `live` answers, whole. A write that lands after a later one landed (a
142// state.set hook held it) put an older timing over a newer one, so it writes the timing again.
143async function save($: EngineInterface, writes: Writes, live: () => KeepaliveTiming): Promise<void> {
144  const write = ++writes.issued
145  await $.state.set(TIMING, { ...live() })
146  const isOvertaken = write < writes.newestLanded
147  writes.newestLanded = Math.max(writes.newestLanded, write)
148  if (isOvertaken) await save($, writes, live)
149}
150
151export const register: Register = on => {
152  // The timing lives here for synchronous checks and is written through to $.state, which
153  // keeps it across a hot reload; session.start reads it back.
154  let t = fresh()
155  const live = () => t
156  const writes: Writes = { issued: 0, newestLanded: 0 }
157  let isBusy = false
158  let isBumpPending = false // a bump was submitted and its turn has not started
159  let isBumpTurn = false // the running main turn is a bump
160  let tick: { cancel: () => void } | undefined
161  let epoch = 0
162
163  on('session.start', async ($, e, next) => {
164    tick?.cancel()
165    tick = undefined
166    if (e.isInteractive) {
167      const { value } = await $.state.get(TIMING)
168      if (value) t = { ...fresh(), ...value }
169      $.ui.status(t.isOff ? 'keepalive off' : undefined)
170      // Claude Code's main-conversation TTL: 1h on a subscription within plan usage, 5m when
171      // set so. Past plan usage it drops to 5m on its own; the first bump then finds the cache
172      // cold and stops the rest (turn.complete below).
173      const force = await $.env.get('FORCE_PROMPT_CACHING_5M')
174      const is5m = force === '1' || force === 'true' || (await $.env.get('CLAUDE_CODE_PROMPT_CACHE_TTL')) === '5m'
175      const ttl = (is5m ? 5 : 60) * 60_000
176
177      tick = $.clock.every(TICK_MS, async () => {
178        const polledAt = await $.clock.now()
179        // Polled only while a bump could still come: idle, not stopped or off, cache not expired.
180        if (!isBusy && !t.isStopped && !t.isOff && t.lastSend > 0 && polledAt < t.lastSend + ttl) {
181          const epochAtRead = epoch
182          const account = await currentAccount($)
183          if (epoch !== epochAtRead) return
184          if (account !== undefined && isAccountChanged(t.account, account)) {
185            t.isStopped = true
186            $.ui.status(accountChangedText(t.account, account))
187            await save($, writes, live)
188            return
189          }
190        }
191        const now = await $.clock.now()
192        if (!isDue({ ...t, now, ttl, isBusy })) return
193        t.bumpedFor = t.lastSend
194        const cap = capFor(t.model)
195        if (t.bumps >= cap) {
196          $.ui.status(`keepalive: done (${cap}/${cap}), cache left to expire`)
197          await save($, writes, live)
198          return
199        }
200        t.bumps++
201        isBumpPending = true
202        $.ui.status(`keepalive ${t.bumps}/${cap}`)
203        try {
204          await $.prompt.submit({ text: bumpText(t.bumps, cap), asUser: true })
205        } catch {
206          isBumpPending = false
207          $.ui.status('keepalive: submit failed')
208        }
209        await save($, writes, live)
210      })
211      // After the timer, so a refused registration costs the command, not the bumps.
212      await $.command.register({
213        name: COMMAND,
214        description: 'Stop or resume the prompt-cache keepalive pings for this conversation',
215        argumentHint: '[off|on]',
216        immediate: true,
217      })
218    }
219    return next(e)
220  })
221
222  on('command.run', { command: COMMAND }, async ($, e) => {
223    const { isOff, text } = keepaliveCommand(e.args, t.isOff)
224    t.isOff = isOff
225    $.ui.status(isOff ? 'keepalive off' : undefined)
226    await save($, writes, live)
227    return { text }
228  })
229
230  on('prompt.submit', async ($, e, next) => {
231    if (PERSON_ORIGINS.has(e.origin.kind)) {
232      epoch++
233      t.bumps = 0
234      t.coldBumps = 0
235      t.isStopped = false
236      isBumpPending = false
237      $.ui.status(t.isOff ? 'keepalive off' : undefined)
238      await save($, writes, live)
239    }
240    return next(e)
241  })
242
243  on('turn.start', async ($, e, next) => {
244    if (e.agentId) return next(e)
245    epoch++
246    isBusy = true
247    isBumpTurn = isBumpPending
248    isBumpPending = false
249    // Any main turn that is not a bump is the person's: the bump count leaves the status line.
250    if (!isBumpTurn) $.ui.status(t.isOff ? 'keepalive off' : undefined)
251    return next(e)
252  })
253
254  on('turn.step', async function* ($, e, next) {
255    if (e.agentId) return yield* next(e)
256    const timing = t
257    const startedAt = await $.clock.now()
258    const read = currentAccount($)
259    let isAnswered = false
260    try {
261      const stream = next(e)
262      for await (const chunk of stream) {
263        if (chunk.kind !== 'engine') isAnswered = true
264        yield chunk
265      }
266      const result = await stream.result
267      if (result.stopReason !== null) isAnswered = true
268      return result
269    } finally {
270      if (isAnswered) {
271        if (timing.lastSend === 0) timing.firstSend = startedAt
272        timing.lastSend = startedAt
273        timing.model = e.model
274        const account = await read
275        if (account !== undefined) timing.account = account
276        if (timing === t) await save($, writes, live)
277      }
278    }
279  })
280
281  on('turn.complete', async ($, e, next) => {
282    if (!e.agentId) {
283      isBusy = false
284      if (isBumpTurn && t.lastSend > 0 && t.lastSend === t.bumpedFor) {
285        $.ui.status('keepalive: ping got no response, cache left to expire')
286      } else if (isBumpTurn && e.usage && !followsFirstRequest(t)) {
287        t.coldBumps = wasCold(e.usage) ? t.coldBumps + 1 : 0
288        if (t.coldBumps >= COLD_LIMIT) {
289          t.isStopped = true
290          $.ui.status(`keepalive: cache cold ${COLD_LIMIT} pings in a row, stopped until your next prompt`)
291        } else if (t.coldBumps > 0) {
292          $.ui.status('keepalive: cache was cold, rebuilt; pings go on')
293        }
294        await save($, writes, live)
295      }
296      isBumpTurn = false
297    }
298    return next(e)
299  })
300
301  on('session.compact', async ($, e, next) => {
302    const result = await next(e)
303    if (isConversationReplaced(e, result)) {
304      t = { ...fresh(), model: t.model, account: t.account, isOff: t.isOff }
305      isBumpPending = false
306      $.ui.status(t.isOff ? 'keepalive off' : undefined)
307      await save($, writes, live)
308    }
309    return result
310  })
311
312  // /clear and an in-session resume end this conversation without a session.start for the next,
313  // so its timing must not carry over.
314  on('session.end', async ($, e, next) => {
315    t = fresh()
316    isBusy = false
317    isBumpPending = false
318    isBumpTurn = false
319    $.ui.status(undefined)
320    await save($, writes, live)
321    return next(e)
322  })
323}
324

1// What the keepalive keeps across a hot reload of its code.
2export type KeepaliveTiming = {
3  lastSend: number // when the main thread's last answered request started, ms; 0 before the first
4  bumpedFor: number // the lastSend the latest bump answered
5  firstSend: number // when the conversation's first answered request started (a fresh one, or the first after a compaction); 0 before it
6  bumps: number // bumps since the person's last prompt
7  coldBumps: number // bumps in a row that found the cache cold
8  model: string // the model the last answered main request named
9  account: string // the account email the last answered main request went under; '' when logged out or not yet read; a read that fails or names no account keeps the one before
10  isStopped: boolean // COLD_LIMIT bumps in a row found the cache cold, or the account changed; wait for the person
11  isOff: boolean // the person ran /keepalive off; no bumps until /keepalive on
12}
13
14declare module 'claude-code' {
15  interface PluginState {
16    'cache-keepalive': { timing: KeepaliveTiming }
17  }
18}
19