cache-warmer

1// cache-warmer: keeps the prompt cache warm while the session sits idle.
2//
3// The prompt cache expires a fixed time after the last request (5 minutes or
4// 1 hour). When the main thread has been quiet for `intervalMinutes`, this mod
5// asks one tiny question through $.model.fork: a tool-less request over the
6// session's own transcript, served from the cache, which refreshes its TTL.
7// Nothing is added to the conversation. After `maxPings` pings in a row with
8// no activity from you, it stops, so an abandoned session does not keep paying.
9
10import type { EngineInterface, Register } from 'claude-code'
11
12const MIN = 60_000
13const PING = 'Cache keep-alive ping. Reply with exactly: ok'
14
15let interval = 50 * MIN // set from options in register
16let maxPings = 3
17let lastAt = 0 // when the main thread last sent a request (ms); 0 = nothing to keep warm
18let pings = 0 // pings since the last real request
19let working = false // a turn is running
20let pinging = false // a ping is in flight
21let lastResult = 'no ping yet'
22
23async function ping($: EngineInterface): Promise<string> {
24  pinging = true
25  try {
26    const r = await $.model.fork({ prompt: PING })
27    const now = await $.clock.now()
28    if (!r.isAnswered) {
29      if (r.reason === 'nothing-to-fork') lastAt = 0
30      else pings++ // count failures too, so a broken API cannot loop us
31      lastResult = 'failed: ' + r.reason
32      return lastResult
33    }
34    const read = r.usage.cache_read_input_tokens ?? 0
35    const wrote = r.usage.cache_creation_input_tokens ?? 0
36    if (read === 0) {
37      // The cache was already cold (or the fork missed it); pinging on just re-buys it
38      pings = maxPings
39      lastResult = `missed the cache (wrote ${wrote} tokens); stopped until your next message`
40      $.ui.toast('cache-warmer: ping missed the cache, so it stopped. Your next message restarts it.')
41    } else {
42      pings++
43      lastAt = now
44      lastResult = `ping ${pings}/${maxPings}: read ${read} cached tokens, wrote ${wrote}`
45    }
46    $.ui.status(`cache warm · ${lastResult}`)
47    return lastResult
48  } finally {
49    pinging = false
50  }
51}
52
53async function tick($: EngineInterface): Promise<void> {
54  if (!lastAt || working || pinging || pings >= maxPings) return
55  if ((await $.clock.now()) - lastAt < interval) return
56  await ping($)
57}
58
59export const register: Register = (on, options) => {
60  interval = Math.max(1, Number(options.intervalMinutes ?? 50)) * MIN
61  maxPings = Math.max(0, Number(options.maxPings ?? 3))
62
63  on('session.start', async ($, e, next) => {
64    await $.command.register({
65      name: 'cache-warmer',
66      description: 'Show the cache-warmer state; "/cache-warmer ping" pings now',
67      argumentHint: '[ping]',
68    })
69    $.clock.every(MIN, () => tick($))
70    return next(e)
71  })
72
73  // A /clear empties the conversation: nothing left to keep warm
74  on('session.end', async ($, e, next) => {
75    lastAt = 0
76    return next(e)
77  })
78
79  on('turn.start', async ($, e, next) => {
80    working = true
81    return next(e)
82  })
83
84  on('turn.complete', async ($, e, next) => {
85    working = false
86    return next(e)
87  })
88
89  // Each real request of the main thread refreshes the cache: restart the clock
90  on('turn.step', async function* ($, e, next) {
91    if (!e.agentId && !pinging) {
92      lastAt = await $.clock.now()
93      pings = 0
94      $.ui.status(undefined)
95    }
96    return yield* next(e)
97  })
98
99  on('command.run', { command: 'cache-warmer' }, async ($, e) => {
100    if ((e.args ?? '').trim() === 'ping') return { text: await ping($) }
101    const idle = lastAt ? Math.round(((await $.clock.now()) - lastAt) / MIN) : null
102    return {
103      text: [
104        `Pings every ${interval / MIN} min idle, at most ${maxPings} in a row.`,
105        idle === null ? 'No request yet in this conversation.' : `Idle ${idle} min, ${pings}/${maxPings} pings used.`,
106        `Last: ${lastResult}`,
107      ].join('\n'),
108    }
109  })
110}
111