keepwarm

1// keepwarm — keeps an idle interactive session's prompt cache warm.
2//
3// The prompt cache of a session lives one hour from the START of the last request that read it.
4// While the session works, every request refreshes it; only idle gaps (you away, or the main thread
5// blocked past the hour on one long subagent) let it lapse, and the next message then rewrites the
6// whole transcript at the cache-write price (2x input on the 1-hour TTL) instead of reading it
7// (0.05x on Opus 5.5). So, once the main thread has sent nothing for IDLE minutes, this mod sends one
8// tool-less fork of the session's own transcript ($.model.fork): the API serves that prefix from the
9// cache, which restarts its hour, and nothing joins the conversation.
10//
11//   prompt.submit   → a request is about to start (and, for the person's own prompt, the 20 h clock restarts)
12//   tool.call       → after a MAIN-thread tool returns, the next request starts at once
13//   session.start   → interactive sessions only: the 30 s tick, the /keepwarm command, the status line
14//   command.run     → /keepwarm on | off | status, per session
15//
16// A refresh that gets no reply (an API error such as 429/529, a dropped connection, an abort) did not
17// touch the cache: the idle clock is left where it was, so the next tick (30 s on) tries again, until
18// the TTL is reached. Stops (until the next prompt) when: the person has not prompted for MAX hours;
19// the idle gap already reached the TTL (the laptop slept, or every retry failed), so a refresh would
20// pay a full write; a refresh reports the cache had lapsed (it read almost nothing); or there is no
21// reply to fork yet. Turns itself off for the rest of the session when a refresh inside the hour had to
22// write more than it read: the fork cannot reach this session's cache (measured on threaded requests,
23// where the server keeps the conversation), so refreshing would only add a write. Never runs headless.
24//
25// Visible without debug mode: the status line under the prompt (last refresh, retrying, paused),
26// /keepwarm status, and ~/.claude/state/keepwarm/<session id>.json (rewritten at session start, on every
27// refresh, failure, pause and /keepwarm command, and every 5 min while the session works).
28// Env: KEEPWARM_OFF (any value) disables; KEEPWARM_IDLE_MIN (57), KEEPWARM_TTL_MIN (60),
29// KEEPWARM_MAX_HOURS (20), KEEPWARM_PROMPT (the ping text) override the defaults.
30
31import type { EngineInterface, HookFailure, Register } from 'claude-code'
32
33import type { KeepwarmBeat, KeepwarmFailure, KeepwarmSession } from '../types'
34
35type Engine = EngineInterface
36
37const SESSION = { plugin: 'keepwarm', key: 'session' } as const
38const MIN = 60_000
39const VERSION = '0.3.1'
40
41export const PING = 'Cache keep-alive ping, not a task. Reply with exactly: .'
42export const DEFAULTS = { idleMin: 57, ttlMin: 60, maxHours: 20, tickMs: 30_000, deadRead: 1_000, keptBeats: 50 }
43
44export type Config = { idleMs: number; ttlMs: number; maxMs: number; prompt: string; isOff: boolean }
45export type Due = { kind: 'wait' } | { kind: 'beat' } | { kind: 'stop'; why: string }
46
47function positive(raw: string | undefined, fallback: number): number {
48  const n = raw === undefined ? Number.NaN : Number(raw)
49  return Number.isFinite(n) && n > 0 ? n : fallback
50}
51
52export async function config($: Engine): Promise<Config> {
53  const off = await $.env.get('KEEPWARM_OFF')
54  const prompt = await $.env.get('KEEPWARM_PROMPT')
55  return {
56    idleMs: positive(await $.env.get('KEEPWARM_IDLE_MIN'), DEFAULTS.idleMin) * MIN,
57    ttlMs: positive(await $.env.get('KEEPWARM_TTL_MIN'), DEFAULTS.ttlMin) * MIN,
58    maxMs: positive(await $.env.get('KEEPWARM_MAX_HOURS'), DEFAULTS.maxHours) * 60 * MIN,
59    prompt: prompt !== undefined && prompt !== '' ? prompt : PING,
60    isOff: off !== undefined && off !== '',
61  }
62}
63
64/** A new session's state; also the defaults a state written by an older version is filled from. */
65export function fresh(now: number, isInteractive: boolean): KeepwarmSession {
66  return {
67    isEnabled: true,
68    isInteractive,
69    lastRequestAt: now,
70    lastPromptAt: now,
71    stoppedBecause: '',
72    offBecause: '',
73    beats: [],
74    failStreak: 0,
75    lastFailure: null,
76  }
77}
78
79/**
80 * The state session.start keeps: a new session's, or the held one with any field an older version of
81 * this mod did not write filled in (a hot reload keeps $.state). Pure, for the tests.
82 */
83export function upgrade(held: Partial<KeepwarmSession> | undefined, now: number, isInteractive: boolean): KeepwarmSession {
84  return { ...fresh(now, isInteractive), ...held, isInteractive }
85}
86
87/** "API error 529 overloaded", "aborted". Pure, for the tests. */
88export function describeFailure(f: KeepwarmFailure): string {
89  if (f.reason !== 'api-error') return f.reason
90  return `API error ${f.status ?? '(no response)'}${f.error !== '' ? ` ${f.error}` : ''}`
91}
92
93/** What the tick should do now. Pure, for the tests. */
94export function decide(s: KeepwarmSession, now: number, cfg: Config): Due {
95  if (!s.isInteractive || !s.isEnabled || cfg.isOff || s.offBecause !== '' || s.stoppedBecause !== '') return { kind: 'wait' }
96  if (now - s.lastPromptAt >= cfg.maxMs) {
97    return { kind: 'stop', why: `no prompt from you for ${Math.round(cfg.maxMs / (60 * MIN))} h` }
98  }
99  const idle = now - s.lastRequestAt
100  if (idle < cfg.idleMs) return { kind: 'wait' }
101  if (idle >= cfg.ttlMs) {
102    const why =
103      s.failStreak > 0 && s.lastFailure !== null
104        ? `${s.failStreak} refresh${s.failStreak === 1 ? '' : 'es'} failed (last: ${describeFailure(s.lastFailure)}) and the cache lapsed`
105        : 'the cache had already lapsed before a refresh could run'
106    return { kind: 'stop', why }
107  }
108  return { kind: 'beat' }
109}
110
111async function load($: Engine): Promise<KeepwarmSession | undefined> {
112  return (await $.state.get(SESSION)).value
113}
114
115async function change($: Engine, fn: (s: KeepwarmSession) => KeepwarmSession): Promise<KeepwarmSession | undefined> {
116  const held = await $.state.get(SESSION)
117  if (held.value === undefined) return undefined
118  const next = fn(held.value)
119  await $.state.set(SESSION, next)
120  return next
121}
122
123// ---- visibility ---------------------------------------------------------------------------------
124
125function hhmm(ms: number): string {
126  const d = new Date(ms)
127  return `${String(d.getHours()).padStart(2, '0')}:${String(d.getMinutes()).padStart(2, '0')}`
128}
129
130function kTokens(n: number): string {
131  return n >= 1000 ? `${Math.round(n / 1000)}k` : String(n)
132}
133
134/**
135 * The status line under the prompt (the engine shows it after "keepwarm: "); undefined shows none.
136 * Pure, for the tests.
137 */
138export function statusBar(s: KeepwarmSession, cfg: Config): string | undefined {
139  if (!s.isInteractive) return undefined
140  if (cfg.isOff) return 'off (KEEPWARM_OFF)'
141  if (!s.isEnabled) return 'off'
142  if (s.offBecause !== '') return "off: this session's cache can't be refreshed"
143  if (s.stoppedBecause !== '') return `paused: ${s.stoppedBecause}`
144  if (s.failStreak > 0 && s.lastFailure !== null) {
145    return `retrying (${describeFailure(s.lastFailure)} at ${hhmm(s.lastFailure.at)})`
146  }
147  const last = s.beats.at(-1)
148  if (last === undefined) return 'on, no refresh yet'
149  return `last refresh ${hhmm(last.at)}, read ${kTokens(last.cacheRead)} cached`
150}
151
152let shownBar: string | undefined | null = null // null: nothing shown yet by this environment
153
154function showBar($: Engine, s: KeepwarmSession, cfg: Config): void {
155  const text = statusBar(s, cfg)
156  if (text === shownBar) return
157  shownBar = text
158  $.ui.status(text)
159}
160
161let sessionId = ''
162let statePath = ''
163let cwd = ''
164// the request time and clock time the state file was last written with, so a tick can bring a stale file
165// up to date (at most every REWRITE_MS) without a write on every prompt or tool call
166let persistedRequestAt = 0
167let persistedAt = 0
168const REWRITE_MS = 5 * MIN
169
170async function persist($: Engine, s: KeepwarmSession, cfg: Config, now: number): Promise<void> {
171  if (statePath === '') return
172  const record = {
173    version: VERSION,
174    sessionId,
175    cwd,
176    updatedAt: new Date(now).toISOString(),
177    status: statusLine(s, now, cfg),
178    nextRefreshAt:
179      s.stoppedBecause === '' && s.offBecause === '' && s.isEnabled && !cfg.isOff ? new Date(s.lastRequestAt + cfg.idleMs).toISOString() : null,
180    config: { idleMin: cfg.idleMs / MIN, ttlMin: cfg.ttlMs / MIN, maxHours: cfg.maxMs / (60 * MIN) },
181    lastRefresh: s.beats.at(-1) ?? null,
182    lastFailure: s.lastFailure,
183    session: { ...s, beats: s.beats.slice(-10) },
184  }
185  persistedRequestAt = s.lastRequestAt
186  persistedAt = now
187  await $.fs
188    .write(statePath, `${JSON.stringify(record, null, 2)}\n`)
189    .catch(err => $.ui.log(`keepwarm: state file not written: ${String(err)}`, { to: 'debug' }))
190}
191
192/** Status line + state file, after anything that changed what they show. */
193async function publish($: Engine, s: KeepwarmSession | undefined, cfg: Config, now: number): Promise<void> {
194  if (s === undefined) return
195  showBar($, s, cfg)
196  await persist($, s, cfg, now)
197}
198
199// ---- the tick -----------------------------------------------------------------------------------
200
201let isRefreshing = false
202
203async function tick($: Engine): Promise<void> {
204  if (isRefreshing) return
205  const s = await load($)
206  if (s === undefined) return
207  const cfg = await config($)
208  const now = await $.clock.now()
209  const due = decide(s, now, cfg)
210  if (due.kind === 'wait') {
211    // the session is working: keep the state file's last request and next refresh within REWRITE_MS of true
212    if (s.lastRequestAt !== persistedRequestAt && now - persistedAt >= REWRITE_MS) await persist($, s, cfg, now)
213    return
214  }
215  if (due.kind === 'stop') {
216    const after = await change($, x => ({ ...x, stoppedBecause: due.why }))
217    $.ui.log(`keepwarm: paused until your next prompt: ${due.why}`, { to: 'debug' })
218    await publish($, after, cfg, now)
219    return
220  }
221  isRefreshing = true
222  try {
223    const r = await $.model.fork({ prompt: cfg.prompt })
224    if (!r.isAnswered && r.reason === 'nothing-to-fork') {
225      const why = 'nothing to keep warm yet (this conversation has no reply)'
226      const after = await change($, x => ({ ...x, stoppedBecause: why }))
227      $.ui.log(`keepwarm: paused until your next prompt: ${why}`, { to: 'debug' })
228      await publish($, after, cfg, now)
229      return
230    }
231    if (!r.isAnswered && (r.reason === 'api-error' || r.reason === 'aborted')) {
232      // no reply, so no proof the cache was read: leave the idle clock alone and retry at the next tick
233      const failure: KeepwarmFailure =
234        r.reason === 'api-error'
235          ? { at: now, reason: r.reason, status: r.status, error: String(r.error) }
236          : { at: now, reason: r.reason, status: null, error: '' }
237      const after = await change($, x => ({ ...x, failStreak: x.failStreak + 1, lastFailure: failure }))
238      $.ui.log(
239        `keepwarm: refresh failed (${describeFailure(failure)}); try ${after?.failStreak ?? '?'}, retrying at the next tick`,
240        { to: 'debug' },
241      )
242      await publish($, after, cfg, now)
243      return
244    }
245    // answered, or a reply with no text: the request reached the API and its usage is the cache's
246    const usage = r.usage
247    const beat: KeepwarmBeat = {
248      at: now,
249      cacheRead: usage.cache_read_input_tokens,
250      input: usage.input_tokens,
251      cacheWrite: usage.cache_creation_input_tokens,
252      output: usage.output_tokens,
253    }
254    const hasLapsed = beat.cacheRead < DEFAULTS.deadRead
255    const after = await change($, x => afterRefresh(x, beat, now))
256    $.ui.log(
257      `keepwarm: refresh cache_read=${beat.cacheRead} input=${beat.input} cache_write=${beat.cacheWrite} ` +
258        `output=${beat.output} answered=${r.isAnswered}${cannotRead(beat) ? ' (could not reach the session cache: off for this session)' : hasLapsed ? ' (cache had lapsed: paused)' : ''}`,
259      { to: 'debug' },
260    )
261    await publish($, after, cfg, now)
262  } finally {
263    isRefreshing = false
264  }
265}
266
267/** The state after a refresh that reached the API (its request started at `sentAt`). Pure, for the tests. */
268export function afterRefresh(x: KeepwarmSession, beat: KeepwarmBeat, sentAt: number): KeepwarmSession {
269  return {
270    ...x,
271    // a prompt that entered while the fork ran is newer: keep it
272    lastRequestAt: Math.max(x.lastRequestAt, sentAt),
273    failStreak: 0,
274    beats: [...x.beats, beat].slice(-DEFAULTS.keptBeats),
275    ...(cannotRead(beat)
276      ? {
277          offBecause:
278            `a refresh inside the hour read ${beat.cacheRead.toLocaleString('en-US')} cached tokens and had to write ` +
279            `${beat.cacheWrite.toLocaleString('en-US')}: it could not reach this session's own cache, so refreshing only adds cost`,
280        }
281      : beat.cacheRead < DEFAULTS.deadRead && { stoppedBecause: 'a refresh found the cache already lapsed' }),
282  }
283}
284
285/**
286 * A refresh that wrote more of the conversation than it read did not find this session's cache. Every
287 * session shares a warm system-and-tools prefix (~26-29k tokens), so a miss reads that much and writes the
288 * rest; a refresh that found the session's cache writes only the last turn (0-1,300 tokens measured).
289 * Pure, for the tests.
290 */
291export function cannotRead(beat: KeepwarmBeat): boolean {
292  return beat.cacheWrite > beat.cacheRead
293}
294
295function ago(ms: number): string {
296  const m = Math.max(0, Math.round(ms / MIN))
297  return m < 60 ? `${m} min` : `${Math.floor(m / 60)} h ${String(m % 60).padStart(2, '0')} min`
298}
299
300/** What /keepwarm status answers with. Pure, for the tests. */
301export function statusLine(s: KeepwarmSession, now: number, cfg: Config): string {
302  const read = s.beats.reduce((n, b) => n + b.cacheRead, 0)
303  const tally = `${s.beats.length} refresh${s.beats.length === 1 ? '' : 'es'} this session, ${read.toLocaleString('en-US')} cached tokens read`
304  const last = s.beats.at(-1)
305  const lastText =
306    last === undefined
307      ? 'No refresh yet'
308      : `Last refresh ${hhmm(last.at)} (${ago(now - last.at)} ago): read ${last.cacheRead.toLocaleString('en-US')} cached tokens, wrote ${last.cacheWrite.toLocaleString('en-US')}`
309  const failText =
310    s.failStreak > 0 && s.lastFailure !== null
311      ? ` Last failure ${hhmm(s.lastFailure.at)}: ${describeFailure(s.lastFailure)} (${s.failStreak} in a row; retrying every ${DEFAULTS.tickMs / 1000} s until ${hhmm(s.lastRequestAt + cfg.ttlMs)}).`
312      : ''
313  const facts = `${lastText}.${failText} ${tally}.`
314  if (!s.isInteractive) return 'off (not an interactive session).'
315  if (cfg.isOff) return `off (KEEPWARM_OFF is set). ${facts}`
316  if (!s.isEnabled) return `off for this session (/keepwarm on to resume). ${facts}`
317  if (s.offBecause !== '') return `off for this session: ${s.offBecause}. /keepwarm on to try again. ${facts}`
318  if (s.stoppedBecause !== '') return `paused until your next prompt (${s.stoppedBecause}). ${facts}`
319  const dueIn = s.lastRequestAt + cfg.idleMs - now
320  const next = dueIn > 0 ? `next refresh in ${ago(dueIn)} (${hhmm(s.lastRequestAt + cfg.idleMs)})` : 'refresh due now'
321  return `on. Last request ${ago(now - s.lastRequestAt)} ago, ${next}; stops ${ago(s.lastPromptAt + cfg.maxMs - now)} from now without a prompt. ${facts}`
322}
323
324let timer: { cancel: () => void } | undefined
325
326/**
327 * The bookkeeping after `next` in prompt.submit and tool.call must never stand in for the prompt or the
328 * tool. The engine already skips a failed hook and keeps what `next` settled to, but reports the failure
329 * on every prompt or tool call; the .catch on each logs it to the debug log instead and passes `next` on.
330 */
331function logSkipped($: Engine, hook: string, error: HookFailure): void {
332  // a re-entry ran nothing of ours, and its $ calls reject
333  if (error.kind === 're-entry') return
334  const why = error.message !== undefined ? `${error.kind}: ${error.message}` : error.kind
335  $.ui.log(`keepwarm: ${hook} bookkeeping skipped (${why})`, { to: 'debug' })
336}
337
338export const register: Register = on => {
339  on('session.start', async ($, e, next) => {
340    const now = await $.clock.now()
341    const held = await load($)
342    const s = upgrade(held, now, e.isInteractive)
343    await $.state.set(SESSION, s)
344    if (e.isInteractive) {
345      await $.command
346        .register({ name: 'keepwarm', description: "Keep this idle session's prompt cache warm: /keepwarm on | off | status" })
347        .catch(err => $.ui.log(`keepwarm: command not registered: ${String(err)}`, { to: 'debug' }))
348      timer?.cancel()
349      timer = $.clock.every(DEFAULTS.tickMs, () => {
350        void tick($).catch(err => $.ui.log(`keepwarm: tick failed: ${String(err)}`, { to: 'debug' }))
351      })
352      try {
353        cwd = e.cwd
354        sessionId = await $.session.id()
355        const home = await $.env.get('HOME')
356        statePath = home !== undefined && sessionId !== '' ? `${home}/.claude/state/keepwarm/${sessionId}.json` : ''
357      } catch (err) {
358        $.ui.log(`keepwarm: no state file: ${String(err)}`, { to: 'debug' })
359      }
360      shownBar = null
361      await publish($, s, await config($), now)
362    }
363    return next(e)
364  })
365
366  on('prompt.submit', async ($, e, next) => {
367    const r = await next(e)
368    if (r.drop === undefined) {
369      const now = await $.clock.now()
370      const isPerson = e.origin === undefined || e.origin.kind === 'composer'
371      // any prompt that enters starts a request, which re-warms the cache, so a pause lifts
372      const before = await load($)
373      const after = await change($, s => ({
374        ...s,
375        lastRequestAt: now,
376        stoppedBecause: '',
377        failStreak: 0,
378        ...(isPerson && { lastPromptAt: now }),
379      }))
380      if (after !== undefined && before !== undefined && (before.stoppedBecause !== '' || before.failStreak > 0)) {
381        await publish($, after, await config($), now)
382      }
383    }
384    return r
385  }).catch(($, e, next) => {
386    logSkipped($, 'prompt.submit', next.error)
387    return next(e)
388  })
389
390  on('tool.call', async ($, e, next) => {
391    const ran = await next(e)
392    // a subagent's requests carry its own transcript and never refresh the main thread's cache
393    if (e.agentId === undefined) {
394      const now = await $.clock.now()
395      await change($, s => ({ ...s, lastRequestAt: now }))
396    }
397    return ran
398  }).catch(($, e, next) => {
399    logSkipped($, 'tool.call', next.error)
400    return next(e)
401  })
402
403  on('command.run', { command: 'keepwarm' }, async ($, e) => {
404    const s = await load($)
405    if (s === undefined) return { text: 'not active in this session.' }
406    const word = e.args.trim().toLowerCase()
407    const now = await $.clock.now()
408    const cfg = await config($)
409    if (word === 'off') {
410      await publish($, await change($, x => ({ ...x, isEnabled: false })), cfg, now)
411      return { text: 'off for this session.' }
412    }
413    if (word === 'on') {
414      await publish($, await change($, x => ({ ...x, isEnabled: true, stoppedBecause: '', offBecause: '' })), cfg, now)
415      return { text: 'on for this session.' }
416    }
417    const where = statePath !== '' ? ` State file: ${statePath}` : ''
418    return { text: `${statusLine(s, now, cfg)}${where}` }
419  })
420}
421

1// Session-scoped state of the keepwarm mod (survives hot reloads, not sessions).
2export type KeepwarmBeat = {
3  /** Epoch ms when the refresh request was sent. */
4  at: number
5  /** Prompt-cache tokens the fork read: the transcript the cache still held. */
6  cacheRead: number
7  /** Uncached input, cache writes and generated tokens of the fork. */
8  input: number
9  cacheWrite: number
10  output: number
11}
12
13/** A refresh that reached no reply: an API error (with its HTTP status and kind) or an abort. */
14export type KeepwarmFailure = {
15  /** Epoch ms when the failed refresh was sent. */
16  at: number
17  reason: string
18  /** HTTP status of an API error; null when no response arrived or not an API error. */
19  status: number | null
20  /** Claude Code's word for the API error (`overloaded`, `rate_limit`, ...); '' when not an API error. */
21  error: string
22}
23
24export type KeepwarmSession = {
25  /** /keepwarm off sets false for this session; default true. */
26  isEnabled: boolean
27  /** session.start's isInteractive; headless sessions never refresh. */
28  isInteractive: boolean
29  /** Epoch ms when the main thread last SENT a model request (prompt entered, a main tool result returned, or a refresh that reached the API). */
30  lastRequestAt: number
31  /** Epoch ms of the person's last own prompt; the 20 h deadline counts from here. */
32  lastPromptAt: number
33  /** Why refreshing stopped until the next prompt, or '' while active. */
34  stoppedBecause: string
35  /**
36   * Why keepwarm turned itself off for the rest of this session, or ''. Set when a refresh inside the hour
37   * had to write more of the conversation than it could read: the session's own cache is not one a fork
38   * can reach (seen on threaded requests), so refreshing only adds cost. /keepwarm on clears it.
39   */
40  offBecause: string
41  beats: KeepwarmBeat[]
42  /** Failed refreshes since the last request that reached the API; each is retried at the next tick. */
43  failStreak: number
44  /** The most recent failed refresh, kept for /keepwarm status. */
45  lastFailure: KeepwarmFailure | null
46}
47
48declare module 'claude-code' {
49  interface PluginState {
50    keepwarm: {
51      session: KeepwarmSession
52    }
53  }
54}
55