lazarus

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register, Timer } from 'claude-code'
3
4import type { LazarusJob, LazarusKind, LazarusStall } from '../types'
5import type { Envelope, JobStatus, Ledger, LedgerEvent, LedgerJob } from '../types/fleet.gen'
6
7const mine = atom({ plugin: 'lazarus', key: 'mine' } as const, [] as LazarusJob[])
8const stoppedByModel = atom({ plugin: 'lazarus', key: 'stoppedByModel' } as const, [] as string[])
9const dead = atom({ plugin: 'lazarus', key: 'dead' } as const, [] as LazarusJob[])
10const diedAt = atom({ plugin: 'lazarus', key: 'diedAt' } as const, 0)
11const stall = atom({ plugin: 'lazarus', key: 'stall' } as const, null as LazarusStall | null)
12const lastTurnAt = atom({ plugin: 'lazarus', key: 'lastTurnAt' } as const, 0)
13
14const PANE = 'lazarus-jobs'
15const LIMIT_ID = 'usage-limit'
16const JITTER_MAX_MS = 60_000
17const RUN_TIMEOUT_MS = 15_000
18const FOLLOWED: readonly LazarusKind[] = ['workflow', 'shell', 'monitor', 'agent']
19const AUTH_ERRORS = new Set(['authentication_failed', 'oauth_org_not_allowed', 'account_on_hold', 'verification_required', 'billing_error'])
20const TASK_ID = /<task-id>([^<]+)<\/task-id>/
21const TASK_STATUS = /<status>([^<]+)<\/status>/
22const WORDS: Record<LazarusKind, readonly [string, string]> = {
23  workflow: ['workflow', 'workflows'],
24  shell: ['shell', 'shells'],
25  monitor: ['monitor', 'monitors'],
26  agent: ['agent', 'agents'],
27}
28
29// The one timer lazarus keeps: the wake at a usage limit's reset. A reload
30// drops it with the module; session.start re-arms it from the ledger.
31let limitTimer: Timer | null = null
32let ledgerWarned = false
33
34// lazarus records the background jobs the main loop launches in the
35// per-session ledger (`vybava fleet ledger`), so that after a crash or a
36// restart one key hands what died back to the model. Every tool call passes
37// through unchanged; the ledger is written only when a background job
38// starts or ends, never on a timer.
39export const register: Register = on => {
40  on('session.start', async ($, e, next) => {
41    await $.command.register({
42      name: 'park',
43      description: 'Is it safe to restart? Lists the background jobs a restart would kill',
44      immediate: true,
45    })
46    await scan($, await $.session.id())
47    return next(e)
48  })
49
50  // `claude --resume` and /resume land here; /resume fires no session.start.
51  on('classic.SessionStart', async ($, e, next) => {
52    const ran = await next(e)
53    if (e.source === 'resume') {
54      await scan($, e.session_id)
55    }
56    return ran
57  })
58
59  on('turn.start', async ($, e, next) => {
60    const now = await $.clock.now()
61    await update($, lastTurnAt, () => now)
62    const stalled = await read($, stall)
63    if (stalled?.kind === 'limit') {
64      // The engine's own auto-continue, the person or our wake started it.
65      await clearStall($, 'completed')
66    }
67    return next(e)
68  })
69
70  on('turn.complete', async ($, e, next) => {
71    const ran = await next(e)
72    if (e.agentId === undefined && e.reason === 'answer' && (await read($, stall))?.kind === 'auth') {
73      await update($, stall, () => null)
74    }
75    return ran
76  })
77
78  on('tool.call', { tool: 'Workflow' }, async ($, e, next) => {
79    const ran = await next(e)
80    if (e.agentId === undefined && ran.deny === undefined && ran.isError !== true && ran.result.status === 'async_launched') {
81      await launched($, {
82        kind: 'workflow',
83        id: ran.result.taskId,
84        runId: ran.result.runId,
85        scriptPath: ran.result.scriptPath,
86        description: ran.result.workflowName ?? 'workflow',
87        startedAt: await $.clock.now(),
88      })
89    }
90    return ran
91  })
92
93  on('tool.call', { tool: 'Bash' }, async ($, e, next) => {
94    const ran = await next(e)
95    if (e.agentId === undefined && e.run_in_background === true && ran.deny === undefined && ran.isError !== true) {
96      const taskId = ran.result.backgroundTaskId
97      if (taskId !== undefined) {
98        // Never the command itself: it can carry a secret, and the ledger
99        // keeps only its digest.
100        const description = e.description ?? 'background shell'
101        await launched($, { kind: 'shell', id: taskId, description, startedAt: await $.clock.now() }, e.command)
102      }
103    }
104    return ran
105  })
106
107  on('tool.call', { tool: 'Monitor' }, async ($, e, next) => {
108    const ran = await next(e)
109    if (e.agentId === undefined && ran.deny === undefined && ran.isError !== true) {
110      await launched($, { kind: 'monitor', id: ran.result.taskId, description: e.description, startedAt: await $.clock.now() }, e.command)
111    }
112    return ran
113  })
114
115  on('tool.call', { tool: 'Agent' }, async ($, e, next) => {
116    const ran = await next(e)
117    // A model's background Agent call answers `async_launched` with its id; a
118    // plugin's spawn answers an AgentCallRecord, which has no status.
119    if (e.agentId === undefined && ran.deny === undefined && ran.isError !== true && 'status' in ran.result && ran.result.status === 'async_launched') {
120      await launched($, { kind: 'agent', id: ran.result.agentId, description: e.description, startedAt: await $.clock.now() })
121    }
122    return ran
123  })
124
125  on('tool.call', { tool: 'TaskStop' }, async ($, e, next) => {
126    const id = e.task_id ?? e.shell_id
127    if (e.agentId === undefined && id !== undefined) {
128      await update($, stoppedByModel, ids => [...ids.filter(one => one !== id), id].slice(-200))
129    }
130    return next(e)
131  })
132
133  on('prompt.submit', async ($, e, next) => {
134    if (e.origin?.kind === 'task-notification') {
135      const id = TASK_ID.exec(e.text)?.[1]
136      const status = TASK_STATUS.exec(e.text)?.[1]
137      if (id !== undefined && status !== undefined) {
138        await ended($, id, status)
139      }
140    }
141    return next(e)
142  })
143
144  // The turn's end carries the engine's list of background work still in
145  // flight: a job of ours missing from it ended without a notification we
146  // saw. Absence proves that only when nothing is in flight, or when the list
147  // names a job of ours (so its ids are the ones the launches answered);
148  // otherwise the job is left to its notification, never closed on a guess.
149  // Agents are always left to their notifications; their task id need not be
150  // the agent id the launch answered.
151  on('classic.Stop', async ($, e, next) => {
152    const ran = await next(e)
153    if (e.agent_id === undefined && e.background_tasks !== undefined) {
154      const live = new Set(e.background_tasks.map(task => task.id))
155      const ours = (await read($, mine)).filter(job => job.kind !== 'agent')
156      const isProven = live.size === 0 || ours.some(job => live.has(job.id))
157      const gone = isProven ? ours.filter(job => !live.has(job.id)) : []
158      for (const job of gone) {
159        await update($, mine, list => list.filter(one => one.id !== job.id))
160        await record($, { kind: job.kind, id: job.id, status: 'stopped' })
161      }
162    }
163    return ran
164  })
165
166  on('classic.StopFailure', async ($, e, next) => {
167    const ran = await next(e)
168    if (e.agent_id !== undefined) {
169      return ran
170    }
171    if (e.error === 'rate_limit') {
172      await limitStall($)
173    } else if (AUTH_ERRORS.has(e.error)) {
174      const now = await $.clock.now()
175      await update($, stall, (): LazarusStall => ({ kind: 'auth', at: now, error: e.error }))
176    }
177    return ran
178  })
179
180  on('command.run', { command: 'park' }, async $ => {
181    return { text: await park($) }
182  })
183
184  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
185    const jobs = await read($, dead)
186    const stalled = await read($, stall)
187    const theirs = await next(e)
188    if (e.props.hasSurvey || (jobs.length === 0 && stalled === null)) {
189      return theirs
190    }
191    const { Box, Text, Button } = $.ui.resolve(e)
192    if (jobs.length > 0) {
193      const at = await read($, diedAt)
194      return (
195        <Box flexDirection="column">
196          <Box flexDirection="row" columnGap={1}>
197            <Text>⟲ {summary(jobs)} died {clock(at)}</Text>
198            <Button key="lazarus-resume" label="resume all" hotkey="1" variant="primary" onPress={() => { void resumeAll($) }} />
199            <Button key="lazarus-list" label="list" hotkey="2" onPress={() => { void $.ui.open({ id: PANE, title: 'lazarus · what died', closeOnEscape: true }) }} />
200            <Button key="lazarus-dismiss" label="dismiss" hotkey="3" onPress={() => { void dismiss($) }} />
201          </Box>
202          {theirs}
203        </Box>
204      )
205    }
206    if (stalled === null) {
207      return theirs
208    }
209    const text = stalled.kind === 'auth'
210      ? `⚠ Claude auth failed (${stalled.error}) — switch the account in a terminal: switcheroo cs <account>`
211      : stalled.until === null
212        ? '⏸ usage limit hit'
213        : `⏸ usage limit — continues by itself at ${clock(stalled.until)}`
214    const go = stalled.kind === 'auth' ? 'retry' : stalled.until === null ? 'continue' : 'continue now'
215    return (
216      <Box flexDirection="column">
217        <Box flexDirection="row" columnGap={1}>
218          <Text>{text}</Text>
219          <Button key="lazarus-continue" label={go} hotkey="1" variant="primary" onPress={() => { void continueNow($) }} />
220          <Button key="lazarus-cancel" label={stalled.kind === 'limit' && stalled.until !== null ? 'cancel' : 'dismiss'} hotkey="3" onPress={() => { void clearStall($, 'stopped') }} />
221        </Box>
222        {theirs}
223      </Box>
224    )
225  })
226
227  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
228    const { Box, Text, Button } = $.ui.resolve(e)
229    const jobs = await read($, dead)
230    return (
231      <Box flexDirection="column">
232        {jobs.length === 0 && <Text dimColor>Nothing died.</Text>}
233        {jobs.map(job => <Text wrap="truncate-end">{describeJob(job)}</Text>)}
234        <Button key="lazarus-close" label="close" role="dismiss" onPress={() => { void $.ui.close({ id: PANE }) }} />
235      </Box>
236    )
237  })
238}
239
240// scan reads a session's ledger: an open job this process did not launch
241// died with the process before it; an open usage-limit wait is re-armed.
242async function scan($: EngineInterface, session: string): Promise<void> {
243  const ledger = await showLedger($, session)
244  if (ledger === null) {
245    return
246  }
247  const known = new Set((await read($, mine)).map(job => job.id))
248  const now = await $.clock.now()
249  const lost = ledger.jobs.filter(job => job.status === 'started' && isFollowed(job.kind) && !known.has(job.id)).map(fromLedger)
250  if (lost.length > 0) {
251    await update($, dead, list => [...list.filter(one => !lost.some(job => job.id === one.id)), ...lost])
252    await update($, diedAt, () => now)
253  }
254  const wait = ledger.jobs.find(job => job.kind === 'limit-wait' && job.status === 'started')
255  if (wait === undefined) {
256    return
257  }
258  const until = wait.until === undefined ? null : Date.parse(wait.until)
259  const at = Date.parse(wait.startedAt)
260  if (until !== null && until > now) {
261    await update($, stall, (): LazarusStall => ({ kind: 'limit', at, until }))
262    armLimit($, until - now)
263  } else {
264    // The reset passed while nothing ran: the person is back, the key decides.
265    await update($, stall, (): LazarusStall => ({ kind: 'limit', at, until: null }))
266  }
267}
268
269async function launched($: EngineInterface, job: LazarusJob, command?: string): Promise<void> {
270  await update($, mine, list => [...list.filter(one => one.id !== job.id), job])
271  await record($, {
272    kind: job.kind,
273    id: job.id,
274    status: 'started',
275    runId: job.runId,
276    scriptPath: job.scriptPath,
277    description: job.description,
278    command,
279  })
280}
281
282// ended closes a job of ours on its notification. A job killed or stopped
283// that the model did not stop itself died while the session ran.
284async function ended($: EngineInterface, id: string, word: string): Promise<void> {
285  const job = (await read($, mine)).find(one => one.id === id)
286  const status = toStatus(word)
287  if (job === undefined || status === null) {
288    return
289  }
290  await update($, mine, list => list.filter(one => one.id !== id))
291  await record($, { kind: job.kind, id, status })
292  if ((status === 'killed' || status === 'stopped') && !(await read($, stoppedByModel)).includes(id)) {
293    await update($, dead, list => [...list.filter(one => one.id !== id), job])
294    const now = await $.clock.now()
295    await update($, diedAt, () => now)
296  }
297}
298
299// resumeAll hands what died to the model as facts: workflows with the exact
300// call that resumes them, the rest for it to judge. The prompt is framed as
301// lazarus's; nothing is re-run behind the model's back.
302async function resumeAll($: EngineInterface): Promise<void> {
303  const jobs = await read($, dead)
304  if (jobs.length === 0) {
305    return
306  }
307  await update($, dead, () => [])
308  if (!(await submit($, resumePrompt(jobs)))) {
309    // Nothing reached the model: the jobs stay offered.
310    await update($, dead, list => [...jobs, ...list.filter(one => !jobs.some(job => job.id === one.id))])
311    return
312  }
313  for (const job of jobs) {
314    await record($, { kind: job.kind, id: job.id, status: 'stopped' })
315  }
316}
317
318// submit starts a turn framed as lazarus's; a refusal is logged and shown,
319// never lost in a press handler or a timer.
320async function submit($: EngineInterface, text: string): Promise<boolean> {
321  try {
322    await $.prompt.submit({ text })
323    return true
324  } catch (err) {
325    $.ui.log(`could not start a turn: ${String(err)}`, { to: 'debug' })
326    $.ui.toast('could not start a turn; see the debug log')
327    return false
328  }
329}
330
331async function dismiss($: EngineInterface): Promise<void> {
332  const jobs = await read($, dead)
333  await update($, dead, () => [])
334  for (const job of jobs) {
335    await record($, { kind: job.kind, id: job.id, status: 'stopped' })
336  }
337  await $.ui.close({ id: PANE })
338}
339
340async function limitStall($: EngineInterface): Promise<void> {
341  const now = await $.clock.now()
342  const usage = await $.session.usage()
343  const resets = usage.rateLimits
344    .filter(limit => limit.resetsAt !== undefined && limit.percentUsed >= 100)
345    .map(limit => Date.parse(limit.resetsAt ?? ''))
346    .filter(at => Number.isFinite(at) && at > now)
347  if (resets.length === 0) {
348    await update($, stall, (): LazarusStall => ({ kind: 'limit', at: now, until: null }))
349    await record($, { kind: 'limit-wait', id: LIMIT_ID, status: 'started' })
350    return
351  }
352  // Spread the sessions one limit stalled over a minute after its reset.
353  const until = Math.max(...resets) + jitter(await $.session.id())
354  await update($, stall, (): LazarusStall => ({ kind: 'limit', at: now, until }))
355  await record($, { kind: 'limit-wait', id: LIMIT_ID, status: 'started', until: new Date(until).toISOString() })
356  armLimit($, until - now)
357}
358
359function armLimit($: EngineInterface, delay: number): void {
360  limitTimer?.cancel()
361  limitTimer = $.clock.after(Math.max(0, delay), () => {
362    limitTimer = null
363    void limitReset($)
364  })
365}
366
367// limitReset is the wake at the reset: it continues the session only when
368// no turn started since the stall (the engine's own auto-continue, or the
369// person, would have cleared the stall already).
370async function limitReset($: EngineInterface): Promise<void> {
371  const stalled = await read($, stall)
372  if (stalled?.kind !== 'limit') {
373    return
374  }
375  const started = (await read($, lastTurnAt)) > stalled.at
376  await clearStall($, 'completed')
377  if (!started) {
378    await submit($, 'continue — the usage limit has reset')
379  }
380}
381
382async function continueNow($: EngineInterface): Promise<void> {
383  const stalled = await read($, stall)
384  if (stalled === null) {
385    return
386  }
387  await clearStall($, 'completed')
388  await submit($, stalled.kind === 'limit' ? 'continue — the usage limit has reset' : 'continue')
389}
390
391async function clearStall($: EngineInterface, status: JobStatus): Promise<void> {
392  const stalled = await read($, stall)
393  limitTimer?.cancel()
394  limitTimer = null
395  await update($, stall, () => null)
396  if (stalled?.kind === 'limit') {
397    await record($, { kind: 'limit-wait', id: LIMIT_ID, status })
398  }
399}
400
401async function park($: EngineInterface): Promise<string> {
402  const session = await $.session.id()
403  const ledger = await showLedger($, session)
404  const known = await read($, mine)
405  const running = ledger === null
406    ? known
407    : ledger.jobs.filter(job => job.status === 'started' && isFollowed(job.kind) && known.some(one => one.id === job.id)).map(fromLedger)
408  const lines = running.map(job => `- ${describeJob(job)}`)
409  const verdict = running.length === 0
410    ? 'Safe to restart: no background job of this session is running.'
411    : `Not safe to restart: ${running.length === 1 ? '1 job' : `${running.length} jobs`} would die (lazarus offers them for resume afterwards).`
412  const parts = [verdict, ...lines]
413  if (ledger === null) {
414    parts.push('(The ledger was unreadable; this lists only what this process launched.)')
415  }
416  await record($, { kind: 'park', id: `park-${await $.clock.now()}`, status: 'completed', description: verdict })
417  return parts.join('\n')
418}
419
420async function showLedger($: EngineInterface, session: string): Promise<Ledger | null> {
421  const out = await vybava($, ['fleet', 'ledger', 'show', '--session', session, '--json'])
422  if (out === null) {
423    return null
424  }
425  try {
426    const envelope = JSON.parse(out) as Envelope<Ledger>
427    if (envelope.ok && envelope.data !== undefined) {
428      return envelope.data
429    }
430    $.ui.log(`fleet ledger show answered not ok: ${envelope.diagnostics.map(d => d.code).join(', ')}`, { to: 'debug' })
431  } catch (err) {
432    $.ui.log(`fleet ledger show printed no envelope: ${String(err)}`, { to: 'debug' })
433  }
434  return null
435}
436
437async function record($: EngineInterface, event: LedgerEvent): Promise<void> {
438  const session = await $.session.id()
439  const out = await vybava($, ['fleet', 'ledger', 'record', '--session', session], JSON.stringify(event))
440  if (out === null && !ledgerWarned) {
441    ledgerWarned = true
442    $.ui.toast('the job ledger cannot be written; crash recovery is off for this session (see the debug log)')
443  }
444}
445
446// vybava runs one fleet verb; a failure is logged to the debug log and
447// answered as null, never thrown into the tool call it rides on.
448async function vybava($: EngineInterface, argv: readonly string[], stdin?: string): Promise<string | null> {
449  try {
450    const ran = await $.process.run(['vybava', ...argv], stdin === undefined ? { timeoutMs: RUN_TIMEOUT_MS } : { stdin, timeoutMs: RUN_TIMEOUT_MS })
451    if (ran.exitCode === 0) {
452      return ran.stdout
453    }
454    $.ui.log(`vybava ${argv.slice(0, 3).join(' ')} exited ${ran.exitCode}: ${firstLine(ran.stderr === '' ? ran.stdout : ran.stderr)}`, { to: 'debug' })
455  } catch (err) {
456    $.ui.log(`vybava ${argv.slice(0, 3).join(' ')} failed: ${String(err)}`, { to: 'debug' })
457  }
458  return null
459}
460
461function resumePrompt(jobs: readonly LazarusJob[]): string {
462  const lines = jobs.map(job => {
463    if (job.kind === 'workflow' && job.runId !== undefined && job.scriptPath !== undefined) {
464      return `- workflow "${job.description ?? job.id}": resume it with Workflow({ scriptPath: ${JSON.stringify(job.scriptPath)}, resumeFromRunId: ${JSON.stringify(job.runId)} })`
465    }
466    return `- ${describeJob(job)}: died; start it again only if it is still needed`
467  })
468  return [
469    `${summary(jobs)} this session started died with it (a crash, a restart or a kill). The person pressed resume.`,
470    ...lines,
471    'Resume the workflows; decide on the rest from where the work stands.',
472  ].join('\n')
473}
474
475function describeJob(job: LazarusJob): string {
476  const what = `${WORDS[job.kind][0]} "${job.description ?? job.id}"`
477  return job.runId === undefined ? `${what} (${job.id})` : `${what} (run ${job.runId})`
478}
479
480function summary(jobs: readonly LazarusJob[]): string {
481  return FOLLOWED.map(kind => {
482    const count = jobs.filter(job => job.kind === kind).length
483    return count === 0 ? null : `${count} ${WORDS[kind][count === 1 ? 0 : 1]}`
484  })
485    .filter((part): part is string => part !== null)
486    .join(' + ')
487}
488
489function fromLedger(job: LedgerJob): LazarusJob {
490  const started = Date.parse(job.startedAt)
491  return {
492    kind: job.kind as LazarusKind,
493    id: job.id,
494    runId: job.runId,
495    scriptPath: job.scriptPath,
496    description: job.description,
497    startedAt: Number.isFinite(started) ? started : 0,
498  }
499}
500
501function isFollowed(kind: string): boolean {
502  return (FOLLOWED as readonly string[]).includes(kind)
503}
504
505function toStatus(word: string): JobStatus | null {
506  switch (word.trim()) {
507    case 'completed':
508      return 'completed'
509    case 'failed':
510      return 'failed'
511    case 'killed':
512      return 'killed'
513    case 'stopped':
514      return 'stopped'
515    default:
516      return null
517  }
518}
519
520// jitter spreads the sessions one limit stalled: a stable 0–60 s per session.
521function jitter(session: string): number {
522  let hash = 0
523  for (const char of session) {
524    hash = (hash * 31 + char.charCodeAt(0)) >>> 0
525  }
526  return hash % JITTER_MAX_MS
527}
528
529function clock(ms: number): string {
530  const at = new Date(ms)
531  return `${String(at.getHours()).padStart(2, '0')}:${String(at.getMinutes()).padStart(2, '0')}`
532}
533
534function firstLine(text: string): string {
535  return (text.trim().split('\n')[0] ?? '').slice(0, 200)
536}
537

1// lazarus's state contract: the session values its hooks keep in $.state.
2
3/** The background work lazarus follows; the ledger's job kinds less its own. */
4export type LazarusKind = 'workflow' | 'shell' | 'monitor' | 'agent'
5
6/** A background job as the ledger keys it (kind + id). */
7export type LazarusJob = {
8  kind: LazarusKind
9  id: string
10  /** A workflow's run, what resumeFromRunId takes. */
11  runId?: string
12  scriptPath?: string
13  description?: string
14  /** When it started, in $.clock.now() milliseconds. */
15  startedAt: number
16}
17
18/**
19 * What stopped the session: a usage limit, which continues by itself at
20 * `until` when the reset is known, or an auth failure, which waits for a key.
21 */
22export type LazarusStall =
23  | { kind: 'limit'; at: number; until: number | null }
24  | { kind: 'auth'; at: number; error: string }
25
26declare module 'claude-code' {
27  interface PluginState {
28    lazarus: {
29      /** Jobs this process launched that have not ended. */
30      mine: LazarusJob[]
31      /** Task ids the model stopped itself (TaskStop): their end is no death. */
32      stoppedByModel: string[]
33      /** Jobs that died with a previous process, or were killed while this one ran. */
34      dead: LazarusJob[]
35      /** When lazarus found them dead, in milliseconds. */
36      diedAt: number
37      stall: LazarusStall | null
38      /** The last main-loop turn's start; a limit wake that sees a newer one stands down. */
39      lastTurnAt: number
40    }
41  }
42}
43