loop-report-probe

1// Throwaway spike, never merged: which quiet routes put a loop report in a Remote Control
2// conversation (phone) and what does each cost the model's context? See ../../README.md.
3//
4// A timer runs the /probe-report command (or another route) once between two real turns, in the
5// phase named by PHASE_FILE (or DEFAULT_PHASE when the file is absent). Every observation is one
6// JSON line in LOG, outside any repository: each session.append row, each turn.start, and
7// $.session.usage() before a run and after the next real model turn. The mod never writes to
8// GitHub or a repository.
9import type { EngineInterface, Register } from 'claude-code'
10
11const LOG = '/persist/claude/spike/loop-report.jsonl'
12const PHASE_FILE = '/persist/claude/spike/loop-report.phase'
13const DEFAULT_PHASE = 'control' // changed by a git pull, or overridden by PHASE_FILE
14const TICK_MS = 60_000
15const MARKER = 'PROBE-REPORT'
16
17const PHASES = [
18  'control', // no report: two real turns with nothing between them
19  'report', // /probe-report through $.command.run, the stored row left as it is
20  'report-trim', // the same, with the stored output row blanked by a session.append hook
21  'append-user', // $.session.append, type user (the model reads it)
22  'append-system', // $.session.append, type system (a notice the model never reads)
23  'ui-log', // $.ui.log to the transcript
24  'ui-toast', // $.ui.toast
25  'ui-status', // $.ui.status
26  'ui-notify', // $.ui.notify
27] as const
28type Phase = (typeof PHASES)[number]
29
30const isPhase = (text: string): text is Phase => (PHASES as readonly string[]).includes(text)
31
32// A sample report in the epic's (#684) shape: an In flight and a Merged table, 10 rows in all.
33const IN_FLIGHT: [string, string, string][] = [
34  ['#701', 'Add the retry budget to the merge queue', 'review'],
35  ['#704', 'Show the lease owner in /factory status', 'build'],
36  ['#709', 'Cap the spawn log at 10 MB', 'CI'],
37  ['#711', 'Name the stalled gate in the PR body', 'build'],
38  ['#715', 'Read the answer file once per pass', 'design'],
39]
40const MERGED: [string, string, string][] = [
41  ['#690', 'Pin the plugin types in the lockfile', '#697'],
42  ['#693', 'Give each step its own scratch folder', '#723'],
43  ['#696', 'Add a read-only config op', '#722'],
44  ['#699', 'Size the test pool from the container', '#725'],
45  ['#702', 'Wait on checks with a 60 s interval', '#721'],
46]
47
48function sampleReport(now: number): string {
49  const stamp = new Date(now).toLocaleString('sv-SE', { timeZone: 'Europe/Amsterdam' })
50  const table = (head: string[], rows: string[][]) =>
51    [`| ${head.join(' | ')} |`, `|${head.map(() => '---').join('|')}|`, ...rows.map(row => `| ${row.join(' | ')} |`)].join('\n')
52  return [
53    `**${MARKER}** - ${stamp} (Europe/Amsterdam)`,
54    '',
55    '### In flight',
56    table(['Issue', 'Title', 'Stage'], IN_FLIGHT),
57    '',
58    '### Merged',
59    table(['Issue', 'Title', 'PR'], MERGED),
60  ].join('\n')
61}
62
63type Pending = { runId: number; phase: Phase; at: string; usageBefore: unknown; turnStarts: number }
64
65let runs = 0
66let pending: Pending | undefined
67let running: Phase | undefined // set while a run is in flight, so the trim hook knows our own row
68let writing: Promise<void> = Promise.resolve()
69
70// Serialized, so lines logged from overlapping hooks are never lost.
71function log($: EngineInterface, kind: string, fields: Record<string, unknown> = {}) {
72  const line = JSON.stringify({ at: new Date().toISOString(), phase: running ?? pending?.phase ?? currentPhase, kind, ...fields }) + '\n'
73  writing = writing
74    .then(async () => {
75      const prev = (await $.fs.exists(LOG)) ? String(await $.fs.read(LOG)) : ''
76      await $.fs.write(LOG, prev + line)
77    })
78    .catch(() => undefined)
79  return writing
80}
81
82let currentPhase: string = DEFAULT_PHASE
83
84async function readPhase($: EngineInterface): Promise<Phase | 'off'> {
85  const text = (await $.fs.exists(PHASE_FILE)) ? String(await $.fs.read(PHASE_FILE)).trim() : DEFAULT_PHASE
86  if (text === 'off') return 'off'
87  if (isPhase(text)) return text
88  void log($, 'bad-phase', { text })
89  return 'off'
90}
91
92const usageOf = async ($: EngineInterface) => {
93  const usage = await $.session.usage()
94  return { contextTokens: usage.context.tokens, percent: usage.context.percent, costUsd: usage.cost?.usd }
95}
96
97async function deliver($: EngineInterface, phase: Phase, report: string) {
98  const block = { type: 'text' as const, text: report }
99  switch (phase) {
100    case 'control':
101      return 'none'
102    case 'report':
103    case 'report-trim':
104      return await $.command.run({ command: 'probe-report' })
105    case 'append-user':
106      return await $.session.append({ message: { type: 'user', content: [block] } })
107    case 'append-system':
108      return await $.session.append({ message: { type: 'system', content: [block] } })
109    case 'ui-log':
110      $.ui.log(report)
111      return 'logged'
112    case 'ui-toast':
113      $.ui.toast(report, { timeoutMs: 20_000 })
114      return 'toasted'
115    case 'ui-status':
116      $.ui.status(report)
117      return 'pinned'
118    case 'ui-notify':
119      return await $.ui.notify(report, { title: 'loop report probe' })
120  }
121}
122
123async function tick($: EngineInterface) {
124  const phase = await readPhase($)
125  currentPhase = phase
126  if (phase === 'off') return
127  if (pending !== undefined) return // one run per gap between two real turns
128  runs += 1
129  const runId = runs
130  const usageBefore = await usageOf($)
131  pending = { runId, phase, at: new Date().toISOString(), usageBefore, turnStarts: 0 }
132  void log($, 'run-start', { runId, usageBefore })
133  running = phase
134  try {
135    const result = await deliver($, phase, sampleReport(await $.clock.now()))
136    void log($, 'run-done', { runId, result })
137  } catch (err) {
138    void log($, 'run-error', { runId, error: String(err) })
139  } finally {
140    running = undefined
141  }
142  void log($, 'run-end', { runId, awaiting: 'next real model turn' })
143}
144
145export const register: Register = on => {
146  on('session.start', async ($, e, next) => {
147    const result = await next(e)
148    await $.command.register({ name: 'probe-report', description: 'Probe: print a sample loop report', immediate: true }).catch(() => undefined)
149    void log($, 'loaded', { version: (await $.session.version()).version, phases: PHASES, defaultPhase: DEFAULT_PHASE })
150    $.clock.every(TICK_MS, () => void tick($))
151    return result
152  })
153
154  on('command.run', { command: 'probe-report' }, async $ => ({ text: sampleReport(await $.clock.now()) }))
155
156  // Every row the conversation keeps. In the report-trim phase the stored output row of our own
157  // run is blanked, as the loop blanks the /rename note, so the report costs the model nothing.
158  on('session.append', async ($, e, next) => {
159    const chars = JSON.stringify(e.message.content ?? '').length
160    const isOurRow = running === 'report-trim' && chars > 0 && JSON.stringify(e.message.content).includes(MARKER)
161    const result = await next(isOurRow ? { ...e, message: { ...e.message, content: [] } } : e)
162    void log($, 'append', {
163      door: e.door,
164      origin: e.origin.kind,
165      type: e.message.type,
166      name: e.message.name,
167      chars,
168      storedChars: JSON.stringify(result.message?.content ?? '').length,
169      trimmed: isOurRow,
170      agentId: e.agentId,
171      duringRun: running !== undefined,
172    })
173    return result
174  }).catch(($, e, next) => next(e))
175
176  on('turn.start', async ($, e, next) => {
177    const result = await next(e)
178    if (e.agentId === undefined && pending !== undefined) pending.turnStarts += 1
179    void log($, 'turn-start', { turnId: e.turnId, agentId: e.agentId, text: e.text.slice(0, 60), duringRun: running !== undefined })
180    return result
181  }).catch(($, e, next) => next(e))
182
183  on('turn.complete', async ($, e, next) => {
184    const result = await next(e)
185    if (e.agentId === undefined && pending !== undefined) {
186      const done = pending
187      pending = undefined
188      void log($, 'turn-end', {
189        runId: done.runId,
190        turnId: e.turnId,
191        reason: e.reason,
192        turnStartsSinceRun: done.turnStarts,
193        usageBefore: done.usageBefore,
194        usageAfter: await usageOf($),
195      })
196    }
197    return result
198  }).catch(($, e, next) => next(e))
199}
200