context-gate-probe

1// context-gate-probe: SPEC Р6 stage 0, docs/PROBE.md "Still LIVE". One observer per unverified mods API
2// point. Every hook passes through unchanged, never throws, appends one observation to an in-memory report
3// and rewrites `<session root>/.claude/<out>` (default probe.json; CONTEXT_GATE_PROBE_OUT overrides the
4// file name, as probe/run-print.sh does for the `-p` run) through $.fs.write.
5// Recorded: metadata and lengths only, except the skill_listing text (the parseSkillListing fixture).
6
7import { atom, read, update } from 'claude-code'
8import type { EngineInterface, Register } from 'claude-code'
9import type { ProbeMarker } from '../types'
10
11export type PointName =
12  | 'skill_listing'
13  | 'tool_describe_mcp'
14  | 'skill_prompt_bang'
15  | 'state_after_clear'
16  | 'filechanged_watchdir'
17  | 'prompt_context_subagent'
18  | 'model_complete_cost'
19  | 'prompt_compose_print'
20
21export const POINTS: readonly PointName[] = [
22  'skill_listing',
23  'tool_describe_mcp',
24  'skill_prompt_bang',
25  'state_after_clear',
26  'filechanged_watchdir',
27  'prompt_context_subagent',
28  'model_complete_cost',
29  'prompt_compose_print',
30]
31
32export type Verdict = 'works' | 'broken' | 'partial'
33export type Sample = Record<string, unknown> & { seq: number; at: string; kind: string }
34export interface Point { status: 'observed' | 'not-fired'; samples: Sample[]; verdict?: Verdict }
35export interface Report {
36  claudeCode: string | null
37  startedAt: string
38  out: string
39  points: Record<PointName, Point>
40}
41
42const MAX_SAMPLES = 60
43const LISTING_CHARS = 20_000
44const WATCH_DIR = '.cursor/rules'
45const WATCH_FILE = '.claude/probe-watch.txt'
46
47const markerAtom = atom({ plugin: 'context-gate-probe', key: 'marker' } as const, '')
48
49export function newReport(now: string): Report {
50  const points = {} as Record<PointName, Point>
51  for (const p of POINTS) points[p] = { status: 'not-fired', samples: [] }
52  return { claudeCode: null, startedAt: now, out: 'probe.json', points }
53}
54
55/** Automatic verdicts where a sample answers the question by itself; the rest stay for the person. */
56export function judge(r: Report): void {
57  const s = (p: PointName) => r.points[p].samples
58  const set = (p: PointName, v: Verdict | undefined) => {
59    if (v) r.points[p].verdict = v
60  }
61  set('tool_describe_mcp', s('tool_describe_mcp').some((x) => x.isDeferred === true) ? 'works' : undefined)
62  const clears = s('state_after_clear').filter((x) => x.kind === 'classic.SessionStart' && x.source === 'clear')
63  if (clears.length) set('state_after_clear', clears.some((x) => typeof x.markerBefore === 'string' && x.markerBefore !== '') ? 'works' : 'broken')
64  const fc = s('filechanged_watchdir').filter((x) => x.kind === 'classic.FileChanged')
65  if (fc.length) set('filechanged_watchdir', fc.some((x) => x.underWatchDir === true) ? 'works' : 'partial')
66  const mc = s('model_complete_cost').filter((x) => x.kind === 'model.complete')
67  if (mc.length) set('model_complete_cost', mc.some((x) => x.isAnswered === true) ? 'works' : 'broken')
68  if (s('prompt_compose_print').some((x) => Array.isArray(x.traits) && (x.traits as unknown[]).includes('print'))) set('prompt_compose_print', 'works')
69}
70
71const lengthOf = (v: unknown): number => (typeof v === 'string' ? v.length : 0)
72const agentKeys = (e: object): string[] => Object.keys(e).filter((k) => /agent/i.test(k))
73
74/** Repo-relative when inside `root`, else the basename: paths only, never content. */
75export function relPath(root: string, path: string): string {
76  if (root && path.startsWith(root.endsWith('/') ? root : root + '/')) return path.slice(root.replace(/\/+$/, '').length + 1)
77  return path.split(/[\\/]/).pop() ?? path
78}
79
80/** `.catch` for every hook: a failed observer degrades to the engine's own behaviour. */
81function pass<E, R>(_$: unknown, e: E, next: (e: E) => R): R {
82  return next(e)
83}
84
85/** The module's own runtime: the report and the write queue (reset on every load). */
86export interface Probe {
87  report: Report
88  seq: number
89  seen: Set<string>
90  writing: Promise<void> | null
91  dirty: boolean
92}
93
94export function newProbe(): Probe {
95  return { report: newReport(new Date().toISOString()), seq: 0, seen: new Set(), writing: null, dirty: false }
96}
97
98async function writeNow($: EngineInterface, rt: Probe): Promise<void> {
99  judge(rt.report)
100  const root = await $.session.root()
101  const name = (await $.env.get('CONTEXT_GATE_PROBE_OUT')) || 'probe.json'
102  rt.report.out = name
103  await $.fs.write(`${root}/.claude/${name}`, JSON.stringify(rt.report, null, 2) + '\n')
104}
105
106/** Overwrite the whole file; writes requested while one is in flight coalesce into one more. */
107async function flush($: EngineInterface, rt: Probe): Promise<void> {
108  rt.dirty = true
109  if (rt.writing) return rt.writing
110  let done: () => void = () => {}
111  rt.writing = new Promise<void>((resolve) => { done = resolve })
112  while (rt.dirty) {
113    rt.dirty = false
114    try {
115      await writeNow($, rt)
116    } catch (err) {
117      try { $.ui.log(`context-gate-probe: write failed: ${String(err)}`, { to: 'debug' }) } catch { /* ignore */ }
118    }
119  }
120  rt.writing = null
121  done()
122}
123
124/** Append one sample (deduplicated by `keyOf` when given) and rewrite probe.json. Never throws. */
125async function observe($: EngineInterface, rt: Probe, point: PointName, kind: string, data: () => Record<string, unknown>, keyOf?: () => string): Promise<void> {
126  try {
127    const key = keyOf?.()
128    if (key !== undefined) {
129      const k = `${point}|${kind}|${key}`
130      if (rt.seen.has(k)) return
131      rt.seen.add(k)
132    }
133    const p = rt.report.points[point]
134    p.status = 'observed'
135    const sample: Sample = { seq: ++rt.seq, at: new Date().toISOString(), kind, ...data() }
136    if (p.samples.length < MAX_SAMPLES) p.samples.push(sample)
137    const { text: _omit, ...logged } = sample
138    try { $.ui.log(`context-gate-probe: ${point} ${JSON.stringify(logged)}`, { to: 'debug' }) } catch { /* ignore */ }
139    await flush($, rt)
140  } catch {
141    /* the probe never breaks the session */
142  }
143}
144
145async function marker($: EngineInterface): Promise<ProbeMarker | null> {
146  try { return await read($, markerAtom) } catch { return null }
147}
148
149async function setMarker($: EngineInterface, value: ProbeMarker): Promise<void> {
150  try { await update($, markerAtom, () => value) } catch { /* ignore */ }
151}
152
153export const register: Register = (on) => {
154  const rt = newProbe()
155  const report = rt.report
156
157  // ── 4 (+ /probe registration): session lifecycle and $.state across /clear ──
158
159  on('session.start', async ($, e, next) => {
160    try {
161      if (report.claudeCode === null) {
162        try { report.claudeCode = (await $.session.version()).version } catch { /* not available */ }
163      }
164      const before = await marker($)
165      await observe($, rt, 'state_after_clear', 'session.start', () => ({ markerBefore: before, surface: e.surface, isInteractive: e.isInteractive }))
166      await setMarker($, `session.start@${new Date().toISOString()}`)
167      await $.command.register({ name: 'probe', description: 'context-gate-probe: classify (time $.model.complete) | dump', argumentHint: 'classify|dump' })
168    } catch { /* ignore */ }
169    return next(e)
170  }).catch(pass)
171
172  on('session.end', async ($, e, next) => {
173    const before = await marker($)
174    await observe($, rt, 'state_after_clear', 'session.end', () => ({ reason: e.reason, markerBefore: before }))
175    return next(e)
176  }).catch(pass)
177
178  // ── 4 + 5: classic SessionStart (source) and watchPaths with a directory entry ──
179
180  on('classic.SessionStart', async ($, e, next) => {
181    const r = await next(e)
182    try {
183      const before = await marker($)
184      await observe($, rt, 'state_after_clear', 'classic.SessionStart', () => ({ source: e.source, markerBefore: before }))
185      await setMarker($, `classic.SessionStart:${e.source}@${new Date().toISOString()}`)
186      const root = await $.session.root()
187      const watch = [`${root}/${WATCH_DIR}`, `${root}/${WATCH_FILE}`]
188      await observe($, rt, 'filechanged_watchdir', 'watchPaths', () => ({ source: e.source, added: watch, existing: r.watchPaths?.length ?? 0 }))
189      return { ...r, watchPaths: [...(r.watchPaths ?? []), ...watch] }
190    } catch {
191      return r
192    }
193  }).catch(pass)
194
195  on('classic.FileChanged', async ($, e, next) => {
196    try {
197      const root = await $.session.root()
198      const path = String(e.file_path)
199      await observe($, rt, 'filechanged_watchdir', 'classic.FileChanged', () => ({
200        file_path: path,
201        event: e.event,
202        underWatchDir: path.startsWith(`${root}/${WATCH_DIR}/`),
203        isWatchFile: path === `${root}/${WATCH_FILE}`,
204      }))
205    } catch { /* ignore */ }
206    return next(e)
207  }).catch(pass)
208
209  // ── 1: the skill listing attachment (passed through unchanged) ──
210
211  on('prompt.attachment', { type: 'skill_listing' }, async ($, e, next) => {
212    const r = await next(e)
213    await observe($, rt, 'skill_listing', 'prompt.attachment', () => ({
214      agentId: e.agentId ?? null,
215      origin: e.origin?.kind ?? null,
216      textLength: e.text.length,
217      resultChanged: r.text !== e.text,
218      hasDetail: e.detail !== undefined,
219      text: e.text.slice(0, LISTING_CHARS),
220    }), () => `${e.agentId ?? 'main'}|${e.text.length}`)
221    return r
222  }).catch(pass)
223
224  // ── 2: tool.describe for MCP tools (deferred?) ──
225
226  on('tool.describe', { tool: /^mcp__/ }, async ($, e, next) => {
227    const r = await next(e)
228    await observe($, rt, 'tool_describe_mcp', 'tool.describe', () => ({
229      tool: e.tool,
230      isDeferred: e.isDeferred === true,
231      descriptionLength: e.description.length,
232      resultIsDeferred: r.isDeferred ?? null,
233      resultDescriptionLength: r.description.length,
234    }), () => `${e.tool}|${e.isDeferred === true}|${e.description.length}`)
235    return r
236  }).catch(pass)
237
238  // ── 3: skill.prompt vs !`…`, Skill tool ordering (seq), command.run ──
239
240  on('skill.prompt', async ($, e, next) => {
241    await observe($, rt, 'skill_prompt_bang', 'skill.prompt', () => ({
242      skill: e.skill,
243      hasBangPattern: /!`/.test(e.text),
244      // the bundled probe-bang skill: its output present without its command means !`…` already ran
245      fixtureExpanded: e.text.includes('probe-bang-expanded') && !e.text.includes('echo probe-bang-expanded'),
246      textLength: e.text.length,
247    }))
248    return next(e)
249  }).catch(pass)
250
251  on('tool.call', { tool: 'Skill' }, async ($, e, next) => {
252    await observe($, rt, 'skill_prompt_bang', 'tool.call:Skill', () => ({ skill: e.skill, hasArgs: lengthOf(e.args) > 0, argsLength: lengthOf(e.args), agentId: e.agentId ?? null }))
253    return next(e)
254  }).catch(pass)
255
256  // ── 7: /probe classify times $.model.complete({ model: 'haiku' }); /probe (dump) summarises ──
257
258  on('command.run', { command: 'probe' }, async ($, e) => {
259    await observe($, rt, 'skill_prompt_bang', 'command.run', () => ({ command: e.command, hasArgs: e.args.trim().length > 0, origin: e.origin?.kind ?? null }))
260    if (e.args.trim() === 'classify') {
261      const t0 = Date.now()
262      try {
263        const res = await $.model.complete({
264          model: 'haiku',
265          system: 'Answer with exactly one label from: frontend, backend, docs.',
266          prompt: 'Task: fix the CSS of the login button.',
267          maxTokens: 16,
268        })
269        const ms = Date.now() - t0
270        await observe($, rt, 'model_complete_cost', 'model.complete', () => (res.isAnswered
271          ? { ms, isAnswered: true, answerLength: res.text.length, usage: res.usage }
272          : { ms, isAnswered: false, reason: res.reason }))
273        return { text: `probe classify: ${ms} ms, ${res.isAnswered ? `usage ${JSON.stringify(res.usage)}` : `not answered (${res.reason})`}` }
274      } catch (err) {
275        const ms = Date.now() - t0
276        await observe($, rt, 'model_complete_cost', 'model.complete', () => ({ ms, isAnswered: false, error: String(err) }))
277        return { text: `probe classify: failed: ${String(err)}` }
278      }
279    }
280    judge(report)
281    const lines = POINTS.map((p) => `${p}: ${report.points[p].status}${report.points[p].verdict ? ` (${report.points[p].verdict})` : ''}, ${report.points[p].samples.length} samples`)
282    return { text: ['context-gate-probe', ...lines, 'usage: /probe classify | dump'].join('\n') }
283  })
284
285  on('command.run', async ($, e, next) => {
286    await observe($, rt, 'skill_prompt_bang', 'command.run', () => ({ command: e.command, hasArgs: e.args.trim().length > 0, origin: e.origin?.kind ?? null }))
287    return next(e)
288  }).catch(pass)
289
290  // ── 6: prompt.context (agentId? instructionFiles?) and turn.step (subagents, cache reads) ──
291
292  on('prompt.context', async ($, e, next) => {
293    const r = await next(e)
294    let root = ''
295    try { root = await $.session.root() } catch { /* relPath falls back to basenames */ }
296    await observe($, rt, 'prompt_context_subagent', 'prompt.context', () => ({
297      blockNames: e.blocks.map((b) => b.name),
298      instructionFilesDefined: e.instructionFiles !== undefined,
299      instructionFilesCount: e.instructionFiles?.length ?? 0,
300      instructionFileKinds: [...new Set((e.instructionFiles ?? []).map((f) => f.kind))],
301      inputKeys: Object.keys(e),
302      agentLikeKeys: agentKeys(e),
303      // what came back from next(e): the plugins beneath (plugin order is unknown) plus the engine
304      resultBlockNames: r.blocks.map((b) => b.name),
305      resultInstructionFiles: (r.instructionFiles ?? []).map((f) => ({ path: relPath(root, f.path), kind: f.kind })),
306      hasCursorRulesBlock: r.blocks.some((b) => b.name === 'cursorRules'),
307    }), () => `${e.blocks.map((b) => b.name).join(',')}|${e.instructionFiles?.length ?? -1}|${r.blocks.map((b) => b.name).join(',')}|${r.instructionFiles?.length ?? -1}`)
308    return r
309  }).catch(pass)
310
311  on('turn.step', async function* ($, e, next) {
312    const r = yield* next(e)
313    await observe($, rt, 'prompt_context_subagent', 'turn.step', () => ({
314      agentId: e.agentId ?? null,
315      model: e.model,
316      index: e.index,
317      usageModel: r?.usage?.model ?? null,
318      cacheReadInputTokens: r?.usage?.cache_read_input_tokens ?? null,
319      stopReason: r?.stopReason ?? null,
320    }))
321    return r
322  })
323
324  // ── 8: prompt.compose traits ('print' under -p, 'sdk-preset') ──
325
326  on('prompt.compose', async ($, e, next) => {
327    const r = await next(e)
328    await observe($, rt, 'prompt_compose_print', 'prompt.compose', () => ({
329      traits: [...e.traits],
330      model: e.model,
331      promptModel: e.promptModel,
332      surfaces: [...e.surfaces],
333      toolsCount: e.tools.length,
334      sectionsCount: r.sections.length,
335    }), () => `${e.traits.join(',')}|${e.model}|${r.sections.length}`)
336    return r
337  }).catch(pass)
338
339  // ── 4 (continued): the marker on each prompt, to see the state right after /clear (no prompt text) ──
340
341  on('prompt.submit', async ($, e, next) => {
342    const before = await marker($)
343    await observe($, rt, 'state_after_clear', 'prompt.submit', () => ({ markerBefore: before, textLength: e.text.length }))
344    return next(e)
345  }).catch(pass)
346}
347

1// context-gate-probe's contract: the one session-state value the probe keeps in `$.state`.
2// Self-contained (no import, no reference), as the mods API requires of a plugin's `types` file.
3
4/** `<event>[:<source>]@<iso>`, or '' before the first write. */
5export type ProbeMarker = string
6
7declare module 'claude-code' {
8  interface PluginState {
9    'context-gate-probe': {
10      /** Written on session.start / classic.SessionStart (`<event>:<source>@<iso>`); read back after /clear. */
11      marker: ProbeMarker
12    }
13  }
14}
15
16