eval-kernel

1import type { EngineInterface, Register } from "claude-code";
2
3// eval-kernel: one `eval` tool backed by a long-lived Bun kernel (../kernel/kernel.ts) that keeps
4// JavaScript state between cells. A cell's host calls (tool.X, completion, agent, tools) come back
5// here as `call` events; this module runs them through $ and answers, so every tool a cell calls
6// passes Claude Code's permission checks and every other plugin's hooks.
7
8const TOOL = "eval";
9const TOOL_ID = "mcp__eval-kernel__eval";
10
11const DESCRIPTION = `Run JavaScript in a persistent Bun kernel, like a notebook cell.
12
13- Top-level \`const\`/\`let\`/\`function\`/\`class\` declarations and \`import\`s at the start of a line persist into later cells. A trailing expression is the cell's value. Top-level await works.
14- Bun, fetch and node:* modules are available. console.log and display(value) are captured.
15- Claude Code tools: \`await tool.Read({ file_path })\` resolves to the tool's text and throws if the tool errors; \`tool.call(name, args, { raw: true })\` returns { text, result, isError } for any tool, MCP tools included. \`await tools()\` lists them. Calls go through the normal permission checks.
16- Models and agents: \`await completion(prompt, { model: "haiku", system, maxTokens })\` returns text; \`await agent(prompt, { description, subagentType, model, schema })\` runs a subagent and returns its answer: the bare text, or with a JSON Schema \`schema\`, the parsed JSON value. Run several at once with Promise.all.
17- Decisions: \`await judge({ type: "noul", instructions: "Is it risky?" }, state)\` asks the decision model (decision-model mod; Jev when configured) one typed question (\`noul\` yes-probability, \`choice\` with \`criteria\` {label: rubric}, \`score\` with ordered \`criteria\`) and returns its answer.
18- Use it to fan work out (read 30 files, run 5 subagents, aggregate), to keep data between steps, or to compute instead of reasoning.
19- Pass \`reset: true\` to restart the kernel with empty state.`;
20
21type Event =
22  | { status: "call"; callId: string; kind: string; payload: any }
23  | { status: "done"; output: string; value?: string }
24  | { status: "error"; output: string; error: string };
25
26/** The kernel's socket; set once the session starts. */
27let sock = "";
28/** Subagents a running cell waits on: agentId → the kernel's callId and the schema its answer must match. */
29const agentCalls = new Map<string, { callId: string; schema?: unknown }>();
30
31/** Every subagent a cell started; their completion notices belong to the cell, not the parent. */
32// ponytail: grows for the session's life (a few ids per agent() call).
33const cellAgents = new Set<string>();
34
35/** The agent ids a delivered notice is about: task-notification tags or a hand-back's sender. */
36function noticeAgents(text: string): string[] {
37  return [...text.matchAll(/<task-id>([\w-]+)<\/task-id>|from="([\w-]+)"/g)].map(m => (m[1] ?? m[2])!);
38}
39
40/** The subagent's hand-back is a report to a program, so ask for the answer alone. */
41function agentPrompt(prompt: string, schema: unknown): string {
42  return schema === undefined
43    ? `${prompt}\n\nYour hand-back report is returned verbatim to a program as your answer: make it only the answer, with no preamble, summary or sign-off.`
44    : `${prompt}\n\nYour hand-back report is parsed by a program: make it only a JSON value matching this JSON Schema, with no prose and no code fences.\n${JSON.stringify(schema)}`;
45}
46
47/** The answer a cell's agent() resolves to; with a schema, the parsed JSON. */
48function agentAnswer(text: string, schema: unknown): { ok: true; value: unknown } | { ok: false; error: string } {
49  if (schema === undefined) return { ok: true, value: text.trim() };
50  // ponytail: parses only, no schema validation. Upgrade path: validate and send the subagent the errors to fix.
51  try {
52    return { ok: true, value: JSON.parse(text.trim().replace(/^```(?:json)?\s*|\s*```$/g, "")) };
53  } catch {
54    return { ok: false, error: `agent: the answer is not JSON: ${text.slice(0, 200)}` };
55  }
56}
57
58async function post($: EngineInterface, path: string, body: unknown = {}): Promise<any> {
59  const res = await $.http.fetch(`http://localhost${path}`, { method: "POST", socketPath: sock, body: JSON.stringify(body), headers: { "Content-Type": "application/json" } });
60  return JSON.parse(res.text);
61}
62
63/** Wait (up to ~5 s) for the kernel to answer /health. */
64async function ready($: EngineInterface): Promise<boolean> {
65  for (let i = 0; i < 50; i++) {
66    try {
67      const res = await $.http.fetch("http://localhost/health", { socketPath: sock });
68      if (res.ok) return true;
69    } catch {}
70    await $.clock.sleep(100);
71  }
72  return false;
73}
74
75/** Keep one kernel alive for the session: a child that exits (reset, crash) is started again. */
76async function supervise($: EngineInterface, argv: string[], cwd: string) {
77  // ponytail: fixed restart cap, no backoff. Upgrade path: back off on quick consecutive exits.
78  for (let starts = 0; starts < 50; starts++) {
79    try {
80      for await (const chunk of $.process.spawn({ argv, cwd })) $.ui.log(`eval-kernel kernel: ${chunk.text.trimEnd()}`, { to: "debug" });
81    } catch (err) {
82      $.ui.log(`eval-kernel kernel failed to start: ${String(err)}`, { to: "debug" });
83      await $.clock.sleep(1000);
84    }
85  }
86}
87
88/** Run one host call a cell made. Throws to make the cell's await reject. */
89async function hostCall($: EngineInterface, kind: string, p: any): Promise<unknown> {
90  if (kind === "tool") {
91    if (p.tool === TOOL_ID) throw new Error("a cell can't call eval itself");
92    const r = await $.tool.call({ tool: String(p.tool), ...(p.args ?? {}) });
93    if (r.deny !== undefined) return { isError: true, text: `denied: ${r.deny}` };
94    return { text: r.text ?? (typeof r.result === "string" ? r.result : JSON.stringify(r.result)), result: r.result, isError: r.isError === true };
95  }
96  if (kind === "tools") return await $.tool.list();
97  if (kind === "complete") {
98    const r = await $.model.complete({ model: String(p.model ?? "haiku"), prompt: String(p.prompt), system: p.system, maxTokens: p.maxTokens, effort: p.effort });
99    if (!r.isAnswered) throw new Error(`completion failed: ${r.reason}`);
100    return r.text;
101  }
102  if (kind === "judge") {
103    // Optional: decision-model adds $.decision when it is loaded; eval-kernel works without it.
104    type Ask = { decision: { ask: (r: object) => Promise<{ answers: Record<string, unknown> }> } };
105    let r: { answers: Record<string, unknown> };
106    try {
107      r = await ($ as unknown as Ask).decision.ask({ purpose: "eval", state: p.state, questions: { q: p.question } });
108    } catch (err) {
109      if (err instanceof TypeError && /decision/.test(err.message)) throw new Error("judge() needs the decision-model mod");
110      throw err;
111    }
112    return r.answers.q;
113  }
114  throw new Error(`unknown host call ${kind}`);
115}
116
117/** Run a cell to its end, answering its host calls. */
118async function runCell($: EngineInterface, code: string, maxCalls: number, signal?: AbortSignal): Promise<Event> {
119  if (!(await ready($))) return { status: "error", output: "", error: "the kernel is not running (is bun installed? see /eval)" };
120  // An interrupt stops the cell: no request can be cancelled, so the kernel is reset, which ends the
121  // wait below and drops the cell's pending subagent answers.
122  // ponytail: interrupting loses kernel state. Upgrade path: an /interrupt endpoint that ends only the running cell.
123  const interrupted = () => {
124    agentCalls.clear();
125    void post($, "/reset").catch(() => {});
126  };
127  signal?.addEventListener("abort", interrupted, { once: true });
128  try {
129    return await drive($, code, maxCalls, signal);
130  } catch (err) {
131    if (signal?.aborted) return { status: "error", output: "", error: "interrupted; the kernel was reset" };
132    throw err;
133  } finally {
134    signal?.removeEventListener("abort", interrupted);
135  }
136}
137
138async function drive($: EngineInterface, code: string, maxCalls: number, signal?: AbortSignal): Promise<Event> {
139  let ev: Event = await post($, "/run", { code, cwd: await $.session.cwd() });
140  let calls = 0;
141  while (ev.status === "call") {
142    if (signal?.aborted) return { status: "error", output: "", error: "interrupted; the kernel was reset" };
143    if (++calls > maxCalls) {
144      await post($, "/reset").catch(() => {});
145      return { status: "error", output: "", error: `the cell made more than ${maxCalls} host calls; the kernel was reset` };
146    }
147    const { callId, kind, payload } = ev;
148    if (kind === "agent") {
149      // Answered later by the subagent's own turn.complete (below); meanwhile take the next event.
150      try {
151        const spawned = await $.agent.spawn({ prompt: agentPrompt(String(payload.prompt), payload.schema), description: String(payload.description ?? "eval cell subagent"), subagentType: payload.subagentType, model: payload.model });
152        if (spawned.deny !== undefined || !spawned.agentId) throw new Error(spawned.deny ?? "no agent id");
153        agentCalls.set(spawned.agentId, { callId, schema: payload.schema });
154        cellAgents.add(spawned.agentId);
155        ev = await post($, "/next");
156      } catch (err) {
157        ev = await post($, "/resume", { callId, ok: false, error: `agent: ${String(err)}` });
158      }
159      continue;
160    }
161    try {
162      ev = await post($, "/resume", { callId, ok: true, value: await hostCall($, kind, payload) });
163    } catch (err) {
164      ev = await post($, "/resume", { callId, ok: false, error: err instanceof Error ? err.message : String(err) });
165    }
166  }
167  return ev;
168}
169
170function render(ev: Event): string {
171  if (ev.status === "error") return [ev.output, `Error: ${ev.error}`].filter(Boolean).join("\n");
172  if (ev.status === "done") return [ev.output, ev.value !== undefined ? `=> ${ev.value}` : ""].filter(Boolean).join("\n") || "(no output)";
173  return "(cell still waiting)";
174}
175
176export const register: Register = (on, options) => {
177  const bun = String(options.bun || "bun");
178  const maxCalls = Number(options.maxCalls ?? 200);
179
180  on("session.start", async ($, e, next) => {
181    const id = await $.session.id();
182    // Unix socket paths are capped near 104 bytes, so /tmp rather than the plugin's folder.
183    sock = `/tmp/eval-kernel-${id.slice(0, 8)}-${Math.random().toString(36).slice(2, 8)}.sock`;
184    void supervise($, [bun, `${$.plugin.root}/kernel/kernel.ts`, sock], e.cwd);
185    await $.tool.register({
186      name: TOOL,
187      description: DESCRIPTION,
188      inputSchema: {
189        type: "object",
190        properties: {
191          code: { type: "string", description: "JavaScript to run as one cell" },
192          title: { type: "string", description: "Short label for the cell" },
193          reset: { type: "boolean", description: "Restart the kernel with empty state before running" },
194        },
195        required: ["code"],
196      },
197    });
198    await $.command.register({ name: "eval", description: "eval-kernel kernel: `/eval vars`, `/eval reset`, or `/eval <code>` to run a cell yourself", argumentHint: "vars | reset | <code>" });
199    return next(e);
200  });
201
202  on("tool.call", { tool: TOOL_ID }, async ($, e, next) => {
203    const input = e as unknown as { code?: unknown; reset?: unknown };
204    if (input.reset === true) {
205      await post($, "/reset").catch(() => {});
206      await $.clock.sleep(200);
207    }
208    if (typeof input.code !== "string" || !input.code.trim()) return input.reset === true ? { result: "kernel reset" } : { isError: true, result: "code is required" };
209    const ev = await runCell($, input.code, maxCalls, next.signal);
210    return ev.status === "error" ? { isError: true, result: render(ev) } : { result: render(ev) };
211  });
212
213  // A subagent hands its report back with the SubagentHandback tool; that report answers the
214  // cell's agent() call.
215  // ponytail: matched by hand because SubagentHandback is missing from this build's typed tool list.
216  on("tool.call", async ($, e, next) => {
217    const call = (e.tool as string) === "SubagentHandback" && e.agentId ? agentCalls.get(e.agentId) : undefined;
218    if (e.agentId && call) {
219      agentCalls.delete(e.agentId);
220      const message = (e as unknown as { message?: unknown }).message;
221      const answer = agentAnswer(typeof message === "string" ? message : JSON.stringify(message), call.schema);
222      await post($, "/answer", { callId: call.callId, ...answer }).catch(() => {});
223      // The cell is the recipient: answering the hand-back here keeps it out of the parent's
224      // conversation, where each report would otherwise start a turn of its own.
225      return { result: "Delivered to the eval cell that started you." };
226    }
227    return next(e);
228  });
229
230  // A cell's subagent runs in the background, so its end is also announced to the parent session,
231  // which would spend a turn on each one. The cell already has the answer: drop those notices.
232  on("prompt.submit", async ($, e, next) => {
233    const ids = e.origin.kind === "task-notification" || e.origin.kind === "peer" || e.origin.kind === "peer-send-message" ? noticeAgents(e.text) : [];
234    if (ids.length && ids.every(id => cellAgents.has(id))) return { drop: "eval-kernel: a cell's subagent finished; the cell has its answer" };
235    return next(e);
236  });
237
238  // A subagent that ended without handing back: its final text (if any) answers the call.
239  on("turn.complete", async ($, e, next) => {
240    const call = e.agentId ? agentCalls.get(e.agentId) : undefined;
241    if (e.agentId && call) {
242      agentCalls.delete(e.agentId);
243      const answer = e.reason === "answer" ? agentAnswer(e.answer, call.schema) : { ok: false, error: `subagent ended: ${e.reason}` };
244      await post($, "/answer", { callId: call.callId, ...answer }).catch(() => {});
245    }
246    return next(e);
247  });
248
249  on("command.run", { command: "eval" }, async ($, e, next) => {
250    const arg = e.args.trim();
251    if (!(await ready($))) return { text: `kernel not running at ${sock} (runtime: ${bun})`, exitCode: 1 };
252    if (arg === "" || arg === "vars") {
253      const res = await $.http.fetch("http://localhost/vars", { socketPath: sock });
254      const vars = (JSON.parse(res.text).vars as { name: string; type: string }[]).map(v => `${v.name}: ${v.type}`);
255      return { text: vars.length ? vars.join("\n") : "no variables yet" };
256    }
257    if (arg === "reset") {
258      await post($, "/reset").catch(() => {});
259      return { text: "kernel reset" };
260    }
261    const ev = await runCell($, arg, maxCalls, next.signal);
262    return { text: render(ev), exitCode: ev.status === "error" ? 1 : 0 };
263  });
264};
265