jev-pilot

1import type { EngineInterface, Register } from "claude-code";
2
3import { buildCandidates, coversStep, sameCall, SKILL_BLOCKLIST_KEY } from "./candidates";
4import type { Candidate } from "./candidates";
5import { clip, previousReply, readTurn, summarize, toolCalls } from "./context";
6import { askJev, choice, noul, score } from "./jev";
7import type { ChoiceAnswer, JevResult, Question } from "./jev";
8import { clearLog, readLog, record, recentDecisions, stats } from "./log";
9import { buildReport } from "./report";
10import { registerView } from "./view";
11import { currentMode, MODES } from "./mode";
12import { registerPromptClassifier } from "./prompt";
13import { registerAgentTypes } from "./agents";
14import { registerCompaction } from "./compact";
15import { summarizeResult, SUMMARY_MODEL } from "./summarize";
16import {
17    bestAction,
18    bashRiskQuestion,
19    difficultyQuestion,
20    singleActionQuestion,
21    doneQuestion,
22    gateBash,
23    gateEdit,
24    inScopeQuestion,
25    adjustEffort,
26    effortKeepsCache,
27    lowerModel,
28    nextActionQuestion,
29    pickCandidate,
30    route,
31    shouldContinue,
32} from "./policy";
33import type { GateThresholds, Route } from "./policy";
34
35const MAX_TAKES_PER_TURN = 3;
36const GATED = new Set(["Bash", "Edit", "Write", "NotebookEdit"]);
37
38/** Short form of a step's tool calls: `Edit app.js ×3, Bash npm test`. */
39function summarizeCalls(
40    calls: readonly { tool: string; input: Record<string, unknown> }[],
41): string {
42    if (!calls.length) return "replied";
43    const parts = new Map<string, number>();
44    for (const c of calls) {
45        const target =
46            typeof c.input.file_path === "string"
47                ? c.input.file_path.split("/").pop()
48                : typeof c.input.command === "string"
49                  ? c.input.command.replace(/\s+/g, " ").slice(0, 30)
50                  : typeof c.input.skill === "string"
51                    ? c.input.skill
52                    : "";
53        const key = target ? `${c.tool} ${target}` : c.tool;
54        parts.set(key, (parts.get(key) ?? 0) + 1);
55    }
56    return [...parts]
57        .map(([k, n]) => (n > 1 ? `${k} ×${n}` : k))
58        .join(", ");
59}
60
61/**
62 * Logs Claude's step next to Jev's top action. `agree` means Jev's action
63 * would have replaced Claude's whole step (one call, same command once output
64 * filters and flags are set aside), so taking it saves one Claude request.
65 */
66async function recordCompare(
67    $: EngineInterface,
68    turnId: string,
69    step: number,
70    request: string,
71    answer: ChoiceAnswer,
72    candidates: readonly Candidate[],
73    toolUses: readonly { name: string; input?: unknown }[],
74    measured: boolean,
75) {
76    const best = bestAction(answer);
77    const claudeCalls = toolUses.map((u) => ({
78        tool: u.name,
79        input: (u.input ?? {}) as Record<string, unknown>,
80    }));
81    const bestCandidate = candidates.find((c) => c.key === best?.key);
82    const agree = Boolean(bestCandidate && coversStep(claudeCalls, bestCandidate));
83    const claudeUsedCandidate = claudeCalls.some((c) =>
84        candidates.some((k) => coversStep([c], k)),
85    );
86    const claudeSummary = summarizeCalls(claudeCalls);
87    await record($, {
88        kind: "compare",
89        applied: false,
90        text: `Claude: ${claudeSummary} · Jev best ${best?.key ?? "none"}${agree ? " ✓" : ""}`,
91        turnId,
92        data: {
93            request: clip(request, 120),
94            step,
95            measured,
96            claudeSummary,
97            pick: answer.choice,
98            best: best?.key,
99            bestP: best?.p,
100            claude: claudeCalls.map((c) => c.tool),
101            claudeCalls: claudeCalls.map((c) =>
102                clip(String(c.input.command ?? c.input.file_path ?? c.input.skill ?? c.input.query ?? ""), 200),
103            ),
104            agree,
105            claudeUsedCandidate,
106        },
107    });
108}
109
110async function blockSkill($: EngineInterface, name: string) {
111    const list = ((await $.store.get(SKILL_BLOCKLIST_KEY)) as string[] | undefined) ?? [];
112    if (!list.includes(name))
113        await $.store.set(SKILL_BLOCKLIST_KEY, [...list, name]);
114}
115
116
117export const register: Register = (on, options) => {
118    const routing =
119        options.routing === "model" || options.routing === "off"
120            ? options.routing
121            : "effort";
122    const takeSteps = options.takeSteps !== false;
123    const gate = options.gate !== false;
124    const doneCheck = options.doneCheck !== false;
125    const takeThreshold =
126        typeof options.takeThreshold === "number"
127            ? options.takeThreshold
128            : 0.85;
129    const subagents = options.subagents !== false;
130    const bypass = options.bypass !== false;
131    // Benchmark (2026-10-14): taking steps inside multi-step coding turns only
132    // added turns (+6, +3% cost); taking pays off when the whole turn is bypassed.
133    const takeOnlyForBypass = options.takeOnlyForBypass !== false;
134    const measureSteps = options.measureSteps !== false;
135    const SINGLE_ACTION = 0.8;
136    const threshold = (key: string, fallback: number) =>
137        typeof options[key] === "number" ? (options[key] as number) : fallback;
138    const gateThresholds: GateThresholds = {
139        dangerAsk: threshold("dangerAsk", 0.5),
140        dangerDeny: threshold("dangerDeny", 0.8),
141        scopeDeny: threshold("scopeDeny", 0.2),
142        editScope: threshold("editScope", 0.15),
143    };
144    const doneThreshold = threshold("doneThreshold", 0.15);
145
146    registerView(on);
147    registerPromptClassifier(on, options, takeThreshold);
148    registerAgentTypes(on, options);
149    registerCompaction(on, options);
150
151    const turns = new Map<
152        string,
153        { route?: Route; taken: number; bypassId?: string; bypassAction?: string }
154    >();
155    // Calls Jev made itself skip the gate: their commands come from a vetted list.
156    const ownCalls = new Set<string>();
157
158    on("session.start", async ($, e, next) => {
159        await $.command.register({
160            name: "jev",
161            description:
162                "Jev pilot: status, /jev report, /jev export, /jev clear, or switch mode with /jev active|shadow|off",
163        });
164        return next(e);
165    });
166
167    on("command.run", { command: "jev" }, async ($, e) => {
168        const arg = e.args.trim();
169        if ((MODES as readonly string[]).includes(arg)) {
170            await $.store.set("mode", arg);
171            return { text: `Jev pilot mode: ${arg}` };
172        }
173        if (arg === "report") {
174            return {
175                text: buildReport(
176                    await readLog($),
177                    takeThreshold,
178                    await currentMode($, options.mode),
179                ),
180            };
181        }
182        if (arg === "candidates") {
183            // Exactly what Jev is offered at step 0 for the last prompt.
184            const ctx = await readTurn($);
185            const calls = ctx ? toolCalls(ctx.steps) : [];
186            const list = await buildCandidates($, ctx?.request ?? "", ctx?.steps ?? [], calls, 0);
187            const kind = (c: Candidate) =>
188                c.tool === "Skill" ? "Skills" : c.tool === "ToolSearch" ? "Tools to load" : c.tool === "Read" ? "File reads" : "Commands";
189            const groups = new Map<string, Candidate[]>();
190            for (const c of list) groups.set(kind(c), [...(groups.get(kind(c)) ?? []), c]);
191            const sections = [...groups].map(
192                ([name, items]) =>
193                    `### ${name} (${items.length})\n\n| Option | Description |\n|---|---|\n` +
194                    items.map((c) => `| \`${c.key.replace(/\|/g, "\\|")}\` | ${c.description.replace(/\|/g, "\\|").replace(/\s+/g, " ")} |`).join("\n"),
195            );
196            return {
197                text: [
198                    `## Jev candidates: ${list.length + 1} options`,
199                    `_For the last prompt: "${clip(ctx?.request ?? "(none)", 60)}". Plus \`claude\`: leave the step to Claude._`,
200                    ...sections,
201                ].join("\n\n"),
202            };
203        }
204        if (arg === "clear") {
205            await clearLog($);
206            return { text: "Jev pilot: log cleared." };
207        }
208        if (arg === "export") {
209            const log = await readLog($);
210            const path = `${await $.session.cwd()}/.jev/decisions.jsonl`;
211            try {
212                await $.fs.write(
213                    path,
214                    log.map((entry) => JSON.stringify(entry)).join("\n") + "\n",
215                );
216                return { text: `Jev pilot: wrote ${log.length} entries to ${path}` };
217            } catch (err) {
218                return {
219                    text: `Jev pilot: could not write ${path}: ${err instanceof Error ? err.message : String(err)}`,
220                };
221            }
222        }
223        const counts =
224            Object.entries(await stats($))
225                .map(([k, v]) => `${k}=${v}`)
226                .join(", ") || "none yet";
227        const lines = recentDecisions()
228            .slice(-10)
229            .map(
230                (d) =>
231                    `- ${d.kind}${d.applied ? "" : " (shadow)"}: ${d.text}${d.ms !== undefined ? ` [${Math.round(d.ms)}ms]` : ""}`,
232            );
233        return {
234            text: [
235                `Jev pilot: mode ${await currentMode($, options.mode)}, routing ${routing}, take steps ${takeSteps} (p ≥ ${takeThreshold}), gate ${gate}, done check ${doneCheck}, subagents ${subagents}`,
236                `Decisions (all sessions): ${counts}`,
237                lines.length
238                    ? `Recent:\n${lines.join("\n")}`
239                    : "No decisions this session yet.",
240                "Full analysis: /jev report",
241            ].join("\n"),
242        };
243    });
244
245    on("turn.complete", async ($, e, next) => {
246        turns.delete(e.turnId);
247        if (e.reason === "aborted" || e.reason === "error")
248            await record($, {
249                kind: "turn",
250                applied: false,
251                text: `turn ${e.reason}`,
252                turnId: e.turnId,
253                data: { reason: e.reason },
254            });
255        return next(e);
256    });
257
258    // Outcome of each call Jev made itself, for the report.
259    on("tool.call", async ($, e, next) => {
260        const result = await next(e);
261        if (e.tool_use_id && ownCalls.has(e.tool_use_id)) {
262            const isError = result.deny !== undefined || result.isError === true;
263            await record($, {
264                kind: "outcome",
265                applied: true,
266                text: `${e.tool} ${isError ? "failed" : "ok"}`,
267                data: { id: e.tool_use_id, isError },
268            });
269            // A skill the Skill tool refused is never offered to Jev again.
270            const skill = (e as { skill?: unknown }).skill;
271            if (isError && e.tool === "Skill" && typeof skill === "string")
272                await blockSkill($, skill);
273        }
274        return result;
275    });
276
277    // Each model request: Jev may take the step itself, and at step 0 rates the
278    // turn's difficulty to lower effort or model. One Jev request carries both.
279    // Every answer is logged; when Claude takes the step, its call is compared
280    // with Jev's top action so shadow runs can calibrate the threshold.
281    on("turn.step", async function* ($, e, next) {
282        const m = await currentMode($, options.mode);
283        if (m === "off" || (e.agentId && !subagents)) return yield* next(e);
284
285        const turn = turns.get(e.turnId) ?? { taken: 0 };
286        turns.set(e.turnId, turn);
287
288        const ctx = await readTurn($, e.agentId);
289        if (!ctx) return yield* next(e);
290        const calls = toolCalls(ctx.steps);
291        const last = calls.at(-1);
292
293        // Whole-turn bypass: Jev took the one action this simple request needs;
294        // a small model reports its result and Claude is not asked at all.
295        if (turn.bypassId && e.index > 0) {
296            const id = turn.bypassId;
297            turn.bypassId = undefined;
298            const done = calls.find((c) => c.tool_use_id === id);
299            if (m === "active" && done?.text !== undefined) {
300                const s = await summarizeResult(
301                    $,
302                    ctx.request,
303                    turn.bypassAction ?? done.tool,
304                    done.text,
305                );
306                await record($, {
307                    kind: "bypass",
308                    applied: s.ok,
309                    text: s.ok
310                        ? `answered with ${SUMMARY_MODEL} after ${turn.bypassAction}`
311                        : `summary failed (${s.reason}), Claude answers`,
312                    turnId: e.turnId,
313                    data: {
314                        action: turn.bypassAction,
315                        model: SUMMARY_MODEL,
316                        ...(s.ok ? { usage: s.usage } : { reason: s.reason }),
317                    },
318                });
319                if (s.ok) {
320                    const usage = { ...s.usage, model: SUMMARY_MODEL };
321                    yield { kind: "text", index: 0, text: s.text };
322                    yield { kind: "stop", stopReason: "end_turn", usage };
323                    return {
324                        turnId: e.turnId,
325                        index: e.index,
326                        answer: s.text,
327                        toolUses: [],
328                        stopReason: "end_turn",
329                        usage,
330                    };
331                }
332            }
333        }
334
335        const questions: Record<string, Question> = {};
336        let candidates: Candidate[] = [];
337        if (
338            takeSteps &&
339            turn.taken < MAX_TAKES_PER_TURN &&
340            (!takeOnlyForBypass || (bypass && e.index === 0 && !e.agentId))
341        ) {
342            candidates = (
343                await buildCandidates($, ctx.request, ctx.steps, calls, e.index)
344            ).filter(
345                (c) =>
346                    !(last && sameCall(last, { tool: c.tool, input: c.input })),
347            );
348            if (candidates.length)
349                questions.next_action = nextActionQuestion(candidates);
350        }
351        if (routing !== "off" && e.index === 0)
352            questions.difficulty = difficultyQuestion;
353        if (bypass && e.index === 0 && questions.next_action && !e.agentId)
354            questions.single_action = singleActionQuestion;
355        // Effort lowered at step 0 holds for every step of the turn.
356        const routed = (step: typeof e) => {
357            if (m !== "active" || !turn.route) return step;
358            const effort = effortKeepsCache(step.model)
359                ? adjustEffort(step.effort, turn.route.effort)
360                : undefined;
361            const model = lowerModel(step.model, turn.route.model);
362            return effort || model
363                ? { ...step, ...(effort ? { effort } : {}), ...(model ? { model } : {}) }
364                : step;
365        };
366
367        if (!Object.keys(questions).length) {
368            // Inside a longer turn Jev doesn't take steps, but is asked in
369            // parallel with Claude which action it would take, so the take
370            // threshold can be measured without adding a delay.
371            let measure: Promise<JevResult> | undefined;
372            let measured: Candidate[] = [];
373            if (measureSteps && takeSteps && e.index > 0 && !e.agentId) {
374                measured = (
375                    await buildCandidates($, ctx.request, ctx.steps, calls, e.index)
376                ).filter((c) => !(last && sameCall(last, { tool: c.tool, input: c.input })));
377                if (measured.length)
378                    measure = askJev(
379                        $,
380                        { request: clip(ctx.request, 4000), recent: summarize(ctx.steps), cwd: await $.session.cwd() },
381                        { next_action: nextActionQuestion(measured) },
382                    );
383            }
384            const result = yield* next(routed(e));
385            const r = await measure;
386            const answer = r?.ok ? choice(r.answers, "next_action") : undefined;
387            if (answer)
388                await recordCompare($, e.turnId, e.index, ctx.request, answer, measured, result.toolUses, true);
389            return result;
390        }
391
392        // A short follow-up ("yes", "do it") is only rated with confidence
393        // when Jev sees what it answers: 0.0–0.2 without, 0.5–0.8 with.
394        const previous =
395            e.index === 0 && !e.agentId ? await previousReply($) : undefined;
396        const state = {
397            request: clip(ctx.request, 4000),
398            ...(previous ? { previous_reply: previous } : {}),
399            recent: summarize(ctx.steps),
400            cwd: await $.session.cwd(),
401        };
402        const r = await askJev($, state, questions);
403        if (!r.ok) {
404            await record($, {
405                kind: "error",
406                applied: false,
407                text: `unavailable: ${r.error}`,
408                ms: r.ms,
409                turnId: e.turnId,
410            });
411            return yield* next(e);
412        }
413
414        if (questions.difficulty) {
415            const difficulty = score(r.answers, "difficulty");
416            turn.route = route(
417                difficulty,
418                routing === "model" ? "model" : "effort",
419            );
420            const effort = effortKeepsCache(e.model)
421                ? adjustEffort(e.effort, turn.route?.effort)
422                : undefined;
423            const model = lowerModel(e.model, turn.route?.model);
424            const raised = effort === "high";
425            const lowered = Boolean((effort && !raised) || model);
426            await record($, {
427                kind: "route",
428                applied: m === "active" && (lowered || raised),
429                text: `difficulty ${(difficulty?.score ?? 0).toFixed(2)} → ${lowered || raised ? `${model ?? e.model}, effort ${effort ?? e.effort}` : effortKeepsCache(e.model) ? "no change" : `no change (effort is left alone on ${e.model})`}`,
430                ms: r.ms,
431                turnId: e.turnId,
432                data: {
433                    score: difficulty?.score,
434                    confidence: difficulty?.confidence,
435                    effort,
436                    model,
437                    lowered,
438                    raised,
439                },
440            });
441        }
442
443        const answer = choice(r.answers, "next_action");
444        const best = bestAction(answer);
445        const singleAction = noul(r.answers, "single_action") ?? 0;
446        const pick =
447            takeOnlyForBypass && singleAction < SINGLE_ACTION
448                ? undefined
449                : pickCandidate(answer, candidates, takeThreshold);
450        if (answer) {
451            const p = answer.probabilities[answer.choice] ?? 0;
452            await record($, {
453                kind: "take",
454                applied: m === "active" && pick !== undefined,
455                text: `${answer.choice} (p=${p.toFixed(2)})${pick ? "" : best ? `; best action ${best.key} p=${best.p.toFixed(2)}` : ""}`,
456                ms: r.ms,
457                turnId: e.turnId,
458                data: {
459                    pick: answer.choice,
460                    p,
461                    best: best?.key,
462                    bestP: best?.p,
463                    candidates: candidates.length,
464                    threshold: takeThreshold,
465                    wouldTake: pick !== undefined,
466                    singleAction: noul(r.answers, "single_action"),
467                    agentId: e.agentId,
468                },
469            });
470        }
471
472        if (pick && m === "active") {
473            turn.taken += 1;
474            const id = `toolu_jev_${crypto.randomUUID().replace(/-/g, "")}`;
475            ownCalls.add(id);
476            if (singleAction >= SINGLE_ACTION) {
477                turn.bypassId = id;
478                turn.bypassAction = pick.key;
479            }
480            yield { kind: "tool", index: 0, id, name: pick.tool };
481            yield { kind: "input", index: 0, json: JSON.stringify(pick.input) };
482            yield { kind: "stop", stopReason: "tool_use", usage: null };
483            return {
484                turnId: e.turnId,
485                index: e.index,
486                answer: "",
487                toolUses: [{ name: pick.tool, input: pick.input }],
488                stopReason: "tool_use",
489                usage: null,
490            };
491        }
492
493        const result = yield* next(routed(e));
494
495        // Compare Claude's own next call with Jev's top action.
496        if (answer)
497            await recordCompare($, e.turnId, e.index, ctx.request, answer, candidates, result.toolUses, false);
498        return result;
499    });
500
501    // Permission gate: only ever tightens what Claude Code already allows.
502    on("tool.check", async ($, e, next) => {
503        const base = await next(e);
504        // Jev's own calls come from the vetted candidate list: approve them so auto
505        // mode's classifier doesn't refuse them. A settings deny rule still wins.
506        if (e.tool_use_id && ownCalls.has(e.tool_use_id)) {
507            if (base.decision === "deny") return base;
508            return {
509                ...base,
510                decision: "allow",
511                reason: "Jev pilot: vetted action from its read-only/project-script list",
512            };
513        }
514        if (
515            !gate ||
516            !GATED.has(e.tool) ||
517            (base.decision !== "allow" && base.decision !== "ask") ||
518            !e.tool_use_id
519        )
520            return base;
521        const m = await currentMode($, options.mode); // ← add
522        if (m === "off") return base;
523        const ctx = await readTurn($, e.agentId);
524        const input = (e.input ?? {}) as Record<string, unknown>;
525        const request = clip(ctx?.request ?? "", 3000);
526        const cwd = await $.session.cwd();
527
528        let verdict;
529        if (e.tool === "Bash") {
530            const r = await askJev(
531                $,
532                { request, command: String(input.command ?? ""), cwd },
533                {
534                    risk: bashRiskQuestion,
535                    in_scope: inScopeQuestion(
536                        "running the command in `command`",
537                    ),
538                },
539            );
540            if (!r.ok) return base;
541            const risk = score(r.answers, "risk");
542            const inScope = noul(r.answers, "in_scope");
543            verdict = gateBash(risk, inScope, gateThresholds);
544            await record($, {
545                kind: "gate",
546                applied: m === "active" && verdict !== undefined,
547                text: `${verdict?.decision ?? "allow"} Bash: ${clip(String(input.command), 50)}`,
548                ms: r.ms,
549                data: {
550                    tool: "Bash",
551                    target: clip(String(input.command), 200),
552                    decision: verdict?.decision ?? "allow",
553                    danger: risk?.probabilities["2"],
554                    inScope,
555                },
556            });
557        } else {
558            const change = clip(
559                String(
560                    input.new_string ?? input.content ?? input.new_source ?? "",
561                ),
562                1200,
563            );
564            const file = String(input.file_path ?? input.notebook_path ?? "");
565            const r = await askJev(
566                $,
567                { request, tool: e.tool, file, change, cwd },
568                {
569                    in_scope: inScopeQuestion(
570                        "this change to the file in `file`",
571                    ),
572                },
573            );
574            if (!r.ok) return base;
575            const inScope = noul(r.answers, "in_scope");
576            verdict = gateEdit(inScope, gateThresholds);
577            await record($, {
578                kind: "gate",
579                applied: m === "active" && verdict !== undefined,
580                text: `${verdict?.decision ?? "allow"} ${e.tool}: ${file}`,
581                ms: r.ms,
582                data: {
583                    tool: e.tool,
584                    target: file,
585                    decision: verdict?.decision ?? "allow",
586                    inScope,
587                },
588            });
589        }
590        // Only ever tighter: allow → ask/deny, ask → deny.
591        if (!verdict || m !== "active") return base;
592        const tighter =
593            verdict.decision === "deny" || base.decision === "allow";
594        return tighter
595            ? { ...base, decision: verdict.decision, reason: verdict.reason }
596            : base;
597    });
598
599    // Done check: when Claude stops, send it back once if the request looks unfinished.
600    on("classic.Stop", async ($, e, next) => {
601        if (!doneCheck || e.stop_hook_active) return next(e);
602        const m = await currentMode($, options.mode);
603        if (m === "off") return next(e);
604        // Waiting on background work (a subagent, a background command) is not
605        // stopping early: that work will wake the session when it finishes.
606        const waitingOn = e.background_tasks?.length ?? 0;
607        if (waitingOn > 0) {
608            await record($, {
609                kind: "done",
610                applied: false,
611                text: `skipped: waiting on ${waitingOn} background task(s)`,
612                data: { skipped: "background", continued: false },
613            });
614            return next(e);
615        }
616
617        const ctx = await readTurn($);
618        if (!ctx) return next(e);
619        const r = await askJev(
620            $,
621            {
622                request: clip(ctx.request, 3000),
623                final_message: clip(e.last_assistant_message ?? "", 2000),
624                recent: summarize(ctx.steps),
625            },
626            { done: doneQuestion },
627        );
628        if (!r.ok) return next(e);
629        const done = noul(r.answers, "done");
630        const unfinished = shouldContinue(done, doneThreshold);
631        await record($, {
632            kind: "done",
633            applied: m === "active" && unfinished,
634            text: unfinished
635                ? `not finished (p=${(done ?? 0).toFixed(2)}), continuing`
636                : `finished (p=${(done ?? 0).toFixed(2)})`,
637            ms: r.ms,
638            data: { p: done, continued: unfinished },
639        });
640        if (!unfinished) return next(e);
641        const below = await next(e);
642        if (m !== "active" || below.block) return below;
643        return {
644            ...below,
645            block: "Jev: the user's request doesn't look finished yet. Continue with what remains. If it is actually complete, say so in one line.",
646        };
647    });
648};
649

1import type { EngineInterface, SessionMessage, ToolUseSummary } from 'claude-code'
2
3// Actions Jev may take itself. Code builds the list; Jev only picks from it, so
4// every argument is a known value and never generated text.
5export type Candidate = {
6  /** Option key shown to Jev, e.g. `read src/app.ts` or `run npm test`. */
7  key: string
8  description: string
9  tool: 'Read' | 'Bash' | 'Skill' | 'ToolSearch'
10  input: Record<string, unknown>
11}
12
13const MAX_READS = 8
14const MAX_SEARCHES = 3
15const SAFE_SCRIPT = /^(test|tests|lint|build|typecheck|type-check|check|test:[\w:-]+|lint:[\w:-]+)$/
16// Built-in CLI commands the command list can report as user commands but the
17// Skill tool refuses (`/diff` did). Failed skills are also blocklisted at run time.
18export const BUILTIN_COMMANDS = new Set(
19  ('add-dir agents bug clear compact config context cost diff doctor exit export help hooks ide init ' +
20    'install-github-app login logout mcp memory model permissions plugin pr-comments release-notes resume ' +
21    'review rewind status statusline terminal-setup theme upgrade usage vim').split(' '),
22)
23
24export const SKILL_BLOCKLIST_KEY = 'skillBlocklist'
25
26const SAFE_MAKE = /^(test|tests|check|lint|build|typecheck)$/
27// Paths like src/app.ts, ./README.md, /abs/x.json: something with a file extension.
28const PATH = /(?:^|[\s`'"(\[])((?:~|\.{1,2})?\/?[\w@.-]+(?:\/[\w@.-]+)*\.[A-Za-z][A-Za-z0-9]{0,7})(?=$|[\s`'"),:;\]])/g
29// Terms the user quoted: `useAuth`, "TODO: fix".
30const QUOTED = /`([^`\n]{3,60})`|"([^"\n]{3,60})"/g
31
32export function mentionedPaths(texts: readonly string[]): string[] {
33  const seen = new Set<string>()
34  for (const text of texts) {
35    for (const m of text.matchAll(PATH)) {
36      const p = m[1]!
37      if (/^\d/.test(p) || /^https?:/.test(p) || p.includes('..')) continue
38      seen.add(p)
39    }
40  }
41  return [...seen]
42}
43
44export function quotedTerms(text: string): string[] {
45  const seen = new Set<string>()
46  for (const m of text.matchAll(QUOTED)) {
47    const term = (m[1] ?? m[2] ?? '').trim()
48    if (term.length >= 3 && !term.includes("'") && !/^[\w./@~-]+\.[A-Za-z]{1,8}$/.test(term)) seen.add(term)
49  }
50  return [...seen].slice(0, MAX_SEARCHES)
51}
52
53// Never offered for reading: Claude Code's temp and task folders (a subagent's
54// `.output` file is its whole transcript and floods the context), system temp
55// dirs, and Claude's own config tree.
56const UNSAFE_PATH = /(^|\/)(tmp|private\/tmp|private\/var\/folders|var\/folders)\/|\/\.claude\/|\/tasks\/[^/]+\.output$|\.output$|\.jsonl$/
57// Inside the project only transcripts are refused, so a project that lives in
58// a temp dir (a scratch copy, a benchmark) can still be read.
59const TRANSCRIPT = /\/tasks\/[^/]+\.output$|\.output$|\.jsonl$/
60export const isSafeToOffer = (file: string, cwd?: string) =>
61  cwd && file.startsWith(`${cwd}/`) ? !TRANSCRIPT.test(file) : !UNSAFE_PATH.test(file)
62
63const absolute = (cwd: string, p: string) => (p.startsWith('/') ? p : `${cwd}/${p.replace(/^\.\//, '')}`)
64
65export function sameCall(a: { tool: string; input: Record<string, unknown> }, b: { tool: string; input: Record<string, unknown> }) {
66  if (a.tool !== b.tool) return false
67  if (a.tool === 'Read') return a.input.file_path === b.input.file_path
68  if (a.tool === 'Bash') return String(a.input.command).trim() === String(b.input.command).trim()
69  if (a.tool === 'Skill') return a.input.skill === b.input.skill
70  if (a.tool === 'ToolSearch') return a.input.query === b.input.query
71  return false
72}
73
74/**
75 * What a shell command does, for comparison: each `&&`/`;`/newline part, cut
76 * at its first pipe (`| tail`, `| grep` only filter the output), without
77 * redirections, `cd` and `echo`, reduced to its command words (flags, paths
78 * and numbers dropped). `git status; git diff --stat 2>&1 | head` and
79 * `git status --short && git diff --stat` both give `git diff|git status`.
80 */
81export function commandKey(command: string): string {
82  const parts = new Set<string>()
83  for (const raw of command.split(/&&|\|\||;|\n/)) {
84    const seg = raw.split(/(?<!\|)\|(?!\|)/)[0]!.replace(/\d?>&\d|\d?>\s*\/dev\/null/g, '').trim()
85    if (!seg || /^(cd|echo|printf|set|export)\b/.test(seg)) continue
86    const tokens = seg.split(/\s+/)
87    const words = [tokens[0]!, ...tokens.slice(1).filter(t => /^[a-z][\w:-]*$/.test(t))]
88    parts.add(words.slice(0, words[1] === 'run' || words[1] === 'exec' ? 3 : 2).join(' '))
89  }
90  return [...parts].sort().join('|')
91}
92
93/** Same action: identical, or for shell commands the same `commandKey`. */
94export function equivalentCall(a: { tool: string; input: Record<string, unknown> }, b: { tool: string; input: Record<string, unknown> }) {
95  if (sameCall(a, b)) return true
96  if (a.tool !== 'Bash' || b.tool !== 'Bash') return false
97  const ka = commandKey(String(a.input.command ?? ''))
98  return ka !== '' && ka === commandKey(String(b.input.command ?? ''))
99}
100
101/** Jev's action would have replaced Claude's whole step: one call, equivalent. */
102export function coversStep(claude: readonly { tool: string; input: Record<string, unknown> }[], jev: { tool: string; input: Record<string, unknown> }) {
103  return claude.length === 1 && equivalentCall(claude[0]!, jev)
104}
105
106const bash = (command: string, description: string): Candidate => ({
107  key: `run ${command}`,
108  description,
109  tool: 'Bash',
110  input: { command, description },
111})
112
113export async function buildCandidates(
114  $: EngineInterface,
115  request: string,
116  steps: readonly SessionMessage[],
117  calls: readonly ToolUseSummary[],
118  stepIndex: number,
119): Promise<Candidate[]> {
120  const cwd = await $.session.cwd()
121  const out: Candidate[] = []
122  const has = (file: string) => $.fs.exists(`${cwd}/${file}`).catch(() => false)
123
124  // Files named in the request or the turn's text/tool output, not yet read this turn.
125  const texts = [request, ...steps.map(m => m.text), ...calls.map(c => c.text ?? '')]
126  const alreadyRead = new Set(calls.filter(c => c.tool === 'Read').map(c => String(c.input.file_path)))
127  let reads = 0
128  for (const p of mentionedPaths(texts)) {
129    if (reads >= MAX_READS) break
130    if (p.startsWith('~')) continue
131    const file = absolute(cwd, p)
132    if (alreadyRead.has(file) || !isSafeToOffer(file, cwd)) continue
133    const stat = await $.fs.stat(file).catch(() => undefined)
134    if (stat?.kind !== 'file' || stat.size > 2_000_000) continue
135    out.push({ key: `read ${p}`, description: `Read the file ${p}`, tool: 'Read', input: { file_path: file } })
136    reads += 1
137  }
138
139  // Searches for terms the user quoted.
140  for (const term of quotedTerms(request)) {
141    out.push(bash(`grep -rnF --exclude-dir=node_modules --exclude-dir=.git -- '${term}' .`, `Search the project for the text "${term}"`))
142  }
143
144  // Read-only git commands.
145  if (await has('.git')) {
146    // One "what changed" option: status and diff --stat as two options split
147    // Jev's probability (0.75/0.24 live); merged it reaches 0.93.
148    out.push(bash('git status --short && git diff --stat', 'Show what changed: which files are modified, staged or untracked, with a per-file count of changed lines'))
149    out.push(bash('git diff', 'Show the full line-by-line diff of unstaged changes'))
150    out.push(bash('git log --oneline -n 15', 'Show the 15 most recent commits'))
151  }
152
153  // The project's own safe scripts and test runners.
154  const pkg = await $.fs.read(`${cwd}/package.json`).catch(() => undefined)
155  if (typeof pkg === 'string') {
156    let scripts: Record<string, string> = {}
157    try {
158      scripts = (JSON.parse(pkg) as { scripts?: Record<string, string> }).scripts ?? {}
159    } catch {}
160    const runner = await packageRunner($, cwd)
161    for (const name of Object.keys(scripts).filter(n => SAFE_SCRIPT.test(n))) {
162      const command = name === 'test' ? `${runner} test` : `${runner} run ${name}`
163      out.push(bash(command, `Run the project's "${name}" script (${clipScript(scripts[name]!)})`))
164    }
165  }
166  const makefile = await $.fs.read(`${cwd}/Makefile`).catch(() => undefined)
167  if (typeof makefile === 'string') {
168    const targets = new Set([...makefile.matchAll(/^([A-Za-z][\w-]*):(?!=)/gm)].map(m => m[1]!))
169    for (const t of [...targets].filter(t => SAFE_MAKE.test(t))) out.push(bash(`make ${t}`, `Run the Makefile's "${t}" target`))
170  }
171  if (await has('Cargo.toml')) {
172    out.push(bash('cargo test', 'Run the Rust tests'))
173    out.push(bash('cargo check', 'Type-check the Rust project'))
174    out.push(bash('cargo clippy', 'Lint the Rust project'))
175  }
176  if (await has('go.mod')) {
177    out.push(bash('go test ./...', 'Run the Go tests'))
178    out.push(bash('go vet ./...', 'Vet the Go code'))
179    out.push(bash('go build ./...', 'Build the Go packages'))
180  }
181  if ((await has('pytest.ini')) || (await has('conftest.py')) || /\[tool\.pytest/.test(String(await $.fs.read(`${cwd}/pyproject.toml`).catch(() => '')))) {
182    out.push(bash('python -m pytest -q', 'Run the Python tests'))
183  }
184
185  // First move of a turn only: a skill to invoke, or a deferred tool to load.
186  if (stepIndex === 0) {
187    const commands = await $.command.list().catch(() => [])
188    const blocked = new Set(((await $.store.get(SKILL_BLOCKLIST_KEY).catch(() => undefined)) as string[] | undefined) ?? [])
189    const allSkills = dedupeSkills(
190      commands.filter(c => (c.source === 'plugin' || c.source === 'user') && c.name !== 'jev' && !BUILTIN_COMMANDS.has(c.name) && !blocked.has(c.name)),
191    )
192    // Every skill when the prompt is about skills; otherwise only related ones.
193    const skills = /\bskills?\b/i.test(request) ? allSkills : relevant(request, allSkills)
194    for (const c of skills) {
195      out.push({ key: `skill ${c.name}`, description: `Use the skill or command "${c.name}": ${clipScript(c.description)}`, tool: 'Skill', input: { skill: c.name } })
196    }
197    const loaded = new Set(calls.filter(c => c.tool === 'ToolSearch').map(c => String(c.input.query)))
198    const tools = (await $.tool.list().catch(() => [])).filter(t => t.mcp && !AUTH_TOOL.test(t.name) && !loaded.has(`select:${t.name}`))
199    for (const t of relevant(request, tools)) {
200      out.push({ key: `load ${t.name}`, description: `Load the tool ${t.name}: ${clipScript(t.description)}`, tool: 'ToolSearch', input: { query: `select:${t.name}`, max_results: 1 } })
201    }
202  }
203
204  // Jev's choice API takes at most 255 options, one of them "claude".
205  return out.slice(0, 254)
206}
207
208// MCP sign-in tools are never the next step of a coding turn.
209const AUTH_TOOL = /__(authenticate|complete_authentication)$/
210const MAX_RELATED = 8
211// Words too common in prompts and skill descriptions to show a relation.
212const STOPWORDS = new Set(
213  ('the and for with this that these those when what which where how why who any all can could would should will ' +
214    'you your use used using user users from into onto about over than then them they its it\'s are was were has have ' +
215    'had not but also only just some more most very like want need make made create new get set show see one two ' +
216    'file files read run runs tool tools skill skills command commands claude code project before after first step ' +
217    'please let lets now ok okay yes here there out other each every such via per our we me my is be to of in on at ' +
218    'an a or do does did done if so as by up no').split(' '),
219)
220
221/** Lowercase content words, light-stemmed so changes/changed/change match. */
222export function words(text: string): Set<string> {
223  const out = new Set<string>()
224  for (const w of text.toLowerCase().split(/[^a-z0-9]+/)) {
225    if (w.length < 3 || STOPWORDS.has(w) || /^\d+$/.test(w)) continue
226    const stem = w.replace(/(ing|ed|es|e|s)$/, '')
227    out.add(stem.length >= 3 && !w.endsWith('ss') ? stem : w)
228  }
229  return out
230}
231
232/**
233 * Skills and tools whose name or description shares words with the prompt,
234 * best first. A name word counts double; one description word alone is not
235 * enough, so generic overlaps ("document", "video") need a second word.
236 */
237export function relevant<T extends { name: string; description: string }>(prompt: string, items: readonly T[]): T[] {
238  const want = words(prompt)
239  if (!want.size) return []
240  const scored = items.map(item => {
241    const name = words(item.name.replace(/^mcp__/, '').replace(/[_:]+/g, ' '))
242    const desc = words(item.description)
243    let score = 0
244    for (const w of want) score += name.has(w) ? 2 : desc.has(w) ? 1 : 0
245    return { item, score }
246  })
247  return scored
248    .filter(s => s.score >= 2)
249    .sort((a, b) => b.score - a.score)
250    .slice(0, MAX_RELATED)
251    .map(s => s.item)
252}
253
254/**
255 * One entry per skill: `anthropic-skills:pdf` and `pdf`, or `youtube-metadata`
256 * and `youtube-plugin:youtube-metadata`, are the same skill listed twice. The
257 * unprefixed name wins.
258 */
259export function dedupeSkills<T extends { name: string }>(skills: readonly T[]): T[] {
260  const base = (n: string) => n.slice(n.lastIndexOf(':') + 1)
261  const byBase = new Map<string, T>()
262  for (const s of skills) {
263    const kept = byBase.get(base(s.name))
264    if (!kept || (kept.name.includes(':') && !s.name.includes(':'))) byBase.set(base(s.name), s)
265  }
266  return skills.filter(s => byBase.get(base(s.name)) === s)
267}
268
269const clipScript = (s: string) => (s.length > 80 ? `${s.slice(0, 80)}…` : s)
270
271async function packageRunner($: EngineInterface, cwd: string) {
272  for (const [file, runner] of [['pnpm-lock.yaml', 'pnpm'], ['yarn.lock', 'yarn'], ['bun.lockb', 'bun'], ['bun.lock', 'bun']] as const) {
273    if (await $.fs.exists(`${cwd}/${file}`).catch(() => false)) return runner
274  }
275  return 'npm'
276}
277

1import type { EngineInterface, SessionMessage } from 'claude-code'
2
3export type TurnContext = {
4  /** The user's request this turn answers. */
5  request: string
6  /** Messages after the request: the turn's steps so far. */
7  steps: SessionMessage[]
8}
9
10export type StepSummary = { assistant?: string; tool?: string; input?: string; result?: string; isError?: true }
11
12const clip = (text: string, max: number) => (text.length > max ? `${text.slice(0, max)}… [${text.length - max} more chars]` : text)
13
14export async function readTurn($: EngineInterface, agentId?: string): Promise<TurnContext | undefined> {
15  const messages = await $.session.messages(agentId ? { agentId } : {})
16  if (!Array.isArray(messages)) return undefined
17  return splitTurn(messages)
18}
19
20// Rows the person didn't type as a request: `!` shell runs, slash-command output,
21// and the engine's notices (background task results, reminders).
22const TRANSCRIPT_ONLY = /^\s*<(bash-input|bash-stdout|bash-stderr|local-command-stdout|local-command-caveat|command-name|command-message|system-reminder|task-notification)>/
23
24/** The last user message that carries text and no tool results is the request. */
25export function splitTurn(messages: readonly SessionMessage[]): TurnContext | undefined {
26  for (let i = messages.length - 1; i >= 0; i--) {
27    const m = messages[i]!
28    if (m.role === 'user' && m.text.trim() !== '' && !(m.toolResults?.length) && !TRANSCRIPT_ONLY.test(m.text)) {
29      return { request: m.text, steps: messages.slice(i + 1) }
30    }
31  }
32  return undefined
33}
34
35/** The turn's tool calls in order, with their outcomes. */
36export function toolCalls(steps: readonly SessionMessage[]) {
37  return steps.flatMap(m => (m.role === 'assistant' ? m.toolUses : []))
38}
39
40/** A compact view of the last `limit` steps for Jev's state. */
41export function summarize(steps: readonly SessionMessage[], limit = 8): StepSummary[] {
42  const out: StepSummary[] = []
43  for (const m of steps) {
44    if (m.role !== 'assistant') continue
45    if (m.text.trim()) out.push({ assistant: clip(m.text, 400) })
46    for (const use of m.toolUses) {
47      out.push({
48        tool: use.tool,
49        input: clip(JSON.stringify(use.input), 300),
50        ...(use.text !== undefined ? { result: clip(use.text, 400) } : {}),
51        ...(use.isError ? { isError: true as const } : {}),
52      })
53    }
54  }
55  return out.slice(-limit)
56}
57
58/** The end of the assistant's last reply, so short follow-ups are judged in context. */
59export async function previousReply($: EngineInterface): Promise<string | undefined> {
60  const messages = await $.session.messages({}).catch(() => undefined)
61  if (!Array.isArray(messages)) return undefined
62  for (let i = messages.length - 1; i >= 0; i--) {
63    const m = messages[i]!
64    if (m.role === 'assistant' && m.text.trim()) return m.text.length > 800 ? `…${m.text.slice(-800)}` : m.text
65  }
66  return undefined
67}
68
69export { clip }
70

1import type { EngineInterface } from 'claude-code'
2
3// TypeSafe System One API: one request carries every question about one state.
4export const JEV_URL = 'https://api.typesafe.ai/v1/systemone'
5const MODEL = 'jev-latest'
6const TIMEOUT_MS = 4000
7
8export type Question =
9  | { type: 'noul'; instructions: unknown; criteria?: { true?: string; false?: string } }
10  | { type: 'choice'; instructions: unknown; criteria: Record<string, string | null> }
11  | { type: 'score'; instructions: unknown; criteria: string[] }
12
13export type NoulAnswer = { type: 'noul'; noul: number }
14export type ChoiceAnswer = { type: 'choice'; choice: string; probabilities: Record<string, number>; confidence: number }
15export type ScoreAnswer = { type: 'score'; score: number; probabilities: Record<string, number>; confidence: number }
16export type Answer = NoulAnswer | ChoiceAnswer | ScoreAnswer
17
18export type JevResult =
19  | { ok: true; answers: Record<string, Answer>; ms: number }
20  | { ok: false; error: string; ms: number }
21
22/** Asks Jev; never throws. Any failure is `{ ok: false }` so callers fall back to Claude Code's own behaviour. */
23export async function askJev($: EngineInterface, state: unknown, questions: Record<string, Question>): Promise<JevResult> {
24  const startedAt = await $.clock.now()
25  const done = async (r: { ok: true; answers: Record<string, Answer> } | { ok: false; error: string }): Promise<JevResult> =>
26    ({ ...r, ms: (await $.clock.now()) - startedAt }) as JevResult
27
28  const key = await $.env.get('TYPESAFE_API_KEY')
29  if (!key) return done({ ok: false, error: 'TYPESAFE_API_KEY is not set' })
30
31  const request = $.http.fetch(JEV_URL, {
32    method: 'POST',
33    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
34    body: JSON.stringify({ model: MODEL, state, questions }),
35  })
36  request.catch(() => {})
37  const timeout = $.clock.sleep(TIMEOUT_MS).then(() => undefined)
38
39  try {
40    const res = await Promise.race([request, timeout])
41    if (!res) return done({ ok: false, error: `timeout after ${TIMEOUT_MS}ms` })
42    if (!res.ok) return done({ ok: false, error: `HTTP ${res.status}` })
43    const body = JSON.parse(res.text) as { answers?: Record<string, Answer> }
44    if (!body.answers) return done({ ok: false, error: 'response had no answers' })
45    return done({ ok: true, answers: body.answers })
46  } catch (err) {
47    return done({ ok: false, error: err instanceof Error ? err.message : String(err) })
48  }
49}
50
51export function noul(answers: Record<string, Answer>, id: string): number | undefined {
52  const a = answers[id]
53  return a?.type === 'noul' ? a.noul : undefined
54}
55
56export function choice(answers: Record<string, Answer>, id: string): ChoiceAnswer | undefined {
57  const a = answers[id]
58  return a?.type === 'choice' ? a : undefined
59}
60
61export function score(answers: Record<string, Answer>, id: string): ScoreAnswer | undefined {
62  const a = answers[id]
63  return a?.type === 'score' ? a : undefined
64}
65

1import type { EngineInterface } from 'claude-code'
2
3// Every Jev decision, agreeing or not, goes to a capped log in $.store so
4// shadow runs can be compared with what Claude did and thresholds tuned.
5export type Kind = 'take' | 'compare' | 'route' | 'gate' | 'done' | 'outcome' | 'turn' | 'bypass' | 'classify' | 'agent' | 'compact' | 'error'
6
7export type Entry = {
8  at: number
9  kind: Kind
10  /** Whether the decision changed what happened (false in shadow mode or for a pass). */
11  applied: boolean
12  text: string
13  ms?: number
14  turnId?: string
15  data?: Record<string, unknown>
16}
17
18const LOG_KEY = 'log'
19const MAX_ENTRIES = 1000
20
21// This session's entries, for the short `/jev` view; a hot reload starts it over.
22const recent: Entry[] = []
23
24export async function record($: EngineInterface, d: Omit<Entry, 'at'>) {
25  const entry: Entry = { ...d, at: await $.clock.now() }
26  recent.push(entry)
27  if (recent.length > 50) recent.shift()
28
29  const log = ((await $.store.get(LOG_KEY)) as Entry[] | undefined) ?? []
30  await $.store.set(LOG_KEY, [...log, entry].slice(-MAX_ENTRIES))
31
32  const stats = ((await $.store.get('stats')) as Record<string, number> | undefined) ?? {}
33  const key = d.applied ? d.kind : `${d.kind}:pass`
34  await $.store.set('stats', { ...stats, [key]: (stats[key] ?? 0) + 1 })
35
36  if (d.kind !== 'outcome' && d.kind !== 'turn') $.ui.status(`jev: ${d.text}`.slice(0, 80))
37}
38
39export function recentDecisions() {
40  return recent.slice()
41}
42
43export async function readLog($: EngineInterface): Promise<Entry[]> {
44  return ((await $.store.get(LOG_KEY)) as Entry[] | undefined) ?? []
45}
46
47export async function clearLog($: EngineInterface) {
48  await $.store.set(LOG_KEY, [])
49  await $.store.set('stats', {})
50}
51
52export async function stats($: EngineInterface) {
53  return ((await $.store.get('stats')) as Record<string, number> | undefined) ?? {}
54}
55

1import type { Entry } from './log'
2
3// `/jev report` as Markdown tables: a summary, Jev vs Claude step by step, the
4// take threshold, then routing, gate and done check. Pure, so tests can feed it
5// entries directly.
6
7const THRESHOLDS = [0.6, 0.7, 0.8, 0.85, 0.9, 0.95]
8const STEP_ROWS = 15
9
10const pct = (n: number, of: number) => (of ? `${Math.round((n / of) * 100)}%` : '—')
11const num = (v: unknown) => (typeof v === 'number' ? v : undefined)
12const str = (v: unknown) => (typeof v === 'string' ? v : undefined)
13const p2 = (v: number | undefined) => (v === undefined ? '—' : v.toFixed(2))
14const cell = (s: string, max = 48) => {
15  const one = s.replace(/\s+/g, ' ').replace(/\|/g, '\\|').trim()
16  return one.length > max ? `${one.slice(0, max - 1)}…` : one
17}
18const bar = (n: number, of: number, width = 10) => {
19  const filled = of ? Math.round((n / of) * width) : 0
20  return '█'.repeat(filled) + '░'.repeat(width - filled)
21}
22
23const pad = (n: number) => String(n).padStart(2, '0')
24const day = (d: Date) => `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`
25
26/** Local `HH:MM`, with `MM-DD` in front when not on the same day as `latest`. */
27export function when(at: number, latest: number): string {
28  const d = new Date(at)
29  const time = `${pad(d.getHours())}:${pad(d.getMinutes())}`
30  return day(d) === day(new Date(latest)) ? time : `${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${time}`
31}
32
33export type Verdict = 'match' | 'differ' | 'missed' | 'both-claude'
34
35/**
36 * How one compared step went, at a given take threshold:
37 * - Jev would take its action → `match` when Claude made the same call, else `differ`.
38 * - Jev leaves it to Claude → `both-claude` when Claude used none of the listed
39 *   actions (agreement), `missed` when Claude did use one.
40 */
41export function verdict(e: Entry, threshold: number): Verdict {
42  const bestP = num(e.data?.bestP) ?? 0
43  const wouldTake = str(e.data?.best) !== undefined && bestP >= threshold
44  if (wouldTake) return e.data?.agree === true ? 'match' : 'differ'
45  return e.data?.claudeUsedCandidate === true ? 'missed' : 'both-claude'
46}
47
48const MARK: Record<Verdict, string> = { match: '✓ same', 'both-claude': '✓ Claude', differ: '✗ differ', missed: '⚠ missed' }
49
50/** What Claude did, from the compare entry (short form for older entries). */
51function claudeDid(e: Entry): string {
52  const summary = str(e.data?.claudeSummary)
53  if (summary) return summary
54  const tools = (e.data?.claude as string[] | undefined) ?? []
55  if (!tools.length) return 'replied'
56  const counts = new Map<string, number>()
57  for (const t of tools) counts.set(t, (counts.get(t) ?? 0) + 1)
58  return [...counts].map(([t, n]) => (n > 1 ? `${t} ×${n}` : t)).join(', ')
59}
60
61export function buildReport(log: readonly Entry[], threshold: number, mode = ''): string {
62  if (!log.length) return '**Jev pilot report:** the log is empty. Use Claude for a while (try `/jev shadow`) and run `/jev report` again.'
63  const of = (kind: Entry['kind']) => log.filter(e => e.kind === kind)
64  const out: string[] = []
65  const since = new Date(log[0]!.at).toISOString().slice(0, 16).replace('T', ' ')
66
67  const takes = of('take')
68  const taken = takes.filter(e => e.applied)
69  const wouldTake = takes.filter(e => !e.applied && e.data?.wouldTake === true)
70  const compares = of('compare')
71  const verdicts = compares.map(e => verdict(e, threshold))
72  const agreed = verdicts.filter(v => v === 'match' || v === 'both-claude').length
73  const errored = of('outcome').filter(e => e.data?.isError === true)
74  const timed = log.map(e => e.ms).filter((m): m is number => typeof m === 'number')
75  const avgMs = timed.length ? Math.round(timed.reduce((a, b) => a + b, 0) / timed.length) : undefined
76
77  out.push('## Jev pilot report')
78  out.push(`_${log.length} log entries since ${since}${mode ? ` · mode **${mode}**` : ''} · take threshold **${threshold}**_`)
79  out.push('')
80  out.push('| | |')
81  out.push('|---|---|')
82  out.push(`| Steps Jev took itself | **${taken.length}** (≈ ${taken.length} Claude requests saved${errored.length ? `, ${errored.length} failed` : ''}) |`)
83  const bypassed = of('bypass').filter(e => e.applied)
84  out.push(`| Steps Jev would have taken (shadow) | **${wouldTake.length}** |`)
85  out.push(`| Turns answered without Claude | **${bypassed.length}** (Jev's action + ${bypassed.length ? 'Haiku' : 'a small model'} summary) |`)
86  out.push(`| Agreement with Claude | **${agreed} / ${compares.length}** (${pct(agreed, compares.length)}) |`)
87  out.push(`| Jev latency | ${avgMs === undefined ? '—' : `${avgMs} ms average`} over ${timed.length} calls · ${of('error').length} errors |`)
88
89  // Jev vs Claude, step by step.
90  if (compares.length) {
91    out.push('', `### Jev vs Claude, step by step${compares.length > STEP_ROWS ? ` (last ${STEP_ROWS})` : ''}`)
92    const latest = log[log.length - 1]!.at
93    out.push('| Time | Request | Jev would | p | Claude did | |')
94    out.push('|---|---|---|---|---|---|')
95    compares.slice(-STEP_ROWS).forEach(e => {
96      const v = verdict(e, threshold)
97      const best = str(e.data?.best)
98      const jev = v === 'match' || v === 'differ' ? best! : `leave to Claude${best ? ` _(best: ${best})_` : ''}`
99      const p = v === 'match' || v === 'differ' ? num(e.data?.bestP) : num(e.data?.bestP)
100      out.push(`| ${when(e.at, latest)} | ${cell(str(e.data?.request) ?? '—', 40)} | ${cell(jev, 56)} | ${p2(p)} | ${cell(claudeDid(e), 40)} | ${MARK[v]} |`)
101    })
102    out.push('', '_✓ same: Jev would make the same call · ✓ Claude: both leave it to Claude · ✗ differ: Jev would act differently · ⚠ missed: Claude used an action Jev had listed_')
103  }
104
105  // Take threshold.
106  const scored = compares.filter(e => num(e.data?.bestP) !== undefined)
107  if (scored.length) {
108    out.push('', '### Take threshold')
109    out.push('| Threshold | Would take | Same call as Claude | |')
110    out.push('|---|---|---|---|')
111    for (const t of THRESHOLDS) {
112      const hit = scored.filter(e => num(e.data?.bestP)! >= t)
113      const ok = hit.filter(e => e.data?.agree === true).length
114      const current = Math.abs(t - threshold) < 1e-9
115      out.push(`| ${current ? `**${t.toFixed(2)} ← current**` : t.toFixed(2)} | ${hit.length} | ${pct(ok, hit.length)} | \`${bar(ok, hit.length)}\` |`)
116    }
117  }
118
119  // Routing, gate, done check.
120  const routes = of('route')
121  const scores = routes.map(e => num(e.data?.score)).filter((s): s is number => s !== undefined)
122  const lowered = routes.filter(e => e.data?.lowered === true)
123  const raised = routes.filter(e => e.data?.raised === true)
124  const gates = of('gate')
125  const by = (d: string) => gates.filter(e => e.data?.decision === d)
126  const dones = of('done')
127  const continued = dones.filter(e => e.data?.continued === true)
128  out.push('', '### Routing, gate and done check')
129  out.push('| Decision | Checked | Result |')
130  out.push('|---|---|---|')
131  out.push(`| Effort routing | ${routes.length} turns | avg difficulty ${scores.length ? (scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(2) : '—'} (0 trivial … 3 hard) · ${lowered.length} lowered, ${raised.length} raised, ${[...lowered, ...raised].filter(e => e.applied).length} applied |`)
132  out.push(`| Gate | ${gates.length} calls | ${by('allow').length} allow · ${by('ask').length} ask · ${by('deny').length} deny |`)
133  out.push(`| Done check | ${dones.length} stops | ${continued.length} unfinished, ${continued.filter(e => e.applied).length} sent back |`)
134  const classified = of('classify')
135  if (classified.length) {
136    const kinds = new Map<string, number>()
137    for (const e of classified) kinds.set(str(e.data?.kind) ?? '?', (kinds.get(str(e.data?.kind) ?? '?') ?? 0) + 1)
138    const hinted = classified.filter(e => (num(e.data?.hints) ?? 0) > 0)
139    out.push(`| Prompt classifier | ${classified.length} prompts | ${[...kinds].map(([k, n]) => `${n} ${k}`).join(' · ')} · ${hinted.length} hinted, ${hinted.filter(e => e.applied).length} applied |`)
140  }
141  const agents = of('agent')
142  if (agents.length) {
143    const switched = agents.filter(e => e.data?.switched === true)
144    out.push(`| Subagent type | ${agents.length} spawns | ${agents.filter(e => e.data?.agree === true).length} agreed with Claude · ${switched.length} switched, ${switched.filter(e => e.applied).length} applied |`)
145  }
146  const compacts = of('compact')
147  if (compacts.length) {
148    const sum = (k: string) => compacts.reduce((a, e) => a + (num(e.data?.[k]) ?? 0), 0)
149    out.push(`| Compaction | ${compacts.length} runs | ${sum('items')} results scored · ${sum('keep')} kept in detail · ${sum('drop')} marked droppable |`)
150  }
151
152  const flagged = [...by('ask'), ...by('deny')].slice(-5)
153  if (flagged.length) {
154    out.push('', '**Recent asks and denies**')
155    const latest = log[log.length - 1]!.at
156    for (const e of flagged) out.push(`- ${when(e.at, latest)} ${e.applied ? '' : '_(shadow)_ '}\`${cell(e.text, 80)}\``)
157  }
158
159  const aborted = of('turn').filter(e => e.data?.reason === 'aborted').length
160  if (aborted) out.push('', `_${aborted} interrupted turn(s)._`)
161  return out.join('\n')
162}
163

1import type { On, RenderElement } from 'claude-code'
2
3// Draws `/jev` output as Markdown, so the report's tables render as tables.
4export function registerView(on: On) {
5  on('ui.render', { component: 'CommandOutput', props: { command: 'jev' } }, ($, e, next) => {
6    if (e.props.isErrored) return next(e)
7    const { Markdown } = $.ui.resolve(e)
8    return h(Markdown, { text: e.props.text }) as RenderElement
9  })
10}
11

1import type { EngineInterface } from 'claude-code'
2
3export type Mode = 'active' | 'shadow' | 'off'
4export const MODES: readonly Mode[] = ['active', 'shadow', 'off']
5
6/** `/jev <mode>` stored in $.store wins over the configured mode. */
7export async function currentMode($: EngineInterface, configured: unknown): Promise<Mode> {
8  const m = (await $.store.get('mode')) ?? configured
9  return MODES.includes(m as Mode) ? (m as Mode) : 'active'
10}
11

1import type { EngineInterface, On, PluginOptions } from 'claude-code'
2
3import { BUILTIN_COMMANDS, SKILL_BLOCKLIST_KEY } from './candidates'
4import { clip, previousReply } from './context'
5import { askJev, choice, noul } from './jev'
6import type { Question } from './jev'
7import { record } from './log'
8import { currentMode } from './mode'
9
10// Phase 3: classify each prompt as it is sent and, when Jev is confident, add
11// a short hint only Claude reads (a skill that fits, or plan first).
12const NONE = 'none'
13const KIND_LEVELS: Record<string, string> = {
14  question: 'Asks for an explanation, information or an opinion; no change is needed',
15  task: 'Asks for a change: writing, editing, fixing or building something',
16  command: 'Asks to run, show or check one specific thing (tests, status, a file)',
17  conversation: 'Small talk, thanks, or a reply to the assistant with no new request',
18}
19const SKILL_HINT = 0.6
20const PLAN_HINT = 0.85
21
22async function skillOptions($: EngineInterface) {
23  const commands = await $.command.list().catch(() => [])
24  const blocked = new Set(((await $.store.get(SKILL_BLOCKLIST_KEY).catch(() => undefined)) as string[] | undefined) ?? [])
25  const criteria: Record<string, string | null> = { [NONE]: 'No listed skill is meant for this request' }
26  for (const c of commands) {
27    if ((c.source === 'plugin' || c.source === 'user') && c.name !== 'jev' && !BUILTIN_COMMANDS.has(c.name) && !blocked.has(c.name))
28      criteria[c.name] = clip(c.description, 160)
29  }
30  return Object.keys(criteria).length > 1 ? criteria : undefined
31}
32
33export function promptQuestions(skills: Record<string, string | null> | undefined): Record<string, Question> {
34  const questions: Record<string, Question> = {
35    kind: {
36      type: 'choice',
37      instructions: "What kind of message is `request`, sent to an AI coding assistant? When `previous_reply` is present, `request` answers it: read a short follow-up like 'fix it' or 'yes' in that light.",
38      criteria: KIND_LEVELS,
39    },
40    needs_plan: {
41      type: 'noul',
42      instructions:
43        "Is `request` a large or ambiguous change (several files, design choices, unclear scope) that the assistant should plan and confirm before editing anything? When `previous_reply` is present, `request` refers to it: a follow-up approving or naming a change already described there is not ambiguous.",
44      criteria: { true: 'Large or ambiguous: plan first', false: 'Small, clear, or not a change at all' },
45    },
46  }
47  if (skills) questions.skill = { type: 'choice', instructions: 'Which listed skill is squarely meant for `request`, if any?', criteria: skills }
48  return questions
49}
50
51export function hints(kind: string | undefined, needsPlan: number | undefined, skill: { name: string; p: number } | undefined, takeThreshold: number): string[] {
52  const out: string[] = []
53  if (skill && skill.name !== NONE && skill.p >= SKILL_HINT && skill.p < takeThreshold)
54    out.push(`Jev routing hint: the skill "${skill.name}" looks like a fit for this request (p=${skill.p.toFixed(2)}). Consider invoking it.`)
55  if (kind === 'task' && (needsPlan ?? 0) >= PLAN_HINT)
56    out.push(`Jev routing hint: this looks like a large or ambiguous change (p=${(needsPlan ?? 0).toFixed(2)}). Outline a short plan and confirm it with the user before editing.`)
57  return out
58}
59
60export function registerPromptClassifier(on: On, options: PluginOptions, takeThreshold: number) {
61  if (options.classify === false) return
62  on('prompt.submit', async ($, e, next) => {
63    if (e.command || e.isMeta || e.text.trim().length < 3) return next(e)
64    const m = await currentMode($, options.mode)
65    if (m === 'off') return next(e)
66
67    const questions = promptQuestions(await skillOptions($))
68    const previous = await previousReply($)
69    const r = await askJev($, { request: clip(e.text, 3000), ...(previous ? { previous_reply: previous } : {}) }, questions)
70    if (!r.ok) return next(e)
71    const kind = choice(r.answers, 'kind')
72    const needsPlan = noul(r.answers, 'needs_plan')
73    const skillAnswer = choice(r.answers, 'skill')
74    const skill = skillAnswer ? { name: skillAnswer.choice, p: skillAnswer.probabilities[skillAnswer.choice] ?? 0 } : undefined
75    const added = hints(kind?.choice, needsPlan, skill, takeThreshold)
76
77    await record($, {
78      kind: 'classify',
79      applied: m === 'active' && added.length > 0,
80      text: `${kind?.choice ?? '?'}${needsPlan !== undefined ? `, plan ${needsPlan.toFixed(2)}` : ''}${skill && skill.name !== NONE ? `, skill ${skill.name} ${skill.p.toFixed(2)}` : ''}${added.length ? ' → hint' : ''}`,
81      ms: r.ms,
82      data: { request: clip(e.text, 120), kind: kind?.choice, kindP: kind?.confidence, needsPlan, skill: skill?.name, skillP: skill?.p, hints: added.length },
83    })
84    if (m !== 'active' || !added.length) return next(e)
85    return next({ ...e, context: [...(e.context ?? []), ...added] })
86  })
87}
88

1import type { On, PluginOptions } from 'claude-code'
2
3import { clip } from './context'
4import { askJev, choice } from './jev'
5import { record } from './log'
6import { currentMode } from './mode'
7
8// Phase 3: when Claude starts a general-purpose subagent, Jev may pick a more
9// specialised agent type. An explicit choice by Claude is only compared, never changed.
10const GENERAL = new Set(['', 'general-purpose', 'claude'])
11const SWITCH_AT = 0.85
12
13export function registerAgentTypes(on: On, options: PluginOptions) {
14  if (options.agentTypes === false) return
15  // Agent types as the engine offers them to the model, with their descriptions.
16  const offered = new Map<string, string>()
17
18  on('agent.offer', async ($, e, next) => {
19    offered.set(e.agent, e.description)
20    return next(e)
21  })
22
23  on('tool.call', { tool: 'Agent' }, async ($, e, next) => {
24    const m = await currentMode($, options.mode)
25    const claudeType = e.subagent_type ?? ''
26    if (m === 'off' || offered.size < 2 || claudeType === 'fork') return next(e)
27
28    const criteria: Record<string, string | null> = {}
29    for (const [name, description] of offered) criteria[name] = clip(description, 200)
30    const r = await askJev(
31      $,
32      { task: clip(e.prompt, 3000), description: e.description },
33      { agent_type: { type: 'choice', instructions: 'Which agent type is best suited to carry out `task`?', criteria } },
34    )
35    if (!r.ok) return next(e)
36    const answer = choice(r.answers, 'agent_type')
37    if (!answer) return next(e)
38    const p = answer.probabilities[answer.choice] ?? 0
39    const canSwitch = GENERAL.has(claudeType) && !GENERAL.has(answer.choice) && p >= SWITCH_AT
40    await record($, {
41      kind: 'agent',
42      applied: m === 'active' && canSwitch,
43      text: `Claude ${claudeType || '(default)'} · Jev ${answer.choice} p=${p.toFixed(2)}${canSwitch ? ' → switched' : ''}`,
44      ms: r.ms,
45      data: { claude: claudeType || 'general-purpose', jev: answer.choice, p, switched: canSwitch, agree: answer.choice === (claudeType || 'general-purpose') },
46    })
47    return m === 'active' && canSwitch ? next({ ...e, subagent_type: answer.choice }) : next(e)
48  })
49}
50

1import type { On, PluginOptions, SessionMessage } from 'claude-code'
2
3import { clip, splitTurn } from './context'
4import { askJev, score } from './jev'
5import type { Question } from './jev'
6import { record } from './log'
7import { currentMode } from './mode'
8
9// Phase 3: before a compaction, Jev scores how relevant each recent tool
10// result is to the current work. No message is dropped (that could break tool
11// call/result pairs); the summarizer is told what to keep in detail.
12const MAX_ITEMS = 30
13const RELEVANCE = [
14  'Irrelevant now: superseded, exploratory, or about finished work',
15  'Background: useful context but details can be summarized',
16  'Relevant: needed to continue the current work',
17  'Essential: exact details (paths, errors, decisions, values) must be kept',
18]
19
20export type Item = { id: string; label: string; result: string }
21
22export function compactItems(messages: readonly SessionMessage[]): Item[] {
23  const items: Item[] = []
24  for (const m of messages) {
25    if (m.role !== 'assistant') continue
26    for (const u of m.toolUses) {
27      const target = typeof u.input.file_path === 'string' ? u.input.file_path : typeof u.input.command === 'string' ? clip(u.input.command, 80) : clip(JSON.stringify(u.input), 80)
28      items.push({ id: u.tool_use_id, label: `${u.tool} ${target}`, result: clip(u.text ?? '', 500) })
29    }
30  }
31  return items.slice(-MAX_ITEMS)
32}
33
34export function compactInstructions(items: readonly Item[], scores: readonly (number | undefined)[]): string | undefined {
35  const keep = items.filter((_, i) => (scores[i] ?? 0) >= 2.2).map(i => i.label)
36  const drop = items.filter((_, i) => scores[i] !== undefined && scores[i]! < 0.8).map(i => i.label)
37  if (!keep.length && !drop.length) return undefined
38  const lines = ['Relevance ranking from Jev:']
39  if (keep.length) lines.push(`Keep exact details (paths, errors, decisions, values) from: ${keep.join('; ')}.`)
40  if (drop.length) lines.push(`These are no longer relevant and can be reduced to a word or dropped: ${drop.join('; ')}.`)
41  return lines.join('\n')
42}
43
44export function registerCompaction(on: On, options: PluginOptions) {
45  if (options.compaction === false) return
46  on('session.compact', async ($, e, next) => {
47    if (e.agentId || (e.trigger !== 'manual' && e.trigger !== 'auto')) return next(e)
48    const m = await currentMode($, options.mode)
49    if (m === 'off') return next(e)
50    const items = compactItems(e.messages)
51    if (items.length < 3) return next(e)
52
53    const request = splitTurn(e.messages)?.request ?? ''
54    const questions: Record<string, Question> = {}
55    items.forEach((it, i) => {
56      questions[`item_${i}`] = {
57        type: 'score',
58        instructions: `How relevant is this earlier tool call and its result to continuing the work on \`request\`? Call: ${it.label}. Result: ${it.result || '(no output)'}`,
59        criteria: RELEVANCE,
60      }
61    })
62    const r = await askJev($, { request: clip(request, 2000), item_count: items.length }, questions)
63    if (!r.ok) return next(e)
64    const scores = items.map((_, i) => score(r.answers, `item_${i}`)?.score)
65    const added = compactInstructions(items, scores)
66    const keep = scores.filter(s => (s ?? 0) >= 2.2).length
67    const drop = scores.filter(s => s !== undefined && s < 0.8).length
68    await record($, {
69      kind: 'compact',
70      applied: m === 'active' && added !== undefined,
71      text: `${e.trigger} compaction: ${items.length} results scored, keep ${keep}, drop ${drop}`,
72      ms: r.ms,
73      data: { trigger: e.trigger, items: items.length, keep, drop },
74    })
75    if (m !== 'active' || !added) return next(e)
76    return next({ ...e, instructions: e.instructions ? `${e.instructions}\n\n${added}` : added })
77  })
78}
79

1import type { EngineInterface, ModelUsage } from 'claude-code'
2
3import { clip } from './context'
4
5// Whole-turn bypass: once Jev has taken the one action a simple request needs,
6// a small model reports the result instead of the main Claude model.
7export const SUMMARY_MODEL = 'claude-haiku-5-5'
8
9const SYSTEM = [
10  'You report the result of one action (a command run or a file read) to a developer, inside Claude Code.',
11  "Answer the developer's request using only the action's output. Lead with the outcome (for tests: how many passed or failed, and which failed).",
12  'Be brief: one to four sentences or a short list, in Markdown. Do not suggest next steps unless something failed. Never invent details that are not in the output.',
13].join(' ')
14
15export type Summary = { ok: true; text: string; usage: ModelUsage } | { ok: false; reason: string }
16
17export async function summarizeResult($: EngineInterface, request: string, action: string, output: string): Promise<Summary> {
18  const r = await $.model.complete({
19    model: SUMMARY_MODEL,
20    system: SYSTEM,
21    prompt: `Request: ${clip(request, 2000)}\nAction: ${action}\nOutput:\n${clip(output, 12000)}`,
22    maxTokens: 600,
23    timeoutMs: 20000,
24  })
25  if (!r.isAnswered) return { ok: false, reason: r.reason }
26  const text = r.text.trim()
27  return text ? { ok: true, text, usage: r.usage } : { ok: false, reason: 'empty-reply' }
28}
29