model-router

1import type { Register } from "claude-code";
2import { route } from "./router.ts";
3
4const HTTP_TIMEOUT_MS = 8000;
5// The debug log keeps 10000 characters of a line, and a body can carry 12000 of task text.
6const DEBUG_PIECE_CHARS = 9000;
7
8type AgentCall = {
9	prompt: string;
10	description: string;
11	subagent_type?: string;
12	model?: string;
13	effort?: string;
14	name?: string;
15};
16
17export const register: Register = (on, options) => {
18	const { api_key: keyOption, model: modelOption } = options;
19	const jev = modelOption === "jev-preview" ? "jev-preview" : "jev-latest";
20	const key = (v: unknown) =>
21		typeof v === "string" && v !== "" ? v : undefined;
22
23	on("tool.call", { tool: "Agent" }, async ($, e, next) => {
24		const call = e as typeof e & AgentCall;
25		// A fork inherits its parent's model, a named agent is a teammate, and a caller-named model keeps its effort too.
26		if (
27			call.subagent_type === "fork" ||
28			call.name !== undefined ||
29			call.model !== undefined
30		)
31			return next(e);
32		// A call the user's rules deny as Claude made it never leaves the machine. The check opens no dialog.
33		const {
34			tool: _tool,
35			tool_use_id: _id,
36			agentId: _agentId,
37			requestMeta: _meta,
38			...input
39		} = e as Record<string, unknown>;
40		const check = await $.tool.check({ tool: "Agent", input });
41		if (check.decision === "deny") return next(e);
42		const keep = call.effort !== undefined;
43		// The key goes only to TypeSafe, which issued it. The sensitive option comes first. The desktop app
44		// reads it back empty after a restart, so the variable stands in when it is empty.
45		const apiKey = key(keyOption) ?? key(await $.env.get("TYPESAFE_API_KEY"));
46
47		const post = async (body: string): Promise<string> => {
48			if (!apiKey) throw new Error("api_key unset");
49			const pieces = Math.ceil(body.length / DEBUG_PIECE_CHARS);
50			for (let i = 0; i < pieces; i++)
51				$.ui.log(
52					`request ${i + 1}/${pieces} ${body.slice(i * DEBUG_PIECE_CHARS, (i + 1) * DEBUG_PIECE_CHARS)}`,
53					{ to: "debug" },
54				);
55			const stop = new AbortController();
56			const timeout = $.clock
57				.sleep(HTTP_TIMEOUT_MS, {
58					signal: AbortSignal.any([stop.signal, next.signal]),
59				})
60				.then(() => {
61					throw new Error(`no answer in ${HTTP_TIMEOUT_MS} ms`);
62				});
63			try {
64				const res = await Promise.race([
65					$.http.fetch("https://api.typesafe.ai/v1/systemone", {
66						method: "POST",
67						headers: {
68							Authorization: `Bearer ${apiKey}`,
69							"Content-Type": "application/json",
70						},
71						body,
72					}),
73					timeout,
74				]);
75				if (!res.ok) throw new Error(`HTTP ${res.status}`);
76				return res.text;
77			} finally {
78				stop.abort();
79			}
80		};
81
82		const decision = await route(
83			{
84				role: call.subagent_type ?? "general-purpose",
85				description: call.description,
86				prompt: call.prompt,
87				model: jev,
88			},
89			post,
90		);
91		if ("skipped" in decision) {
92			$.ui.log(`skipped, ${decision.skipped}`);
93			return next(e);
94		}
95		const { level, model, workIndex, effort, probabilities } = decision;
96		const p = `${probabilities.tier[level].toFixed(2)}/${probabilities.work[workIndex].toFixed(2)}`;
97		const pick = `L${level} ${model} · ${keep ? "caller effort" : effort} · p=${p}`;
98		$.ui.log(pick);
99		const r = await next(keep ? { ...e, model } : { ...e, model, effort });
100		if ("deny" in r && r.deny !== undefined) $.ui.log("denied");
101		return r;
102	}).catch((_, e, next) => next(e));
103};
104

1export const BUDGET_CHARS = 12000;
2export const TIE_MARGIN = 0.1;
3export const WORK_PRIOR = [0.5, 1, 0.5] as const;
4
5export type Model = "haiku" | "sonnet" | "opus";
6export type Effort = "low" | "medium" | "high" | "xhigh";
7export type Level = 0 | 1 | 2;
8export type Probabilities = Record<`${Level}`, number>;
9
10export const TIERS = [
11	{
12		model: "haiku",
13		text: "Lookup or transcription: the brief names the exact files to read or string to find, or holds the exact code, config or prose to write. It gives the commands to run and the check that proves the work done. No choice is left.",
14	},
15	{
16		model: "sonnet",
17		text: "Bounded judgment: the brief fixes the goal and a way to check the result. Check work against stated requirements, fix named defects, implement from a given plan or spec, or answer a question from a few named files.",
18	},
19	{
20		model: "opus",
21		text: "Open judgment: the brief leaves the approach open. Decide a design, create content with no plan or spec to follow, review a multi-file change for risks no checklist names, or debug with an unclear cause.",
22	},
23] as const satisfies readonly { model: Model; text: string }[];
24
25export const WORK_LEVELS: readonly string[] = [
26	"A short task: one or two mechanical steps in one place, nothing to diagnose, and the given command is the whole check.",
27	"An ordinary task: reading or diagnosing code, a few steps or files, and a result to verify.",
28	"A long or delicate task: many steps or files, hidden edge cases, or a wrong result that would look right.",
29];
30
31// Effort is calibrated per model. Haiku's row sits one step higher: at low it stops early on agentic prompts.
32export const EFFORT_LADDER: Record<Model, readonly [Effort, Effort, Effort]> = {
33	haiku: ["medium", "high", "xhigh"],
34	sonnet: ["low", "medium", "high"],
35	opus: ["low", "medium", "high"],
36};
37
38const TIER_INSTRUCTIONS =
39	"How much judgment does completing `task` require from an agent whose role is `role`? `description` is the caller's one-line summary of the task.";
40const WORK_INSTRUCTIONS =
41	"How much work does completing `task` take for an agent whose role is `role`, once the judgment is settled? Count the steps, the files and the edge cases to verify.";
42
43// The task text goes to a third party, and nothing marks which strings are secrets, so redaction matches shapes.
44const REDACT_PATTERNS: readonly [RegExp, string][] = [
45	[/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer ***"],
46	[/\bsk-[A-Za-z0-9_-]{8,}/g, "***"],
47	[
48		/\b([A-Za-z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD))(["']?\s*[=:]\s*)(?:"[^"]*"?|'[^']*'?|[^\s"']+)/gi,
49		"$1$2***",
50	],
51];
52
53export type RouteInput = {
54	role: string;
55	description: string;
56	prompt: string;
57	model: string;
58};
59export type Post = (body: string) => Promise<string>;
60export type Decision = {
61	model: Model;
62	level: Level;
63	workIndex: Level;
64	effort: Effort;
65	probabilities: { tier: Probabilities; work: Probabilities };
66};
67export type Skipped = { skipped: string };
68
69export async function route(
70	input: RouteInput,
71	post: Post,
72): Promise<Decision | Skipped> {
73	const task = redact(condense(input.prompt));
74	const body = JSON.stringify({
75		model: input.model,
76		state: { role: input.role, description: input.description, task },
77		questions: {
78			tier: {
79				type: "score",
80				instructions: TIER_INSTRUCTIONS,
81				criteria: TIERS.map((t) => t.text),
82			},
83			work: {
84				type: "score",
85				instructions: WORK_INSTRUCTIONS,
86				criteria: WORK_LEVELS,
87			},
88		},
89	});
90	let text: string;
91	try {
92		text = await post(body);
93	} catch (err) {
94		return { skipped: err instanceof Error ? err.message : String(err) };
95	}
96	let answers: unknown;
97	try {
98		answers = JSON.parse(text)?.answers;
99	} catch {
100		return { skipped: "malformed answer" };
101	}
102	const tier = parseScore(answers, "tier");
103	const work = parseScore(answers, "work");
104	if (!tier || !work) return { skipped: "malformed answer" };
105	const level = pickTier(tier);
106	const workIndex = pickWork(work);
107	const model = TIERS[level].model;
108	return {
109		model,
110		level,
111		workIndex,
112		effort: EFFORT_LADDER[model][workIndex],
113		probabilities: { tier, work },
114	};
115}
116
117function condense(text: string): string {
118	return text
119		.replace(/```[^\n]*\n([\s\S]*?)```/g, (_, block: string) => {
120			const lines = block.replace(/\n$/, "").split("\n").length;
121			return `[an exact block of ${lines} lines is given verbatim]`;
122		})
123		.replace(/\n{3,}/g, "\n\n")
124		.slice(0, BUDGET_CHARS);
125}
126
127function redact(text: string): string {
128	return REDACT_PATTERNS.reduce(
129		(t, [pattern, replacement]) => t.replace(pattern, replacement),
130		text,
131	);
132}
133
134const isRecord = (v: unknown): v is Record<string, unknown> =>
135	typeof v === "object" && v !== null;
136
137function parseScore(answers: unknown, key: string): Probabilities | undefined {
138	const a = isRecord(answers) ? answers[key] : undefined;
139	if (!isRecord(a)) return undefined;
140	const { type, probabilities } = a;
141	if (type !== "score" || !isRecord(probabilities)) return undefined;
142	const { 0: p0, 1: p1, 2: p2 } = probabilities;
143	if (
144		typeof p0 !== "number" ||
145		typeof p1 !== "number" ||
146		typeof p2 !== "number"
147	)
148		return undefined;
149	return { 0: p0, 1: p1, 2: p2 };
150}
151
152// A near tie goes to the highest level within the margin: when unsure, the larger model.
153function pickTier(p: Probabilities): Level {
154	const best = ([0, 1, 2] as Level[]).reduce((a, b) => (p[b] > p[a] ? b : a));
155	for (const i of [2, 1] as Level[])
156		if (i > best && p[best] - p[i] < TIE_MARGIN) return i;
157	return best;
158}
159
160// The middle is the resting point: an end wins only with more than twice its probability. Equal ends go to the heavier one.
161function pickWork(p: Probabilities): Level {
162	let best: Level = 1;
163	for (const i of [2, 0] as Level[]) {
164		if (p[i] * WORK_PRIOR[i] > p[best] * WORK_PRIOR[best]) best = i;
165	}
166	return best;
167}
168