model-advisor

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface as Engine, Register } from 'claude-code'
3
4import type { Classification, LogEntry, Pending, Tier } from '../types'
5import {
6  advice,
7  jevCall,
8  MODELS,
9  OPT_OUT,
10  parseJev,
11  relayContext,
12  suggest,
13  TIMEOUT_MS,
14  tierOf,
15  verdict,
16} from './advice'
17
18const pending = atom({ plugin: 'model-advisor', key: 'pending' } as const, null)
19const decided = atom({ plugin: 'model-advisor', key: 'decided' } as const, false)
20
21const LOG = 'log'
22const LOG_MAX = 200
23const NO_HUMAN = /^(remote_trigger|sdk)/
24
25async function log($: Engine, entry: LogEntry) {
26  const all = ((await $.store.get(LOG)) ?? []) as LogEntry[]
27  await $.store.set(LOG, [...all, entry].slice(-LOG_MAX))
28}
29
30async function settle($: Engine, outcome: LogEntry['outcome']) {
31  const all = ((await $.store.get(LOG)) ?? []) as LogEntry[]
32  const last = all.at(-1)
33  if (last && !last.outcome) await $.store.set(LOG, [...all.slice(0, -1), { ...last, outcome }])
34}
35
36async function classify($: Engine, text: string): Promise<Classification> {
37  const call = jevCall(
38    {
39      typesafeKey: await $.env.get('TYPESAFE_API_KEY'),
40      cfAccount: await $.env.get('CF_ACCOUNT_ID'),
41      cfToken: await $.env.get('CF_API_TOKEN'),
42      cfGateway: await $.env.get('CF_GATEWAY_ID'),
43    },
44    text,
45    (await $.session.repo())?.root.split('/').at(-1) ?? null,
46  )
47  if (!call) throw new Error('no Jev transport (TYPESAFE_API_KEY or CF_ACCOUNT_ID+CF_API_TOKEN)')
48  const res = await Promise.race([
49    $.http.fetch(call.url, { method: 'POST', headers: call.headers, body: call.body }),
50    $.clock.sleep(TIMEOUT_MS).then(() => {
51      throw new Error(`timeout after ${TIMEOUT_MS} ms`)
52    }),
53  ])
54  if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.text.slice(0, 120)}`)
55
56  return parseJev(res.text, call.gateway)
57}
58
59/** Sends the held prompt, switching to `tier` first when one is given. */
60async function send($: Engine, p: Pending, tier: Tier | null) {
61  await update($, pending, () => null)
62  await settle($, tier ? 'switched' : 'kept')
63  if (tier) await $.command.run({ command: 'model', args: MODELS[tier].alias })
64  await $.prompt.submit({ text: p.text, asUser: true })
65}
66
67export const register: Register = (on, options) => {
68  const autoSwitch = Boolean(options.autoSwitch)
69  const downgrade = Boolean(options.downgrade)
70  const quiet = Boolean(options.quiet)
71
72  on('session.start', async ($, e, next) => {
73    // The hub's UserPromptSubmit hook (scripts/model_advisor.py) stands down on this.
74    await $.env.set('MODEL_ADVISOR_MOD', '1')
75    await $.command.register({
76      name: 'advisor',
77      description: 'Model advisor: the held prompt, and recent decisions',
78      argumentHint: '[switch|send]',
79    })
80
81    return next(e)
82  })
83
84  on('prompt.submit', async ($, e, next) => {
85    // Only a person's own first prompt: the resend, a plugin's or a scheduled prompt pass.
86    // The engine stamps every origin; an unstamped one (the test kit's) reads as the composer.
87    const kind = e.origin?.kind ?? 'composer'
88    if (kind !== 'composer' && kind !== 'bridge') return next(e)
89    if (await read($, decided)) return next(e)
90    const text = e.text
91    if (!text.trim() || text.trimStart().startsWith('/') || text.includes(OPT_OUT)) return next(e)
92    await update($, decided, () => true)
93    // A resumed session already has a cache worth keeping on its model.
94    let history: unknown = []
95    try {
96      history = await $.session.messages()
97    } catch {
98      // No transcript to read (a host without one): treat as a fresh session.
99    }
100    if (Array.isArray(history) && history.some(m => m?.role === 'assistant')) return next(e)
101    const off = (await $.env.get('MODEL_ADVISOR'))?.toLowerCase()
102    if (off === 'off' || off === '0' || off === 'false') return next(e)
103    if (NO_HUMAN.test((await $.env.get('CLAUDE_CODE_ENTRYPOINT')) ?? '')) return next(e)
104
105    const model = await $.session.model()
106    const current = tierOf(model)
107    const t0 = await $.clock.now()
108    let cls: Classification
109    try {
110      cls = await classify($, text)
111    } catch (err) {
112      // Fail open: a broken advisor never eats a prompt.
113      if (!quiet) $.ui.status(`advisor: classifier failed (${String(err).slice(0, 80)}); prompt sent unchanged`)
114
115      return next(e)
116    }
117    const ms = (await $.clock.now()) - t0
118    const { tier: rec, rule } = suggest(cls, current, downgrade)
119    // A prompt that cannot be held and resent intact (a phone, an attachment) is relayed instead.
120    const relay = kind === 'bridge' || (e.attachments?.length ?? 0) > 0
121    await log($, {
122      ts: t0,
123      model,
124      current,
125      suggest: rec,
126      rule,
127      ms,
128      origin: kind,
129      outcome: rec && relay ? 'relayed' : undefined,
130      head: text.slice(0, 120),
131    })
132
133    if (rec === null) {
134      if (quiet) return next(e)
135      const line = verdict(rule, current, cls)
136      $.ui.status(line)
137      if (kind !== 'bridge') return next(e)
138      const note = `The model advisor plugin wrote a note this user's client does not display. Open your reply with it verbatim as a blockquote, then answer as usual:\n${line}`
139
140      return next({ ...e, context: [...(e.context ?? []), note] })
141    }
142
143    const line = advice(rec, rule, current, cls)
144    if (relay) return next({ ...e, context: [...(e.context ?? []), relayContext(rec, current, line)] })
145
146    const held: Pending = { text, tier: rec, note: line }
147    if (autoSwitch) {
148      $.ui.toast(`${line} Switching to ${MODELS[rec].name} and resending.`)
149      // Commands queue until the session is idle, which it is once this prompt is dropped.
150      $.clock.after(0, () => void send($, held, rec))
151    } else {
152      await update($, pending, () => held)
153      $.ui.toast(line)
154    }
155
156    return { drop: line }
157  })
158
159  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
160    const p = await read($, pending)
161    if (!p) return next(e)
162    const { Box, Button, Text } = $.ui.resolve(e)
163    const now = tierOf(await $.session.model())
164    const keep = now ? MODELS[now].name : 'this model'
165
166    return (
167      <Box>
168        <Text>{p.note} </Text>
169        <Button key="switch" label={`Switch to ${MODELS[p.tier].name} and send`} onPress={() => send($, p, p.tier)} />
170        <Button key="keep" label={`Send on ${keep}`} onPress={() => send($, p, null)} />
171        <Button
172          key="edit"
173          label="Edit"
174          onPress={async () => {
175            await update($, pending, () => null)
176            await settle($, 'kept')
177            await $.prompt.fill({ text: p.text })
178          }}
179        />
180      </Box>
181    )
182  })
183
184  on('command.run', { command: 'advisor' }, async ($, e) => {
185    const p = await read($, pending)
186    const arg = e.args.trim()
187    if (p && (arg === 'switch' || arg === 'send')) {
188      await send($, p, arg === 'switch' ? p.tier : null)
189
190      return { text: arg === 'switch' ? `Switching to ${MODELS[p.tier].name} and sending.` : 'Sending unchanged.' }
191    }
192    const all = ((await $.store.get(LOG)) ?? []) as LogEntry[]
193    const fired = all.filter(r => r.suggest)
194    const lines = fired
195      .slice(-10)
196      .map(r => `  ${r.current ?? '?'} -> ${r.suggest} [${r.outcome ?? 'open'}] ${r.head.slice(0, 60)}`)
197    const held = p ? `Held: ${p.note}\n/advisor switch or /advisor send.\n` : ''
198
199    return {
200      text: `${held}First prompts classified ${all.length}, advised ${fired.length}.${lines.length ? `\n${lines.join('\n')}` : ''}`,
201    }
202  })
203}
204

1import type { Classification, Scope, Tier } from '../types'
2
3// Port of claude-workspace scripts/model_advisor.py: the same classifier contract,
4// thresholds and rules. Change both together.
5
6export const TIERS: readonly Tier[] = ['haiku', 'sonnet', 'opus', 'fable']
7const SCOPES: readonly Scope[] = ['quick', 'task', 'project']
8
9/** Display name, /model alias, first-party input $/MTok (2026-10; output is 5x on every tier). */
10export const MODELS: Record<Tier, { name: string; alias: string; price: number }> = {
11  haiku: { name: 'Haiku 5.5', alias: 'haiku', price: 0.1 },
12  sonnet: { name: 'Sonnet 5.5', alias: 'sonnet', price: 2 },
13  opus: { name: 'Opus 5.5', alias: 'opus', price: 4 },
14  fable: { name: 'Fable 5.1', alias: 'fable', price: 10 },
15}
16
17// Sonnet 5.5 finishes a named deliverable well but stops to ask on a goal it has to
18// work out (2026-10-07), so the sonnet/opus line is whether the work is named.
19export const CRITERIA: Record<Tier, string> = {
20  haiku:
21    'A mechanical or fully specified task a small fast model completes reliably: run a named command or script, look something up, rename or reformat, extract or classify, summarise one item, apply edits the prompt spells out. No judgment, design or debugging expected.',
22  sonnet:
23    'A bounded task whose deliverable the prompt names: implement or fix a described change in known code, review one document, issue or PR, answer a factual question, a routine lookup or write-up. Done when the named thing is done; nothing has to be found out first.',
24  opus:
25    'Work the prompt sets as a goal and leaves to be discovered: find what needs changing and then change it, audit and update, follow through across many files or repositories, design an approach, debug an unknown cause, assess claims critically, synthesise research across sources. Needs initiative to keep going without asking.',
26  fable:
27    'Frontier-hard work where depth matters more than cost: novel research or experiments with uncertain outcomes, subtle correctness or security reasoning, long autonomous projects spanning many hours.',
28}
29const QUESTION = 'Which is the least capable model tier that would reliably complete this task well?'
30const JUDGEABLE =
31  'The prompt itself describes the work to be done, so its difficulty can be judged without opening a linked issue, memory id, file, paper or web page.'
32const SCOPE_CRITERIA: Record<Scope, string> = {
33  quick: 'A single answer, a short conversation or one small change, done within minutes.',
34  task: 'A focused task of several steps, finished within about an hour.',
35  project: 'An extended session: research, experiments or many changes over hours.',
36}
37
38export const THETA_UP = 0.75
39export const THETA_DOWN = 0.75
40export const JUDGEABLE_MIN = 0.5
41export const QUICK_MAX = 0.5
42export const PROMPT_CHARS = 4000
43export const TIMEOUT_MS = 6000
44export const OPT_OUT = '#no-advice'
45
46export const tierOf = (model: string | undefined): Tier | null =>
47  TIERS.find(t => (model ?? '').toLowerCase().includes(t)) ?? null
48
49const rank = (t: Tier) => TIERS.indexOf(t)
50const atMost = (p: Record<Tier, number>, t: Tier) => TIERS.slice(0, rank(t) + 1).reduce((s, u) => s + p[u], 0)
51const atLeast = (p: Record<Tier, number>, t: Tier) => TIERS.slice(rank(t)).reduce((s, u) => s + p[u], 0)
52export const best = (p: Record<Tier, number>): Tier => TIERS.reduce((a, b) => (p[b] > p[a] ? b : a))
53
54// ------------------------------------------------------------------ transport
55
56export type JevEnv = {
57  typesafeKey?: string
58  cfAccount?: string
59  cfToken?: string
60  cfGateway?: string
61}
62
63export type JevCall = { url: string; headers: Record<string, string>; body: string; gateway: boolean }
64
65/** The Jev request for `prompt`, or null with no transport configured. */
66export const jevCall = (env: JevEnv, prompt: string, repo: string | null): JevCall | null => {
67  const state = {
68    setting:
69      'The first message of a Claude Code session. The model works autonomously with tools (shell, files, web, GitHub) for as many turns as the task takes.',
70    repository: repo,
71    prompt: prompt.slice(0, PROMPT_CHARS),
72  }
73  const questions = {
74    tier: { type: 'choice', instructions: QUESTION, criteria: CRITERIA },
75    judgeable: { type: 'noul', instructions: JUDGEABLE },
76    scope: { type: 'choice', instructions: 'How much work will this session take?', criteria: SCOPE_CRITERIA },
77  }
78  if (env.typesafeKey) {
79    return {
80      url: 'https://api.typesafe.ai/v1/systemone',
81      headers: { Authorization: `Bearer ${env.typesafeKey}`, 'Content-Type': 'application/json' },
82      body: JSON.stringify({ model: 'jev-latest', state, questions }),
83      gateway: false,
84    }
85  }
86  if (env.cfToken && env.cfAccount) {
87    const headers: Record<string, string> = {
88      Authorization: `Bearer ${env.cfToken}`,
89      'Content-Type': 'application/json',
90    }
91    if (env.cfGateway) headers['cf-aig-gateway-id'] = env.cfGateway
92
93    return {
94      url: `https://api.cloudflare.com/client/v4/accounts/${env.cfAccount}/ai/run`,
95      headers,
96      body: JSON.stringify({ model: 'typesafe/jev', input: { state, questions } }),
97      gateway: true,
98    }
99  }
100
101  return null
102}
103
104/** Reads Jev's answer; throws with a short reason on any shape it does not expect. */
105export const parseJev = (text: string, gateway: boolean): Classification => {
106  let ans: any
107  try {
108    const res = JSON.parse(text)
109    ans = (gateway ? res.result.result : res).answers
110  } catch {
111    throw new Error(`unreadable answer: ${text.slice(0, 120)}`)
112  }
113  const num = (v: unknown) => {
114    const n = Number(v)
115    if (v === undefined || v === null || Number.isNaN(n)) throw new Error(`unexpected answer shape: ${JSON.stringify(ans).slice(0, 120)}`)
116
117    return n
118  }
119  const tier = Object.fromEntries(TIERS.map(t => [t, num(ans?.tier?.probabilities?.[t])])) as Record<Tier, number>
120  const scope = Object.fromEntries(SCOPES.map(s => [s, num(ans?.scope?.probabilities?.[s])])) as Record<Scope, number>
121
122  return { tier, scope, judgeable: num(ans?.judgeable?.noul) }
123}
124
125// ------------------------------------------------------------------- decision
126
127/** The tier to suggest, or null, and the rule that decided. */
128export const suggest = (
129  cls: Classification,
130  current: Tier | null,
131  downgrade = false,
132): { tier: Tier | null; rule: string } => {
133  const p = cls.tier
134  let rec: Tier | null = null
135  if (current === null) {
136    if (downgrade) rec = TIERS.slice(0, 2).find(t => atMost(p, t) >= THETA_DOWN) ?? null
137  } else {
138    if (downgrade) rec = TIERS.slice(0, rank(current)).find(t => atMost(p, t) >= THETA_DOWN) ?? null
139    if (rec === null) rec = [...TIERS.slice(rank(current) + 1)].reverse().find(t => atLeast(p, t) >= THETA_UP) ?? null
140  }
141  if (rec === null) {
142    if (current === null) return { tier: null, rule: 'unknown-model' }
143    if (cls.judgeable < JUDGEABLE_MIN) return { tier: null, rule: 'opaque' }
144    const b = best(p)
145    if (b === current) return { tier: null, rule: 'agrees' }
146
147    return { tier: null, rule: rank(b) < rank(current) && !downgrade ? 'cheaper' : 'unsure' }
148  }
149  const down = current === null || rank(rec) < rank(current)
150  if (cls.judgeable < JUDGEABLE_MIN) {
151    if (current === 'fable' && down) return { tier: 'opus', rule: 'opaque-fable-prior' }
152
153    return { tier: null, rule: 'opaque' }
154  }
155  if (down && cls.scope.quick > QUICK_MAX) return { tier: null, rule: 'quick' }
156
157  return { tier: rec, rule: 'tier' }
158}
159
160// ------------------------------------------------------------------- messages
161
162const pct = (x: number) => `${Math.round(x * 100)}%`
163const a = (name: string) => (/^[AEIOU]/.test(name) ? `an ${name}` : `a ${name}`)
164
165export const WHY_SILENT: Record<string, string> = {
166  agrees: "agrees with the session's model",
167  unsure: 'leans another way, but not confidently enough to interrupt',
168  'unknown-model': "couldn't read the session's model",
169  cheaper: 'a cheaper model looks enough, and the advisor only suggests upgrades',
170  opaque: 'the prompt points at the work rather than describing it, so no call',
171  quick: 'looks like a few-minute session, not worth interrupting',
172}
173
174/** One line for a suggestion: the tier, how sure, and the cost ratio. */
175export const advice = (rec: Tier, rule: string, current: Tier | null, cls: Classification): string => {
176  const { name } = MODELS[rec]
177  const down = current === null || rank(rec) < rank(current)
178  const sure = down ? atMost(cls.tier, rec) : atLeast(cls.tier, rec)
179  const why =
180    rule === 'opaque-fable-prior'
181      ? 'the prompt points at the work rather than describing it, and in 32 graded past sessions none needed Fable'
182      : `${pct(sure)} sure`
183  if (current === null) return `Model advisor: this looks like ${a(name)} task (${why}).`
184  const ratio = MODELS[current].price / MODELS[rec].price
185  const cost = ratio > 1 ? `${ratio.toFixed(1)}x less` : `${(1 / ratio).toFixed(1)}x more`
186
187  return `Model advisor: this looks like ${a(name)} task (${why}); the session is on ${MODELS[current].name}, and ${name} costs about ${cost} per token.`
188}
189
190/** Context for the model when the person's client cannot show a choice (phone, web). */
191export const relayContext = (rec: Tier, current: Tier | null, line: string): string => {
192  const { name, alias } = MODELS[rec]
193  const down = current === null || rank(rec) < rank(current)
194  const todo = down
195    ? 'then answer as usual.'
196    : `then do the task, running its substantive work (the design, the hard reasoning, the implementation) in a subagent through the Agent tool with model "${alias}". Keep the main loop to coordinating, checking the subagent's result, and the reply.`
197  const tail = down ? ` /model ${alias} would likely do, for next time.` : ` Handing the heavy part to ${a(name)} subagent; /model ${alias} moves the whole session.`
198
199  return `The model advisor plugin classified this prompt; the user's client does not display plugin output. Open your reply with this line verbatim as a blockquote, ${todo}\n${line}${tail}`
200}
201
202/** The status-line verdict when the advisor stays silent. */
203export const verdict = (rule: string, current: Tier | null, cls: Classification): string => {
204  const b = best(cls.tier)
205  const on = current ? MODELS[current].name : 'unknown model'
206
207  return `advisor: ${MODELS[b].name}-shaped (${pct(cls.tier[b])}), on ${on}; quick ${pct(cls.scope.quick)}, describes the work ${pct(cls.judgeable)}. Silent: ${WHY_SILENT[rule] ?? rule}.`
208}
209

1/** Cheapest first; the index is the rank. */
2export type Tier = 'haiku' | 'sonnet' | 'opus' | 'fable'
3
4export type Scope = 'quick' | 'task' | 'project'
5
6/** Jev's answer: P(least capable tier that does the job), P(prompt describes the work), P(scope). */
7export type Classification = {
8  tier: Record<Tier, number>
9  judgeable: number
10  scope: Record<Scope, number>
11}
12
13/** A first prompt held back while the person chooses a model. */
14export type Pending = { text: string; tier: Tier; note: string }
15
16/** One decision, kept in the plugin store for /advisor. */
17export type LogEntry = {
18  ts: number
19  model: string
20  current: Tier | null
21  suggest: Tier | null
22  rule: string
23  ms: number
24  origin: string
25  outcome?: 'switched' | 'kept' | 'relayed'
26  head: string
27}
28
29declare module 'claude-code' {
30  interface PluginState {
31    'model-advisor': { pending: Pending | null; decided: boolean }
32  }
33}
34