model-hint

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register, Timer } from 'claude-code'
3
4import type { Suggestion } from '../types'
5import { classify, isSkippable, type Bias } from './classify'
6import * as judge from './judge'
7import { satisfies, tierOf } from './model'
8
9const COMMAND = 'model-hint'
10
11const suggestion = atom({ plugin: 'model-hint', key: 'suggestion' } as const, null)
12const isEnabled = atom({ plugin: 'model-hint', key: 'isEnabled' } as const, true)
13const isDismissed = atom({ plugin: 'model-hint', key: 'isDismissed' } as const, false)
14const judgeNote = atom({ plugin: 'model-hint', key: 'judgeNote' } as const, null)
15
16const BIASES: readonly Bias[] = ['balanced', 'cost', 'quality']
17const SHOWS = ['always', 'when-different'] as const
18const ENGINES = ['model', 'rules'] as const
19type Show = (typeof SHOWS)[number]
20type Engine = (typeof ENGINES)[number]
21
22/**
23 * How long the person must pause before the model reads the draft. A judge call
24 * measured 0.8 to 1.4 s (median about 0.95 s), so 300 ms keeps the verdict near
25 * 1.25 s after typing stops; a stray keystroke only aborts the call in flight.
26 */
27export const PAUSE_MS = 300
28
29/** Verdicts kept per session, so a draft judged once answers instantly. */
30const CACHE_SIZE = 64
31
32type Settings = { bias: Bias; show: Show; engine: Engine; judgeModel: string }
33
34function pick<T extends string>(value: unknown, allowed: readonly T[], fallback: T): T {
35  return typeof value === 'string' && (allowed as readonly string[]).includes(value) ? (value as T) : fallback
36}
37
38function sameSuggestion(a: Suggestion | null, b: Suggestion | null): boolean {
39  if (a === null || b === null) return a === b
40  return a.tier === b.tier && a.reasons.join('|') === b.reasons.join('|')
41}
42
43/** Writes the hint only when it changed, so a stable classification never redraws; a change also clears a Hide. */
44async function publish($: EngineInterface, next: Suggestion | null, force: boolean): Promise<void> {
45  let changed = false
46  await update($, suggestion, prev => {
47    changed = !sameSuggestion(prev, next)
48    return changed ? next : prev
49  })
50  if (changed || force) await update($, isDismissed, () => false)
51}
52
53// One pending model pass at a time, for the draft as it stood when the person
54// paused. Module state: a hot reload drops it, which is the right outcome.
55let pending: Timer | undefined
56let inflight: AbortController | undefined
57let generation = 0
58
59// Verdicts by judge model and trimmed draft, oldest first. Backspacing to a
60// draft already judged, or pausing twice on the same text, costs no call.
61const verdicts = new Map<string, Suggestion>()
62
63// Wall-clock time of the last edit, for the debug log's latency line only.
64let editedAt = 0
65
66function cacheKey(settings: Settings, draft: string): string {
67  return `${settings.judgeModel}\u0000${draft.trim()}`
68}
69
70function remember(key: string, verdict: Suggestion): void {
71  verdicts.delete(key)
72  verdicts.set(key, verdict)
73  if (verdicts.size > CACHE_SIZE) {
74    const oldest = verdicts.keys().next().value
75    if (oldest !== undefined) verdicts.delete(oldest)
76  }
77}
78
79// Whether this session draws the band. Learned at the first AbovePrompt render,
80// not from the surface's name: until then the status line under the prompt
81// carries the hint. That covers surfaces with a status line but no band; it does
82// nothing in the VS Code panel, where prompt.edit never fires.
83let hasBand = false
84
85async function mirror($: EngineInterface, settings: Settings): Promise<void> {
86  if (hasBand) return
87  const [hint, enabled, note] = await Promise.all([read($, suggestion), read($, isEnabled), read($, judgeNote)])
88  if (hint === null || !enabled) {
89    $.ui.status(undefined)
90    return
91  }
92  if (hint.source === 'rules' && settings.engine === 'model' && note === null) {
93    $.ui.status('model-hint: reading the draft...')
94    return
95  }
96  const current = tierOf(await $.session.model())
97  const covered = satisfies(current, hint.tier)
98  if (covered && settings.show === 'when-different') {
99    $.ui.status(undefined)
100    return
101  }
102  const tail = covered && current ? `covered by ${current}` : `/model ${hint.tier} to switch`
103  $.ui.status(`model-hint: ${hint.tier} · ${hint.reasons.join(', ')}${note ? ` (${note}, rules' guess)` : ''} · ${tail}`)
104}
105
106function cancelPending(): number {
107  pending?.cancel()
108  inflight?.abort()
109  generation += 1
110  return generation
111}
112
113/**
114 * The instant pass. The rules answer at once; a model verdict already on
115 * screen is kept while the person keeps typing, so the band does not flip
116 * between the two sources on every keystroke. Then the model pass is armed.
117 */
118async function onDraft($: EngineInterface, draft: string, settings: Settings): Promise<void> {
119  const mine = cancelPending()
120  editedAt = Date.now()
121
122  const rules = classify(draft, settings.bias)
123  await update($, judgeNote, () => null)
124
125  const known = settings.engine === 'model' && rules !== null ? verdicts.get(cacheKey(settings, draft)) : undefined
126  if (known) {
127    await publish($, known, false)
128    await mirror($, settings)
129    return
130  }
131
132  const held = await read($, suggestion)
133  const keepModelVerdict = held?.source === 'model' && rules !== null
134  if (!keepModelVerdict) await publish($, rules, isSkippable(draft))
135  await mirror($, settings)
136
137  if (settings.engine !== 'model' || rules === null || draft.trim().length < judge.MIN_LENGTH) return
138  pending = $.clock.after(PAUSE_MS, () => {
139    void ask($, draft, mine, settings)
140  })
141}
142
143/** The model pass. A verdict lands only while it is still the latest and the draft has not moved on. */
144async function ask($: EngineInterface, draft: string, mine: number, settings: Settings): Promise<void> {
145  const stop = new AbortController()
146  inflight = stop
147  const firedAt = Date.now()
148  // A failure is shown in the band and logged; it never sits silently on "reading".
149  const fail = async (note: string, detail: string) => {
150    $.ui.log(`model-hint: ${note}: ${detail}`, { to: 'debug' })
151    if (mine !== generation) return
152    await update($, judgeNote, () => note)
153    await mirror($, settings)
154  }
155  let reply
156  try {
157    reply = await $.model.complete(judge.request(draft, settings.judgeModel), { signal: stop.signal })
158  } catch (err) {
159    await fail('judge call failed', String(err))
160    return
161  }
162  if (mine !== generation) return
163  if (!reply.isAnswered) {
164    if (reply.reason === 'aborted') return
165    const detail = reply.reason === 'api-error' ? `${reply.status ?? ''} ${reply.error}` : reply.reason
166    await fail(reply.reason === 'api-error' ? 'judge API error' : 'judge gave no answer', detail)
167    return
168  }
169
170  const verdict = judge.parse(reply.text)
171  if (verdict === null) {
172    await fail('judge reply unreadable', reply.text.slice(0, 200))
173    return
174  }
175  const box = await $.prompt.read()
176  if (box.text !== draft) {
177    $.ui.log(`model-hint: draft moved on before the verdict landed (${box.text.length} vs ${draft.length} chars)`, { to: 'debug' })
178    return
179  }
180  remember(cacheKey(settings, draft), verdict)
181  const now = Date.now()
182  $.ui.log(
183    `model-hint: verdict ${verdict.tier} (${verdict.reasons[0]}) ${now - editedAt} ms after the last edit ` +
184      `(timer ${firedAt - editedAt} ms, judge call ${now - firedAt} ms)`,
185    { to: 'debug' },
186  )
187  await publish($, verdict, false)
188  await mirror($, settings)
189}
190
191export const register: Register = (on, options) => {
192  const settings: Settings = {
193    bias: pick(options.bias, BIASES, 'balanced'),
194    show: pick<Show>(options.show, SHOWS, 'always'),
195    engine: pick<Engine>(options.engine, ENGINES, 'model'),
196    judgeModel: typeof options.judge === 'string' && options.judge.trim() ? options.judge.trim() : 'haiku',
197  }
198
199  on('session.start', async ($, e, next) => {
200    await $.command.register({
201      name: COMMAND,
202      description: 'Toggle the model hint band above the prompt',
203    })
204    return next(e)
205  })
206
207  on('command.run', { command: COMMAND }, async $ => {
208    const enabled = await update($, isEnabled, was => !was)
209    await mirror($, settings)
210    return { text: enabled ? 'Model hint on.' : 'Model hint off for this session.' }
211  }).catch(() => ({ text: 'Model hint: could not toggle, see the debug log.' }))
212
213  // Fires on each edit the composer makes; `next(e)` resolves the box after it.
214  on('prompt.edit', async ($, e, next) => {
215    const box = await next(e)
216    await onDraft($, box.text, settings)
217    return box
218  })
219
220  // A plugin's fill replaces the draft without a prompt.edit, so re-read it.
221  on('prompt.fill', async ($, e, next) => {
222    const filled = await next(e)
223    if (filled.isFilled) {
224      const box = await $.prompt.read()
225      await onDraft($, box.text, settings)
226    }
227    return filled
228  })
229
230  // A failure here must never hold a prompt back: let it through untouched.
231  on('prompt.submit', async ($, e, next) => {
232    cancelPending()
233    await update($, suggestion, () => null)
234    await update($, isDismissed, () => false)
235    await update($, judgeNote, () => null)
236    $.ui.status(undefined)
237    return next(e)
238  }).catch((_$, e, next) => next(e))
239
240  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
241    if (!hasBand) {
242      hasBand = true
243      $.ui.status(undefined)
244    }
245    if (e.props.hasSurvey) return next(e)
246
247    const [hint, enabled, dismissed, note] = await Promise.all([
248      read($, suggestion),
249      read($, isEnabled),
250      read($, isDismissed),
251      read($, judgeNote),
252    ])
253    if (hint === null || !enabled || dismissed) return next(e)
254
255    const current = tierOf(await $.session.model())
256    const covered = satisfies(current, hint.tier)
257    if (covered && settings.show === 'when-different') return next(e)
258
259    const { Box, Button, Text } = $.ui.resolve(e)
260    const tier = hint.tier
261    const hide = <Button key="hide" label="Hide" hotkey="h" onPress={() => update($, isDismissed, () => true)} />
262
263    // The rules' guess is never shown as a tier while the model is still reading.
264    if (hint.source === 'rules' && settings.engine === 'model' && note === null) {
265      return (
266        <Box gap={1}>
267          <Text dimColor>reading the draft...</Text>
268          {hide}
269        </Box>
270      )
271    }
272
273    const switchTo = async () => {
274      const rows = await $.config.list()
275      const row = rows.find(r => r.key === 'model')
276      if (!row || row.isLocked) {
277        $.ui.toast(`Type /model ${tier} to switch`)
278        return
279      }
280      const set = await $.config.set({ key: 'model', value: tier })
281      if (set.deny) $.ui.toast(`Could not switch: ${set.deny}. Type /model ${tier}`)
282      else await update($, isDismissed, () => true)
283    }
284
285    const detail = [
286      hint.reasons.join(', '),
287      note ? `(${note}, rules' guess)` : '',
288      covered && current ? `· covered by ${current}` : '',
289    ]
290      .filter(Boolean)
291      .join(' ')
292
293    return (
294      <Box gap={1}>
295        <Text color={covered ? 'success' : 'suggestion'} bold>
296          {tier}
297        </Text>
298        <Text dimColor wrap="truncate-end">
299          {detail}
300        </Text>
301        {!covered && <Button key="switch" label={`Switch to ${tier}`} hotkey="s" onPress={switchTo} />}
302        {hide}
303      </Box>
304    )
305  })
306}
307

1import type { Suggestion, Tier } from '../types'
2
3/**
4 * Scores a draft prompt and picks a tier. Pure: no engine, no model call, so it
5 * runs on every keystroke without cost and is unit-tested in isolation.
6 *
7 * The score is a sum of signals. Negative leans haiku, positive leans opus,
8 * the middle is sonnet. `bias` shifts the two thresholds so a cost-conscious
9 * person needs more evidence before being pushed up a tier, and vice versa.
10 */
11
12export type Bias = 'balanced' | 'cost' | 'quality'
13
14type Signal = {
15  /** Shown to the person as the reason for the pick; keep it short. */
16  label: string
17  weight: number
18  test: (draft: string, lower: string) => boolean
19}
20
21const words = (list: string[]) => new RegExp(`\\b(?:${list.join('|')})\\b`, 'i')
22
23const HEAVY_VERBS = words([
24  'architect', 'design', 'redesign', 'rearchitect', 'restructure', 'migrate', 'migration',
25  'refactor', 'rewrite', 'port', 'overhaul', 'modernize', 'modernise', 'decompose',
26  'modulari[sz]e', 'made modular', 'make .{1,30} modular', 'decouple', 'untangle', 'separat(?:e|ed|ing)',
27  'split (?:up|out|into|apart)', 'break (?:up|apart|out)', 'extract (?:into|out|a|the)', 'consolidate',
28])
29const HARD_PROBLEMS = words([
30  'race condition', 'deadlock', 'memory leak', 'flaky', 'intermittent', 'heisenbug',
31  'root cause', 'regression', 'data loss', 'corrupt(?:ed|ion)?', 'nondeterministic',
32  'performance', 'latency', 'throughput', 'scal(?:e|ing|ability)', 'concurren(?:cy|t)',
33])
34const JUDGMENT = words([
35  'trade-?offs?', 'strategy', 'approach', 'options?', 'compare', 'evaluate', 'assess',
36  'recommend', 'should (?:we|i)', 'pros and cons', 'which is better', 'critique', 'review',
37  'audit', 'security', 'threat model', 'plan', 'roadmap', 'spec', 'rfc', 'proposal',
38])
39const WIDE_SCOPE = words([
40  'across', 'entire', 'whole', 'all (?:the )?(?:files|modules|packages|services|tests)',
41  'codebase', 'monorepo', 'end[- ]to[- ]end', 'every', 'throughout', 'system[- ]wide',
42  '(?:multiple|several|many) (?:features|modules|files|services|components|packages|places)',
43])
44const DEBUG = words(['debug(?:ging|ged)?', 'investigate', 'troubleshoot(?:ing)?', 'diagnose', 'figure out', 'why (?:does|is|do|did|are|was|were)', "doesn't work", 'not working', 'broken', 'crash(?:es|ing)?'])
45
46const TRIVIAL = words([
47  'typo', 'rename', 'spelling', 'capitali[sz]e', 'lowercase', 'uppercase', 'indent',
48  'reformat', 'prettier', 'lint', 'alphabeti[sz]e', 'reorder',
49  'add a comment', 'remove (?:the )?comment', 'bump (?:the )?version', 'changelog',
50  'git (?:status|log|diff|add|commit|push|pull|stash|branch)', 'commit this', 'commit message',
51  'run (?:the )?tests?', 'run (?:the )?build', 'run (?:the )?linter', 'npm (?:run|test|install)',
52  'what does .{1,40} (?:do|mean|return)', 'what is .{1,40}\\?', 'where is', 'where are', 'which file',
53  'list (?:the|all)', 'show me', 'how many',
54  'translate', 'convert .{1,30} to (?:json|yaml|csv|markdown|md)', 'one[- ]liner', 'quick question',
55])
56const SMALL_SCOPE = words(['this line', 'this function', 'this file', 'one line', 'single line', 'tiny', 'minor', 'quick'])
57
58/** Past this length a draft is a brief, not a mechanical edit, whatever words it uses. */
59const MECHANICAL_MAX = 140
60
61const SIGNALS: Signal[] = [
62  { label: 'architecture or large refactor', weight: 3, test: (_, l) => HEAVY_VERBS.test(l) },
63  { label: 'hard bug class', weight: 3, test: (_, l) => HARD_PROBLEMS.test(l) },
64  { label: 'needs judgment', weight: 3, test: (_, l) => JUDGMENT.test(l) },
65  { label: 'wide scope', weight: 2, test: (_, l) => WIDE_SCOPE.test(l) },
66  { label: 'debugging', weight: 2, test: (_, l) => DEBUG.test(l) },
67  { label: 'multi-step', weight: 3, test: d => countSteps(d) >= 3 },
68  { label: 'long brief', weight: 2, test: d => d.length >= 600 },
69  { label: 'detailed brief', weight: 1, test: d => d.length >= 250 && d.length < 600 },
70  { label: 'mechanical edit', weight: -3, test: (d, l) => d.length < MECHANICAL_MAX && TRIVIAL.test(l) },
71  { label: 'small scope', weight: -1, test: (_, l) => SMALL_SCOPE.test(l) },
72  { label: 'short ask', weight: -1, test: d => d.trim().length < 40 },
73]
74
75/** Numbered items, bullets, or "then"/"after that" chains count as steps. */
76export function countSteps(draft: string): number {
77  const listed = (draft.match(/(?:^|\n)\s*(?:\d+[.)]|[-*•])\s+\S/g) ?? []).length
78  const chained = (draft.match(/\b(?:then|after that|next|finally|afterwards|once that)\b/gi) ?? []).length
79  return Math.max(listed, chained + 1)
80}
81
82/** Drafts the model never sees: slash commands, shell escapes, nothing typed. */
83export function isSkippable(draft: string): boolean {
84  const t = draft.trimStart()
85  return t.length === 0 || t.startsWith('/') || t.startsWith('!')
86}
87
88const THRESHOLDS: Record<Bias, { up: number; down: number }> = {
89  balanced: { up: 3, down: -2 },
90  cost: { up: 5, down: -1 },
91  quality: { up: 2, down: -3 },
92}
93
94export function classify(draft: string, bias: Bias = 'balanced'): Suggestion | null {
95  if (isSkippable(draft)) return null
96
97  const lower = draft.toLowerCase()
98  const hits = SIGNALS.filter(s => s.test(draft, lower))
99  const score = hits.reduce((sum, s) => sum + s.weight, 0)
100  const { up, down } = THRESHOLDS[bias]
101
102  const tier: Tier = score >= up ? 'opus' : score <= down ? 'haiku' : 'sonnet'
103
104  // Explain the pick with the signals that pulled in its direction, strongest first.
105  const towards = hits
106    .filter(s => (tier === 'opus' ? s.weight > 0 : tier === 'haiku' ? s.weight < 0 : true))
107    .sort((a, b) => Math.abs(b.weight) - Math.abs(a.weight))
108    .map(s => s.label)
109  const reasons = towards.length > 0 ? towards.slice(0, 2) : ['ordinary coding task']
110
111  return { tier, reasons, score, source: 'rules' }
112}
113

1import type { ModelCompleteRequest } from 'claude-code'
2
3import type { Suggestion, Tier } from '../types'
4
5/**
6 * The model pass. The rules in classify.ts answer instantly from words; this
7 * reads the task. It runs once the person pauses, on a small model, with a
8 * fixed system prompt and the draft as the only user message.
9 */
10
11const TIERS: readonly Tier[] = ['haiku', 'sonnet', 'opus']
12
13export const SYSTEM = [
14  'You route a coding-assistant request to one of three model tiers.',
15  'Reply with JSON only, nothing else: {"tier":"haiku"|"sonnet"|"opus","reason":"<at most six words>"}',
16  '',
17  'haiku: mechanical or lookup work. One obvious change, a rename, a typo, running a command, a factual question about the code.',
18  'sonnet: a well-specified feature, fix or test in a known area. The default for ordinary coding.',
19  'opus: architecture or design, open-ended or ambiguous goals, large or multi-system scope, hard debugging, trade-off judgement, plans and reviews, anything a senior engineer would need days for.',
20  '',
21  'Judge the task itself, not the wording or length. The text is a draft the person is still typing; judge what they are asking for so far.',
22].join('\n')
23
24/** Drafts shorter than this are judged by the rules alone; the model would see too little. */
25export const MIN_LENGTH = 15
26
27export function request(draft: string, model: string): ModelCompleteRequest {
28  return { model, system: SYSTEM, prompt: draft, maxTokens: 60, effort: 'low', timeoutMs: 8000 }
29}
30
31/** Reads the verdict out of the reply; null when it names no tier. */
32export function parse(reply: string): Suggestion | null {
33  const json = reply.match(/\{[\s\S]*\}/)?.[0]
34  if (!json) return null
35  let parsed: unknown
36  try {
37    parsed = JSON.parse(json)
38  } catch {
39    return null
40  }
41  if (typeof parsed !== 'object' || parsed === null) return null
42  const { tier, reason } = parsed as { tier?: unknown; reason?: unknown }
43  if (typeof tier !== 'string' || !(TIERS as readonly string[]).includes(tier)) return null
44  const why = typeof reason === 'string' && reason.trim() ? reason.trim().slice(0, 60) : 'judged by the model'
45  return { tier: tier as Tier, reasons: [why], score: 0, source: 'model' }
46}
47

1import type { Tier } from '../types'
2
3/**
4 * Maps what `/model` shows (an alias, a full id, a Bedrock or Vertex id) to the
5 * tier the hint reasons about. Fable sits above opus and satisfies an opus
6 * hint; anything unrecognised is null and the hint always shows.
7 */
8export type Current = Tier | 'fable' | null
9
10export function tierOf(model: string): Current {
11  const m = model.toLowerCase()
12  if (m.includes('fable')) return 'fable'
13  if (m.includes('opus')) return 'opus'
14  if (m.includes('sonnet')) return 'sonnet'
15  if (m.includes('haiku')) return 'haiku'
16  return null
17}
18
19/** Whether the session's model already covers the suggested tier. */
20export function satisfies(current: Current, suggested: Tier): boolean {
21  if (current === null) return false
22  if (current === suggested) return true
23  return current === 'fable' && suggested === 'opus'
24}
25

1/** The three tiers the hint picks between. */
2export type Tier = 'haiku' | 'sonnet' | 'opus'
3
4/** What produced a suggestion: the instant word rules, or the small model that reads the draft after a pause. */
5export type Source = 'rules' | 'model'
6
7/** What the classifier concluded about the current draft. */
8export type Suggestion = {
9  tier: Tier
10  /** Short, human-readable grounds for the pick, e.g. "typo fix, one line". */
11  reasons: string[]
12  /** The rules' net score: negative leans haiku, positive leans opus. 0 for a model verdict. */
13  score: number
14  source: Source
15}
16
17declare module 'claude-code' {
18  interface PluginState {
19    'model-hint': {
20      /** The hint for the draft as it stands; null while the box is empty or the draft is a command. */
21      suggestion: Suggestion | null
22      /** Session toggle, flipped by /model-hint. */
23      isEnabled: boolean
24      /** True once the person hides the band for the current draft; reset when the draft is submitted or emptied. */
25      isDismissed: boolean
26      /** Why the last model pass gave no verdict, shown in the band in place of "(reading...)"; null while none failed. */
27      judgeNote: string | null
28    }
29  }
30}
31