jev-skill-suggestion

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface as Engine, Register } from 'claude-code'
3
4import type { Mode, Ranked, Skill, Verdict } from '../types'
5
6// The two-request approach of docs.typesafe.ai/cookbooks/skill_suggestion:
7// rank every skill and ask whether the request needs one at all, then re-read
8// the top three with their SKILL.md openings and check each one really fits.
9const API = 'https://api.typesafe.ai/v1/systemone'
10const MODEL = 'jev-latest'
11const SHORTLIST = 3
12const EXCERPT_CHARS = 700
13const GATE_THRESHOLD = 0.3
14const FITS_THRESHOLD = 0.3
15const DEBOUNCE_MS = 600
16const MIN_CHARS = 8
17const PANE = 'jev-skill-suggestion'
18const MODES: Mode[] = ['observe', 'block', 'steer']
19
20const GATE_QUESTIONS: Record<string, string> = {
21  acts_on_user_system:
22    "Is the assistant being asked to act on the user's files, accounts, devices, or online services, rather than only to explain or advise?",
23  would_follow_documented_procedure:
24    'Would a careful expert answering this consult a specific documented procedure or set of commands, rather than answering from general understanding?',
25  prose_suffices:
26    "Could a knowledgeable generalist fully satisfy this request in prose, with no tools, no documentation, and no access to the user's files or accounts?",
27}
28const INVERTED = new Set(['prose_suffices'])
29
30const mode = atom({ plugin: 'jev-skill-suggestion', key: 'mode' } as const, 'observe')
31const disabled = atom({ plugin: 'jev-skill-suggestion', key: 'disabled' } as const, [])
32const roster = atom({ plugin: 'jev-skill-suggestion', key: 'roster' } as const, [])
33const verdict = atom({ plugin: 'jev-skill-suggestion', key: 'verdict' } as const, null)
34const status = atom({ plugin: 'jev-skill-suggestion', key: 'status' } as const, '')
35const enabled = atom({ plugin: 'jev-skill-suggestion', key: 'enabled' } as const, true)
36
37const unquote = (value: string) => value.trim().replace(/^(['"])([\s\S]*)\1$/, '$2')
38
39// Enough YAML for a SKILL.md header: `name`, and a `description` that is
40// inline, quoted, folded (`>`) or literal (`|`), continued on indented lines.
41export function parseSkillMd(text: string, fallbackName: string): Skill | null {
42  const match = text.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/)
43  if (!match) return null
44  const lines = (match[1] ?? '').split(/\r?\n/)
45  const field = (key: string) => {
46    const at = lines.findIndex(line => line.startsWith(`${key}:`))
47    if (at === -1) return ''
48    const head = (lines[at] ?? '').slice(key.length + 1).trim()
49    const rest: string[] = []
50    for (const line of lines.slice(at + 1)) {
51      if (!/^\s+\S/.test(line)) break
52      rest.push(line.trim())
53    }
54    const parts = /^[|>][-+]?$/.test(head) ? rest : [head, ...rest]
55    return unquote(parts.join(' '))
56  }
57  const name = field('name') || fallbackName
58  return { name, description: field('description'), body: (match[2] ?? '').trim().slice(0, EXCERPT_CHARS) }
59}
60
61// The engine's skill listing: `- name: description` per skill, a description
62// sometimes running on over following lines. Plugin skills carry `plugin:`.
63export function parseListing(text: string): Skill[] {
64  const skills: Skill[] = []
65  for (const line of text.split('\n')) {
66    const entry = line.match(/^- (\S+?)(?:: (.*))?$/)
67    const last = skills[skills.length - 1]
68    if (entry?.[1]) {
69      skills.push({ name: entry[1].replace(/:$/, ''), description: entry[2] ?? '', body: '' })
70    } else if (last && line.trim() && !line.startsWith('The following')) {
71      last.description += ` ${line.trim()}`
72    }
73  }
74  return skills
75}
76
77// Disk skills bring their SKILL.md body; the listing brings the plugin and
78// built-in ones the disk scan cannot see. The listing decides who exists.
79export function mergeRoster(disk: Skill[], listed: Skill[]): Skill[] {
80  if (listed.length === 0) return disk
81  const byName = new Map(disk.map(skill => [skill.name, skill]))
82  return listed.map(skill => {
83    const local = byName.get(skill.name)
84    return local ? { ...skill, description: local.description || skill.description, body: local.body } : skill
85  })
86}
87
88export function gateOf(values: Record<string, number>): number {
89  const oriented = Object.entries(values).map(([key, value]) => (INVERTED.has(key) ? 1 - value : value))
90  return oriented.length === 0 ? 0 : oriented.reduce((a, b) => a + b, 0) / oriented.length
91}
92
93const runtime: {
94  apiKey: string
95  asked: number
96  disk: Skill[]
97  listed: Skill[]
98  timer?: { cancel: () => void }
99} = { apiKey: '', asked: 0, disk: [], listed: [] }
100
101async function ask($: Engine, state: unknown, questions: Record<string, unknown>) {
102  const response = await $.http.fetch(API, {
103    method: 'POST',
104    headers: { Authorization: `Bearer ${runtime.apiKey}`, 'Content-Type': 'application/json' },
105    body: JSON.stringify({ state, model: MODEL, questions }),
106  })
107  if (!response.ok) throw new Error(`Jev ${response.status}: ${response.text.slice(0, 160)}`)
108  return JSON.parse(response.text).answers as Record<
109    string,
110    { choice?: string; probabilities?: Record<string, number>; noul?: number }
111  >
112}
113
114const suggest = async ($: Engine, text: string): Promise<Verdict> => {
115  const startedAt = await $.clock.now()
116  const off = new Set(await read($, disabled))
117  const skills = (await read($, roster)).filter(skill => !off.has(skill.name))
118  const state = { request: text, recent_context: '' }
119  if (skills.length === 0) return { text, gate: 0, ranked: [], fits: {}, winner: null, ms: 0 }
120
121  const wideQuestions: Record<string, unknown> = {
122    which: {
123      type: 'choice',
124      instructions: "Which of these skills, if any, is the right one to load to help with the user's latest request?",
125      criteria: Object.fromEntries(skills.map(skill => [skill.name, skill.description.slice(0, 250) || skill.name])),
126    },
127  }
128  for (const [key, instructions] of Object.entries(GATE_QUESTIONS)) {
129    wideQuestions[`gate::${key}`] = { type: 'noul', instructions }
130  }
131  const wide = await ask($, state, wideQuestions)
132  const ranked: Ranked[] = Object.entries(wide.which?.probabilities ?? {})
133    .map(([name, probability]) => ({ name, probability }))
134    .sort((a, b) => b.probability - a.probability)
135  const gate = gateOf(
136    Object.fromEntries(
137      Object.entries(wide)
138        .filter(([key]) => key.startsWith('gate::'))
139        .map(([key, answer]) => [key.slice('gate::'.length), answer.noul ?? 0]),
140    ),
141  )
142  const done = async (fits: Record<string, number>, winner: string | null): Promise<Verdict> => ({
143    text,
144    gate,
145    ranked: ranked.slice(0, 12),
146    fits,
147    winner,
148    ms: (await $.clock.now()) - startedAt,
149  })
150  if (gate < GATE_THRESHOLD) return done({}, null)
151
152  const shortlist = ranked.slice(0, SHORTLIST).map(entry => skills.find(skill => skill.name === entry.name)!)
153  const rerankQuestions: Record<string, unknown> = {
154    which: {
155      type: 'choice',
156      instructions:
157        "Exactly one of these skills is the right one to load for the user's latest request. Which one? Read what each actually does, not just its name.",
158      criteria: Object.fromEntries(
159        shortlist.map(skill => [skill.name, `${skill.description}${skill.body ? ` — ${skill.body}` : ''}`]),
160      ),
161    },
162  }
163  for (const skill of shortlist) {
164    rerankQuestions[`fits::${skill.name}`] = {
165      type: 'noul',
166      instructions: `Does the skill '${skill.name}' do the specific thing the user's request asks for? It is described as: ${skill.description}`,
167    }
168  }
169  const narrow = await ask($, state, rerankQuestions)
170  const fits = Object.fromEntries(
171    shortlist.map(skill => [skill.name, narrow[`fits::${skill.name}`]?.noul ?? 0]),
172  )
173  const best = Math.max(...Object.values(fits))
174  return done(fits, best < FITS_THRESHOLD ? null : (narrow.which?.choice ?? null))
175}
176
177// Scores the draft; a newer draft's answer wins, an older one is dropped.
178const score = async ($: Engine, text: string) => {
179  const ticket = ++runtime.asked
180  if (!runtime.apiKey) {
181    await update($, status, () => 'no API key: set TYPESAFE_API_KEY or the plugin option')
182    return null
183  }
184  await update($, status, () => 'scoring…')
185  try {
186    const result = await suggest($, text)
187    if (ticket === runtime.asked) {
188      await update($, verdict, () => result)
189      await update($, status, () => '')
190    }
191    return result
192  } catch (error) {
193    if (ticket === runtime.asked) await update($, status, () => String((error as Error).message ?? error))
194    return null
195  }
196}
197
198const scanDisk = async ($: Engine, cwd: string) => {
199  const home = (await $.env.get('HOME')) ?? ''
200  const found: Skill[] = []
201  for (const root of [`${home}/.claude/skills`, `${cwd}/.claude/skills`]) {
202    const entries = await $.fs.list(root).catch(() => [])
203    for (const entry of entries) {
204      if (entry.kind !== 'dir' && !entry.isLink) continue
205      const text = await $.fs.read(`${root}/${entry.name}/SKILL.md`).catch(() => null)
206      const skill = typeof text === 'string' ? parseSkillMd(text, entry.name) : null
207      if (skill && !found.some(other => other.name === skill.name)) found.push(skill)
208    }
209  }
210  return found
211}
212
213const setMode = async ($: Engine, next: Mode) => {
214  await update($, mode, () => next)
215  await $.store.set('mode', next)
216  $.ui.invalidate('prompt.attachment')
217}
218
219const setEnabled = async ($: Engine, on: boolean) => {
220  runtime.timer?.cancel()
221  runtime.asked++
222  await update($, enabled, () => on)
223  await update($, status, () => '')
224  if (!on) await update($, verdict, () => null)
225  await $.store.set('enabled', on)
226  $.ui.invalidate('prompt.attachment')
227}
228
229const toggle = async ($: Engine, name: string) => {
230  const list = await update($, disabled, list =>
231    list.includes(name) ? list.filter(one => one !== name) : [...list, name],
232  )
233  await $.store.set('disabled', list)
234  $.ui.invalidate('prompt.attachment')
235}
236
237
238export const register: Register = (on, options) => {
239  runtime.apiKey = typeof options.apiKey === 'string' ? options.apiKey : ''
240
241  on('session.start', async ($, e, next) => {
242    runtime.apiKey ||= (await $.env.get('TYPESAFE_API_KEY')) ?? ''
243    const storedMode = await $.store.get('mode')
244    if (typeof storedMode === 'string' && MODES.includes(storedMode as Mode)) {
245      await update($, mode, () => storedMode as Mode)
246    }
247    if ((await $.store.get('enabled')) === false) await update($, enabled, () => false)
248    const storedDisabled = await $.store.get('disabled')
249    if (Array.isArray(storedDisabled)) await update($, disabled, () => storedDisabled.map(String))
250    runtime.disk = await scanDisk($, e.cwd)
251    await update($, roster, () => mergeRoster(runtime.disk, runtime.listed))
252    await $.command.register({
253      name: 'jev',
254      description: 'Jev skill scores: open the skills pane, `/jev observe|block|steer`, or `/jev off|on`',
255    })
256    return next(e)
257  })
258
259  on('command.run', { command: 'jev' }, async ($, e) => {
260    const arg = e.args.trim()
261    if (arg === 'off' || arg === 'on') {
262      await setEnabled($, arg === 'on')
263      return { text: arg === 'on' ? 'Jev skills: on.' : 'Jev skills: off. Nothing is scored and Claude picks skills as usual. `/jev on` turns it back on.' }
264    }
265    if (MODES.includes(arg as Mode)) {
266      await setMode($, arg as Mode)
267      return { text: `Jev skills: mode is now ${arg}.` }
268    }
269    await $.ui.open({ id: PANE, title: 'Jev skills' })
270    return { text: 'Jev skills pane opened.' }
271  })
272
273  // Live scoring as the person types, once the draft has been still a moment.
274  on('prompt.edit', async ($, e, next) => {
275    const box = await next(e)
276    if (box.text === e.text || !(await read($, enabled))) return box
277    runtime.timer?.cancel()
278    const draft = box.text.trim()
279    // Short drafts and slash commands keep the last verdict on show.
280    if (draft.length < MIN_CHARS || draft.startsWith('/')) {
281      runtime.asked++
282      await update($, status, () => '')
283      return box
284    }
285    runtime.timer = $.clock.after(DEBOUNCE_MS, () => void score($, draft))
286    return box
287  })
288
289  // Steer: the winner rides along as a hidden context block on the prompt.
290  on('prompt.submit', async ($, e, next) => {
291    runtime.timer?.cancel()
292    const text = e.text.trim()
293    if (!(await read($, enabled)) || text.startsWith('/')) return next(e)
294    if ((await read($, mode)) !== 'steer' || text.length < MIN_CHARS) return next(e)
295    const known = await read($, verdict)
296    const result = known?.text === text ? known : await score($, text)
297    if (!result?.winner) return next(e)
298    const note =
299      `<skill_relevance>\nRelevant to the current request: ${result.winner}. ` +
300      'Load it with the Skill tool before you start. Ignore this if it does not fit what the user actually asked for.\n</skill_relevance>'
301    return next({ ...e, context: [...(e.context ?? []), note] })
302  })
303
304  // The listing the model reads: learn the roster from it, then hide it
305  // (block, steer) or drop the disabled skills from it (observe).
306  on('prompt.attachment', { type: 'skill_listing' }, async ($, e, next) => {
307    runtime.listed = parseListing(e.text)
308    await update($, roster, () => mergeRoster(runtime.disk, runtime.listed))
309    if (!(await read($, enabled))) return next(e)
310    const current = await read($, mode)
311    if (current !== 'observe') return { text: null }
312    const off = new Set(await read($, disabled))
313    if (off.size === 0) return next(e)
314    const kept = e.text
315      .split('\n')
316      .filter(line => {
317        const entry = line.match(/^- (\S+?)(?::|$)/)
318        return !entry?.[1] || !off.has(entry[1])
319      })
320      .join('\n')
321    return next({ ...e, text: kept })
322  })
323
324  // The model's own skill loads: blocked outright, held to the steer pick,
325  // and never a disabled skill.
326  on('tool.call', { tool: 'Skill' }, async ($, e, next) => {
327    const name = e.skill
328    if (!(await read($, enabled))) return next(e)
329    const current = await read($, mode)
330    if ((await read($, disabled)).includes(name)) {
331      return { deny: `jev-skill-suggestion: the skill '${name}' is turned off. Carry on without it.` }
332    }
333    if (current === 'block') {
334      return { deny: 'jev-skill-suggestion: skill loading is blocked in this session. Carry on without a skill.' }
335    }
336    if (current === 'steer') {
337      const pick = (await read($, verdict))?.winner
338      if (pick !== name) {
339        return { deny: `jev-skill-suggestion: only the skill Jev picked for this request may load${pick ? ` (${pick})` : ' (none)'}.` }
340      }
341    }
342    return next(e)
343  }).catch(($, e, next) => (next.called ? next(e) : { deny: 'jev-skill-suggestion: its guard failed.' }))
344
345  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
346    // Off, or nothing to show: the other bands draw as usual
347    if (!(await read($, enabled))) return next(e)
348    const result = await read($, verdict)
349    const note = await read($, status)
350    const current = await read($, mode)
351    if (e.props.hasSurvey || (result === null && note === '')) return next(e)
352    const { Box, Button, Text } = $.ui.resolve(e)
353    const width = Math.max(10, Math.min(24, e.props.bodyColumns - 50))
354    const rows = Math.max(1, Math.min(5, e.props.maxRows - 2))
355    const bar = (p: number) => '█'.repeat(Math.round(p * width)).padEnd(width, '·')
356    const nextMode = MODES[(MODES.indexOf(current) + 1) % MODES.length] ?? 'observe'
357    // Bands of other mods, drawn under this one
358    const below = await next(e)
359
360    return (
361      <Box flexDirection="column">
362        <Box>
363          <Box flexGrow={1}>
364            <Text wrap="truncate-end">
365              <Text bold color="magenta">
366                Jev
367              </Text>
368              <Text dimColor> {current}</Text>
369              {result && (
370                <Text>
371                  {' '}· gate <Text bold>{result.gate.toFixed(2)}</Text> ·{' '}
372                  {result.winner ? (
373                    <Text color="green">
374                      {current === 'block' ? 'would load' : current === 'steer' ? 'loads' : 'pick'} {result.winner}
375                    </Text>
376                  ) : (
377                    <Text dimColor>no skill</Text>
378                  )}
379                  <Text dimColor> · {result.ms}ms</Text>
380                </Text>
381              )}
382              {note !== '' && <Text color="yellow"> {note}</Text>}
383            </Text>
384          </Box>
385          <Button key="mode" hotkey="m" label={`mode: ${nextMode}`} onPress={() => setMode($, nextMode)} />
386          <Button key="pane" hotkey="s" label="skills" onPress={() => $.ui.open({ id: PANE, title: 'Jev skills' })} />
387        </Box>
388        {result?.ranked.slice(0, rows).map(entry => (
389          <Text key={entry.name} wrap="truncate-end" color={entry.name === result.winner ? 'green' : undefined}>
390            {entry.probability.toFixed(2)} {bar(entry.probability)} {entry.name}
391            {result.fits[entry.name] !== undefined ? (
392              <Text dimColor> fits {result.fits[entry.name]?.toFixed(2)}</Text>
393            ) : null}
394          </Text>
395        ))}
396        {below}
397      </Box>
398    )
399  })
400
401  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
402    const { Box, Button, Text } = $.ui.resolve(e)
403    const skills = await read($, roster)
404    const off = new Set(await read($, disabled))
405    const current = await read($, mode)
406    const result = await read($, verdict)
407    const isOn = await read($, enabled)
408    const probability = new Map(result?.ranked.map(entry => [entry.name, entry.probability]) ?? [])
409    const sorted = [...skills].sort(
410      (a, b) => (probability.get(b.name) ?? -1) - (probability.get(a.name) ?? -1) || a.name.localeCompare(b.name),
411    )
412    const columns = e.props.bodyColumns ?? 80
413
414    return (
415      <Box flexDirection="column">
416        <Box>
417          <Button key="power" label={isOn ? 'on' : 'off'} variant={isOn ? 'primary' : undefined} onPress={() => setEnabled($, !isOn)} />
418          <Text> Mode </Text>
419          {MODES.map(one => (
420            <Button
421              key={one}
422              label={one}
423              variant={one === current ? 'primary' : undefined}
424              onPress={() => setMode($, one)}
425            />
426          ))}
427        </Box>
428        <Text dimColor>
429          observe: Claude picks as usual · block: no skill loads · steer: only Jev's pick loads
430        </Text>
431        <Text dimColor>
432          {skills.length} skills, {off.size} off{result ? ` · last: "${result.text.slice(0, 40)}"` : ''}
433        </Text>
434        {sorted.map(skill => (
435          <Box key={skill.name}>
436            <Button
437              key={`toggle:${skill.name}`}
438              plain
439              label={off.has(skill.name) ? '[ ]' : '[x]'}
440              onPress={() => toggle($, skill.name)}
441            />
442            <Text dimColor={off.has(skill.name)} wrap="truncate-end">
443              {' '}
444              {probability.has(skill.name) ? probability.get(skill.name)!.toFixed(2) : '    '}{' '}
445              {skill.name}
446              <Text dimColor> {skill.description.slice(0, Math.max(0, columns - skill.name.length - 14))}</Text>
447            </Text>
448          </Box>
449        ))}
450      </Box>
451    )
452  })
453}
454

1export type Mode = 'observe' | 'block' | 'steer'
2
3export type Skill = { name: string; description: string; body: string }
4
5export type Ranked = { name: string; probability: number }
6
7export type Verdict = {
8  text: string
9  gate: number
10  ranked: Ranked[]
11  fits: Record<string, number>
12  winner: string | null
13  ms: number
14}
15
16declare module 'claude-code' {
17  interface PluginState {
18    'jev-skill-suggestion': {
19      mode: Mode
20      enabled: boolean
21      disabled: string[]
22      roster: Skill[]
23      verdict: Verdict | null
24      status: string
25    }
26  }
27}
28