jev-skill-suggestion

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface as Engine, Register } from 'claude-code'
3
4import type { Mode, Ranked, Skill, Verdict } from '../types'
5
6// The two-request approach of docs.typesafe.ai/cookbooks/skill_suggestion:
7// rank every skill and ask whether the request needs one at all, then re-read
8// the top three with their SKILL.md openings and check each one really fits.
9const API = 'https://api.typesafe.ai/v1/systemone'
10const MODEL = 'jev-latest'
11const SHORTLIST = 3
12const EXCERPT_CHARS = 700
13const GATE_THRESHOLD = 0.3
14const FITS_THRESHOLD = 0.3
15const DEBOUNCE_MS = 600
16const MIN_CHARS = 8
17const PANE = 'jev-skill-suggestion'
18const MODES: Mode[] = ['observe', 'block', 'steer']
19
20const GATE_QUESTIONS: Record<string, string> = {
21  acts_on_user_system:
22    "Is the assistant being asked to act on the user's files, accounts, devices, or online services, rather than only to explain or advise?",
23  would_follow_documented_procedure:
24    'Would a careful expert answering this consult a specific documented procedure or set of commands, rather than answering from general understanding?',
25  prose_suffices:
26    "Could a knowledgeable generalist fully satisfy this request in prose, with no tools, no documentation, and no access to the user's files or accounts?",
27}
28const INVERTED = new Set(['prose_suffices'])
29
30const mode = atom({ plugin: 'jev-skill-suggestion', key: 'mode' } as const, 'observe')
31const disabled = atom({ plugin: 'jev-skill-suggestion', key: 'disabled' } as const, [])
32const roster = atom({ plugin: 'jev-skill-suggestion', key: 'roster' } as const, [])
33const verdict = atom({ plugin: 'jev-skill-suggestion', key: 'verdict' } as const, null)
34const status = atom({ plugin: 'jev-skill-suggestion', key: 'status' } as const, '')
35const enabled = atom({ plugin: 'jev-skill-suggestion', key: 'enabled' } as const, true)
36
37const unquote = (value: string) => value.trim().replace(/^(['"])([\s\S]*)\1$/, '$2')
38
39// Enough YAML for a SKILL.md header: `name`, and a `description` that is
40// inline, quoted, folded (`>`) or literal (`|`), continued on indented lines.
41export function parseSkillMd(text: string, fallbackName: string): Skill | null {
42  const match = text.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/)
43  if (!match) return null
44  const lines = (match[1] ?? '').split(/\r?\n/)
45  const field = (key: string) => {
46    const at = lines.findIndex(line => line.startsWith(`${key}:`))
47    if (at === -1) return ''
48    const head = (lines[at] ?? '').slice(key.length + 1).trim()
49    const rest: string[] = []
50    for (const line of lines.slice(at + 1)) {
51      if (!/^\s+\S/.test(line)) break
52      rest.push(line.trim())
53    }
54    const parts = /^[|>][-+]?$/.test(head) ? rest : [head, ...rest]
55    return unquote(parts.join(' '))
56  }
57  const name = field('name') || fallbackName
58  return { name, description: field('description'), body: (match[2] ?? '').trim().slice(0, EXCERPT_CHARS) }
59}
60
61// The engine's skill listing: `- name: description` per skill, a description
62// sometimes running on over following lines. Plugin skills carry `plugin:`.
63export function parseListing(text: string): Skill[] {
64  const skills: Skill[] = []
65  for (const line of text.split('\n')) {
66    const entry = line.match(/^- (\S+?)(?:: (.*))?$/)
67    const last = skills[skills.length - 1]
68    if (entry?.[1]) {
69      skills.push({ name: entry[1].replace(/:$/, ''), description: entry[2] ?? '', body: '' })
70    } else if (last && line.trim() && !line.startsWith('The following')) {
71      last.description += ` ${line.trim()}`
72    }
73  }
74  return skills
75}
76
77// Disk skills bring their SKILL.md body; the listing brings the plugin and
78// built-in ones the disk scan cannot see. The listing decides who exists.
79export function mergeRoster(disk: Skill[], listed: Skill[]): Skill[] {
80  if (listed.length === 0) return disk
81  const byName = new Map(disk.map(skill => [skill.name, skill]))
82  return listed.map(skill => {
83    const local = byName.get(skill.name)
84    return local ? { ...skill, description: local.description || skill.description, body: local.body } : skill
85  })
86}
87
88export function gateOf(values: Record<string, number>): number {
89  const oriented = Object.entries(values).map(([key, value]) => (INVERTED.has(key) ? 1 - value : value))
90  return oriented.length === 0 ? 0 : oriented.reduce((a, b) => a + b, 0) / oriented.length
91}
92
93const runtime: {
94  apiKey: string
95  asked: number
96  disk: Skill[]
97  listed: Skill[]
98  timer?: { cancel: () => void }
99} = { apiKey: '', asked: 0, disk: [], listed: [] }
100
101async function ask($: Engine, state: unknown, questions: Record<string, unknown>) {
102  const response = await $.http.fetch(API, {
103    method: 'POST',
104    headers: { Authorization: `Bearer ${runtime.apiKey}`, 'Content-Type': 'application/json' },
105    body: JSON.stringify({ state, model: MODEL, questions }),
106  })
107  if (!response.ok) throw new Error(`Jev ${response.status}: ${response.text.slice(0, 160)}`)
108  return JSON.parse(response.text).answers as Record<
109    string,
110    { choice?: string; probabilities?: Record<string, number>; noul?: number }
111  >
112}
113
114const suggest = async ($: Engine, text: string): Promise<Verdict> => {
115  const startedAt = await $.clock.now()
116  const off = new Set(await read($, disabled))
117  const skills = (await read($, roster)).filter(skill => !off.has(skill.name))
118  const state = { request: text, recent_context: '' }
119  if (skills.length === 0) return { text, gate: 0, ranked: [], fits: {}, winner: null, ms: 0 }
120
121  const wideQuestions: Record<string, unknown> = {
122    which: {
123      type: 'choice',
124      instructions: "Which of these skills, if any, is the right one to load to help with the user's latest request?",
125      criteria: Object.fromEntries(skills.map(skill => [skill.name, skill.description.slice(0, 250) || skill.name])),
126    },
127  }
128  for (const [key, instructions] of Object.entries(GATE_QUESTIONS)) {
129    wideQuestions[`gate::${key}`] = { type: 'noul', instructions }
130  }
131  const wide = await ask($, state, wideQuestions)
132  const ranked: Ranked[] = Object.entries(wide.which?.probabilities ?? {})
133    .map(([name, probability]) => ({ name, probability }))
134    .sort((a, b) => b.probability - a.probability)
135  const gate = gateOf(
136    Object.fromEntries(
137      Object.entries(wide)
138        .filter(([key]) => key.startsWith('gate::'))
139        .map(([key, answer]) => [key.slice('gate::'.length), answer.noul ?? 0]),
140    ),
141  )
142  const done = async (fits: Record<string, number>, winner: string | null): Promise<Verdict> => ({
143    text,
144    gate,
145    ranked: ranked.slice(0, 12),
146    fits,
147    winner,
148    ms: (await $.clock.now()) - startedAt,
149  })
150  if (gate < GATE_THRESHOLD) return done({}, null)
151
152  const shortlist = ranked.slice(0, SHORTLIST).map(entry => skills.find(skill => skill.name === entry.name)!)
153  const rerankQuestions: Record<string, unknown> = {
154    which: {
155      type: 'choice',
156      instructions:
157        "Exactly one of these skills is the right one to load for the user's latest request. Which one? Read what each actually does, not just its name.",
158      criteria: Object.fromEntries(
159        shortlist.map(skill => [skill.name, `${skill.description}${skill.body ? ` — ${skill.body}` : ''}`]),
160      ),
161    },
162  }
163  for (const skill of shortlist) {
164    rerankQuestions[`fits::${skill.name}`] = {
165      type: 'noul',
166      instructions: `Does the skill '${skill.name}' do the specific thing the user's request asks for? It is described as: ${skill.description}`,
167    }
168  }
169  const narrow = await ask($, state, rerankQuestions)
170  const fits = Object.fromEntries(
171    shortlist.map(skill => [skill.name, narrow[`fits::${skill.name}`]?.noul ?? 0]),
172  )
173  const best = Math.max(...Object.values(fits))
174  return done(fits, best < FITS_THRESHOLD ? null : (narrow.which?.choice ?? null))
175}
176
177// Scores the draft; a newer draft's answer wins, an older one is dropped.
178const score = async ($: Engine, text: string) => {
179  const ticket = ++runtime.asked
180  if (!runtime.apiKey) {
181    await update($, status, () => 'no API key: set TYPESAFE_API_KEY or the plugin option')
182    return null
183  }
184  await update($, status, () => 'scoring…')
185  try {
186    const result = await suggest($, text)
187    if (ticket === runtime.asked) {
188      await update($, verdict, () => result)
189      await update($, status, () => '')
190    }
191    return result
192  } catch (error) {
193    if (ticket === runtime.asked) await update($, status, () => String((error as Error).message ?? error))
194    return null
195  }
196}
197
198// A native Windows terminal sets USERPROFILE, not HOME
199export const homeDir = (home: string | undefined, profile: string | undefined) => home ?? profile ?? ''
200
201const scanDisk = async ($: Engine, cwd: string) => {
202  const home = homeDir(await $.env.get('HOME'), await $.env.get('USERPROFILE'))
203  const found: Skill[] = []
204  for (const root of [`${home}/.claude/skills`, `${cwd}/.claude/skills`]) {
205    const entries = await $.fs.list(root).catch(() => [])
206    for (const entry of entries) {
207      if (entry.kind !== 'dir' && !entry.isLink) continue
208      const text = await $.fs.read(`${root}/${entry.name}/SKILL.md`).catch(() => null)
209      const skill = typeof text === 'string' ? parseSkillMd(text, entry.name) : null
210      if (skill && !found.some(other => other.name === skill.name)) found.push(skill)
211    }
212  }
213  return found
214}
215
216const setMode = async ($: Engine, next: Mode) => {
217  await update($, mode, () => next)
218  await $.store.set('mode', next)
219  $.ui.invalidate('prompt.attachment')
220}
221
222const setEnabled = async ($: Engine, on: boolean) => {
223  runtime.timer?.cancel()
224  runtime.asked++
225  await update($, enabled, () => on)
226  await update($, status, () => '')
227  if (!on) await update($, verdict, () => null)
228  await $.store.set('enabled', on)
229  $.ui.invalidate('prompt.attachment')
230}
231
232const toggle = async ($: Engine, name: string) => {
233  const list = await update($, disabled, list =>
234    list.includes(name) ? list.filter(one => one !== name) : [...list, name],
235  )
236  await $.store.set('disabled', list)
237  $.ui.invalidate('prompt.attachment')
238}
239
240
241export const register: Register = (on, options) => {
242  runtime.apiKey = typeof options.apiKey === 'string' ? options.apiKey : ''
243
244  on('session.start', async ($, e, next) => {
245    runtime.apiKey ||= (await $.env.get('TYPESAFE_API_KEY')) ?? ''
246    const storedMode = await $.store.get('mode')
247    if (typeof storedMode === 'string' && MODES.includes(storedMode as Mode)) {
248      await update($, mode, () => storedMode as Mode)
249    }
250    if ((await $.store.get('enabled')) === false) await update($, enabled, () => false)
251    const storedDisabled = await $.store.get('disabled')
252    if (Array.isArray(storedDisabled)) await update($, disabled, () => storedDisabled.map(String))
253    runtime.disk = await scanDisk($, e.cwd)
254    await update($, roster, () => mergeRoster(runtime.disk, runtime.listed))
255    await $.command.register({
256      name: 'jev',
257      description: 'Jev skill scores: open the skills pane, `/jev observe|block|steer`, or `/jev off|on`',
258    })
259    return next(e)
260  })
261
262  on('command.run', { command: 'jev' }, async ($, e) => {
263    const arg = e.args.trim()
264    if (arg === 'off' || arg === 'on') {
265      await setEnabled($, arg === 'on')
266      return { text: arg === 'on' ? 'Jev skills: on.' : 'Jev skills: off. Nothing is scored and Claude picks skills as usual. `/jev on` turns it back on.' }
267    }
268    if (MODES.includes(arg as Mode)) {
269      await setMode($, arg as Mode)
270      return { text: `Jev skills: mode is now ${arg}.` }
271    }
272    await $.ui.open({ id: PANE, title: 'Jev skills' })
273    return { text: 'Jev skills pane opened.' }
274  })
275
276  // Live scoring as the person types, once the draft has been still a moment.
277  on('prompt.edit', async ($, e, next) => {
278    const box = await next(e)
279    if (box.text === e.text || !(await read($, enabled))) return box
280    runtime.timer?.cancel()
281    const draft = box.text.trim()
282    // Short drafts and slash commands keep the last verdict on show.
283    if (draft.length < MIN_CHARS || draft.startsWith('/')) {
284      runtime.asked++
285      await update($, status, () => '')
286      return box
287    }
288    runtime.timer = $.clock.after(DEBOUNCE_MS, () => void score($, draft))
289    return box
290  })
291
292  // Steer: the winner rides along as a hidden context block on the prompt.
293  on('prompt.submit', async ($, e, next) => {
294    runtime.timer?.cancel()
295    const text = e.text.trim()
296    if (!(await read($, enabled)) || text.startsWith('/')) return next(e)
297    if ((await read($, mode)) !== 'steer' || text.length < MIN_CHARS) return next(e)
298    const known = await read($, verdict)
299    const result = known?.text === text ? known : await score($, text)
300    if (!result?.winner) return next(e)
301    const note =
302      `<skill_relevance>\nRelevant to the current request: ${result.winner}. ` +
303      'Load it with the Skill tool before you start. Ignore this if it does not fit what the user actually asked for.\n</skill_relevance>'
304    return next({ ...e, context: [...(e.context ?? []), note] })
305  })
306
307  // The listing the model reads: learn the roster from it, then hide it
308  // (block, steer) or drop the disabled skills from it (observe).
309  on('prompt.attachment', { type: 'skill_listing' }, async ($, e, next) => {
310    runtime.listed = parseListing(e.text)
311    await update($, roster, () => mergeRoster(runtime.disk, runtime.listed))
312    if (!(await read($, enabled))) return next(e)
313    const current = await read($, mode)
314    if (current !== 'observe') return { text: null }
315    const off = new Set(await read($, disabled))
316    if (off.size === 0) return next(e)
317    const kept = e.text
318      .split('\n')
319      .filter(line => {
320        const entry = line.match(/^- (\S+?)(?::|$)/)
321        return !entry?.[1] || !off.has(entry[1])
322      })
323      .join('\n')
324    return next({ ...e, text: kept })
325  })
326
327  // The model's own skill loads: blocked outright, held to the steer pick,
328  // and never a disabled skill.
329  on('tool.call', { tool: 'Skill' }, async ($, e, next) => {
330    const name = e.skill
331    if (!(await read($, enabled))) return next(e)
332    const current = await read($, mode)
333    if ((await read($, disabled)).includes(name)) {
334      return { deny: `jev-skill-suggestion: the skill '${name}' is turned off. Carry on without it.` }
335    }
336    if (current === 'block') {
337      return { deny: 'jev-skill-suggestion: skill loading is blocked in this session. Carry on without a skill.' }
338    }
339    if (current === 'steer') {
340      const pick = (await read($, verdict))?.winner
341      if (pick !== name) {
342        return { deny: `jev-skill-suggestion: only the skill Jev picked for this request may load${pick ? ` (${pick})` : ' (none)'}.` }
343      }
344    }
345    return next(e)
346  }).catch(($, e, next) => (next.called ? next(e) : { deny: 'jev-skill-suggestion: its guard failed.' }))
347
348  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
349    // Off, or nothing to show: the other bands draw as usual
350    if (!(await read($, enabled))) return next(e)
351    const result = await read($, verdict)
352    const note = await read($, status)
353    const current = await read($, mode)
354    if (e.props.hasSurvey || (result === null && note === '')) return next(e)
355    const { Box, Button, Text } = $.ui.resolve(e)
356    const width = Math.max(10, Math.min(24, e.props.bodyColumns - 50))
357    const rows = Math.max(1, Math.min(5, e.props.maxRows - 2))
358    const bar = (p: number) => '█'.repeat(Math.round(p * width)).padEnd(width, '·')
359    const nextMode = MODES[(MODES.indexOf(current) + 1) % MODES.length] ?? 'observe'
360    // Bands of other mods, drawn under this one
361    const below = await next(e)
362
363    return (
364      <Box flexDirection="column">
365        <Box>
366          <Box flexGrow={1}>
367            <Text wrap="truncate-end">
368              <Text bold color="magenta">
369                Jev
370              </Text>
371              <Text dimColor> {current}</Text>
372              {result && (
373                <Text>
374                  {' '}· gate <Text bold>{result.gate.toFixed(2)}</Text> ·{' '}
375                  {result.winner ? (
376                    <Text color="green">
377                      {current === 'block' ? 'would load' : current === 'steer' ? 'loads' : 'pick'} {result.winner}
378                    </Text>
379                  ) : (
380                    <Text dimColor>no skill</Text>
381                  )}
382                  <Text dimColor> · {result.ms}ms</Text>
383                </Text>
384              )}
385              {note !== '' && <Text color="yellow"> {note}</Text>}
386            </Text>
387          </Box>
388          <Button key="mode" hotkey="m" label={`mode: ${nextMode}`} onPress={() => setMode($, nextMode)} />
389          <Button key="pane" hotkey="s" label="skills" onPress={() => $.ui.open({ id: PANE, title: 'Jev skills' })} />
390        </Box>
391        {result?.ranked.slice(0, rows).map(entry => (
392          <Text key={entry.name} wrap="truncate-end" color={entry.name === result.winner ? 'green' : undefined}>
393            {entry.probability.toFixed(2)} {bar(entry.probability)} {entry.name}
394            {result.fits[entry.name] !== undefined ? (
395              <Text dimColor> fits {result.fits[entry.name]?.toFixed(2)}</Text>
396            ) : null}
397          </Text>
398        ))}
399        {below}
400      </Box>
401    )
402  })
403
404  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
405    const { Box, Button, Text } = $.ui.resolve(e)
406    const skills = await read($, roster)
407    const off = new Set(await read($, disabled))
408    const current = await read($, mode)
409    const result = await read($, verdict)
410    const isOn = await read($, enabled)
411    const probability = new Map(result?.ranked.map(entry => [entry.name, entry.probability]) ?? [])
412    const sorted = [...skills].sort(
413      (a, b) => (probability.get(b.name) ?? -1) - (probability.get(a.name) ?? -1) || a.name.localeCompare(b.name),
414    )
415    const columns = e.props.bodyColumns ?? 80
416
417    return (
418      <Box flexDirection="column">
419        <Box>
420          <Button key="power" label={isOn ? 'on' : 'off'} variant={isOn ? 'primary' : undefined} onPress={() => setEnabled($, !isOn)} />
421          <Text> Mode </Text>
422          {MODES.map(one => (
423            <Button
424              key={one}
425              label={one}
426              variant={one === current ? 'primary' : undefined}
427              onPress={() => setMode($, one)}
428            />
429          ))}
430        </Box>
431        <Text dimColor>
432          observe: Claude picks as usual · block: no skill loads · steer: only Jev's pick loads
433        </Text>
434        <Text dimColor>
435          {skills.length} skills, {off.size} off{result ? ` · last: "${result.text.slice(0, 40)}"` : ''}
436        </Text>
437        {sorted.map(skill => (
438          <Box key={skill.name}>
439            <Button
440              key={`toggle:${skill.name}`}
441              plain
442              label={off.has(skill.name) ? '[ ]' : '[x]'}
443              onPress={() => toggle($, skill.name)}
444            />
445            <Text dimColor={off.has(skill.name)} wrap="truncate-end">
446              {' '}
447              {probability.has(skill.name) ? probability.get(skill.name)!.toFixed(2) : '    '}{' '}
448              {skill.name}
449              <Text dimColor> {skill.description.slice(0, Math.max(0, columns - skill.name.length - 14))}</Text>
450            </Text>
451          </Box>
452        ))}
453      </Box>
454    )
455  })
456}
457

1export type Mode = 'observe' | 'block' | 'steer'
2
3export type Skill = { name: string; description: string; body: string }
4
5export type Ranked = { name: string; probability: number }
6
7export type Verdict = {
8  text: string
9  gate: number
10  ranked: Ranked[]
11  fits: Record<string, number>
12  winner: string | null
13  ms: number
14}
15
16declare module 'claude-code' {
17  interface PluginState {
18    'jev-skill-suggestion': {
19      mode: Mode
20      enabled: boolean
21      disabled: string[]
22      roster: Skill[]
23      verdict: Verdict | null
24      status: string
25    }
26  }
27}
28