jev-skill-scout

1import type { Register } from 'claude-code'
2
3import { readRoster } from '../lib/roster.js'
4import { contextLine, suggest } from '../lib/scout.js'
5
6type Skill = { name: string; description: string; excerpt: string; path: string; source: string }
7type Init = { method?: string; headers?: Record<string, string>; body?: string }
8type Suggestion = {
9  suggestion: string | null
10  verify: { fits: Record<string, number> } | null
11  ms: number
12}
13type Options = {
14  apiKey?: string
15  enabled?: boolean
16  gateThreshold?: number
17  fitsThreshold?: number
18  timeoutMs?: number
19  model?: string
20  quiet?: boolean
21  shadow?: boolean
22}
23
24const ROSTER_KEY = 'roster.v1'
25const ROSTER_TTL_MS = 10 * 60 * 1000
26
27export const register: Register = (on, options) => {
28  const opt = (options ?? {}) as Options
29  const enabled = opt.enabled !== false
30
31  on('session.start', async ($, e, next) => {
32    const r = await next(e)
33    if (!enabled) return r
34    const home = await $.env.get('HOME')
35    if (!home) return r
36    const roster = await readRoster(
37      { list: p => $.fs.list(p), read: p => $.fs.read(p), exists: p => $.fs.exists(p) },
38      { home, cwd: e.cwd },
39    )
40    await $.store.set(ROSTER_KEY, { at: Date.now(), cwd: e.cwd, skills: roster })
41    if (!opt.quiet) $.ui.log(`jev-skill-scout: ${roster.length} skills indexed`)
42    return r
43  })
44
45  on('prompt.submit', async ($, e, next) => {
46    const typed = e.origin.kind === 'composer' || e.origin.kind === 'bridge'
47    if (!enabled || !typed || e.text.length < 12 || e.text.startsWith('/')) return next(e)
48    const key = opt.apiKey || (await $.env.get('TYPESAFE_API_KEY')) || (await $.env.get('TYPESAFE_KEY'))
49    if (!key) return next(e)
50    const cached = (await $.store.get(ROSTER_KEY)) as { at: number; cwd: string; skills: Skill[] } | undefined
51    let roster = cached?.skills ?? []
52    if (!cached || Date.now() - cached.at > ROSTER_TTL_MS) {
53      const home = await $.env.get('HOME')
54      const cwd = await $.session.cwd()
55      if (home) {
56        roster = await readRoster(
57          { list: p => $.fs.list(p), read: p => $.fs.read(p), exists: p => $.fs.exists(p) },
58          { home, cwd },
59        )
60        await $.store.set(ROSTER_KEY, { at: Date.now(), cwd, skills: roster })
61      }
62    }
63    if (!roster.length) return next(e)
64
65    const started = await $.clock.now()
66    let result: Suggestion | null = null
67    try {
68      result = (await Promise.race([
69        suggest({
70          // The engine's init has no abort signal; the race below is the timeout.
71          fetchImpl: (url: string, init: Init) => $.http.fetch(url, { method: init.method, headers: init.headers, body: init.body }),
72          key,
73          roster,
74          request: e.text,
75          options: {
76            model: opt.model,
77            gateThreshold: opt.gateThreshold,
78            fitsThreshold: opt.fitsThreshold,
79            timeoutMs: opt.timeoutMs ?? 4000,
80          },
81        }),
82        $.clock.sleep(opt.timeoutMs ?? 4000).then(() => null),
83      ])) as Suggestion | null
84    } catch (err) {
85      if (!opt.quiet) $.ui.log(`jev-skill-scout: off for this turn (${String(err).slice(0, 120)})`)
86      return next(e)
87    }
88    const ms = (await $.clock.now()) - started
89    if (!result) {
90      if (!opt.quiet) $.ui.status(`jev-skill-scout: no answer in ${ms} ms, turn left alone`)
91      return next(e)
92    }
93    if (!result.suggestion) {
94      if (!opt.quiet) $.ui.status(`jev-skill-scout: no skill (${ms} ms)`)
95      return next(e)
96    }
97    const fit = result.verify?.fits?.[result.suggestion] ?? 0
98    const shadow = opt.shadow || (await $.env.get('JEV_SKILL_SCOUT_SHADOW')) === '1'
99    if (shadow) {
100      $.ui.status(`jev-skill-scout (shadow): would suggest ${result.suggestion} (fit ${fit.toFixed(2)}, ${ms} ms)`)
101      return next(e)
102    }
103    if (!opt.quiet) $.ui.status(`jev-skill-scout: ${result.suggestion} (fit ${fit.toFixed(2)}, ${ms} ms)`)
104    return next({ ...e, context: [...(e.context ?? []), contextLine(result.suggestion)] })
105  }).catch(($, e, next) => {
106    // Core has a side effect on prompt.submit: enter the prompt exactly once.
107    if (next.called) return undefined
108    return next(e)
109  })
110}
111

1// Finds every SKILL.md Claude Code can load and reads its frontmatter.
2// Pure over an injected fs so the mod (engine $.fs) and the CLI (node:fs) share it.
3
4const PLUGIN_ROOT = ['.claude', 'plugins', 'cache'];
5
6export function parseFrontmatter(text) {
7  const lines = text.split('\n');
8  let i = 0;
9  while (i < lines.length && lines[i].trim() === '') i++;
10  if ((lines[i] ?? '').trim() !== '---') return { fields: {}, body: text };
11  i++;
12  const fields = {};
13  let key = '';
14  let buf = [];
15  const flush = () => {
16    if (key) fields[key] = buf.join(' ').replace(/\s+/g, ' ').trim();
17    key = '';
18    buf = [];
19  };
20  for (; i < lines.length; i++) {
21    const line = lines[i];
22    if (line.trim() === '---') { i++; break; }
23    const m = /^([A-Za-z_][\w-]*):[ \t]*(.*)$/.exec(line);
24    if (m && !/^[ \t]/.test(line)) {
25      flush();
26      key = m[1];
27      const value = m[2].trim();
28      if (value && !/^[>|][-+]?$/.test(value)) buf.push(value.replace(/^["']|["']$/g, ''));
29    } else if (key && line.trim()) {
30      buf.push(line.trim());
31    }
32  }
33  flush();
34  return { fields, body: lines.slice(i).join('\n') };
35}
36
37// Version directories sort as 1.10.0 > 1.9.0, not as strings.
38function versionKey(v) {
39  return v.split(/[.-]/).map(p => (/^\d+$/.test(p) ? Number(p) : -1));
40}
41function newestVersion(names) {
42  return [...names].sort((a, b) => {
43    const ka = versionKey(a), kb = versionKey(b);
44    for (let i = 0; i < Math.max(ka.length, kb.length); i++) {
45      const d = (kb[i] ?? -1) - (ka[i] ?? -1);
46      if (d) return d;
47    }
48    return 0;
49  })[0];
50}
51
52/**
53 * @param {{ list(p:string):Promise<{name:string,kind:string}[]>, read(p:string):Promise<string>, exists(p:string):Promise<boolean> }} fs
54 * @param {{ home:string, cwd?:string, bodyChars?:number }} where
55 * @returns {Promise<{ name:string, description:string, excerpt:string, path:string, source:string }[]>}
56 */
57export async function readRoster(fs, { home, cwd, bodyChars = 700 }) {
58  const out = new Map();
59  const add = async (skillDir, name, source) => {
60    const p = `${skillDir}/SKILL.md`;
61    if (!(await fs.exists(p))) return;
62    let text;
63    try { text = await fs.read(p); } catch { return; }
64    const { fields, body } = parseFrontmatter(text.slice(0, 12000));
65    const description = fields.description ?? '';
66    if (!description) return;
67    const key = name;
68    if (out.has(key)) return;
69    out.set(key, {
70      name: key,
71      description,
72      excerpt: body.replace(/\s+/g, ' ').trim().slice(0, bodyChars),
73      path: p,
74      source,
75    });
76  };
77  const dirs = async p => {
78    try { return (await fs.list(p)).filter(e => !e.name.startsWith('.')); } catch { return []; }
79  };
80
81  if (cwd) for (const e of await dirs(`${cwd}/.claude/skills`)) await add(`${cwd}/.claude/skills/${e.name}`, e.name, 'project');
82  for (const e of await dirs(`${home}/.claude/skills`)) await add(`${home}/.claude/skills/${e.name}`, e.name, 'user');
83
84  // Only enabled plugins reach the model's skill list; a cached but disabled
85  // one must not be suggested. Without readable settings, every plugin counts.
86  let enabled = null;
87  try {
88    const settings = JSON.parse(await fs.read(`${home}/.claude/settings.json`));
89    if (settings && typeof settings.enabledPlugins === 'object') enabled = settings.enabledPlugins;
90  } catch { enabled = null; }
91
92  const cache = [home, ...PLUGIN_ROOT].join('/');
93  for (const market of await dirs(cache)) {
94    for (const plugin of await dirs(`${cache}/${market.name}`)) {
95      if (enabled && enabled[`${plugin.name}@${market.name}`] !== true) continue;
96      const versions = (await dirs(`${cache}/${market.name}/${plugin.name}`)).map(v => v.name);
97      if (!versions.length) continue;
98      const v = newestVersion(versions);
99      const skillsDir = `${cache}/${market.name}/${plugin.name}/${v}/skills`;
100      for (const s of await dirs(skillsDir)) await add(`${skillsDir}/${s.name}`, `${plugin.name}:${s.name}`, `plugin ${market.name}`);
101    }
102  }
103  return [...out.values()];
104}
105
106/**
107 * The roster as Claude Code itself listed it in a transcript (`skill_listing`
108 * attachments): one `- name: description` per skill, long descriptions wrapped.
109 */
110export function parseListing(content) {
111  const out = [];
112  for (const raw of String(content ?? '').split('\n')) {
113    const m = /^- ([^\s:]+(?::[^\s:]+)?): ?(.*)$/.exec(raw);
114    if (m) out.push({ name: m[1], description: m[2].trim() });
115    else if (out.length && raw.trim()) out[out.length - 1].description += ` ${raw.trim()}`;
116  }
117  return out;
118}
119
120// Skill tool calls name plugin skills as plugin:skill, sometimes as just skill.
121export function sameSkill(a, b) {
122  if (!a || !b) return false;
123  if (a === b) return true;
124  return a.split(':').pop() === b.split(':').pop();
125}
126

1// The judgment. Two TypeSafe requests at most, following the skill-suggestion
2// cookbook (https://docs.typesafe.ai/cookbooks/skill_suggestion): rank the whole
3// roster and gate the turn, then re-read the top few properly and allow a reject.
4// Pure: the caller supplies fetch, so the mod and the audit make the same calls.
5
6export const NONE = '__none__';
7export const ENDPOINT = 'https://api.typesafe.ai/v1/systemone';
8// A Choice takes at most 255 options; rosters past this are ranked in chunks.
9export const CHUNK = 250;
10
11export const DEFAULTS = {
12  model: 'jev-latest',
13  shortlist: 3,
14  gateThreshold: 0.3,   // mean of the oriented gate nouls; below it, no skill
15  fitsThreshold: 0.3,   // the winner's "really fits" noul; below it, no skill
16  descriptionChars: 400,
17  excerptChars: 700,
18  contextChars: 400,
19  timeoutMs: 4000,
20  minPromptChars: 12,
21};
22
23const GATES = {
24  acts: 'Does `request` ask the agent to do something with the user\'s files, repositories, accounts, sites, documents or data, rather than only answer from general knowledge?',
25  procedure: 'Would a careful agent answer `request` better by following a specific written procedure, checklist or house style, rather than by general skill alone?',
26  prose: 'Can `request` be fully satisfied by a short reply in prose, with no tools, no files and no procedure?',
27};
28const INVERTED = new Set(['prose']);
29
30const cut = (s, n) => {
31  const flat = String(s ?? '').replace(/\s+/g, ' ').trim();
32  return flat.length <= n ? flat : `${flat.slice(0, n - 3).trimEnd()}...`;
33};
34
35export function buildRank(roster, request, recent, o = DEFAULTS) {
36  const criteria = {};
37  for (const s of roster) criteria[s.name] = cut(s.description, o.descriptionChars);
38  criteria[NONE] = 'None of the skills above is what this request needs. Also the answer for small talk, a question answerable from what is already on screen, or a small direct edit that needs no procedure.';
39  const questions = {
40    which: {
41      type: 'choice',
42      instructions: 'Which single skill should the agent read before answering `request`? Judge each skill only by what its description says it is for, and prefer the skill whose trigger conditions the request matches most specifically.',
43      criteria,
44    },
45  };
46  for (const [k, text] of Object.entries(GATES)) questions[`gate_${k}`] = { type: 'noul', instructions: text };
47  return {
48    model: o.model,
49    state: { request: cut(request, 4000), recent_context: cut(recent, o.contextChars) },
50    questions,
51  };
52}
53
54export function readRank(answers) {
55  const which = answers.which;
56  const ranked = Object.entries(which.probabilities ?? {}).sort((a, b) => b[1] - a[1]);
57  const gates = {};
58  let sum = 0, n = 0;
59  for (const [k] of Object.entries(GATES)) {
60    const v = answers[`gate_${k}`]?.noul;
61    if (typeof v !== 'number') continue;
62    gates[k] = v;
63    sum += INVERTED.has(k) ? 1 - v : v;
64    n++;
65  }
66  return { ranked, gate: n ? sum / n : 0, gates, confidence: which.confidence ?? null };
67}
68
69export function buildVerify(candidates, request, recent, o = DEFAULTS) {
70  const criteria = {};
71  for (const c of candidates) criteria[c.name] = `${cut(c.description, o.descriptionChars)} Instructions begin: ${cut(c.excerpt, o.excerptChars)}`;
72  criteria[NONE] = 'None of these skills should be loaded for this request.';
73  const questions = {
74    which: {
75      type: 'choice',
76      instructions: 'Now that each candidate skill\'s real instructions are visible, which one should the agent load before answering `request`?',
77      criteria,
78    },
79  };
80  candidates.forEach((c, i) => {
81    questions[`fits_${i}`] = {
82      type: 'noul',
83      instructions: `Would loading skill \`candidates[${i}].name\` change how the agent handles \`request\` for the better, judged by the skill's own instructions, not its name?`,
84      criteria: { true: 'The skill\'s instructions cover this request and following them would change the work.', false: 'Wrong domain, or the request would be handled the same way without it.' },
85    };
86  });
87  return {
88    model: o.model,
89    state: {
90      request: cut(request, 4000),
91      recent_context: cut(recent, o.contextChars),
92      candidates: candidates.map(c => ({ name: c.name, description: cut(c.description, o.descriptionChars), instructions: cut(c.excerpt, o.excerptChars) })),
93    },
94    questions,
95  };
96}
97
98export function readVerify(answers, candidates) {
99  const which = answers.which;
100  const fits = {};
101  candidates.forEach((c, i) => { fits[c.name] = answers[`fits_${i}`]?.noul ?? 0; });
102  return { winner: which.choice, probabilities: which.probabilities ?? {}, confidence: which.confidence ?? null, fits };
103}
104
105/**
106 * Ask Jev. `fetchImpl(url, init)` resolves `{ ok, status, text }` (the engine's
107 * $.http.fetch and a wrapper over global fetch both fit).
108 */
109export async function ask(fetchImpl, key, body, timeoutMs = DEFAULTS.timeoutMs, retries = 0) {
110  for (let attempt = 0; ; attempt++) {
111    try {
112      return await askOnce(fetchImpl, key, body, timeoutMs);
113    } catch (e) {
114      const msg = String(e?.message ?? e);
115      const retryable = /aborted|429|50\d|fetch failed|ECONN|ETIMEDOUT/i.test(msg);
116      if (!retryable || attempt >= retries) throw e;
117      await new Promise(r => setTimeout(r, 500 * (attempt + 1)));
118    }
119  }
120}
121
122async function askOnce(fetchImpl, key, body, timeoutMs) {
123  const controller = typeof AbortController === 'function' ? new AbortController() : null;
124  let timer = null;
125  // Two guards: the abort for transports that honour it, the race for those that do not.
126  const deadline = new Promise((_, reject) => { timer = setTimeout(() => { controller?.abort(); reject(new Error('aborted: timeout')); }, timeoutMs); });
127  const work = (async () => {
128    const r = await fetchImpl(ENDPOINT, {
129      method: 'POST',
130      headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
131      body: JSON.stringify(body),
132      signal: controller?.signal,
133    });
134    const text = typeof r.text === 'function' ? await r.text() : r.text;
135    if (!r.ok) throw new Error(`typesafe ${r.status}: ${String(text).slice(0, 200)}`);
136    return JSON.parse(text);
137  })();
138  try {
139    return await Promise.race([work, deadline]);
140  } finally {
141    if (timer) clearTimeout(timer);
142  }
143}
144
145/** Re-applies the thresholds to a stored judgment, so a rerun with new ones costs nothing. */
146export function decide(res, options = {}) {
147  const o = { ...DEFAULTS, ...options };
148  if (!res || !res.rank) return null;
149  if (res.rank.gate < o.gateThreshold) return null;
150  const v = res.verify;
151  if (!v || !v.winner || v.winner === NONE) return null;
152  return (v.fits[v.winner] ?? 0) >= o.fitsThreshold ? v.winner : null;
153}
154
155/**
156 * The whole judgment for one request. Returns what the mod would attach and
157 * everything the audit needs to explain it.
158 */
159export async function suggest({ fetchImpl, key, roster, request, recent = '', options = {} }) {
160  const o = { ...DEFAULTS, ...options };
161  const t0 = Date.now();
162  const chunks = [];
163  for (let i = 0; i < roster.length; i += CHUNK) chunks.push(roster.slice(i, i + CHUNK));
164  const replies = await Promise.all(chunks.map(c => ask(fetchImpl, key, buildRank(c, request, recent, o), o.timeoutMs, o.retries ?? 0)));
165  const usage = { input: 0, output: 0, calls: replies.length };
166  for (const r of replies) { usage.input += r.usage?.input_tokens ?? 0; usage.output += r.usage?.output_tokens ?? 0; }
167  const rank = mergeRanks(replies.map(r => readRank(r.answers)), o.shortlist);
168  const out = { suggestion: null, rank, verify: null, usage, ms: 0, stage: 'gate' };
169  if (rank.gate < o.gateThreshold) { out.ms = Date.now() - t0; return out; }
170  const short = rank.ranked.filter(([n]) => n !== NONE).slice(0, o.shortlist).map(([n]) => roster.find(s => s.name === n)).filter(Boolean);
171  if (!short.length) { out.ms = Date.now() - t0; return out; }
172  const r2 = await ask(fetchImpl, key, buildVerify(short, request, recent, o), o.timeoutMs, o.retries ?? 0);
173  const verify = readVerify(r2.answers, short);
174  usage.input += r2.usage?.input_tokens ?? 0;
175  usage.output += r2.usage?.output_tokens ?? 0;
176  usage.calls += 1;
177  out.verify = verify;
178  out.stage = 'verify';
179  out.ms = Date.now() - t0;
180  out.suggestion = decide(out, o);
181  return out;
182}
183
184// Probabilities from different chunks are not comparable, so each chunk keeps
185// only its own top few and the verify stage settles it with real excerpts.
186function mergeRanks(ranks, shortlist) {
187  if (ranks.length === 1) return ranks[0];
188  const ranked = [];
189  let gate = 0;
190  for (const r of ranks) {
191    ranked.push(...r.ranked.filter(([n]) => n !== NONE).slice(0, shortlist));
192    gate += r.gate;
193  }
194  ranked.sort((a, b) => b[1] - a[1]);
195  ranked.push([NONE, Math.max(...ranks.map(r => r.ranked.find(([n]) => n === NONE)?.[1] ?? 0))]);
196  return { ranked, gate: gate / ranks.length, gates: ranks[0].gates, confidence: null };
197}
198
199export function contextLine(name) {
200  return `<skill_relevance>\nRelevant to this request: ${name}. Load it with the Skill tool before answering. Ignore this if it does not fit what the user actually asked for.\n</skill_relevance>`;
201}
202