skill-stats

1// skill-stats: how often each skill runs, which never trigger, and a one key
2// hand off to skill-creator for a weak description.
3//
4// Two sources of use, kept apart:
5// - history: Skill tool calls and /name commands found in the transcripts of the
6//   config dirs the config_dirs setting names (default: this account's), under
7//   <dir>/projects/**/*.jsonl. Covers the past, but Claude Code deletes
8//   transcripts after cleanupPeriodDays.
9// "Your" skills are the ones from a skills folder, plus the plugin skills from the
10// marketplaces the own_marketplaces setting names.
11// - live: every skill.prompt event from now on, kept in $.store, which is per
12//   account and survives transcript cleanup.
13
14// Settings, read once in register
15let configDirs = []
16let ownMarketplaces = []
17let sourceDirs = []
18function list(v) {
19  return String(v ?? '').split(',').map((x) => x.trim()).filter(Boolean)
20}
21// GREP is the same pattern for grep -E, which has no \/ escape
22const GREP = '"name":"Skill","input":\\{"skill":"[^"]+"|<command-name>/[^<]+</command-name>'
23const USE = /"name":"Skill","input":\{"skill":"([^"]+)"|<command-name>\/([^<\s]+)<\/command-name>/
24const NOT_MINE = ['plugin', 'built-in', 'mcp']
25// A listing entry this small carries the name and no description. Measured on
26// 2.1.289: entries with a description are 47 tokens and up, bare ones 4 to 13.
27const NAME_ONLY_TOKENS = 20
28
29export function register(on, options) {
30  configDirs = list(options?.config_dirs)
31  ownMarketplaces = list(options?.own_marketplaces)
32  sourceDirs = list(options?.source_dirs)
33  on('session.start', async ($, e, next) => {
34    await $.command.register({
35      name: 'skill-stats',
36      description: 'Skill use counts and the skills that never trigger (add "all" for plugin skills)',
37    })
38    await $.command.register({
39      name: 'skill-fix',
40      description: 'Put a skill-creator request for one skill\'s description in the prompt box',
41      argumentHint: '<skill>',
42    })
43    return next(e)
44  })
45
46  // Live count: fires for /name, the Skill tool, and skills preloaded into a subagent
47  on('skill.prompt', async ($, e, next) => {
48    const live = { ...((await $.store.get('live')) ?? {}) }
49    const prev = live[e.skill] ?? { n: 0, last: 0 }
50    live[e.skill] = { n: prev.n + 1, last: await $.clock.now() }
51    await $.store.set('live', live)
52    return next(e)
53  })
54
55  on('command.run', { command: 'skill-stats' }, async ($, e) => {
56    const all = e.args.trim() === 'all'
57    const rows = await collect($, all)
58    return { text: render(rows.list, rows.listing, all) }
59  })
60
61  on('command.run', { command: 'skill-fix' }, async ($, e) => {
62    const name = e.args.trim()
63    if (!name) return { text: 'Usage: /skill-fix <skill>. /skill-stats lists the candidates.' }
64    const rows = await collect($, true)
65    const row = rows.list.find((r) => r.name === name || r.key === name)
66    if (!row) return { text: 'No skill named ' + name + ' in this session.' }
67    const source = await findSource($, row.name)
68    const filled = await $.prompt.fill({ text: fixPrompt(row, rows.listing, source) })
69    return { text: filled.isFilled ? 'The skill-creator request is in the prompt box. Read it, then press Enter.' : fixPrompt(row, rows.listing, source) }
70  })
71}
72
73// Every skill this session has, with its history and live counts
74async function collect($, all) {
75  const usage = await $.session.usage({ breakdown: 'summary' })
76  const skills = usage.context.breakdown?.skills
77  const listing = skills
78    ? { total: skills.totalSkills, included: skills.includedSkills, tokens: skills.tokens }
79    : null
80  const listed = skills?.skillFrontmatter ?? []
81
82  const history = await scanHistory($)
83  const live = (await $.store.get('live')) ?? {}
84  const marketplaceOf = await pluginMarketplaces($)
85  const mine = (s) => !NOT_MINE.includes(s.source) ||
86    (s.source === 'plugin' && ownMarketplaces.includes(marketplaceOf.get(s.pluginName) ?? ''))
87
88  const rows = listed
89    .filter((s) => all || mine(s))
90    .map((s) => {
91      // A plugin skill is called as plugin:name, a user skill by its bare name
92      const key = s.pluginName && !s.name.includes(':') ? s.pluginName + ':' + s.name : s.name
93      // Synced claude.ai skills are called with a prefix (anthropic-skills:pdf), so add up every key ending in :name too
94      const past = sum(history, key, s.name)
95      const l = sum(new Map(Object.entries(live)), key, s.name)
96      return { name: s.name, key, source: s.source, tokens: s.tokens, nameOnly: s.tokens < NAME_ONLY_TOKENS, uses: past.n, liveUses: l.n, last: Math.max(past.last, l.last) }
97    })
98    .sort((a, b) => b.uses + b.liveUses - (a.uses + a.liveUses) || a.name.localeCompare(b.name))
99  return { list: rows, listing }
100}
101
102// skill name -> { n, last }, last being the newest transcript file it appears in
103async function scanHistory($) {
104  const home = await $.env.get('HOME')
105  const own = (await $.env.get('CLAUDE_CONFIG_DIR')) || home + '/.claude'
106  const dirs = (configDirs.length ? configDirs : [own]).map((d) => d.replace(/^~(?=\/|$)/, home).replace(/\/+$/, '') + '/projects')
107  const grep = await $.process.run(
108    ['grep', '-roE', '--include=*.jsonl', GREP, ...dirs],
109    { timeoutMs: 60000 },
110  )
111  // grep exits 1 when nothing matched, 2 on an error such as a missing dir
112  const found = new Map()
113  const files = new Set()
114  for (const line of grep.stdout.split('\n')) {
115    const at = line.indexOf('.jsonl:')
116    if (at < 0) continue
117    const file = line.slice(0, at + 6)
118    const m = USE.exec(line.slice(at + 7))
119    if (!m) continue
120    const name = m[1] ?? m[2]
121    const entry = found.get(name) ?? { n: 0, files: new Set() }
122    entry.n += 1
123    entry.files.add(file)
124    found.set(name, entry)
125    files.add(file)
126  }
127
128  const mtime = new Map()
129  if (files.size) {
130    const stat = await $.process.run(['stat', '-f', '%m %N', ...files])
131    for (const line of stat.stdout.split('\n')) {
132      const sp = line.indexOf(' ')
133      if (sp > 0) mtime.set(line.slice(sp + 1), Number(line.slice(0, sp)) * 1000)
134    }
135  }
136
137  const out = new Map()
138  for (const [name, entry] of found) {
139    let last = 0
140    for (const f of entry.files) last = Math.max(last, mtime.get(f) ?? 0)
141    out.set(name, { n: entry.n, last })
142  }
143  return out
144}
145
146function sum(map, key, name) {
147  let n = 0, last = 0
148  for (const [k, v] of map) {
149    if (k === key || k === name || (!name.includes(':') && k.endsWith(':' + name))) {
150      n += v.n
151      last = Math.max(last, v.last)
152    }
153  }
154  return { n, last }
155}
156
157function day(ms) {
158  return ms ? new Date(ms).toISOString().slice(0, 10) : 'never'
159}
160
161function render(list, listing, all) {
162  const lines = []
163  if (listing) {
164    lines.push('Skill listing: ' + listing.included + ' of ' + listing.total + ' skills fit its budget (' + listing.tokens + ' tokens).')
165    if (listing.included < listing.total) {
166      lines.push('Skills past the budget are listed without their description, so Claude rarely picks them. Fewer plugin skills helps more than a better description there.')
167    }
168    lines.push('')
169  }
170  const width = Math.max(5, ...list.map((r) => r.key.length))
171  lines.push('skill'.padEnd(width) + '  history  live  last seen   tokens  listed as  source')
172  for (const r of list) {
173    lines.push(
174      r.key.padEnd(width) + '  ' + String(r.uses).padStart(7) + '  ' + String(r.liveUses).padStart(4) +
175      '  ' + day(r.last).padEnd(10) + '  ' + String(r.tokens).padStart(6) + '  ' + (r.nameOnly ? 'name only' : 'full     ') + '  ' + r.source,
176    )
177  }
178  const never = list.filter((r) => r.uses + r.liveUses === 0).map((r) => r.key)
179  lines.push('')
180  lines.push(never.length ? 'Never triggered (' + never.length + '): ' + never.join(', ') : 'Every listed skill has triggered at least once.')
181  const bare = list.filter((r) => r.nameOnly).map((r) => r.key)
182  if (bare.length) lines.push('Listed by name only, no description (' + bare.length + '): ' + bare.join(', ') + '. Claude can only match these by name.')
183  lines.push('History comes from the transcripts in ' + (configDirs.length ? configDirs.join(', ') : 'this account') + ', which Claude Code deletes after cleanupPeriodDays. Live counts start when this mod loads.')
184  if (never.length) lines.push('/skill-fix <skill> puts a skill-creator request for one of them in the prompt box.')
185  if (!all) lines.push('/skill-stats all adds plugin and built-in skills.')
186  return lines.join('\n')
187}
188
189// plugin name -> marketplace, from the enabled plugins in the merged settings
190async function pluginMarketplaces($) {
191  const out = new Map()
192  try {
193    const settings = await $.settings.read()
194    for (const id of Object.keys(settings?.enabledPlugins ?? {})) {
195      const at = id.lastIndexOf('@')
196      if (at > 0) out.set(id.slice(0, at), id.slice(at + 1))
197    }
198  } catch {
199    // No settings: only skills from a skills folder count as yours
200  }
201  return out
202}
203
204// The SKILL.md to edit, found under the source_dirs setting (an installed plugin is a cached copy)
205async function findSource($, name) {
206  if (!sourceDirs.length || !/^[\w.-]+$/.test(name)) return null
207  const home = await $.env.get('HOME')
208  const roots = sourceDirs.map((d) => d.replace(/^~(?=\/|$)/, home))
209  const found = await $.process.run(['find', ...roots, '-path', '*/skills/' + name + '/SKILL.md', '-not', '-path', '*/node_modules/*'])
210  return found.stdout.split('\n').find(Boolean) ?? null
211}
212
213function fixPrompt(row, listing, source) {
214  const facts = [
215    'uses in transcripts: ' + row.uses,
216    'live uses: ' + row.liveUses,
217    'last seen: ' + day(row.last),
218    'listing tokens: ' + row.tokens + (row.nameOnly ? ' (listed by name only, the description does not reach Claude; find out why before rewriting it)' : ''),
219    'source: ' + row.source,
220  ]
221  if (listing && listing.included < listing.total) {
222    facts.push('the skill listing holds ' + listing.included + ' of ' + listing.total + ' skills, so some are listed without a description')
223  }
224  return (
225    '/anthropic-skills:skill-creator Improve the description of the ' + row.key +
226    ' skill so it triggers when it should. ' + facts.join('; ') +
227    '. Read its SKILL.md first' +
228    (source ? ' (the source to edit is ' + source + '; the installed plugin is a cached copy, so after the edit bump that plugin\'s version and push)' : '') +
229    ', propose the new description, and wait for my yes before writing it.'
230  )
231}
232