skill-nudge

1import { atom, read, update } from "claude-code";
2import type { EngineInterface, Register } from "claude-code";
3
4import {
5  SYSTEM,
6  catalog,
7  listing,
8  parse,
9  transcript,
10  usedSkills,
11} from "../lib/nudge.ts";
12import {
13  parseEvents,
14  promptVersion,
15  report,
16  summarize,
17} from "../lib/telemetry.ts";
18import type { NudgeEvent, TipFate } from "../lib/telemetry.ts";
19
20const COMMAND = "skill-nudge";
21
22const ENABLED_KEY = "enabled";
23
24const MODEL = "haiku";
25
26const INSTALL_KEY = "installId";
27
28// One store key per session, so concurrent sessions never overwrite each other's events.
29const EVENTS_PREFIX = "events:";
30
31const MAX_EVENTS_PER_SESSION = 500;
32
33const MAX_SESSIONS = 50;
34
35// Haiku judges at most once per this many finished turns.
36const TURNS_BETWEEN_JUDGMENTS = 2;
37
38const nudge = atom({ plugin: "skill-nudge", key: "nudge" } as const, null);
39
40const offered = atom({ plugin: "skill-nudge", key: "offered" } as const, []);
41
42const turnsSinceJudgment = atom(
43  { plugin: "skill-nudge", key: "turnsSinceJudgment" } as const,
44  0,
45);
46
47async function isEnabled($: EngineInterface): Promise<boolean> {
48  return (await $.store.get(ENABLED_KEY)) !== false;
49}
50
51let isInteractive = false;
52
53let isJudging = false;
54
55// Bumped by each prompt, so a judgment that lands after the user moved on is dropped.
56let epoch = 0;
57
58// Appends run one at a time, so this session's events never overwrite each other.
59let recording: Promise<void> = Promise.resolve();
60
61function record($: EngineInterface, event: NudgeEvent): Promise<void> {
62  recording = recording.then(() => append($, event)).catch(() => undefined);
63
64  return recording;
65}
66
67async function append($: EngineInterface, event: NudgeEvent): Promise<void> {
68  const key = EVENTS_PREFIX + (await $.session.id());
69  const events = await eventsAt($, key);
70  await $.store.set(
71    key,
72    JSON.stringify([...events, event].slice(-MAX_EVENTS_PER_SESSION)),
73  );
74}
75
76async function eventsAt(
77  $: EngineInterface,
78  key: string,
79): Promise<NudgeEvent[]> {
80  return parseEvents(String((await $.store.get(key)) ?? "[]"));
81}
82
83async function fate(
84  $: EngineInterface,
85  type: TipFate["type"],
86  skill: string,
87): Promise<void> {
88  await record($, {
89    type,
90    skill,
91    at: await $.clock.now(),
92    prompt: await promptVersion(SYSTEM),
93  });
94}
95
96async function dismissTip($: EngineInterface, skill: string): Promise<void> {
97  await update($, nudge, () => null);
98  await fate($, "dismissed", skill);
99}
100
101async function prepareTelemetry($: EngineInterface): Promise<void> {
102  if ((await $.store.get(INSTALL_KEY)) === undefined)
103    await $.store.set(INSTALL_KEY, crypto.randomUUID());
104
105  const sessions = (await $.store.keys()).filter((key) =>
106    key.startsWith(EVENTS_PREFIX),
107  );
108
109  for (const key of sessions.slice(0, -MAX_SESSIONS)) await $.store.delete(key);
110}
111
112async function stats($: EngineInterface): Promise<string> {
113  await recording;
114
115  const keys = (await $.store.keys()).filter((key) =>
116    key.startsWith(EVENTS_PREFIX),
117  );
118
119  const sessions: NudgeEvent[][] = [];
120
121  for (const key of keys) sessions.push(await eventsAt($, key));
122
123  const summary = summarize(String(await $.store.get(INSTALL_KEY)), sessions);
124  const json = JSON.stringify(summary, null, 2);
125  const { isCopied } = await $.ui.copy({ text: json });
126
127  return `${report(summary)}\n\n${
128    isCopied
129      ? "Copied the JSON summary to the clipboard."
130      : `JSON summary:\n${json}`
131  }`;
132}
133
134async function judge($: EngineInterface, at: number): Promise<void> {
135  const messages = await $.session.messages();
136
137  const exclude = new Set([
138    ...usedSkills(messages),
139    ...(await read($, offered)),
140  ]);
141
142  const skills = catalog(await $.command.list(), exclude);
143
144  if (skills.length === 0) return;
145
146  const startedAt = await $.clock.now();
147
148  const reply = await $.model.complete({
149    model: MODEL,
150    system: SYSTEM,
151    prompt: [
152      { text: listing(skills), cache: true },
153      { text: transcript(messages) },
154    ],
155    maxTokens: 200,
156    timeoutMs: 20000,
157  });
158
159  const now = await $.clock.now();
160  const found = reply.isAnswered ? parse(reply.text, skills) : undefined;
161  const isStale = at !== epoch;
162  const shows = found !== undefined && !isStale;
163
164  // The tip shows first; recording it never delays it.
165  if (shows) {
166    await update($, nudge, () => found);
167    await update($, offered, (list) => [...list, found.skill]);
168  }
169
170  await record($, {
171    type: "judged",
172    at: now,
173    prompt: await promptVersion(SYSTEM),
174    outcome: reply.isAnswered
175      ? isStale
176        ? "stale"
177        : found === undefined
178          ? "none"
179          : "nudge"
180      : "error",
181    skill: found?.skill ?? null,
182    skills: skills.length,
183    latencyMs: now - startedAt,
184    inputTokens: reply.usage.input_tokens,
185    outputTokens: reply.usage.output_tokens,
186    cacheReadTokens: reply.usage.cache_read_input_tokens,
187  });
188}
189
190export const register: Register = (on) => {
191  // Each load starts fresh, as a reload's new environment does.
192  isInteractive = false;
193  isJudging = false;
194  epoch = 0;
195  recording = Promise.resolve();
196
197  on("session.start", async ($, e, next) => {
198    isInteractive = e.isInteractive;
199    await $.command.register({
200      name: COMMAND,
201      description: "Turn skill nudges on or off; `stats` shows how tips fared",
202      argumentHint: "[stats]",
203    });
204    await prepareTelemetry($).catch(() => undefined);
205
206    return next(e);
207  });
208
209  on("command.run", { command: COMMAND }, async ($, e) => {
210    if (e.args.trim() === "stats") return { text: await stats($) };
211
212    const isOn = !(await isEnabled($));
213    await $.store.set(ENABLED_KEY, isOn);
214    const shown = await read($, nudge);
215
216    if (!isOn && shown !== null) await dismissTip($, shown.skill);
217
218    return { text: `Skill nudges ${isOn ? "on" : "off"}.` };
219  });
220
221  on("prompt.submit", async ($, e, next) => {
222    epoch += 1;
223    const shown = await read($, nudge);
224
225    if (shown !== null) {
226      await update($, nudge, () => null);
227      const [command = ""] = e.text.trimStart().split(/\s/, 1);
228      const typed = command === `/${shown.skill}`;
229      await fate($, typed ? "typed" : "ignored", shown.skill);
230    }
231
232    return next(e);
233  });
234
235  on("turn.complete", async ($, e, next) => {
236    const result = await next(e);
237
238    if (e.agentId !== undefined || e.reason !== "answer" || !isInteractive)
239      return result;
240
241    const turns = (await read($, turnsSinceJudgment)) + 1;
242    await update($, turnsSinceJudgment, () => turns);
243
244    if (isJudging || turns < TURNS_BETWEEN_JUDGMENTS || !(await isEnabled($)))
245      return result;
246
247    isJudging = true;
248    await update($, turnsSinceJudgment, () => 0);
249    const at = epoch;
250    $.clock.after(0, () => {
251      void judge($, at)
252        .catch(() => undefined)
253        .finally(() => {
254          isJudging = false;
255        });
256    });
257
258    return result;
259  });
260
261  on("ui.render", { component: "AbovePrompt" }, async ($, e, next) => {
262    const shown = await read($, nudge);
263
264    if (shown === null || e.props.hasSurvey) return next(e);
265
266    const { Box, Button, Text } = $.ui.resolve(e);
267
268    const dismiss = () => dismissTip($, shown.skill);
269
270    const use = async () => {
271      // A draft the user already typed becomes the skill's arguments.
272      const { text } = await $.prompt.read();
273      await $.prompt.fill({ text: `/${shown.skill} ${text.trimStart()}` });
274      await update($, nudge, () => null);
275      await fate($, "used", shown.skill);
276    };
277
278    // Digit hotkeys, as a survey's: a bare digit in the empty prompt presses
279    // them, while a letter works only after the person focuses the band.
280    return (
281      <Box flexDirection="column" padding={1}>
282        <Text>
283          <Text color="suggestion" bold>
284            ✦ Skill tip
285          </Text>
286          <Text dimColor> · </Text>
287          <Text bold>/{shown.skill}</Text>
288        </Text>
289        <Box paddingLeft={2} flexDirection="column">
290          <Text>{shown.reason}</Text>
291          <Box columnGap={2}>
292            <Button key="use" label="Use" hotkey="1" plain onPress={use} />
293            <Button
294              key="dismiss"
295              label="Dismiss"
296              hotkey="2"
297              plain
298              role="dismiss"
299              onPress={dismiss}
300            />
301          </Box>
302        </Box>
303      </Box>
304    );
305  });
306};
307

1// Structural types, not the `claude-code` ones, so evals/skill-nudge can run this
2// prompt code under plain Node.
3export type Command = {
4  name: string;
5  description: string;
6  source: string;
7  plugin?: string;
8};
9
10export type Message = {
11  role: "user" | "assistant";
12  text: string;
13  toolUses: readonly { tool: string; input: { skill?: unknown } }[];
14};
15
16export type Skill = { name: string; description: string };
17
18export type Suggestion = { skill: string; reason: string };
19
20type Json = null | boolean | number | string | Json[] | JsonObject;
21
22type JsonObject = { [key: string]: Json };
23
24const RECENT_MESSAGES = 8;
25
26const MESSAGE_CHARS = 1500;
27
28// The band wraps long text; this cap only stops a runaway reply.
29const REASON_CHARS = 300;
30
31export const SYSTEM = `You watch a Claude Code session from the side. After each turn you decide whether the user should run one of their installed skills next, and which one.
32
33Each skill's description says what situations it is for ("Use when...", "Apply when..."). Match the situation in the latest exchange to those descriptions, not just the topic.
34
35Suggest a skill when:
36- the user's goal is what a skill is for, and the assistant answered without that skill's process: it guessed, gave a quick summary, or proposed a shortcut;
37- the work just done calls for a check that a skill provides, and the assistant has not done that check;
38- the user is about to start, plan, pause, resume, hand off, or ship work, and a skill covers that step.
39
40Suggest nothing when:
41- the turn was small talk, a trivial edit, or a complete self-contained answer;
42- the assistant already did what the skill would do: ran it, verified against the real artifact, or wrote and ran the regression test;
43- the assistant is mid-task and simply continuing.
44
45Pick the single most specific skill.
46
47Reply with one JSON object and nothing else:
48{"skill": "<a name exactly as listed>", "reason": "<one short sentence to the user, under 120 characters, saying why now; address them as \\"you\\", never as \\"the user\\">"}
49or, when nothing fits:
50{"skill": null}`;
51
52/** The user's own skills: plugin and user slash commands, less the ones already used or offered. */
53export function catalog(
54  commands: readonly Command[],
55  exclude: ReadonlySet<string>,
56): Skill[] {
57  return commands.flatMap(({ name, description, source, plugin }) =>
58    (source === "plugin" || source === "user") &&
59    plugin !== "skill-nudge" &&
60    !exclude.has(name)
61      ? [{ name, description }]
62      : [],
63  );
64}
65
66/** Skills the session already ran, by the Skill tool or typed as `/name`. */
67export function usedSkills(messages: readonly Message[]): Set<string> {
68  const used = new Set<string>();
69
70  for (const message of messages) {
71    for (const call of message.toolUses) {
72      const skill = call.tool === "Skill" ? String(call.input.skill ?? "") : "";
73
74      if (skill) used.add(skill);
75    }
76
77    const typed = /^\/([\w:-]+)/.exec(message.text.trimStart());
78
79    if (message.role === "user" && typed) used.add(typed[1]!);
80  }
81
82  return used;
83}
84
85export function listing(skills: readonly Skill[]): string {
86  const lines = skills.map(
87    ({ name, description }) => `- ${name}: ${description}`,
88  );
89
90  return `Installed skills:\n${lines.join("\n")}\n\n`;
91}
92
93export function transcript(messages: readonly Message[]): string {
94  const lines = messages.slice(-RECENT_MESSAGES).map((message) => {
95    const tools = message.toolUses.map((call) => call.tool);
96    const text = clip(message.text.trim(), MESSAGE_CHARS);
97    const used = tools.length > 0 ? ` [tools: ${tools.join(", ")}]` : "";
98
99    return `${message.role}: ${text}${used}`;
100  });
101
102  return `Recent conversation, oldest first:\n${lines.join("\n\n")}`;
103}
104
105/** Reads the model's reply; anything but a listed skill with a reason is no nudge. */
106export function parse(
107  reply: string,
108  skills: readonly Skill[],
109): Suggestion | undefined {
110  const source = /\{[\s\S]*\}/.exec(reply)?.[0];
111
112  if (source === undefined) return undefined;
113
114  let value: Json;
115
116  try {
117    value = JSON.parse(source);
118  } catch {
119    return undefined;
120  }
121
122  if (!isJsonObject(value)) return undefined;
123
124  const { skill, reason } = value;
125
126  if (!isString(skill) || !isString(reason) || reason.trim() === "")
127    return undefined;
128
129  if (!skills.some((one) => one.name === skill)) return undefined;
130
131  return { skill, reason: clip(reason.trim(), REASON_CHARS) };
132}
133
134function isJsonObject(value: Json | undefined): value is JsonObject {
135  return value === Object(value) && !Array.isArray(value);
136}
137
138function isString(value: Json | undefined): value is string {
139  return value === String(value);
140}
141
142function clip(text: string, limit: number): string {
143  return text.length > limit ? `${text.slice(0, limit - 1)}…` : text;
144}
145

1// Outcome events, kept on the user's machine. They hold no conversation text and
2// no Haiku reason: only what happened, which skill, and what it cost.
3
4export type Judgment = {
5  type: "judged";
6  at: number;
7  prompt: string;
8  // `stale`: Haiku answered after the user had already sent another prompt.
9  outcome: "nudge" | "none" | "error" | "stale";
10  skill: string | null;
11  skills: number;
12  latencyMs: number;
13  inputTokens: number;
14  outputTokens: number;
15  cacheReadTokens: number;
16};
17
18export type TipFate = {
19  // `typed`: the user ran the suggested skill without pressing Use.
20  type: "used" | "typed" | "dismissed" | "ignored";
21  at: number;
22  prompt: string;
23  skill: string;
24};
25
26export type NudgeEvent = Judgment | TipFate;
27
28type Fates = {
29  shown: number;
30  used: number;
31  typed: number;
32  dismissed: number;
33  ignored: number;
34};
35
36export type Summary = {
37  installId: string;
38  sessions: number;
39  from: string | null;
40  to: string | null;
41  judgments: { total: number } & Record<Judgment["outcome"], number>;
42  averageLatencyMs: number;
43  tokens: { input: number; output: number; cacheRead: number };
44  tips: Fates;
45  bySkill: Record<string, Fates>;
46  byPrompt: Record<string, Fates & { judged: number }>;
47};
48
49const emptyFates = (): Fates => ({
50  shown: 0,
51  used: 0,
52  typed: 0,
53  dismissed: 0,
54  ignored: 0,
55});
56
57type Json = null | boolean | number | string | Json[] | JsonObject;
58
59type JsonObject = { [key: string]: Json };
60
61const OUTCOMES = ["nudge", "none", "error", "stale"] as const;
62
63const FATES = ["used", "typed", "dismissed", "ignored"] as const;
64
65/** Reads a session's stored events, keeping only well-formed ones. */
66export function parseEvents(text: string): NudgeEvent[] {
67  let value: Json;
68
69  try {
70    value = JSON.parse(text);
71  } catch {
72    return [];
73  }
74
75  return Array.isArray(value) ? value.flatMap(eventOf) : [];
76}
77
78function eventOf(value: Json): NudgeEvent[] {
79  if (!isJsonObject(value)) return [];
80
81  const row = value;
82  const { type, at, prompt, skill } = row;
83
84  if (!isNumber(at) || !isString(prompt)) return [];
85
86  if (type === "judged") {
87    const outcome = OUTCOMES.find((one) => one === row.outcome);
88
89    const numbers = [
90      row.skills,
91      row.latencyMs,
92      row.inputTokens,
93      row.outputTokens,
94      row.cacheReadTokens,
95    ];
96
97    if (outcome === undefined || !numbers.every(isNumber)) return [];
98
99    return [
100      {
101        type,
102        at,
103        prompt,
104        outcome,
105        skill: isString(skill) ? skill : null,
106        skills: Number(row.skills),
107        latencyMs: Number(row.latencyMs),
108        inputTokens: Number(row.inputTokens),
109        outputTokens: Number(row.outputTokens),
110        cacheReadTokens: Number(row.cacheReadTokens),
111      },
112    ];
113  }
114
115  const fate = FATES.find((one) => one === type);
116
117  return fate !== undefined && isString(skill)
118    ? [{ type: fate, at, prompt, skill }]
119    : [];
120}
121
122function isJsonObject(value: Json): value is JsonObject {
123  return value === Object(value) && !Array.isArray(value);
124}
125
126function isNumber(value: Json | undefined): value is number {
127  return Number.isFinite(value);
128}
129
130function isString(value: Json | undefined): value is string {
131  return value === String(value);
132}
133
134/** A short, stable name for a system prompt, so results compare across prompt changes. */
135export async function promptVersion(system: string): Promise<string> {
136  const digest = await crypto.subtle.digest(
137    "SHA-256",
138    new TextEncoder().encode(system),
139  );
140
141  return [...new Uint8Array(digest).slice(0, 4)]
142    .map((byte) => byte.toString(16).padStart(2, "0"))
143    .join("");
144}
145
146export function summarize(
147  installId: string,
148  sessions: readonly (readonly NudgeEvent[])[],
149): Summary {
150  const events = sessions.flat().sort((a, b) => a.at - b.at);
151  const judgments = { total: 0, nudge: 0, none: 0, error: 0, stale: 0 };
152  const tokens = { input: 0, output: 0, cacheRead: 0 };
153  const tips = emptyFates();
154  const bySkill: Record<string, Fates> = {};
155  const byPrompt: Record<string, Fates & { judged: number }> = {};
156  let latency = 0;
157
158  for (const event of events) {
159    const prompt = (byPrompt[event.prompt] ??= { judged: 0, ...emptyFates() });
160
161    if (event.type === "judged") {
162      judgments.total += 1;
163      judgments[event.outcome] += 1;
164      prompt.judged += 1;
165      latency += event.latencyMs;
166      tokens.input += event.inputTokens;
167      tokens.output += event.outputTokens;
168      tokens.cacheRead += event.cacheReadTokens;
169
170      if (event.outcome !== "nudge" || event.skill === null) continue;
171
172      tips.shown += 1;
173      prompt.shown += 1;
174      (bySkill[event.skill] ??= emptyFates()).shown += 1;
175      continue;
176    }
177
178    tips[event.type] += 1;
179    prompt[event.type] += 1;
180    (bySkill[event.skill] ??= emptyFates())[event.type] += 1;
181  }
182
183  return {
184    installId,
185    sessions: sessions.filter((one) => one.length > 0).length,
186    from: dateOf(events.at(0)),
187    to: dateOf(events.at(-1)),
188    judgments,
189    averageLatencyMs:
190      judgments.total === 0 ? 0 : Math.round(latency / judgments.total),
191    tokens,
192    tips,
193    bySkill,
194    byPrompt,
195  };
196}
197
198export function report(summary: Summary): string {
199  const { judgments, tips } = summary;
200
201  if (judgments.total === 0) return "Skill Nudge has not judged a turn yet.";
202
203  const taken = tips.used + tips.typed;
204
205  const skills = Object.entries(summary.bySkill)
206    .sort(([, a], [, b]) => b.shown - a.shown)
207    .map(
208      ([skill, fates]) =>
209        `  /${skill}: shown ${fates.shown}, taken ${fates.used + fates.typed}, dismissed ${fates.dismissed}, ignored ${fates.ignored}`,
210    );
211
212  return [
213    `Skill Nudge, ${summary.from} to ${summary.to} (${summary.sessions} sessions)`,
214    `Judgments: ${judgments.total} (${judgments.nudge} tips, ${judgments.none} quiet, ${judgments.error} errors, ${judgments.stale} too late), ${summary.averageLatencyMs} ms average`,
215    `Tips: ${tips.shown} shown, ${taken} taken (${percent(taken, tips.shown)}), ${tips.dismissed} dismissed, ${tips.ignored} ignored`,
216    ...skills,
217  ].join("\n");
218}
219
220function dateOf(event: NudgeEvent | undefined): string | null {
221  return event === undefined
222    ? null
223    : new Date(event.at).toISOString().slice(0, 10);
224}
225
226function percent(part: number, whole: number): string {
227  return whole === 0 ? "0%" : `${Math.round((part / whole) * 100)}%`;
228}
229

1export type Nudge = { skill: string; reason: string };
2
3declare module "claude-code" {
4  interface PluginState {
5    "skill-nudge": {
6      nudge: Nudge | null;
7      offered: string[];
8      turnsSinceJudgment: number;
9    };
10  }
11}
12