prompt-coach

1import { atom, read, update } from 'claude-code'
2import type { Register, Hook, Color } from 'claude-code'
3
4import type { Recent, Rubric, Score } from '../types'
5
6const PANE = 'prompt-coach'
7const KEEP = 30
8const MIN_WORDS = 6
9const GAUGE_CELLS = 20
10
11const scores = atom({ plugin: 'prompt-coach', key: 'scores' } as const, [])
12const selected = atom({ plugin: 'prompt-coach', key: 'selected' } as const, null)
13const recent = atom({ plugin: 'prompt-coach', key: 'recent' } as const, { prompts: [], answer: '' })
14const isHidden = atom({ plugin: 'prompt-coach', key: 'isHidden' } as const, false)
15const isPending = atom({ plugin: 'prompt-coach', key: 'isPending' } as const, false)
16const lastError = atom({ plugin: 'prompt-coach', key: 'lastError' } as const, null)
17const sessionId = atom({ plugin: 'prompt-coach', key: 'sessionId' } as const, null)
18
19const KEEP_SESSIONS = 30
20
21/** The store keys to drop so at most `keep` sessions stay, oldest first. */
22export const staleKeys = (keys: readonly string[], keep: number): string[] => {
23  const mine = keys.filter(k => k.startsWith('session:'))
24  return mine.length <= keep ? [] : mine.slice(0, mine.length - keep)
25}
26
27type Saved = { scores: Score[]; recent: Recent }
28
29/** Writes this session's history to the plugin's own store, so a resumed session gets it back. */
30async function persist($: Parameters<Hook<'prompt.submit'>>[0]) {
31  const id = await read($, sessionId)
32  if (id === null) return
33  const saved: Saved = { scores: await read($, scores), recent: await read($, recent) }
34  await $.store.set(`session:${id}`, saved)
35}
36
37
38const RULES = `You grade prompts that a software developer sends to an AI coding assistant, inside an ongoing conversation.
39You are given the conversation so far (the developer's previous prompts and the assistant's last reply) and then the new prompt. Grade the NEW prompt in that context: anything the assistant already knows from earlier turns counts as given, and a short follow-up that is unambiguous in context scores well. Do not invent a different domain than the conversation shows.
40First decide the prompt's kind: "task" (asks for work: build, fix, change, write, run), "question" (asks to know or understand something), or "decision" (answers a question the assistant asked, or picks an option). Grade each kind by what a good one needs. For a question, criteria means "is it clear what a satisfying answer contains", scope means "is it one question, bounded to one thing", and constraints means "are the things that would change the answer named"; a question that the assistant can answer fully with nothing more scores 16 or more on all three. For a decision, a plain unambiguous answer scores 16 or more everywhere. Only a task is judged on files, platforms, tests and done criteria.
41Score five criteria, each 0 to 20:
42- clarity: is the ask unambiguous, with the task stated up front?
43- context: does the assistant have what it needs (from this prompt or earlier turns): what is known, where things are, what was tried?
44- criteria: is it clear what "done" looks like, how to verify, or what to return?
45- scope: is the work bounded (which files, which platform, what not to touch)?
46- constraints: are rules, examples, style or decisions already made named where they matter?
47Scoring anchors, apply them literally: 20 = nothing more could reasonably be asked on this criterion for a task of this size; 16-19 = covered, one small thing could be sharper; 11-15 = partly covered; 5-10 = mostly missing; 0-4 = absent. A prompt that names the file or module, the symptom, what is already in use, the platform, the test to add and the command to run is 16 or more on every criterion. Do not withhold points for details the developer could not know yet (the root cause, which fix is right). Do not withhold points for brevity when the ask is already unambiguous. A follow-up that continues work agreed in earlier turns inherits the criteria, scope and constraints already established there; "keep the tests passing" names a success criterion. Leave "better" empty when nothing material is missing.
48Then write "rewrite": the same ask, rewritten as the developer should have sent it, in the developer's voice, keeping their intent and length within reason (one to four sentences). Never pad with things the assistant already knows.
49Answer with JSON only, no prose and no code fence:
50{"kind":"task|question|decision","clarity":n,"context":n,"criteria":n,"scope":n,"constraints":n,"tip":"one sentence, under 90 characters, the single most useful change","better":["up to three short bullets on what to add or change"],"rewrite":"the improved prompt"}`
51
52/** A prompt worth grading: not a slash command, not a bare "yes". */
53export const isGradable = (text: string): boolean => {
54  const t = text.trim()
55  if (t.length === 0 || t.startsWith('/')) return false
56  return t.split(/\s+/).length >= MIN_WORDS
57}
58
59const clamp = (n: unknown, max: number): number => {
60  const v = typeof n === 'number' && Number.isFinite(n) ? Math.round(n) : 0
61  return Math.max(0, Math.min(max, v))
62}
63
64/** The grader's JSON, tolerant of a fence or stray prose around it. */
65export const parseGrade = (text: string): Omit<Score, 'at' | 'prompt'> | null => {
66  const start = text.indexOf('{')
67  const end = text.lastIndexOf('}')
68  if (start < 0 || end <= start) return null
69  let raw: Record<string, unknown>
70  try {
71    raw = JSON.parse(text.slice(start, end + 1)) as Record<string, unknown>
72  } catch {
73    return null
74  }
75  const rubric: Rubric = {
76    clarity: clamp(raw.clarity, 20),
77    context: clamp(raw.context, 20),
78    criteria: clamp(raw.criteria, 20),
79    scope: clamp(raw.scope, 20),
80    constraints: clamp(raw.constraints, 20),
81  }
82  const sum = rubric.clarity + rubric.context + rubric.criteria + rubric.scope + rubric.constraints
83  const score = Math.max(1, Math.min(100, sum))
84  const tip = typeof raw.tip === 'string' ? raw.tip.trim() : ''
85  const better = Array.isArray(raw.better)
86    ? raw.better.filter((b): b is string => typeof b === 'string').map(b => b.trim()).filter(Boolean).slice(0, 3)
87    : []
88  const rewrite = typeof raw.rewrite === 'string' ? raw.rewrite.trim() : ''
89  const kind = raw.kind === 'question' || raw.kind === 'decision' ? raw.kind : 'task'
90  return { score, rubric, tip, better, rewrite, kind }
91}
92
93/** Red at 1, amber at 50, green at 100, as a hex colour. */
94export const colorFor = (score: number): Color => {
95  const t = Math.max(0, Math.min(1, (score - 1) / 99))
96  const mix = (a: number, b: number, k: number) => Math.round(a + (b - a) * k)
97  const [r, g, b] =
98    t < 0.5
99      ? [mix(0xd9, 0xe6, t * 2), mix(0x3c, 0xa8, t * 2), mix(0x3c, 0x2e, t * 2)]
100      : [mix(0xe6, 0x2e, (t - 0.5) * 2), mix(0xa8, 0xb8, (t - 0.5) * 2), mix(0x2e, 0x5a, (t - 0.5) * 2)]
101  const hex = (n: number) => n.toString(16).padStart(2, '0')
102  return `#${hex(r)}${hex(g)}${hex(b)}`
103}
104
105/** The conversation so far, as the grader reads it, then the new prompt. */
106export const gradingPrompt = (ctx: Recent, text: string): string => {
107  const earlier = ctx.prompts.length === 0 ? '(none)' : ctx.prompts.map((p, i) => `${i + 1}. ${p}`).join('\n')
108  const answer = ctx.answer === '' ? '(none)' : ctx.answer
109  return `Developer's previous prompts, oldest first:\n${earlier}\n\nAssistant's last reply (excerpt):\n${answer}\n\nNEW PROMPT to grade:\n${text}`
110}
111
112async function grade($: Parameters<Hook<'prompt.submit'>>[0], text: string, at: number) {
113  await update($, isPending, () => true)
114  const ctx = await read($, recent)
115  const r = await $.model.complete({
116    model: 'sonnet',
117    effort: 'low',
118    maxTokens: 700,
119    timeoutMs: 25000,
120    system: [{ text: RULES, cache: true }],
121    prompt: gradingPrompt(ctx, text),
122  })
123  await update($, isPending, () => false)
124
125  if (!r.isAnswered) {
126    await update($, lastError, () => `grader ${r.reason}`)
127    return
128  }
129  const parsed = parseGrade(r.text)
130  if (parsed === null) {
131    await update($, lastError, () => 'grader returned no JSON')
132    return
133  }
134  const entry: Score = { at, prompt: text.replace(/\s+/g, ' ').slice(0, 80), ...parsed }
135  await update($, lastError, () => null)
136  await update($, scores, list => [...list, entry].slice(-KEEP))
137  await update($, selected, () => null)
138  await persist($)
139}
140
141export const register: Register = on => {
142  on('session.start', async ($, e, next) => {
143    const id = await $.session.id()
144    await update($, sessionId, () => id)
145    const saved = (await $.store.get(`session:${id}`)) as Saved | undefined
146    if (saved !== undefined && Array.isArray(saved.scores)) {
147      await update($, scores, () => saved.scores)
148      await update($, recent, () => saved.recent ?? { prompts: [], answer: '' })
149    } else {
150      await $.store.set(`session:${id}`, { scores: [], recent: { prompts: [], answer: '' } })
151    }
152    for (const stale of staleKeys(await $.store.keys(), KEEP_SESSIONS)) await $.store.delete(stale)
153    await $.command.register({
154      name: 'prompt-coach',
155      description: 'Open the prompt coach pane: your last scores, the rubric and what to do better',
156    })
157    void $.ui.open({ id: PANE, title: 'Prompt coach' })
158
159    return next(e)
160  })
161
162  on('command.run', { command: 'prompt-coach' }, async $ => {
163    await update($, isHidden, () => false)
164    const opened = await $.ui.open({ id: PANE, title: 'Prompt coach', focus: true })
165
166    return { text: opened.isPlaced ? 'Prompt coach pane opened.' : 'Prompt coach pane is waiting for room; widen the window.' }
167  })
168
169  on('prompt.submit', async ($, e, next) => {
170    const text = e.text
171    if (!text.trim().startsWith('/')) {
172      await update($, recent, r => ({ ...r, prompts: [...r.prompts, text.replace(/\s+/g, ' ').slice(0, 300)].slice(-3) }))
173    }
174    if (!isGradable(text)) return next(e)
175
176    const started = await $.clock.now()
177    void grade($, text, started).catch(() => update($, isPending, () => false))
178
179    return next(e)
180  })
181
182  on('turn.complete', async ($, e, next) => {
183    if (e.agentId === undefined) {
184      const answer = e.answer.replace(/\s+/g, ' ').slice(0, 600)
185      await update($, recent, r => ({ ...r, answer }))
186      await persist($)
187    }
188
189    return next(e)
190  })
191
192  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
193    const list = await read($, scores)
194    const latest = list[list.length - 1]
195    const pending = await read($, isPending)
196    const error = await read($, lastError)
197    const hidden = await read($, isHidden)
198    if (e.props.hasSurvey || hidden || (latest === undefined && !pending && error === null)) {
199      return next(e)
200    }
201
202    const { Box, Button, Text } = $.ui.resolve(e)
203
204    return (
205      <Box flexDirection="row" gap={1}>
206        {latest !== undefined && <Gauge Text={Text} Box={Box} score={latest.score} />}
207        {latest !== undefined && (
208          <Text bold color={colorFor(latest.score)}>
209            {String(latest.score)}
210          </Text>
211        )}
212        {latest !== undefined && (
213          <Text dimColor wrap="truncate-end">
214            {latest.tip}
215          </Text>
216        )}
217        {pending && <Text dimColor>scoring…</Text>}
218        {error !== null && !pending && <Text color="warning">{error}</Text>}
219        <Button key="hide" label="Hide" plain onPress={() => update($, isHidden, () => true)} />
220      </Box>
221    )
222  })
223
224  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
225    const { Box, Button, Text } = $.ui.resolve(e)
226    const list = await read($, scores)
227    const pick = await read($, selected)
228    const shownIndex = pick !== null && pick >= 0 && pick < list.length ? pick : list.length - 1
229    const latest = list[shownIndex]
230    const rows = e.viewport?.rows ?? 24
231    const historyRoom = Math.max(1, rows - 16)
232
233    if (latest === undefined) {
234      return (
235        <Box flexDirection="column">
236          <Text dimColor>No prompts scored yet. Prompts of {String(MIN_WORDS)}+ words are graded as you send them.</Text>
237        </Box>
238      )
239    }
240
241    const avg = Math.round(list.reduce((s, one) => s + one.score, 0) / list.length)
242    const r = latest.rubric
243
244    return (
245      <Box flexDirection="column" gap={1}>
246        <Box flexDirection="row" gap={1}>
247          <Text bold>{shownIndex === list.length - 1 ? 'Latest' : `Prompt ${String(shownIndex + 1)}`}</Text>
248          <Gauge Text={Text} Box={Box} score={latest.score} />
249          <Text bold color={colorFor(latest.score)}>{String(latest.score)}</Text>
250          <Text dimColor>avg {String(avg)} over {String(list.length)}</Text>
251          <Text dimColor>{latest.kind}</Text>
252        </Box>
253        <Text dimColor wrap="truncate-end">"{latest.prompt}"</Text>
254        <Box flexDirection="column">
255          <RubricRow Text={Text} Box={Box} label="Clarity" value={r.clarity} />
256          <RubricRow Text={Text} Box={Box} label="Context" value={r.context} />
257          <RubricRow Text={Text} Box={Box} label="Criteria" value={r.criteria} />
258          <RubricRow Text={Text} Box={Box} label="Scope" value={r.scope} />
259          <RubricRow Text={Text} Box={Box} label="Constraints" value={r.constraints} />
260        </Box>
261        <Box flexDirection="column">
262          <Text bold>Do better</Text>
263          <Text>{latest.tip}</Text>
264          {latest.better.map(b => (
265            <Text dimColor wrap="wrap">
266              {'  - '}
267              {b}
268            </Text>
269          ))}
270        </Box>
271        {latest.rewrite !== '' && (
272          <Box flexDirection="column">
273            <Text bold color="success">Try instead</Text>
274            <Text wrap="wrap">{latest.rewrite}</Text>
275          </Box>
276        )}
277        <Box flexDirection="column">
278          <Text bold>History</Text>
279          <Text dimColor>Pick a row to see its rubric, tip and rewrite.</Text>
280          {list
281            .map((one, i) => ({ one, i }))
282            .slice(-historyRoom)
283            .reverse()
284            .map(({ one, i }) => (
285              <Button key={`h${String(i)}`} plain onPress={() => update($, selected, () => i)}>
286                <Text color={colorFor(one.score)}>{String(one.score).padStart(3)}</Text>
287                {' '}
288                <Text dimColor>{i === shownIndex ? '> ' : '  '}</Text>
289                <Text dimColor={i !== shownIndex} wrap="truncate-end">{one.prompt}</Text>
290              </Button>
291            ))}
292        </Box>
293      </Box>
294    )
295  })
296}
297
298
299/** A bar of cells, each coloured where it sits on the red-to-green run, filled up to the score. */
300function Gauge(props: { Text: any; Box: any; score: number; cells?: number }) {
301  const { Text, Box, score } = props
302  const cells = props.cells ?? GAUGE_CELLS
303  const filled = Math.round((score / 100) * cells)
304  const parts = []
305  for (let i = 0; i < cells; i += 1) {
306    const at = Math.round(((i + 1) / cells) * 100)
307    parts.push(
308      i < filled ? (
309        <Text color={colorFor(at)}>▰</Text>
310      ) : (
311        <Text dimColor>▱</Text>
312      ),
313    )
314  }
315  return <Box flexDirection="row">{parts}</Box>
316}
317
318function RubricRow(props: { Text: any; Box: any; label: string; value: number }) {
319  const { Text, Box, label, value } = props
320  const pct = value * 5
321  return (
322    <Box flexDirection="row" gap={1}>
323      <Text dimColor>{label.padEnd(12)}</Text>
324      <Gauge Text={Text} Box={Box} score={pct} cells={10} />
325      <Text color={colorFor(pct)}>{String(value).padStart(2)}/20</Text>
326    </Box>
327  )
328}
329

1export type Rubric = {
2  clarity: number
3  context: number
4  criteria: number
5  scope: number
6  constraints: number
7}
8
9export type Score = {
10  at: number
11  prompt: string
12  score: number
13  rubric: Rubric
14  tip: string
15  better: string[]
16  rewrite: string
17  kind: 'task' | 'question' | 'decision'
18}
19
20export type Recent = {
21  prompts: string[]
22  answer: string
23}
24
25declare module 'claude-code' {
26  interface PluginState {
27    'prompt-coach': {
28      scores: Score[]
29      selected: number | null
30      recent: Recent
31      sessionId: string | null
32      isHidden: boolean
33      isPending: boolean
34      lastError: string | null
35    }
36  }
37}
38