prompt-polish

1import type { EngineInterface, ModelCompleteResult, ModelEffort, Register } from 'claude-code'
2
3export const MODEL = 'claude-opus-5-5'
4/** A prompt starting with this is sent as typed (the backslash removed). */
5export const RAW = '\\'
6/** Opus's reply when the prompt needs no change: one character back instead of the whole prompt, so a clean prompt costs no wait for its echo. */
7export const SAME = '='
8
9/**
10 * A call with no answer by now gets an identical second call, and the first answer wins (the other is cut): a stall
11 * (13.8 s and 60 s among the first 48 real prompts) then costs about this plus one normal call. Past 4 s it fired on 14%
12 * of a live bench's calls and won 2 of 17, as most were long rewrites still writing; past 6 s, 2.5% (2026-10-06).
13 */
14export const HEDGE_MS = 6000
15
16/** Which benchmarked instructions SYSTEM is (tune/BENCH.md); recorded on every event, so the audit can tell what a session ran. */
17export const INSTRUCTIONS = 'v6'
18
19export const SYSTEM = `You rewrite a prompt that a developer is about to send to Claude Code (Claude Opus 5.5, an agentic coding assistant working in their project). Your rewrite replaces their prompt, so Claude Code acts on exactly what you write. You never answer or carry out the prompt.
20
211. Fix spelling, punctuation and grammar.
222. Then make it clear and direct, as Anthropic's guidance for prompting Claude Code and Opus 5.5 advises:
23- When the developer wants something done, write an instruction ("Add…", "Fix…", "Find…"), not a wish ("I want to be able to…") or a request for suggestions: Opus 5.5 follows wording closely, and "can you suggest…" gets suggestions instead of changes.
24- Keep every file, symptom, expected result, reason, constraint and check they gave visible: Claude Code works from each one.
25- Put several asks in a short numbered list.
26
27Rules:
28- Keep the developer's meaning, scope and voice, pronouns included (I, we, you). Add nothing they did not give, such as requirements, facts, files, steps, tests or "double-check" lines (Opus 5.5 already checks its own work), and drop nothing they did.
29- Copy code, commands, file paths, URLs, identifiers, error messages and quoted text exactly as written, and pasted text (logs, output, code blocks, anything inside <pasted_content> tags, the tags included) character for character, typos and all: Claude Code searches for and runs them as written.
30- A question stays a question. A short prompt stays short.
31- Use plain wording, without capitals for emphasis or CRITICAL and MUST: plain instructions work best with Opus 5.5.
32- Reply with the rewritten prompt only, with no preface, notes, quotes or code fence around it.
33- If the prompt is already correct and clear, reply with ${SAME} alone.`
34
35/** What the tuner (tune/tune.py) may change; the mod reads it on every prompt, these defaults when it has none. */
36export type Tuning = { effort: ModelEffort; minWords: number; timeoutMs: number; retries?: number }
37export const DEFAULTS: Tuning = { effort: 'low', minWords: 4, timeoutMs: 60_000 }
38
39/** One polished prompt, as the tuner reads it from the events folder. */
40export type Event = {
41  at: number
42  words: number
43  ms: number
44  effort: ModelEffort
45  outcome: 'rewritten' | 'unchanged' | 'guarded' | 'failed' | 'timeout'
46  /** INSTRUCTIONS at the time */
47  v?: string
48  guard?: string
49  /** why the first rewrite was refused, when one more call fixed it */
50  repaired?: string
51  /** a call ran past HEDGE_MS and got a second one: whether the second answered first */
52  hedged?: 'won' | 'lost'
53  original?: string
54  rewrite?: string
55}
56
57/**
58 * Whether the user typed the prompt (stamped `composer`) in a session drawn on a surface, the terminal or the desktop app:
59 * a bare `claude -p` run has none, and a phone's prompts come as `bridge`.
60 */
61export const typed = (origin: { kind: string }, surfaces: readonly string[]) => origin.kind === 'composer' && surfaces.length > 0
62
63const words = (text: string) => text.trim().split(/\s+/).filter(Boolean).length
64
65/**
66 * A slash command (/compact, /plugin:cmd) or a shell (!) line. A prompt opening with a file path is neither: 2026-10-06, one
67 * starting /home/me/... went unpolished as a "command".
68 */
69export const COMMAND = /^\s*(?:!|\/[\w:.-]+(?:\s|$))/
70
71/** Prompts left alone: commands, shell lines, and replies under `minWords` words (yes do it). */
72export const skip = (text: string, minWords = DEFAULTS.minWords) => COMMAND.test(text) || words(text) < minWords
73
74// What a rewrite must carry over exactly: code spans, URLs, paths, --flags, snake_case names, file names, quoted text
75const VERBATIM = [
76  /`[^`\n]+`/g,
77  /\bhttps?:\/\/[^\s<>()"'`]+/g,
78  /(?<![\w.~/:-])(?:~|\.{1,2})?\/[\w.@-]+(?:\/[\w.@-]+)*/g,
79  /(?<![\w.~/:-])[\w.-]+(?:\/[\w.-]+)+\.\w{1,6}\b/g,
80  /(?<![\w-])--[a-zA-Z][\w-]*/g,
81  /\b[A-Za-z]\w*_\w+\b/g,
82]
83// A bare file name may change case only (Claude.md is CLAUDE.md); a path's case is kept exactly
84// A name with a letter in it: "3.js" is the library Three.js said aloud, not a file
85const FILE_NAME = /\b(?=[\w-]*[A-Za-z])[\w-]+\.(?:ts|tsx|js|mjs|cjs|py|json|jsonl|md|sh|css|html|txt|ya?ml|toml)\b/g
86const META = /\b(the (original )?prompt|rewrit\w*|reply with|corrected version|already (clear|correct|fine)|missing (full stop|period|comma|punctuation)|no change)\b/i
87// "reply with only" is instruction talk only before its "=": a prompt asking for a one-word reply says it too
88const SELF_TALK = /\b(the instructions say|must reply|reply with only\s*["'`]?=|output must be|wait, the prompt)/i
89const QUOTED = /"([^"\n]{1,200})"/g
90// Pasted text (Claude Code's <pasted_content> blocks) and fenced code: a log, an error or code goes through untouched
91const PASTED = /<pasted_content\b[^>]*>[\s\S]*?<\/pasted_content\b[^>]*>|```[\s\S]*?```/g
92
93/** Why a rewrite may not replace the prompt (it lost something to be kept exactly, added a preface, or ballooned); undefined when it may. */
94export function guard(original: string, rewrite: string): string | undefined {
95  if (/^(here('s| is) (the |your )?(rewritten|revised|improved|cleaned|polished|corrected|optimi[sz]ed)\b|sure\b|certainly\b|(the )?rewritten prompt\b)/i.test(rewrite)) return 'added a preface'
96  // The whole prompt back with talk about the rewrite around it: the model's own deliberation leaked into the reply
97  // (an intro line before a pasted error is fine; "already clear, missing period only" is not)
98  const flat = (t: string) => t.replace(/\s+/g, ' ').trim()
99  const [r, o] = [flat(rewrite), flat(original)]
100  if (o && r.includes(o) && META.test(r.replace(o, ' ')) && !META.test(o)) return 'added commentary'
101  // Talk about its own instructions is never part of a prompt, wherever it sits
102  if (SELF_TALK.test(r) && !SELF_TALK.test(o)) return 'added commentary'
103  if (/^```/.test(rewrite) && !/^```/.test(original.trim())) return 'wrapped in a code fence'
104  for (const [m] of original.matchAll(PASTED)) if (!rewrite.includes(m)) return 'changed pasted text'
105  for (const re of VERBATIM)
106    for (const [m] of original.matchAll(re)) {
107      const token = m.replace(/[.,;:!?)]+$/, '')
108      if (token.length > 1 && !rewrite.includes(token)) return `lost ${token}`
109    }
110  for (const [name] of original.matchAll(FILE_NAME))
111    if (!rewrite.toLowerCase().includes(name.toLowerCase())) return `lost ${name}`
112  for (const [, quoted] of original.matchAll(QUOTED)) {
113    // Its closing comma or full stop may move outside the quotes as the rewrite punctuates
114    const inner = quoted?.replace(/[.,;:!?]+$/, '')
115    if (inner && !rewrite.includes(inner)) return `lost "${inner}"`
116  }
117  if (words(rewrite) > 3 * words(original) + 30) return 'padded'
118  return undefined
119}
120
121/** Every string the guard would call lost, not only the first it stops at. */
122export function lostByGuard(original: string, rewrite: string): string[] {
123  const out: string[] = []
124  let r = rewrite
125  for (let i = 0; i < 30; i++) {
126    const why = guard(original, r)
127    if (!why?.startsWith('lost ')) break
128    const token = why.slice(5).replace(/^"(.*)"$/, '$1')
129    if (out.includes(token)) break
130    out.push(token)
131    r += `\n${token}`
132  }
133  return out
134}
135
136/** The one repair call's instruction: what was wrong with the refused rewrite, exactly (as polish-lab tested it). */
137export function repairNote(original: string, rewrite: string, why: string): string {
138  if (why.startsWith('lost ')) {
139    const lost = lostByGuard(original, rewrite)
140    return `Your previous rewrite dropped or changed text that has to be kept exactly. Write the rewrite again, keeping everything else you had, and put each of these strings back in its natural place, exactly as written here (not as a list at the end):\n${(lost.length ? lost : [why.slice(5)]).map(s => `- ${s}`).join('\n')}`
141  }
142  if (why === 'changed pasted text') return 'Your previous rewrite changed pasted text. Write the rewrite again with every pasted block (logs, output, code, <pasted_content> blocks with their tags) copied character for character.'
143  if (why === 'padded') return `Your previous rewrite was too long: ${rewrite.length} characters against a budget of ${Math.max(200, Math.round(2.5 * original.length))}. Write it again, shorter, without dropping anything the developer said.`
144  return 'Your previous rewrite added a preface, notes or a code fence. Reply with the rewritten prompt only.'
145}
146
147/** The tuner's settings over the defaults; anything missing or out of range keeps its default. */
148export function tuning(json: string | undefined): Tuning {
149  try {
150    const t = JSON.parse(json ?? '{}') as Partial<Tuning>
151    return {
152      effort: t.effort && ['low', 'medium', 'high', 'xhigh', 'max'].includes(t.effort) ? t.effort : DEFAULTS.effort,
153      minWords: typeof t.minWords === 'number' && Number.isInteger(t.minWords) && t.minWords >= 1 ? t.minWords : DEFAULTS.minWords,
154      timeoutMs: typeof t.timeoutMs === 'number' && Number.isInteger(t.timeoutMs) && t.timeoutMs >= 5000 ? t.timeoutMs : DEFAULTS.timeoutMs,
155    }
156  } catch {
157    return DEFAULTS
158  }
159}
160
161/**
162 * One rewrite through Opus, as a prompt gets it: the reply, how it came out, why the guard refused it, how long it took.
163 * A refused rewrite gets one more call naming what went wrong, within what is left of the time limit; only a second
164 * refusal sends the prompt as typed (polish-lab, 2026-10-06: it fixed 23 of 23 refusals, +0.5 to +0.9 over the prompt as typed).
165 */
166export async function polish($: EngineInterface, text: string, t: Tuning, system = SYSTEM, model = MODEL) {
167  // Instructions may carry a reminder for after the prompt, below a <<<AFTER>>> line (the bench tries instructions placed after the input)
168  const [head, after] = system.split(/\n?<<<AFTER>>>\n?/)
169  const started = await $.clock.now()
170  const turn = `<prompt>\n${text}\n</prompt>${after ? `\n\n${after}` : ''}`
171  let hedged: Event['hedged']
172  // Not answered by HEDGE_MS: an identical second call (within the same time limit), and the first answer wins
173  const ask = (prompt = turn, timeoutMs = t.timeoutMs) =>
174    new Promise<ModelCompleteResult>((resolve, reject) => {
175      const calls: AbortController[] = []
176      let open = 0
177      let done = false
178      const end = (settle: () => void, second: boolean) => {
179        done = true
180        timer.cancel()
181        if (calls.length > 1 && hedged !== 'won') hedged = second ? 'won' : 'lost'
182        for (const c of calls) c.abort()
183        settle()
184      }
185      const call = (ms: number) => {
186        const stop = new AbortController()
187        const second = calls.push(stop) > 1
188        open++
189        $.model.complete({ model, system: head!, prompt, maxTokens: 8000, effort: t.effort, timeoutMs: ms }, { signal: stop.signal }).then(
190          r => {
191            open--
192            // A failed call waits for the other one, if it still runs
193            if (!done && (r.isAnswered || open === 0)) end(() => resolve(r), second)
194          },
195          (err: unknown) => done || end(() => reject(err), second),
196        )
197      }
198      const timer = $.clock.after(HEDGE_MS, () => {
199        if (!done && timeoutMs - HEDGE_MS >= HEDGE_MS) call(timeoutMs - HEDGE_MS)
200      })
201      call(timeoutMs)
202    })
203  let r = await ask()
204  // An empty reply or an API error is sometimes a blip: retry it (a timeout is not, as the time is spent)
205  for (let n = 0; n < (t.retries ?? 0) && !r.isAnswered && (r.reason === 'empty-reply' || r.reason === 'api-error'); n++) r = await ask()
206  const read = (raw: string) => raw.trim().replace(/^<(prompt|rewrite)>\s*|\s*<\/(prompt|rewrite)>$/g, '')
207  // A line of = alone is Opus's "no change", even with its own deliberation around it
208  const same = (reply: string) => reply.split('\n').some(l => l.trim() === SAME) || reply.replace(/\s+/g, ' ') === text.trim().replace(/\s+/g, ' ')
209  let reply = r.isAnswered ? read(r.text) : ''
210  const unchanged = same(reply)
211  let why = reply && !unchanged ? guard(text, reply) : undefined
212  const refused = why
213  // Only a refused rewrite needs the clock read here (one round trip fewer for every other prompt)
214  const left = why ? t.timeoutMs - ((await $.clock.now()) - started) : 0
215  if (why && left >= 5000) {
216    const again = await ask(`${turn}\n\n<previous_rewrite>\n${reply}\n</previous_rewrite>\n\n${repairNote(text, reply, why)}`, left)
217    const fixed = again.isAnswered ? read(again.text) : ''
218    if (fixed && !same(fixed) && !guard(text, fixed)) [reply, why] = [fixed, undefined]
219  }
220  const ms = (await $.clock.now()) - started
221  const outcome: Event['outcome'] = !r.isAnswered
222    ? r.reason === 'aborted' ? 'timeout' : 'failed'
223    : !reply ? 'failed' : unchanged ? 'unchanged' : why ? 'guarded' : 'rewritten'
224  return { reply, outcome, why, repaired: why ? undefined : refused, hedged, ms, reason: r.isAnswered ? 'empty reply' : r.reason }
225}
226
227// The bench's judge: how a rewrite scores against Anthropic's guidance for prompting Claude Code and Opus 5.5
228const GRADES = ['fidelity', 'language', 'claude_code', 'brevity'] as const
229export const JUDGE = `You grade an automatic rewrite of a developer's prompt to Claude Code (Claude Opus 5.5). Score each 1-5:
230fidelity: same meaning and intent; nothing added that the developer did not say or clearly imply (no invented files, steps, tests, checks or constraints); nothing dropped; pasted text, logs and code blocks left exactly as given.
231language: spelling, grammar and punctuation are correct.
232claude_code: follows Anthropic's guidance for prompting Claude Code: clear and direct; states the action to take (do it, not suggest it) when the developer wants action; keeps the scope, files, symptoms and the check the developer gave visible; several asks as a short list; no aggressive emphasis (CAPS, CRITICAL); no added "double-check" lines, since Opus 5.5 checks its own work.
233brevity: no longer than it needs to be; a short prompt stays short; no preface or notes.
234Reply with JSON only: {"fidelity":n,"language":n,"claude_code":n,"brevity":n,"note":"one short sentence on the biggest flaw, or empty"}`
235
236// The source this module was loaded from: when the file on disk differs, the session runs an older mod
237let loaded: string | undefined
238let warned = false
239const source = ($: EngineInterface) => $.fs.read(`${$.plugin.root}/hooks/register.tsx`).then(s => (typeof s === 'string' ? s : undefined), () => undefined)
240
241export const register: Register = on => {
242  on('session.start', async ($, e, next) => {
243    loaded = await source($)
244    warned = false
245    await $.command.register({ name: 'polish-bench', description: 'Bench prompt-polish: /polish-bench <prompts.json> [system.txt|-] [effort] [model] [judge]; writes <prompts.json>.out.json', argumentHint: '<prompts.json> [system.txt|-] [effort] [model] [judge]' })
246    return next(e)
247  })
248
249  // The bench: every prompt of a JSON list through the same rewrite and guard, 6 at a time. The judge grades what Claude Code would
250  // receive for every prompt (the rewrite, or the prompt as typed when it was left alone, refused or failed), so runs that rewrite more
251  // or less are graded on the same prompts; each row records the model, effort and judge. Judge "-": a timing run, nothing graded
252  on('command.run', { command: 'polish-bench' }, async ($, e) => {
253    const [file, systemFile, effort, model, judge] = e.args.trim().split(/\s+/)
254    if (!file) return { text: 'usage: /polish-bench <prompts.json> [system.txt|-] [effort] [model] [judge]' }
255    const prompts = JSON.parse(String(await $.fs.read(file))) as string[]
256    const system = systemFile && systemFile !== '-' ? String(await $.fs.read(systemFile)) : SYSTEM
257    // A bench measures each effort's own time and quality: a longer limit than a typed prompt gets, so slow ones are not cut off
258    // An effort like low+retry also retries a failed reply once
259    const [level, plus] = (effort ?? '').split('+')
260    const t: Tuning = { ...DEFAULTS, timeoutMs: 180_000, ...(level ? { effort: level as ModelEffort } : {}), retries: plus === 'retry' ? 1 : 0 }
261    const setup = { model: model ?? MODEL, effort: t.effort, judge: judge ?? MODEL, ...(t.retries ? { retries: t.retries } : {}) }
262    const rows: object[] = []
263    let i = 0
264    const worker = async () => {
265      while (i < prompts.length) {
266        const text = prompts[i++]!
267        if (skip(text, t.minWords)) {
268          rows.push({ text, outcome: 'skipped', ...setup })
269          continue
270        }
271        const p = await polish($, text, t, system, setup.model)
272        const delivered = p.outcome === 'rewritten' ? p.reply : text
273        if (setup.judge === '-') {
274          rows.push({ text, ...p, delivered, grade: null, ...setup })
275          continue
276        }
277        // Each judge grades blind (the original and what would be delivered, nothing about the setup); several are averaged.
278        // Thinking counts toward maxTokens: room for it, so a grade is not cut off
279        const grades: Record<string, Record<string, number>> = {}
280        for (const j of setup.judge.split(',')) {
281          const g = await $.model.complete({ model: j, system: JUDGE, prompt: `<original>\n${text}\n</original>\n<rewrite>\n${delivered}\n</rewrite>`, maxTokens: 4000, effort: 'low' })
282          try {
283            const parsed = g.isAnswered ? JSON.parse(g.text.slice(g.text.indexOf('{'), g.text.lastIndexOf('}') + 1)) : null
284            if (parsed && GRADES.every(k => typeof parsed[k] === 'number')) grades[j] = parsed
285          } catch {
286            // an unreadable grade counts as none from that judge
287          }
288        }
289        const given = Object.values(grades)
290        const grade = given.length
291          ? { ...Object.fromEntries(GRADES.map(k => [k, given.reduce((n, g) => n + g[k]!, 0) / given.length])), note: given.map(g => g['note']).filter(Boolean).join(' / ') }
292          : null
293        rows.push({ text, ...p, delivered, grade, ...(given.length > 1 || setup.judge.includes(',') ? { grades } : {}), ...setup })
294      }
295    }
296    await Promise.all(Array.from({ length: 6 }, worker))
297    await $.fs.write(`${file}.out.json`, JSON.stringify(rows, null, 1))
298    return { text: `polish-bench: ${rows.length} prompts, results in ${file}.out.json` }
299  })
300
301  on('prompt.submit', async ($, e, next) => {
302    if (e.text.startsWith(RAW)) return next({ ...e, text: e.text.slice(RAW.length) })
303    if (COMMAND.test(e.text) || words(e.text) < 2) return next(e)
304    // Every engine read the decision needs, at once (each is a round trip to the engine)
305    const [surfaces, off, home, own] = await Promise.all([$.session.surfaces(), $.env.get('PROMPT_POLISH_OFF'), $.env.get('HOME'), $.env.get('PROMPT_POLISH_STATE')])
306    // Scripts, SDK and headless runs send their prompts exactly as written
307    if (!typed(e.origin, surfaces) || off) return next(e)
308    // A session keeps the mod it loaded until /reload-plugins: say so once when a later save has not reached it (the prompt does not wait)
309    if (loaded !== undefined && !warned) {
310      warned = true
311      void source($).then(now => {
312        if (now === undefined || now === loaded) warned = false
313        else return $.ui.toast('prompt-polish was updated after this session loaded it: run /reload-plugins to use the new version')
314      })
315    }
316    // PROMPT_POLISH_STATE: a latency bench keeps its events and the tuner's changes out of the real folder
317    const state = own || `${home}/.local/state/prompt-polish`
318    const t = tuning(await $.fs.read(`${state}/tuning.json`).then(s => (typeof s === 'string' ? s : undefined), () => undefined))
319    if (skip(e.text, t.minWords)) return next(e)
320
321    const { reply, outcome, why, repaired, hedged, ms, reason } = await polish($, e.text, t)
322    // The record and the toasts go on their own: the prompt does not wait for them
323    const wrote = $.clock.now().then(at => {
324      const event: Event = { at, words: words(e.text), ms, effort: t.effort, outcome, v: INSTRUCTIONS, ...(why ? { guard: why, original: e.text, rewrite: reply } : {}), ...(repaired ? { repaired } : {}), ...(hedged ? { hedged } : {}) }
325      return $.fs.write(`${state}/events/${at}-${Math.random().toString(36).slice(2, 8)}.json`, JSON.stringify(event))
326    }).catch(() => {})
327    if (outcome === 'failed' || outcome === 'timeout') void $.ui.toast(`Prompt sent as typed: Opus did not rewrite it (${reason})`)
328    if (why) void $.ui.toast(`Prompt sent as typed: the rewrite ${why}`)
329    const entered = outcome === 'rewritten'
330      ? await next({ ...e, text: reply, context: [...(e.context ?? []), `prompt-polish: Opus rewrote the user's prompt above. Their own words, in case the rewrite lost something:\n${e.text}`] })
331      : await next(e)
332
333    // After the prompt went in and its record was written: the tuner reads the latest numbers and changes tuning.json only when they call for it
334    void wrote.then(() => $.process.run(['python3', `${$.plugin.root}/tune/tune.py`, '--state', state], { timeoutMs: 20_000 })).then(
335      ({ stdout }) => (stdout.trim() ? $.ui.toast(`prompt-polish tuned: ${stdout.trim()}`) : undefined),
336      () => undefined,
337    )
338    return entered
339  })
340}
341