jev-effort

1/**
2 * jev-effort — sets the main loop's reasoning effort per prompt with TypeSafe's
3 * Jev (via OpenRouter by default). Forked from claude-code-templates'
4 * jev-model-router v0.4.2 (MIT), reduced to effort routing only.
5 *
6 *   prompt.submit  read recent context, ask Jev one score question
7 *   turn.step      step 0 applies the decision; later steps reuse it
8 *
9 * `shadow` (default on) classifies and logs but never changes a request.
10 * Fails open: on any error the turn keeps its effort (see `sticky`).
11 * Privacy: prompt + reply excerpts and tool names go to `endpoint`; never tool I/O.
12 */
13import type { Register } from 'claude-code'
14import {
15  askingLine,
16  buildState,
17  decide,
18  logLine,
19  readAnswer,
20  requestBody,
21  requestHeaders,
22  setupLine,
23  skipReason,
24  SPARK_LENGTH,
25  sparkline,
26  statusLine,
27  sticky,
28} from './policy.ts'
29import type { Decision, Effort, JevAnswer } from './policy.ts'
30
31interface Classification {
32  answer: JevAnswer | null
33  note: string | null
34  jevMs?: number
35}
36
37type SessionEffort = Effort | 'max' | number | undefined
38
39const clamp = (v: number, lo: number, hi: number) => Math.min(hi, Math.max(lo, v))
40
41export const register: Register = (on, options) => {
42  const text = (key: string, fallback: string) =>
43    typeof options[key] === 'string' && options[key] ? (options[key] as string) : fallback
44  const number = (key: string, fallback: number) =>
45    typeof options[key] === 'number' ? (options[key] as number) : fallback
46  const flag = (key: string, fallback: boolean) =>
47    typeof options[key] === 'boolean' ? (options[key] as boolean) : fallback
48
49  const configuredKey = text('apiKey', '')
50  const shadow = flag('shadow', true)
51  const sendContext = flag('sendContext', true)
52  const logDecisions = flag('logDecisions', true)
53  const model = text('model', '~typesafe/jev-latest')
54  const endpoint = text('endpoint', 'https://openrouter.ai/api/v1/systemone')
55  const timeoutMs = clamp(number('timeoutMs', 800), 100, 5000)
56  const bars = {
57    up: clamp(number('minUpgradeConfidence', 0.3), 0, 1),
58    down: clamp(number('minDowngradeConfidence', 0.6), 0, 1),
59  }
60
61  // Module state; a hot reload drops it, which fails open.
62  let announced = false
63  let apiKey = ''
64  let pending: Classification | null = null
65  let turn: { id: string; applied: Effort | null } | null = null
66  let prev: { session: SessionEffort; sent: SessionEffort } | null = null
67  const scores: number[] = []
68
69  on('prompt.submit', async ($, e, next) => {
70    if (!announced) {
71      announced = true
72      // Env fallback keeps the key out of settings.json.
73      apiKey = configuredKey || ((await $.env.get('OPENROUTER_API_KEY')) ?? '')
74      if (logDecisions) $.ui.log(setupLine({ apiKey, shadow, model, endpoint, sendContext, bars }))
75    }
76    if (!apiKey) return next(e)
77
78    const skip = skipReason(e.text, e.origin, e.turnId)
79    if (skip) {
80      pending = { answer: null, note: `skipped: ${skip}` }
81      return next(e)
82    }
83
84    $.ui.status(askingLine(shadow, sparkline(scores)))
85    try {
86      const state = buildState(e.text, sendContext ? await $.session.messages() : [], sendContext)
87      const t0 = await $.clock.now()
88      const response = await Promise.race([
89        $.http.fetch(endpoint, { method: 'POST', headers: requestHeaders(apiKey), body: requestBody(model, state) }),
90        $.clock.sleep(timeoutMs),
91      ])
92      const jevMs = (await $.clock.now()) - t0
93      const answer = response && response.ok ? readAnswer(response.text) : null
94      const note = !response
95        ? `timeout ${timeoutMs}ms`
96        : !response.ok
97          ? `HTTP ${response.status}`
98          : answer
99            ? null
100            : 'bad response'
101      if (answer) {
102        scores.push(answer.score)
103        if (scores.length > SPARK_LENGTH) scores.shift()
104      }
105      pending = { answer, note, jevMs }
106    } catch (error) {
107      pending = { answer: null, note: `error: ${String(error)}`.slice(0, 120) }
108    }
109    return next(e)
110  })
111
112  on('turn.step', async function* ($, e, next) {
113    if (e.agentId || !apiKey) return yield* next(e)
114
115    if (e.index > 0) {
116      const applied = turn?.id === e.turnId ? turn.applied : null
117      return yield* next(applied ? { ...e, effort: applied } : e)
118    }
119
120    const c = pending ?? { answer: null, note: 'no prompt' }
121    pending = null
122    const session = e.effort
123    let decision: Decision = decide(c.answer, session, bars)
124    if (!c.answer && c.note) decision = { ...decision, reason: c.note }
125    const stuck = c.answer ? null : sticky(session, prev)
126    const applied = shadow ? null : (decision.target ?? stuck)
127
128    turn = { id: e.turnId, applied }
129    prev = { session, sent: applied ?? session }
130
131    const line = { shadow, session, answer: c.answer, note: c.note, jevMs: c.jevMs, decision, stuck }
132    if (logDecisions) $.ui.log(logLine(line))
133    $.ui.status(statusLine(line, sparkline(scores)))
134
135    return yield* next(applied ? { ...e, effort: applied } : e)
136  })
137}
138

1/**
2 * jev-effort — pure logic. No `$`, no I/O: context → Jev state, the request,
3 * the answer, the decision, and the log/status formatters.
4 *
5 * Wire shape (OpenRouter and TypeSafe-direct are identical):
6 *   POST {endpoint} { model, state, questions } → { answers: { effort: { score, confidence } } }
7 */
8import type { PromptOrigin, SessionMessage } from 'claude-code'
9
10/** The only values the mod ever sets, cheapest first. Never `max`. */
11export const EFFORTS = ['low', 'medium', 'high', 'xhigh'] as const
12export type Effort = (typeof EFFORTS)[number]
13type SessionEffort = Effort | 'max' | number | undefined
14
15// ponytail: fixed caps; promote to options only if shadow data asks for it.
16export const LIMITS = {
17  promptChars: 2000,
18  replyChars: 1500,
19  earlierPrompts: 3,
20  earlierPromptChars: 300,
21  toolsChars: 300,
22}
23export type Limits = typeof LIMITS
24
25export interface JevState {
26  prompt: string
27  last_reply?: string
28  earlier_prompts?: string[]
29  last_turn_tools?: string
30}
31
32export interface JevAnswer {
33  score: number
34  confidence: number
35}
36
37export interface Bars {
38  up: number
39  down: number
40}
41
42export interface Decision {
43  wanted: Effort | null
44  target: Effort | null
45  reason: string
46}
47
48// ---------------------------------------------------------------- skips
49
50const ROUTED_ORIGINS = new Set(['composer', 'bridge', 'sdk'])
51
52/** Why a submission isn't classified, or null to classify it. */
53export function skipReason(text: string, origin: PromptOrigin, turnId: string | undefined): string | null {
54  if (!ROUTED_ORIGINS.has(origin.kind)) return `origin ${origin.kind}`
55  if (turnId) return 'mid-turn prompt'
56  const t = text.trim()
57  if (!t) return 'empty prompt'
58  // Jev would only see the command's name, never the skill body it expands to.
59  if (/^\/[^\s/]+$/.test(t)) return 'bare command'
60  return null
61}
62
63// ---------------------------------------------------------------- context
64
65const norm = (s: string) => s.replace(/\s+/g, ' ').trim()
66
67/** Head and tail: keeps the task statement and the closing question. */
68export function clip(text: string, max: number): string {
69  if (text.length <= max) return text
70  const head = Math.ceil((max - 3) / 2)
71  const tail = Math.floor((max - 3) / 2)
72  return `${text.slice(0, head)} … ${text.slice(text.length - tail)}`
73}
74
75// ponytail: engine-injected rows (<system-reminder>, <command-name>, ...) are told
76// apart by a leading '<'; a user prompt starting with '<' is missed. Verify live.
77const isPromptRow = (m: SessionMessage) =>
78  m.role === 'user' && !m.toolResults?.length && norm(m.text) !== '' && !norm(m.text).startsWith('<')
79
80/** Tool names only, never inputs or outputs: `Read×3, Edit, Bash×2 (1 failed)`. */
81function summarizeTools(uses: { tool: string; isError?: true }[], max: number): string {
82  const counts = new Map<string, { n: number; failed: number }>()
83  for (const u of uses) {
84    const c = counts.get(u.tool) ?? { n: 0, failed: 0 }
85    c.n += 1
86    if (u.isError) c.failed += 1
87    counts.set(u.tool, c)
88  }
89  const parts = [...counts].map(
90    ([tool, c]) => `${tool}${c.n > 1 ? `×${c.n}` : ''}${c.failed ? ` (${c.failed} failed)` : ''}`,
91  )
92  return clip(parts.join(', '), max)
93}
94
95/** What Jev sees. Reads only role, text, tool names, error flags and whether toolResults exist. */
96export function buildState(
97  prompt: string,
98  messages: readonly SessionMessage[],
99  sendContext: boolean,
100  limits: Limits = LIMITS,
101): JevState {
102  const state: JevState = { prompt: clip(norm(prompt), limits.promptChars) }
103  if (!sendContext) return state
104
105  let end = messages.length
106  const last = messages[end - 1]
107  if (last && last.role === 'user' && norm(last.text) === norm(prompt)) end -= 1
108
109  let reply: string | undefined
110  const uses: { tool: string; isError?: true }[] = []
111  const earlier: string[] = []
112  let inLastTurn = true
113  for (let i = end - 1; i >= 0 && earlier.length < limits.earlierPrompts; i--) {
114    const m = messages[i]!
115    if (isPromptRow(m)) {
116      earlier.unshift(clip(norm(m.text), limits.earlierPromptChars))
117      inLastTurn = false
118      continue
119    }
120    if (m.role !== 'assistant') continue
121    if (reply === undefined && norm(m.text)) reply = clip(norm(m.text), limits.replyChars)
122    if (inLastTurn) uses.unshift(...m.toolUses.map((u) => ({ tool: u.tool, isError: u.isError })))
123  }
124
125  if (reply) state.last_reply = reply
126  if (earlier.length) state.earlier_prompts = earlier
127  if (uses.length) state.last_turn_tools = summarizeTools(uses, limits.toolsChars)
128  return state
129}
130
131// ---------------------------------------------------------------- request / response
132
133const QUESTION = {
134  type: 'score',
135  instructions:
136    'How much step-by-step reasoning does the coding assistant need to do the work the user asks for in `prompt` well? If `prompt` is a short reply such as "yes", "do it" or "continue", rate the work it approves or continues, as described in `last_reply` and `last_turn_tools`. The other fields are earlier conversation, given for context only.',
137  criteria: [
138    'Almost none: a direct answer, a confirmation, a lookup, running one command, or a small change whose exact edit is already spelled out.',
139    'Some: an ordinary, well-specified change or explanation within a few files, where the approach is obvious.',
140    'A lot: a multi-step change across several files, a bug with a clear symptom but unclear cause, or a review that weighs trade-offs.',
141    'As much as possible: open-ended design or architecture, a failure whose cause is still unknown after earlier attempts, or work where a mistake is costly or hard to undo (security, concurrency, data migration, production systems).',
142  ],
143}
144
145export const requestBody = (model: string, state: JevState) =>
146  JSON.stringify({ model, state, questions: { effort: QUESTION } })
147
148export const requestHeaders = (apiKey: string): Record<string, string> => ({
149  'content-type': 'application/json',
150  authorization: `Bearer ${apiKey}`,
151})
152
153const finite = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v)
154
155/** Null on anything unexpected; a missing confidence counts as no answer. */
156export function readAnswer(responseText: string): JevAnswer | null {
157  let body: any
158  try {
159    body = JSON.parse(responseText)
160  } catch {
161    return null
162  }
163  const a = body?.answers?.effort
164  if (!finite(a?.score) || !finite(a?.confidence)) return null
165  return { score: a.score, confidence: a.confidence }
166}
167
168// ---------------------------------------------------------------- decision
169
170export const effortLevel = (score: number): Effort =>
171  EFFORTS[Math.min(EFFORTS.length - 1, Math.max(0, Math.round(score)))]!
172
173const isEffort = (v: unknown): v is Effort => EFFORTS.includes(v as Effort)
174
175/** The session's `/effort` is the baseline: neither floor nor ceiling. */
176export function decide(answer: JevAnswer | null, session: SessionEffort, bars: Bars): Decision {
177  if (!answer) return { wanted: null, target: null, reason: 'no answer' }
178  const wanted = effortLevel(answer.score)
179  const keep = (reason: string): Decision => ({ wanted, target: null, reason })
180  if (session === undefined) return keep('model takes no effort')
181  if (typeof session === 'number') return keep('numeric budget left alone')
182  if (session === 'max') return keep('max left alone')
183  if (!isEffort(session)) return keep(`unknown effort ${session}`)
184  if (wanted === session) return keep(`already ${session}`)
185  const bar = EFFORTS.indexOf(wanted) < EFFORTS.indexOf(session) ? bars.down : bars.up
186  const cmp = `conf ${answer.confidence.toFixed(2)} ${answer.confidence >= bar ? '≥' : '<'} ${bar.toFixed(2)}`
187  return answer.confidence >= bar ? { wanted, target: wanted, reason: cmp } : keep(cmp)
188}
189
190/**
191 * With no answer (failure, skip, no prompt), repeat the previous turn's effort
192 * rather than snap back to the session's: the effort stays steady through an outage.
193 * Only while the user's own `/effort` hasn't changed since that turn.
194 */
195export function sticky(
196  session: SessionEffort,
197  prev: { session: SessionEffort; sent: SessionEffort } | null,
198): Effort | null {
199  if (!prev || prev.session !== session || !isEffort(session) || !isEffort(prev.sent)) return null
200  return prev.sent === session ? null : prev.sent
201}
202
203// ---------------------------------------------------------------- formatting
204
205const showEffort = (e: SessionEffort) => (e === undefined ? 'default' : String(e))
206
207export interface LineInput {
208  shadow: boolean
209  session: SessionEffort
210  answer: JevAnswer | null
211  note: string | null
212  jevMs?: number
213  decision: Decision
214  /** Set when sticky repeated the previous effort. */
215  stuck?: Effort | null
216}
217
218function outcome(i: LineInput): string {
219  const s = showEffort(i.session)
220  if (i.decision.target) return `${s} → ${i.decision.target}`
221  if (i.stuck) return `${s} → ${i.stuck} (sticky)`
222  return `${s} kept`
223}
224
225/** One transcript line per main turn. */
226export function logLine(i: LineInput): string {
227  const head = i.shadow ? '[jev-effort] shadow · ' : '[jev-effort] '
228  if (!i.answer) return `${head}${i.note ?? 'no answer'} · ${outcome(i)}`
229  const a = i.answer
230  const jev = `jev ${a.score.toFixed(2)} → ${i.decision.wanted} (conf ${a.confidence.toFixed(2)}${i.jevMs === undefined ? '' : `, ${Math.round(i.jevMs)}ms`})`
231  const tail = i.decision.target ? '' : `: ${i.decision.reason}`
232  return `${head}${jev} · ${outcome(i)}${tail}`
233}
234
235const BARS = '▁▂▃▄▅▆▇█'
236export const SPARK_LENGTH = 8
237
238/** Recent Jev scores (0..3) as bars, oldest first. */
239export const sparkline = (scores: readonly number[]) =>
240  scores.map((s) => BARS[Math.round((Math.min(3, Math.max(0, s)) / 3) * (BARS.length - 1))]).join('')
241
242const statusHead = (shadow: boolean, glyph: string, spark: string) =>
243  `${shadow ? 'shadow ' : ''}${glyph}${spark ? ` ${spark}` : ''}`
244
245/** While Jev is being asked. */
246export const askingLine = (shadow: boolean, spark: string) => `${statusHead(shadow, '◌', spark)} asking…`
247
248/** The persistent status line: ● answered, ✕ failed, – nothing asked. */
249export function statusLine(i: LineInput, spark: string): string {
250  if (!i.answer) {
251    const quiet = i.note === null || i.note === 'no prompt' || i.note.startsWith('skipped')
252    return `${statusHead(i.shadow, quiet ? '–' : '✕', spark)} ${i.note ?? 'no answer'} · ${outcome(i)}`
253  }
254  const { decision: d, answer: a } = i
255  const why = d.target
256    ? ` ${a.confidence.toFixed(2)}`
257    : d.reason.startsWith('conf')
258      ? ` (wanted ${d.wanted}, ${d.reason.slice(5)})`
259      : ` (${d.reason})`
260  const ms = i.jevMs === undefined ? '' : ` · ${Math.round(i.jevMs)}ms`
261  return `${statusHead(i.shadow, '●', spark)} ${outcome(i)}${why}${ms}`
262}
263
264/** The once-per-session setup line. */
265export function setupLine(o: {
266  apiKey: string
267  shadow: boolean
268  model: string
269  endpoint: string
270  sendContext: boolean
271  bars: Bars
272}): string {
273  if (!o.apiKey) return '[jev-effort] no apiKey set; doing nothing'
274  const host = /^\w+:\/\/([^/]+)/.exec(o.endpoint)?.[1] ?? o.endpoint
275  return `[jev-effort] ready · ${o.shadow ? 'shadow' : 'route'} · ${o.model} via ${host} · context ${o.sendContext ? 'on' : 'off'} · bars ↑${o.bars.up.toFixed(2)} ↓${o.bars.down.toFixed(2)}`
276}
277