auto-effort

1// auto-effort: sets Claude Code's effort once per prompt. Jev (TypeSafe's System One model)
2// rates how demanding the request is (0-3) at turn.start; the policy in policy.ts smooths that
3// into a level, and turn.step sends the turn's model requests at that level. The session's own
4// effort setting (/effort, effortLevel) is the ceiling. Subagent loops are left alone, and the
5// level never changes inside a turn, so its tool follow-ups keep their prompt cache.
6//
7// The host reads on(...) and $.noun.method(...) from source, so calls are spelled in full and
8// helpers that take $ are top-level functions in this file.
9import type { EngineInterface, On } from 'claude-code'
10
11import { DEFAULT_POLICY, type EffortState, LEVELS, decide, levelIndex } from './policy'
12import { ENDPOINT, type MessageLike, QUESTIONS, effortState, readJudgment } from './request'
13import { DEFAULT_SETTINGS, type Settings, resolveSettings, settingsFiles } from './settings'
14
15const COMMAND = 'auto-effort'
16
17type JevOutcome =
18  | { ok: true; judgment: { score: number; confidence: number; ack: number }; latencyMs: number }
19  | { ok: false; reason: string }
20
21export function register(on: On): void {
22  let settings: Settings = DEFAULT_SETTINGS
23  let sessionOn = true
24  let state: EffortState = { dwell: DEFAULT_POLICY.minDwell }
25  // The level auto-effort set last; undefined before the first prompt.
26  let level: string | undefined
27  // The session's own effort, read off the main loop's last step: the ceiling.
28  let ceiling = 'xhigh'
29  let lastReason = ''
30
31  on('session.start', async ($, e, next) => {
32    const result = await next(e)
33    await $.command.register({
34      name: COMMAND,
35      description: 'auto-effort: status, on, or off (this session)',
36      argumentHint: '[status|on|off]',
37      immediate: true,
38    })
39    return result
40  })
41
42  on('turn.start', async ($, e, next) => {
43    if (e.text.trim()) settings = await loadSettings($)
44    if (sessionOn && settings.enabled && e.text.trim()) {
45      const messages = (await $.session.messages()) as readonly MessageLike[]
46      const outcome = await askJev($, effortState(e.text, messages), settings)
47      const current = level ?? ceiling
48      const decision = decide(current, ceiling, state, outcome.ok ? outcome.judgment : undefined, settings.policy)
49      state = decision.state
50      level = decision.level
51      lastReason = outcome.ok ? decision.reason : `unavailable: ${outcome.reason}`
52      const detail = outcome.ok
53        ? ` (score ${outcome.judgment.score.toFixed(2)}, confidence ${outcome.judgment.confidence.toFixed(2)}, go-ahead ${outcome.judgment.ack.toFixed(2)}, ${outcome.latencyMs} ms)`
54        : ''
55      $.ui.log(`${current} -> ${level}: ${lastReason}${detail}`, { to: 'debug' })
56      $.ui.status(`effort: ${applied(level, ceiling)} (auto)`)
57    }
58    return next(e)
59  })
60
61  on('turn.step', async function* ($, e, next) {
62    // Subagents, a budget instead of a level, and a model without effort pass untouched.
63    if (e.agentId !== undefined || typeof e.effort !== 'string' || levelIndex(e.effort) < 0) {
64      return yield* next(e)
65    }
66    ceiling = e.effort
67    if (!sessionOn || !settings.enabled || level === undefined) return yield* next(e)
68    return yield* next({ ...e, effort: applied(level, ceiling) as (typeof LEVELS)[number] })
69  })
70
71  on('command.run', { command: COMMAND }, async ($, e) => {
72    const arg = e.args.trim()
73    if (arg === 'on' || arg === 'off') {
74      sessionOn = arg === 'on'
75      $.ui.status(sessionOn && settings.enabled && level !== undefined ? `effort: ${applied(level, ceiling)} (auto)` : undefined)
76      return { text: `auto-effort ${arg} for this session${sessionOn ? '' : `; back to ${ceiling}`}` }
77    }
78    const active = sessionOn && settings.enabled
79    return {
80      text: [
81        `auto-effort ${active ? 'on' : settings.enabled ? 'off' : 'off (settings)'}: ${active && level !== undefined ? applied(level, ceiling) : ceiling} now, ceiling ${ceiling} (the session's effort), floor ${settings.policy.floor}.`,
82        `Running average ${state.e === undefined ? '-' : state.e.toFixed(2)}, ${state.dwell} prompt${state.dwell === 1 ? '' : 's'} at this level${lastReason ? `, last decision: ${lastReason}` : ''}.`,
83      ].join('\n'),
84    }
85  })
86}
87
88/** The level a step is sent at: auto-effort's, never above the session's own. */
89function applied(level: string, ceiling: string): string {
90  const i = levelIndex(level)
91  const c = levelIndex(ceiling)
92  return i < 0 || c < 0 || i > c ? ceiling : level
93}
94
95/** The shared settings files merged over the defaults (settings.ts); a missing file is skipped. */
96async function loadSettings($: EngineInterface): Promise<Settings> {
97  const home = (await $.env.get('HOME')) ?? ''
98  const xdg = await $.env.get('XDG_CONFIG_HOME')
99  const root = await $.session.root()
100  const texts: (string | undefined)[] = []
101  for (const file of settingsFiles(home, xdg, root)) {
102    texts.push(await $.fs.read(file).catch(() => undefined))
103  }
104  return resolveSettings(texts)
105}
106
107/** One System One request. Never throws: every failure is `{ ok: false, reason }`. */
108async function askJev($: EngineInterface, jevState: Record<string, unknown>, settings: Settings): Promise<JevOutcome> {
109  const key = (await $.env.get('TYPESAFE_API_KEY'))?.trim()
110  if (!key) return { ok: false, reason: 'TYPESAFE_API_KEY is not set' }
111  const started = Date.now()
112  const request = $.http
113    .fetch(ENDPOINT, {
114      method: 'POST',
115      headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
116      body: JSON.stringify({ model: settings.jev.model, state: jevState, questions: QUESTIONS }),
117    })
118    .then(
119      (response) => response,
120      (error: unknown) => (error instanceof Error ? error.message : String(error)),
121    )
122  const timeout = $.clock.sleep(settings.jev.timeoutMs).then(() => 'timed out')
123  const response = await Promise.race([request, timeout])
124  if (typeof response === 'string') return { ok: false, reason: response.slice(0, 160) }
125  if (!response.ok) {
126    return { ok: false, reason: response.status === 401 ? 'invalid API key' : `HTTP ${response.status}` }
127  }
128  let body: unknown
129  try {
130    body = JSON.parse(response.text)
131  } catch {
132    return { ok: false, reason: 'unreadable response' }
133  }
134  const judgment = readJudgment(body)
135  if (!judgment) return { ok: false, reason: 'response without depth and go-ahead answers' }
136  return { ok: true, judgment, latencyMs: Date.now() - started }
137}
138

1// The level policy, ported unchanged from pi-auto-effort: a running average of Jev's depth
2// scores, a margin before moving, a jump rule for clearly harder work, a minimum dwell before
3// going down, and a go-ahead ("yes", "continue") that keeps the level.
4
5/** Effort levels auto-effort moves between, lowest first; the index is the depth score. */
6export const LEVELS = ['low', 'medium', 'high', 'xhigh', 'max'] as const
7export type Level = (typeof LEVELS)[number]
8
9export interface Policy {
10  /** Lowest level auto-effort sets. */
11  floor: Level
12  /** Weight of the newest score in the running average (1 = no smoothing). */
13  alpha: number
14  /** The average must differ from the current level by this much to move. */
15  margin: number
16  /** A single score this far above the current level jumps straight to it... */
17  jump: number
18  /** ...when Jev is at least this confident. */
19  jumpConfidence: number
20  /** Messages a level must hold before it can go down. */
21  minDwell: number
22  /** A go-ahead message above this probability keeps the level. */
23  ackThreshold: number
24}
25
26export const DEFAULT_POLICY: Policy = {
27  floor: 'low',
28  alpha: 0.5,
29  margin: 0.6,
30  jump: 1.5,
31  jumpConfidence: 0.6,
32  minDwell: 2,
33  ackThreshold: 0.7,
34}
35
36export interface EffortState {
37  /** Running average of depth scores; undefined before the first judgment. */
38  e?: number
39  /** Messages since the level last changed. */
40  dwell: number
41}
42
43export interface Judgment {
44  /** Depth score 0-3. */
45  score: number
46  confidence: number
47  /** Probability the message is a go-ahead relying on the earlier plan. */
48  ack: number
49}
50
51export interface Decision {
52  level: string
53  state: EffortState
54  reason: 'ack' | 'jump' | 'up' | 'down' | 'hold' | 'unavailable'
55}
56
57export function levelIndex(level: string): number {
58  return LEVELS.indexOf(level as Level)
59}
60
61/**
62 * The next level for a message. `current` is the level auto-effort set last (or the ceiling
63 * before the first message); `ceiling` is the session's own effort setting. Levels outside the
64 * managed range, or a ceiling below the floor, are left alone.
65 */
66export function decide(
67  current: string,
68  ceilingLevel: string,
69  state: EffortState,
70  judgment: Judgment | undefined,
71  policy: Policy,
72): Decision {
73  const floor = levelIndex(policy.floor)
74  const ceiling = levelIndex(ceilingLevel)
75  const c = Math.min(levelIndex(current), ceiling)
76  if (c < 0 || ceiling < 0 || ceiling < floor) {
77    return { level: ceilingLevel, state: { ...state, dwell: state.dwell + 1 }, reason: 'hold' }
78  }
79  const held = LEVELS[c] as string
80  if (!judgment) return { level: held, state: { ...state, dwell: state.dwell + 1 }, reason: 'unavailable' }
81  const s = judgment.score
82  const e = state.e === undefined ? s : policy.alpha * s + (1 - policy.alpha) * state.e
83  let target = c
84  let reason: Decision['reason'] = 'hold'
85  if (judgment.ack > policy.ackThreshold) reason = 'ack'
86  else if (s - c >= policy.jump && judgment.confidence >= policy.jumpConfidence) {
87    target = Math.round(s)
88    reason = 'jump'
89  } else if (e - c >= policy.margin) {
90    target = Math.round(e)
91    reason = 'up'
92  } else if (c - e >= policy.margin && state.dwell >= policy.minDwell) {
93    target = Math.round(e)
94    reason = 'down'
95  }
96  target = Math.min(Math.max(target, floor), ceiling)
97  if (target === c) return { level: held, state: { e, dwell: state.dwell + 1 }, reason: reason === 'ack' ? 'ack' : 'hold' }
98  return { level: LEVELS[target] as string, state: { e, dwell: 0 }, reason }
99}
100

1// The Jev request for a new prompt: the questions, the state built from the transcript, and
2// reading the answers. Pure functions; the HTTP call is in register.ts, where `$` is.
3
4export const ENDPOINT = 'https://api.typesafe.ai/v1/systemone'
5
6export const QUESTIONS = {
7  depth: {
8    type: 'score',
9    instructions:
10      'How much careful reasoning does the work asked for in `request` need from a coding agent, given `previous_requests`, `last_outcome` and `signals`?',
11    criteria: [
12      'A lookup, a trivial answer, or a mechanical change',
13      'A routine single-file change or a clear question',
14      'A multi-step change, debugging with clear symptoms, or design inside a known pattern',
15      'Subtle design, a cross-cutting refactor, hard debugging, or correctness-critical work',
16    ],
17  },
18  ack: {
19    type: 'noul',
20    instructions:
21      'Is `request` a short go-ahead or acknowledgement (like "yes", "go ahead", "continue", "do it") that relies on the plan already discussed, rather than a new task?',
22    criteria: { true: 'A go-ahead for work already under way', false: 'A new or changed request' },
23  },
24} as const
25
26const EDIT_TOOLS = new Set(['Edit', 'Write', 'MultiEdit', 'NotebookEdit'])
27
28/** The part of a SessionMessage this reads. */
29export interface MessageLike {
30  role: 'user' | 'assistant'
31  text: string
32  toolUses?: readonly { tool: string; isError?: true }[]
33  toolResults?: readonly { isError?: true }[]
34}
35
36/** The Jev state for a new prompt: the request, the two before it, the last answer, and what the last run did. */
37export function effortState(prompt: string, messages: readonly MessageLike[]): Record<string, unknown> {
38  // A user message carrying tool results is the engine's, not a prompt.
39  const prompts: number[] = []
40  messages.forEach((m, i) => {
41    if (m.role === 'user' && !m.toolResults?.length && m.text.trim()) prompts.push(i)
42  })
43  // The new prompt is usually already in the transcript; the previous run is before it.
44  const lastPrompt = prompts.at(-1)
45  if (lastPrompt !== undefined && messages[lastPrompt]?.text.trim() === prompt.trim()) prompts.pop()
46  const runStart = prompts.at(-1)
47  const previous = prompts.slice(-2).map((i) => clip(messages[i]?.text.trim() ?? '', 1_000))
48  let lastOutcome = ''
49  const signals = { last_run_errors: 0, files_edited: 0, tool_calls: 0 }
50  if (runStart !== undefined) {
51    for (const m of messages.slice(runStart + 1)) {
52      if (m.role === 'user') {
53        if (!m.toolResults?.length && m.text.trim()) break
54        signals.last_run_errors += m.toolResults?.filter((r) => r.isError).length ?? 0
55        continue
56      }
57      if (m.text.trim()) lastOutcome = m.text.trim()
58      for (const use of m.toolUses ?? []) {
59        signals.tool_calls++
60        if (EDIT_TOOLS.has(use.tool) && !use.isError) signals.files_edited++
61      }
62    }
63  }
64  return {
65    request: clip(prompt, 8_000),
66    previous_requests: previous.length ? previous : ['(none: this is the first request)'],
67    last_outcome: lastOutcome ? clip(lastOutcome, 1_500) : '(none)',
68    signals,
69  }
70}
71
72/** Depth and go-ahead from a System One response body; undefined when either is missing. */
73export function readJudgment(body: unknown): { score: number; confidence: number; ack: number } | undefined {
74  const answers = (body as { answers?: Record<string, Record<string, unknown>> } | undefined)?.answers
75  const depth = answers?.depth
76  const ack = answers?.ack
77  if (depth?.type !== 'score' || ack?.type !== 'noul') return undefined
78  const score = Number(depth.score)
79  const confidence = Number(depth.confidence)
80  const yes = Number(ack.noul)
81  if (![score, confidence, yes].every(Number.isFinite)) return undefined
82  return { score, confidence, ack: yes }
83}
84
85export function clip(text: string, max: number): string {
86  return text.length > max ? `${text.slice(0, max - 1)}…` : text
87}
88

1// Settings shared with the pi and opencode ports: `~/.config/agents/auto-effort.json` (or under
2// $XDG_CONFIG_HOME) and `<project>/.agents/auto-effort.json`, with Claude Code's own
3// `~/.claude/auto-effort.json` and `<project>/.claude/auto-effort.json` as overrides. Later files
4// win; objects merge, other values replace; keys this port does not know are ignored.
5import { DEFAULT_POLICY, type Policy } from './policy'
6
7export const NAME = 'auto-effort'
8
9export interface Settings {
10  enabled: boolean
11  jev: { model: string; timeoutMs: number }
12  policy: Policy
13}
14
15export const DEFAULT_SETTINGS: Settings = {
16  enabled: true,
17  jev: { model: 'jev-latest', timeoutMs: 1_500 },
18  policy: DEFAULT_POLICY,
19}
20
21/** The settings files, lowest precedence first. */
22export function settingsFiles(home: string, xdgConfigHome: string | undefined, root: string): string[] {
23  const shared = xdgConfigHome || `${home}/.config`
24  return [
25    `${shared}/agents/${NAME}.json`,
26    `${home}/.claude/${NAME}.json`,
27    `${root}/.agents/${NAME}.json`,
28    `${root}/.claude/${NAME}.json`,
29  ]
30}
31
32/** The defaults with each file's JSON merged on top in order; unreadable or invalid text is skipped. */
33export function resolveSettings(texts: readonly (string | undefined)[]): Settings {
34  let merged: Record<string, unknown> = structuredClone(DEFAULT_SETTINGS) as unknown as Record<string, unknown>
35  for (const text of texts) {
36    if (text === undefined) continue
37    try {
38      const value: unknown = JSON.parse(text)
39      if (isRecord(value)) merged = merge(merged, value)
40    } catch {
41      // Invalid JSON: keep what the earlier files said.
42    }
43  }
44  return merged as unknown as Settings
45}
46
47function merge(base: Record<string, unknown>, over: Record<string, unknown>): Record<string, unknown> {
48  const out: Record<string, unknown> = { ...base }
49  for (const [key, value] of Object.entries(over)) {
50    const current = out[key]
51    out[key] = isRecord(current) && isRecord(value) ? merge(current, value) : value
52  }
53  return out
54}
55
56function isRecord(value: unknown): value is Record<string, unknown> {
57  return typeof value === 'object' && value !== null && !Array.isArray(value)
58}
59