ways-judge

1// The relevance gate's Claude Code internal engine (ADR-708).
2//
3// The prompt lane's gate runs in the settings UserPromptSubmit hooks, which
4// `next(e)` runs at `prompt.submit`. Under `gate.engine: claude-code` the
5// ways scan there writes the judge request as a job in the session's
6// directory and waits for the answer. This module carries the job: it
7// starts `next(e)` without awaiting it and, alongside, waits on
8// `ways judge take`. On a job it sends the request through
9// `$.model.complete` unchanged and hands the reply's text and usage, or the
10// failure reason, to `ways judge answer`. The hook side parses and
11// validates the reply; this module never reads it. On `standby` (the
12// session resolved to a key engine) it calls no model. When `next(e)`
13// settles first, the take is stopped with the stream's `return()`, since
14// `$.process.run` has no abort (ADR-708 §5, ADR-709).
15//
16// The module registers itself as the session's judge host before each
17// session id's first prompt reaches `next(e)`, not at `session.start`: a
18// /clear or an in-session /resume changes the id with no `session.start`
19// (§3). It hooks no classic.* event, which an organization's policy plugin
20// sends past the plugins a person installs.
21
22import type { EngineInterface, HookStream, ProcessSpawnChunk, ProcessSpawnResult, PromptSubmitInput, PromptSubmitResult, Register } from 'claude-code'
23
24import {
25  ABORTED,
26  REFUSED,
27  STOP_GRACE_MS,
28  VERB_TIMEOUT_MS,
29  answerArgv,
30  answerOf,
31  hostArgv,
32  parseTake,
33  requestOf,
34  takeArgv,
35} from './lib.ts'
36import type { JudgeAnswer, JudgeJob } from './lib.ts'
37
38type Next = (e: PromptSubmitInput) => Promise<PromptSubmitResult>
39type TakeStream = HookStream<ProcessSpawnChunk, ProcessSpawnResult>
40
41const debug = ($: EngineInterface, text: string) => $.ui.log(`ways-judge: ${text}`, { to: 'debug' })
42
43export const register: Register = on => {
44  // Each session id this module has registered as judge host, and whether
45  // `ways judge host` took it. A reload starts the map over and registers
46  // again, which the verb takes as the same registration.
47  const hosted = new Map<string, boolean>()
48
49  // Every prompt, whatever its origin: the settings hook that writes the job
50  // cannot see the origin, so the mod takes for each one. A take with no
51  // job costs one spawn, stopped when next(e) settles.
52  on('prompt.submit', async ($, e, next) => {
53    let id: string
54    try {
55      id = await $.session.id()
56    } catch (err) {
57      debug($, `no session id, so no judge for this prompt: ${String(err)}`)
58      return next(e)
59    }
60    if (!(await host($, hosted, id))) return next(e)
61    return bracket($, id, e, next)
62  })
63}
64
65/** Registers the session id as judge host once: a host that registers
66 *  after the session resolved stays idle until the next session (§2), so
67 *  a failure is not retried under the same id. */
68async function host($: EngineInterface, hosted: Map<string, boolean>, id: string): Promise<boolean> {
69  const known = hosted.get(id)
70  if (known !== undefined) return known
71  let isHost = false
72  try {
73    const r = await $.process.run(hostArgv(id), { timeoutMs: VERB_TIMEOUT_MS })
74    isHost = r.exitCode === 0
75    if (!isHost) debug($, `ways judge host failed (${r.exitCode}): ${r.stderr.trim()}`)
76  } catch (err) {
77    debug($, `ways judge host did not run: ${String(err)}`)
78  }
79  hosted.set(id, isHost)
80  return isHost
81}
82
83/** Runs `next(e)` with `ways judge take` beside it, and serves the job the
84 *  take hands over. Resolves to what `next(e)` resolved to. */
85async function bracket($: EngineInterface, id: string, e: PromptSubmitInput, next: Next): Promise<PromptSubmitResult> {
86  const settled = next(e)
87  const take = $.process.spawn({ argv: takeArgv(id) })
88  const stop = new AbortController()
89  const served = serve($, id, take, stop.signal)
90  try {
91    return await settled
92  } finally {
93    // The prompt went on: a take still waiting has nothing to wait for, and
94    // a model call still running has no reader.
95    stop.abort()
96    await Promise.race([take.return(undefined as never).catch(() => undefined), $.clock.sleep(STOP_GRACE_MS)])
97    await served
98  }
99}
100
101/** Reads the take to its end; on a job, judges it and writes the answer. */
102async function serve($: EngineInterface, id: string, take: TakeStream, stopped: AbortSignal): Promise<void> {
103  let out = ''
104  let code: number | null = null
105  try {
106    for (;;) {
107      const r = await take.next()
108      if (r.done) {
109        code = r.value?.code ?? null
110        break
111      }
112      if (r.value.stream === 'stdout') out += r.value.text
113    }
114  } catch (err) {
115    if (!stopped.aborted) debug($, `ways judge take did not run: ${String(err)}`)
116    return
117  }
118  const t = parseTake(out, code)
119  if (t.kind === 'none' && !stopped.aborted) debug($, t.why)
120  if (t.kind !== 'job') return
121  // A claimed job is always answered, so its gate fails open at once rather
122  // than at its deadline; once the prompt went on there is no call to make.
123  const answer = stopped.aborted ? ABORTED : await judge($, t.job, stopped)
124  try {
125    const w = await $.process.run(answerArgv(id, t.job.id), { stdin: JSON.stringify(answer), timeoutMs: VERB_TIMEOUT_MS })
126    if (w.exitCode !== 0) debug($, `ways judge answer failed (${w.exitCode}): ${w.stderr.trim()}`)
127  } catch (err) {
128    debug($, `ways judge answer did not run: ${String(err)}`)
129  }
130}
131
132async function judge($: EngineInterface, job: JudgeJob, stopped: AbortSignal): Promise<JudgeAnswer> {
133  try {
134    return answerOf(await $.model.complete(requestOf(job), { signal: stopped }))
135  } catch (err) {
136    debug($, `the model call was refused: ${String(err)}`)
137    return REFUSED
138  }
139}
140

1// The ways-judge mod's plain parts: the ways CLI's argv, what `ways judge
2// take` prints, and the answer `ways judge answer` reads (ADR-708 §4, §5).
3// The shapes mirror JudgeJob and JudgeAnswer in
4// tools/ways-agent-core/src/protocol.rs.
5
6import type { ModelCompleteRequest, ModelCompleteResult, ModelUsage } from 'claude-code'
7
8export const WAYS = 'ways'
9
10/** Registers the mod as the session's judge host, once per session id. */
11export function hostArgv(id: string): string[] {
12  return [WAYS, 'judge', 'host', '--session', id]
13}
14
15/** How long a take may wait: the prompt hook's own timeout, so a take whose
16 *  stop was missed cannot outlive the prompt that started it. */
17export const TAKE_WAIT_MS = 30_000
18
19/** Waits for the session's next job; prints it, or `standby`. */
20export function takeArgv(id: string): string[] {
21  return [WAYS, 'judge', 'take', '--session', id, '--wait', '--timeout-ms', String(TAKE_WAIT_MS)]
22}
23
24/** Reads a JudgeAnswer as JSON on stdin, for the job `take` printed when it
25 *  carried an id, so an answer can never land on a newer job. */
26export function answerArgv(id: string, job?: string): string[] {
27  const argv = [WAYS, 'judge', 'answer', '--session', id]
28  return job ? [...argv, '--job', job] : argv
29}
30
31/** How long the host and answer verbs may run. Each is a file write. */
32export const VERB_TIMEOUT_MS = 5_000
33
34/** How long stopping `take` may hold the prompt once `next(e)` settled. */
35export const STOP_GRACE_MS = 3_000
36
37/** A judge request, as `ways judge take` prints it. */
38export type JudgeJob = {
39  id?: string
40  model: string
41  system: string
42  prompt: string
43  max_tokens: number
44  timeout_ms: number
45}
46
47/** What `ways judge answer` reads: the model's text and usage, or why there
48 *  is none. The mod never reads the verdict in the text. */
49export type JudgeAnswer = {
50  text?: string
51  usage?: ModelUsage
52  reason?: string
53}
54
55export type Take = { kind: 'job'; job: JudgeJob } | { kind: 'standby' } | { kind: 'none'; why: string }
56
57/** What `take` printed and how it exited: a job, `standby`, or neither. */
58export function parseTake(stdout: string, code: number | null): Take {
59  const out = stdout.trim()
60  if (code !== 0) return { kind: 'none', why: `take exited ${code ?? 'on a signal'}` }
61  if (out === 'standby') return { kind: 'standby' }
62  let v: unknown
63  try {
64    v = JSON.parse(out)
65  } catch {
66    return { kind: 'none', why: `take printed neither a job nor standby: ${out.slice(0, 80)}` }
67  }
68  return isJob(v) ? { kind: 'job', job: v } : { kind: 'none', why: 'take printed a job without its fields' }
69}
70
71function isJob(v: unknown): v is JudgeJob {
72  if (typeof v !== 'object' || v === null) return false
73  const j = v as Record<string, unknown>
74  return (
75    typeof j.model === 'string' &&
76    typeof j.system === 'string' &&
77    typeof j.prompt === 'string' &&
78    typeof j.max_tokens === 'number' &&
79    typeof j.timeout_ms === 'number'
80  )
81}
82
83/** The job as `$.model.complete` takes it, unchanged. */
84export function requestOf(job: JudgeJob): ModelCompleteRequest {
85  return { model: job.model, system: job.system, prompt: job.prompt, maxTokens: job.max_tokens, timeoutMs: job.timeout_ms }
86}
87
88/** The model's result as the answer carries it: text and usage when it
89 *  answered, else the reason (`aborted` for a call past `timeoutMs`). */
90export function answerOf(r: ModelCompleteResult): JudgeAnswer {
91  return r.isAnswered ? { text: r.text, usage: r.usage } : { reason: r.reason, usage: r.usage }
92}
93
94/** The answer for a call the engine refused to send (a blocked model, a
95 *  cap or a time limit it does not take). */
96export const REFUSED: JudgeAnswer = { reason: 'error' }
97
98/** The answer for a job claimed after the prompt went on. */
99export const ABORTED: JudgeAnswer = { reason: 'aborted' }
100