decision-watch

1// decision-watch: runs the Stop-time decision watch (hooks/decision_watch.py) with the session's
2// own model login, and writes the verdict to a log. A FLAG or UNKNOWN verdict shows in the session
3// with $.ui.toast. An ALLOW verdict shows nothing. Each config notice shows as its own toast, marked
4// as a notice (decision decision-watch-mod-limits). It never blocks and never changes the Stop result.
5//
6// classic.Stop  returns next(e) at once. The judging runs in the background and is skipped when
7//               stop_hook_active is set.
8// /dw           runs the same judgment on the latest turn and shows the result.
9//
10// The python script has two phases: --emit-prompt prints the prompt when a model call is due,
11// then --model-text or --model-error finishes the verdict. This mod makes the model call between.
12// Phase 1 prints a "key" for the evidence it judged. Phase 2 gets it back as --key after the reply,
13// so evidence that moved between the phases reads UNKNOWN, never a silent ALLOW. No key reads UNKNOWN.
14// The script prints `verdict` and `notices` apart, so the mod never parses joined text.
15// The log holds the verdict line, the notices and timings, never the prompt.
16//
17// The host reads on(...) and $.noun.method(...) from source, so they are spelled literally.
18
19let lastStop; // the last classic.Stop input this session, for /dw
20let busy = false; // true while a background judge runs
21
22// The prefix of every UNKNOWN this mod makes itself; python's own UNKNOWN text comes through as printed.
23const UNKNOWN = "Decision watch: UNKNOWN, ";
24// The mark on a config notice toast. It never matches a FLAG or UNKNOWN verdict.
25const NOTICE = "Decision watch notice: ";
26const NOTHING = "nothing to say (no protected change, no model call)";
27const msg = (err) => String((err && err.message) || err).slice(0, 300);
28// $.model.complete returns {isAnswered, text, usage}. Accept a plain string too.
29const textOf = (r) => (typeof r === "string" ? r : String((r && r.text) ?? ""));
30
31export function register(on) {
32  on("session.start", async ($, e, next) => {
33    await $.command.register({
34      name: "dw",
35      description: "Run the decision watch on the latest turn with this session's own model login",
36    });
37    return next(e);
38  });
39
40  on("command.run", { command: "dw" }, async ($, e) => {
41    const lines = [];
42    try {
43      const stop = lastStop ?? (await deriveStop($));
44      lines.push(`stop input: session ${stop.session_id}`);
45      const out = await judge($, stop, "dw");
46      lines.push(...out.trace, `RESULT: ${out.result}`, ...out.notices.map((n) => `NOTICE: ${n}`));
47    } catch (err) {
48      lines.push(`RESULT: UNKNOWN (mod error): ${msg(err)}`);
49    }
50    lines.push(`Stops seen: ${await stopsSeen($)}`);
51    return { text: lines.join("\n") };
52  });
53
54  // Background judge. Waits at most STOP_WAIT_MS for one "STOP seen" line, then returns next(e).
55  // The judging is fire-and-forget, one judge at a time. A Stop that finds one running logs "SKIP busy".
56  on("classic.Stop", async ($, e, next) => {
57    lastStop = stopOf(e);
58    const note = e.stop_hook_active ? " (stop_hook_active, no judge)" : "";
59    try {
60      await within(log($, `STOP seen ${e.session_id}${note}`), STOP_WAIT_MS);
61    } catch {
62      // the log never holds a Stop
63    }
64    if (!e.stop_hook_active) {
65      if (busy) {
66        void log($, `SKIP busy ${e.session_id}`);
67      } else {
68        busy = true;
69        void shadow($, lastStop).finally(() => {
70          busy = false;
71        });
72      }
73    }
74    return next(e);
75  });
76}
77
78function stopOf(e) {
79  return {
80    session_id: e.session_id,
81    transcript_path: e.transcript_path,
82    cwd: e.cwd,
83    hook_event_name: "Stop",
84    stop_hook_active: false,
85  };
86}
87
88async function configDir($) {
89  const given = await $.env.get("CLAUDE_CONFIG_DIR");
90  if (given) return given;
91  const home = (await $.env.get("USERPROFILE")) || (await $.env.get("HOME"));
92  return `${home}/.claude`;
93}
94
95// No Stop seen yet: rebuild transcript_path from the session id and cwd the way the app names it.
96async function deriveStop($) {
97  const id = await $.session.id();
98  const cwd = await $.session.cwd();
99  const path = `${await configDir($)}/projects/${cwd.replace(/[^A-Za-z0-9]/g, "-")}/${id}.jsonl`;
100  if (!(await $.fs.exists(path))) throw new Error("no Stop seen yet and no transcript found for this session");
101  return { session_id: id, transcript_path: path, cwd, hook_event_name: "Stop", stop_hook_active: false };
102}
103
104// First python on PATH that is not the WindowsApps Store stub (settings.json's launcher does the same).
105async function pickPython($) {
106  for (const argv of [["where.exe", "python3", "python"], ["which", "-a", "python3", "python"]]) {
107    try {
108      const r = await deadline($.process.run(argv, { timeoutMs: FIND_MS }), FIND_MS + 1000);
109      const hit = r.stdout.split(/\r?\n/).map((s) => s.trim()).find((s) => s && !/WindowsApps/i.test(s));
110      if (r.exitCode === 0 && hit) return hit;
111    } catch {
112      // try the next finder
113    }
114  }
115  throw new Error("no python on PATH other than the WindowsApps stub");
116}
117
118// ASCII-only JSON: python -I may read stdin in a legacy code page, so escape every non-ASCII character.
119const asciiJson = (v) => JSON.stringify(v).replace(/[\u0080-￿]/g, (c) => "\\u" + c.charCodeAt(0).toString(16).padStart(4, "0"));
120
121// The background judge and /dw each get their own state folder, so neither mutes the other's
122// "already judged" cache.
123const stateDir = (cfg, who) => `${cfg}/decision-watch-${who}`;
124
125async function dwRun($, stop, extra, who) {
126  const cfg = await configDir($);
127  const py = await pickPython($);
128  const t0 = Date.now();
129  // The mod's own timer frees the slot if the host does not stop the child. It does not kill the child.
130  const r = await deadline($.process.run([py, "-I", `${cfg}/hooks/decision_watch.py`, ...extra], {
131    stdin: asciiJson(stop),
132    env: { CLAUDE_CONFIG_DIR: stateDir(cfg, who), DW_REAL_CONFIG_DIR: cfg },
133    timeoutMs: PY_MS,
134  }), PY_MS + 1000);
135  return { ...r, ms: Date.now() - t0 };
136}
137
138// Two phases. Every failure is UNKNOWN, never a pass: the python side prints the same
139// "Decision watch: UNKNOWN ..." text for a model error it is handed with --model-error.
140async function judge($, stop, who) {
141  const trace = [];
142  const p1 = await dwRun($, stop, ["--emit-prompt"], who);
143  trace.push(`phase 1 (python, cheap path): exit ${p1.exitCode}, ${p1.ms} ms`);
144  if (p1.exitCode !== 0) return unknown(trace, `decision_watch exited ${p1.exitCode}: ${p1.stderr.slice(0, 200)}`);
145  const raw = p1.stdout.trim();
146  if (raw === "") return { trace, result: NOTHING, shown: "", notices: [] };
147  let out;
148  try {
149    out = JSON.parse(raw);
150  } catch {
151    return unknown(trace, `phase 1 output was not JSON: ${raw.slice(0, 200)}`);
152  }
153  if (!out.needs_model) {
154    const { verdict, notices } = partsOf(out);
155    const result = verdict ? `message from the cheap path: ${verdict}` : notices.length ? NOTHING : `message from the cheap path: ${raw.slice(0, 300)}`;
156    return { trace, result, shown: verdict, notices };
157  }
158  if (typeof out.key !== "string" || out.key === "") {
159    return unknown(trace, "phase 1 printed no key, so the reply cannot be tied to the evidence judged (an old decision_watch.py?)");
160  }
161
162  trace.push(`model due: ${out.model}, prompt ${out.prompt.length} chars`);
163  let args;
164  let modelFailed = false;
165  const t0 = Date.now();
166  try {
167    // The host's timeoutMs is unproven, so the mod enforces the limit itself: the race rejects, phase 2
168    // gets --model-error, and the one-judge slot frees. A late reply is dropped.
169    const reply = await deadline($.model.complete({ model: out.model, prompt: out.prompt, maxTokens: 512, timeoutMs: MODEL_MS }), MODEL_MS);
170    const text = textOf(reply);
171    if (text === "") throw new Error("the model reply carried no text");
172    trace.push(`model call: OK ${Date.now() - t0} ms, reply ${text.length} chars`);
173    args = ["--model-text", text.slice(0, 4000)];
174  } catch (err) {
175    modelFailed = true;
176    trace.push(`model call: FAIL ${Date.now() - t0} ms`);
177    args = ["--model-error", `model call: ${msg(err)}`];
178  }
179  const p2 = await dwRun($, stop, [...args, "--key", out.key], who);
180  trace.push(`phase 2 (python, verdict): exit ${p2.exitCode}, ${p2.ms} ms`);
181  if (p2.exitCode !== 0) return unknown(trace, `decision_watch exited ${p2.exitCode} in phase 2: ${p2.stderr.slice(0, 200)}`);
182  let parts = { verdict: "", notices: [] };
183  try {
184    if (p2.stdout.trim() !== "") parts = partsOf(JSON.parse(p2.stdout));
185  } catch {
186    return unknown(trace, `phase 2 output was not JSON: ${p2.stdout.slice(0, 200)}`);
187  }
188  if (parts.verdict === "" && modelFailed) return unknown(trace, "the model call failed and the script printed no verdict", parts.notices);
189  return { trace, result: parts.verdict === "" ? "ALLOW (silent)" : parts.verdict, shown: parts.verdict, notices: parts.notices };
190}
191
192// The script prints `verdict` and `notices` under a split flag. Output with no `notices` array
193// (a script that prints systemMessage only) is all verdict.
194function partsOf(o) {
195  if (Array.isArray(o.notices)) {
196    return { verdict: typeof o.verdict === "string" ? o.verdict : "", notices: o.notices.filter((n) => typeof n === "string" && n) };
197  }
198  return { verdict: typeof o.systemMessage === "string" ? o.systemMessage : "", notices: [] };
199}
200
201// A failure the mod itself sees is UNKNOWN, and it shows like the script's own UNKNOWN.
202function unknown(trace, why, notices = []) {
203  return { trace, result: `UNKNOWN: ${why}`, shown: UNKNOWN + why, notices };
204}
205
206async function shadow($, stop) {
207  await log($, `START ${stop.session_id}`);
208  let line;
209  let shown = "";
210  let notices = [];
211  try {
212    const out = await judge($, stop, "shadow");
213    // Each notice goes on its own log lines, before END, so the END cut at MAX_LINE never drops one.
214    // A longer notice is split over several lines with the same prefix. The pieces, joined, give the notice with
215    // its whitespace folded. A cut never splits a surrogate pair, and no piece after the first starts with a space,
216    // because log() would fold that space into the prefix. Whitespace at the very start of a notice is lost.
217    const prefix = `NOTICE ${stop.session_id} | NOTICE: `;
218    const room = Math.max(MAX_LINE - TIME_LEN - 1 - prefix.length, 100);
219    for (const n of out.notices) {
220      const text = n.replace(/\s+/g, " ");
221      let i = 0;
222      do {
223        let j = Math.min(i + room, text.length);
224        while (j < text.length && j > i + 1 && (text[j] === " " || /[\uDC00-\uDFFF]/.test(text[j]))) j--;
225        await log($, prefix + text.slice(i, j));
226        i = j;
227      } while (i < text.length);
228    }
229    line = `END ${stop.session_id} | ${out.trace.join(" | ")} | ${out.result}`;
230    shown = out.shown;
231    notices = out.notices;
232  } catch (err) {
233    line = `END ${stop.session_id} | MOD ERROR, reads as UNKNOWN: ${msg(err)}`;
234    shown = UNKNOWN + `the mod raised (${msg(err)})`;
235  }
236  await log($, line);
237  // A FLAG or UNKNOWN verdict toasts. An ALLOW verdict is silent. Each notice toasts on its own line.
238  if (/^Decision watch: (FLAG|UNKNOWN)/.test(shown)) {
239    await toast($, stop, shown);
240  } else if (shown) {
241    await log($, `NOT SHOWN ${stop.session_id} | not a FLAG or UNKNOWN | text: ${shown}`);
242  }
243  for (const n of notices) await toast($, stop, NOTICE + n);
244}
245
246async function toast($, stop, text) {
247  try {
248    await deadline(Promise.resolve($.ui.toast(text)), TOAST_MS);
249  } catch (err) {
250    await log($, `TOAST FAILED ${stop.session_id} | ${msg(err)} | text: ${text}`);
251  }
252}
253
254const MAX_LOG_LINES = 500;
255const MAX_LINE = 2000;
256const TIME_LEN = 24; // length of an ISO timestamp, the start of each log line
257const LOG_WAIT_MS = 2000;
258const STOP_WAIT_MS = 500;
259const MODEL_MS = 50000;
260const PY_MS = 30000; // each python run (unmeasured; split mode calls no model); the mod's timer adds 1 s
261const FIND_MS = 5000; // each python finder; the mod's timer adds 1 s
262const TOAST_MS = 5000;
263
264// Like within, but a late promise rejects, so the caller can fail the step and free its slot.
265async function deadline(work, ms) {
266  let timer;
267  try {
268    return await Promise.race([work, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`no reply within ${ms} ms`)), ms); })]);
269  } finally {
270    clearTimeout(timer);
271  }
272}
273
274// Race a promise against a timer, and free the timer either way.
275async function within(work, ms) {
276  let timer;
277  try {
278    return await Promise.race([work, new Promise((resolve) => { timer = setTimeout(resolve, ms); })]);
279  } finally {
280    clearTimeout(timer);
281  }
282}
283
284async function logFile($) {
285  return `${stateDir(await configDir($), "shadow")}/shadow.log`;
286}
287
288// Writes go one at a time, so two lines never read the same old file. $.fs.write makes a missing folder.
289// The file keeps its newest MAX_LOG_LINES lines. A file the host will not read (too big) starts over.
290let logChain = Promise.resolve();
291function log($, line) {
292  const work = async () => {
293    try {
294      const file = await logFile($);
295      let old = [];
296      try {
297        if (await $.fs.exists(file)) old = String(await $.fs.read(file)).split("\n").filter(Boolean);
298      } catch {
299        old = [];
300      }
301      // Whitespace is folded, as guard_core.log_cut does, so no text can forge a second log line.
302      const next = `${new Date().toISOString()} ${line.replace(/\s+/g, " ")}`.slice(0, MAX_LINE);
303      await $.fs.write(file, [...old.slice(-(MAX_LOG_LINES - 1)), next].join("\n") + "\n");
304    } catch {
305      // the log never fails the Stop
306    }
307  };
308  const done = logChain.then(() => within(work(), LOG_WAIT_MS));
309  logChain = done;
310  return done;
311}
312
313async function stopsSeen($) {
314  try {
315    const file = await logFile($);
316    if (!(await $.fs.exists(file))) return 0;
317    return String(await $.fs.read(file)).split("\n").filter((l) => /^\S+ STOP seen /.test(l)).length;
318  } catch {
319    return "unknown";
320  }
321}
322