Workspace tool runtime

<img src="assets/banner.png" alt="Chaos Monkey for Coding Agents" width="100%">
How coding-agent harnesses recover when their tools fail. A fault-injection benchmark.
CMCA injects seeded faults into tool calls at two layers: an MCP proxy in front of mock services (Track S) and each harness's own built-in tools (Track N). It measures task success under faults, recovery behavior, the cost of recovery, and whether the agent reports its outcome honestly. The full design is in SPEC.md.
Results and build log: chaos-monkey-coding-agents.vercel.app (source in packages/site).
M0 (foundations) is implemented and M1 (conformance and fixed-model injectors) is in progress. Native injectors for Claude Code, Pi and OpenCode pass conformance; their capability files are in capabilities/. Track N has 3 pilot tasks in datasets/cmca-n. Smoke and clean pilot runs with cc-claude are in results/; no leaderboard results are published yet. Track S does not exist yet. Decisions taken during implementation are in DECISIONS.md; open verification items are in docs/verification.
Requirements: Node 24 (22.12 or later works locally), pnpm 10, uv.
pnpm install
pnpm build
pnpm test
pnpm lint
pnpm bench:decide
uv sync --all-packages
uv run pytest
Layout:
| Path | Contents |
|---|---|
packages/fault-engine | Fault schedule, trial state store, catalog, schemas, tt-decide CLI |
packages/injector-claude-code, injector-pi, injector-opencode | Native injectors for Track N |
packages/cli | cmca CLI (lint today) |
packages/site | Results site (Next.js static export for Vercel) |
python/cmca_harbor | Harbor custom agents, conformance runner, trial planner |
python/cmca_analysis | Analysis pipeline (event log validation and trial summaries today) |
profiles/ | Fault profiles |
configs/ | Harness configurations |
datasets/ | Task datasets, conformance probes and the canary |
capabilities/ | Capability files generated from conformance |
results/ | Conformance, task validation, smoke and pilot evidence |
docs/verification/ | Evidence for third-party behavior (V-01 to V-21) |
To be completed by the maintainers before the first public release (SPEC 13.2).
Code: Apache-2.0 (LICENSE). Task data: CC BY 4.0 (LICENSE-DATA); task files carry a canary string and must not be used for model training.
hooks/register.ts 10 lines1import type { Register } from 'claude-code';
2import { handle, type CallEvent, type CallResult, type Next } from './core.js';
3
4export const register: Register = (on) => {
5 on('tool.call', async ($, e, next) => {
6 const run = (argv: readonly string[], init: { stdin: string; timeoutMs: number }) => $.process.run(argv, init);
7 return (await handle(e as unknown as CallEvent, next as unknown as Next, run)) as Awaited<ReturnType<typeof next>> & CallResult;
8 });
9};
10hooks/core.ts 166 lines1// Runs inside the Claude Code hooks environment: no Node, no imports outside
2// this folder. Decisions come from the shared engine through `tt-decide`.
3
4export const TT_DECIDE = ['node', '/opt/tt/bin/tt-decide.cjs'] as const;
5
6export interface RunResult {
7 exitCode: number;
8 stdout: string;
9}
10export type Run = (argv: readonly string[], init: { stdin: string; timeoutMs: number }) => Promise<RunResult>;
11
12export interface CallEvent {
13 tool: string;
14 tool_use_id: string;
15 agentId?: string;
16 [argument: string]: unknown;
17}
18
19export type CallResult =
20 | { deny: string }
21 | { result: unknown; isError?: undefined; deny?: undefined; [k: string]: unknown }
22 | { isError: true; result: unknown; deny?: undefined; [k: string]: unknown };
23
24export type Next = (e: CallEvent) => Promise<CallResult>;
25
26interface PostRef {
27 type: string;
28 payload_id: string;
29 poison_index?: number;
30 drift_index?: number;
31}
32
33interface Decision {
34 apply: boolean;
35 type?: string;
36 phase?: 'pre' | 'post';
37 text?: string;
38 post?: PostRef;
39}
40
41const TOOL_CLASS: Record<string, string> = {
42 Read: 'fs.read',
43 Write: 'fs.write',
44 Edit: 'fs.edit',
45 Grep: 'fs.search',
46 Glob: 'fs.search',
47 Bash: 'shell',
48};
49
50export function capabilityOf(tool: string): string | null {
51 const cls = TOOL_CLASS[tool];
52 if (cls) return cls;
53 const m = /^mcp__([A-Za-z0-9_-]+?)__([A-Za-z0-9_-]+)$/.exec(tool);
54 return m ? `mcp.${m[1]}.${m[2]}` : null;
55}
56
57export function argsOf(e: CallEvent): Record<string, unknown> {
58 const { tool: _t, tool_use_id: _id, agentId: _a, ...args } = e;
59 return args;
60}
61
62export function pathsOf(tool: string, args: Record<string, unknown>): string[] {
63 const keys = tool === 'Grep' || tool === 'Glob' ? ['path'] : ['file_path'];
64 return keys.map((k) => args[k]).filter((v): v is string => typeof v === 'string' && v.startsWith('/'));
65}
66
67/**
68 * The model-visible text of a tool result and a way to put a transformed
69 * text back while keeping the result valid for the tool's output schema.
70 * Returns null for shapes with no text to transform.
71 */
72export function textView(tool: string, result: unknown): { text: string; with(text: string): unknown } | null {
73 if (result === null || typeof result !== 'object') {
74 if (typeof result === 'string') return { text: result, with: (t) => t };
75 return null;
76 }
77 const r = result as Record<string, unknown>;
78 const lines = (t: string) => (t === '' ? 0 : t.split('\n').length);
79 if (tool === 'Bash' && typeof r.stdout === 'string') {
80 return { text: r.stdout, with: (t) => ({ ...r, stdout: t }) };
81 }
82 if (tool === 'Read' && r.type === 'text') {
83 const file = r.file as Record<string, unknown>;
84 if (typeof file?.content !== 'string') return null;
85 // The mapper reports totalLines when content is empty or short, so it
86 // must describe the replaced text, or the real file size leaks.
87 const start = typeof file.startLine === 'number' && file.startLine > 1 ? file.startLine : 1;
88 return {
89 text: file.content,
90 with: (t) => ({ ...r, file: { ...file, content: t, numLines: lines(t), totalLines: start - 1 + lines(t) } }),
91 };
92 }
93 if (tool === 'Grep' && typeof r.content === 'string') {
94 return { text: r.content, with: (t) => ({ ...r, content: t, numLines: lines(t) }) };
95 }
96 if ((tool === 'Grep' || tool === 'Glob') && Array.isArray(r.filenames)) {
97 return {
98 text: (r.filenames as string[]).join('\n'),
99 with: (t) => {
100 const filenames = t === '' ? [] : t.split('\n');
101 return { ...r, filenames, numFiles: filenames.length };
102 },
103 };
104 }
105 if (Array.isArray(result)) {
106 const blocks = result as Array<Record<string, unknown>>;
107 const i = blocks.findIndex((b) => b?.type === 'text' && typeof b.text === 'string');
108 if (i < 0) return null;
109 const text = blocks
110 .filter((b) => b?.type === 'text' && typeof b.text === 'string')
111 .map((b) => b.text as string)
112 .join('\n');
113 return {
114 text,
115 with: (t) => [
116 { type: 'text', text: t },
117 ...blocks.filter((b) => !(b?.type === 'text')),
118 ],
119 };
120 }
121 return null;
122}
123
124async function decide(run: Run, payload: unknown): Promise<Decision> {
125 const r = await run(TT_DECIDE, { stdin: JSON.stringify(payload), timeoutMs: 5000 });
126 if (r.exitCode !== 0) return { apply: false };
127 return JSON.parse(r.stdout) as Decision;
128}
129
130async function applyPost(run: Run, post: PostRef, output: string): Promise<string> {
131 const r = await run([...TT_DECIDE, '--apply'], { stdin: JSON.stringify({ post, output }), timeoutMs: 5000 });
132 if (r.exitCode !== 0) return output;
133 return (JSON.parse(r.stdout) as { output: string }).output;
134}
135
136/**
137 * The `tool.call` handler. Pre-phase faults refuse the call without running
138 * it. Post-phase faults run the call, then either replace the result text
139 * (`empty`, `trunc`, `poison`) or report an error for a call that did run
140 * (`falsefail`).
141 */
142export async function handle(e: CallEvent, next: Next, run: Run): Promise<CallResult> {
143 const capability = capabilityOf(e.tool);
144 if (!capability) return next(e);
145 const args = argsOf(e);
146 const d = await decide(run, {
147 tool_name: e.tool,
148 capability,
149 args,
150 paths: pathsOf(e.tool, args),
151 ...(e.agentId ? { agent_id: e.agentId } : {}),
152 });
153 if (!d.apply) return next(e);
154 if (d.phase === 'pre') return { deny: d.text ?? '' };
155
156 const res = await next(e);
157 if ('deny' in res && res.deny !== undefined) return res;
158 if (res.isError) return res;
159 if (d.type === 'falsefail') return { deny: d.text ?? '' };
160 if (!d.post) return res;
161 const view = textView(e.tool, res.result);
162 if (!view) return res;
163 const out = await applyPost(run, d.post, view.text);
164 return { result: view.with(out) };
165}
166