codex-computer-use

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { CodexCuPending } from '../types'
5
6// Codex's computer-use engine is cua_repl from the ChatGPT app. The bridge tool reaches it
7// through a small daemon in ~/.claude/mcp/codex-cu that keeps one server alive on a Unix
8// socket. Codex asks before using each app; the daemon answers yes only for apps the person
9// allowed in the pane below, so approval works the same on every surface. (A function hook
10// cannot answer an MCP elicitation, so the session's own codex-cu connection is not used.)
11const SERVER = 'codex-cu'
12const CODEX_JS = 'mcp__codex-cu__js'
13// The mod's own tools. The plugin is not named codex-cu: its tools would then sit in the
14// codex-cu server's mcp__codex-cu__ namespace, which the engine refuses.
15const BRIDGE = 'mcp__codex-computer-use__cua'
16const BRIDGE_RESET = 'mcp__codex-computer-use__cua_reset'
17const CODEX_TOOLS = [CODEX_JS, 'mcp__codex-cu__js_reset', BRIDGE, BRIDGE_RESET]
18const OWN_COMPUTER_USE = /^mcp__computer-use__/
19const NODE = '/Applications/ChatGPT.app/Contents/Resources/cua_node/bin/node'
20const PANE = 'codex-approval'
21// The daemon this mod speaks to: one cua_repl server per Claude session (and per subagent),
22// so several sessions, such as threads, can drive different apps at the same time.
23const DAEMON_VERSION = 'v2'
24// The person's standing approvals, shared with the daemon:
25// { "apps": [...always allowed...], "autoApproveAll": true|false }.
26const ALWAYS_FILE = '.claude/mcp/codex-cu/always-allowed.json'
27
28type Standing = { apps: string[]; autoApproveAll: boolean }
29
30async function readStanding($: EngineInterface, home: string): Promise<Standing> {
31  try {
32    const parsed = JSON.parse(String(await $.fs.read(`${home}/${ALWAYS_FILE}`)))
33    return { apps: Array.isArray(parsed.apps) ? parsed.apps : [], autoApproveAll: parsed.autoApproveAll === true }
34  } catch {
35    return { apps: [], autoApproveAll: false }
36  }
37}
38
39async function readAlways($: EngineInterface, home: string): Promise<string[]> {
40  return (await readStanding($, home)).apps
41}
42
43async function writeAlways($: EngineInterface, home: string, apps: string[]) {
44  const standing = await readStanding($, home)
45  await $.fs.write(`${home}/${ALWAYS_FILE}`, `${JSON.stringify({ ...standing, apps }, null, 2)}\n`)
46}
47
48async function writeAutoApprove($: EngineInterface, home: string, autoApproveAll: boolean) {
49  const standing = await readStanding($, home)
50  await $.fs.write(`${home}/${ALWAYS_FILE}`, `${JSON.stringify({ ...standing, autoApproveAll }, null, 2)}\n`)
51}
52
53const pending = atom({ plugin: 'codex-computer-use', key: 'pending' } as const, null as CodexCuPending)
54const allowed = atom({ plugin: 'codex-computer-use', key: 'allowed' } as const, [] as string[])
55
56const ROUTING = `# Computer use goes through Codex (codex-cu mod is on)
57
58For any task that needs to see or operate a desktop app on this Mac, use Codex's computer-use engine through \`${BRIDGE}\` (reset with \`${BRIDGE_RESET}\`). The mcp__computer-use__* tools are blocked while this mode is on.
59
60How to drive it well:
61- It runs JavaScript in a persistent REPL with a \`cua\` object. First call (or after a reset): exactly one entry call, such as \`let app = await cua.getApp("Calculator");\`, and nothing else. Its result carries the API documentation and the app's accessibility tree. Read it before the next call and use only documented APIs.
62- Prefer element indices from the accessibility tree (\`app.click(12)\`, \`app.setValue(9, "text")\`) over coordinates, and keyboard shortcuts (\`app.pressKey("cmd+t")\`) where the app has them.
63- Batch deterministic steps in one call and end it with \`await app.getAXState()\`, which returns a diff of what changed. Re-read indices after every action; never reuse stale ones. Ask for \`{ disableDiffing: true }\` only when you need the full tree.
64- Return only what you need: observation calls print their result by default, so to filter, read with \`{ emit: false }\` (e.g. \`const t = await app.getAXState({ emit: false, disableDiffing: true })\`) and print just the relevant lines with \`nodeRepl.write(...)\`.
65- It works in the background with real clicks, typing and drags, without moving the person's cursor. There is no hover.
66- After any error or surprise, read the state before retrying: an action can fail yet still apply, and an app dialog may have opened. Answer a dialog that would change the person's settings or data with its least-change option (Not Now, Cancel) unless the task calls for it.
67- When the person wants to watch, bring the app to the front first (\`open -a <App>\` from the shell activates it), then pace the steps with short pauses so each change is visible.
68- Leave apps as you found them (close tabs or windows you opened) unless the task says otherwise.
69- Because it works in the background, the app stays behind the person's other windows. When the result is something they will want to look at (a note, a document, a form you filled), bring the app forward at the end: raise its window with \`performSecondaryAction(<window index>, "Raise")\` and say where the result is.
70- Each app needs approval. If the result says the person is being asked to approve an app, stop and wait for their answer; the mod tells you when they decide. Never try to get around a denial.
71- Other Claude sessions may be using computer use at the same time, each in its own Codex session. An app one of them is driving is leased to it: if the result says an app is in use by another session, work in a different app, or wait about a minute and retry. Never fight over an app.
72- Prefer purpose-built tools first (a CLI, an API, an MCP connector, the built-in browser or Claude in Chrome for web pages). Use Codex computer use for native apps and anything only the GUI can do.`
73
74export const register: Register = on => {
75  let isOn = true
76  let isUsedThisTurn = false
77  let home = ''
78
79  on('session.start', async ($, e, next) => {
80    isOn = (await $.store.get('enabled')) !== false
81    home = (await $.process.run(['printenv', 'HOME'])).stdout.trim()
82    try {
83      await $.command.register({
84        name: 'codex-cu',
85        description: 'Route computer use through Codex: on, off, status, auto on/off, forget <app>',
86        argumentHint: '[on|off|status|auto on|auto off|forget <app>]',
87        immediate: true,
88      })
89    } catch {
90      // The command file still offers it as /codex-computer-use:codex-cu.
91    }
92    $.ui.status(isOn ? 'Codex CU on' : undefined)
93    try {
94      await registerBridge()
95    } catch (error) {
96      $.ui.toast(`codex-cu: bridge tool not registered (${String(error).slice(0, 120)})`)
97    }
98
99    return next(e)
100
101    async function registerBridge() {
102      await $.tool.register({
103      name: 'cua',
104      description:
105        "Codex computer use: control native Mac apps in the background (real clicks, typing, drags; no hover). Runs JavaScript in Codex's persistent cua_repl with a `cua` object. First call (or after cua_reset): exactly one entry call, e.g. `let app = await cua.getApp(\"Calculator\");` or `await cua.getState();`. Its result carries the API docs and UI state; read it, then use only documented APIs. Apps need the person's approval.",
106      inputSchema: {
107        type: 'object',
108        properties: {
109          code: { type: 'string', description: 'JavaScript to execute using the initialized cua_repl runtime.' },
110          timeout_ms: { type: 'integer', minimum: 1, description: 'Execution timeout in milliseconds (default 30000).' },
111          title: { type: 'string', maxLength: 80, description: 'Short user-facing description of what the code does.' },
112        },
113        required: ['code'],
114      },
115    })
116      await $.tool.register({
117        name: 'cua_reset',
118        description: 'Reset the Codex cua_repl JavaScript session. Does not close apps or tabs.',
119      })
120    }
121  })
122
123  on('tool.call', { tool: [BRIDGE, BRIDGE_RESET] }, async ($, e) => {
124    const { tool, tool_use_id, agentId, ...args } = e as typeof e & { agentId?: string }
125    // its own Codex session per Claude session, and per subagent inside it
126    const session = `${await $.session.id()}${agentId ? `/${agentId}` : ''}`
127    const isReset = tool === BRIDGE_RESET
128    isUsedThisTurn = true
129    const fail = (text: string) => ({ result: text, text, isError: true as const })
130
131    // The daemon, started on first use.
132    const dir = `${home}/.claude/mcp/codex-cu`
133    const socketPath = `${dir}/daemon.sock`
134    let isUp = false
135    try {
136      const health = await $.http.fetch('http://codex-cu/health', { socketPath })
137      isUp = health.ok && health.text.trim() === `ok ${DAEMON_VERSION}`
138      // an older daemon shares one Codex session among every caller: replace it
139      if (health.ok && !isUp) {
140        await $.process.run(['pkill', '-f', `${dir}/daemon.mjs`])
141        await $.clock.sleep(300)
142      }
143    } catch {
144      isUp = false
145    }
146    if (!isUp) {
147      await $.process.run([
148        '/bin/sh',
149        '-c',
150        `nohup "${NODE}" "${dir}/daemon.mjs" > "${dir}/daemon.log" 2>&1 < /dev/null &`,
151      ])
152      for (let i = 0; i < 40 && !isUp; i++) {
153        await $.clock.sleep(250)
154        try {
155          isUp = (await $.http.fetch('http://codex-cu/health', { socketPath })).ok
156        } catch {
157          isUp = false
158        }
159      }
160      if (!isUp) return fail(`The Codex computer-use daemon did not start. See ${dir}/daemon.log.`)
161    }
162
163    const approve = await read($, allowed)
164    const res = await $.http.fetch(`http://codex-cu/${isReset ? 'reset' : 'js'}`, {
165      method: 'POST',
166      socketPath,
167      body: JSON.stringify(isReset ? { session } : { ...args, approve, session }),
168    })
169    const busy: { app: string; holder: string; idleSeconds: number }[] = JSON.parse(res.headers['x-codex-busy'] ?? '[]')
170    if (busy.length > 0) {
171      const { app, idleSeconds } = busy[0]
172      return fail(
173        `${app} is being driven by another Claude session right now (its last call was ${idleSeconds}s ago), so it is leased to that session. Work in a different app, or wait about a minute and retry; the lease lapses 2 minutes after that session's last call, or when it resets or ends. Do not try to get around it.`,
174      )
175    }
176    const declined: string[] = JSON.parse(res.headers['x-codex-declined'] ?? '[]')
177    if (declined.length > 0) {
178      const app = declined[0]
179      await update($, pending, () => ({ app }))
180      await $.ui.open({ id: PANE, title: 'Codex computer use', focus: true, closeOnEscape: true, rows: 6 })
181      return fail(
182        `Codex needs the person's approval to use ${app}. They are being asked in the codex-cu prompt now. Stop and wait: a message will say whether they allowed it, and then you can retry the same call.`,
183      )
184    }
185
186    return res.ok ? { result: res.text, text: res.text } : fail(res.text)
187  })
188
189  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
190    const { Box, Button, Text } = $.ui.resolve(e)
191    const ask = await read($, pending)
192    if (ask === null) return <Text dimColor>No approval waiting.</Text>
193    const app = ask.app
194
195    return (
196      <Box flexDirection="column">
197        <Text>Allow Codex computer use to use {app}?</Text>
198        <Text dimColor>It sends real clicks and keystrokes to the app in the background.</Text>
199        <Box>
200          <Button
201            key="allow"
202            label="This session"
203            hotkey="a"
204            variant="primary"
205            autoFocus
206            onPress={async () => {
207              await update($, allowed, list => (list.includes(app) ? list : [...list, app]))
208              await update($, pending, () => null)
209              await $.ui.close({ id: PANE })
210              await $.prompt.submit({ text: `I allowed Codex computer use to use ${app} for this session. Retry the last call and continue the task.`, asUser: true })
211            }}
212          />
213          <Text> </Text>
214          <Button
215            key="always"
216            label="Always allow"
217            hotkey="l"
218            onPress={async () => {
219              const always = await readAlways($, home)
220              if (!always.includes(app)) await writeAlways($, home, [...always, app])
221              await update($, allowed, list => (list.includes(app) ? list : [...list, app]))
222              await update($, pending, () => null)
223              await $.ui.close({ id: PANE })
224              await $.prompt.submit({ text: `I always allowed Codex computer use to use ${app}. Retry the last call and continue the task.`, asUser: true })
225            }}
226          />
227          <Text> </Text>
228          <Button
229            key="deny"
230            label="Deny"
231            hotkey="d"
232            role="dismiss"
233            onPress={async () => {
234              await update($, pending, () => null)
235              await $.ui.close({ id: PANE })
236              await $.prompt.submit({ text: `I denied Codex computer use for ${app}. Do not use ${app}; tell me another way or stop.`, asUser: true })
237            }}
238          />
239        </Box>
240      </Box>
241    )
242  })
243
244  // `/codex-cu` is registered at session start; the plugin's commands/codex-cu.md lists it in
245  // every menu as `/codex-computer-use:codex-cu`. Both land here.
246  on('command.run', { command: ['codex-cu', 'codex-computer-use:codex-cu'] }, async ($, e) => {
247    const arg = e.args.trim().toLowerCase()
248    if (arg === 'auto on' || arg === 'auto off') {
249      const isAuto = arg === 'auto on'
250      await writeAutoApprove($, home, isAuto)
251      return {
252        text: isAuto
253          ? 'Auto-approve is on: Codex computer use may use any app without asking (Codex\'s own safety and organization blocks still apply).'
254          : 'Auto-approve is off: apps not on the always-allowed list ask in the approval pane.',
255      }
256    }
257    if (arg.startsWith('forget')) {
258      const target = e.args.trim().slice('forget'.length).trim()
259      if (target === '') return { text: 'Usage: /codex-cu forget <app name> (or: all)' }
260      const always = await readAlways($, home)
261      const kept = target.toLowerCase() === 'all' ? [] : always.filter(one => one.toLowerCase() !== target.toLowerCase())
262      await writeAlways($, home, kept)
263      await update($, allowed, list => (target.toLowerCase() === 'all' ? [] : list.filter(one => one.toLowerCase() !== target.toLowerCase())))
264      return { text: kept.length === always.length ? `${target} was not on the always-allowed list.` : `Removed. Always allowed now: ${kept.length > 0 ? kept.join(', ') : 'none'}.` }
265    }
266    if (arg === 'on' || arg === 'off') {
267      isOn = arg === 'on'
268      await $.store.set('enabled', isOn)
269      $.ui.status(isOn ? 'Codex CU on' : undefined)
270    } else if (arg !== '' && arg !== 'status') {
271      return { text: 'Usage: /codex-cu [on|off|status|auto on|auto off|forget <app>]' }
272    }
273
274    const tools = await $.tool.list()
275    const isConnected = tools.some(tool => tool.name === CODEX_JS)
276    const apps = await read($, allowed)
277    const mode = isOn
278      ? 'On. Desktop computer use goes through Codex; my own computer-use tools are blocked.'
279      : 'Off. I use my own computer-use tools.'
280    const route = `Route: the ${BRIDGE} bridge through the codex-cu daemon (${DAEMON_VERSION}: one Codex session per Claude session, apps leased to one session at a time)${isConnected ? '; the codex-cu server is also connected directly' : ''}.`
281    const standing = await readStanding($, home)
282    const always = standing.apps
283    const autoLine = standing.autoApproveAll
284      ? 'Auto-approve: ON, every app is allowed without asking (/codex-cu auto off to ask again).'
285      : 'Auto-approve: off (/codex-cu auto on to allow every app without asking).'
286    const allowedLine = apps.length > 0 ? `Allowed this session: ${apps.join(', ')}.` : 'No apps allowed through the pane yet this session.'
287    const alwaysLine = `Always allowed: ${always.length > 0 ? always.join(', ') : 'none'} (revoke with /codex-cu forget <app>). Apps set to "Always" inside Codex are allowed too.`
288
289    let usersLine = 'Sessions using it now: none (the daemon is not running).'
290    try {
291      const res = await $.http.fetch('http://codex-cu/sessions', { socketPath: `${home}/.claude/mcp/codex-cu/daemon.sock` })
292      const me = await $.session.id()
293      const users: { session: string; busy: boolean; leases: string[]; idleSeconds: number }[] = res.ok ? JSON.parse(res.text) : []
294      usersLine = users.length === 0
295        ? 'Sessions using it now: none.'
296        : `Sessions using it now (each with its own Codex session): ${users
297            .map(u => `${u.session.startsWith(me) ? 'this session' : u.session.slice(0, 8)}${u.session.includes('/') ? ' (subagent)' : ''}${u.busy ? ' working' : ` idle ${u.idleSeconds}s`}${u.leases.length > 0 ? `, holds ${u.leases.join(', ')}` : ''}`)
298            .join('; ')}.`
299    } catch {
300      // daemon not running
301    }
302
303    return { text: `Codex computer use: ${mode}\n${route}\n${autoLine}\n${allowedLine}\n${alwaysLine}\n${usersLine}` }
304  })
305
306  // a session that ends frees its Codex sessions and app leases at once
307  on('session.end', async ($, e, next) => {
308    const done = await next(e)
309    try {
310      await $.http.fetch('http://codex-cu/end', {
311        method: 'POST',
312        socketPath: `${home}/.claude/mcp/codex-cu/daemon.sock`,
313        body: JSON.stringify({ session: e.sessionId }),
314      })
315    } catch {
316      // no daemon running: nothing to free
317    }
318
319    return done
320  })
321
322  on('prompt.compose', async ($, e, next) => {
323    const composed = await next(e)
324    if (!isOn) return composed
325
326    return {
327      ...composed,
328      sections: [...composed.sections, { id: 'codex-cu:routing', text: ROUTING, scope: 'session' as const }],
329    }
330  })
331
332  on('tool.call', { tool: OWN_COMPUTER_USE }, ($, e, next) =>
333    isOn
334      ? {
335          deny: `codex-cu mode is on, so desktop computer use goes through Codex. Use ${BRIDGE} instead (load it with ToolSearch if needed). The person can switch back with /codex-cu off.`,
336        }
337      : next(e),
338  )
339
340  on('tool.call', { tool: CODEX_TOOLS }, async ($, e, next) => {
341    isUsedThisTurn = true
342    const ran = await next(e)
343    // The server's own tool can only ask through the engine's prompt; point at the bridge,
344    // which asks through the pane, when that approval did not come through.
345    if (e.tool === CODEX_JS && /was not approved to use/.test(String(ran.text ?? ''))) {
346      return { ...ran, text: `${ran.text}\n\n[codex-cu] Use ${BRIDGE} with the same code instead: it asks the person for approval in its own prompt.` }
347    }
348
349    return ran
350  })
351
352  // Codex tells its computer-use runtime when a turn ends so it can release the
353  // session; do the same on the direct connection.
354  on('turn.complete', async ($, e, next) => {
355    const done = await next(e)
356    if (isUsedThisTurn && e.agentId === undefined) {
357      isUsedThisTurn = false
358      try {
359        await $.mcp.call(SERVER, 'turn_ended', {
360          hook_event_name: 'Stop',
361          session_id: await $.session.id(),
362          turn_id: e.turnId,
363        })
364      } catch {
365        // No direct connection: the daemon's server releases on its own.
366      }
367    }
368
369    return done
370  })
371}
372

1export type CodexCuPending = { app: string } | null
2
3declare module 'claude-code' {
4  interface PluginState {
5    'codex-computer-use': { pending: CodexCuPending; allowed: string[] }
6  }
7}
8