Codex Computer Use

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { CodexCuPending } from '../types'
5
6// Codex's computer-use engine (node_repl + @oai/sky on Windows, cua_repl on macOS) drives apps in
7// the background on macOS; on Windows it reads apps through UI Automation but takes the foreground to act. The
8// mod reaches it through a small daemon (bridge/daemon.mjs, shipped in this plugin) that keeps one
9// engine per Claude session. Codex asks before using each app; the daemon answers yes only for
10// apps the person allowed in the pane below. The daemon listens on 127.0.0.1 and requires the
11// token it writes to <home>/.claude/codex-cu/daemon.json.
12const BRIDGE = 'mcp__codex-computer-use__cua'
13const BRIDGE_RESET = 'mcp__codex-computer-use__cua_reset'
14const OWN_COMPUTER_USE = /^mcp__(computer-use__|remote-devices__computer)/
15const MAC_NODE = '/Applications/ChatGPT.app/Contents/Resources/cua_node/bin/node'
16const PANE = 'codex-approval'
17const DAEMON_VERSION = 'v3'
18
19type Host = { home: string; data: string; isWindows: boolean }
20type Daemon = { url: string; token: string }
21type Standing = { apps: string[]; autoApproveAll: boolean }
22
23const alwaysFile = (host: Host) => `${host.data}/always-allowed.json`
24
25async function readStanding($: EngineInterface, host: Host): Promise<Standing> {
26  try {
27    const parsed = JSON.parse(String(await $.fs.read(alwaysFile(host))))
28    return { apps: Array.isArray(parsed.apps) ? parsed.apps : [], autoApproveAll: parsed.autoApproveAll === true }
29  } catch {
30    return { apps: [], autoApproveAll: false }
31  }
32}
33
34async function writeStanding($: EngineInterface, host: Host, change: Partial<Standing>) {
35  const standing = await readStanding($, host)
36  await $.fs.write(alwaysFile(host), `${JSON.stringify({ ...standing, ...change }, null, 2)}\n`)
37}
38
39const same = (a: string, b: string) => a.toLowerCase() === b.toLowerCase()
40
41async function daemonInfo($: EngineInterface, host: Host): Promise<Daemon | null> {
42  try {
43    const state = JSON.parse(String(await $.fs.read(`${host.data}/daemon.json`)))
44    return { url: `http://127.0.0.1:${state.port}`, token: String(state.token) }
45  } catch {
46    return null
47  }
48}
49
50async function daemonFetch($: EngineInterface, daemon: Daemon, path: string, body?: unknown) {
51  return $.http.fetch(`${daemon.url}${path}`, {
52    method: body === undefined ? 'GET' : 'POST',
53    headers: { 'x-codex-cu-token': daemon.token },
54    body: body === undefined ? undefined : JSON.stringify(body),
55  })
56}
57
58async function healthOf($: EngineInterface, daemon: Daemon | null): Promise<string | null> {
59  if (daemon === null) return null
60  try {
61    const res = await daemonFetch($, daemon, '/health')
62    return res.ok ? res.text.trim() : null
63  } catch {
64    return null
65  }
66}
67
68// Node to run the bridge with: the one Codex's current runtime ships (read fresh from Codex's own
69// config, so a Codex update never leaves it pointing at an old folder), else node on PATH.
70async function nodeCandidates($: EngineInterface, host: Host): Promise<string[]> {
71  const found: string[] = []
72  if (host.isWindows) {
73    try {
74      const toml = String(await $.fs.read(`${host.home}/.codex/config.toml`))
75      const path = /NODE_REPL_NODE_PATH\s*=\s*(?:'([^']+)'|"([^"]+)")/.exec(toml)
76      const node = path?.[1] ?? path?.[2]?.replace(/\\\\/g, '\\')
77      if (node) found.push(node)
78    } catch {
79      // no Codex config: node on PATH below
80    }
81  } else {
82    found.push(MAC_NODE)
83  }
84  return [...found, 'node']
85}
86
87async function ensureDaemon($: EngineInterface, host: Host): Promise<Daemon | string> {
88  let daemon = await daemonInfo($, host)
89  const health = await healthOf($, daemon)
90  if (daemon !== null && health === `ok ${DAEMON_VERSION}`) return daemon
91  // an older daemon speaks another protocol: ask it to leave
92  if (daemon !== null && health !== null) {
93    try {
94      await daemonFetch($, daemon, '/quit', {})
95    } catch {
96      // already gone
97    }
98    await $.clock.sleep(300)
99  }
100
101  let started = ''
102  for (const node of await nodeCandidates($, host)) {
103    try {
104      const ran = await $.process.run([node, `${$.plugin.root}/bridge/start.mjs`, host.data], { cwd: host.home })
105      if (ran.exitCode === 0) {
106        started = node
107        break
108      }
109    } catch {
110      // this node could not start: try the next
111    }
112  }
113  if (started === '') return `Could not start the Codex computer-use bridge: no working Node found (Codex runtime or node on PATH). Is the Codex app installed?`
114
115  for (let i = 0; i < 40; i++) {
116    await $.clock.sleep(250)
117    daemon = await daemonInfo($, host)
118    if ((await healthOf($, daemon)) === `ok ${DAEMON_VERSION}` && daemon !== null) return daemon
119  }
120  return `The Codex computer-use bridge did not start. See ${host.data}/daemon.log.`
121}
122
123const pending = atom({ plugin: 'codex-computer-use', key: 'pending' } as const, null as CodexCuPending)
124const allowed = atom({ plugin: 'codex-computer-use', key: 'allowed' } as const, [] as string[])
125
126const WINDOWS_ROUTING = `# Desktop apps go through Codex computer use (codex-cu mod is on)
127
128For any task that needs to see or operate a desktop app on this Windows PC, use Codex's Windows computer-use engine through \`${BRIDGE}\` (reset with \`${BRIDGE_RESET}\`; load them with ToolSearch if they are deferred). Your own mcp__computer-use__* tools are blocked while this mode is on. The point of this mode: Codex reads apps through Windows UI Automation (exact values, not screenshots) and asks the person before using each app. On Windows it is not background automation: typing and clicks bring the app to the front and take over input while they run, so tell the person to leave the machine alone during a run.
129
130How to drive it:
131- The tool runs JavaScript in a persistent node_repl. Print with \`nodeRepl.write(string)\` (JSON.stringify objects). Keep values across calls on \`globalThis\`; top-level const/let cannot be redeclared by a later call.
132- First call (or after a reset): \`if (!globalThis.sky) { const { sky } = await import("@oai/sky"); globalThis.sky = sky; }\` then \`globalThis.apps = await sky.list_apps();\` and print the id, displayName and window count of the apps you need.
133- Use the app \`id\` list_apps returned (Calculator is \`Microsoft.WindowsCalculator_8wekyb3d8bbwe!App\`). If it has no window: \`await sky.launch_app({ app: id })\`, wait about 1.5 s, call list_apps again and take the returned window object. Never build a window object yourself.
134- Observe: \`globalThis.state = await sky.get_window_state({ window, include_screenshot: false, include_text: true })\`, then print \`state.accessibility.tree\` (filter its lines when it is long). \`document_text\`, \`focused_element\` and \`selected_text\` are on \`state.accessibility\` too. Element indexes are valid only for the latest state: observe again after every action that changes the UI.
135- Act mostly by keyboard: \`sky.press_key({ window, key: "Control_L+b" })\` (X keysym names: Return, Tab, Escape, Up, Down, Alt_L, Control_L+a, Control_L+Home, KP_0..KP_9; press ribbon KeyTips one key per call, e.g. Alt_L then a then m) and \`sky.type_text({ window, text })\` once the focus is confirmed. \`sky.set_value({ window, element_index, value })\` sets an edit field directly (in Excel, set the Name Box to a range like "A1:F1" and press Return instead of using Go To). \`sky.click({ window, element_index })\` can fail with "geometry unavailable" or "not in cached state"; fall back to the keyboard. Coordinate clicks (\`sky.click({ window, screenshotId, x, y })\`) move the person's real cursor: use them only when nothing else works.
136- Pop-up dialogs (Excel's Remove Duplicates, Greater Than, Save As) often do not appear in the accessibility tree or in list_windows. Take a screenshot to see where the keyboard focus is before pressing Return (Return presses the focused button, not always OK), and if keys stop having any effect, the dialog has lost the foreground: \`sky.activate_window({ window })\` first.
137- Screenshots: pass \`include_screenshot: true\` only when the accessibility tree is not enough; the result names a saved image you can open with Read.
138- After an error or a surprise, observe before retrying: an action can fail yet still apply, and a dialog may have opened (find it with \`sky.list_windows()\`).
139- Each app needs the person's approval. If the result says they are being asked, stop and wait for their answer; a message will say what they chose. Never try to get around a denial.
140- Codex's own limits apply: never automate terminals, the Run dialog, password managers, security apps, sign-in dialogs or the Codex/ChatGPT app, and never press the Windows key.
141- Other Claude sessions may use computer use at the same time; an app one of them is driving is leased to it. If told an app is in use, use another app or wait about a minute.
142- Prefer purpose-built tools first (a CLI, an API, a connector, the browser tools for web pages). Use this for native apps and anything only the GUI can do. Say where the result is when you finish.`
143
144const MAC_ROUTING = `# Desktop apps go through Codex computer use (codex-cu mod is on)
145
146For any task that needs to see or operate a desktop app on this Mac, use Codex's computer-use engine through \`${BRIDGE}\` (reset with \`${BRIDGE_RESET}\`). The mcp__computer-use__* tools are blocked while this mode is on.
147
148- It runs JavaScript in a persistent REPL with a \`cua\` object. First call (or after a reset): exactly one entry call, such as \`let app = await cua.getApp("Calculator");\`, and nothing else. Its result carries the API documentation and the app's accessibility tree; use only documented APIs.
149- Prefer element indices from the accessibility tree over coordinates, and keyboard shortcuts where the app has them. Re-read indices after every action.
150- It works in the background without moving the person's cursor.
151- Each app needs approval. If the result says the person is being asked, stop and wait for their answer. Never get around a denial.`
152
153// Records the person's answer to an app approval and closes the pane; resolves the message that
154// tells Claude what they chose.
155async function grant($: EngineInterface, host: Host, app: string, choice: 'session' | 'always' | 'deny'): Promise<string> {
156  if (choice !== 'deny') {
157    if (choice === 'always') {
158      const standing = await readStanding($, host)
159      if (!standing.apps.some(one => same(one, app))) await writeStanding($, host, { apps: [...standing.apps, app] })
160    }
161    await update($, allowed, list => (list.some(one => same(one, app)) ? list : [...list, app]))
162  }
163  await update($, pending, () => null)
164  try {
165    await $.ui.close({ id: PANE })
166  } catch {
167    // the pane was not open
168  }
169  return choice === 'session'
170    ? `I allowed Codex computer use to use ${app} for this chat. Retry the last call and continue the task.`
171    : choice === 'always'
172      ? `I always allowed Codex computer use to use ${app}. Retry the last call and continue the task.`
173      : `I did not allow Codex computer use to use ${app}. Do not use ${app}; tell me another way or stop.`
174}
175
176// Hands the person's answer to Claude as their message. A command hook cannot submit while it
177// holds the turn, so it calls this without awaiting and the message enters once the command is done.
178async function resume($: EngineInterface, text: string) {
179  try {
180    await $.clock.sleep(150)
181    await $.prompt.submit({ text, asUser: true })
182  } catch (error) {
183    $.ui.toast(`codex-cu: could not tell Claude (${String(error).slice(0, 100)}). Say "continue".`)
184  }
185}
186
187export const register: Register = on => {
188  let isOn = true
189  let host: Host = { home: '', data: '', isWindows: false }
190
191  on('session.start', async ($, e, next) => {
192    isOn = (await $.store.get('enabled')) !== false
193    const isWindows = (await $.env.get('OS')) === 'Windows_NT'
194    const home = ((isWindows ? await $.env.get('USERPROFILE') : await $.env.get('HOME')) ?? '').replace(/\\/g, '/')
195    host = { home, data: `${home}/.claude/codex-cu`, isWindows }
196    try {
197      await $.command.register({
198        name: 'codex-cu',
199        description: 'Codex computer use: on, off, status, allow/always/deny <app>, auto on/off, forget <app>',
200        argumentHint: '[on|off|status|allow <app>|always <app>|deny <app>|auto on|auto off|forget <app>]',
201        immediate: true,
202      })
203    } catch {
204      // offered as /codex-computer-use:codex-cu instead
205    }
206    $.ui.status(isOn ? 'Codex CU on' : undefined)
207    try {
208      await $.tool.register({
209        name: 'cua',
210        description: isWindows
211          ? "Codex computer use for Windows desktop apps (reads exact values through UI Automation; acting brings the app to the front). Runs JavaScript in Codex's persistent node_repl; import `sky` from \"@oai/sky\" and call sky.list_apps(), sky.launch_app(), sky.get_window_state(), sky.click({ window, element_index }), sky.set_value(), sky.press_key(), sky.type_text(). Print with nodeRepl.write(). Apps need the person's approval."
212          : "Codex computer use: control native Mac apps in the background. Runs JavaScript in Codex's persistent cua_repl with a `cua` object. First call: exactly one entry call, e.g. `let app = await cua.getApp(\"Calculator\");`. Apps need the person's approval.",
213        inputSchema: {
214          type: 'object',
215          properties: {
216            code: { type: 'string', description: 'JavaScript to run in the Codex computer-use REPL.' },
217            timeout_ms: { type: 'integer', minimum: 1, description: 'Execution timeout in milliseconds (default 30000).' },
218            title: { type: 'string', maxLength: 80, description: 'Short user-facing description of what the code does.' },
219          },
220          required: ['code'],
221        },
222      })
223      await $.tool.register({
224        name: 'cua_reset',
225        description: 'Reset the Codex computer-use JavaScript session. Does not close apps or windows.',
226      })
227    } catch (error) {
228      $.ui.toast(`codex-cu: tool not registered (${String(error).slice(0, 120)})`)
229    }
230
231    return next(e)
232  })
233
234  on('tool.call', { tool: [BRIDGE, BRIDGE_RESET] }, async ($, e) => {
235    const { tool, tool_use_id, agentId, ...args } = e as typeof e & { agentId?: string }
236    const fail = (text: string) => ({ result: text, text, isError: true as const })
237    if (!isOn) {
238      return fail('codex-cu is off, so desktop computer use goes through your own computer-use tools (mcp__computer-use__*). The person can switch Codex back on with /codex-cu on.')
239    }
240    // its own Codex engine per Claude session, and per subagent inside it
241    const session = `${await $.session.id()}${agentId ? `/${agentId}` : ''}`
242    const isReset = tool === BRIDGE_RESET
243
244    const daemon = await ensureDaemon($, host)
245    if (typeof daemon === 'string') return fail(daemon)
246
247    const approve = await read($, allowed)
248    const res = await daemonFetch($, daemon, isReset ? '/reset' : '/js', isReset ? { session } : { ...args, approve, session })
249    const busy: { app: string; idleSeconds: number }[] = JSON.parse(res.headers['x-codex-busy'] ?? '[]')
250    if (busy.length > 0) {
251      const { app, idleSeconds } = busy[0]
252      return fail(
253        `${app} is being driven by another Claude session right now (its last call was ${idleSeconds}s ago), so it is leased to that session. Work in a different app, or wait about a minute and retry. Do not try to get around it.`,
254      )
255    }
256    const declined: string[] = JSON.parse(res.headers['x-codex-declined'] ?? '[]')
257    if (declined.length > 0) {
258      const app = declined[0]
259      await update($, pending, () => ({ app }))
260      let isShown = false
261      try {
262        isShown = (await $.ui.open({ id: PANE, title: 'Codex computer use', focus: true, closeOnEscape: true, rows: 6 })).isPlaced
263      } catch {
264        isShown = false
265      }
266      $.ui.toast(`Codex wants to use ${app}: allow for this chat, always allow, or don't allow`)
267      return fail(
268        `Codex needs the person's approval to use ${app}. They are being asked now${isShown ? ' in the Codex computer use panel' : ''} (they can also answer with /codex-cu allow ${app}, /codex-cu always ${app} or /codex-cu deny ${app}). Stop and wait: a message will say what they chose, and then you can retry the same call.`,
269      )
270    }
271
272    return res.ok ? { result: res.text, text: res.text } : fail(res.text || `The bridge answered ${res.status}.`)
273  })
274
275  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
276    const { Box, Button, Text } = $.ui.resolve(e)
277    const ask = await read($, pending)
278    if (ask === null) return <Text dimColor>No approval waiting.</Text>
279    const app = ask.app
280
281    return (
282      <Box flexDirection="column">
283        <Text>Allow Codex computer use to use {app}?</Text>
284        <Text dimColor>It sends real clicks and keystrokes to {app}; on Windows the app comes to the front while it works.</Text>
285        <Box>
286          <Button key="allow" label="Allow for this chat" hotkey="a" variant="primary" autoFocus onPress={async () => resume($, await grant($, host, app, 'session'))} />
287          <Text> </Text>
288          <Button key="always" label="Always allow" hotkey="l" onPress={async () => resume($, await grant($, host, app, 'always'))} />
289          <Text> </Text>
290          <Button key="deny" label="Don't allow" hotkey="d" role="dismiss" onPress={async () => resume($, await grant($, host, app, 'deny'))} />
291        </Box>
292      </Box>
293    )
294  })
295
296  on('command.run', { command: ['codex-cu', 'codex-computer-use:codex-cu'] }, async ($, e) => {
297    const raw = e.args.trim()
298    const arg = raw.toLowerCase()
299    const verb = arg.split(/\s+/)[0] ?? ''
300    const target = raw.slice(verb.length).trim()
301
302    if (verb === 'allow' || verb === 'always' || verb === 'deny') {
303      const ask = await read($, pending)
304      const app = target || ask?.app
305      if (!app) return { text: `Usage: /codex-cu ${verb} <app name>` }
306      void resume($, await grant($, host, app, verb === 'allow' ? 'session' : verb))
307      return { text: verb === 'deny' ? `Not allowed: ${app}.` : `Allowed${verb === 'always' ? ' always' : ' for this chat'}: ${app}.` }
308    }
309    if (arg === 'auto on' || arg === 'auto off') {
310      const isAuto = arg === 'auto on'
311      await writeStanding($, host, { autoApproveAll: isAuto })
312      return {
313        text: isAuto
314          ? "Auto-approve is on: Codex computer use may use any app without asking (Codex's own safety blocks still apply)."
315          : 'Auto-approve is off: apps not on the always-allowed list ask first.',
316      }
317    }
318    if (verb === 'forget') {
319      if (target === '') return { text: 'Usage: /codex-cu forget <app name> (or: all)' }
320      const always = (await readStanding($, host)).apps
321      const isAll = same(target, 'all')
322      const kept = isAll ? [] : always.filter(one => !same(one, target))
323      await writeStanding($, host, { apps: kept })
324      await update($, allowed, list => (isAll ? [] : list.filter(one => !same(one, target))))
325      return { text: kept.length === always.length && !isAll ? `${target} was not on the always-allowed list.` : `Removed. Always allowed now: ${kept.length > 0 ? kept.join(', ') : 'none'}.` }
326    }
327    if (arg === 'on' || arg === 'off') {
328      isOn = arg === 'on'
329      await $.store.set('enabled', isOn)
330      $.ui.status(isOn ? 'Codex CU on' : undefined)
331    } else if (arg !== '' && arg !== 'status') {
332      return { text: 'Usage: /codex-cu [on|off|status|allow <app>|always <app>|deny <app>|auto on|auto off|forget <app>]' }
333    }
334
335    const apps = await read($, allowed)
336    const standing = await readStanding($, host)
337    const mode = isOn
338      ? 'On. Desktop apps go through Codex (exact values from UI Automation, per-app approval); my own computer-use tools are blocked. On Windows the app comes to the front while Codex works.'
339      : 'Off. I use my own computer-use tools.'
340    const lines = [
341      `Codex computer use: ${mode}`,
342      standing.autoApproveAll ? 'Auto-approve: ON, every app is allowed without asking (/codex-cu auto off to ask again).' : 'Auto-approve: off.',
343      apps.length > 0 ? `Allowed this chat: ${apps.join(', ')}.` : 'No apps allowed yet in this chat.',
344      `Always allowed: ${standing.apps.length > 0 ? standing.apps.join(', ') : 'none'} (revoke with /codex-cu forget <app>).`,
345    ]
346    const daemon = await daemonInfo($, host)
347    if ((await healthOf($, daemon)) === null || daemon === null) {
348      lines.push('Bridge: not running (it starts on first use).')
349    } else {
350      try {
351        const me = await $.session.id()
352        const users: { session: string; busy: boolean; leases: string[]; idleSeconds: number }[] = JSON.parse((await daemonFetch($, daemon, '/sessions')).text)
353        lines.push(
354          users.length === 0
355            ? 'Bridge: running, no sessions using it.'
356            : `Bridge: ${users.map(u => `${u.session.startsWith(me) ? 'this chat' : u.session.slice(0, 8)}${u.busy ? ' working' : ` idle ${u.idleSeconds}s`}${u.leases.length > 0 ? `, holds ${u.leases.join(', ')}` : ''}`).join('; ')}.`,
357        )
358      } catch {
359        lines.push('Bridge: running.')
360      }
361    }
362    return { text: lines.join('\n') }
363  })
364
365  // a session that ends frees its Codex engines and app leases at once
366  on('session.end', async ($, e, next) => {
367    const done = await next(e)
368    try {
369      const daemon = await daemonInfo($, host)
370      if (daemon !== null) await daemonFetch($, daemon, '/end', { session: e.sessionId })
371    } catch {
372      // no bridge running: nothing to free
373    }
374    return done
375  })
376
377  on('prompt.compose', async ($, e, next) => {
378    const composed = await next(e)
379    if (!isOn) return composed
380    return {
381      ...composed,
382      sections: [...composed.sections, { id: 'codex-cu:routing', text: host.isWindows ? WINDOWS_ROUTING : MAC_ROUTING, scope: 'session' as const }],
383    }
384  })
385
386  on('tool.call', { tool: OWN_COMPUTER_USE }, ($, e, next) =>
387    isOn
388      ? {
389          deny: `codex-cu mode is on, so desktop computer use goes through Codex. Use ${BRIDGE} instead (load it with ToolSearch if needed). The person can switch back with /codex-cu off.`,
390        }
391      : next(e),
392  )
393}
394

1export type CodexCuPending = { app: string } | null
2
3declare module 'claude-code' {
4  interface PluginState {
5    'codex-computer-use': { pending: CodexCuPending; allowed: string[] }
6  }
7}
8