sleight

1import { atom, read, update } from 'claude-code'
2import type { Register } from 'claude-code'
3
4import type { Frame, LogEntry, ViewStatus } from '../types'
5import { FRAME_MARKER, snapshotCode } from './snapshot'
6
7// The server's own name in plugin.json; the session lists it as
8// plugin:sleight:computer, and its tools as mcp__plugin_sleight_computer__*.
9const SERVER = 'computer'
10const JS_TOOL = 'mcp__plugin_sleight_computer__js'
11const TURN_END_TOOL = 'mcp__plugin_sleight_computer__turn_ended'
12const PANE = 'sleight'
13const LOG_LIMIT = 200
14
15const log = atom({ plugin: 'sleight', key: 'log' } as const, [] as LogEntry[])
16const frame = atom({ plugin: 'sleight', key: 'frame' } as const, null as Frame | null)
17const view = atom({ plugin: 'sleight', key: 'view' } as const, { kind: 'idle' } as ViewStatus)
18const stopped = atom({ plugin: 'sleight', key: 'stopped' } as const, false)
19
20// Not drawn from, so plain module variables (a reload resets them, harmlessly).
21let usedThisTurn = false
22let turnRunning = false
23let actions = 0
24let lastApp: string | undefined
25// Set while one of this mod's own calls is in flight, so its tool.call hooks
26// let the call through and don't log it.
27let ownCall = false
28// The exact snapshot code in flight, for the tool.check below.
29let snapshotInFlight: string | undefined
30// Apps the pane snapshotted since Claude last used sleight. The engine diffs
31// UI state against the latest read of an app, whoever made it, so Claude's
32// next diff would be against the pane's read; the next prompt says so.
33const snapshottedApps = new Set<string>()
34// The pane's last drawn size, for sizing the next snapshot.
35let paneColumns = 48
36let paneRows = 20
37// The surface that last drew the pane; a snapshot carries only what it draws.
38let paneSurface = 'terminal'
39
40const now = () => new Date().toTimeString().slice(0, 8)
41
42function appFrom(code: string, text: string | undefined): string | undefined {
43  return code.match(/getApp\(\s*["'`]([^"'`]+)["'`]/)?.[1] ?? text?.match(/App: ([^.\n]+)\./)?.[1]
44}
45
46function setStatus($: { ui: { status: (text: string | undefined) => void } }) {
47  $.ui.status(lastApp ? `sleight · ${lastApp} · ${actions} action${actions === 1 ? '' : 's'}` : undefined)
48}
49
50// Runs one call on the engine as this mod.
51async function call($: any, tool: string, args: Record<string, unknown>) {
52  const conn = await $.mcp.connect(SERVER)
53  if (!conn.isConnected) throw new Error(conn.message)
54  ownCall = true
55  try {
56    return await $.mcp.call(conn.server, tool, args)
57  } finally {
58    ownCall = false
59  }
60}
61
62// Tells the engine the turn is over so it releases what it holds, as Codex
63// does at the end of each turn. The relay fills in the session and turn ids.
64async function endEngineTurn($: any, event: 'Stop' | 'Interrupt') {
65  try {
66    const ended = await call($, 'turn_ended', { hook_event_name: event })
67    $.ui.log(`sleight: turn_ended ${ended.isError ? `failed: ${JSON.stringify(ended.content)}` : 'sent'}`, { to: 'debug' })
68    // The relay names an app sleight drove that holds a keyboard event tap, once per session.
69    for (const block of ended.content ?? []) {
70      const text = typeof block?.text === 'string' ? block.text : ''
71      if (text.startsWith('sleight-warning: ')) $.ui.toast(text.slice('sleight-warning: '.length))
72    }
73  } catch (err) {
74    $.ui.log(`sleight: could not end the turn: ${(err as Error).message}`, { to: 'debug' })
75  }
76}
77
78async function snapshot($: any, app: string) {
79  await update($, view, () => ({ kind: 'snapshotting', app }) as ViewStatus)
80  try {
81    // Leave rows for the status line, buttons, "Actions" and three log lines.
82    const code = snapshotCode(app, Math.max(8, paneColumns - 2), Math.max(4, paneRows - 7), paneSurface === 'terminal')
83    snapshotInFlight = code
84    const result = await call($, 'js', { code, title: 'sleight pane snapshot' }).finally(() => { snapshotInFlight = undefined })
85    const text = result.content.map((block: { text?: string }) => block.text ?? '').join('\n')
86    const line = text.split('\n').find((l: string) => l.startsWith(FRAME_MARKER))
87    if (result.isError || !line) {
88      throw new Error(result.isError ? text.slice(0, 200) : 'no frame in the result')
89    }
90    const shot = JSON.parse(line.slice(FRAME_MARKER.length))
91    snapshottedApps.add(app)
92    await update($, frame, () => ({ ...shot, at: now() }) as Frame)
93    await update($, view, () => ({ kind: 'idle' }) as ViewStatus)
94  } catch (err) {
95    await update($, view, () => ({ kind: 'error', message: (err as Error).message }) as ViewStatus)
96  }
97}
98
99async function isPaneOpen($: any): Promise<boolean> {
100  const panes = await $.ui.panes()
101  return panes.some((pane: { id: string }) => pane.id === PANE)
102}
103
104// In Auto mode the classifier refused the pane's own snapshot, since no request of the person's
105// asked for it (2026-10-08). This mod allows exactly two calls: its snapshot, by the code it just
106// built, and turn_ended. next.origin is set by the host, so only this plugin's own calls pass;
107// Claude's calls (origin engine) go to the normal check.
108function ownCallVerdict(e: { tool: string; input: unknown }, origin: { plugin: string } | undefined) {
109  if (origin?.plugin !== 'sleight') return undefined
110  if (e.tool === TURN_END_TOOL) return { decision: 'allow', reason: 'sleight ends the engine turn' } as const
111  const code = (e.input as { code?: unknown } | undefined)?.code
112  if (e.tool === JS_TOOL && snapshotInFlight !== undefined && code === snapshotInFlight) {
113    return { decision: 'allow', reason: 'sleight pane snapshot' } as const
114  }
115  return undefined
116}
117
118export const register: Register = on => {
119  on('tool.check', async ($, e, next) => ownCallVerdict(e, next.origin) ?? next(e))
120    // Failing open to the normal check only ever tightens. A call from inside this mod's own
121    // frame lands here as a re-entry, decided from e and next alone.
122    .catch(($, e, next) => (next.error?.kind === 're-entry' ? ownCallVerdict(e, next.origin) : undefined) ?? next(e))
123  on('session.start', async ($, e, next) => {
124    await $.command.register({
125      name: 'sleight',
126      description: 'Open the sleight pane, and send any text after it to Claude; `stop` halts its computer use',
127      argumentHint: '[stop | prompt]',
128      immediate: true,
129    })
130    return next(e)
131  })
132
133  // A new prompt lifts a stop, and tells Claude about pane snapshots that
134  // moved the engine's diff baseline.
135  on('prompt.submit', async ($, e, next) => {
136    await update($, stopped, () => false)
137    if (snapshottedApps.size === 0) return next(e)
138    const apps = [...snapshottedApps].join(', ')
139    snapshottedApps.clear()
140    const note =
141      `sleight: its pane re-read the UI state of ${apps} after your last sleight call. ` +
142      'The engine diffs against the latest read, so pass { disableDiffing: true } to your next ' +
143      'getAXState() for that app before relying on a diff.'
144    return next({ ...e, context: [...(e.context ?? []), note] })
145  })
146
147  on('turn.start', async ($, e, next) => {
148    turnRunning = true
149    return next(e)
150  })
151
152  on('tool.call', { tool: JS_TOOL }, async ($, e, next) => {
153    if (ownCall) return next(e)
154    if (await read($, stopped)) {
155      return { deny: 'The user stopped sleight with /sleight stop. Do not use it again until they ask.' }
156    }
157    usedThisTurn = true
158    snapshottedApps.clear()
159    const code = String(e.code ?? '')
160    const entry: LogEntry = {
161      id: e.tool_use_id,
162      title: String(e.title ?? '') || (code.split('\n')[0] ?? '').slice(0, 80),
163      status: 'running',
164      at: now(),
165    }
166    await update($, log, list => [...list, entry].slice(-LOG_LIMIT))
167    const ran = await next(e)
168    const app = appFrom(code, ran.text)
169    if (app) lastApp = app
170    actions++
171    const status: LogEntry['status'] = ran.deny !== undefined ? 'refused' : ran.isError ? 'error' : 'done'
172    await update($, log, list => list.map(one => (one.id === entry.id ? { ...one, status, app: app ?? one.app } : one)))
173    setStatus($)
174    return ran
175  })
176
177  // turn_ended stays listed so this mod can call it; Claude may not.
178  on('tool.call', { tool: TURN_END_TOOL }, async ($, e, next) =>
179    ownCall ? next(e) : { deny: 'turn_ended is internal to sleight; its hooks call it when a turn ends.' },
180  )
181
182  on('turn.complete', async ($, e, next) => {
183    const result = await next(e)
184    if (e.agentId !== undefined) return result
185    turnRunning = false
186    if (!usedThisTurn) return result
187    usedThisTurn = false
188    // Refresh the pane while the engine turn is still open, then end it.
189    if (lastApp && !e.isAborted && (await isPaneOpen($))) await snapshot($, lastApp)
190    await endEngineTurn($, e.isAborted ? 'Interrupt' : 'Stop')
191    return result
192  })
193
194  on('command.run', { command: 'sleight' }, async ($, e) => {
195    if (e.args.trim() === 'stop') {
196      await update($, stopped, () => true)
197      await endEngineTurn($, 'Interrupt')
198      $.ui.status('sleight · stopped')
199      return { text: 'sleight stopped: Claude can’t use it again until your next message. Press Esc to stop the rest of the turn.' }
200    }
201    await $.ui.open({ id: PANE, title: 'sleight' })
202    // `/sleight do this` opens the pane and sends "do this" to Claude, as if
203    // typed on its own line. The engine refuses a submit from inside this hook
204    // (the prompt would wait on the command submitting it), so it goes out
205    // just after; it waits for any running turn to finish.
206    const prompt = e.args.trim()
207    if (!prompt) return { text: 'sleight pane opened.' }
208    $.clock.after(0, async () => {
209      // A new session may not have connected sleight yet. Sent first, the prompt ran with no
210      // sleight tools and Claude used the shell instead (2026-10-08).
211      await $.mcp.connect(SERVER).catch(() => undefined)
212      $.prompt.submit({ text: prompt, asUser: true }).catch(err => {
213        $.ui.toast(`sleight: couldn't send your prompt (${(err as Error).message}). Send it on its own line.`)
214      })
215    })
216    return { text: 'sleight pane opened. Sending your prompt to Claude.' }
217  })
218
219  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
220    const els = $.ui.resolve(e)
221    const { Box, Text, Button } = els
222    paneSurface = e.surface
223    paneColumns = Math.max(8, Math.min(120, e.props.bodyColumns ?? 48))
224    // The rows the surface gave the pane: about a third of the terminal inline,
225    // its full height docked. A picture sized to more pushes the buttons and
226    // log out of view.
227    paneRows = Math.max(10, Math.min(40, e.props.scroll?.bodyRows ?? Math.floor((e.viewport?.rows ?? 30) / 2)))
228
229    const [entries, shot, status, isStopped] = await Promise.all([read($, log), read($, frame), read($, view), read($, stopped)])
230
231    const picture = (() => {
232      if (!shot) return <Text dimColor>No picture yet. It appears after Claude uses an app, or press Refresh.</Text>
233      if (e.surface === 'terminal' && 'Raster' in els && shot.cells) {
234        const { Raster } = els as any
235        return <Raster key="frame" columns={shot.columns} rows={shot.rows} cells={shot.cells} />
236      }
237      if ('Svg' in els && shot.image) {
238        const { Svg } = els as any
239        // Not `h`: JSX compiles to calls of the global h, and a local h broke the drawing.
240        const width = 320
241        const height = Math.round((shot.height / shot.width) * width)
242        const source =
243          `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}">` +
244          `<image href="data:${shot.image.mime};base64,${shot.image.base64}" width="${width}" height="${height}"/></svg>`
245        return <Svg source={source} alt={`Screenshot of ${shot.app}`} width={width} height={height} />
246      }
247      return <Text dimColor>{`${shot.app}: picture not shown on this surface. Press Refresh.`}</Text>
248    })()
249
250    const statusLine =
251      status.kind === 'snapshotting' ? `Refreshing ${status.app}…`
252      : status.kind === 'error' ? `Couldn't refresh: ${status.message}`
253      : shot ? `${shot.app} at ${shot.at}`
254      : ''
255    // Size the log to the rows the pane shows, not the whole terminal: picture,
256    // status line, buttons and the "Actions" heading take the rest.
257    const bodyRows = e.props.scroll?.bodyRows ?? e.viewport?.rows ?? 30
258    const room = Math.max(3, bodyRows - (shot?.rows ?? 1) - 4)
259    const mark = { running: '…', done: '✓', error: '✗', refused: '⊘' } as const
260
261    return (
262      <Box flexDirection="column">
263        {picture}
264        <Text dimColor>{isStopped ? 'Stopped until your next message.' : statusLine}</Text>
265        <Box flexDirection="row">
266          <Button
267            key="refresh"
268            label={turnRunning ? 'Refresh (after this turn)' : 'Refresh'}
269            hotkey="r"
270            onPress={() => {
271              if (turnRunning || !lastApp) return
272              const app = lastApp
273              void snapshot($, app).then(() => endEngineTurn($, 'Stop'))
274            }}
275          />
276          <Button
277            key="stop"
278            label="Stop"
279            hotkey="s"
280            onPress={() => {
281              void update($, stopped, () => true)
282              void endEngineTurn($, 'Interrupt')
283              $.ui.status('sleight · stopped')
284            }}
285          />
286        </Box>
287        <Text bold>Actions</Text>
288        {entries.length === 0 && <Text dimColor>None yet.</Text>}
289        {/* Newest first, so the latest action stays visible however short the pane is. */}
290        {entries.slice(-room).reverse().map(entry => (
291          <Text dimColor={entry.status === 'done'} wrap="truncate-end">
292            {`${mark[entry.status]} ${entry.at} ${entry.app ? `${entry.app}: ` : ''}${entry.title}`}
293          </Text>
294        ))}
295      </Box>
296    )
297  })
298}
299

1// The JavaScript the mod runs in the computer-use engine's session (through
2// the `js` tool) to draw its pane. It must stay self-contained: it runs in the
3// engine, not here. snapshotCode() fills in the app and the frame size.
4//
5// It writes one line, `SLEIGHT_FRAME <json>`: the app's screenshot as
6// terminal cells (upper half blocks, two pixels per cell) for a terminal pane,
7// or else as an image small enough to embed in an SVG. Only one, to keep the
8// result small. The engine's sandbox can't load `sharp`, so it decodes with
9// the bundled pure-JS jpeg-js and pngjs.
10
11const SNAPSHOT_JS = `await (async () => {
12  // Claude Code replaces an MCP result over its token limit (25,000 by default)
13  // with a notice, and base64 counts heavily: a 51,135-character image never
14  // reached the desktop pane (2026-10-08). So the image stays near 16,000.
15  const APP = __APP__, COLS = __COLS__, ROWS = __ROWS__, TERMINAL = __TERMINAL__, MAX_IMAGE_B64 = 16000;
16  const app = await cua.getApp(APP);
17  const shot = Buffer.from(await app.getScreenshot({ emit: false }));
18  const isJpeg = shot[0] === 0xff && shot[1] === 0xd8;
19  const decoded = isJpeg
20    ? (await import('jpeg-js')).default.decode(shot, { useTArray: true })
21    : (await import('pngjs')).PNG.sync.read(shot);
22  const { width, height, data } = decoded;
23
24  // Fit the screenshot into COLS x ROWS*2 pixels, keeping its aspect ratio,
25  // averaging each source block.
26  const scale = Math.max(width / COLS, height / (ROWS * 2));
27  const outW = Math.max(1, Math.min(COLS, Math.round(width / scale)));
28  const outH = Math.max(2, Math.min(ROWS * 2, Math.round(height / scale)));
29  const px = (x, y) => {
30    const x0 = Math.floor(x * width / outW), x1 = Math.max(x0 + 1, Math.floor((x + 1) * width / outW));
31    const y0 = Math.floor(y * height / outH), y1 = Math.max(y0 + 1, Math.floor((y + 1) * height / outH));
32    let r = 0, g = 0, b = 0, n = 0;
33    for (let yy = y0; yy < y1; yy++) for (let xx = x0; xx < x1; xx++) {
34      const i = (yy * width + xx) * 4;
35      r += data[i]; g += data[i + 1]; b += data[i + 2]; n++;
36    }
37    return (Math.round(r / n) << 16) | (Math.round(g / n) << 8) | Math.round(b / n);
38  };
39  const rows = Math.ceil(outH / 2);
40  const cells = Buffer.alloc(TERMINAL ? outW * rows * 12 : 0);
41  if (TERMINAL) for (let r = 0; r < rows; r++) for (let c = 0; c < outW; c++) {
42    const o = (r * outW + c) * 12;
43    cells.writeUInt32LE(0x2580, o);
44    cells.writeUInt32LE(px(c, r * 2), o + 4);
45    cells.writeUInt32LE(r * 2 + 1 < outH ? px(c, r * 2 + 1) : 0x01000000, o + 8);
46  }
47
48  let image = { mime: isJpeg ? 'image/jpeg' : 'image/png', base64: shot.toString('base64') };
49  if (TERMINAL) image = undefined;
50  else if (image.base64.length > MAX_IMAGE_B64) {
51    // Too big to embed: re-encode as a JPEG, from 1.5 times the 320 px the desktop
52    // pane draws, smaller and rougher until it fits.
53    const { encode } = (await import('jpeg-js')).default;
54    for (const [maxW, quality] of [[480, 70], [400, 60], [320, 55], [240, 50], [160, 45]]) {
55      const w = Math.min(width, maxW), h = Math.max(1, Math.round(height * w / width));
56      const small = Buffer.alloc(w * h * 4);
57      for (let y = 0; y < h; y++) for (let x = 0; x < w; x++) {
58        const si = (Math.floor(y * height / h) * width + Math.floor(x * width / w)) * 4, di = (y * w + x) * 4;
59        small[di] = data[si]; small[di + 1] = data[si + 1]; small[di + 2] = data[si + 2]; small[di + 3] = 255;
60      }
61      image = { mime: 'image/jpeg', base64: Buffer.from(encode({ data: small, width: w, height: h }, quality).data).toString('base64') };
62      if (image.base64.length <= MAX_IMAGE_B64) break;
63    }
64  }
65
66  nodeRepl.write('SLEIGHT_FRAME ' + JSON.stringify({
67    app: APP, width, height, columns: outW, rows, cells: TERMINAL ? cells.toString('base64') : undefined, image,
68  }));
69})()
70`
71
72export const FRAME_MARKER = 'SLEIGHT_FRAME '
73
74export function snapshotCode(app: string, columns: number, rows: number, terminal: boolean): string {
75  return SNAPSHOT_JS
76    .replace('__APP__', JSON.stringify(app))
77    .replace('__TERMINAL__', String(terminal))
78    .replace('__COLS__', String(columns))
79    .replace('__ROWS__', String(rows))
80}
81

1// What the sleight mod keeps in $.state for its pane.
2
3/** One action Claude took through sleight, newest last. */
4export type LogEntry = {
5  id: string
6  /** The short description Claude gave the call, or the start of its code. */
7  title: string
8  app?: string
9  status: 'running' | 'done' | 'error' | 'refused'
10  /** Local time, HH:MM:SS. */
11  at: string
12}
13
14/** The pane's latest picture of the app, from lib's snapshot script. */
15export type Frame = {
16  app: string
17  /** Terminal cells for a Raster: columns * rows little-endian u32 triplets, base64. Terminal panes only. */
18  columns: number
19  rows: number
20  cells?: string
21  /** The screenshot, small enough to embed in an SVG. Other surfaces only. */
22  image?: { mime: string; base64: string }
23  width: number
24  height: number
25  at: string
26}
27
28export type ViewStatus =
29  | { kind: 'idle' }
30  | { kind: 'snapshotting'; app: string }
31  | { kind: 'error'; message: string }
32
33declare module 'claude-code' {
34  interface PluginState {
35    sleight: {
36      log: LogEntry[]
37      frame: Frame | null
38      view: ViewStatus
39      stopped: boolean
40    }
41  }
42}
43