simpane

1import type { EngineInterface, Register } from 'claude-code'
2import * as dev from './devices'
3import type { Choice, Device, Key, Platform } from './devices'
4import type { OverlayProps } from './overlay'
5
6const PANE = 'simpane'
7const FRAME_MS = 200 // a frame is taken as soon as the last one is in, at most this often
8const C = { fg: '#dadada', muted: '#6b6b6b', hot: '#a882ff', err: '#e06c75' }
9const KEYS: Record<string, Key> = { return: 'return', enter: 'return', backspace: 'backspace', delete: 'backspace', tab: 'tab', up: 'up', down: 'down', left: 'left', right: 'right', escape: 'home' }
10
11const S = {
12  platform: 'ios' as Platform,
13  found: { ios: { booted: [] as Device[], off: [] as Choice[] }, android: { booted: [] as Device[], off: [] as Choice[] } },
14  device: null as Device | null,
15  size: null as { width: number; height: number } | null, // the frame, in pixels
16  density: 1, // pixels per input unit: points on iOS, 1 on Android
17  axe: true,
18  frame: null as { file: string; generation: number } | null,
19  box: { cols: 0, rows: 0 },
20  down: null as [number, number] | null,
21  typed: '',
22  status: '',
23  error: '',
24  loop: null as { cancel: () => void } | null,
25  busy: false,
26  ticks: 0,
27  tmp: '/tmp',
28}
29const RUN_ID = Math.random().toString(36).slice(2, 8)
30const redraw = ($: EngineInterface) => $.ui.invalidate('ui.render')
31
32async function run($: EngineInterface, argv: string[], timeoutMs = 10_000) {
33  return $.process.run(argv, { timeoutMs })
34}
35
36async function discover($: EngineInterface) {
37  try {
38    const ios = await run($, dev.listIos)
39    if (ios.exitCode === 0) S.found.ios = dev.parseIos(ios.stdout)
40  } catch { /* no Xcode */ }
41  try {
42    S.found.android = dev.parseAndroid((await run($, dev.listAndroid)).stdout)
43  } catch { /* no Android SDK */ }
44  S.axe = (await run($, dev.hasAxe).catch(() => ({ exitCode: 1 }))).exitCode === 0
45  pick(S.platform)
46}
47
48function pick(p: Platform) {
49  S.platform = p
50  const next = S.found[p].booted[0] ?? null
51  if (next?.id !== S.device?.id) {
52    S.device = next
53    S.size = null
54    S.frame = null
55    S.density = 1
56  }
57}
58
59// One frame: the device writes a PNG, the terminal reads it. Two files take
60// turns, so the terminal never reads one half-written.
61async function tick($: EngineInterface) {
62  if (S.busy) return
63  S.busy = true
64  try {
65    if (++S.ticks % 25 === 0 && !(await $.ui.panes()).some(p => p.id === PANE)) return stop()
66    const d = S.device
67    if (!d) return
68    const gen = (S.frame?.generation ?? 0) + 1
69    const file = `${S.tmp}/simpane-${RUN_ID}-${gen % 2}.png`
70    const r = await run($, dev.capture(d, file))
71    if (d !== S.device) return
72    if (r.exitCode !== 0) {
73      S.error = `no frame from ${d.name}: is it still running? (↻)`
74      return redraw($)
75    }
76    const size = dev.pngSize((await run($, dev.pngHead(file))).stdout)
77    if (!size) return
78    const resized = !S.size || S.size.width !== size.width || S.size.height !== size.height
79    S.size = size
80    S.frame = { file, generation: gen }
81    if (d.platform === 'ios' && S.axe && S.density === 1) {
82      const r = await run($, dev.describeIos(d.id)).catch(() => null)
83      S.density = r?.exitCode === 0 ? dev.densityOf(r.stdout, size.width) : 3
84    }
85    if (S.error) { S.error = ''; return redraw($) }
86    if (resized || !(await blit($))) redraw($)
87  } catch (err) {
88    S.error = String(err)
89    redraw($)
90  } finally {
91    S.busy = false
92  }
93}
94
95async function blit($: EngineInterface): Promise<boolean> {
96  if (!S.frame) return false
97  const r = await $.ui.blit({ requestId: PANE, key: 'screen', source: { file: S.frame.file, format: 'png', generation: S.frame.generation } })
98  return !('deny' in r && r.deny)
99}
100
101function stop() {
102  S.loop?.cancel()
103  S.loop = null
104}
105
106async function openPane($: EngineInterface, p?: Platform) {
107  await $.ui.open({ id: PANE, title: 'Simulator', focus: true, rows: 40, columns: 48 })
108  S.tmp = ((await $.env.get('TMPDIR')) || '/tmp').replace(/\/$/, '')
109  await refresh($, p)
110  S.loop ??= $.clock.every(FRAME_MS, () => void tick($))
111}
112
113// Finds what's running and picks a device on platform `p` (or wherever one runs).
114async function refresh($: EngineInterface, p?: Platform) {
115  await discover($)
116  if (p) pick(p)
117  else if (!S.device) pick(S.found.ios.booted.length || !S.found.android.booted.length ? 'ios' : 'android')
118  redraw($)
119}
120
121// Input goes out in the order it came; the next frame shows what it did.
122let queue = Promise.resolve()
123function send($: EngineInterface, argv: string[] | null, what: string) {
124  if (!argv) return
125  if (S.platform === 'ios' && !S.axe) {
126    S.error = 'iOS input needs AXe: brew install cameroncooke/axe/axe'
127    return redraw($)
128  }
129  queue = queue.then(async () => {
130    const r = await run($, argv).catch(err => ({ exitCode: 1, stderr: String(err) }))
131    S.status = r.exitCode === 0 ? what : ''
132    S.error = r.exitCode === 0 ? '' : `${what} failed: ${String(r.stderr).trim().split('\n').pop()}`
133    redraw($)
134  })
135}
136
137const toDevice = (at: [number, number]): [number, number] => [
138  (Math.min(1, Math.max(0, at[0])) * S.size!.width) / S.density,
139  (Math.min(1, Math.max(0, at[1])) * S.size!.height) / S.density,
140]
141
142function release($: EngineInterface, up: [number, number]) {
143  const down = S.down
144  S.down = null
145  if (!down || !S.device || !S.size) return
146  const a = toDevice(down)
147  const b = toDevice(up)
148  const cell = 1 / Math.max(1, S.box.cols) // a move under about a cell is a tap
149  if (Math.hypot(up[0] - down[0], (up[1] - down[1]) / 2) < cell) send($, dev.tap(S.device, ...a), `tap ${a.map(Math.round).join(',')}`)
150  else send($, dev.swipe(S.device, a, b, 250), 'swipe')
151}
152
153// Typed letters gather for a moment, then go as one text input.
154let typing: { cancel: () => void } | null = null
155function flush($: EngineInterface) {
156  typing?.cancel()
157  typing = null
158  if (S.typed && S.device) send($, dev.text(S.device, S.typed), `typed “${S.typed}”`)
159  S.typed = ''
160}
161
162function keyIn($: EngineInterface, k: string) {
163  if (!S.device) return
164  const ch = k === 'space' ? ' ' : k
165  if ([...ch].length === 1) {
166    S.typed += ch
167    typing?.cancel()
168    typing = $.clock.after(150, () => flush($))
169    return
170  }
171  flush($)
172  const named = KEYS[k]
173  if (named) send($, dev.key(S.device, named), named)
174}
175
176let wheel = 0
177let wheelTimer: { cancel: () => void } | null = null
178function scroll($: EngineInterface, by: number) {
179  if (!S.device || !S.size) return
180  wheel += by
181  wheelTimer ??= $.clock.after(120, () => {
182    wheelTimer = null
183    const d = S.device
184    if (!d || !S.size) return
185    // Wheel down reads further: the finger moves up.
186    const dy = Math.max(-0.35, Math.min(0.35, -wheel * 0.06))
187    wheel = 0
188    send($, dev.swipe(d, toDevice([0.5, 0.5 - dy / 2]), toDevice([0.5, 0.5 + dy / 2]), 150), 'scroll')
189  })
190}
191
192export const register: Register = on => {
193  on('session.start', async ($, e, next) => {
194    const r = await next(e)
195    await $.command.register({ name: 'simpane', description: 'iPhone / Android simulator screen in a pane: tap, swipe, type; args: [ios|android]' })
196    return r
197  })
198
199  on('command.run', { command: 'simpane' }, async ($, e) => {
200    const arg = (e.args ?? '').trim().toLowerCase()
201    await openPane($, arg === 'android' || arg === 'ios' ? arg : undefined)
202    const d = S.device
203    return { text: d ? `simpane: ${d.name}. Click to tap, drag to swipe, click the pane and type to type.` : `simpane: no ${S.platform === 'ios' ? 'simulator' : 'emulator'} running; boot one from the pane.` }
204  })
205
206  on('ui.message', async ($, e, next) => {
207    if (e.requestId !== PANE || e.element !== 'touch' || !e.data || typeof e.data !== 'object') return next(e)
208    const data = e.data as { down?: [number, number]; up?: [number, number]; key?: string }
209    if (data.down) S.down = data.down
210    else if (data.up) release($, data.up)
211    else if (typeof data.key === 'string') keyIn($, data.key)
212    return {}
213  })
214
215  on('ui.scroll', { component: 'Pane', requestId: PANE }, async ($, e) => {
216    scroll($, e.by)
217    return {}
218  })
219
220  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e, next) => {
221    if (e.surface !== 'terminal') return next(e)
222    const { Box, Text, Button, Client, Image } = $.ui.resolve(e)
223    const cols = Math.max(20, (e.props?.bodyColumns ?? 48) - 1)
224    const height = Math.max(10, e.props?.scroll?.bodyRows ?? 36)
225    const d = S.device
226    const tab = (p: Platform, label: string) => (
227      <Button key={`tab-${p}`} plain dimColor={S.platform !== p} label={S.platform === p ? `[ ${label} ]` : `  ${label}  `} onPress={() => { pick(p); void refresh($, p) }} />
228    )
229    const tool = (id: string, label: string, k: Key) => <Button key={id} plain dimColor label={label} onPress={() => { if (d) send($, dev.key(d, k), k) }} />
230    const header = (
231      <Box flexDirection="row">
232        {tab('ios', 'iPhone')}
233        {tab('android', 'Android')}
234        <Box flexGrow={1} />
235        {d && tool('home', ' ⌂ ', 'home')}
236        {d?.platform === 'android' && tool('back', ' ◁ ', 'back')}
237        {d?.platform === 'android' && tool('apps', ' ▢ ', 'apps')}
238        <Button key="refresh" plain dimColor label=" ↻" onPress={() => void refresh($, S.platform)} />
239      </Box>
240    )
241    const status = <Text color={S.error ? C.err : C.muted} wrap="truncate-end">{S.error || S.status || ' '}</Text>
242
243    if (!d) {
244      const off = S.found[S.platform].off
245      const none = S.platform === 'ios'
246        ? 'No iOS simulator found. Install Xcode, then: xcrun simctl list'
247        : 'No Android emulator found. brew install --cask android-commandlinetools, then sdkmanager + avdmanager (or Android Studio → Device Manager).'
248      return (
249        <Box flexDirection="column" minHeight={height}>
250          {header}
251          <Text color={C.fg}>{off.length ? `Nothing running. Boot one (headless, it shows here):` : none}</Text>
252          {off.slice(0, Math.max(1, height - 5)).map(c => (
253            <Button key={`boot-${c.id}`} plain label={`  ▸ ${c.name}`} onPress={() => {
254              S.status = `booting ${c.name}…`
255              redraw($)
256              void run($, dev.boot(S.platform, c.id), 60_000).then(async r => {
257                S.error = r.exitCode === 0 ? '' : `boot failed: ${r.stderr.trim()}`
258                // The emulator takes a while to show up in adb.
259                for (let i = 0; i < 30 && !S.device && !S.error; i++) {
260                  await new Promise(res => $.clock.after(2000, () => res(null)))
261                  await refresh($, S.platform)
262                }
263                S.status = ''
264                redraw($)
265              })
266            }} />
267          ))}
268          <Box flexGrow={1} />
269          {status}
270        </Box>
271      )
272    }
273
274    const box = S.size ? dev.fit(S.size.width, S.size.height, cols, height - 4) : { cols: Math.min(cols, 24), rows: height - 4 }
275    S.box = box
276    const screen = S.frame
277      ? <Image key="screen" source={{ file: S.frame.file, format: 'png', generation: S.frame.generation }} columns={box.cols} rows={box.rows} alt={d.name} />
278      : <Box width={box.cols} height={box.rows}><Text color={C.muted}>waiting for the first frame…</Text></Box>
279    return (
280      <Box flexDirection="column" minHeight={height}>
281        {header}
282        <Text color={C.hot} bold wrap="truncate-end">{`${d.name}${S.typed ? `  ⌨ ${S.typed}` : ''}`}</Text>
283        <Box flexDirection="row">
284          <Box flexGrow={1} />
285          <Box key="stage" position="relative" width={box.cols} height={box.rows}>
286            {screen}
287            <Box key="hit" position="absolute" top={0} left={0}>
288              <Client key="touch" module="./overlay.tsx" props={{ cols: box.cols, rows: box.rows } satisfies OverlayProps} />
289            </Box>
290          </Box>
291          <Box flexGrow={1} />
292        </Box>
293        {status}
294        <Text color={C.muted} wrap="truncate-end">click tap · drag swipe · wheel scroll · type into the focused pane · esc home</Text>
295      </Box>
296    )
297  })
298}
299

1// What simpane runs: finding devices, taking a frame, and sending a tap, swipe,
2// text or key. Pure command builders and parsers, so they're easy to test.
3// iOS: AXe (frames, input; github.com/cameroncooke/AXe) and xcrun simctl (boot,
4// frames without AXe). Android: adb and emulator.
5
6export type Platform = 'ios' | 'android'
7export type Device = { platform: Platform; id: string; name: string }
8export type Choice = { id: string; name: string } // a device to boot
9export type Argv = string[]
10
11// The tools live outside the engine's PATH more often than not.
12const EXTRA = '$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$HOME/Library/Android/sdk/platform-tools:$HOME/Library/Android/sdk/emulator:$ANDROID_HOME/platform-tools:$ANDROID_HOME/emulator'
13export const sh = (script: string, ...args: string[]): Argv => ['/bin/sh', '-c', `PATH="$PATH:${EXTRA}"; ${script}`, 'sh', ...args]
14
15export const listIos = sh('xcrun simctl list devices available -j')
16export const listAndroid = sh('adb devices 2>/dev/null; echo ---; emulator -list-avds 2>/dev/null')
17export const hasAxe = sh('command -v axe >/dev/null')
18
19export function parseIos(json: string): { booted: Device[]; off: Choice[] } {
20  const booted: Device[] = []
21  const off: Choice[] = []
22  const all = (JSON.parse(json) as { devices: Record<string, { udid: string; name: string; state: string }[]> }).devices
23  for (const [runtime, list] of Object.entries(all)) {
24    if (!/iOS/.test(runtime)) continue
25    for (const d of list) {
26      if (d.state === 'Booted') booted.push({ platform: 'ios', id: d.udid, name: d.name })
27      else if (!off.some(o => o.name === d.name)) off.push({ id: d.udid, name: d.name })
28    }
29  }
30  const phonesFirst = (a: { name: string }, b: { name: string }) => Number(b.name.startsWith('iPhone')) - Number(a.name.startsWith('iPhone'))
31  return { booted: booted.sort(phonesFirst), off: off.sort(phonesFirst) }
32}
33
34export function parseAndroid(out: string): { booted: Device[]; off: Choice[] } {
35  const [devs = '', avds = ''] = out.split('---')
36  const booted = devs.split('\n').map(l => l.trim().split(/\s+/)).filter(p => p[1] === 'device').map(p => ({ platform: 'android' as const, id: p[0]!, name: p[0]! }))
37  const off = avds.split('\n').map(l => l.trim()).filter(Boolean).map(n => ({ id: n, name: n.replace(/_/g, ' ') }))
38  return { booted, off }
39}
40
41// Boots headless: the screen lives in the pane, not in a window.
42export const boot = (p: Platform, id: string): Argv =>
43  p === 'ios'
44    ? sh('xcrun simctl boot "$1"', id)
45    : sh('nohup emulator -avd "$1" -no-window -no-audio -no-boot-anim >/dev/null 2>&1 &', id)
46
47export const capture = (d: Device, file: string): Argv =>
48  d.platform === 'ios'
49    ? sh('if command -v axe >/dev/null; then axe screenshot --udid "$1" --output "$2"; else xcrun simctl io "$1" screenshot --type=png "$2"; fi >/dev/null 2>&1', d.id, file)
50    : sh('adb -s "$1" exec-out screencap -p > "$2"', d.id, file)
51
52// The first 24 bytes of a PNG hold its width and height (big-endian, at 16 and 20).
53export const pngHead = (file: string): Argv => sh('head -c 24 "$1" | od -An -tu1', file)
54export function pngSize(od: string): { width: number; height: number } | null {
55  const b = od.trim().split(/\s+/).map(Number)
56  if (b.length < 24 || b[1] !== 80 || b[2] !== 78 || b[3] !== 71) return null // \x89PNG
57  const u32 = (i: number) => ((b[i]! << 24) >>> 0) + (b[i + 1]! << 16) + (b[i + 2]! << 8) + b[i + 3]!
58  return { width: u32(16), height: u32(20) }
59}
60
61// iOS input is in points; the frame is in pixels. The app's frame is in points.
62export const describeIos = (id: string): Argv => sh('axe describe-ui --udid "$1"', id)
63export function densityOf(json: string, pixelWidth: number): number {
64  const points = Number((JSON.parse(json) as { frame?: { width?: number } }[])[0]?.frame?.width)
65  return points > 0 ? pixelWidth / points : 3
66}
67
68const r = Math.round
69export function tap(d: Device, x: number, y: number): Argv {
70  return d.platform === 'ios'
71    ? sh('axe tap -x "$2" -y "$3" --udid "$1"', d.id, String(r(x)), String(r(y)))
72    : sh('adb -s "$1" shell input tap "$2" "$3"', d.id, String(r(x)), String(r(y)))
73}
74
75export function swipe(d: Device, a: [number, number], b: [number, number], ms: number): Argv {
76  const n = [a[0], a[1], b[0], b[1]].map(v => String(r(v)))
77  return d.platform === 'ios'
78    ? sh('axe swipe --start-x "$2" --start-y "$3" --end-x "$4" --end-y "$5" --duration "$6" --udid "$1"', d.id, ...n, String(ms / 1000))
79    : sh('adb -s "$1" shell input swipe "$2" "$3" "$4" "$5" "$6"', d.id, ...n, String(r(ms)))
80}
81
82// `adb shell input text` goes through the device's shell: spaces become %s and
83// the whole thing is single-quoted there.
84export function text(d: Device, s: string): Argv {
85  if (d.platform === 'ios') return sh('printf %s "$2" | axe type --stdin --udid "$1"', d.id, s)
86  const quoted = "'" + s.replace(/%/g, '\\%').replace(/ /g, '%s').replace(/'/g, `'\\''`) + "'"
87  return sh('adb -s "$1" shell input text "$2"', d.id, quoted)
88}
89
90export type Key = 'home' | 'back' | 'lock' | 'return' | 'backspace' | 'tab' | 'up' | 'down' | 'left' | 'right' | 'apps'
91// iOS: HID usage codes for keys, AXe buttons for hardware. Android: keyevents.
92const IOS_KEY: Partial<Record<Key, number>> = { return: 40, backspace: 42, tab: 43, right: 79, left: 80, down: 81, up: 82 }
93const IOS_BUTTON: Partial<Record<Key, string>> = { home: 'home', lock: 'lock' }
94const ANDROID_KEY: Record<Key, number> = { home: 3, back: 4, lock: 26, return: 66, backspace: 67, tab: 61, up: 19, down: 20, left: 21, right: 22, apps: 187 }
95
96export function key(d: Device, k: Key): Argv | null {
97  if (d.platform === 'android') return sh('adb -s "$1" shell input keyevent "$2"', d.id, String(ANDROID_KEY[k]))
98  if (IOS_BUTTON[k]) return sh('axe button "$2" --udid "$1"', d.id, IOS_BUTTON[k]!)
99  if (IOS_KEY[k]) return sh('axe key "$2" --udid "$1"', d.id, String(IOS_KEY[k]))
100  return null // no back or app switcher button on iOS
101}
102
103// The biggest box of cells that keeps the frame's shape; a cell is about twice
104// as tall as wide.
105export function fit(width: number, height: number, cols: number, rows: number): { cols: number; rows: number } {
106  const a = height / width
107  const byRows = Math.max(4, Math.min(rows, Math.floor((cols * a) / 2)))
108  return { cols: Math.max(4, Math.min(cols, Math.round((byRows * 2) / a))), rows: byRows }
109}
110

1import type { ClientModule } from 'claude-code'
2
3// An empty box over the device screen: it reports where a press starts and
4// ends (as fractions of the box), and the keys, so the mod can tap or swipe.
5export type OverlayProps = { cols: number; rows: number }
6
7const Overlay: ClientModule<OverlayProps> = (props, surface) => {
8  const { Box } = surface.elements
9  surface.onPointer(e => {
10    const at: [number, number] = [(e.fine?.x ?? e.x + 0.5) / props.cols, (e.fine?.y ?? e.y + 0.5) / props.rows]
11    if (e.type === 'down' && (e.button ?? 'left') === 'left') surface.post({ down: at })
12    else if (e.type === 'up') surface.post({ up: at })
13  })
14  surface.onKey(e => surface.post({ key: e.key }))
15  return <Box width={props.cols} height={props.rows} />
16}
17
18export default Overlay
19