terminal-browser

1/* @jsx h */
2import type { EngineInterface, Register } from 'claude-code'
3import type { Browser } from './browser'
4import { MAX_IMAGE_CELLS } from './surface.tsx'
5import { normalizeUrl } from './urls.ts'
6import {
7  isBridgeState,
8  isInputMessage,
9  isLaunchReport,
10  isSizeMessage,
11  takenItems,
12  type BridgeState,
13  type Frame,
14} from './bridge-protocol.ts'
15
16
17
18const PANE = 'browser'
19const IMAGE_KEY = 'view'
20const IMAGE_ALT = ''
21const OPEN_TOOL = 'mcp__terminal-browser__open'
22const CLOSE_TOOL = 'mcp__terminal-browser__close'
23const START_URL = 'terminal-browser://start'
24const INSTALL_URL = 'https://terminal-browser.sh'
25const REQUIRED_CAPABILITIES = ['embedding', 'image-frames']
26const WATCH_RETRY_MS = 250
27const DENIED_REDRAW_MS = 500
28const DENIALS_BEFORE_LOGGING = 3
29
30
31const state = {
32  port: null as number | null,
33  token: null as string | null,
34  open: false,
35  pendingUrl: null as string | null,
36  region: null as { cols: number; rows: number } | null,
37  last: null as BridgeState | null,
38  frame: null as Frame | null,
39  mounted: null as { cols: number; rows: number } | null,
40  watcher: 0,
41  // a hack to programatically trigger agent input focus
42  viewGeneration: 0,
43}
44
45const bridgeUrl = (path: string) => `http://127.0.0.1:${state.port}${path}`
46const authHeaders = () => ({ authorization: `Bearer ${state.token}` })
47const viewKey = () => `view${state.viewGeneration}`
48
49
50async function terminalBrowserCommand($: EngineInterface): Promise<string[]> {
51  const checkoutCli = `${$.plugin.root}/../cli/dist/main.js`
52  if (await $.fs.exists(checkoutCli)) return ['node', checkoutCli] // dev case
53  return ['terminal-browser']
54}
55
56async function checkCapabilities($: EngineInterface, command: string[]): Promise<{ ok: true } | { ok: false; installed: boolean }> {
57  let result: { stdout: string; exitCode: number }
58  try {
59    result = await $.process.run([...command, 'capabilities'], { timeoutMs: 10_000 })
60  } catch {
61    return { ok: false, installed: false }
62  }
63  let capabilities: string[] = []
64  if (result.exitCode === 0) {
65    const line = result.stdout.split('\n').find((text: string) => text.startsWith('{'))
66    try {
67      const parsed = line ? JSON.parse(line) : null
68      if (parsed && Array.isArray(parsed.capabilities)) capabilities = parsed.capabilities
69    } catch {}
70  }
71  const ok = REQUIRED_CAPABILITIES.every(need => capabilities.includes(need))
72  return ok ? { ok: true } : { ok: false, installed: true }
73}
74
75async function startBridge($: EngineInterface): Promise<{ ok: true } | { ok: false; error: string }> {
76  const command = await terminalBrowserCommand($)
77  const check = await checkCapabilities($, command)
78  if (!check.ok) {
79    return {
80      ok: false,
81      error: check.installed
82        ? 'Newer terminal-browser version required, run terminal-browser upgrade'
83        : `Please install terminal-browser first - ${INSTALL_URL}`,
84    }
85  }
86  let report: unknown = null
87  try {
88    const { stdout, stderr, exitCode } = await $.process.run([...command, 'claude-bridge', 'launch'], { timeoutMs: 20_000 })
89    const line = stdout.split('\n').find((text: string) => text.startsWith('{'))
90    report = line ? JSON.parse(line) : { error: stderr.trim() || `exit ${exitCode}`, code: 'start' }
91  } catch (err) {
92    report = { error: String(err), code: 'start' }
93  }
94  if (!isLaunchReport(report) || !('port' in report)) {
95    const detail = isLaunchReport(report) && 'error' in report ? report.error : 'terminal-browser could not start'
96    return { ok: false, error: `${detail}` }
97  }
98  state.port = report.port
99  state.token = report.token
100  void watchBridge($)
101  return { ok: true }
102}
103
104async function post($: EngineInterface, path: string, body: unknown): Promise<unknown> {
105  if (state.port === null) return null
106  try {
107    const response = await $.http.fetch(bridgeUrl(path), {
108      method: 'POST',
109      headers: { 'content-type': 'application/json', ...authHeaders() },
110      body: JSON.stringify(body),
111    })
112    return response.ok ? JSON.parse(response.text || '{}') : null
113  } catch {
114    return null
115  }
116}
117
118async function fetchState($: EngineInterface, version: number): Promise<BridgeState | null> {
119  if (state.port === null) return null
120  try {
121    const response = await $.http.fetch(bridgeUrl(`/state?version=${version}`), { headers: authHeaders() })
122    const parsed: unknown = response.ok ? JSON.parse(response.text) : null
123    return isBridgeState(parsed) ? parsed : null
124  } catch {
125    return null
126  }
127}
128
129
130function imageSource(frame: Frame) {
131  return { shm: frame.shm, format: frame.format, width: frame.width, height: frame.height, generation: frame.generation }
132}
133
134function imageGrid(frame: Frame, cols: number, rows: number) {
135  return {
136    cols: Math.max(1, Math.min(frame.cols, cols, MAX_IMAGE_CELLS)),
137    rows: Math.max(1, Math.min(frame.rows, rows, MAX_IMAGE_CELLS)),
138  }
139}
140
141function fitsMounted(frame: Frame): boolean {
142  const mounted = state.mounted
143  if (!mounted || !state.region) return false
144  const grid = imageGrid(frame, state.region.cols, state.region.rows)
145  return grid.cols === mounted.cols && grid.rows === mounted.rows
146}
147
148async function watchBridge($: EngineInterface): Promise<void> {
149  const watcher = ++state.watcher
150  const live = () => state.watcher === watcher && state.port !== null
151  let version = -1
152  let denials = 0
153  let lastDeniedRedraw = 0
154  while (live()) {
155    const fresh = await fetchState($, version)
156    if (!live()) return
157    if (!fresh) {
158      await $.clock.sleep(WATCH_RETRY_MS)
159      continue
160    }
161    version = fresh.version
162    const previous = state.last
163    state.last = fresh
164    if (fresh.inbox > 0) await deliverAgentText($)
165    if (!state.open) continue
166    const pictureGone = fresh.frame === null && state.frame !== null
167    if (pictureGone) state.frame = null
168    const paneChanged =
169      !previous
170      || pictureGone
171      || previous.title !== fresh.title
172      || previous.alive !== fresh.alive
173      || previous.error !== fresh.error
174    if (paneChanged) {
175      $.ui.invalidate('ui.render')
176      if (fresh.title) await $.ui.open({ id: PANE, title: fresh.title.slice(0, 40) })
177    }
178    const frame = fresh.frame
179    if (!frame || frame.generation === state.frame?.generation) continue
180    state.frame = frame
181    if (!fitsMounted(frame)) {
182      $.ui.invalidate('ui.render')
183      continue
184    }
185    const result = await $.ui.blit({ requestId: PANE, key: IMAGE_KEY, source: imageSource(frame) })
186    if (!result.deny) {
187      denials = 0
188      continue
189    }
190    denials += 1
191    if (denials === DENIALS_BEFORE_LOGGING) $.ui.log(`terminal-browser: frame blit refused: ${result.deny}`)
192    const now = Date.now()
193    if (now - lastDeniedRedraw >= DENIED_REDRAW_MS) {
194      lastDeniedRedraw = now
195      $.ui.invalidate('ui.render')
196    }
197  }
198}
199
200async function openBrowser($: EngineInterface, raw: string | null): Promise<{ ok: true; url: string } | { ok: false; error: string }> {
201  if (state.port === null) {
202    const started = await startBridge($)
203    if (!started.ok) return started
204  }
205  const alive = state.last?.alive === true
206  const url = raw ? normalizeUrl(raw) : (alive && state.last?.url ? state.last.url : START_URL)
207  state.pendingUrl = url
208  state.open = true
209  await $.ui.open({ id: PANE, title: 'browser', focus: true, rows: 24 })
210  $.ui.invalidate('ui.render')
211  if (state.region) {
212    state.pendingUrl = null
213    await post($, '/open', { url, ...state.region })
214  }
215  return { ok: true, url }
216}
217
218async function closeBrowser($: EngineInterface): Promise<boolean> {
219  if (!state.open) return false
220  await $.ui.close({ id: PANE })
221  return true
222}
223
224async function browserClosed($: EngineInterface): Promise<void> {
225  state.open = false
226  state.pendingUrl = null
227  state.region = null
228  state.frame = null
229  state.mounted = null
230  await post($, '/browser/close', {})
231}
232
233async function deliverAgentText($: EngineInterface): Promise<void> {
234  const oneLine = (text: string) => text.replace(/[\x00-\x1f\x7f-\x9f]/g, ' ').replace(/\s+/g, ' ').trim()
235  const lines = takenItems(await post($, '/inbox/take', {}))
236    .filter(item => item.text.trim() !== '')
237    .flatMap(item => [oneLine(item.text), ...(item.screenshot ? [oneLine(item.screenshot)] : [])])
238  if (lines.length === 0) return
239  const { isFilled } = await $.prompt.fill({ text: `${lines.join('\n')}\n` })
240  if (!isFilled) {
241    // would need to see a case this happens before shipping
242    // $.ui.toast('copied to clipboard')
243    return
244  }
245  state.viewGeneration += 1
246  $.ui.invalidate('ui.render')
247}
248
249
250export const register: Register = (on, options) => {
251  const agentToolEnabled = options.agentTool === true
252
253// this is a hack because the plugin api does not allow $ or the result of next to be passed to functions
254  on('engine.create', async ($, e, next) => {
255    const built = await next(e)
256    const servedByHooks = () => { throw new Error('the browser noun is served by its hooks') }
257    const browser: Browser = { open: servedByHooks, close: servedByHooks }
258    return { ...built, browser }
259  })
260
261  on('browser.open', async ($, e) => {
262    const opened = await openBrowser($, e.url ?? null)
263    return { value: opened }
264  })
265
266  on('browser.close', async ($) => {
267    return { value: await closeBrowser($) }
268  })
269
270  on('session.start', async ($, e, next) => {
271    const r = await next(e)
272    let surfaces: readonly string[] = []
273    try {
274      surfaces = await $.session.surfaces()
275    } catch {}
276    if (!surfaces.includes('terminal')) return r
277    await $.command.register({
278      name: 'browser',
279      description: 'Open a browser to the right',
280      argumentHint: '[url]',
281      immediate: true,
282    }).catch(err => $.ui.log(`terminal-browser: /browser not registered: ${err}`))
283    if (agentToolEnabled) {
284      await $.tool.register({
285        name: 'open',
286        description: 'Open terminal-browser directly inside claude code. Control the open page with the terminal-browser action CLI.',
287        inputSchema: { type: 'object', properties: { url: { type: 'string', description: 'The page to open, as a full url or a host name' } } },
288      }).catch(err => $.ui.log(`terminal-browser: open tool not registered: ${err}`))
289      await $.tool.register({ name: 'close', description: 'Close the terminal-browser pane.' }).catch(err => $.ui.log(`terminal-browser: close tool not registered: ${err}`))
290    }
291    return r
292  })
293
294  on('command.run', { command: 'browser' }, async ($, e) => {
295    const arg = e.args.trim()
296    if (arg === 'close' || (!arg && state.open)) {
297      await closeBrowser($)
298      return { text: 'Closed terminal-browser' }
299    }
300    const opened = await openBrowser($, arg || null)
301    return { text: opened.ok ? 'Opened terminal-browser' : opened.error }
302  })
303
304  on('tool.call', { tool: new RegExp(`^${OPEN_TOOL}$`) }, async ($, e) => {
305    const url = (e as { url?: unknown }).url
306    const opened = await openBrowser($, typeof url === 'string' ? url : null)
307    if (!opened.ok) return { deny: `could not open the browser: ${opened.error}` }
308    return { result: [{ type: 'text', text: `opened ${opened.url} ` }] }
309  })
310
311  on('tool.call', { tool: new RegExp(`^${CLOSE_TOOL}$`) }, async ($) => {
312    const closed = await closeBrowser($)
313    return { result: [{ type: 'text', text: closed ? 'browser pane closed' : 'no browser pane was open (maybe the user closed it?)' }] }
314  })
315
316  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
317    if (e.surface !== 'terminal') {
318      const { Box } = await $.ui.resolve(e)
319      return <Box />
320    }
321    const { Box, Client, Image } = await $.ui.resolve(e)
322    const rows = e.props.scroll.bodyRows > 0 ? e.props.scroll.bodyRows : Math.max(8, (e.viewport?.rows ?? 30) - 8)
323    const cols = Math.max(1, e.props.bodyColumns > 0 ? e.props.bodyColumns : (e.viewport?.columns ?? 80))
324    const frame = state.frame
325    const grid = frame ? imageGrid(frame, cols, rows) : null
326    state.mounted = grid
327    return (
328      <Box flexDirection="column" width={cols} height={rows}>
329        {frame && grid ? (
330          <Image key={IMAGE_KEY} source={imageSource(frame)} columns={grid.cols} rows={grid.rows} alt={IMAGE_ALT} />
331        ) : null}
332        <Box position="absolute" top={0} left={0} width={cols} height={rows}>
333          <Client key={viewKey()} module="./surface.tsx" width={cols} height={rows} />
334        </Box>
335      </Box>
336    )
337  })
338
339  on('ui.message', async ($, e, next) => {
340    if (e.requestId !== PANE || state.port === null) return next(e)
341    if (isSizeMessage(e.data)) {
342      state.region = { cols: e.data.cols, rows: e.data.rows }
343      if (state.pendingUrl) {
344        const url = state.pendingUrl
345        state.pendingUrl = null
346        await post($, '/open', { url, ...state.region })
347      } else {
348        await post($, '/size', state.region)
349      }
350    } else if (isInputMessage(e.data)) {
351      await post($, '/input', { events: e.data.events })
352    }
353    return next(e)
354  })
355
356  on('ui.scroll', { requestId: PANE }, async ($, e) => {
357    if (state.port === null || !state.open || e.by === 0) return {}
358    const kind = e.by < 0 ? 'scrollup' : 'scrolldown'
359    const ticks = Math.min(Math.round(Math.abs(e.by)) || 1, 10)
360    const x = e.pointer?.column ?? Math.floor((state.region?.cols ?? 0) / 2)
361    const y = e.pointer?.row ?? Math.floor((state.region?.rows ?? 0) / 2)
362    const events = Array.from({ length: ticks }, () => ({ type: 'mouse', kind, x, y }))
363    await post($, '/input', { events })
364    return {}
365  })
366
367  on('ui.close', { id: PANE }, async ($, e, next) => {
368    const r = await next(e)
369    await browserClosed($)
370    return r
371  })
372}
373

1/* @jsx h */
2import type { ClientKeyEvent, ClientPointerEvent, ClientSurface } from 'claude-code'
3
4export const MAX_IMAGE_CELLS = 255
5
6type State = { cols: number; rows: number }
7
8type InputEvent =
9  | { type: 'mouse'; kind: ClientPointerEvent['type']; button?: string; x: number; y: number; mods: Mods }
10  | { type: 'key'; key: string; text?: string; mods: Mods }
11type Mods = { shift: boolean; alt: boolean; ctrl: boolean; super: boolean }
12
13const KEY_NAMES: Record<string, string> = {
14  return: 'enter',
15  enter: 'enter',
16  backspace: 'backspace',
17  delete: 'delete',
18  tab: 'tab',
19  up: 'up',
20  down: 'down',
21  left: 'left',
22  right: 'right',
23  home: 'home',
24  end: 'end',
25  pageup: 'pageup',
26  pagedown: 'pagedown',
27  insert: 'insert',
28  space: ' ',
29}
30
31function keyEvent(event: ClientKeyEvent): InputEvent | null {
32  const mods: Mods = { shift: Boolean(event.shift), alt: false, ctrl: Boolean(event.ctrl), super: Boolean(event.meta) }
33  const named = KEY_NAMES[event.key.toLowerCase()]
34  if (named !== undefined) {
35    return { type: 'key', key: named, text: named.length === 1 && !mods.ctrl ? named : undefined, mods }
36  }
37  if ([...event.key].length === 1) {
38    return { type: 'key', key: event.key, text: mods.ctrl || mods.super ? undefined : event.key, mods }
39  }
40  const fn = /^f(\d{1,2})$/i.exec(event.key)
41  if (fn) return { type: 'key', key: `f${fn[1]}`, mods }
42  return null
43}
44
45function pointerEvent(event: ClientPointerEvent): InputEvent | null {
46  const mods: Mods = { shift: Boolean(event.shift), alt: Boolean(event.alt), ctrl: Boolean(event.ctrl), super: false }
47  if (event.type === 'enter' || event.type === 'leave') return { type: 'mouse', kind: 'move', x: event.x, y: event.y, mods }
48  return { type: 'mouse', kind: event.type, button: event.button, x: event.x, y: event.y, mods }
49}
50
51export default function Browser(_props: unknown, surface: ClientSurface<State>) {
52  const { Box } = surface.elements
53  const queue: InputEvent[] = []
54
55  if (surface.state === undefined) {
56    surface.setState({ cols: 0, rows: 0 })
57    surface.onPointer(event => {
58      const mapped = pointerEvent(event)
59      if (mapped) queue.push(mapped)
60    })
61    surface.onKey(event => {
62      const mapped = keyEvent(event)
63      if (mapped) queue.push(mapped)
64    })
65
66    surface.every(20, () => {
67      if (queue.length === 0) return
68      surface.post({ type: 'input', events: queue.splice(0, queue.length) as unknown as never })
69    })
70  }
71
72  const cols = Math.min(surface.columns, MAX_IMAGE_CELLS)
73  const rows = Math.min(surface.rows, MAX_IMAGE_CELLS)
74  if (cols > 0 && rows > 0 && surface.state && (surface.state.cols !== cols || surface.state.rows !== rows)) {
75    surface.setState({ cols, rows })
76    surface.post({ type: 'size', cols, rows })
77  }
78
79  return <Box flexDirection="column" height="100%" />
80}
81

1export function normalizeUrl(raw: string): string {
2  const text = raw.trim()
3  if (/^[a-z][a-z0-9+.-]*:/i.test(text)) return text
4  if (/^localhost(:\d+)?(\/|$)/.test(text) || /^\d+\.\d+\.\d+\.\d+/.test(text)) return `http://${text}`
5  return `https://${text}`
6}
7

1// we cannot use libraries inside the claude code sandbox, hence this gross code
2
3
4export type Frame = {
5  shm: string
6  format: 'rgba' | 'rgb'
7  width: number
8  height: number
9  generation: number
10  cols: number
11  rows: number
12}
13
14export type BridgeState = {
15  version: number
16  frame: Frame | null
17  title: string
18  url: string | null
19  alive: boolean
20  error: string | null
21  inbox: number
22}
23
24export type LaunchReport =
25  | { port: number; token: string }
26  | { error: string; code: 'tty' | 'start' }
27
28export type SizeMessage = { type: 'size'; cols: number; rows: number }
29export type InputMessage = { type: 'input'; events: unknown[] }
30
31const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === 'object' && value !== null
32
33export const isFrame = (value: unknown): value is Frame =>
34  isRecord(value)
35  && typeof value.shm === 'string'
36  && (value.format === 'rgba' || value.format === 'rgb')
37  && Number.isInteger(value.width)
38  && Number.isInteger(value.height)
39  && Number.isInteger(value.generation)
40  && Number.isInteger(value.cols)
41  && Number.isInteger(value.rows)
42
43export const isBridgeState = (value: unknown): value is BridgeState =>
44  isRecord(value)
45  && Number.isInteger(value.version)
46  && typeof value.alive === 'boolean'
47  && 'frame' in value
48  && (value.frame === null || isFrame(value.frame))
49
50export const isLaunchReport = (value: unknown): value is LaunchReport =>
51  isRecord(value) && (typeof value.port === 'number' || typeof value.error === 'string')
52
53export const isSizeMessage = (data: unknown): data is SizeMessage =>
54  isRecord(data) && data.type === 'size' && Number.isInteger(data.cols) && Number.isInteger(data.rows)
55
56export const isInputMessage = (data: unknown): data is InputMessage =>
57  isRecord(data) && data.type === 'input' && Array.isArray(data.events)
58
59export type AgentText = { text: string; screenshot: string | null }
60
61const isAgentText = (value: unknown): value is AgentText =>
62  isRecord(value) && typeof value.text === 'string' && (value.screenshot === null || typeof value.screenshot === 'string')
63
64export const takenItems = (value: unknown): AgentText[] =>
65  isRecord(value) && Array.isArray(value.items) ? value.items.filter(isAgentText) : []
66

1export type BrowserOpenInput = { url?: string }
2
3export type BrowserOpenResult =
4  | { ok: true; url: string }
5  | { ok: false; error: string }
6
7export type Browser = {
8  open: (input: BrowserOpenInput) => Promise<BrowserOpenResult>
9  close: (input?: Record<string, never>) => Promise<boolean>
10}
11
12declare module 'claude-code' {
13  interface EngineInterface {
14    browser: Browser
15  }
16}
17