mobile-mcp

1import { atom, read, update } from "claude-code";
2import type { EngineInterface, Register } from "claude-code";
3
4import type { Shot, Target } from "../types";
5import { keysToActions } from "./keys";
6import { fitInside, pngSize } from "./png";
7
8// the key of the MCP server in this plugin's manifest
9const MCP_SERVER = "mobile-mcp";
10const COMMAND = "mobile-mirror";
11const PANE = "mobile-mirror";
12const VIEW_KEY = "view";
13const TAP_KEY = "tap";
14// frames rotate through a few files, so the terminal never reads one being rewritten
15const FRAME_FILES = 3;
16// large enough that the terminal never upscales it on a Retina pane; smaller is faster
17const SHOT_MAX_SIZE = 1600;
18const RETRY_AFTER_ERROR_MS = 1000;
19// the toolbar row, plus the URL field's row while it shows
20const TOOLBAR_ROWS = 1;
21const URL_FIELD_ROWS = 1;
22
23// kept in session state, not module variables, so they survive a reload of the mod
24const target = atom({ plugin: "mobile-mcp", key: "target" } as const, null as Target | null);
25// bumped to stop the running frame loop: a loop runs only while it holds the current value
26const streamId = atom({ plugin: "mobile-mcp", key: "streamId" } as const, 0);
27const isAskingUrl = atom({ plugin: "mobile-mcp", key: "isAskingUrl" } as const, false);
28const shot = atom({ plugin: "mobile-mcp", key: "shot" } as const, { file: "", generation: 0, width: 1, height: 1 } as Shot);
29
30type Device = { id: string; platform: string; state: string };
31type TapMessage = { x: number; y: number };
32type KeysMessage = { keys: string[] };
33// the runtime has Uint8Array.fromBase64; the TS lib does not declare it yet
34type Base64Decoder = { fromBase64(base64: string): Uint8Array };
35type McpResult = Awaited<ReturnType<EngineInterface["mcp"]["call"]>>;
36
37function firstText(content: unknown): string {
38	const block = (content as { type: string; text?: string }[]).find(b => b.type === "text");
39
40	return block?.text ?? "";
41}
42
43function frameFile(generation: number): string {
44	return `/tmp/mobile-mirror-${generation % FRAME_FILES}.png`;
45}
46
47// calls a mobile-mcp tool by the name this session runs the server under, connecting it on first use
48async function callMobileMcp($: EngineInterface, tool: string, args: Record<string, unknown> = {}): Promise<McpResult> {
49	const connection = await $.mcp.connect(MCP_SERVER);
50	if (!connection.isConnected) {
51		throw new Error(`mobile-mcp is not connected: ${connection.message}`);
52	}
53
54	return $.mcp.call(connection.server, tool, args);
55}
56
57// runs a toolbar action, and toasts only when it fails
58async function runAction($: EngineInterface, what: string, tool: string, args: Record<string, unknown>): Promise<void> {
59	try {
60		const result = await callMobileMcp($, tool, args);
61		if (result.isError) {
62			$.ui.toast(`${what} failed: ${firstText(result.content)}`);
63		}
64
65	} catch (error) {
66		$.ui.toast(`${what} failed: ${String(error)}`);
67	}
68}
69
70function pickDevice(devices: Device[], wanted: string | undefined): Device | undefined {
71	const online = devices.filter(d => d.state === "online");
72	if (wanted) {
73		return online.find(d => d.id === wanted);
74	}
75
76	return online.find(d => d.platform === "android") ?? online[0];
77}
78
79export const register: Register = on => {
80	// key batches are sent one after another, so typed text keeps its order
81	let typing: Promise<void> = Promise.resolve();
82
83	on("session.start", async ($, e, next) => {
84		await $.command.register({ name: COMMAND, description: "Mirror a device screen: live picture, taps, keys and buttons", argumentHint: "[device-id]" });
85
86		return next(e);
87	});
88
89	on("command.run", { command: COMMAND }, async ($, e) => {
90		const wanted = e.args.trim() || undefined;
91		const listed = await callMobileMcp($, "mobile_list_available_devices");
92		const { devices } = JSON.parse(firstText(listed.content)) as { devices: Device[] };
93		const device = pickDevice(devices, wanted);
94		if (!device) {
95			const online = devices.filter(d => d.state === "online").map(d => d.id);
96			return { text: `No online device${wanted ? ` "${wanted}"` : ""}. Online: ${online.join(", ") || "none"}` };
97		}
98
99		const sized = await callMobileMcp($, "mobile_get_screen_size", { device: device.id });
100		const match = /(\d+)x(\d+)/.exec(firstText(sized.content));
101		if (!match) {
102			return { text: `Could not read the screen size of ${device.id}.` };
103		}
104
105		await update($, target, () => ({
106			id: device.id,
107			platform: device.platform === "android" ? "android" : "ios",
108			screenWidth: Number(match[1]),
109			screenHeight: Number(match[2]),
110		}));
111		await update($, isAskingUrl, () => false);
112		const id = await update($, streamId, n => n + 1);
113
114		const captureFrame = async (generation: number): Promise<boolean> => {
115			const file = frameFile(generation);
116			const saved = await callMobileMcp($, "mobile_save_screenshot", { device: device.id, saveTo: file, maxSize: SHOT_MAX_SIZE });
117			if (saved.isError) {
118				return false;
119			}
120
121			const { base64 } = await $.fs.read(file, { as: "bytes" });
122			const size = pngSize((Uint8Array as unknown as Base64Decoder).fromBase64(base64));
123			const current = await read($, shot);
124			// a new shape (first frame, rotation) needs a full redraw; otherwise swap the pixels in place
125			if (current.generation === 0 || current.width !== size.width || current.height !== size.height) {
126				await update($, shot, () => ({ file, generation, ...size }));
127			} else {
128				await $.ui.blit({ requestId: PANE, key: VIEW_KEY, source: { file, format: "png", generation } }).catch(() => {});
129			}
130
131			return true;
132		};
133
134		const stream = async () => {
135			// the next frame is asked for as soon as the previous one is shown
136			for (let generation = 1; id === (await read($, streamId)); generation++) {
137				const isShown = await captureFrame(generation).catch(() => false);
138				if (!isShown) {
139					await $.clock.sleep(RETRY_AFTER_ERROR_MS);
140				}
141
142			}
143		};
144
145		await update($, shot, () => ({ file: "", generation: 0, width: 1, height: 1 }));
146		void stream();
147		await $.ui.open({ id: PANE, title: `mobile-mirror · ${device.id}` });
148
149		return { text: `Mirroring ${device.id}. Click the picture to tap, then type to send keys.` };
150	});
151
152	on("ui.message", async ($, e) => {
153		const device = await read($, target);
154		if (e.element !== TAP_KEY || !device) {
155			return {};
156		}
157
158		if ("keys" in (e.data as object)) {
159			const actions = keysToActions((e.data as KeysMessage).keys, device.platform);
160			typing = typing.then(async () => {
161				for (const action of actions) {
162					if (action.type === "text") {
163						await runAction($, "Typing", "mobile_type_keys", { device: device.id, text: action.text, submit: false });
164					} else {
165						await runAction($, `Press ${action.button}`, "mobile_press_button", { device: device.id, button: action.button });
166					}
167
168				}
169			});
170
171			return {};
172		}
173
174		const tap = e.data as TapMessage;
175		const x = Math.round(tap.x * device.screenWidth);
176		const y = Math.round(tap.y * device.screenHeight);
177		await runAction($, "Tap", "mobile_click_on_screen_at_coordinates", { device: device.id, x, y });
178
179		return {};
180	});
181
182	on("ui.close", async ($, e, next) => {
183		if (e.id === PANE) {
184			await update($, streamId, n => n + 1);
185		}
186
187		return next(e);
188	});
189
190	on("ui.render", { component: "Pane", requestId: PANE }, async ($, e) => {
191		const ui = $.ui.resolve(e);
192		const { Box, Text } = ui;
193		if (e.surface !== "terminal" || !("Image" in ui)) {
194			return <Text>mobile-mirror needs the terminal, in kitty or Ghostty.</Text>;
195		}
196
197		const { file, generation, width, height } = await read($, shot);
198		if (generation === 0) {
199			return <Text dimColor>Waiting for the first frame…</Text>;
200		}
201
202		const device = await read($, target);
203		if (!device) {
204			return <Text dimColor>Run /mobile-mirror to pick a device.</Text>;
205		}
206
207		const isAndroid = device.platform === "android";
208		const isAsking = await read($, isAskingUrl);
209		const roomForImage = e.props.scroll.bodyRows - TOOLBAR_ROWS - (isAsking ? URL_FIELD_ROWS : 0);
210		const size = fitInside(e.props.bodyColumns, roomForImage, width, height);
211		const { Button, Input } = ui;
212
213		const press = (button: string) => () => runAction($, `Press ${button}`, "mobile_press_button", { device: device.id, button });
214
215		const openUrl = async (url: string) => {
216			await update($, isAskingUrl, () => false);
217			if (!url.trim()) {
218				return;
219			}
220
221			await runAction($, `Open ${url.trim()}`, "mobile_open_url", { device: device.id, url: url.trim() });
222		};
223
224		return (
225			<Box flexDirection='column'>
226				<Box flexDirection='row' gap={1}>
227					<Button key='home' label='Home' onPress={press("HOME")} />
228					{isAndroid && <Button key='back' label='Back' onPress={press("BACK")} />}
229					{isAndroid && <Button key='app-switch' label='App Switch' onPress={press("APP_SWITCH")} />}
230					<Button key='url' label='URL' onPress={() => update($, isAskingUrl, asking => !asking)} />
231				</Box>
232				{isAsking && <Input key='url-field' label='URL ' placeholder='https://example.com' submitLabel='open' autoFocus onSubmit={openUrl} />}
233				<Box>
234					<ui.Image key={VIEW_KEY} source={{ file, format: "png", generation }} columns={size.columns} rows={size.rows} alt='device screen' />
235					<Box position='absolute' top={0} left={0}>
236						<ui.Client key={TAP_KEY} module='./tap.tsx' width={size.columns} height={size.rows} />
237					</Box>
238				</Box>
239			</Box>
240		);
241	});
242};
243

1import type { Platform } from "../types";
2
3export type KeyAction = { type: "text"; text: string } | { type: "button"; button: string };
4
5// special key names the terminal reports, mapped to mobile_press_button buttons
6const ANDROID_BUTTONS: Record<string, string> = {
7	return: "ENTER",
8	backspace: "BACKSPACE",
9	delete: "BACKSPACE",
10	up: "DPAD_UP",
11	down: "DPAD_DOWN",
12	left: "DPAD_LEFT",
13	right: "DPAD_RIGHT",
14};
15
16const IOS_BUTTONS: Record<string, string> = {
17	return: "ENTER",
18};
19
20// ponytail: iOS has no BACKSPACE button in mobilecli, so it is typed as \b; unverified on a device
21const IOS_TEXT: Record<string, string> = {
22	backspace: "\b",
23	delete: "\b",
24};
25
26function actionFor(key: string, platform: Platform): KeyAction | undefined {
27	const buttons = platform === "android" ? ANDROID_BUTTONS : IOS_BUTTONS;
28	const button = buttons[key];
29	if (button) {
30		return { type: "button", button };
31	}
32
33	const text = platform === "ios" ? IOS_TEXT[key] : undefined;
34	if (text) {
35		return { type: "text", text };
36	}
37
38	if (key === "space") {
39		return { type: "text", text: " " };
40	}
41
42	// a single character is typed as is; any other named key is dropped
43	if ([...key].length === 1) {
44		return { type: "text", text: key };
45	}
46
47	return undefined;
48}
49
50// turns pressed keys into device actions, joining consecutive text into one type call
51export function keysToActions(keys: string[], platform: Platform): KeyAction[] {
52	const actions: KeyAction[] = [];
53	for (const key of keys) {
54		const action = actionFor(key, platform);
55		if (!action) {
56			continue;
57		}
58
59		const last = actions[actions.length - 1];
60		if (action.type === "text" && last?.type === "text") {
61			actions[actions.length - 1] = { type: "text", text: last.text + action.text };
62		} else {
63			actions.push(action);
64		}
65
66	}
67
68	return actions;
69}
70

1// width and height from a PNG's IHDR chunk, big-endian at bytes 16..23
2export function pngSize(bytes: Uint8Array): { width: number; height: number } {
3	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
4
5	return { width: view.getUint32(16), height: view.getUint32(20) };
6}
7
8const MAX_CELLS = 255;
9// cell height / cell width of the terminal font, measured 17.3px / 8.1px; no API reports it, so tune per font
10export const CELL_ASPECT = 2.14;
11
12function clampCells(n: number): number {
13	return Math.max(1, Math.min(MAX_CELLS, Math.floor(n)));
14}
15
16// the largest box of cells that keeps the picture's aspect and fits inside maxColumns x maxRows
17export function fitInside(columnsLimit: number, rowsLimit: number, width: number, height: number): { columns: number; rows: number } {
18	const maxColumns = clampCells(columnsLimit);
19	const maxRows = clampCells(rowsLimit);
20	// a broken PNG header can report zero; without an aspect to keep, fill the box instead of dividing by zero
21	if (width <= 0 || height <= 0) {
22		return { columns: maxColumns, rows: maxRows };
23	}
24
25	const rowsAtFullWidth = (maxColumns * height) / width / CELL_ASPECT;
26	if (rowsAtFullWidth <= maxRows) {
27		return { columns: clampCells(maxColumns), rows: clampCells(rowsAtFullWidth) };
28	}
29
30	const columnsAtFullHeight = (maxRows * width * CELL_ASPECT) / height;
31
32	return { columns: clampCells(columnsAtFullHeight), rows: clampCells(maxRows) };
33}
34

1import type { ClientKeyEvent, ClientPointerEvent, ClientSurface } from "claude-code";
2
3type Pending = { keys: string[] };
4
5// a post per frame at most, and a later one replaces an undelivered one, so keys are sent in batches
6const FLUSH_MS = 50;
7
8// Laid over the screenshot: posts each left click as a fraction of the picture (0..1 on both axes),
9// and, once clicked, the keys typed while it holds the focus
10export default function TapInput(_props: unknown, surface: ClientSurface<Pending>) {
11	if (surface.state === undefined) {
12		const pending: Pending = { keys: [] };
13
14		surface.onPointer((e: ClientPointerEvent) => {
15			if (e.type !== "down" || e.button !== "left" || surface.columns === 0 || surface.rows === 0) {
16				return;
17			}
18
19			const x = e.fine?.x ?? e.x + 0.5;
20			const y = e.fine?.y ?? e.y + 0.5;
21			surface.post({ x: x / surface.columns, y: y / surface.rows });
22		});
23
24		surface.onKey((e: ClientKeyEvent) => {
25			pending.keys.push(e.key);
26		});
27
28		surface.every(FLUSH_MS, () => {
29			if (pending.keys.length === 0) {
30				return;
31			}
32
33			surface.post({ keys: pending.keys.splice(0) });
34		});
35
36		surface.setState(pending);
37	}
38
39	const { Box } = surface.elements;
40
41	return <Box width='100%' height='100%' />;
42}
43

1export type Shot = { file: string; generation: number; width: number; height: number };
2export type Platform = "android" | "ios";
3export type Target = { id: string; platform: Platform; screenWidth: number; screenHeight: number };
4
5declare module "claude-code" {
6	interface PluginState {
7		"mobile-mcp": { shot: Shot; target: Target | null; streamId: number; isAskingUrl: boolean }
8	}
9}
10