3dgs-view

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, HookStream, ProcessSpawnChunk, ProcessSpawnResult, Register } from 'claude-code'
3
4import type { View } from '../types'
5import { pack, walk, type PadMessage } from './pad'
6
7const PANE = 'splat'
8const TMP = '/tmp/claude-splat-preview'
9const USAGE = 'Usage: /splat <file.ply|.splat|.spz|.ksplat|.sog>'
10const view = atom({ plugin: '3dgs-view', key: 'view' } as const, null as View | null)
11const offer = atom({ plugin: '3dgs-view', key: 'offer' } as const, null as string | null)
12const SPLAT_PATH = /[^\s"'`()[\]]+\.(?:ply|splat|spz|ksplat|sog)\b/gi
13// ponytail: module-level, so a reload forgets what was declined; good enough
14const seen = new Set<string>()
15
16// ponytail: module-level, what the pane last drew at; a reload starts from 80x25 until the next draw
17let grid = { columns: 80, rows: 25 }
18// ponytail: no capability query in the API, so the terminal's name says whether it draws images
19const drawsImages = (program: string, term: string) => program === 'ghostty' || program === 'WezTerm' || term === 'xterm-kitty'
20
21// ponytail: module-level, one server per session; a reload drops the handle and the engine kills the child
22let server: HookStream<ProcessSpawnChunk, ProcessSpawnResult> | undefined
23
24const resolve = async ($: EngineInterface, arg: string) => {
25  if (arg.startsWith('/')) return arg
26  if (arg.startsWith('~/')) return ((await $.env.get('HOME')) ?? '') + arg.slice(1)
27  return `${await $.session.cwd()}/${arg}`
28}
29
30// Any splat path in `text` that exists and was not offered yet becomes the band's offer.
31const detect = async ($: EngineInterface, text: string) => {
32  for (const hit of text.match(SPLAT_PATH) ?? []) {
33    const file = await resolve($, hit)
34    if (seen.has(file) || (await read($, view))?.file === file) continue
35    const stat = await $.fs.stat(file).catch(() => undefined)
36    if (stat?.kind !== 'file') continue
37    seen.add(file)
38    await update($, offer, () => file)
39    return
40  }
41}
42
43const patch = ($: EngineInterface, fn: (v: View) => Partial<View>) =>
44  update($, view, v => (v ? { ...v, ...fn(v) } : v))
45
46type Camera = { az: number; el: number; d: number; px: number; py: number; pz: number; mode: View['mode'] }
47const STEP = 0.08 // a w/s step in look mode, in units of the scene radius
48
49const camQuery = (at: Camera) => `az=${at.az}&el=${at.el}&d=${at.d.toFixed(2)}&px=${at.px}&py=${at.py}&pz=${at.pz}${at.mode === 'look' ? '&look=1' : ''}`
50
51const snap = async ($: EngineInterface, cam: Partial<Camera>, fast = false) => {
52  const v = await read($, view)
53  if (!v?.url) return
54  const mode = cam.mode ?? v.mode
55  const at: Camera = { az: cam.az ?? v.az, el: cam.el ?? v.el, d: Math.min(4, Math.max(0.1, cam.d ?? v.d)), px: cam.px ?? v.px, py: cam.py ?? v.py, pz: cam.pz ?? v.pz, mode }
56  await patch($, () => ({ ...at, status: fast ? 'dragging' : 'rendering' }))
57  // ponytail: curl is on every mac/linux box; the server owns the Chrome process
58  const ar = (grid.columns / (grid.rows * 2)).toFixed(3) // cells are about 1:2
59  await $.process.run(['curl', '-s', `${v.url}snap?${camQuery(at)}&ar=${ar}${fast ? '&fast=1' : ''}`], { timeoutMs: 5000 })
60}
61
62export const register: Register = on => {
63  on('session.start', async ($, e, next) => {
64    // a (re)load means any earlier server of ours is gone with the old module: start the pane clean
65    await update($, view, () => null)
66    await update($, offer, () => null)
67    await $.command.register({
68      name: 'splat',
69      description: 'Preview a 3D Gaussian splat file in a pane (Spark renderer)',
70    })
71    return next(e)
72  })
73
74  on('command.run', { command: 'splat' }, async ($, e) => {
75    const arg = e.args.trim()
76    if (!arg) return { text: USAGE }
77    const file = await resolve($, arg)
78    const stat = await $.fs.stat(file).catch(() => undefined)
79    if (stat?.kind !== 'file') return { text: `splat: not a file: ${file}` }
80    seen.add(file)
81    await update($, offer, () => null)
82
83    const hasImages = drawsImages((await $.env.get('TERM_PROGRAM')) ?? '', (await $.env.get('TERM')) ?? '')
84
85    await server?.return(undefined as never)
86    const shot = `${TMP}/${Date.now()}.png` // fresh name per splat: the terminal caches by path
87    const child = $.process.spawn({
88      argv: ['python3', `${$.plugin.root}/hooks/serve.py`, `${$.plugin.root}/hooks/viewer.html`, file, shot],
89    })
90    server = child
91    await update($, view, () => ({
92      file, size: stat.size, splats: 0, url: '', shot, generation: 0, az: 30, el: 20, d: 1, px: 0, py: 0, pz: 0, mode: 'orbit', hasImages, status: 'starting server',
93    }))
94
95    void (async () => {
96      try {
97        for await (const { stream, text } of child) {
98          if (stream !== 'stdout') continue
99          for (const line of text.split('\n').filter(Boolean)) {
100            if (/^\d+$/.test(line)) {
101              await patch($, () => ({ url: `http://localhost:${line}/`, status: 'loading splat' }))
102              void snap($, {})
103            } else if (line === 'SHOT') {
104              await patch($, v => ({ generation: v.generation + 1, status: 'ready' }))
105            } else if (line.startsWith('LOG N ')) {
106              await patch($, () => ({ splats: Number(line.slice(6)) }))
107            } else {
108              await patch($, () => ({ status: line.slice(4) }))
109            }
110          }
111        }
112      } catch (err) {
113        await patch($, () => ({ status: `server failed: ${String(err)}` }))
114      }
115    })()
116
117    await $.ui.open({ id: PANE, title: 'Splat', focus: true })
118    return { text: `Previewing ${file}` }
119  })
120
121  // /clear, /resume and /branch reset every $.state value but leave the server: stop it and close the pane
122  on('classic.SessionStart', { source: ['clear', 'resume', 'fork'] }, async ($, e, next) => {
123    await server?.return(undefined as never)
124    server = undefined
125    await $.ui.close({ id: PANE })
126    return next(e)
127  })
128
129  on('prompt.submit', async ($, e, next) => {
130    await detect($, e.text)
131    return next(e)
132  })
133
134  on('tool.call', async ($, e, next) => {
135    await detect($, JSON.stringify(e))
136    return next(e)
137  })
138
139  on('ui.message', { element: 'pad' }, async ($, e, next) => {
140    const m = e.data as PadMessage
141    void snap($, { az: m.az, el: m.el, px: m.px, py: m.py, pz: m.pz, mode: m.mode }, m.fast)
142    return next(e)
143  })
144
145  // the wheel over the pane zooms instead of scrolling it: down zooms out, up zooms in.
146  // ponytail: a trackpad fires many small events per gesture, so 3% a row and a clamp, not 15% a tick
147  on('ui.scroll', { requestId: PANE }, async ($, e) => {
148    const v = await read($, view)
149    // look mode walks instead: wheel down steps back, wheel up steps forward
150    if (v) void snap($, v.mode === 'look' ? walk(v, -0.02 * e.by, 0) : { d: v.d * Math.pow(1.03, e.by) })
151    return {}
152  })
153
154  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
155    const file = await read($, offer)
156    if (!file || e.props.hasSurvey) return next(e)
157    const { Box, Text, Button } = $.ui.resolve(e)
158    const dismiss = () => update($, offer, () => null)
159    const theirs = await next(e) // what the mods after this one draw in the band stays
160    return (
161      <Box flexDirection="column">
162        <Box>
163          <Text color="claude">◆ </Text>
164          <Text>Preview {file.split('/').pop()} in the splat pane? </Text>
165          <Button hotkey="1" plain onPress={() => void $.command.run({ command: 'splat', args: file })}>Yes</Button>
166          <Text> </Text>
167          <Button hotkey="2" plain onPress={() => void dismiss()}>No</Button>
168        </Box>
169        {theirs}
170      </Box>
171    )
172  })
173
174  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
175    const { Box, Text, Button, Link } = $.ui.resolve(e)
176    const v = await read($, view)
177    if (!v) return <Text dimColor>{USAGE}</Text>
178    const name = v.file.split('/').pop()
179    const columns = Math.min(e.viewport?.columns ?? 80, 160) - 2
180    const isBusy = v.status !== 'ready'
181    const isLook = v.mode === 'look'
182    // orbit: a/d circle the scene; look: they turn the head, which runs the other way round
183    const turn = isLook ? 1 : -1
184    const statusColor = v.status.includes('failed') || v.status.startsWith('ERR') ? 'error' : isBusy ? 'warning' : 'success'
185
186    // the footer is laid out here, not by the terminal, so the picture can take exactly the rows it leaves
187    // the browser opens where the pane looks, in the same mode
188    const ext = v.file.split('.').pop()?.toLowerCase() ?? ''
189    const openBrowser = () => void $.process.run(['open', `${v.url}?ext=${ext}&${camQuery(v)}`])
190    const keys: [hotkey: string, label: string, onPress: () => void][] = [
191      ['a', isLook ? '◀ turn' : '◀ orbit', () => void snap($, { az: v.az + 20 * turn })],
192      ['d', isLook ? 'turn ▶' : 'orbit ▶', () => void snap($, { az: v.az - 20 * turn })],
193      ['q', isLook ? '▲ look up' : '▲ up', () => void snap($, { el: Math.max(-85, Math.min(85, v.el - 15 * turn)) })],
194      ['e', isLook ? '▼ look down' : '▼ down', () => void snap($, { el: Math.max(-85, Math.min(85, v.el + 15 * turn)) })],
195      ['w', isLook ? 'forward' : 'zoom in', () => void snap($, isLook ? walk(v, STEP, 0) : { d: v.d * 0.7 })],
196      ['s', isLook ? 'back' : 'zoom out', () => void snap($, isLook ? walk(v, -STEP, 0) : { d: v.d / 0.7 })],
197      ['m', isLook ? 'mode: look around' : 'mode: orbit', () => void snap($, { mode: isLook ? 'orbit' : 'look', d: 1, px: 0, py: 0, pz: 0 })],
198      ['r', 'reset', () => void snap($, { az: 30, el: 20, d: 1, px: 0, py: 0, pz: 0 })],
199      ['o', 'open in browser', openBrowser],
200    ]
201    const KEY_GAP = 3
202    const lines = pack(keys, ([, label]) => label.length + 3, columns, KEY_GAP) // a plain Button draws "k: label"
203    const readout = `${isLook ? 'look around' : 'orbit'} · az ${v.az}° · el ${v.el}°${isLook ? '' : ` · dist ${v.d.toFixed(2)}`}`
204    // the longest hint that fits beside the readout, else none
205    const hint = [
206      `${isLook ? 'drag looks · right-drag slides · wheel walks' : 'drag orbits · right-drag pans · wheel zooms'} · esc to prompt`,
207      'esc to prompt',
208    ].find(h => readout.length + 2 + h.length <= columns)
209    // the pane's body less its frame, our header and status rows and the key lines
210    const rows = Math.max(8, Math.min(120, (e.viewport?.rows ?? 24) - 11 - lines.length))
211    grid = { columns, rows }
212    const header = (
213      <Box>
214        <Text color="claude" bold>◆ {name}</Text>
215        <Text dimColor>  {(v.size / 1e6).toFixed(0)} MB{v.splats ? ` · ${(v.splats / 1e6).toFixed(1)}M splats` : ''}</Text>
216        <Text>  </Text>
217        <Text color={statusColor}>● {v.status}</Text>
218      </Box>
219    )
220    const browser = <Button hotkey="o" plain onPress={openBrowser}>open in browser</Button>
221
222    // a terminal without image support gets the link, not a picture
223    if (!v.hasImages || e.surface !== 'terminal') {
224      return (
225        <Box flexDirection="column">
226          {header}
227          <Text dimColor>This terminal draws no images. Ghostty, kitty and WezTerm show the preview inline.</Text>
228          {v.url && <Link href={v.url} label="interactive viewer (orbit with the mouse)" />}
229          {v.url && <Box>{browser}</Box>}
230        </Box>
231      )
232    }
233
234    const { Image, Client } = $.ui.resolve(e)
235    const picture =
236      v.generation > 0 ? (
237        <Box width={columns} height={rows}>
238          <Image
239            key="shot"
240            source={{ file: v.shot, format: 'png', generation: v.generation }}
241            columns={columns}
242            rows={rows}
243            alt="splat preview: this terminal did not draw the image; press o for the browser"
244          />
245          <Box position="absolute" top={0} left={0}>
246            <Client key="pad" module="./pad.tsx" props={{ az: v.az, el: v.el, px: v.px, py: v.py, pz: v.pz, mode: v.mode }} width={columns} height={rows} />
247          </Box>
248        </Box>
249      ) : (
250        <Box width={columns} height={rows} borderStyle="round" borderColor="subtle" alignItems="center" justifyContent="center">
251          <Text dimColor>{v.status}…</Text>
252        </Box>
253      )
254
255    return (
256      <Box flexDirection="column">
257        {header}
258        {picture}
259        <Box width={columns} justifyContent="space-between">
260          <Text dimColor>{readout}</Text>
261          {hint && <Text dimColor>{hint}</Text>}
262        </Box>
263        {lines.map(line => (
264          <Box key={line.map(([hotkey]) => hotkey).join()} columnGap={KEY_GAP}>
265            {line.map(([hotkey, label, onPress]) => <Button key={hotkey} hotkey={hotkey} plain onPress={onPress}>{label}</Button>)}
266          </Box>
267        ))}
268      </Box>
269    )
270  })
271}
272

1import type { ClientModule } from 'claude-code'
2
3export type PadProps = { az: number; el: number; px: number; py: number; pz: number; mode: 'orbit' | 'look' }
4export type PadMessage = PadProps & { fast: boolean }
5
6const DEG_PER_COL = 1.5
7const DEG_PER_ROW = 3
8const PAN_PER_COL = 0.01 // in units of the scene radius
9const PAN_PER_ROW = 0.02
10
11const round = (n: number) => +n.toFixed(3) || 0 // no -0 in the query string
12
13/** Look mode: the eye after stepping `forward` and `right` along the ground, facing `az`, in units of the scene radius. */
14export const walk = <C extends { az: number; px: number; pz: number }>(cam: C, forward: number, right: number): C => {
15  const a = (cam.az * Math.PI) / 180 // the eye faces (-sin a, 0, -cos a); its right is (cos a, 0, -sin a)
16  return { ...cam, px: round(cam.px - forward * Math.sin(a) + right * Math.cos(a)), pz: round(cam.pz - forward * Math.cos(a) - right * Math.sin(a)) }
17}
18
19/**
20 * Camera after dragging `dx` columns and `dy` rows: the left button turns, the right one pans.
21 * Both modes drag the world with the pointer; looking out from the eye turns the opposite way to circling it.
22 */
23export const dragTo = (start: PadProps, button: 'left' | 'right', dx: number, dy: number): PadProps => {
24  const sign = start.mode === 'look' ? -1 : 1
25  if (button === 'left')
26    return { ...start, az: Math.round(start.az - sign * dx * DEG_PER_COL), el: Math.max(-85, Math.min(85, Math.round(start.el + sign * dy * DEG_PER_ROW))) }
27  // look mode pans in the world, so the spot stays put while the head turns
28  if (start.mode === 'look') return { ...walk(start, 0, -dx * PAN_PER_COL), py: round(start.py + dy * PAN_PER_ROW) }
29  return { ...start, px: round(start.px - dx * PAN_PER_COL), py: round(start.py + dy * PAN_PER_ROW) }
30}
31
32/** Greedy line breaking: `items` split into lines, `gap` cells apart, none wider than `columns` unless one item alone is. */
33export const pack = <T,>(items: T[], width: (item: T) => number, columns: number, gap: number): T[][] => {
34  const lines: T[][] = []
35  let line: T[] = []
36  let used = 0
37  for (const item of items) {
38    const w = width(item)
39    if (line.length > 0 && used + gap + w <= columns) { line.push(item); used += gap + w; continue }
40    line = [item]
41    used = w
42    lines.push(line)
43  }
44  return lines
45}
46
47// ponytail: one pad per pane, so module scope holds the latest camera the pane drew
48let latest: PadProps = { az: 0, el: 20, px: 0, py: 0, pz: 0, mode: 'orbit' }
49
50const Pad: ClientModule<PadProps, { isWired: true }> = (props, s) => {
51  latest = props
52  if (!s.state) {
53    let start: { x: number; y: number; button: 'left' | 'right'; from: PadProps } | undefined
54    let sentAt = 0
55    s.onPointer(e => {
56      const x = e.fine?.x ?? e.x
57      const y = e.fine?.y ?? e.y
58      if (e.type === 'down' && (e.button === 'left' || e.button === 'right')) start = { x, y, button: e.button, from: latest }
59      if (!start) return
60      const at = dragTo(start.from, start.button, x - start.x, y - start.y)
61      if (e.type === 'move' && Date.now() - sentAt > 120) {
62        sentAt = Date.now()
63        s.post({ ...at, fast: true } satisfies PadMessage)
64      }
65      if (e.type === 'up') {
66        start = undefined
67        s.post({ ...at, fast: false } satisfies PadMessage)
68      }
69    })
70    s.setState({ isWired: true })
71  }
72  return s.elements.Box({ width: s.columns, height: s.rows })
73}
74
75export default Pad
76

1export type View = {
2  file: string
3  size: number
4  splats: number
5  url: string
6  shot: string
7  generation: number
8  az: number
9  el: number
10  d: number
11  px: number
12  py: number
13  /** look mode only: px, py, pz are the eye's world offset from the scene center */
14  pz: number
15  /** orbit circles the scene; look stands at a spot, turns in place and walks */
16  mode: 'orbit' | 'look'
17  status: string
18  hasImages: boolean
19}
20
21declare module 'claude-code' {
22  interface PluginState {
23    '3dgs-view': { view: View | null; offer: string | null }
24  }
25}
26