meshy-viewer

1import { atom, read, update } from 'claude-code'
2import type { HookStream, McpToolResult, ProcessSpawnChunk, ProcessSpawnResult, Register, Timer } from 'claude-code'
3
4import type { ViewerMode, ViewerView } from '../types'
5import { parseBmp, parseGlb } from './glb'
6import type { ByteReader, ImageDecoder } from './glb'
7import { cellFrame, infoLine, isTextured, makeModel, svgFrame, viewerPage } from './model'
8import type { Model } from './model'
9import { fromBase64 } from './render'
10
11const PANE = 'meshy-viewer'
12/** Degrees per rotate press, per dragged column and per auto-rotate tick (vector): 36 frames a turn. */
13const STEP = 10
14const SPIN_MS = 300
15/** Live mode turns in finer steps, since every frame is a real render. */
16const LIVE_STEP = 4
17const LIVE_SPIN_MS = 120
18const DRAG_DEGREES_PER_COLUMN = 3
19const DRAG_DEGREES_PER_ROW = 6
20const ZOOM_FACTOR = 0.85
21/** Per wheel tick over the picture; a touchpad sends many. */
22const WHEEL_ZOOM = 0.92
23/** `$.fs.read` takes files up to 4 MiB; past that, ranges come through the shell. */
24const SMALL_FILE = 4 * 1024 * 1024
25const CHUNK = 3 * 1024 * 1024
26const TOAST_MS = 6000
27/** Base-color textures are sampled from a copy this many pixels square. */
28const TEXTURE_SIZE = 256
29// Cuts the image out of the GLB and converts it to BMP: sips on macOS, else ImageMagick.
30const DECODE_SCRIPT =
31  'd=$(mktemp -d) || exit 1; trap \'rm -rf "${d:?}"\' EXIT; ' +
32  'tail -c +"$2" "$1" | head -c "$3" > "$d/in" && ' +
33  'if command -v sips >/dev/null 2>&1; then sips -s format bmp -z "$4" "$4" "$d/in" --out "$d/out.bmp" >/dev/null 2>&1; ' +
34  'elif command -v magick >/dev/null 2>&1; then magick "$d/in" -resize "$4x$4!" "BMP3:$d/out.bmp"; else exit 2; fi && ' +
35  'base64 < "$d/out.bmp" | tr -d "\\n"'
36/** Where the live renderer's Chrome may be, unless MESHY_VIEWER_CHROME names one. */
37const CHROME_CANDIDATES = [
38  '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
39  '/Applications/Chromium.app/Contents/MacOS/Chromium',
40  '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
41  '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser',
42  '/usr/bin/google-chrome',
43  '/usr/bin/chromium',
44  '/usr/bin/chromium-browser',
45  '/snap/bin/chromium',
46]
47const NODE_CANDIDATES = ['/opt/homebrew/bin/node', '/usr/local/bin/node', '/usr/bin/node']
48/** Pixels rendered per terminal cell in live mode, and the frame's cap. */
49const CELL_PIXELS = { x: 10, y: 20 }
50const MAX_FRAME = { width: 1280, height: 960 }
51/**
52 * The session's browser pane, where the desktop shows the real viewer: the tool as the model names it,
53 * and the MCP server and tool behind it, which the mod calls itself.
54 */
55const BROWSER_PANES: readonly BrowserPane[] = [{ name: 'mcp__Claude_Browser__navigate', server: 'Claude_Browser', tool: 'navigate' }]
56const SERVER_START_MS = 20_000
57
58const CLOSED: ViewerView = { path: null, angle: 0, phi: 75, zoom: 105, isAutoRotating: false, isOpen: false, mode: 'vector' }
59const view = atom({ plugin: 'meshy-viewer', key: 'view' } as const, CLOSED)
60const hint = atom({ plugin: 'meshy-viewer', key: 'hint' } as const, null)
61const dismissed = atom({ plugin: 'meshy-viewer', key: 'dismissed' } as const, [])
62// meshy-tray's tasks, when that mod is loaded: a finished one earns a "View" offer.
63const trayTasks = { plugin: 'meshy-tray', key: 'tasks' } as const
64
65const snap = (angle: number) => (((Math.round(angle / STEP) * STEP) % 360) + 360) % 360
66const wrap = (angle: number) => ((Math.round(angle) % 360) + 360) % 360
67/**
68 * "Rotate right" (direction 1) moves the model's front to the right. The
69 * vector renderer's angle grows that way; model-viewer's theta grows the
70 * other way, so each mode turns its own way.
71 */
72const turnBy = (now: ViewerView, direction: number) =>
73  now.mode === 'live' ? wrap(now.angle - direction * LIVE_STEP) : snap(now.angle + direction * STEP)
74const tiltBy = (now: ViewerView, direction: number) => clampPhi(now.phi + direction * DRAG_DEGREES_PER_ROW)
75const clampPhi = (phi: number) => Math.max(5, Math.min(175, Math.round(phi)))
76const clampZoom = (zoom: number) => Math.max(25, Math.min(400, Math.round(zoom)))
77const errorText = (error: unknown) => (error instanceof Error ? error.message : String(error))
78/** An MCP result's text blocks as one line. */
79const mcpText = (result: McpToolResult) => result.content.map(block => block.text ?? '').join(' ').trim()
80const baseName = (path: string) => path.split('/').pop() ?? path
81/** A view as stored by any version of this mod, every field filled in. */
82const whole = (stored: Partial<ViewerView> | undefined): ViewerView => ({ ...CLOSED, ...stored })
83
84/** The renderer process, while live mode runs. */
85type Live = {
86  socket: string
87  outDir: string
88  stream: HookStream<ProcessSpawnChunk, ProcessSpawnResult>
89  ready: Promise<void>
90  isGone: boolean
91  /** The model the page shows, once /load answered. */
92  loadedPath?: string
93  /** The camera of the last frame asked for. */
94  camera?: Camera
95  generation: number
96  width: number
97  height: number
98}
99
100/** The local page server, while the desktop's browser pane shows a model. */
101type Served = {
102  path: string
103  url: string
104  stream: HookStream<ProcessSpawnChunk, ProcessSpawnResult>
105}
106
107type Camera = { theta: number; phi: number; radius: number }
108type BrowserPane = { name: `mcp__${string}__${string}`; server: string; tool: string }
109
110export const register: Register = on => {
111  // Parsed models outlive a redraw but not a reload; $.state keeps which one is shown.
112  const models = new Map<string, Model>()
113  const loading = new Map<string, Promise<Model>>()
114  let loadModel: ((path: string) => Promise<Model>) | undefined
115  let openInBrowser: ((path: string) => Promise<string>) | undefined
116  let startSpin: (() => void) | undefined
117  let spin: Timer | undefined
118  let live: Live | undefined
119  let isGraphicsTerminal = false
120  /** The rows the picture took at the last terminal draw: the wheel zooms over them. */
121  let pictureRows = 0
122  /** Shows the view's camera in live mode; resolves true once a new frame for it is on disk. */
123  let showLive: ((path: string, size: { width: number; height: number }) => Promise<boolean>) | undefined
124  let stopLive: (() => Promise<void>) | undefined
125  /** Opens the model in the session's browser pane (desktop) through the pane's own MCP server; resolves the page's URL. */
126  let openInPane: ((path: string) => Promise<string>) | undefined
127  let served: Served | undefined
128  let browserPane: BrowserPane | undefined
129
130  on('session.start', async ($, e, next) => {
131    const readerFor = async (path: string): Promise<{ size: number; read: ByteReader }> => {
132      const { size } = await $.fs.stat(path)
133      if (size <= SMALL_FILE) {
134        const { base64 } = await $.fs.read(path, { as: 'bytes' })
135        const bytes = fromBase64(base64)
136
137        return { size, read: async (offset, length) => bytes.subarray(offset, offset + length) }
138      }
139
140      return {
141        size,
142        read: async (offset, length) => {
143          const out = new Uint8Array(length)
144          for (let at = 0; at < length; at += CHUNK) {
145            const count = Math.min(CHUNK, length - at)
146            // Positional arguments, so the path is never parsed by the shell.
147            const ran = await $.process.run(
148              ['sh', '-c', 'tail -c +"$2" "$1" | head -c "$3" | base64 | tr -d "\\n"', 'sh', path, String(offset + at + 1), String(count)],
149              { timeoutMs: 120_000 },
150            )
151            if (ran.exitCode !== 0) throw new Error(`could not read ${baseName(path)}: ${ran.stderr.trim()}`)
152            out.set(fromBase64(ran.stdout).subarray(0, count), at)
153          }
154
155          return out
156        },
157      }
158    }
159
160    const decoderFor =
161      (path: string): ImageDecoder =>
162      async ({ offset, length, mimeType }) => {
163        if (!/^image\/(jpeg|png)$/.test(mimeType)) return undefined
164        const ran = await $.process
165          .run(['sh', '-c', DECODE_SCRIPT, 'sh', path, String(offset + 1), String(length), String(TEXTURE_SIZE)], {
166            timeoutMs: 60_000,
167          })
168          .catch(() => undefined)
169
170        // No converter, or one that failed: the model keeps its clay color.
171        return ran?.exitCode === 0 && ran.stdout ? parseBmp(fromBase64(ran.stdout)) : undefined
172      }
173
174    loadModel = path => {
175      const cached = models.get(path)
176      if (cached) return Promise.resolve(cached)
177      let pending = loading.get(path)
178      if (!pending) {
179        pending = (async () => {
180          const { size, read: readBytes } = await readerFor(path)
181          const model = makeModel(path, size, await parseGlb(readBytes, size, decoderFor(path)))
182          // Decimate now, so drawing a frame is only the projection.
183          svgFrame(model, 0)
184          models.set(path, model)
185
186          return model
187        })().finally(() => loading.delete(path))
188        loading.set(path, pending)
189      }
190
191      return pending
192    }
193
194    openInBrowser = async path => {
195      const name = baseName(path)
196      const folder = `${path.slice(0, path.length - name.length)}.meshy-viewer`
197      const page = `${folder}/${name.replace(/\.glb$/i, '')}.html`
198      const { head, tail } = viewerPage(name)
199      await $.process.run(['mkdir', '-p', folder])
200      // The GLB streams through base64 between the page's two halves: no size limit here.
201      const wrote = await $.process.run(
202        ['sh', '-c', '{ printf "%s" "$2"; base64 < "$1" | tr -d "\\n"; printf "%s" "$3"; } > "$4"', 'sh', path, head, tail, page],
203        { timeoutMs: 120_000 },
204      )
205      if (wrote.exitCode !== 0) throw new Error(`could not write ${page}: ${wrote.stderr.trim()}`)
206      const opened = await $.process.run(['open', page]).catch(() => undefined)
207      if (opened?.exitCode !== 0) {
208        const fallback = await $.process.run(['xdg-open', page]).catch(() => undefined)
209        if (fallback?.exitCode !== 0) throw new Error(`wrote ${page} but could not open a browser`)
210      }
211
212      return page
213    }
214
215    // --- live mode: a headless Chrome draws the frames ----------------------
216
217    const termProgram = (await $.env.get('TERM_PROGRAM')) ?? ''
218    const term = (await $.env.get('TERM')) ?? ''
219    isGraphicsTerminal =
220      termProgram === 'ghostty' || /ghostty|kitty/i.test(term) || (await $.env.get('KITTY_WINDOW_ID')) !== undefined
221
222    const findChrome = async () => {
223      const named = await $.env.get('MESHY_VIEWER_CHROME')
224      for (const candidate of named ? [named, ...CHROME_CANDIDATES] : CHROME_CANDIDATES) {
225        if (await $.fs.exists(candidate)) return candidate
226      }
227      throw new Error('no Chrome found; install Google Chrome or set MESHY_VIEWER_CHROME to a Chromium binary')
228    }
229    const findNode = async () => {
230      for (const candidate of NODE_CANDIDATES) if (await $.fs.exists(candidate)) return candidate
231
232      return 'node'
233    }
234
235    const ask = async (renderer: Live, route: string, body: unknown): Promise<Record<string, unknown>> => {
236      const response = await $.http.fetch(`http://meshy-viewer${route}`, {
237        method: 'POST',
238        headers: { 'content-type': 'application/json' },
239        body: JSON.stringify(body),
240        socketPath: renderer.socket,
241      })
242      let json: Record<string, unknown> = {}
243      try {
244        json = JSON.parse(response.text)
245      } catch {
246        // An empty or broken answer reads as a failure below.
247      }
248      if (!response.ok) throw new Error(typeof json.error === 'string' ? json.error : `the renderer answered HTTP ${response.status}`)
249
250      return json
251    }
252
253    const startRenderer = async (width: number, height: number): Promise<Live> => {
254      const chrome = await findChrome()
255      const node = await findNode()
256      const tag = `${Date.now().toString(36)}${Math.floor(Math.random() * 1e6).toString(36)}`
257      const outDir = `/tmp/meshy-viewer-${tag}`
258      const socket = `/tmp/meshy-viewer-${tag}.sock`
259      const stream = $.process.spawn({
260        argv: [
261          node,
262          `${$.plugin.root}/bin/render.mjs`,
263          '--chrome', chrome,
264          '--socket', socket,
265          '--out', outDir,
266          '--width', String(width),
267          '--height', String(height),
268        ],
269      })
270      let markReady!: () => void
271      let markFailed!: (error: Error) => void
272      const ready = new Promise<void>((resolve, reject) => {
273        markReady = resolve
274        markFailed = reject
275      })
276      ready.catch(() => undefined)
277      const renderer: Live = { socket, outDir, stream, ready, isGone: false, generation: 0, width, height }
278      // The loop is the child's life: it runs on in the background until the renderer exits.
279      void (async () => {
280        let stderr = ''
281        try {
282          for await (const chunk of stream) {
283            if (chunk.stream === 'stdout' && chunk.text.includes('ready')) markReady()
284            else if (chunk.stream === 'stderr') stderr = (stderr + chunk.text).slice(-2000)
285          }
286        } catch (error) {
287          stderr = `${stderr}\n${errorText(error)}`
288        } finally {
289          renderer.isGone = true
290          const reason = stderr.trim().split('\n').pop() ?? ''
291          markFailed(new Error(`the live renderer exited${reason ? `: ${reason}` : ''}`))
292          if (live === renderer) {
293            live = undefined
294            $.ui.invalidate('ui.render')
295          }
296        }
297      })()
298      // Never let a renderer that fails to start hang the viewer.
299      $.clock.after(30_000, () => markFailed(new Error('the live renderer did not start within 30s')))
300
301      return renderer
302    }
303
304    let inFlight: Promise<void> | undefined
305    let wanted: Camera | undefined
306
307    const take = (renderer: Live, answer: Record<string, unknown>) => {
308      if (typeof answer.generation === 'number') renderer.generation = answer.generation
309    }
310    // Two draws before the first renderer is up must not start two Chromes.
311    let starting: Promise<Live> | undefined
312
313    showLive = async (path, { width, height }) => {
314      if (!live || live.isGone) {
315        starting ??= startRenderer(width, height).finally(() => {
316          starting = undefined
317        })
318        live = await starting
319      }
320      const renderer = live
321      await renderer.ready
322      if (renderer.loadedPath !== path || renderer.width !== width || renderer.height !== height) {
323        const answer = await ask(renderer, '/load', { path, width, height })
324        renderer.loadedPath = path
325        renderer.width = width
326        renderer.height = height
327        renderer.camera = undefined
328        take(renderer, answer)
329      }
330      const shown = whole(await read($, view))
331      // Closed, or replaced, while the load ran: nothing more to ask of it.
332      if (live !== renderer || renderer.isGone) return false
333      const camera: Camera = { theta: shown.angle, phi: shown.phi, radius: shown.zoom }
334      const last = renderer.camera
335      // The frame on disk already shows this camera: a redraw asks for nothing.
336      if (last && last.theta === camera.theta && last.phi === camera.phi && last.radius === camera.radius) return false
337      wanted = camera
338      // One request at a time; the camera the latest change asked for wins.
339      if (!inFlight) {
340        inFlight = (async () => {
341          try {
342            while (wanted && live === renderer && !renderer.isGone) {
343              const asked = wanted
344              wanted = undefined
345              renderer.camera = asked
346              take(renderer, await ask(renderer, '/frame', asked))
347            }
348          } finally {
349            inFlight = undefined
350          }
351        })()
352      }
353      await inFlight
354
355      return true
356    }
357
358    stopLive = async () => {
359      const renderer = live
360      live = undefined
361      if (!renderer || renderer.isGone) return
362      await ask(renderer, '/quit', {}).catch(() => undefined)
363      await renderer.stream.return({ code: 0, signal: null }).catch(() => undefined)
364    }
365
366    const stopServer = async () => {
367      const running = served
368      served = undefined
369      await running?.stream.return({ code: 0, signal: null }).catch(() => undefined)
370    }
371
372    openInPane = async path => {
373      if (!browserPane) throw new Error('this session has no browser pane')
374      if (served?.path !== path) {
375        await stopServer()
376        const node = await findNode()
377        const stream = $.process.spawn({
378          argv: [node, `${$.plugin.root}/bin/serve.mjs`, '--model', path, '--cache', '/tmp/meshy-viewer-cache'],
379        })
380        let settle!: (url: string) => void
381        let fail!: (error: Error) => void
382        const url = new Promise<string>((resolve, reject) => {
383          settle = resolve
384          fail = reject
385        })
386        url.catch(() => undefined)
387        void (async () => {
388          let stderr = ''
389          try {
390            for await (const chunk of stream) {
391              const match = chunk.stream === 'stdout' ? /http:\/\/127\.0\.0\.1:\d+\//.exec(chunk.text) : null
392              if (match) settle(match[0])
393              else if (chunk.stream === 'stderr') stderr = (stderr + chunk.text).slice(-2000)
394            }
395          } catch (error) {
396            stderr = `${stderr}\n${errorText(error)}`
397          } finally {
398            const reason = stderr.trim().split('\n').pop() ?? ''
399            fail(new Error(`the model server exited${reason ? `: ${reason}` : ''}`))
400            if (served?.stream === stream) served = undefined
401          }
402        })()
403        $.clock.after(SERVER_START_MS, () => fail(new Error('the model server did not start within 20s')))
404        served = { path, url: await url, stream }
405      }
406      // The mod calls the pane's MCP server itself: a plugin's own call is its grant, so the pane
407      // opens at once. (A $.tool.call goes through the model-tool permission path, whose auto-mode
408      // classifier holds no verdict for a call no model turn produced.)
409      let opened: McpToolResult
410      try {
411        opened = await $.mcp.call(browserPane.server, browserPane.tool, { url: served.url })
412      } catch (error) {
413        // The server could not be called by name: let a model turn open the page with the tool,
414        // submitted once this hook has answered (a hook holding the turn may not submit).
415        const text = `Open ${served.url} in the built-in browser pane with the ${browserPane.name} tool. It is meshy-viewer's page for ${baseName(path)}; reply with one short line.`
416        $.clock.after(50, () => void $.prompt.submit({ text }))
417
418        return served.url
419      }
420      if (opened.isError) throw new Error(mcpText(opened) || 'the browser pane refused the page')
421
422      return served.url
423    }
424
425    const tick = async () => {
426      const shown = whole(await read($, view))
427      if (!shown.isOpen || !shown.isAutoRotating) {
428        spin?.cancel()
429        spin = undefined
430        return
431      }
432      const isLive = shown.mode === 'live'
433      await update($, view, current => ({
434        ...whole(current),
435        angle: isLive ? wrap(whole(current).angle + LIVE_STEP) : snap(whole(current).angle + STEP),
436      }))
437    }
438    startSpin = () => {
439      if (spin) return
440      void (async () => {
441        const shown = whole(await read($, view))
442        if (spin) return
443        spin = $.clock.every(shown.mode === 'live' ? LIVE_SPIN_MS : SPIN_MS, () => void tick())
444      })()
445    }
446
447    await $.command.register({
448      name: 'meshy-view',
449      description: 'View and spin a GLB model in a pane (default: the newest in ./meshy-models; --live or --vector picks the renderer)',
450      argumentHint: '[path] [--live|--vector]',
451    })
452
453    const tools = new Set((await $.tool.list()).map(tool => tool.name))
454    browserPane = BROWSER_PANES.find(pane => tools.has(pane.name))
455
456    // After a reload: parse the model on screen again and keep it turning.
457    const shown = whole(await read($, view))
458    if (shown.path) {
459      void loadModel(shown.path)
460        .then(() => $.ui.invalidate('ui.render'))
461        .catch(() => undefined)
462    }
463    if (shown.isOpen && shown.isAutoRotating) startSpin()
464
465    return next(e)
466  })
467
468  on('command.run', { command: 'meshy-view' }, async ($, e) => {
469    if (!loadModel) return { text: 'meshy-viewer is still starting; try again in a moment.' }
470    const cwd = await $.session.cwd()
471    let mode: ViewerMode | undefined
472    const rest: string[] = []
473    for (const word of e.args.trim().split(/\s+/).filter(Boolean)) {
474      if (word === '--live') mode = 'live'
475      else if (word === '--vector') mode = 'vector'
476      else rest.push(word)
477    }
478    let path = rest.join(' ').replace(/^(["'])(.*)\1$/, '$2')
479    if (!path) {
480      const folder = `${cwd}/meshy-models`
481      const glbs = (await $.fs.exists(folder))
482        ? (await $.fs.list(folder)).filter(entry => entry.kind === 'file' && /\.glb$/i.test(entry.name))
483        : []
484      const newest = glbs.sort((a, b) => b.mtimeMs - a.mtimeMs)[0]
485      if (!newest) return { text: 'No .glb files in ./meshy-models. Usage: /meshy-view [path to a .glb] [--live|--vector]' }
486      path = `${folder}/${newest.name}`
487    } else if (path.startsWith('~/')) {
488      path = `${await $.env.get('HOME')}${path.slice(1)}`
489    } else if (!path.startsWith('/')) {
490      path = `${cwd}/${path}`
491    }
492    if (!(await $.fs.exists(path))) return { text: `No such file: ${path}` }
493
494    // The desktop pane cannot draw pixels, but the session's browser pane can run the real viewer.
495    const surfaces = await $.session.surfaces()
496    if (surfaces.includes('desktop') && openInPane && mode === undefined) {
497      try {
498        const url = await openInPane(path)
499
500        return { text: `Viewing ${baseName(path)} in the browser pane (${url}). Drag to orbit, scroll to zoom.` }
501      } catch (error) {
502        $.ui.toast(`Browser pane failed: ${errorText(error)}; opening the pane view`, { timeoutMs: TOAST_MS })
503      }
504    }
505
506    let model: Model
507    try {
508      model = await loadModel(path)
509    } catch (error) {
510      return { text: `Could not read ${baseName(path)}: ${errorText(error)}` }
511    }
512    const chosen = path
513    const chosenMode: ViewerMode = mode ?? (isGraphicsTerminal ? 'live' : 'vector')
514    await update($, view, () => ({ ...CLOSED, path: chosen, isOpen: true, mode: chosenMode }))
515    const placed = await $.ui.open({ id: PANE, title: model.name, focus: true, closeOnEscape: true })
516    await update($, hint, () => (placed.isPlaced ? null : { path: chosen, reason: placed.reason }))
517
518    return {
519      text: placed.isPlaced
520        ? `Viewing ${model.name}${chosenMode === 'live' ? ' (live)' : ''}.`
521        : `The viewer pane is waiting (${placed.reason}); the line above the prompt can open ${model.name} in your browser instead.`,
522    }
523  })
524
525  on('ui.close', { id: PANE }, async ($, e, next) => {
526    await stopLive?.()
527    await update($, view, current => ({ ...whole(current), isOpen: false, isAutoRotating: false }))
528
529    return next(e)
530  })
531
532  // The drag layer posts how far the pointer moved, a key's turn or tilt, or a zoom step.
533  on('ui.message', { component: 'Pane', requestId: PANE }, async ($, e) => {
534    const data = (e.data ?? {}) as { dx?: unknown; dy?: unknown; turn?: unknown; tilt?: unknown; zoom?: unknown }
535    const dx = Number(data.dx) || 0
536    const dy = Number(data.dy) || 0
537    const turn = Number(data.turn) || 0
538    const tilt = Number(data.tilt) || 0
539    const zoom = Number(data.zoom) || 0
540    if (dx === 0 && dy === 0 && turn === 0 && tilt === 0 && zoom === 0) return {}
541    await update($, view, current => {
542      const now = { ...whole(current), isAutoRotating: false }
543      if (turn !== 0) now.angle = turnBy(now, turn)
544      if (now.mode !== 'live') {
545        // The model follows the pointer: a drag left turns it left.
546        return { ...now, angle: dx === 0 ? now.angle : snap(now.angle + dx * STEP) }
547      }
548      if (tilt !== 0) now.phi = tiltBy(now, tilt)
549
550      return {
551        ...now,
552        // The model follows the pointer sideways; dragging down brings its top into view.
553        angle: wrap(now.angle - dx * DRAG_DEGREES_PER_COLUMN),
554        phi: clampPhi(now.phi - dy * DRAG_DEGREES_PER_ROW),
555        zoom: zoom === 0 ? now.zoom : clampZoom(now.zoom * (zoom > 0 ? ZOOM_FACTOR : 1 / ZOOM_FACTOR)),
556      }
557    })
558
559    return {}
560  })
561
562  // The wheel over the picture zooms the live view; elsewhere it scrolls the pane.
563  on('ui.scroll', { component: 'Pane', requestId: PANE }, async ($, e, next) => {
564    const shown = whole(await read($, view))
565    if (e.origin.kind !== 'person' || !e.pointer || shown.mode !== 'live' || !shown.isOpen) return next(e)
566    // The tree's row under the pointer: the window before this move, plus the cell.
567    const row = e.offset - e.by + e.pointer.row
568    if (row < 0 || row >= pictureRows || e.pointer.column < 0) return next(e)
569    const factor = (e.by < 0 ? WHEEL_ZOOM : 1 / WHEEL_ZOOM) ** Math.min(4, Math.abs(e.by))
570    await update($, view, current => ({ ...whole(current), zoom: clampZoom(whole(current).zoom * factor) }))
571
572    return {}
573  })
574
575  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
576    const { Box, Text, Button } = $.ui.resolve(e)
577    const shown = whole(await read($, view))
578    if (!shown.path) {
579      return <Text dimColor>Run /meshy-view [path] to open a model.</Text>
580    }
581    const path = shown.path
582    const model = models.get(path)
583    if (!model) {
584      return <Text dimColor>{`Loading ${baseName(path)}...`}</Text>
585    }
586    const isLive = shown.mode === 'live' && e.surface === 'terminal'
587    const move = (change: Partial<ViewerView>) =>
588      update($, view, current => ({ ...whole(current), isAutoRotating: false, ...change }))
589    const turn = (direction: number) =>
590      update($, view, current => {
591        const now = whole(current)
592
593        return { ...now, isAutoRotating: false, angle: turnBy(now, direction) }
594      })
595    const toggleSpin = async () => {
596      const now = await update($, view, current => ({ ...whole(current), isAutoRotating: !whole(current).isAutoRotating }))
597      if (now.isAutoRotating) startSpin?.()
598    }
599    const setMode = async (mode: ViewerMode) => {
600      if (mode === 'vector') await stopLive?.()
601      await update($, view, current => ({ ...whole(current), mode, isAutoRotating: false }))
602    }
603    // A close from this plugin's own button skips its ui.close hook, so it tidies up here.
604    const close = async () => {
605      await stopLive?.()
606      await update($, view, current => ({ ...whole(current), isOpen: false, isAutoRotating: false }))
607      await $.ui.close({ id: PANE })
608    }
609    const browse = async () => {
610      try {
611        const page = await openInBrowser?.(path)
612        $.ui.toast(`Opened in your browser: ${page}`, { timeoutMs: TOAST_MS })
613      } catch (error) {
614        $.ui.toast(`Open in browser failed: ${errorText(error)}`, { timeoutMs: TOAST_MS })
615      }
616    }
617
618    const controls = (
619      <Box flexDirection="row" gap={1}>
620        <Button key="left" label="Rotate left" hotkey="h" onPress={() => turn(-1)} />
621        <Button key="right" label="Rotate right" hotkey="l" onPress={() => turn(1)} />
622        {isLive ? <Button key="up" label="Tilt up" hotkey="k" onPress={() => move({ phi: tiltBy(shown, -1) })} /> : null}
623        {isLive ? <Button key="down" label="Tilt down" hotkey="j" onPress={() => move({ phi: tiltBy(shown, 1) })} /> : null}
624        {isLive ? <Button key="in" label="Zoom in" hotkey="i" onPress={() => move({ zoom: clampZoom(shown.zoom * ZOOM_FACTOR) })} /> : null}
625        {isLive ? <Button key="out" label="Zoom out" hotkey="u" onPress={() => move({ zoom: clampZoom(shown.zoom / ZOOM_FACTOR) })} /> : null}
626        <Button key="auto" label={shown.isAutoRotating ? 'Stop' : 'Auto-rotate'} hotkey="a" onPress={toggleSpin} />
627        <Button key="reset" label="Reset" hotkey="r" onPress={() => move({ angle: 0, phi: CLOSED.phi, zoom: CLOSED.zoom })} />
628        {e.surface === 'terminal' ? (
629          <Button key="mode" label={isLive ? 'Vector' : 'Live'} hotkey="v" onPress={() => setMode(isLive ? 'vector' : 'live')} />
630        ) : null}
631        {e.surface === 'desktop' && browserPane ? (
632          <Button key="pane" label="Open in browser pane" hotkey="p" variant="primary" onPress={() => openInPane?.(path).catch(error => $.ui.toast(`Browser pane failed: ${errorText(error)}`, { timeoutMs: TOAST_MS }))} />
633        ) : null}
634        <Button key="browser" label="Open in browser" hotkey="o" onPress={browse} />
635        <Button key="close" label="Close" hotkey="q" role="dismiss" onPress={close} />
636      </Box>
637    )
638    const status = isLive
639      ? `${shown.angle} deg, tilt ${shown.phi}, zoom ${shown.zoom}%${shown.isAutoRotating ? ', auto-rotating' : ''}`
640      : `${shown.angle} deg${shown.isAutoRotating ? ', auto-rotating' : ''}`
641    const footer = (shownFaces: number, note?: string) => (
642      <Box flexDirection="column">
643        <Text dimColor wrap="truncate-end">{infoLine(model, shownFaces)}</Text>
644        {note ? <Text dimColor wrap="truncate-end">{note}</Text> : null}
645      </Box>
646    )
647    const textureNote = isTextured(model)
648      ? model.mesh.colors
649        ? 'Texture colors are sampled once per face; Live or Open in browser shows the full texture.'
650        : 'Textures are not drawn here; Live or Open in browser shows them.'
651      : undefined
652
653    if (e.surface === 'terminal') {
654      const { Raster, Client, Image } = $.ui.resolve(e)
655      const columns = Math.max(16, Math.min(255, e.props.bodyColumns))
656      const room = Math.max(6, (e.viewport?.rows ?? 30) - 8)
657      const rows = Math.max(6, Math.min(255, room, Math.round(columns * 0.375)))
658      pictureRows = rows
659      // A transparent Client the size of the picture, laid over it: drag the model itself.
660      const dragLayer = (
661        <Box position="absolute" top={0} left={0}>
662          <Client key="drag" module="./drag.tsx" width={columns} height={rows} props={{}} />
663        </Box>
664      )
665      const strip = (
666        <Text dimColor>{`${isLive ? 'drag the model to spin, scroll or +/- to zoom' : 'drag the model to spin'}  ${status}`}</Text>
667      )
668
669      if (isLive) {
670        const renderer = live
671        const isCurrent = renderer !== undefined && !renderer.isGone && renderer.loadedPath === path
672        // Ask for this camera's frame; the redraw that follows draws it. A closing pane asks for nothing.
673        if (shown.isOpen) {
674          void showLive?.(path, {
675            width: Math.min(MAX_FRAME.width, columns * CELL_PIXELS.x),
676            height: Math.min(MAX_FRAME.height, rows * CELL_PIXELS.y),
677          })
678            .then(isNew => isNew && $.ui.invalidate('ui.render'))
679            .catch(error => {
680              $.ui.toast(`Live view failed: ${errorText(error)}; showing the vector view`, { timeoutMs: TOAST_MS })
681              void setMode('vector')
682            })
683        }
684
685        return (
686          <Box flexDirection="column">
687            <Box flexDirection="column" height={rows}>
688              {isCurrent && renderer.generation > 0 ? (
689                <Image
690                  key="live"
691                  source={{ file: `${renderer.outDir}/frame.png`, format: 'png', generation: renderer.generation }}
692                  columns={columns}
693                  rows={rows}
694                  alt={`${model.name}: the live view needs a kitty-graphics terminal such as Ghostty; press v for the vector view`}
695                />
696              ) : (
697                <Text dimColor>{`Starting the live renderer for ${model.name}...`}</Text>
698              )}
699              {dragLayer}
700            </Box>
701            {strip}
702            {controls}
703            {footer(model.mesh.triangleCount, 'Live: rendered by headless Chrome with model-viewer.')}
704          </Box>
705        )
706      }
707
708      const frame = cellFrame(model, shown.angle, columns, rows)
709
710      return (
711        <Box flexDirection="column">
712          <Box flexDirection="column" height={rows}>
713            <Raster key="model" columns={columns} rows={rows} cells={frame.cells} />
714            {dragLayer}
715          </Box>
716          {strip}
717          {controls}
718          {footer(frame.shown, textureNote)}
719        </Box>
720      )
721    }
722
723    const frame = svgFrame(model, shown.angle)
724    const alt = `${model.name}, turned ${shown.angle} degrees`
725    if (e.surface === 'desktop') {
726      const { Svg, Client } = $.ui.resolve(e)
727
728      return (
729        <Box flexDirection="column">
730          <Svg source={frame.svg} alt={alt} />
731          <Client key="drag" module="./drag.tsx" props={{ label: `<< drag here to spin >>  ${status}` }} />
732          {controls}
733          {footer(frame.shown, textureNote)}
734        </Box>
735      )
736    }
737    const { Svg } = $.ui.resolve(e)
738
739    return (
740      <Box flexDirection="column">
741        <Svg source={frame.svg} alt={alt} />
742        <Text dimColor>{status}</Text>
743        {controls}
744        {footer(frame.shown, textureNote)}
745      </Box>
746    )
747  })
748
749  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
750    if (e.props.hasSurvey) return next(e)
751    const waiting = await read($, hint)
752    const taken = await read($, dismissed)
753    const finished = ((await read($, trayTasks)) ?? []).filter(
754      task => task.status === 'SUCCEEDED' && task.file && !taken.includes(task.id),
755    )
756    const offer = finished[finished.length - 1]
757    if (!waiting && !offer) return next(e)
758
759    const { Box, Text, Button } = $.ui.resolve(e)
760    const below = await next(e)
761
762    const viewOffer = async () => {
763      if (!offer?.file || !loadModel) return
764      const path = `${offer.cwd}/${offer.file.replace(/^\.\//, '')}`
765      await update($, dismissed, list => [...list, offer.id])
766      if (e.surface === 'desktop' && openInPane) {
767        try {
768          await openInPane(path)
769          return
770        } catch (error) {
771          $.ui.toast(`Browser pane failed: ${errorText(error)}; opening the pane view`, { timeoutMs: TOAST_MS })
772        }
773      }
774      try {
775        await loadModel(path)
776      } catch (error) {
777        $.ui.toast(`Could not read ${baseName(path)}: ${errorText(error)}`, { timeoutMs: TOAST_MS })
778        return
779      }
780      const mode: ViewerMode = isGraphicsTerminal ? 'live' : 'vector'
781      await update($, view, () => ({ ...CLOSED, path, isOpen: true, mode }))
782      const placed = await $.ui.open({ id: PANE, title: baseName(path), focus: true, closeOnEscape: true })
783      await update($, hint, () => (placed.isPlaced ? null : { path, reason: placed.reason }))
784    }
785    const browseWaiting = async () => {
786      if (!waiting) return
787      try {
788        const page = await openInBrowser?.(waiting.path)
789        $.ui.toast(`Opened in your browser: ${page}`, { timeoutMs: TOAST_MS })
790        await update($, hint, () => null)
791      } catch (error) {
792        $.ui.toast(`Open in browser failed: ${errorText(error)}`, { timeoutMs: TOAST_MS })
793      }
794    }
795
796    return (
797      <Box flexDirection="column">
798        {offer ? (
799          <Box key="offer" flexDirection="row" gap={1}>
800            <Text wrap="truncate-end">{`Meshy model ready: ${offer.file}`}</Text>
801            <Button key="view" label="View" hotkey="1" variant="primary" onPress={viewOffer} />
802            <Button
803              key="skip"
804              label="Dismiss"
805              hotkey="2"
806              onPress={() => update($, dismissed, list => [...list, offer.id])}
807            />
808          </Box>
809        ) : null}
810        {waiting ? (
811          <Box key="hint" flexDirection="row" gap={1}>
812            <Text wrap="truncate-end">{`Viewer pane is waiting (${waiting.reason}).`}</Text>
813            <Button key="browse" label="Open in browser" hotkey="3" onPress={browseWaiting} />
814            <Button key="unhint" label="Dismiss" hotkey="4" onPress={() => update($, hint, () => null)} />
815          </Box>
816        ) : null}
817        {below}
818      </Box>
819    )
820  })
821}
822

1// A GLB (binary glTF 2.0) reader for what the viewer draws: every triangle of
2// the default scene in world space, and each material's base color.
3// https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html
4
5/** Reads `length` bytes of the file from `offset`. */
6export type ByteReader = (offset: number, length: number) => Promise<Uint8Array>
7
8export type Material = {
9  name: string
10  /** sRGB, 0..1. */
11  color: [number, number, number]
12  isTextured: boolean
13  isDoubleSided: boolean
14}
15
16/** A decoded image, row 0 at the top. */
17export type Texture = { width: number; height: number; rgb: Uint8Array }
18
19/** Decodes an embedded image, found at `offset` in the file; undefined when it cannot. */
20export type ImageDecoder = (image: { offset: number; length: number; mimeType: string }) => Promise<Texture | undefined>
21
22export type Mesh = {
23  /** Nine floats per triangle: three world-space vertices. */
24  positions: Float32Array
25  /** The palette index of each triangle. */
26  materials: Uint16Array
27  /** sRGB 0..255, three per triangle, when a base-color texture was sampled. */
28  colors?: Uint8Array
29  palette: Material[]
30  triangleCount: number
31  min: [number, number, number]
32  max: [number, number, number]
33}
34
35type Json = Record<string, any>
36
37const MAGIC = 0x46546c67 // 'glTF'
38const JSON_CHUNK = 0x4e4f534a
39const BIN_CHUNK = 0x004e4942
40const COMPONENT_SIZE: Record<number, number> = { 5120: 1, 5121: 1, 5122: 2, 5123: 2, 5125: 4, 5126: 4 }
41const TYPE_SIZE: Record<string, number> = { SCALAR: 1, VEC2: 2, VEC3: 3, VEC4: 4, MAT4: 16 }
42const UNSUPPORTED = ['KHR_draco_mesh_compression', 'EXT_meshopt_compression']
43// Untextured clay, for a material whose color lives in a texture this viewer does not decode.
44const CLAY: [number, number, number] = [0.8, 0.74, 0.66]
45
46const IDENTITY = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]
47
48const multiply = (a: number[], b: number[]) => {
49  const out = new Array<number>(16).fill(0)
50  for (let column = 0; column < 4; column += 1) {
51    for (let row = 0; row < 4; row += 1) {
52      let sum = 0
53      for (let k = 0; k < 4; k += 1) sum += a[k * 4 + row]! * b[column * 4 + k]!
54      out[column * 4 + row] = sum
55    }
56  }
57
58  return out
59}
60
61const nodeMatrix = (node: Json): number[] => {
62  if (Array.isArray(node.matrix) && node.matrix.length === 16) return node.matrix
63  const [tx, ty, tz] = node.translation ?? [0, 0, 0]
64  const [x, y, z, w] = node.rotation ?? [0, 0, 0, 1]
65  const [sx, sy, sz] = node.scale ?? [1, 1, 1]
66
67  return [
68    (1 - 2 * (y * y + z * z)) * sx, 2 * (x * y + w * z) * sx, 2 * (x * z - w * y) * sx, 0,
69    2 * (x * y - w * z) * sy, (1 - 2 * (x * x + z * z)) * sy, 2 * (y * z + w * x) * sy, 0,
70    2 * (x * z + w * y) * sz, 2 * (y * z - w * x) * sz, (1 - 2 * (x * x + y * y)) * sz, 0,
71    tx, ty, tz, 1,
72  ]
73}
74
75const determinant3 = (m: number[]) =>
76  m[0]! * (m[5]! * m[10]! - m[9]! * m[6]!) -
77  m[4]! * (m[1]! * m[10]! - m[9]! * m[2]!) +
78  m[8]! * (m[1]! * m[6]! - m[5]! * m[2]!)
79
80const toSrgb = (linear: number) => {
81  const c = Math.max(0, Math.min(1, linear))
82
83  return c <= 0.0031308 ? c * 12.92 : 1.055 * c ** (1 / 2.4) - 0.055
84}
85
86const readComponent = (view: DataView, offset: number, type: number, isNormalized: boolean) => {
87  switch (type) {
88    case 5126:
89      return view.getFloat32(offset, true)
90    case 5125:
91      return view.getUint32(offset, true)
92    case 5123: {
93      const v = view.getUint16(offset, true)
94      return isNormalized ? v / 65535 : v
95    }
96    case 5122: {
97      const v = view.getInt16(offset, true)
98      return isNormalized ? Math.max(v / 32767, -1) : v
99    }
100    case 5121: {
101      const v = view.getUint8(offset)
102      return isNormalized ? v / 255 : v
103    }
104    case 5120: {
105      const v = view.getInt8(offset)
106      return isNormalized ? Math.max(v / 127, -1) : v
107    }
108    default:
109      throw new Error(`unsupported accessor component type ${type}`)
110  }
111}
112
113export const parseGlb = async (read: ByteReader, size: number, decodeImage?: ImageDecoder): Promise<Mesh> => {
114  if (size < 20) throw new Error('not a GLB: the file is too short')
115  const header = new DataView((await read(0, 20)).slice().buffer)
116  if (header.getUint32(0, true) !== MAGIC) throw new Error('not a GLB: no glTF header (a .gltf text file is not supported)')
117  if (header.getUint32(4, true) !== 2) throw new Error(`glTF version ${header.getUint32(4, true)} is not supported`)
118  const jsonLength = header.getUint32(12, true)
119  if (header.getUint32(16, true) !== JSON_CHUNK) throw new Error('not a GLB: the first chunk is not JSON')
120  const gltf: Json = JSON.parse(new TextDecoder().decode(await read(20, jsonLength)))
121
122  const required: string[] = gltf.extensionsRequired ?? []
123  const blocked = required.filter(name => UNSUPPORTED.includes(name))
124  if (blocked.length > 0) {
125    throw new Error(`the mesh is compressed (${blocked.join(', ')}); use Open in browser`)
126  }
127
128  const binStart = 20 + jsonLength + 8
129  if (binStart <= size) {
130    const binHeader = new DataView((await read(20 + jsonLength, 8)).slice().buffer)
131    if (binHeader.getUint32(4, true) !== BIN_CHUNK) throw new Error('not a GLB: no binary chunk')
132  }
133  if ((gltf.buffers ?? []).some((buffer: Json, i: number) => i > 0 || buffer.uri !== undefined)) {
134    throw new Error('the GLB refers to external buffers, which the viewer does not read')
135  }
136
137  const readAccessor = async (index: number) => {
138    const accessor: Json = gltf.accessors?.[index]
139    if (!accessor) throw new Error(`accessor ${index} is missing`)
140    if (accessor.sparse) throw new Error('sparse accessors are not supported')
141    const width = TYPE_SIZE[accessor.type] ?? 1
142    const componentSize = COMPONENT_SIZE[accessor.componentType] ?? 4
143    const count: number = accessor.count
144    const values = new Float64Array(count * width)
145    if (accessor.bufferView === undefined) return { values, width, count }
146    const bufferView: Json = gltf.bufferViews[accessor.bufferView]
147    const elementSize = width * componentSize
148    const stride: number = bufferView.byteStride ?? elementSize
149    const start = (bufferView.byteOffset ?? 0) + (accessor.byteOffset ?? 0)
150    const length = count === 0 ? 0 : stride * (count - 1) + elementSize
151    const bytes = await read(binStart + start, length)
152    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
153    for (let i = 0; i < count; i += 1) {
154      for (let c = 0; c < width; c += 1) {
155        values[i * width + c] = readComponent(view, i * stride + c * componentSize, accessor.componentType, accessor.normalized === true)
156      }
157    }
158
159    return { values, width, count }
160  }
161
162  const palette: Material[] = (gltf.materials ?? []).map((material: Json, i: number): Material => {
163    const pbr: Json = material.pbrMetallicRoughness ?? {}
164    const factor: number[] = pbr.baseColorFactor ?? [1, 1, 1, 1]
165    const isTextured = pbr.baseColorTexture !== undefined
166    const isWhite = factor.slice(0, 3).every(v => v > 0.98)
167
168    return {
169      name: material.name ?? `material ${i}`,
170      color: isTextured && isWhite ? CLAY : [toSrgb(factor[0]!), toSrgb(factor[1]!), toSrgb(factor[2]!)],
171      isTextured,
172      isDoubleSided: material.doubleSided === true,
173    }
174  })
175  let defaultMaterial = -1
176
177  // One decode per image, shared by every material that samples it.
178  const textures = new Map<number, Promise<Texture | undefined>>()
179  const textureOf = (materialIndex: number) => {
180    const info: Json | undefined = gltf.materials?.[materialIndex]?.pbrMetallicRoughness?.baseColorTexture
181    const imageIndex: number | undefined = info === undefined ? undefined : gltf.textures?.[info.index]?.source
182    const image: Json | undefined = imageIndex === undefined ? undefined : gltf.images?.[imageIndex]
183    if (!decodeImage || imageIndex === undefined || image?.bufferView === undefined) return undefined
184    let pending = textures.get(imageIndex)
185    if (!pending) {
186      const bufferView: Json = gltf.bufferViews[image.bufferView]
187      pending = decodeImage({
188        offset: binStart + (bufferView.byteOffset ?? 0),
189        length: bufferView.byteLength,
190        mimeType: image.mimeType ?? '',
191      }).catch(() => undefined)
192      textures.set(imageIndex, pending)
193    }
194
195    return pending.then(texture => (texture ? { texture, texCoord: (info?.texCoord as number | undefined) ?? 0 } : undefined))
196  }
197  let isSampled = false
198
199  const chunks: { positions: Float32Array; materials: Uint16Array; colors: Uint8Array }[] = []
200  let triangleCount = 0
201
202  const addPrimitive = async (primitive: Json, matrix: number[]) => {
203    const mode = primitive.mode ?? 4
204    if (mode !== 4 && mode !== 5 && mode !== 6) return
205    if (primitive.attributes?.POSITION === undefined) return
206    const position = await readAccessor(primitive.attributes.POSITION)
207    const indices =
208      primitive.indices === undefined
209        ? Array.from({ length: position.count }, (_, i) => i)
210        : Array.from((await readAccessor(primitive.indices)).values)
211
212    let corners: number[] = []
213    if (mode === 4) {
214      corners = indices.length % 3 === 0 ? indices : indices.slice(0, indices.length - (indices.length % 3))
215    } else {
216      for (let i = 2; i < indices.length; i += 1) {
217        if (mode === 6) corners.push(indices[0]!, indices[i - 1]!, indices[i]!)
218        else if (i % 2 === 0) corners.push(indices[i - 2]!, indices[i - 1]!, indices[i]!)
219        else corners.push(indices[i - 1]!, indices[i - 2]!, indices[i]!)
220      }
221    }
222
223    let material = primitive.material
224    if (material === undefined) {
225      if (defaultMaterial < 0) {
226        defaultMaterial = palette.length
227        palette.push({ name: 'default', color: CLAY, isTextured: false, isDoubleSided: false })
228      }
229      material = defaultMaterial
230    }
231
232    const isFlipped = determinant3(matrix) < 0
233    const count = corners.length / 3
234    const positions = new Float32Array(count * 9)
235    const p = position.values
236    const m = matrix
237    for (let t = 0; t < count; t += 1) {
238      for (let k = 0; k < 3; k += 1) {
239        // A mirroring transform reverses the winding; swap two corners to keep faces facing out.
240        const corner = corners[t * 3 + (isFlipped && k > 0 ? 3 - k : k)]! * 3
241        const x = p[corner]!
242        const y = p[corner + 1]!
243        const z = p[corner + 2]!
244        const o = t * 9 + k * 3
245        positions[o] = m[0]! * x + m[4]! * y + m[8]! * z + m[12]!
246        positions[o + 1] = m[1]! * x + m[5]! * y + m[9]! * z + m[13]!
247        positions[o + 2] = m[2]! * x + m[6]! * y + m[10]! * z + m[14]!
248      }
249    }
250    const colors = new Uint8Array(count * 3)
251    const base = palette[material]?.color ?? CLAY
252    const sampled = palette[material]?.isTextured ? await textureOf(material) : undefined
253    const uvs = sampled ? primitive.attributes?.[`TEXCOORD_${sampled.texCoord}`] : undefined
254    if (sampled && uvs !== undefined) {
255      isSampled = true
256      const uv = (await readAccessor(uvs)).values
257      const { width, height, rgb } = sampled.texture
258      const factor = (gltf.materials?.[material]?.pbrMetallicRoughness?.baseColorFactor as number[] | undefined) ?? [1, 1, 1, 1]
259      const tint = [toSrgb(factor[0]!), toSrgb(factor[1]!), toSrgb(factor[2]!)]
260      for (let t = 0; t < count; t += 1) {
261        // The texel under the face's centre, wrapped as REPEAT does.
262        let u = 0
263        let v = 0
264        for (let k = 0; k < 3; k += 1) {
265          const corner = corners[t * 3 + k]!
266          u += uv[corner * 2]! / 3
267          v += uv[corner * 2 + 1]! / 3
268        }
269        const x = Math.min(width - 1, Math.floor((u - Math.floor(u)) * width))
270        const y = Math.min(height - 1, Math.floor((v - Math.floor(v)) * height))
271        const texel = (y * width + x) * 3
272        for (let c = 0; c < 3; c += 1) colors[t * 3 + c] = Math.round(rgb[texel + c]! * tint[c]!)
273      }
274    } else {
275      for (let t = 0; t < count; t += 1) {
276        for (let c = 0; c < 3; c += 1) colors[t * 3 + c] = Math.round(base[c]! * 255)
277      }
278    }
279    chunks.push({ positions, materials: new Uint16Array(count).fill(material), colors })
280    triangleCount += count
281  }
282
283  const visit = async (index: number, parent: number[], depth: number) => {
284    if (depth > 64) throw new Error('the node hierarchy is too deep')
285    const node: Json = gltf.nodes?.[index]
286    if (!node) return
287    const matrix = multiply(parent, nodeMatrix(node))
288    if (node.mesh !== undefined) {
289      for (const primitive of gltf.meshes?.[node.mesh]?.primitives ?? []) await addPrimitive(primitive, matrix)
290    }
291    for (const child of node.children ?? []) await visit(child, matrix, depth + 1)
292  }
293
294  const scene: Json | undefined = gltf.scenes?.[gltf.scene ?? 0]
295  let roots: number[] = scene?.nodes ?? []
296  if (!scene) {
297    const children = new Set<number>((gltf.nodes ?? []).flatMap((node: Json) => node.children ?? []))
298    roots = (gltf.nodes ?? []).map((_: unknown, i: number) => i).filter((i: number) => !children.has(i))
299  }
300  for (const root of roots) await visit(root, IDENTITY, 0)
301  if (triangleCount === 0) throw new Error('the GLB has no triangles to draw')
302
303  const positions = new Float32Array(triangleCount * 9)
304  const materials = new Uint16Array(triangleCount)
305  const colors = new Uint8Array(triangleCount * 3)
306  let at = 0
307  for (const chunk of chunks) {
308    positions.set(chunk.positions, at * 9)
309    materials.set(chunk.materials, at)
310    colors.set(chunk.colors, at * 3)
311    at += chunk.materials.length
312  }
313
314  const min: [number, number, number] = [Infinity, Infinity, Infinity]
315  const max: [number, number, number] = [-Infinity, -Infinity, -Infinity]
316  for (let i = 0; i < positions.length; i += 3) {
317    for (let c = 0; c < 3; c += 1) {
318      const v = positions[i + c]!
319      if (v < min[c]!) min[c] = v
320      if (v > max[c]!) max[c] = v
321    }
322  }
323
324  return { positions, materials, ...(isSampled ? { colors } : {}), palette, triangleCount, min, max }
325}
326
327/** A 24- or 32-bit uncompressed (or bitfield) BMP, as `sips` and ImageMagick write it. */
328export const parseBmp = (bytes: Uint8Array): Texture | undefined => {
329  if (bytes.length < 54 || bytes[0] !== 0x42 || bytes[1] !== 0x4d) return undefined
330  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
331  const dataOffset = view.getUint32(10, true)
332  const width = view.getInt32(18, true)
333  const rawHeight = view.getInt32(22, true)
334  const bitsPerPixel = view.getUint16(28, true)
335  const compression = view.getUint32(30, true)
336  if (width <= 0 || rawHeight === 0 || (bitsPerPixel !== 24 && bitsPerPixel !== 32)) return undefined
337  if (compression !== 0 && compression !== 3) return undefined
338  const height = Math.abs(rawHeight)
339  const stride = Math.floor((bitsPerPixel * width + 31) / 32) * 4
340  if (dataOffset + stride * height > bytes.length) return undefined
341
342  const masks =
343    bitsPerPixel === 32 && compression === 3
344      ? [view.getUint32(54, true), view.getUint32(58, true), view.getUint32(62, true)]
345      : undefined
346  const channel = (pixel: number, mask: number) => {
347    if (mask === 0) return 0
348    let shift = 0
349    while (((mask >>> shift) & 1) === 0) shift += 1
350
351    return (((pixel & mask) >>> shift) * 255) / (mask >>> shift)
352  }
353
354  const rgb = new Uint8Array(width * height * 3)
355  for (let y = 0; y < height; y += 1) {
356    // Rows run bottom-up unless the height is negative.
357    const row = dataOffset + (rawHeight < 0 ? y : height - 1 - y) * stride
358    for (let x = 0; x < width; x += 1) {
359      const o = (y * width + x) * 3
360      if (masks) {
361        const pixel = view.getUint32(row + x * 4, true)
362        rgb[o] = Math.round(channel(pixel, masks[0]!))
363        rgb[o + 1] = Math.round(channel(pixel, masks[1]!))
364        rgb[o + 2] = Math.round(channel(pixel, masks[2]!))
365      } else {
366        const p = row + x * (bitsPerPixel / 8)
367        rgb[o] = bytes[p + 2]!
368        rgb[o + 1] = bytes[p + 1]!
369        rgb[o + 2] = bytes[p]!
370      }
371    }
372  }
373
374  return { width, height, rgb }
375}
376

1import type { Mesh } from './glb'
2import { decimate, project, toCells, toSvg } from './render'
3import type { Faces } from './render'
4
5/** The SVG element's limit is 131072 characters. */
6const SVG_LIMIT = 130_000
7const SVG_BUDGET = 3200
8const CELL_BUDGET = 20_000
9export const SVG_WIDTH = 640
10export const SVG_HEIGHT = 480
11
12export type Model = {
13  path: string
14  name: string
15  size: number
16  mesh: Mesh
17  /** Faces per budget, and drawn frames per angle and size. */
18  faces: Map<number, Faces>
19  frames: Map<string, string>
20  /** The faces the SVG frames draw, once the first one found its budget. */
21  svgBudget: number
22}
23
24export const makeModel = (path: string, size: number, mesh: Mesh): Model => ({
25  path,
26  name: path.split('/').pop() ?? path,
27  size,
28  mesh,
29  faces: new Map(),
30  frames: new Map(),
31  svgBudget: SVG_BUDGET,
32})
33
34const facesFor = (model: Model, budget: number) => {
35  let faces = model.faces.get(budget)
36  if (!faces) {
37    faces = decimate(model.mesh, budget)
38    model.faces.set(budget, faces)
39  }
40
41  return faces
42}
43
44const remember = (model: Model, key: string, value: string) => {
45  // 36 angles on two surfaces is all a session needs; a resize starts over.
46  if (model.frames.size > 96) model.frames.clear()
47  model.frames.set(key, value)
48
49  return value
50}
51
52/** The SVG frame at `angle`, decimated until it fits the element's limit. */
53export const svgFrame = (model: Model, angle: number) => {
54  const key = `svg:${angle}`
55  for (;;) {
56    const faces = facesFor(model, model.svgBudget)
57    const cached = model.frames.get(key)
58    if (cached) return { svg: cached, shown: faces.count }
59    const svg = toSvg(project(model.mesh, faces, angle), SVG_WIDTH, SVG_HEIGHT)
60    if (svg.length <= SVG_LIMIT || model.svgBudget < 200) {
61      return { svg: remember(model, key, svg), shown: faces.count }
62    }
63    // Every angle draws the same faces, so a smaller budget clears the frames drawn so far.
64    model.svgBudget = Math.floor(model.svgBudget * 0.75)
65    for (const name of [...model.frames.keys()]) if (name.startsWith('svg:')) model.frames.delete(name)
66  }
67}
68
69/** The terminal frame at `angle`, as Raster cells. */
70export const cellFrame = (model: Model, angle: number, columns: number, rows: number) => {
71  const key = `cells:${angle}:${columns}x${rows}`
72  const faces = facesFor(model, CELL_BUDGET)
73  const cached = model.frames.get(key)
74  if (cached) return { cells: cached, shown: faces.count }
75
76  return { cells: remember(model, key, toCells(project(model.mesh, faces, angle), columns, rows)), shown: faces.count }
77}
78
79export const formatBytes = (bytes: number) => {
80  if (bytes < 1024) return `${bytes} B`
81  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
82
83  return `${(bytes / 1024 / 1024).toFixed(1)} MB`
84}
85
86const plural = (n: number, word: string) => `${n.toLocaleString('en-US')} ${word}${n === 1 ? '' : 's'}`
87
88/** The info strip: name, triangles, materials, size and bounding box. */
89export const infoLine = (model: Model, shown: number) => {
90  const { mesh } = model
91  const dims = [0, 1, 2].map(c => (mesh.max[c]! - mesh.min[c]!).toFixed(2)).join(' x ')
92  const triangles =
93    shown < mesh.triangleCount
94      ? `${plural(mesh.triangleCount, 'triangle')} (${shown.toLocaleString('en-US')} drawn)`
95      : plural(mesh.triangleCount, 'triangle')
96
97  return `${model.name}  ${triangles}  ${plural(mesh.palette.length, 'material')}  ${formatBytes(model.size)}  ${dims} m`
98}
99
100export const isTextured = (model: Model) => model.mesh.palette.some(material => material.isTextured)
101
102const escapeHtml = (text: string) =>
103  text.replace(/[&<>"']/g, ch => `&#${ch.charCodeAt(0)};`)
104
105/**
106 * The page "Open in browser" writes, split around the GLB's base64 so a
107 * shell can stream the file between them: <model-viewer> orbits, zooms
108 * and draws the textures.
109 */
110export const viewerPage = (name: string) => {
111  const title = escapeHtml(name)
112
113  return {
114    head:
115      `<!doctype html><html><head><meta charset="utf-8"><title>${title}</title>` +
116      '<meta name="viewport" content="width=device-width, initial-scale=1">' +
117      '<script type="module" src="https://unpkg.com/@google/model-viewer@4/dist/model-viewer.min.js"></script>' +
118      '<style>html,body{margin:0;height:100%;background:#202124;color:#ddd;font:13px system-ui}' +
119      'model-viewer{width:100%;height:100%}p{position:fixed;left:12px;bottom:4px;margin:0;opacity:.7}</style>' +
120      `</head><body><model-viewer camera-controls auto-rotate shadow-intensity="1" alt="${title}" ` +
121      'src="data:model/gltf-binary;base64,',
122    tail: `"></model-viewer><p>${title} - drag to orbit, scroll to zoom</p></body></html>\n`,
123  }
124}
125

1// A small flat-shaded renderer: decimate by vertex clustering, turn and tilt,
2// cull back faces, shade by normal, sort back to front, then draw the faces as
3// SVG paths (desktop) or as half-block cells (terminal).
4
5import type { Mesh } from './glb'
6
7export type Faces = {
8  positions: Float32Array
9  materials: Uint16Array
10  /** sRGB 0..255, three per face, when the mesh has sampled colors. */
11  colors?: Uint8Array
12  count: number
13}
14
15/** A face on screen: corners in -1..1 (y up), depth, sRGB color 0..255. */
16type Projected = {
17  x: [number, number, number]
18  y: [number, number, number]
19  z: number
20  r: number
21  g: number
22  b: number
23}
24
25export const PITCH_DEGREES = 20
26const LIGHT = (() => {
27  const v = [-0.35, 0.55, 0.75]
28  const n = Math.hypot(v[0]!, v[1]!, v[2]!)
29
30  return v.map(c => c / n) as [number, number, number]
31})()
32const AMBIENT = 0.32
33// Shade in steps, so neighbouring faces of one color share a path.
34const SHADE_LEVELS = 24
35// The model's bounding sphere spans this share of the frame's shorter side, at any turn.
36const FILL = 0.47
37
38/**
39 * At most `budget` faces: the mesh itself when it fits, else a vertex
40 * clustering on an ever coarser grid until it does.
41 */
42export const decimate = (mesh: Mesh, budget: number): Faces => {
43  if (mesh.triangleCount <= budget) {
44    return { positions: mesh.positions, materials: mesh.materials, colors: mesh.colors, count: mesh.triangleCount }
45  }
46  const size = Math.max(mesh.max[0] - mesh.min[0], mesh.max[1] - mesh.min[1], mesh.max[2] - mesh.min[2]) || 1
47  // A closed surface clustered on a G-cell grid keeps about 4 G^2 faces; start a little finer.
48  let grid = Math.max(2, Math.min(256, Math.round(Math.sqrt(budget / 4) * 1.4)))
49  // Settle on the finest grid that fits: shrink while over budget, grow while well under it.
50  let best: Faces | undefined
51  const tried = new Set<number>()
52  for (let attempt = 0; attempt < 12 && !tried.has(grid); attempt += 1) {
53    tried.add(grid)
54    const faces = cluster(mesh, grid, size)
55    const fits = faces.count <= budget
56    if (fits && (!best || faces.count > best.count)) best = faces
57    if (fits && (faces.count >= budget * 0.7 || grid >= 256)) break
58    if (!fits && grid <= 2) break
59    const next = grid * Math.sqrt(budget / Math.max(1, faces.count)) * 0.95
60    grid = Math.max(2, Math.min(256, fits ? Math.ceil(next) : Math.floor(next)))
61  }
62  if (best) return best
63
64  // Coarser than a 2-cell grid cannot merge more; keep the first faces that fit.
65  const coarsest = cluster(mesh, 2, size)
66  const count = Math.min(coarsest.count, budget)
67
68  return {
69    positions: coarsest.positions.slice(0, count * 9),
70    materials: coarsest.materials.slice(0, count),
71    colors: coarsest.colors?.slice(0, count * 3),
72    count,
73  }
74}
75
76const cluster = (mesh: Mesh, grid: number, size: number): Faces => {
77  const cellOf = new Map<number, number>()
78  const sums: number[] = []
79  const counts: number[] = []
80  const vertexCells = new Uint32Array(mesh.triangleCount * 3)
81  const p = mesh.positions
82  const [minX, minY, minZ] = mesh.min
83  const top = grid - 1
84  const perUnit = grid / size
85  for (let v = 0; v < mesh.triangleCount * 3; v += 1) {
86    const cx = Math.min(top, Math.floor((p[v * 3]! - minX) * perUnit))
87    const cy = Math.min(top, Math.floor((p[v * 3 + 1]! - minY) * perUnit))
88    const cz = Math.min(top, Math.floor((p[v * 3 + 2]! - minZ) * perUnit))
89    const key = (cx * grid + cy) * grid + cz
90    let index = cellOf.get(key)
91    if (index === undefined) {
92      index = counts.length
93      cellOf.set(key, index)
94      counts.push(0)
95      sums.push(0, 0, 0)
96    }
97    counts[index]! += 1
98    sums[index * 3]! += p[v * 3]!
99    sums[index * 3 + 1]! += p[v * 3 + 1]!
100    sums[index * 3 + 2]! += p[v * 3 + 2]!
101    vertexCells[v] = index
102  }
103
104  const seen = new Set<string>()
105  const kept: number[] = []
106  for (let t = 0; t < mesh.triangleCount; t += 1) {
107    const a = vertexCells[t * 3]!
108    const b = vertexCells[t * 3 + 1]!
109    const c = vertexCells[t * 3 + 2]!
110    if (a === b || b === c || a === c) continue
111    const key = [a, b, c].sort((m, n) => m - n).join(',')
112    if (seen.has(key)) continue
113    seen.add(key)
114    kept.push(t)
115  }
116
117  const positions = new Float32Array(kept.length * 9)
118  const materials = new Uint16Array(kept.length)
119  const colors = mesh.colors ? new Uint8Array(kept.length * 3) : undefined
120  kept.forEach((t, i) => {
121    colors?.set(mesh.colors!.subarray(t * 3, t * 3 + 3), i * 3)
122    for (let k = 0; k < 3; k += 1) {
123      const cell = vertexCells[t * 3 + k]!
124      for (let axis = 0; axis < 3; axis += 1) {
125        positions[i * 9 + k * 3 + axis] = sums[cell * 3 + axis]! / counts[cell]!
126      }
127    }
128    materials[i] = mesh.materials[t]!
129  })
130
131  return { positions, materials, colors, count: kept.length }
132}
133
134/** The visible faces at a turn of `angle` degrees, farthest first, inside the unit circle. */
135export const project = (mesh: Mesh, faces: Faces, angle: number): Projected[] => {
136  const center = [0, 1, 2].map(c => (mesh.min[c]! + mesh.max[c]!) / 2)
137  const size = Math.max(mesh.max[0] - mesh.min[0], mesh.max[1] - mesh.min[1], mesh.max[2] - mesh.min[2]) || 1
138  const yaw = (angle * Math.PI) / 180
139  const pitch = (PITCH_DEGREES * Math.PI) / 180
140  const [cy, sy, cp, sp] = [Math.cos(yaw), Math.sin(yaw), Math.cos(pitch), Math.sin(pitch)]
141  // The farthest corner from the centre: turning never moves the model past it.
142  let radius = 0
143  for (let o = 0; o < faces.count * 9; o += 3) {
144    const d = Math.hypot(faces.positions[o]! - center[0]!, faces.positions[o + 1]! - center[1]!, faces.positions[o + 2]! - center[2]!)
145    if (d > radius) radius = d
146  }
147  const unit = radius || size
148
149  const out: Projected[] = []
150  const vx = [0, 0, 0]
151  const vy = [0, 0, 0]
152  const vz = [0, 0, 0]
153  for (let t = 0; t < faces.count; t += 1) {
154    for (let k = 0; k < 3; k += 1) {
155      const o = t * 9 + k * 3
156      const x = (faces.positions[o]! - center[0]!) / unit
157      const y = (faces.positions[o + 1]! - center[1]!) / unit
158      const z = (faces.positions[o + 2]! - center[2]!) / unit
159      const x1 = x * cy + z * sy
160      const z1 = -x * sy + z * cy
161      vx[k] = x1
162      vy[k] = y * cp - z1 * sp
163      vz[k] = y * sp + z1 * cp
164    }
165    const ax = vx[1]! - vx[0]!
166    const ay = vy[1]! - vy[0]!
167    const az = vz[1]! - vz[0]!
168    const bx = vx[2]! - vx[0]!
169    const by = vy[2]! - vy[0]!
170    const bz = vz[2]! - vz[0]!
171    let nx = ay * bz - az * by
172    let ny = az * bx - ax * bz
173    let nz = ax * by - ay * bx
174    const length = Math.hypot(nx, ny, nz)
175    if (length === 0) continue
176    // Back faces go even on double-sided materials: a closed model hides them anyway,
177    // and skipping them leaves the budget for the side that shows.
178    if (nz <= 0) continue
179    const lambert = Math.max(0, (nx * LIGHT[0] + ny * LIGHT[1] + nz * LIGHT[2]) / length)
180    const shade = Math.round((AMBIENT + (1 - AMBIENT) * lambert) * SHADE_LEVELS) / SHADE_LEVELS
181    let [r, g, b] = (mesh.palette[faces.materials[t]!]?.color ?? [0.8, 0.8, 0.8]).map(c => c * 255)
182    if (faces.colors) [r, g, b] = [faces.colors[t * 3]!, faces.colors[t * 3 + 1]!, faces.colors[t * 3 + 2]!]
183    out.push({
184      x: [vx[0]!, vx[1]!, vx[2]!],
185      y: [vy[0]!, vy[1]!, vy[2]!],
186      z: (vz[0]! + vz[1]! + vz[2]!) / 3,
187      r: Math.round(Math.min(255, r! * shade)),
188      g: Math.round(Math.min(255, g! * shade)),
189      b: Math.round(Math.min(255, b! * shade)),
190    })
191  }
192
193  return out.sort((m, n) => m.z - n.z)
194}
195
196const hex = (r: number, g: number, b: number) =>
197  `#${[r, g, b].map(c => c.toString(16).padStart(2, '0')).join('')}`
198
199/**
200 * The faces as one SVG document: consecutive faces of one color share a path.
201 * `width` and `height` are the viewBox; the element scales it to its slot.
202 */
203export const toSvg = (faces: Projected[], width: number, height: number) => {
204  const scale = Math.min(width, height) * FILL
205  const parts: string[] = [
206    `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${width} ${height}" width="${width}" height="${height}">`,
207    // A hairline of each face's own color closes the seams between faces.
208    '<g stroke-width=".8" stroke-linejoin="round">',
209  ]
210  let fill = ''
211  let d = ''
212  const flush = () => {
213    if (d) parts.push(`<path fill="${fill}" stroke="${fill}" d="${d}"/>`)
214    d = ''
215  }
216  for (const face of faces) {
217    const color = hex(face.r, face.g, face.b)
218    if (color !== fill) {
219      flush()
220      fill = color
221    }
222    const point = (k: number) =>
223      `${Math.round(width / 2 + face.x[k]! * scale)} ${Math.round(height / 2 - face.y[k]! * scale)}`
224    d += `M${point(0)} ${point(1)} ${point(2)}z`
225  }
226  flush()
227  parts.push('</g></svg>')
228
229  return parts.join('')
230}
231
232const DEFAULT_COLOR = 0x01000000
233
234/**
235 * The faces drawn into `columns` x `rows` terminal cells, two pixels per cell
236 * (the upper and lower half block), as RasterProps `cells`.
237 */
238export const toCells = (faces: Projected[], columns: number, rows: number) => {
239  const width = columns
240  const height = rows * 2
241  const pixels = new Int32Array(width * height).fill(-1)
242  const scale = Math.min(width, height) * FILL
243  for (const face of faces) {
244    const xs = face.x.map(x => width / 2 + x * scale)
245    const ys = face.y.map(y => height / 2 - y * scale)
246    const color = (face.r << 16) | (face.g << 8) | face.b
247    const minX = Math.max(0, Math.floor(Math.min(...xs)))
248    const maxX = Math.min(width - 1, Math.ceil(Math.max(...xs)))
249    const minY = Math.max(0, Math.floor(Math.min(...ys)))
250    const maxY = Math.min(height - 1, Math.ceil(Math.max(...ys)))
251    const edge = (a: number, b: number, px: number, py: number) =>
252      (xs[b]! - xs[a]!) * (py - ys[a]!) - (ys[b]! - ys[a]!) * (px - xs[a]!)
253    const area = edge(0, 1, xs[2]!, ys[2]!)
254    if (area === 0) continue
255    for (let py = minY; py <= maxY; py += 1) {
256      for (let px = minX; px <= maxX; px += 1) {
257        const cx = px + 0.5
258        const cy = py + 0.5
259        const w0 = edge(1, 2, cx, cy) / area
260        const w1 = edge(2, 0, cx, cy) / area
261        const w2 = edge(0, 1, cx, cy) / area
262        if (w0 >= 0 && w1 >= 0 && w2 >= 0) pixels[py * width + px] = color
263      }
264    }
265  }
266
267  const words = new Uint32Array(columns * rows * 3)
268  for (let row = 0; row < rows; row += 1) {
269    for (let column = 0; column < columns; column += 1) {
270      const top = pixels[row * 2 * width + column]!
271      const bottom = pixels[(row * 2 + 1) * width + column]!
272      const o = (row * columns + column) * 3
273      if (top < 0 && bottom < 0) {
274        words.set([0x20, DEFAULT_COLOR, DEFAULT_COLOR], o)
275      } else if (top < 0) {
276        words.set([0x2584, bottom, DEFAULT_COLOR], o) // lower half block
277      } else {
278        words.set([0x2580, top, bottom < 0 ? DEFAULT_COLOR : bottom], o) // upper half block
279      }
280    }
281  }
282
283  return toBase64(littleEndian(words))
284}
285
286const littleEndian = (words: Uint32Array) => {
287  const bytes = new Uint8Array(words.length * 4)
288  const view = new DataView(bytes.buffer)
289  words.forEach((word, i) => view.setUint32(i * 4, word, true))
290
291  return bytes
292}
293
294const ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
295
296export const toBase64 = (bytes: Uint8Array) => {
297  let out = ''
298  for (let i = 0; i < bytes.length; i += 3) {
299    const a = bytes[i]!
300    const b = bytes[i + 1] ?? 0
301    const c = bytes[i + 2] ?? 0
302    out += ALPHABET[a >> 2]! + ALPHABET[((a & 3) << 4) | (b >> 4)]!
303    out += i + 1 < bytes.length ? ALPHABET[((b & 15) << 2) | (c >> 6)]! : '='
304    out += i + 2 < bytes.length ? ALPHABET[c & 63]! : '='
305  }
306
307  return out
308}
309
310const DECODE = new Map([...ALPHABET].map((ch, i) => [ch, i]))
311
312export const fromBase64 = (text: string) => {
313  const clean = text.replace(/[^A-Za-z0-9+/]/g, '')
314  const bytes = new Uint8Array(Math.floor((clean.length * 3) / 4))
315  let at = 0
316  for (let i = 0; i < clean.length; i += 4) {
317    const [a, b, c, d] = [0, 1, 2, 3].map(k => DECODE.get(clean[i + k] ?? 'A') ?? 0) as [number, number, number, number]
318    const n = (a << 18) | (b << 12) | (c << 6) | d
319    if (at < bytes.length) bytes[at++] = n >> 16
320    if (at < bytes.length) bytes[at++] = (n >> 8) & 255
321    if (at < bytes.length) bytes[at++] = n & 255
322  }
323
324  return bytes
325}
326

1import type { ClientModule } from 'claude-code'
2
3type DragProps = { label?: string }
4
5/**
6 * The surface that turns the model while dragged: each move posts the cells
7 * the pointer travelled, and keys post turns and zooms; the hooks module
8 * turns those into camera moves. With a `label` it is a strip of text; without
9 * one it draws nothing, a transparent layer the terminal lays over the picture.
10 */
11const DragStrip: ClientModule<DragProps, { isReady: true }> = (props, surface) => {
12  if (surface.state === undefined) {
13    let last: { x: number; y: number } | undefined
14    surface.onPointer(event => {
15      const x = event.fine?.x ?? event.x
16      const y = event.fine?.y ?? event.y
17      if (event.type === 'down') {
18        last = { x, y }
19      } else if (event.type === 'move' && last && event.button !== undefined) {
20        const dx = Math.trunc(x - last.x)
21        const dy = Math.trunc(y - last.y)
22        if (dx !== 0 || dy !== 0) {
23          surface.post({ dx, dy })
24          last = { x: last.x + dx, y: last.y + dy }
25        }
26      } else if (event.type === 'up') {
27        last = undefined
28      }
29    })
30    // Keys name the move, as the buttons do; a drag posts the pointer's travel.
31    surface.onKey(event => {
32      if (event.key === 'left') surface.post({ turn: -1 })
33      else if (event.key === 'right') surface.post({ turn: 1 })
34      else if (event.key === 'up') surface.post({ tilt: -1 })
35      else if (event.key === 'down') surface.post({ tilt: 1 })
36      else if (event.key === '+' || event.key === '=') surface.post({ zoom: 1 })
37      else if (event.key === '-') surface.post({ zoom: -1 })
38    })
39    surface.setState({ isReady: true })
40  }
41  const { Box, Text } = surface.elements
42  if (props.label === undefined) return <Box width={surface.columns || 1} height={surface.rows || 1} />
43
44  return (
45    <Box>
46      <Text dimColor>{props.label}</Text>
47    </Box>
48  )
49}
50
51export default DragStrip
52

1export type ViewerMode = 'vector' | 'live'
2
3export type ViewerView = {
4  /** The GLB on screen, absolute; null before /meshy-view. */
5  path: string | null
6  /** Turn around the vertical axis, degrees. */
7  angle: number
8  /** Tilt, degrees from straight above: 90 is level (live mode). */
9  phi: number
10  /** Camera distance, percent of the fitted distance (live mode). */
11  zoom: number
12  isAutoRotating: boolean
13  /** True while the viewer pane is open. */
14  isOpen: boolean
15  /** `live` draws real frames from headless Chrome in a kitty-graphics terminal; `vector` draws faces. */
16  mode: ViewerMode
17}
18
19/** Shown above the prompt when the pane waits unplaced. */
20export type ViewerHint = { path: string; reason: string }
21
22declare module 'claude-code' {
23  interface PluginState {
24    'meshy-viewer': {
25      view: ViewerView
26      hint: ViewerHint | null
27      /** meshy-tray task ids whose "View" offer was taken or dismissed. */
28      dismissed: string[]
29    }
30  }
31}
32