blender-live

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { BlenderFrame, BlenderStep } from '../types'
5import { renderSvg } from './view3d'
6import type { Model } from './view3d'
7
8type $ = EngineInterface
9
10const PLUGIN = 'blender-live'
11const PANE = 'blender-live'
12const TITLE = 'Blender Live'
13const TICK_MS = 110
14const SPIN_DEG = 3
15const SPIN_AFTER_STEP_MS = 40_000
16const GLOW_MS = 7_000
17const REVEAL_MS = 2_600
18// how long each streamed snapshot stays; faster when many are waiting
19const FRAME_MS = 700
20const FAST_FRAME_MS = 280
21
22const steps = atom({ plugin: 'blender-live', key: 'steps' } as const, [])
23const viewing = atom({ plugin: 'blender-live', key: 'viewing' } as const, null)
24const angle = atom({ plugin: 'blender-live', key: 'angle' } as const, 35)
25const spinUntil = atom({ plugin: 'blender-live', key: 'spinUntil' } as const, 0)
26const isWire = atom({ plugin: 'blender-live', key: 'isWire' } as const, false)
27const busy = atom({ plugin: 'blender-live', key: 'busy' } as const, null)
28const flashUntil = atom({ plugin: 'blender-live', key: 'flashUntil' } as const, 0)
29const revealStart = atom({ plugin: 'blender-live', key: 'revealStart' } as const, 0)
30const isBandHidden = atom({ plugin: 'blender-live', key: 'isBandHidden' } as const, false)
31const frame = atom({ plugin: 'blender-live', key: 'frame' } as const, null)
32const frameStart = atom({ plugin: 'blender-live', key: 'frameStart' } as const, 0)
33const queue = atom({ plugin: 'blender-live', key: 'queue' } as const, [])
34const note = atom({ plugin: 'blender-live', key: 'note' } as const, null)
35
36const STEP_DESCRIPTION = `Run one modeling step in Blender (headless) and stream it to the user's live feed.
37
38The user watches a 3D feed of the scene: a simplified preview streams while your code runs, and when the step finishes the feed swaps in the full-detail model with its real materials (a glTF the step exports, with procedural textures baked into images, lit by the scene's own world/sky). So build in visible steps with short labels ("Hull", "Canopy", "Nose gear"). The scene is saved between calls. Any size works, from N64 low-poly to multi-million-triangle models.
39
40\`code\` is Python run inside Blender with these ready to use: bpy, bmesh, Vector, math, np (numpy), and
41  mat(name, (r, g, b))  -> get/create a simple material (linear RGB 0..1); any real node material works too
42  paint(obj, material)  -> give an object that material
43  snap("what")          -> take a live frame now (also works inside your own helper functions)
44  focus("Armory*")      -> point the feed camera at these objects (globs, comma separated) from here on
45Objects whose names start with "_" are helpers and are hidden. The ground is at z = 0. Folders and prelude files set with blender_configure (an art kit, gear code) are importable/loaded in every step.
46
47Live frames: by default one after every top-level line (and every line directly in a top-level loop) that changes the scene, paced automatically so big scenes stay fast. Options: \`mode\` "objects" (a frame only when parts are added/removed: good for scripts that make hundreds of parts) or "end" (one frame per step); \`pace\` seconds between frames; \`focus\` to look at part of a big model (everything around it turns see-through in the feed); \`render\` "cycles" (or "cycles:128:1920x1080") to also render the step realistically with Cycles.
48
49If the code raises, nothing is saved and you get the traceback: fix it and call again. On success you get the triangle count, the objects, and a list of files to send to the feed (do that right away, as the result says).`
50
51let workDir = 'blender-live'
52let ticker: { cancel: () => void } | undefined
53const models = new Map<string, Model>()
54
55function run($: $, args: string[], timeoutMs = 300_000) {
56  return $.process.run(['sh', `${$.plugin.root}/engine/bstep`, ...args], {
57    env: { BSTEP_DIR: workDir },
58    timeoutMs,
59  })
60}
61
62async function openPane($: $) {
63  return $.ui.open({ id: PANE, title: TITLE })
64}
65
66function compactLine(input: unknown, output: unknown, isRunning: boolean, isErrored: boolean) {
67  const label = String((input as { label?: unknown } | null)?.label ?? '')
68  const text = typeof output === 'string' ? output : JSON.stringify(output ?? '')
69  const tris = /(\d+) triangles/.exec(text)?.[1]
70  if (isRunning) return `Blender: building "${label}"...`
71  if (isErrored) return `Blender: "${label}" failed, fixing it`
72  return `Blender: "${label}" done${tris ? ` (${tris} tris)` : ''}, watch it in Blender Live`
73}
74
75async function spinFor($: $, ms: number) {
76  const now = await $.clock.now()
77  await update($, spinUntil, () => now + ms)
78}
79
80async function loadModel($: $, path: string): Promise<Model | null> {
81  const cached = models.get(path)
82  if (cached) return cached
83  try {
84    const raw = JSON.parse(await $.fs.read(path)) as Model & { format?: number; objects: unknown[] }
85    const model = raw.format === 3 ? fromFormat3(raw as unknown as Format3) : raw
86    models.set(path, model)
87    return model
88  } catch {
89    return null
90  }
91}
92
93type Format3Entry = { n: string; c: number[]; o?: boolean; lo: number[]; hi: number[]; v: string; i: string; w: number }
94type Format3 = { step: number; label: string; tris: number; objects: Format3Entry[] }
95
96function bytesOf(b64: string): Uint8Array {
97  const bin = atob(b64)
98  const out = new Uint8Array(bin.length)
99  for (let k = 0; k < bin.length; k++) out[k] = bin.charCodeAt(k)
100  return out
101}
102
103// the engine's compact frames (16-bit positions + indices) as flat triangles for the SVG view
104function fromFormat3(doc: Format3): Model {
105  const objects = doc.objects.map(o => {
106    const vb = bytesOf(o.v)
107    const q = new Int16Array(vb.buffer, vb.byteOffset, vb.byteLength / 2)
108    const ib = bytesOf(o.i)
109    const idx = o.w === 4 ? new Uint32Array(ib.buffer, ib.byteOffset, ib.byteLength / 4) : new Uint16Array(ib.buffer, ib.byteOffset, ib.byteLength / 2)
110    const tris: number[] = []
111    for (const v of idx) {
112      for (let a = 0; a < 3; a++) {
113        const lo = o.lo[a] ?? 0
114        const hi = o.hi[a] ?? 0
115        tris.push(((q[v * 3 + a] ?? 0) + 32768) / 65535 * (hi - lo) + lo)
116      }
117    }
118    return { name: o.n, color: o.c, open: o.o ?? true, tris }
119  })
120  return { step: doc.step, label: doc.label, tris: doc.tris, objects }
121}
122
123const fileName = (p: string) => p.split('/').pop()?.replace(/\.json$/, '') ?? p
124
125// what Claude sends to the feed page after a step: files to upload, then database writes
126async function feedInstructions($: $, n: number, stdout: string, feedFrames: string[]) {
127  const assets = [...stdout.matchAll(/^BSTEP_ASSET (.+)$/gm)].map(m => m[1]!.trim())
128  const gltf = /^BSTEP_GLTF (.+)$/m.exec(stdout)?.[1]?.trim()
129  const sky = /^BSTEP_SKY (.+)$/m.exec(stdout)?.[1]?.trim()
130  const render = /^BSTEP_RENDER (.+?)\t(.*)$/m.exec(stdout)
131  const frameset = /^BSTEP_FRAMESET (.+)$/m.exec(stdout)?.[1]?.trim()
132  const warns = [...stdout.matchAll(/^BSTEP_WARN (.+)$/gm)].map(m => m[1])
133  const baked = /^BSTEP_BAKED (.+)$/m.exec(stdout)?.[1]
134  const out: string[] = ['', '--- Send to the feed now ---']
135  const uploads = [...assets, ...(render ? [render[1]!] : [])]
136  if (uploads.length) {
137    out.push(`1. Upload to the feed artifact (Artifact publish, url = the feed, asset: true, file_paths up to 25 per call): ${uploads.join(' ')}`)
138    out.push('   Each .js file is a part of a packed file: its key is in the file (window.BLD["<key>/<part>"]). ' +
139      'Write one "files" doc per key: {run, key, urls: [url of part 0, part 1, ...]}, doc id = key with "/" replaced by "_".')
140  }
141  // frames in batches under the 1 MiB / 50 writes limits
142  const batches: string[][] = [[]]
143  let size = 0
144  for (const p of feedFrames) {
145    let bytes = 0
146    try {
147      bytes = (await $.fs.stat(p)).size
148    } catch {}
149    const last = batches[batches.length - 1]!
150    if (last.length >= 45 || size + bytes > 900_000) {
151      batches.push([])
152      size = 0
153    }
154    batches[batches.length - 1]!.push(p)
155    size += bytes
156  }
157  const writes = batches.filter(b => b.length).map(b => b.map(p => `frames/${fileName(p)} <- ${p}`).join(', '))
158  out.push(`${uploads.length ? 2 : 1}. One ArtifactData batch (set; file_path where a file is named; doc ids start with the run id, e.g. <run>_s${n}, so earlier models never clash):`)
159  if (frameset) out.push(`   framesets/<run>_s${n} <- ${frameset}  (this step's ${feedFrames.length} live frames, packed in the uploaded frames_s${String(n).padStart(2, '0')} file)`)
160  else writes.forEach((w, k) => out.push(`   frames batch ${k + 1}: ${w}`))
161  if (gltf) out.push(`   also: gltf/<run>_s${n} <- ${gltf}`)
162  if (sky) out.push(`   also: sky/<run>_s${n} <- ${sky}`)
163  if (render) out.push(`   also: renders/<run>_s${n} = {run, step: ${n}, url: <uploaded url of ${render[1]}>, samples, size, time} (${render[2]!.split('\t').join(', ')})`)
164  out.push('   and one chat/mNNNN note {seq, at, text} saying what you built.')
165  if (baked) out.push(`Baked textures: ${baked}`)
166  if (warns.length) out.push(`Warnings: ${warns.join(' | ')}`)
167  return '\n' + out.join('\n')
168}
169
170// a failed call is a deny: the model reads the text as an error result
171function textResult(text: string, isError = false) {
172  return isError ? { deny: text } : { result: text }
173}
174
175// plays the next waiting snapshot once the current one has had its time
176async function playNext($: $) {
177  const waiting = await read($, queue)
178  if (waiting.length === 0) return
179  const now = await $.clock.now()
180  const hold = waiting.length > 10 ? FAST_FRAME_MS : FRAME_MS
181  if (now - (await read($, frameStart)) < hold) return
182  const [next, ...rest] = waiting
183  await update($, queue, () => rest)
184  await update($, frame, () => next ?? null)
185  await update($, frameStart, () => now)
186  await update($, viewing, () => null)
187}
188
189// what the pane's buttons do
190async function act($: $, key: string) {
191  const list = await read($, steps)
192  const current = (await read($, viewing)) ?? list.at(-1)?.n ?? 0
193  const go = (n: number | null) => update($, viewing, () => (n === null || n >= (list.at(-1)?.n ?? 0) ? null : Math.max(1, n)))
194
195  if (key === 'prev') await go(current - 1)
196  else if (key === 'next') await go(current + 1)
197  else if (key === 'latest') await go(null)
198  else if (key === 'wire') await update($, isWire, w => !w)
199  else if (key === 'spin') {
200    const now = await $.clock.now()
201    await update($, spinUntil, until => (until > now ? 0 : now + 10 * 60_000))
202  } else if (key === 'replay' && list.length > 0) {
203    const all = list.flatMap(st => st.frames ?? [])
204    await update($, frame, () => null)
205    await update($, queue, () => all.map((f, i) => ({ ...f, prevPath: i === 0 ? null : all[i - 1]!.path })))
206    await spinFor($, all.length * FAST_FRAME_MS + 20_000)
207  } else if (key === 'hide') await update($, isBandHidden, () => true)
208  else if (key === 'pane') await openPane($)
209  else if (key.startsWith('s')) await go(Number(key.slice(1)))
210}
211
212// everything the live view draws from, read in one place (reads subscribe the drawing)
213async function liveView($: $) {
214  const list = await read($, steps)
215  const at = await read($, viewing)
216  const shown = at === null ? list.at(-1) : list.find(s => s.n === at)
217  const working = await read($, busy)
218  const now = await $.clock.now()
219  const isSpinning = now < (await read($, spinUntil))
220  const fraction = (now - (await read($, revealStart))) / REVEAL_MS
221  const isWireOn = await read($, isWire)
222  const live = at === null ? await read($, frame) : null
223  let svg: string
224  if (live) {
225    const sinceFrame = now - (await read($, frameStart))
226    svg = renderSvg(await loadModel($, live.path), {
227      angle: await read($, angle),
228      isWire: isWireOn,
229      glow: [],
230      caption: live.label,
231      transition: {
232        prev: live.prevPath ? await loadModel($, live.prevPath) : null,
233        t: sinceFrame / Math.min(FRAME_MS * 0.8, 520),
234      },
235      codeLine: live.code,
236      live: { building: working },
237    })
238  } else {
239    const before = shown ? list.find(st => st.n === shown.n - 1) : undefined
240    svg = renderSvg(shown ? await loadModel($, shown.json) : null, {
241      angle: await read($, angle),
242      isWire: isWireOn,
243      glow: shown && now < (await read($, flashUntil)) ? shown.added : [],
244      caption: shown ? `${shown.n}. ${shown.label}` : 'empty scene',
245      transition: shown
246        ? { prev: before ? await loadModel($, before.json) : null, t: fraction }
247        : undefined,
248      live: { building: working },
249    })
250  }
251  const status = working
252    ? `Building: ${working}...`
253    : shown
254      ? `Step ${shown.n} of ${list.length}: ${shown.label}`
255      : 'Waiting for the first step. Ask Claude to model something!'
256  return { list, shown, working, isSpinning, isWireOn, svg, status, message: await read($, note) }
257}
258
259
260export const register: Register = on => {
261  on('session.start', async ($, e, next) => {
262    const started = await next(e)
263    workDir = `${started.cwd}/blender-live`
264
265    await $.command.register({ name: 'blender', description: 'Show the Blender Live view above the prompt' })
266    await $.command.register({ name: 'blender-pane', description: 'Open Blender Live in a big side pane' })
267    await $.command.register({
268      name: 'blender-setup',
269      description: 'Install Blender for the Blender Live mod (one time, about a minute). Add a version, e.g. 4.2, for that one',
270    })
271
272    await $.tool.register({
273      name: 'blender_step',
274      description: STEP_DESCRIPTION,
275      isDeferred: false,
276      inputSchema: {
277        type: 'object',
278        properties: {
279          label: { type: 'string', description: 'Short name of this step, shown to the user' },
280          code: { type: 'string', description: 'Python (bpy) to run on the scene' },
281          focus: { type: 'string', description: 'Objects the feed camera looks at, e.g. "Armory*" or "NoseGear*,GearDoor*"' },
282          mode: { type: 'string', enum: ['line', 'objects', 'end'], description: 'When to take live frames (default line)' },
283          pace: { type: 'number', description: 'At least this many seconds between live frames' },
284          render: { type: 'string', description: '"cycles", or "cycles:<samples>:<W>x<H>", to also render this step with Cycles' },
285        },
286        required: ['label', 'code'],
287      },
288    })
289    await $.tool.register({
290      name: 'blender_configure',
291      description:
292        'Change Blender Live settings for this project (kept in config.json, used by every later step). ' +
293        'paths: folders to put on sys.path (your helper modules). prelude: .py files run before every step (an art kit). ' +
294        'blender: path to a Blender executable to use instead of the bundled bpy (e.g. a 4.2 LTS install). ' +
295        'bpy_version: which bundled bpy to use (run /blender-setup <version> first, e.g. 4.2). ' +
296        'preview_tris: triangles per live frame (default 12000). pace, mode: live frame defaults. ' +
297        'realtime/bake/sky: the realistic feed view (default on). bake_size: baked texture size (default 1024). ' +
298        'render: "cycles" to render every step, samples, render_size [W, H].',
299      isDeferred: false,
300      inputSchema: {
301        type: 'object',
302        properties: {
303          settings: { type: 'object', description: 'Settings to change, e.g. {"paths": ["/repo/kit"], "mode": "objects"}' },
304        },
305        required: ['settings'],
306      },
307    })
308    await $.tool.register({
309      name: 'blender_undo',
310      description: 'Undo the last Blender Live step: the scene goes back to how it was before it.',
311      isDeferred: false,
312    })
313    await $.tool.register({
314      name: 'blender_reset',
315      description: 'Start a new, empty Blender Live scene (clears every step). Call this before starting a new model.',
316      isDeferred: false,
317    })
318    await $.tool.register({
319      name: 'blender_finish',
320      description:
321        'Finish the Blender Live model: makes a timelapse GIF of all steps, a contact sheet, and exports model.glb. Returns the file paths (scene.blend is there too).',
322      isDeferred: false,
323    })
324
325    // the turntable: turn a few degrees per tick while spinning
326    ticker?.cancel()
327    ticker = $.clock.every(TICK_MS, async () => {
328      await playNext($)
329      const now = await $.clock.now()
330      if (now < (await read($, spinUntil))) {
331        await update($, angle, a => (a + SPIN_DEG) % 360)
332      }
333    })
334
335    return started
336  })
337
338  on('command.run', { command: 'blender' }, async $ => {
339    await update($, isBandHidden, () => false)
340    await spinFor($, SPIN_AFTER_STEP_MS)
341    return { text: 'Blender Live is showing above the prompt.' }
342  })
343
344  on('command.run', { command: 'blender-pane' }, async $ => {
345    await openPane($)
346    await spinFor($, SPIN_AFTER_STEP_MS)
347    return { text: 'Blender Live pane opened.' }
348  })
349
350  on('command.run', { command: 'blender-setup' }, async ($, e) => {
351    const version = e.args.trim()
352    $.ui.toast(`Installing Blender${version ? ' ' + version : ''}... this takes about a minute`)
353    const out = await run($, version ? ['setup', version] : ['setup'], 600_000)
354    const ok = out.stdout.includes('BSTEP_READY')
355    return { text: ok ? 'Blender is installed and ready.' : `Setup failed:\n${out.stdout}\n${out.stderr}` }
356  })
357
358  on('tool.call', { tool: 'mcp__blender-live__blender_step' }, async ($, e, next) => {
359    const label = String(e.label ?? 'Step').slice(0, 60)
360    const code = String(e.code ?? '')
361    await update($, isBandHidden, () => false)
362    await openPane($)
363    await spinFor($, SPIN_AFTER_STEP_MS)
364    await update($, busy, () => label)
365    await update($, note, () => null)
366    try {
367      // stream Blender's output: every BSTEP_SNAP line is a frame to play now
368      const lines: string[] = []
369      const stepFrames: BlenderFrame[] = []
370      let lastPath = (await read($, queue)).at(-1)?.path ?? (await read($, frame))?.path ?? null
371      let pending = ''
372      const env: Record<string, string> = { BSTEP_DIR: workDir, BSTEP_STREAM: '1' }
373      if (e.focus) env.BSTEP_FOCUS = String(e.focus)
374      if (e.mode) env.BSTEP_MODE = String(e.mode)
375      if (e.pace !== undefined) env.BSTEP_PACE = String(e.pace)
376      if (e.render) env.BSTEP_RENDER = String(e.render)
377      const feedFrames: string[] = []
378      const child = $.process.spawn({
379        argv: ['sh', `${$.plugin.root}/engine/bstep`, 'step', label, '-c', code],
380        env,
381      })
382      for await (const chunk of child) {
383        pending += chunk.text
384        const parts = pending.split('\n')
385        pending = parts.pop() ?? ''
386        for (const line of parts) {
387          if (!line.startsWith('BSTEP_')) continue
388          lines.push(line)
389          const snap = /^BSTEP_SNAP (.+?)\t(\d+)\t([^\t]*)(?:\t(.*))?$/.exec(line)
390          if (snap) {
391            if (snap[4]) feedFrames.push(snap[4])
392            const f: BlenderFrame = { path: snap[1]!, prevPath: lastPath, line: Number(snap[2]), code: snap[3]!, label }
393            lastPath = f.path
394            stepFrames.push(f)
395            await update($, queue, q => [...q, f])
396          }
397        }
398      }
399      if (pending.startsWith('BSTEP_')) lines.push(pending)
400      const exit = await child.result
401      const stdout = lines.join('\n')
402      if (exit.code === 3 || stdout.includes('BSTEP_NEEDS_SETUP')) {
403        return textResult('Blender is not installed yet. Ask the user to run /blender-setup.', true)
404      }
405      const ok = /BSTEP_OK step=(\d+) tris=(\d+) png=(.+)/.exec(stdout)
406      if (!ok) {
407        // the scene was not saved: drop this step's frames from the stream
408        await update($, queue, q => q.filter(f => !stepFrames.includes(f)))
409        await update($, note, () => `Step "${label}" failed, fixing it...`)
410        const tb = lines.filter(l => l.startsWith('BSTEP_TB ')).map(l => l.slice(9)).join('\n')
411        return textResult(`The step failed and the scene was not changed.\n${tb || stdout}`.trim(), true)
412      }
413      const n = Number(ok[1])
414      const png = ok[3]!.trim()
415      const objects = (/BSTEP_OBJECTS (.*)/.exec(stdout)?.[1] ?? '').split(', ').filter(Boolean)
416      const before = (await read($, steps)).filter(s => s.n < n)
417      const known = new Set(before.at(-1)?.objects ?? [])
418      const step: BlenderStep = {
419        n,
420        label,
421        tris: Number(ok[2]),
422        png,
423        json: png.replace(/\.png$/, '.json'),
424        objects,
425        added: objects.filter(o => !known.has(o)),
426        frames: stepFrames,
427      }
428      models.delete(step.json)
429      await update($, steps, () => [...before, step])
430      await update($, viewing, () => null)
431      const now = await $.clock.now()
432      await update($, revealStart, () => now)
433      await update($, flashUntil, () => now + GLOW_MS)
434      await spinFor($, SPIN_AFTER_STEP_MS)
435
436
437      const summary =
438        `Step ${n} "${label}" done: ${step.tris} triangles in the scene.\n` +
439        `Objects: ${objects.join(', ') || '(none)'}\n` +
440        (step.added.length ? `New this step: ${step.added.join(', ')}` : 'No new objects (existing ones changed).') +
441        (await feedInstructions($, n, stdout, feedFrames))
442      return textResult(summary)
443    } finally {
444      await update($, busy, () => null)
445    }
446  })
447
448  on('tool.call', { tool: 'mcp__blender-live__blender_configure' }, async ($, e) => {
449    const settings = (e.settings ?? {}) as Record<string, unknown>
450    const pairs = Object.entries(settings).map(([k, v]) => `${k}=${JSON.stringify(v)}`)
451    if (!pairs.length) return textResult('Nothing to change: pass settings, e.g. {"mode": "objects"}.', true)
452    const out = await run($, ['configure', ...pairs])
453    if (out.exitCode !== 0) return textResult(`Could not save the settings.\n${out.stdout}${out.stderr}`, true)
454    return textResult(`Saved. Settings now: ${out.stdout.replace(/^BSTEP_CONFIG /, '').trim()}`)
455  })
456
457  on('tool.call', { tool: 'mcp__blender-live__blender_undo' }, async $ => {
458    const out = await run($, ['undo'])
459    if (out.exitCode !== 0) return textResult(`Nothing to undo.\n${out.stdout}`.trim(), true)
460    const list = await read($, steps)
461    const gone = list.at(-1)
462    await update($, steps, l => l.slice(0, -1))
463    await update($, queue, () => [])
464    await update($, frame, () => null)
465    await update($, viewing, () => null)
466    await update($, note, () => (gone ? `Undid step ${gone.n}: ${gone.label}` : null))
467    return textResult(gone ? `Undid step ${gone.n} "${gone.label}".` : 'Undone.')
468  })
469
470  on('tool.call', { tool: 'mcp__blender-live__blender_reset' }, async $ => {
471    const out = await run($, ['reset'])
472    if (out.exitCode === 3) return textResult('Blender is not installed yet. Ask the user to run /blender-setup.', true)
473    models.clear()
474    await update($, queue, () => [])
475    await update($, frame, () => null)
476    await update($, steps, () => [])
477    await update($, viewing, () => null)
478    await update($, note, () => null)
479    await update($, isBandHidden, () => false)
480    return textResult(`New empty scene ready in ${workDir}.`)
481  })
482
483  on('tool.call', { tool: 'mcp__blender-live__blender_finish' }, async $ => {
484    const movie = await run($, ['movie'])
485    const glb = await run($, ['export'])
486    const files = [
487      /gif=(\S+)/.exec(movie.stdout)?.[1],
488      /sheet=(\S+)/.exec(movie.stdout)?.[1],
489      /glb=(\S+)/.exec(glb.stdout)?.[1],
490      /blend=(\S+)/.exec(glb.stdout)?.[1],
491    ].filter(Boolean)
492    await update($, note, () => `Finished! Files are in ${workDir}`)
493    await spinFor($, 120_000)
494    return textResult(
495      files.length
496        ? `Finished. Files:\n${files.join('\n')}\nSend the GIF and .blend to the user.`
497        : `Could not finish.\n${movie.stdout}${movie.stderr}\n${glb.stdout}${glb.stderr}`,
498      files.length === 0,
499    )
500  })
501
502  // keep the chat tidy: one short line per Blender call, the pane is the window
503  on('ui.render', { component: 'ToolUse' }, ($, e, next) => {
504    if (!e.props.tool.startsWith('mcp__blender-live__')) return next(e)
505    const { Text } = $.ui.resolve(e)
506    const p = e.props
507    const line = p.tool.endsWith('blender_step')
508      ? compactLine(p.input, p.output, p.isRunning, p.isErrored)
509      : `Blender: ${p.tool.replace('mcp__blender-live__blender_', '')}${p.isRunning ? '...' : ' done'}`
510    return <Text color={p.isErrored ? 'red' : undefined} dimColor={!p.isRunning && !p.isErrored}>{line}</Text>
511  })
512
513  on('ui.render', { component: 'ToolResult' }, async ($, e, next) => {
514    if (!e.props.tool.startsWith('mcp__blender-live__') || e.props.isErrored) return next(e)
515    const { Box } = $.ui.resolve(e)
516    return <Box />
517  })
518
519  // the live view, pinned above the prompt so it stays on screen while the chat scrolls
520  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
521    if (e.props.hasSurvey || (await read($, isBandHidden))) return next(e)
522    const v = await liveView($)
523    if (!v.shown && !v.working) return next(e)
524
525    if (e.surface === 'terminal') {
526      const { Box, Text, Button, Image } = $.ui.resolve(e)
527      const cols = Math.max(20, Math.min(56, (e.props.bodyColumns ?? 60) - 2))
528      return (
529        <Box flexDirection="column">
530          {v.shown && (
531            <Image key="band-shot" source={{ file: v.shown.png, format: 'png' }} columns={cols}
532              rows={Math.round(cols * 0.375)} alt={v.status} />
533          )}
534          <Box flexDirection="row" gap={1}>
535            <Text bold color={v.working ? 'yellow' : 'red'}>{v.working ? '● building' : '● LIVE'}</Text>
536            <Text>{v.status}</Text>
537            <Button key="hide" label="Hide" onPress={() => act($, 'hide')} />
538          </Box>
539        </Box>
540      )
541    }
542
543    const { Box, Text, Button, Svg } = $.ui.resolve(e)
544    return (
545      <Box flexDirection="column">
546        <Svg key="band-view" source={v.svg} alt={v.status} height={250} isInteractive />
547        <Box flexDirection="row" flexWrap="wrap" gap={1} alignItems="center">
548          <Text bold color={v.working ? 'yellow' : undefined}>{v.status}</Text>
549          <Button key="replay" label="Replay" onPress={() => act($, 'replay')} />
550          <Button key="spin" label={v.isSpinning ? 'Stop spin' : 'Spin'} onPress={() => act($, 'spin')} />
551          <Button key="wire" label={v.isWireOn ? 'Solid' : 'Wireframe'} onPress={() => act($, 'wire')} />
552          <Button key="pane" label="Bigger" onPress={() => act($, 'pane')} />
553          <Button key="hide" label="Hide" onPress={() => act($, 'hide')} />
554        </Box>
555      </Box>
556    )
557  })
558
559  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
560    const { list, shown, working, isSpinning, svg, status, message } = await liveView($)
561    const wire = await read($, isWire)
562
563    const controls = (Box: any, Button: any) => (
564      <Box flexDirection="row" flexWrap="wrap" gap={1}>
565        <Button key="prev" label="< Prev" onPress={() => act($, 'prev')} />
566        <Button key="next" label="Next >" onPress={() => act($, 'next')} />
567        <Button key="latest" label="Latest" onPress={() => act($, 'latest')} />
568        <Button key="replay" label="Replay build" variant="primary" onPress={() => act($, 'replay')} />
569        <Button key="spin" label={isSpinning ? 'Stop spin' : 'Spin'} onPress={() => act($, 'spin')} />
570        <Button key="wire" label={wire ? 'Solid' : 'Wireframe'} onPress={() => act($, 'wire')} />
571      </Box>
572    )
573    const timeline = (Box: any, Button: any, Text: any) => (
574      <Box flexDirection="row" flexWrap="wrap" gap={1}>
575        {list.map(s => (
576          <Button key={`s${s.n}`} plain onPress={() => act($, `s${s.n}`)}>
577            <Text bold={shown?.n === s.n} color={shown?.n === s.n ? 'yellow' : undefined}>
578              {s.n}. {s.label}
579            </Text>
580            <Text dimColor> {s.tris}</Text>
581          </Button>
582        ))}
583      </Box>
584    )
585
586    if (e.surface === 'terminal') {
587      const { Box, Text, Button, Image } = $.ui.resolve(e)
588      const cols = Math.max(20, Math.min(80, (e.props.bodyColumns ?? 60) - 2))
589      return (
590        <Box flexDirection="column" gap={1}>
591          <Text bold color={working ? 'yellow' : undefined}>{status}</Text>
592          {shown && (
593            <Image
594              key="shot"
595              source={{ file: shown.png, format: 'png' }}
596              columns={cols}
597              rows={Math.round(cols * 0.375)}
598              alt={`Render of step ${shown.n}: ${shown.label}`}
599            />
600          )}
601          {message && <Text color="cyan">{message}</Text>}
602          {controls(Box, Button)}
603          {timeline(Box, Button, Text)}
604        </Box>
605      )
606    }
607
608    const { Box, Text, Button, Svg } = $.ui.resolve(e)
609    return (
610      <Box flexDirection="column" gap={1}>
611        <Text bold color={working ? 'yellow' : undefined}>{status}</Text>
612        <Svg key="view" source={svg} alt={status} isInteractive />
613        {message && <Text color="cyan">{message}</Text>}
614        {controls(Box, Button)}
615        {timeline(Box, Button, Text)}
616      </Box>
617    )
618  })
619}
620

1// A tiny software 3D renderer: turns a step's triangles into one SVG,
2// flat-shaded like an N64 viewport, seen from an orbiting camera.
3
4export type ModelObject = { name: string; color: number[]; tris: number[]; open?: boolean }
5export type Model = { step: number; label: string; tris: number; objects: ModelObject[] }
6
7export type ViewOptions = {
8  /** Turntable angle, degrees. */
9  angle: number
10  isWire: boolean
11  /** Object names drawn with a glowing outline. */
12  glow: readonly string[]
13  /** Caption drawn in the corner. */
14  caption: string
15  /**
16   * Animates from the previous frame: parts that only moved or changed shape
17   * slide there (`t` 0..1), new triangles build up from the bottom and glow.
18   */
19  transition?: { prev: Model | null; t: number }
20  /** The line of code that made this frame, shown like a live terminal. */
21  codeLine?: string
22  /** The stream badge: a blinking LIVE dot, and what Blender is busy with. */
23  live?: { building: string | null }
24}
25
26const W = 640
27const H = 440
28const SVG_LIMIT = 125_000
29// the sign that marks a back face in camera space (its axes are mirrored)
30const FACING = 1
31const PITCH = (24 * Math.PI) / 180
32
33type Face = { d: string; depth: number; fill: string; isGlow: boolean; area: number }
34
35function toHex(linear: number, light: number): string {
36  // Blender colours are linear; screens want sRGB
37  const v = Math.round(Math.min(1, Math.pow(Math.max(0, linear), 1 / 2.2) * light) * 255)
38  return v.toString(16).padStart(2, '0')
39}
40
41type V3 = [number, number, number]
42
43function bounds(model: Model) {
44  let x0 = Infinity, y0 = Infinity, z0 = Infinity
45  let x1 = -Infinity, y1 = -Infinity, z1 = -Infinity
46  for (const o of model.objects) {
47    const t = o.tris
48    for (let i = 0; i + 2 < t.length; i += 3) {
49      const x = t[i]!, y = t[i + 1]!, z = t[i + 2]!
50      x0 = Math.min(x0, x); y0 = Math.min(y0, y); z0 = Math.min(z0, z)
51      x1 = Math.max(x1, x); y1 = Math.max(y1, y); z1 = Math.max(z1, z)
52    }
53  }
54  if (x0 === Infinity) return { center: [0, 0, 0.5] as V3, radius: 1, floor: 0 }
55  const center: V3 = [(x0 + x1) / 2, (y0 + y1) / 2, (z0 + z1) / 2]
56  const radius = Math.max(0.5, Math.hypot(x1 - x0, y1 - y0, z1 - z0) / 2)
57  return { center, radius, floor: Math.min(0, z0) }
58}
59
60export function renderSvg(model: Model | null, opt: ViewOptions): string {
61  const { center, radius, floor } = model ? bounds(model) : { center: [0, 0, 0.5] as V3, radius: 1, floor: 0 }
62  const yaw = (opt.angle * Math.PI) / 180
63  const cy = Math.cos(yaw)
64  const sy = Math.sin(yaw)
65  const cp = Math.cos(PITCH)
66  const sp = Math.sin(PITCH)
67  const dist = radius * 3.6
68  const focal = (H * 0.42 * dist) / radius
69
70  // world -> camera space: [right, up, depth]
71  const view = (x: number, y: number, z: number): V3 => {
72    const dx = x - center[0]
73    const dy = y - center[1]
74    const dz = z - center[2]
75    const rx = dx * cy - dy * sy
76    const ry = dx * sy + dy * cy
77    return [rx, dz * cp + ry * sp, ry * cp - dz * sp]
78  }
79  const project = (v: V3): [number, number] => {
80    const s = focal / (dist + v[2])
81    return [W / 2 + v[0] * s, H / 2 + 20 - v[1] * s]
82  }
83
84  // match each object to the previous frame: same shape -> slide, else new triangles build up
85  const t = Math.max(0, Math.min(1, opt.transition?.t ?? 1))
86  const ease = t * t * (3 - 2 * t)
87  const prevEntries = new Map<string, number[]>()
88  const prevKeys = new Set<string>()
89  const prevNames = new Set<string>()
90  const key3 = (a: number[], i: number) =>
91    `${a[i]!.toFixed(3)},${a[i + 1]!.toFixed(3)},${a[i + 2]!.toFixed(3)}`
92  if (opt.transition?.prev && t < 1) {
93    const seen = new Map<string, number>()
94    for (const o of opt.transition.prev.objects) {
95      const k = seen.get(o.name) ?? 0
96      seen.set(o.name, k + 1)
97      prevEntries.set(`${o.name}#${k}`, o.tris)
98      prevNames.add(o.name)
99      for (let i = 0; i + 8 < o.tris.length; i += 9) {
100        prevKeys.add(`${o.name}|${key3(o.tris, i)}|${key3(o.tris, i + 3)}|${key3(o.tris, i + 6)}`)
101      }
102    }
103  }
104  const animating = prevNames.size > 0 || (opt.transition?.prev === null && t < 1)
105  // new triangles appear lowest first; collect their heights to find the cut
106  const isNewTri: boolean[][] = []
107  const lows: number[] = []
108  const seenCur = new Map<string, number>()
109  const slideFrom: (number[] | undefined)[] = []
110  for (const o of model?.objects ?? []) {
111    const k = seenCur.get(o.name) ?? 0
112    seenCur.set(o.name, k + 1)
113    const before = prevEntries.get(`${o.name}#${k}`)
114    const slides = animating && before !== undefined && before.length === o.tris.length
115    slideFrom.push(slides ? before : undefined)
116    const flags: boolean[] = []
117    for (let i = 0; i + 8 < o.tris.length; i += 9) {
118      const isNew =
119        animating && !slides &&
120        !prevKeys.has(`${o.name}|${key3(o.tris, i)}|${key3(o.tris, i + 3)}|${key3(o.tris, i + 6)}`)
121      flags.push(isNew)
122      if (isNew) lows.push(Math.min(o.tris[i + 2]!, o.tris[i + 5]!, o.tris[i + 8]!))
123    }
124    isNewTri.push(flags)
125  }
126  lows.sort((p, q) => p - q)
127  const shownNew = Math.floor(lows.length * ease)
128  const cutZ = shownNew === 0 ? -Infinity : lows[shownNew - 1]!
129  const edgeZ = lows[Math.max(0, Math.floor(shownNew * 0.8) - 1)] ?? -Infinity
130
131  const faces: Face[] = []
132  const glow = new Set(opt.glow)
133  let total = 0
134  let objIndex = -1
135  for (const o of model?.objects ?? []) {
136    objIndex++
137    const isGlow = glow.has(o.name)
138    const flags = isNewTri[objIndex] ?? []
139    const from = slideFrom[objIndex]
140    const t = from
141      ? o.tris.map((v, i) => from[i]! + (v - from[i]!) * ease)
142      : o.tris
143    const [cr = 0.7, cg = 0.7, cb = 0.7] = o.color
144    for (let i = 0; i + 8 < t.length; i += 9) {
145      total++
146      let isEdge = false
147      if (flags[i / 9]) {
148        const low = Math.min(t[i + 2]!, t[i + 5]!, t[i + 8]!)
149        if (low > cutZ) continue
150        isEdge = low >= edgeZ || ease < 1
151      }
152      const a = view(t[i]!, t[i + 1]!, t[i + 2]!)
153      const b = view(t[i + 3]!, t[i + 4]!, t[i + 5]!)
154      const c = view(t[i + 6]!, t[i + 7]!, t[i + 8]!)
155      // flat shading from the face normal; two-sided so open meshes read well
156      const ux = b[0] - a[0], uy = b[1] - a[1], uz = b[2] - a[2]
157      const vx = c[0] - a[0], vy = c[1] - a[1], vz = c[2] - a[2]
158      const nx = uy * vz - uz * vy, ny = uz * vx - ux * vz, nz = ux * vy - uy * vx
159      // closed meshes: skip faces turned away from the camera (no overlap glitches)
160      const toCam = -a[0] * nx - a[1] * ny + (-dist - a[2]) * nz
161      if (!o.open && toCam * FACING > 0) continue
162      const len = Math.hypot(nx, ny, nz) || 1
163      const facing = Math.abs((nx * -0.45 + ny * 0.75 + nz * -0.5) / len)
164      const light = 0.38 + 0.72 * facing
165      const fill = '#' + toHex(cr, light) + toHex(cg, light) + toHex(cb, light)
166      const pa = project(a), pb = project(b), pc = project(c)
167      const area = Math.abs((pb[0] - pa[0]) * (pc[1] - pa[1]) - (pc[0] - pa[0]) * (pb[1] - pa[1]))
168      const r = (n: number) => Math.round(n * 10) / 10
169      faces.push({
170        d: `M${r(pa[0])} ${r(pa[1])}L${r(pb[0])} ${r(pb[1])}L${r(pc[0])} ${r(pc[1])}Z`,
171        depth: (a[2] + b[2] + c[2]) / 3,
172        fill: isEdge ? '#fff3b0' : fill,
173        isGlow: isGlow || isEdge,
174        area,
175      })
176    }
177  }
178
179  // ground grid and a soft shadow
180  const grid: string[] = []
181  const half = radius * 1.6
182  const steps = 8
183  for (let i = 0; i <= steps; i++) {
184    const t = -half + (2 * half * i) / steps
185    const lines: [number, number, number, number][] = [
186      [t, -half, t, half],
187      [-half, t, half, t],
188    ]
189    for (const [px, py, qx, qy] of lines) {
190      const s = project(view(center[0] + px, center[1] + py, floor))
191      const e = project(view(center[0] + qx, center[1] + qy, floor))
192      grid.push(`M${s[0].toFixed(1)} ${s[1].toFixed(1)}L${e[0].toFixed(1)} ${e[1].toFixed(1)}`)
193    }
194  }
195  const shadow = project(view(center[0], center[1], floor))
196  const shadowR = (focal / dist) * radius * 0.85
197
198  faces.sort((p, q) => q.depth - p.depth)
199  const stroke = opt.isWire ? 'stroke="#0d0f14" stroke-width=".7"' : ''
200  const head =
201    `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${W} ${H}" width="${W}" height="${H}">` +
202    `<defs><radialGradient id="bg" cx="50%" cy="35%" r="75%"><stop offset="0" stop-color="#3a4150"/>` +
203    `<stop offset="1" stop-color="#14161c"/></radialGradient>` +
204    `<radialGradient id="sh"><stop offset="0" stop-color="#000" stop-opacity=".45"/>` +
205    `<stop offset="1" stop-color="#000" stop-opacity="0"/></radialGradient>` +
206    `<style>.g{stroke:#ffd84a;stroke-width:1.6;stroke-linejoin:round;animation:p 1s ease-in-out infinite alternate}` +
207    `@keyframes p{from{stroke-opacity:1}to{stroke-opacity:.25}}` +
208    `.blink{animation:b 1s steps(2,start) infinite}@keyframes b{to{visibility:hidden}}</style></defs>` +
209    `<rect width="${W}" height="${H}" fill="url(#bg)"/>` +
210    `<path d="${grid.join('')}" stroke="#ffffff" stroke-opacity=".08" fill="none"/>` +
211    `<ellipse cx="${shadow[0].toFixed(1)}" cy="${shadow[1].toFixed(1)}" rx="${shadowR.toFixed(1)}" ` +
212    `ry="${(shadowR * sp).toFixed(1)}" fill="url(#sh)"/>`
213  const badge = opt.live
214    ? `<circle class="blink" cx="22" cy="22" r="6" fill="#ff3b4a"/>` +
215      `<text x="34" y="27" font-family="monospace" font-size="14" font-weight="bold" fill="#ff6b77">LIVE</text>` +
216      (opt.live.building
217        ? `<text class="blink" x="${W - 14}" y="27" text-anchor="end" font-family="monospace" font-size="14" ` +
218          `fill="#ffd84a">building: ${escapeXml(opt.live.building.slice(0, 40))}</text>`
219        : '')
220    : ''
221  const code = opt.codeLine
222    ? `<rect x="0" y="${H - 58}" width="${W}" height="22" fill="#000" fill-opacity=".45"/>` +
223      `<text x="14" y="${H - 42}" font-family="monospace" font-size="13" fill="#7dff9b">&#8250; ` +
224      `${escapeXml(opt.codeLine.slice(0, 80))}</text>`
225    : ''
226  const foot =
227    code +
228    badge +
229    `<text x="14" y="${H - 14}" font-family="monospace" font-size="14" fill="#cfd6e4">` +
230    `${escapeXml(opt.caption)}</text>` +
231    `<text x="${W - 14}" y="${H - 14}" text-anchor="end" font-family="monospace" font-size="14" ` +
232    `fill="#8fc1ff">${total} tris</text></svg>`
233
234  // keep inside the SVG size limit: drop the smallest faces first
235  const tag = (f: Face) =>
236    `<path d="${f.d}" fill="${f.fill}" ` +
237    (f.isGlow ? 'class="g"' : opt.isWire ? stroke : `stroke="${f.fill}" stroke-width=".5"`) +
238    '/>'
239  const budget = SVG_LIMIT - head.length - foot.length
240  const tags = faces.map(tag)
241  let size = tags.reduce((n, t) => n + t.length, 0)
242  const keep = faces.map(() => true)
243  if (size > budget) {
244    const order = faces.map((_, i) => i).sort((p, q) => faces[p]!.area - faces[q]!.area)
245    for (const i of order) {
246      if (size <= budget) break
247      keep[i] = false
248      size -= tags[i]!.length
249    }
250  }
251  const body = tags.filter((_, i) => keep[i]).join('')
252  return head + body + foot
253}
254
255function escapeXml(s: string): string {
256  return s.replace(/[<>&"]/g, c => ({ '<': '&lt;', '>': '&gt;', '&': '&amp;', '"': '&quot;' })[c] ?? c)
257}
258

1/** One snapshot of the scene, taken after a line of Claude's code ran. */
2export type BlenderFrame = {
3  path: string
4  /** The frame before it, to animate from; null for the first. */
5  prevPath: string | null
6  line: number
7  code: string
8  label: string
9}
10
11/** One finished modeling step, as the pane lists it. */
12export type BlenderStep = {
13  /** 1-based step number. */
14  n: number
15  label: string
16  tris: number
17  /** Absolute paths of the step's picture and triangle data. */
18  png: string
19  json: string
20  /** Every object in the scene after this step. */
21  objects: string[]
22  /** Objects that first appeared in this step (highlighted when shown). */
23  added: string[]
24  /** The live snapshots taken while this step ran, in order. */
25  frames: BlenderFrame[]
26}
27
28declare module 'claude-code' {
29  interface PluginState {
30    'blender-live': {
31      steps: BlenderStep[]
32      /** The step on screen; null follows the newest one. */
33      viewing: number | null
34      /** Turntable angle in degrees. */
35      angle: number
36      /** The turntable spins until this time (ms since epoch). */
37      spinUntil: number
38      isWire: boolean
39      /** The label of the step Blender is working on right now. */
40      busy: string | null
41      /** New parts glow until this time (ms since epoch). */
42      flashUntil: number
43      /** The streamed snapshot on screen, and when it started animating in. */
44      frame: BlenderFrame | null
45      frameStart: number
46      /** Snapshots waiting to be played. */
47      queue: BlenderFrame[]
48      /** New parts build up, layer by layer, from this time (ms since epoch). */
49      revealStart: number
50      /** The person hid the live view above the prompt. */
51      isBandHidden: boolean
52      /** A message under the viewer (errors, finished files). */
53      note: string | null
54    }
55  }
56}
57