meshy-tray

1import { atom, read, update } from 'claude-code'
2import type { Register, Timer } from 'claude-code'
3
4import type { MeshyKind, MeshyTask } from '../types'
5import {
6  BASE_URL,
7  ENDPOINTS,
8  KIND_LABELS,
9  MeshyError,
10  describeImage,
11  formatElapsed,
12  imageMime,
13  isActive,
14  isRemoteImage,
15  keyFromDotenv,
16  parseResponse,
17  progressBar,
18  shortId,
19  statusLabel,
20} from './meshy'
21import type { MeshyTaskObject } from './meshy'
22
23const PANE = 'meshy-tasks'
24const TOOL_NAME = 'meshy_generate'
25const TOOL = 'mcp__meshy-tray__meshy_generate'
26const TICK_MS = 1000
27const POLL_EVERY_TICKS = 5
28const TOAST_MS = 8000
29const MAX_TASKS = 100
30
31const tasks = atom({ plugin: 'meshy-tray', key: 'tasks' } as const, [])
32
33const MODES: Record<string, MeshyKind> = {
34  'image-to-3d': 'image',
35  'text-to-3d': 'text',
36  'multi-image-to-3d': 'multi',
37}
38
39const INPUT_SCHEMA = {
40  type: 'object',
41  properties: {
42    mode: {
43      type: 'string',
44      enum: Object.keys(MODES),
45      description: 'Which Meshy generation to start.',
46    },
47    prompt: {
48      type: 'string',
49      description: 'text-to-3d: what to make (max 800 characters).',
50    },
51    image: {
52      type: 'string',
53      description:
54        'image-to-3d: a local .png/.jpg/.jpeg path (relative to the working directory or absolute, up to 4 MiB), an https URL, or a data URI.',
55    },
56    images: {
57      type: 'array',
58      items: { type: 'string' },
59      minItems: 1,
60      maxItems: 4,
61      description:
62        'multi-image-to-3d: 1 to 4 views of the same object, each a local path, URL or data URI; the first is the front view.',
63    },
64    ai_model: {
65      type: 'string',
66      description: 'meshy-6-lite, meshy-6, meshy-7.1 or latest (the default).',
67    },
68    texture: {
69      type: 'boolean',
70      description:
71        'Default true. image and multi-image: should_texture. text-to-3d: run the refine (texturing) stage after the untextured preview.',
72    },
73    texture_prompt: { type: 'string', description: 'Guides texturing (max 800 characters).' },
74    enable_pbr: { type: 'boolean', description: 'Also generate metallic, roughness and normal maps.' },
75    should_remesh: { type: 'boolean' },
76    topology: { type: 'string', enum: ['triangle', 'quad'] },
77    target_polycount: { type: 'integer', minimum: 100, maximum: 300000 },
78    pose_mode: { type: 'string', enum: ['', 'a-pose', 't-pose'] },
79    options: {
80      type: 'object',
81      description:
82        'Any other Meshy create-request fields (e.g. geometry_resolution, texture_resolution, auto_size), sent as given.',
83    },
84  },
85  required: ['mode'],
86}
87
88const DESCRIPTION = [
89  'Start a Meshy 3D generation: image-to-3d (one image), text-to-3d (a prompt) or multi-image-to-3d (1-4 views).',
90  'Returns the task id at once; the task runs in the background, shown in the band above the prompt.',
91  'When it succeeds the GLB is saved to ./meshy-models/<task-id>.glb and the user gets a toast; do not poll or wait for it.',
92  'The user can list tasks with /meshy-tasks. Each generation spends Meshy credits.',
93].join(' ')
94
95type Args = Record<string, unknown>
96
97type Plan = {
98  kind: MeshyKind
99  body: Record<string, unknown>
100  source: string
101  images: string[]
102  imageField?: 'image_url' | 'image_urls'
103  shouldRefine?: boolean
104  refineOptions?: Record<string, string | number | boolean>
105}
106
107const str = (value: unknown) => (typeof value === 'string' && value.trim() ? value.trim() : undefined)
108
109/** The optional create fields the tool's schema names, when given. */
110const commonFields = (args: Args) => {
111  const fields: Record<string, unknown> = {}
112  for (const name of ['ai_model', 'should_remesh', 'topology', 'target_polycount', 'pose_mode']) {
113    if (args[name] !== undefined) fields[name] = args[name]
114  }
115  const options = args.options
116  if (options && typeof options === 'object' && !Array.isArray(options)) {
117    Object.assign(fields, options)
118  }
119
120  return fields
121}
122
123const textureFields = (args: Args) => {
124  const fields: Record<string, string | number | boolean> = {}
125  if (typeof args.enable_pbr === 'boolean') fields.enable_pbr = args.enable_pbr
126  const prompt = str(args.texture_prompt)
127  if (prompt) fields.texture_prompt = prompt
128
129  return fields
130}
131
132/** The request a tool call asks for, its images still as given. */
133const plan = (args: Args): Plan => {
134  const kind = MODES[String(args.mode)]
135  if (!kind) throw new Error(`mode must be one of ${Object.keys(MODES).join(', ')}`)
136
137  if (kind === 'text') {
138    const prompt = str(args.prompt)
139    if (!prompt) throw new Error('text-to-3d needs a prompt')
140
141    return {
142      kind,
143      source: prompt,
144      images: [],
145      body: { ...commonFields(args), mode: 'preview', prompt },
146      shouldRefine: args.texture !== false,
147      refineOptions: textureFields(args),
148    }
149  }
150
151  const body = {
152    ...commonFields(args),
153    ...textureFields(args),
154    ...(typeof args.texture === 'boolean' ? { should_texture: args.texture } : {}),
155  }
156
157  if (kind === 'image') {
158    const image = str(args.image)
159    if (!image) throw new Error('image-to-3d needs an image (a local path or a URL)')
160
161    return { kind, source: describeImage(image), images: [image], imageField: 'image_url', body }
162  }
163
164  const images = Array.isArray(args.images) ? (args.images.map(str).filter(Boolean) as string[]) : []
165  if (images.length < 1 || images.length > 4) {
166    throw new Error('multi-image-to-3d needs 1 to 4 images')
167  }
168
169  return { kind, source: images.map(describeImage).join(', '), images, imageField: 'image_urls', body }
170}
171
172const errorText = (error: unknown) => (error instanceof Error ? error.message : String(error))
173
174export const register: Register = (on, options) => {
175  // Built in session.start over the session's own `$`, which outlives any one
176  // dispatch; the tool call hands its work to it.
177  let generate: ((args: Args) => Promise<string>) | undefined
178  let timer: Timer | undefined
179  let ticks = 0
180  let isPolling = false
181  const downloading = new Set<string>()
182
183  on('session.start', async ($, e, next) => {
184    const loadKey = async () => {
185      // The plugin's own option (kept in secure storage), else the environment, else the project's .env.
186      const fromOptions = typeof options.meshy_api_key === 'string' ? options.meshy_api_key.trim() : ''
187      if (fromOptions) return fromOptions
188      const fromEnv = (await $.env.get('MESHY_API_KEY'))?.trim()
189      if (fromEnv) return fromEnv
190      for (const dir of new Set([await $.session.cwd(), await $.session.root()])) {
191        const path = `${dir}/.env`
192        if (!(await $.fs.exists(path))) continue
193        const key = keyFromDotenv(await $.fs.read(path))
194        if (key) return key
195      }
196
197      return undefined
198    }
199
200    const request = async (key: string, method: 'GET' | 'POST', path: string, body?: unknown) => {
201      const response = await $.http.fetch(`${BASE_URL}${path}`, {
202        method,
203        headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
204        body: body === undefined ? undefined : JSON.stringify(body),
205      })
206
207      return parseResponse(response, key)
208    }
209
210    const toImageUrl = async (image: string) => {
211      if (isRemoteImage(image)) return image
212      const mime = imageMime(image)
213      const path = image.startsWith('/') ? image : `${await $.session.cwd()}/${image}`
214      if (!(await $.fs.exists(path))) throw new Error(`${image}: no such file`)
215      try {
216        const { base64 } = await $.fs.read(path, { as: 'bytes' })
217
218        return `data:${mime};base64,${base64}`
219      } catch {
220        throw new Error(`${image}: could not be read (files over 4 MiB cannot be sent; pass a URL instead)`)
221      }
222    }
223
224    const patch = (id: string, change: Partial<MeshyTask>) =>
225      update($, tasks, list => list.map(task => (task.id === id ? { ...task, ...change } : task)))
226
227    const fail = async (task: MeshyTask, message: string) => {
228      await patch(task.id, { status: 'FAILED', error: message, finishedAt: await $.clock.now() })
229      $.ui.toast(`Meshy ${KIND_LABELS[task.kind]} ${shortId(task.id)} failed: ${message}`, {
230        timeoutMs: TOAST_MS,
231      })
232    }
233
234    const download = async (task: MeshyTask, key: string) => {
235      if (downloading.has(task.id)) return
236      downloading.add(task.id)
237      try {
238        // Asked again here so a download restarted after a reload gets a fresh signed URL.
239        const object = (await request(key, 'GET', `${ENDPOINTS[task.kind]}/${task.apiId}`)) as MeshyTaskObject
240        const url = object.model_urls?.glb
241        if (!url) {
242          await fail(task, 'the task succeeded but returned no GLB')
243          return
244        }
245        const relative = `meshy-models/${task.id}.glb`
246        // The asset URL is pre-signed: the key never reaches curl's argv.
247        const ran = await $.process.run(
248          ['curl', '-fsSL', '--retry', '2', '--create-dirs', '-o', relative, url],
249          { cwd: task.cwd, timeoutMs: 10 * 60 * 1000 },
250        )
251        if (ran.exitCode !== 0) {
252          await fail(task, `download failed: ${ran.stderr.trim() || `curl exited ${ran.exitCode}`}`)
253          return
254        }
255        const file = `./${relative}`
256        await patch(task.id, { status: 'SUCCEEDED', progress: 100, file, finishedAt: await $.clock.now() })
257        $.ui.toast(`Meshy model ready: ${file}`, { timeoutMs: TOAST_MS })
258      } catch (error) {
259        await fail(task, `download failed: ${errorText(error)}`)
260      } finally {
261        downloading.delete(task.id)
262      }
263    }
264
265    const poll = async (task: MeshyTask, key: string) => {
266      let object: MeshyTaskObject
267      try {
268        object = (await request(key, 'GET', `${ENDPOINTS[task.kind]}/${task.apiId}`)) as MeshyTaskObject
269      } catch (error) {
270        // A rate limit, a 5xx or a network blip is retried next round.
271        const isFinal =
272          error instanceof MeshyError && error.status >= 400 && error.status < 500 && error.status !== 429
273        if (isFinal) await fail(task, error.message)
274        return
275      }
276
277      if (object.status === 'PENDING' || object.status === 'IN_PROGRESS') {
278        await patch(task.id, { status: object.status, progress: object.progress ?? 0 })
279        return
280      }
281      if (object.status === 'FAILED' || object.status === 'CANCELED') {
282        const fallback = object.status === 'CANCELED' ? 'canceled' : 'no reason given'
283        await fail(task, object.task_error?.message || fallback)
284        return
285      }
286      if (object.status !== 'SUCCEEDED') return
287
288      if (task.kind === 'text' && task.stage === 'preview' && task.shouldRefine) {
289        try {
290          const created = (await request(key, 'POST', ENDPOINTS.text, {
291            ...task.refineOptions,
292            mode: 'refine',
293            preview_task_id: task.apiId,
294          })) as { result?: string }
295          if (!created?.result) throw new Error('Meshy returned no task id')
296          await patch(task.id, { apiId: created.result, stage: 'refine', status: 'PENDING', progress: 0 })
297        } catch (error) {
298          await fail(task, `refine: ${errorText(error)}`)
299        }
300        return
301      }
302
303      await patch(task.id, { status: 'DOWNLOADING', progress: 100 })
304      await download(task, key)
305    }
306
307    const tick = async () => {
308      const active = (await read($, tasks)).filter(isActive)
309      if (active.length === 0) {
310        timer?.cancel()
311        timer = undefined
312        return
313      }
314      // The elapsed times move every second; the state only every poll.
315      $.ui.invalidate('ui.render')
316      ticks += 1
317      if (isPolling || ticks % POLL_EVERY_TICKS !== 0) return
318      const key = await loadKey()
319      if (!key) return
320      isPolling = true
321      try {
322        await Promise.all(
323          active.map(task => (task.status === 'DOWNLOADING' ? download(task, key) : poll(task, key))),
324        )
325      } finally {
326        isPolling = false
327      }
328    }
329
330    const ensureTicking = () => {
331      if (timer) return
332      ticks = 0
333      timer = $.clock.every(TICK_MS, () => void tick())
334    }
335
336    generate = async args => {
337      const key = await loadKey()
338      if (!key) {
339        throw new Error('MESHY_API_KEY is not set: export it, or add MESHY_API_KEY=... to the project .env file')
340      }
341      const planned = plan(args)
342      const imageUrls: string[] = []
343      for (const image of planned.images) imageUrls.push(await toImageUrl(image))
344      const body =
345        planned.imageField === 'image_url'
346          ? { ...planned.body, image_url: imageUrls[0] }
347          : planned.imageField === 'image_urls'
348            ? { ...planned.body, image_urls: imageUrls }
349            : planned.body
350      const answer = (await request(key, 'POST', ENDPOINTS[planned.kind], body)) as { result?: string }
351      const id = answer?.result
352      if (!id) throw new Error('Meshy returned no task id')
353
354      const task: MeshyTask = {
355        id,
356        kind: planned.kind,
357        apiId: id,
358        status: 'PENDING',
359        progress: 0,
360        source: planned.source,
361        cwd: await $.session.cwd(),
362        startedAt: await $.clock.now(),
363        ...(planned.kind === 'text'
364          ? { stage: 'preview' as const, shouldRefine: planned.shouldRefine, refineOptions: planned.refineOptions }
365          : {}),
366      }
367      await update($, tasks, list => [...list, task].slice(-MAX_TASKS))
368      ensureTicking()
369
370      const stages = planned.kind === 'text' && planned.shouldRefine ? ' (preview, then texturing)' : ''
371
372      return `Started ${String(args.mode)} task ${id}${stages}. It runs in the background and shows in the band above the prompt; when it succeeds the model is saved to ./meshy-models/${id}.glb and the user is notified. Do not poll for it.`
373    }
374
375    await $.tool.register({ name: TOOL_NAME, description: DESCRIPTION, inputSchema: INPUT_SCHEMA })
376    await $.command.register({
377      name: 'meshy-tasks',
378      description: "List this session's Meshy generations in a pane",
379    })
380    // Tasks kept in $.state outlive a reload; pick their polling back up.
381    if ((await read($, tasks)).some(isActive)) ensureTicking()
382
383    return next(e)
384  })
385
386  on('tool.call', { tool: TOOL }, async (_$, e) => {
387    if (!generate) return { deny: 'meshy-tray is still starting; try again in a moment.' }
388    try {
389      return { result: await generate(e as Args) }
390    } catch (error) {
391      return { deny: `meshy_generate: ${errorText(error)}` }
392    }
393  })
394
395  on('command.run', { command: 'meshy-tasks' }, async $ => {
396    await $.ui.open({ id: PANE, title: 'Meshy tasks', closeOnEscape: true })
397
398    return { text: 'Opened the Meshy tasks pane.' }
399  })
400
401  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
402    if (e.props.hasSurvey) return next(e)
403    const active = (await read($, tasks)).filter(isActive)
404    if (active.length === 0) return next(e)
405
406    const { Box, Text } = $.ui.resolve(e)
407    const now = await $.clock.now()
408
409    return (
410      <Box flexDirection="column">
411        {active.map(task => {
412          const { filled, track } = progressBar(task.progress)
413
414          return (
415            <Box key={task.id} flexDirection="row">
416              <Text>
417                {KIND_LABELS[task.kind].padEnd(6)}
418                {shortId(task.id)} {statusLabel(task).padEnd(8)}{' '}
419              </Text>
420              <Text color="cyan">{filled}</Text>
421              <Text dimColor>{track}</Text>
422              <Text>{` ${`${task.progress}%`.padStart(4)} `}</Text>
423              <Text dimColor>{formatElapsed(now - task.startedAt)}</Text>
424            </Box>
425          )
426        })}
427      </Box>
428    )
429  })
430
431  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
432    const { Box, Text } = $.ui.resolve(e)
433    const list = [...(await read($, tasks))].reverse()
434    const now = await $.clock.now()
435
436    if (list.length === 0) {
437      return (
438        <Box flexDirection="column">
439          <Text dimColor>No Meshy tasks in this session yet.</Text>
440          <Text dimColor>Ask Claude to make a 3D model; it calls meshy_generate.</Text>
441        </Box>
442      )
443    }
444
445    const running = list.filter(isActive).length
446
447    return (
448      <Box flexDirection="column">
449        <Text dimColor>
450          {list.length} task{list.length === 1 ? '' : 's'}, {running} running
451        </Text>
452        {list.map(task => {
453          const color =
454            task.status === 'SUCCEEDED' ? 'green' : task.status === 'FAILED' || task.status === 'CANCELED' ? 'red' : 'cyan'
455          const elapsed = formatElapsed((task.finishedAt ?? now) - task.startedAt)
456
457          return (
458            <Box key={task.id} flexDirection="column" marginTop={1}>
459              <Box flexDirection="row">
460                <Text bold>{`${KIND_LABELS[task.kind]} ${shortId(task.id)} `}</Text>
461                <Text color={color}>{statusLabel(task)}</Text>
462                <Text>{isActive(task) ? ` ${task.progress}%` : ''} </Text>
463                <Text dimColor>{elapsed}</Text>
464              </Box>
465              <Text dimColor wrap="truncate-end">
466                {`${task.kind === 'text' ? 'prompt' : 'image'}: ${task.source}`}
467              </Text>
468              {task.file ? <Text wrap="truncate-start">{`file: ${task.file}`}</Text> : null}
469              {task.error ? <Text color="red">{`error: ${task.error}`}</Text> : null}
470              <Text dimColor wrap="truncate">{`id: ${task.id}`}</Text>
471            </Box>
472          )
473        })}
474      </Box>
475    )
476  })
477}
478

1import type { HttpResponse } from 'claude-code'
2
3import type { MeshyKind, MeshyStatus, MeshyTask } from '../types'
4
5export const BASE_URL = 'https://api.meshy.ai'
6
7// Create and retrieve share a path per kind: POST <path>, GET <path>/<id>.
8export const ENDPOINTS: Record<MeshyKind, string> = {
9  image: '/openapi/v1/image-to-3d',
10  multi: '/openapi/v1/multi-image-to-3d',
11  text: '/openapi/v2/text-to-3d',
12}
13
14export const KIND_LABELS: Record<MeshyKind, string> = {
15  image: 'image',
16  multi: 'multi',
17  text: 'text',
18}
19
20export type MeshyTaskObject = {
21  id: string
22  status: MeshyStatus
23  progress?: number
24  model_urls?: { glb?: string }
25  task_error?: { message?: string }
26}
27
28export class MeshyError extends Error {
29  status: number
30
31  constructor(status: number, message: string) {
32    super(message)
33    this.status = status
34  }
35}
36
37/** True while the task still needs polling or downloading. */
38export const isActive = (task: MeshyTask) =>
39  task.status === 'PENDING' ||
40  task.status === 'IN_PROGRESS' ||
41  task.status === 'DOWNLOADING'
42
43/** The last 8 characters: Meshy ids are UUIDv7, so their heads repeat. */
44export const shortId = (id: string) => id.replace(/-/g, '').slice(-8)
45
46const LEVELS = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']
47
48/**
49 * A bar `width` cells wide: full cells, one partial cell from the eighths,
50 * and the empty track, returned apart so the track can be drawn dim.
51 */
52export const progressBar = (percent: number, width = 10) => {
53  const eighths = Math.round((Math.max(0, Math.min(100, percent)) / 100) * width * 8)
54  const full = Math.floor(eighths / 8)
55  const part = eighths % 8
56  const filled = '█'.repeat(full) + (part > 0 ? LEVELS[part - 1] : '')
57
58  return { filled, track: '▁'.repeat(width - filled.length) }
59}
60
61export const formatElapsed = (ms: number) => {
62  const seconds = Math.max(0, Math.floor(ms / 1000))
63  if (seconds < 60) return `${seconds}s`
64  const minutes = Math.floor(seconds / 60)
65  if (minutes < 60) return `${minutes}m${String(seconds % 60).padStart(2, '0')}s`
66
67  return `${Math.floor(minutes / 60)}h${String(minutes % 60).padStart(2, '0')}m`
68}
69
70export const statusLabel = (task: MeshyTask) => {
71  if (task.status === 'PENDING') return 'queued'
72  if (task.status === 'DOWNLOADING') return 'saving'
73  if (task.status === 'IN_PROGRESS') return task.stage ?? 'running'
74
75  return task.status.toLowerCase()
76}
77
78/** Reads MESHY_API_KEY from a .env file's text, or undefined. */
79export const keyFromDotenv = (text: string) => {
80  for (const line of text.split(/\r?\n/)) {
81    const match = /^\s*(?:export\s+)?MESHY_API_KEY\s*=\s*(.*)$/.exec(line)
82    if (!match) continue
83    let value = (match[1] ?? '').trim()
84    const quote = value[0]
85    if ((quote === '"' || quote === "'") && value.indexOf(quote, 1) > 0) {
86      value = value.slice(1, value.indexOf(quote, 1))
87    } else {
88      value = value.replace(/\s+#.*$/, '')
89    }
90    if (value) return value
91  }
92
93  return undefined
94}
95
96/** Never lets the key reach a message, whatever an error echoes back. */
97export const scrub = (text: string, key: string) =>
98  key ? text.split(key).join('***') : text
99
100/** A Meshy response's JSON, or a MeshyError with the API's own message. */
101export const parseResponse = (response: HttpResponse, key: string): unknown => {
102  let json: unknown
103  try {
104    json = JSON.parse(response.text)
105  } catch {
106    json = undefined
107  }
108  if (!response.ok) {
109    const message = (json as { message?: unknown } | undefined)?.message
110    const text = typeof message === 'string' && message ? message : 'request failed'
111    throw new MeshyError(response.status, scrub(`${text} (HTTP ${response.status})`, key))
112  }
113
114  return json
115}
116
117const MIME: Record<string, string> = {
118  png: 'image/png',
119  jpg: 'image/jpeg',
120  jpeg: 'image/jpeg',
121}
122
123/** True for an image Meshy fetches itself: an http(s) URL or a data URI. */
124export const isRemoteImage = (image: string) => /^(https?:|data:)/i.test(image)
125
126/** The MIME type of a local image Meshy accepts, or an error. */
127export const imageMime = (image: string) => {
128  const mime = MIME[image.split('.').pop()?.toLowerCase() ?? '']
129  if (!mime) throw new Error(`${image}: Meshy takes .png, .jpg or .jpeg images`)
130
131  return mime
132}
133
134/** How an image argument shows in the pane: data URIs are not worth printing. */
135export const describeImage = (image: string) =>
136  image.startsWith('data:') ? '(data URI)' : image
137

1export type MeshyKind = 'image' | 'text' | 'multi'
2
3export type MeshyStatus =
4  | 'PENDING'
5  | 'IN_PROGRESS'
6  | 'DOWNLOADING'
7  | 'SUCCEEDED'
8  | 'FAILED'
9  | 'CANCELED'
10
11export type MeshyTask = {
12  /** The id meshy_generate returned; names the GLB file. */
13  id: string
14  kind: MeshyKind
15  /** The Meshy task being polled now (a text task's refine stage has its own). */
16  apiId: string
17  /** Text-to-3D only: which of the two stages is running. */
18  stage?: 'preview' | 'refine'
19  /** Text-to-3D only: start a refine (texturing) task once the preview succeeds. */
20  shouldRefine?: boolean
21  /** Refine request fields (enable_pbr, texture_prompt, ...). */
22  refineOptions?: { [field: string]: string | number | boolean }
23  status: MeshyStatus
24  progress: number
25  /** The prompt, or the image path or URL(s). */
26  source: string
27  /** The session directory the GLB is saved under. */
28  cwd: string
29  startedAt: number
30  finishedAt?: number
31  /** './meshy-models/<id>.glb' once downloaded. */
32  file?: string
33  error?: string
34}
35
36declare module 'claude-code' {
37  interface PluginState {
38    'meshy-tray': { tasks: MeshyTask[] }
39  }
40}
41