sfx-gen

1// sfx-gen: generate sound effects from text with a local Stable Audio model.
2// Shells out to sa3.cpp's sa3-generate binary. No Python, no API key.
3
4import { atom, read, update } from 'claude-code'
5import type { EngineInterface, Register } from 'claude-code'
6
7import type { Clip } from '../types'
8
9type Engine = EngineInterface
10
11const PANE = 'sfx'
12const PANE_TITLE = 'Sound effects'
13const STORE_KEY = 'clips'
14const MAX_CLIPS = 50
15
16const clipsRef = { plugin: 'sfx-gen', key: 'clips' } as const
17const clips = atom(clipsRef, [])
18const pending = atom({ plugin: 'sfx-gen', key: 'pending' } as const, null)
19const dismissed = atom({ plugin: 'sfx-gen', key: 'dismissed' } as const, null)
20
21const DEFAULTS = {
22  model: 'small-sfx',
23  device: 'auto',
24  encoding: 'q5_k_m',
25  duration: 8,
26}
27
28type Config = typeof DEFAULTS & { sa3Dir?: string }
29let config: Config = { ...DEFAULTS }
30
31type GenOpts = {
32  prompt: string
33  duration?: number | string | null
34  seed?: number | string | null
35  steps?: number | string | null
36}
37
38type GenResult =
39  | { ok: true; clip: Clip; base64: string }
40  | { ok: false; error: string }
41
42function pickBinary(root: string) {
43  return root ? root + '/build-metal/bin/sa3-generate' : ''
44}
45
46// Parse a prompt string, pulling out --duration/--dur, --seed, --steps flags.
47function parseArgs(raw: string | undefined) {
48  let text = (raw || '').trim()
49  const grab = (re: RegExp) => {
50    const m = text.match(re)
51    if (!m) return null
52    text = text.replace(re, ' ').trim()
53    return m[1] ?? null
54  }
55  const duration = grab(/(?:^|\s)--(?:duration|dur)[ =](-?\d+(?:\.\d+)?)/i) ?? grab(/(?:^|\s)-d[ =](\d+(?:\.\d+)?)/i)
56  const seed = grab(/(?:^|\s)--seed[ =](\d+)/i)
57  const steps = grab(/(?:^|\s)--steps[ =](\d+)/i)
58  return { prompt: text, duration, seed, steps }
59}
60
61function errText(err: unknown) {
62  return err instanceof Error ? err.message : String(err)
63}
64
65function slug(text: string) {
66  return text.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'sfx'
67}
68
69async function firstExisting($: Engine, paths: string[]) {
70  for (const p of paths) if (await $.fs.exists(p)) return p
71  return null
72}
73
74async function saveClips($: Engine, list: Clip[]) {
75  try {
76    await $.store.set(STORE_KEY, list)
77  } catch (err) {
78    $.ui.toast('sfx-gen: could not persist clips: ' + errText(err))
79  }
80}
81
82async function addClip($: Engine, clip: Clip) {
83  await update($, clips, list => [...(list ?? []), clip].slice(-MAX_CLIPS))
84  await saveClips($, await read($, clips))
85}
86
87async function generate($: Engine, opts: GenOpts): Promise<GenResult> {
88  const envDir = await $.env.get('SA3_CPP_DIR')
89  const sa3Dir = (config.sa3Dir || envDir || '').trim()
90  const { model, encoding, device } = config
91  const duration = Number(opts.duration ?? config.duration)
92  const prompt = (opts.prompt || '').trim()
93
94  if (!prompt) return { ok: false, error: 'empty prompt' }
95
96  const bin =
97    (await firstExisting($, [sa3Dir + '/build-metal/bin/sa3-generate', sa3Dir + '/build/bin/sa3-generate'])) ??
98    pickBinary(sa3Dir)
99  const modelsDir = sa3Dir + '/models'
100
101  if (!(await $.fs.exists(bin))) {
102    return { ok: false, error: `sa3-generate not found at ${bin}. Set the sa3_dir config or install sa3.cpp.` }
103  }
104  if (!(await $.fs.exists(modelsDir))) {
105    return { ok: false, error: `models dir not found: ${modelsDir}. Run sa3.cpp's ./models.sh first.` }
106  }
107
108  const cwd = await $.session.cwd()
109  const id = String(Date.now())
110  const outPath = `${cwd}/.sfx-${id}.wav`
111
112  const argv = [
113    bin,
114    '--model', model,
115    '--encoding', encoding,
116    '--models-dir', modelsDir,
117    '--prompt', prompt,
118    '--duration', String(duration),
119    '--out', outPath,
120  ]
121  if (opts.seed != null) argv.push('--seed', String(opts.seed))
122  if (opts.steps != null) argv.push('--steps', String(opts.steps))
123
124  const init: { timeoutMs: number; env?: Record<string, string> } = { timeoutMs: 180000 }
125  if (device === 'cpu') init.env = { SA3_DEVICE: 'cpu' }
126
127  const started = Date.now()
128  $.ui.status(`generating ${duration}s sfx with ${model} (${device})...`)
129  await update($, pending, () => ({ prompt, duration, model, device }))
130
131  try {
132    let run
133    try {
134      run = await $.process.run(argv, init)
135    } catch (err) {
136      return { ok: false, error: `sa3-generate failed to start: ${errText(err)}` }
137    }
138    if (run.exitCode !== 0) {
139      return { ok: false, error: `sa3-generate exited ${run.exitCode}: ${run.stderr || run.stdout}` }
140    }
141
142    let bytes
143    try {
144      bytes = await $.fs.read(outPath, { as: 'bytes' })
145    } catch (err) {
146      return { ok: false, error: `wrote ${outPath} but could not read it back: ${errText(err)}` }
147    }
148
149    const elapsed = ((Date.now() - started) / 1000).toFixed(1)
150    const clip: Clip = { id, prompt, path: outPath, duration, elapsed, model, at: started }
151    await addClip($, clip)
152    return { ok: true, clip, base64: bytes.base64 }
153  } finally {
154    $.ui.status(undefined)
155    await update($, pending, () => null)
156  }
157}
158
159// Shared tail of /sfx and the tool: toast, play, show the pane.
160async function announce($: Engine, r: GenResult, play: boolean) {
161  if (!r.ok) {
162    $.ui.toast('sfx-gen: ' + r.error)
163    return
164  }
165  $.ui.toast(`sfx ready: ${r.clip.prompt} (${r.clip.elapsed}s)`)
166  void $.ui.open({ id: PANE, title: PANE_TITLE })
167  if (play) await $.audio.play({ base64: r.base64, mime: 'audio/wav' })
168}
169
170async function playClip($: Engine, clip: Clip) {
171  try {
172    const bytes = await $.fs.read(clip.path, { as: 'bytes' })
173    await $.audio.play({ base64: bytes.base64, mime: 'audio/wav' })
174  } catch (err) {
175    $.ui.toast(`sfx-gen: cannot play ${clip.path}: ${errText(err)}`)
176  }
177}
178
179// WhatsApp treats audio-only mp4 as a document, so render a waveform video track.
180async function shareClip($: Engine, clip: Clip) {
181  const ffmpeg = await firstExisting($, ['/opt/homebrew/bin/ffmpeg', '/usr/local/bin/ffmpeg', '/usr/bin/ffmpeg'])
182  if (!ffmpeg) {
183    $.ui.toast('sfx-gen: ffmpeg not found, install it with brew install ffmpeg')
184    return
185  }
186  if (!(await $.fs.exists(clip.path))) {
187    $.ui.toast(`sfx-gen: clip is gone: ${clip.path}`)
188    return
189  }
190
191  const home = await $.env.get('HOME')
192  const out = `${home}/Downloads/sfx-${slug(clip.prompt)}-${clip.id}.mp4`
193  $.ui.toast('sfx-gen: converting to mp4...')
194
195  try {
196    const run = await $.process.run(
197      [
198        ffmpeg, '-y', '-loglevel', 'error',
199        '-i', clip.path,
200        '-filter_complex', '[0:a]showwaves=s=720x720:mode=cline:colors=white,format=yuv420p[v]',
201        '-map', '[v]', '-map', '0:a',
202        '-c:v', 'libx264', '-preset', 'veryfast',
203        '-c:a', 'aac', '-b:a', '192k',
204        '-movflags', '+faststart',
205        out,
206      ],
207      { timeoutMs: 120000 },
208    )
209    if (run.exitCode !== 0) {
210      $.ui.toast(`sfx-gen: ffmpeg exited ${run.exitCode}: ${run.stderr}`)
211      return
212    }
213  } catch (err) {
214    $.ui.toast('sfx-gen: ffmpeg failed: ' + errText(err))
215    return
216  }
217
218  // A file reference on the clipboard pastes as an attachment in WhatsApp desktop.
219  let copied = false
220  try {
221    const cp = await $.process.run([
222      '/usr/bin/osascript',
223      '-e', 'on run argv',
224      '-e', 'set the clipboard to (POSIX file (item 1 of argv))',
225      '-e', 'end run',
226      out,
227    ])
228    copied = cp.exitCode === 0
229  } catch {}
230
231  const where = '~/Downloads/' + out.split('/').pop()
232  $.ui.toast(copied ? `mp4 saved to ${where} and copied. Paste it into WhatsApp.` : `mp4 saved to ${where}`)
233}
234
235export const register: Register = (on, options) => {
236  if (options.sa3_dir) config.sa3Dir = String(options.sa3_dir)
237  if (options.model) config.model = String(options.model)
238  if (options.device) config.device = String(options.device)
239  if (options.encoding) config.encoding = String(options.encoding)
240  if (options.duration != null) config.duration = Number(options.duration)
241
242  on('session.start', async ($, e, next) => {
243    // Seed session state from the store once; a hot reload keeps state as is.
244    if ((await $.state.get(clipsRef)).version === 0) {
245      const saved = await $.store.get(STORE_KEY)
246      const list = Array.isArray(saved) ? (saved as Clip[]).slice(-MAX_CLIPS) : []
247      await update($, clips, () => list)
248      // old clips stay in the pane but don't take over the band
249      const last = list[list.length - 1]
250      if (last) await update($, dismissed, () => last.id)
251    }
252
253    await $.command.register({
254      name: 'sfx',
255      description: 'Generate a sound effect from a text prompt with a local Stable Audio model',
256      argumentHint: '[description] [--duration N] [--seed N]',
257    })
258    await $.command.register({
259      name: 'sfx-pane',
260      description: 'Show generated sound effects in a pane',
261    })
262    await $.tool.register({
263      name: 'sfx',
264      description:
265        'Generate a sound effect from a text prompt using a local Stable Audio model (sa3.cpp). ' +
266        'No API key, no Python. Returns the path to a WAV file. Use for foley and SFX: ' +
267        '"metal impact with a deep sub boom", "whoosh across the screen", "footsteps on gravel". ' +
268        'Keep prompts to one sound each and layer clips for complex scenes.',
269      inputSchema: {
270        type: 'object',
271        properties: {
272          prompt: { type: 'string', description: 'Text description of the sound effect to generate' },
273          duration: { type: 'number', description: 'Clip length in seconds (1-30), default from config' },
274          seed: { type: 'number', description: 'Random seed for reproducibility' },
275          steps: { type: 'number', description: 'Diffusion steps, default 8' },
276          play: { type: 'boolean', description: 'Play the generated clip after writing (default true)' },
277        },
278        required: ['prompt'],
279      },
280    })
281    return next(e)
282  })
283
284  on('command.run', { command: 'sfx' }, async ($, e) => {
285    const a = parseArgs(e.args)
286    if (!a.prompt) return { text: 'Usage: /sfx <description> [--duration N] [--seed N]' }
287    const r = await generate($, a)
288    await announce($, r, true)
289    if (!r.ok) return { text: 'sfx-gen: ' + r.error }
290    return { text: `sfx-gen: generated ${r.clip.duration}s in ${r.clip.elapsed}s -> ${r.clip.path}\n  "${r.clip.prompt}"` }
291  })
292
293  on('command.run', { command: 'sfx-pane' }, async $ => {
294    const opened = await $.ui.open({ id: PANE, title: PANE_TITLE })
295    return { text: opened.isPlaced ? 'Sound effects pane opened.' : 'Sound effects pane is waiting for room.' }
296  })
297
298  on('tool.call', { tool: 'mcp__sfx-gen__sfx' }, async ($, e) => {
299    const input = e as unknown as GenOpts & { play?: boolean }
300    const r = await generate($, input)
301    await announce($, r, input.play !== false)
302    if (!r.ok) return { result: r.error }
303    return { result: `Generated ${r.clip.duration}s sound effect "${r.clip.prompt}" in ${r.clip.elapsed}s -> ${r.clip.path}` }
304  })
305
306  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
307    const { Box, Text, Button } = $.ui.resolve(e)
308    const list = (await read($, clips)) ?? []
309    const now = await read($, pending)
310    const room = Math.max(1, (e.viewport?.rows ?? 24) - 6)
311    const recent = list.slice(-room).reverse()
312
313    return (
314      <Box flexDirection="column">
315        {now && (
316          <Text color="yellow">
317            generating {now.duration}s "{now.prompt}" with {now.model}...
318          </Text>
319        )}
320        {list.length === 0 && !now && <Text dimColor>No clips yet. Try /sfx a whoosh across the screen</Text>}
321        {recent.map(clip => (
322          <Box key={'row-' + clip.id} flexDirection="column" marginBottom={1}>
323            <Text wrap="truncate-end">{clip.prompt}</Text>
324            <Box>
325              <Text dimColor>
326                {clip.duration}s, made in {clip.elapsed}s{' '}
327              </Text>
328              <Button key={'play-' + clip.id} label="Play" onPress={() => void playClip($, clip)} />
329              <Text> </Text>
330              <Button key={'share-' + clip.id} label="Share mp4" onPress={() => void shareClip($, clip)} />
331            </Box>
332          </Box>
333        ))}
334      </Box>
335    )
336  })
337
338  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
339    if (e.props.hasSurvey) return next(e)
340
341    const { Box, Text, Button } = $.ui.resolve(e)
342    const now = await read($, pending)
343    if (now) {
344      return (
345        <Box>
346          <Text color="yellow">
347            sfx: generating {now.duration}s "{now.prompt}" with {now.model} ({now.device})...
348          </Text>
349        </Box>
350      )
351    }
352
353    const list = (await read($, clips)) ?? []
354    const last = list[list.length - 1]
355    if (!last || (await read($, dismissed)) === last.id) return next(e)
356
357    return (
358      <Box>
359        <Text wrap="truncate-end">sfx: "{last.prompt}" {last.duration}s </Text>
360        <Button key="play" label="Play" onPress={() => void playClip($, last)} />
361        <Text> </Text>
362        <Button key="share" label="Share mp4" onPress={() => void shareClip($, last)} />
363        <Text> </Text>
364        <Button key="dismiss" label="Dismiss" role="dismiss" onPress={() => void update($, dismissed, () => last.id)} />
365      </Box>
366    )
367  })
368}
369

1export type Clip = {
2  id: string
3  prompt: string
4  path: string
5  duration: number
6  elapsed: string
7  model: string
8  at: number
9}
10
11export type Pending = { prompt: string; duration: number; model: string; device: string }
12
13declare module 'claude-code' {
14  interface PluginState {
15    'sfx-gen': {
16      clips: Clip[]
17      pending: Pending | null
18      // id of the last clip the band was dismissed for
19      dismissed: string | null
20    }
21  }
22}
23