mya

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, PluginOptions, Register } from 'claude-code'
3
4import type { Phase } from '../types'
5
6type Mode = 'dictate' | 'translate'
7
8const phase = atom({ plugin: 'mya', key: 'phase' } as const, 'idle' as Phase)
9const frame = atom({ plugin: 'mya', key: 'frame' } as const, 0)
10
11let options: PluginOptions = {}
12let listening: Mode | undefined
13let root = ''
14let isSpeaking = false
15let stopSpeaking = false
16let isVoiceTurn = false
17let isMuted = false
18let isKeysOn = true // the Right Ctrl hotkey: on, or off while you copy-paste elsewhere
19let currentPhase: Phase = 'idle'
20let ticker: { cancel: () => void } | undefined
21const stopFile = `/tmp/mya-${Math.random().toString(36).slice(2)}.stop` // one per session
22
23const text = (key: string, fallback = '') => String(options[key] ?? fallback)
24const python = () => text('python', 'python3')
25const envFileArgs = () => (text('envFile') ? ['--env', text('envFile')] : [])
26const keysEnv = (): Record<string, string> => {
27  const env: Record<string, string> = {}
28  if (text('deepgramApiKey')) env.DEEPGRAM_API_KEY = text('deepgramApiKey')
29  if (text('cartesiaApiKey')) env.CARTESIA_API_KEY = text('cartesiaApiKey')
30  return env
31}
32const message = (error: unknown) => (error instanceof Error ? error.message : String(error))
33const rest = (): Phase => (!isKeysOn ? 'off' : isMuted ? 'muted' : 'idle')
34
35// Claude has finished: the phone rings until you answer.
36async function waitForYou($: EngineInterface, isAnswer: boolean) {
37  if (listening || isSpeaking) return
38  await setPhase($, isAnswer && !isMuted ? 'waiting' : rest())
39  if (isAnswer) ring($)
40}
41
42async function setPhase($: EngineInterface, next: Phase) {
43  currentPhase = next
44  await update($, phase, () => next)
45}
46
47// Switch the hotkey on or off (the mic stays untouched until you tap the key again).
48async function toggleKeys($: EngineInterface, on?: boolean) {
49  isKeysOn = on ?? !isKeysOn
50  if (currentPhase === 'idle' || currentPhase === 'off' || currentPhase === 'muted') await setPhase($, rest())
51  $.ui.toast(isKeysOn ? 'MYA: hotkey on' : 'MYA: hotkey off (Right Ctrl + Right Shift to turn it on)')
52}
53
54// One tap on the key: start listening, or, if already listening, finish (the text is then sent).
55async function pressed($: EngineInterface, mode: Mode) {
56  if (isSpeaking) stopSpeaking = true // a tap cuts MYA's voice, then we listen
57  try {
58    if (listening) await $.fs.write(stopFile, 'stop')
59    else await listen($, mode, 'submit', 'hotkey')
60  } catch (error) {
61    listening = undefined
62    await setPhase($, rest())
63    $.ui.toast(`MYA: ${message(error)}`)
64  }
65}
66
67async function listen($: EngineInterface, mode: Mode, then: 'submit' | 'fill', how: 'hotkey' | 'command') {
68  listening = mode
69  await setPhase($, mode === 'translate' ? 'translating' : 'listening')
70  let heard = ''
71  let problem = ''
72  try {
73    const manualStop = how === 'hotkey' ? ['--silence-seconds', '3600', '--wait-seconds', '120', '--max-seconds', '180'] : []
74    const recording = $.process.spawn({
75      argv: [
76        python(), `${root}/bin/mic_stt.py`,
77        '--stop-file', stopFile,
78        ...envFileArgs(),
79        '--language', text('language', 'en'),
80        '--keyterms', text('keyterms', 'Mya'),
81        '--device', text('micDevice', 'auto'),
82        ...manualStop,
83      ],
84      env: keysEnv(),
85    })
86    for await (const piece of recording) {
87      if (piece.stream !== 'stdout') continue
88      for (const line of piece.text.split('\n')) {
89        const [kind, ...parts] = line.split('\t')
90        if (kind === 'TEXT') heard = parts.join('\t').trim()
91        if (kind === 'ERR') problem = parts.join('\t').trim()
92      }
93    }
94  } catch (error) {
95    problem = message(error)
96  } finally {
97    listening = undefined
98  }
99  if (!heard) {
100    await setPhase($, rest())
101    $.ui.toast(`MYA: ${problem || 'nothing received'}`)
102    if (options.phrases !== false && text('voiceEngine', 'cartesia') === 'machine') void talk($, 'I did not hear you. Please, repeat.').catch(() => undefined)
103    return
104  }
105
106  if (mode === 'translate') {
107    await setPhase($, 'thinking')
108    const done = await $.model.complete({
109      model: 'haiku',
110      prompt:
111        `Translate the text between the <t> tags into natural ${text('translateTo', 'English')}. ` +
112        'Reply with the translation only, no quotes, no commentary.\n<t>' + heard + '</t>',
113    })
114    if (!done.isAnswered) {
115      await setPhase($, rest())
116      $.ui.toast(`MYA: translation failed (${done.reason})`)
117      await $.prompt.fill({ text: heard, mode: 'append' }) // keep what was said
118      return
119    }
120    heard = done.text.trim()
121  }
122
123  if (then === 'submit') {
124    isVoiceTurn = true
125    await setPhase($, 'thinking')
126    await $.prompt.submit({ text: heard })
127  } else {
128    await setPhase($, rest())
129    await $.prompt.fill({ text: heard, mode: 'append' })
130  }
131}
132
133const LANGUAGES: Record<string, string> = { en: 'English', fr: 'French', de: 'German', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch' }
134// The language a machine voice speaks, from its eSpeak name: "en-us+klatt4" -> English.
135const machineLanguage = () => {
136  const code = text('machineVoice', 'en-us+klatt4').split('+')[0]!.split('-')[0]!
137  return LANGUAGES[code] ?? code
138}
139
140// The language a Piper voice speaks, from its file name: "fr_FR-siwis-medium.onnx" -> French.
141const piperLanguage = () => {
142  const code = /^([a-z]{2})[_-]/.exec(text('piperModel').split('/').pop() ?? '')?.[1] ?? text('language', 'en')
143  return LANGUAGES[code] ?? code
144}
145
146// MYA reads Claude's reply aloud. A machine or Piper voice speaks ONE language (a French voice reading English is unintelligible), so the
147// reply is first turned into a short spoken message in that language; a human-like cloud voice only condenses a long reply.
148async function say($: EngineInterface, answer: string) {
149  const engine = text('voiceEngine', 'cartesia')
150  const isMachine = engine === 'machine'
151  const voiceLanguage = isMachine ? machineLanguage() : engine === 'piper' ? piperLanguage() : undefined
152  let spoken = answer
153  const isPlainAscii = !/[^\x00-\x7f]/.test(answer)
154  // A machine voice speaks short plain ASCII as it is; Piper cannot tell the language of a text, so it always goes through the model.
155  if (isMachine ? !(isPlainAscii && answer.length <= 350) : voiceLanguage ? true : answer.length > 350) {
156    const language = voiceLanguage ? `in ${voiceLanguage}` : 'in the same language'
157    const short = await $.model.complete({
158      model: 'haiku',
159      prompt:
160        `Here is a coding assistant's reply to the user. Turn it into a short spoken message ${language}: ` +
161        (voiceLanguage ? `the message MUST be written entirely ${language}, translating anything that is in another language (keep only names of files or tools as they are). ` : '') +
162        '2 to 4 short plain sentences, one idea each, with commas where a speaker would pause, like an old computer reading carefully; ' +
163        'no markdown, no code, no paths or commands to spell out, no abbreviations or symbols ' +
164        'a speech synthesizer would stumble on. Give the gist, and say clearly if the user has to do something. ' +
165        'Reply with the message only.\n<r>' + answer.slice(0, 6000) + '</r>',
166    })
167    if (short.isAnswered) spoken = short.text
168    else if (voiceLanguage) {
169      $.ui.toast('MYA: could not prepare the spoken message')
170      return
171    }
172  }
173  await talk($, spoken)
174}
175
176// Speaks one text aloud with MYA's voice, showing the 'speaking' phase and honouring a tap that cuts it off.
177async function talk($: EngineInterface, spoken: string) {
178  isSpeaking = true
179  stopSpeaking = false
180  await setPhase($, 'speaking')
181  try {
182    const voice = $.process.spawn({
183      argv: [
184        python(), `${root}/bin/speak.py`,
185        ...envFileArgs(),
186        '--engine', text('voiceEngine', 'cartesia'),
187        '--piper-model', text('piperModel'),
188        '--machine-voice', text('machineVoice', 'en-us+klatt4'),
189        '--machine-rate', String(Number(options.machineRate ?? 155)),
190        '--machine-wordgap', String(Number(options.machineWordGap ?? 4)),
191        '--language', text('language', 'en'),
192        '--voice', text('voiceId'),
193        '--style', text('voiceStyle', 'plain'),
194        '--speed', String(Number(options.voiceSpeed ?? 1)),
195        '--device', text('speakerDevice', 'default'),
196      ],
197      env: keysEnv(),
198      input: spoken,
199    })
200    for await (const piece of voice) {
201      if (stopSpeaking) break // leaving the loop stops playback
202      if (piece.stream === 'stdout' && piece.text.startsWith('ERR')) $.ui.toast(`MYA: ${piece.text.split('\t')[1] ?? ''}`)
203    }
204  } finally {
205    isSpeaking = false
206    stopSpeaking = false
207    if (!listening) await setPhase($, rest())
208  }
209}
210
211// The phone: a bell now and then while Claude waits for you.
212function ring($: EngineInterface) {
213  if (options.ringSound !== true) return
214  void (async () => {
215    try {
216      const bell = $.process.spawn({
217        argv: [python(), `${root}/bin/ring.py`, '--times', '2', '--device', text('speakerDevice', 'default')],
218      })
219      for await (const _piece of bell) { /* plays to the end */ }
220    } catch { /* a missing bell is not worth an error */ }
221  })()
222}
223
224const LABEL: Record<Phase, string> = {
225  off: 'hotkey off, Right Ctrl + Right Shift or /mya on',
226  waiting: 'your turn, tap Right Ctrl to answer',
227  idle: 'tap Right Ctrl to talk',
228  listening: 'listening, tap Right Ctrl to send',
229  translating: 'translating, tap Right Ctrl to send',
230  thinking: 'thinking',
231  speaking: 'speaking, tap Right Ctrl to interrupt',
232  muted: 'muted (/mya mute)',
233}
234const COLOR: Record<Phase, string> = { off: 'red', waiting: 'yellow', idle: 'gray', listening: 'green', translating: 'yellow', thinking: 'magenta', speaking: 'cyan', muted: 'gray' }
235const BARS = '▁▂▃▄▅▆▇█'
236// A telephone that rings: the handset rocks and the sound waves come and go.
237const PHONE = ['  ☎  ', ' ((☎)) ', '(((☎)))', ' ((☎)) ']
238const phone = (tick: number) => PHONE[tick % PHONE.length]!
239
240// A little level meter that moves while MYA listens or speaks.
241const meter = (tick: number, width = 12) =>
242  Array.from({ length: width }, (_, i) => BARS[Math.min(7, Math.floor(Math.abs(Math.sin(tick * 0.9 + i * 1.7)) * 8))]).join('')
243
244export const register: Register = (on, config) => {
245  options = config
246  isKeysOn = options.hotkeyOnStart !== false
247
248  on('turn.complete', async ($, e, next) => {
249    const done = await next(e)
250    if (isVoiceTurn && !e.agentId) {
251      isVoiceTurn = false
252      const willSpeak = options.speakReplies !== false && !isMuted && e.reason === 'answer' && e.answer.trim() !== ''
253      if (willSpeak) {
254        void say($, e.answer)
255          .catch(error => $.ui.toast(`MYA: ${message(error)}`))
256          .finally(() => { void waitForYou($, e.reason === 'answer') })
257        return done
258      }
259    }
260    if (!e.agentId) await waitForYou($, e.reason === 'answer')
261    return done
262  })
263
264  on('prompt.submit', async ($, e, next) => {
265    if (!listening) await setPhase($, 'thinking')
266    return next(e)
267  })
268
269  on('session.start', async ($, e, next) => {
270    root = $.plugin.root
271    await $.command.register({
272      name: 'mya',
273      description: 'MYA voice: /mya (dictate into the prompt), /mya go (dictate and send), /mya stop, /mya mute, /mya off|on (the Right Ctrl hotkey).',
274    })
275
276    if (options.phrases !== false && text('voiceEngine', 'cartesia') === 'machine') {
277      void talk($, 'Mya online. I am ready, when you are.').catch(() => undefined)
278    }
279    await setPhase($, rest())
280    ticker?.cancel()
281    ticker = $.clock.every(300, () => {
282      if (currentPhase !== 'idle' && currentPhase !== 'muted' && currentPhase !== 'off') void update($, frame, n => (n ?? 0) + 1) // the band animates
283    })
284
285    // The hotkey: tap = dictate / send, with the chord key held = translate.
286    // The listener (xinput) can die — keyboard replugged, X restarted — and MYA would stay deaf for good. So it is started again, waiting longer
287    // after each quick death (and giving up after repeated failures to start at all).
288    void (async () => {
289      let quickDeaths = 0
290      let startFailures = 0
291      while (startFailures < 5) {
292        const startedAt = Date.now()
293        try {
294          const keys = $.process.spawn({
295            argv: [python(), `${root}/bin/hotkey.py`, String(options.toggleKeycode ?? 105), String(options.chordKeycode ?? 65), String(options.lockKeycode ?? 62), String(options.maxTapSeconds ?? 0.8)],
296          })
297          for await (const piece of keys) {
298            if (piece.stream !== 'stdout') continue
299            for (const line of piece.text.split('\n')) {
300              const [word, ...parts] = line.trim().split('\t')
301              if (word === 'LOCK') void toggleKeys($)
302              else if (word === 'TOGGLE' && isKeysOn) void pressed($, 'dictate')
303              else if (word === 'TRANSLATE' && isKeysOn) void pressed($, 'translate')
304              else if (word === 'ERR') $.ui.toast(`MYA: hotkey ${parts.join(' ')}`)
305            }
306          }
307          startFailures = 0
308        } catch {
309          startFailures += 1
310          if (startFailures === 5) $.ui.toast('MYA: the hotkey is unavailable')
311        }
312        quickDeaths = Date.now() - startedAt > 10_000 ? 0 : quickDeaths + 1
313        if (quickDeaths === 4) $.ui.toast('MYA: the hotkey listener keeps stopping, retrying')
314        try {
315          await $.clock.sleep(Math.min(Number(options.hotkeyRetryMs ?? 2000) * 2 ** Math.min(quickDeaths, 4), 30_000))
316        } catch {
317          return // the module was unloaded
318        }
319      }
320    })()
321
322    return next(e)
323  })
324
325  on('command.run', { command: 'mya' }, async ($, e) => {
326    const arg = e.args.trim().toLowerCase()
327    if (arg === 'stop') {
328      if (!listening) return { text: 'MYA is not listening.' }
329      await $.fs.write(stopFile, 'stop')
330      return { text: 'MYA: finishing…' }
331    }
332    if (arg === 'off' || arg === 'on' || arg === 'keys') {
333      await toggleKeys($, arg === 'keys' ? undefined : arg === 'on')
334      return { text: isKeysOn ? 'MYA hotkey is on.' : 'MYA hotkey is off: Right Ctrl does nothing until /mya on, or Right Ctrl + Right Shift.' }
335    }
336    if (arg === 'mute') {
337      isMuted = !isMuted
338      if (isMuted) stopSpeaking = true
339      if (currentPhase === 'idle' || currentPhase === 'muted') await setPhase($, rest())
340      return { text: isMuted ? 'MYA will stay quiet.' : 'MYA will speak again.' }
341    }
342    if (arg !== '' && arg !== 'go') return { text: 'Usage: /mya, /mya go, /mya stop, /mya mute, /mya off or /mya on.' }
343    if (listening) return { text: 'MYA is already listening: speak, or /mya stop.' }
344    void listen($, 'dictate', arg === 'go' ? 'submit' : 'fill', 'command').catch(async error => {
345      listening = undefined
346      await setPhase($, rest())
347      $.ui.toast(`MYA: ${message(error)}`)
348    })
349    return { text: 'MYA is listening…' }
350  })
351
352  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
353    if (options.showBand === false || e.props.hasSurvey) return next(e)
354    const { Box, Text } = $.ui.resolve(e)
355    const now = (await read($, phase)) ?? 'idle'
356    const tick = (await read($, frame)) ?? 0
357    const color = COLOR[now]
358    const live = now === 'listening' || now === 'translating' || now === 'speaking'
359    const dots = '.'.repeat((tick % 3) + 1)
360
361    return (
362      <Box>
363        <Text bold inverse color={color}> MYA </Text>
364        <Text color={color}> {live ? meter(tick) : now === 'thinking' ? dots.padEnd(3) : now === 'waiting' ? phone(tick) : '·'} </Text>
365        <Text dimColor>{isKeysOn || now === 'off' ? LABEL[now] : LABEL[now].replace(/, tap Right Ctrl.*$/, '')}</Text>
366      </Box>
367    )
368  })
369}
370

1export type Phase = 'idle' | 'listening' | 'translating' | 'thinking' | 'speaking' | 'waiting' | 'muted' | 'off'
2
3declare module 'claude-code' {
4  interface PluginState {
5    mya: { phase: Phase; frame: number }
6  }
7}
8