mcp-gaps

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { GapCandidate, McpCall, ToolFamily } from '../types'
5
6// mcp-gaps: a read-only observer. It never calls $.fs, $.process or $.http.
7// It watches MCP tool calls, keeps a session log, and flags three signals that
8// usually mean "the MCP could not do what the person wanted":
9//   1. browser-fallback: AdRoll MCP calls and browser-automation calls in the same turn
10//   2. tool-error: an AdRoll MCP tool answered an error or was denied
11//   3. stated-gap: the answer says the MCP/tool/API doesn't support something
12// Gap candidates persist on this machine through $.store so a week of dogfooding adds up.
13
14const PANE = 'mcp-gaps'
15const STORE_KEY = 'gaps'
16
17const calls = atom({ plugin: 'mcp-gaps', key: 'calls' } as const, [] as McpCall[])
18const gaps = atom({ plugin: 'mcp-gaps', key: 'gaps' } as const, [] as GapCandidate[])
19
20// Tool names of the AdRoll MCP server. Matching by name, not server prefix, because the
21// prefix differs between the desktop app (a connector id) and the CLI (whatever you named it).
22const ADROLL_TOOLS = new Set([
23  'whoami', 'get_entity_list', 'get_entity_details_batch', 'get_entity_metrics_batch',
24  'get_abm_metrics_batch', 'get_account_lists', 'get_account_list_groups', 'get_account_list_items',
25  'get_account_list_definitions', 'get_account_field_values', 'explore_account_count',
26  'explore_account_list', 'get_account_spikes', 'get_account_spike_details_batch',
27  'get_optimized_recommendations', 'get_revenue_impact', 'get_revenue_impact_opportunities',
28  'search_geo', 'upload_ad', 'edit_ad', 'edit_strategy', 'create_strategy', 'campaigns_preview',
29  'ads_preview', 'account_list_explorer', 'call_dev_tool', 'dev_tools',
30])
31const BROWSER_SERVER = /chrome|browser/i
32
33// "not supported", "can't be done through the MCP", "no tool for", "isn't available via the API"...
34const NEGATED_CAPABILITY = /\b(?:not|n['’]t|cannot|no|unable)\b[^.!?\n]{0,40}?\b(?:support|supported|available|expose[sd]?|offer|provide|allow|possible|exist|tool|endpoint|api|way)\b/i
35const MCP_CONTEXT = /\b(?:MCP|tool|API|server|connector|programmatically)\b/i
36
37// Per-turn scratch. Reset on each prompt; a hot reload loses it, which only costs one turn.
38let turnPrompt = ''
39let turnAdroll: string[] = []
40let turnBrowser = 0
41let turnErrors: McpCall[] = []
42
43function parseTool(tool: string): { server: string; name: string } | null {
44  const m = /^mcp__(.+?)__(.+)$/.exec(tool)
45  return m ? { server: m[1], name: m[2] } : null
46}
47
48function familyOf(server: string, name: string): ToolFamily {
49  if (ADROLL_TOOLS.has(name)) return 'adroll'
50  if (BROWSER_SERVER.test(server)) return 'browser'
51  return 'other'
52}
53
54function unique<T>(items: readonly T[]): T[] {
55  return [...new Set(items)]
56}
57
58function shorten(server: string): string {
59  return server.length > 24 ? `${server.slice(0, 8)}…` : server
60}
61
62function statedGap(answer: string): string | undefined {
63  for (const sentence of answer.split(/(?<=[.!?])\s+|\n+/)) {
64    if (sentence.length < 12 || sentence.length > 400) continue
65    if (NEGATED_CAPABILITY.test(sentence) && MCP_CONTEXT.test(sentence)) return sentence.trim()
66  }
67  return undefined
68}
69
70type ServerStat = { label: string; calls: number; errors: number; avgMs: number }
71
72function summarize(list: readonly McpCall[]): ServerStat[] {
73  const map = new Map<string, { calls: number; errors: number; ms: number; family: ToolFamily }>()
74  for (const c of list) {
75    const cur = map.get(c.server) ?? { calls: 0, errors: 0, ms: 0, family: c.family }
76    cur.calls += 1
77    cur.ms += c.ms
78    if (c.status !== 'ok') cur.errors += 1
79    if (c.family === 'adroll') cur.family = 'adroll'
80    map.set(c.server, cur)
81  }
82  return [...map.entries()]
83    .map(([server, s]) => ({
84      label: s.family === 'adroll' ? `AdRoll MCP (${shorten(server)})` : shorten(server),
85      calls: s.calls,
86      errors: s.errors,
87      avgMs: Math.round(s.ms / s.calls),
88    }))
89    .sort((a, b) => b.calls - a.calls)
90}
91
92async function refreshStatus($: EngineInterface): Promise<void> {
93  const list = await read($, calls)
94  const found = await read($, gaps)
95  const errors = list.filter(c => c.status !== 'ok').length
96  $.ui.status(`${list.length} MCP calls · ${errors} errors · ${found.length} gap candidates`)
97}
98
99function stamp(ms: number): string {
100  return `${new Date(ms).toISOString().slice(0, 16).replace('T', ' ')} UTC`
101}
102
103async function report($: EngineInterface): Promise<string> {
104  const list = await read($, calls)
105  const found = await read($, gaps)
106  const stats = summarize(list)
107  const lines: string[] = ['## MCP gap log', '']
108  lines.push(`This session: ${list.length} MCP tool call(s) across ${stats.length} server(s).`)
109  for (const s of stats) lines.push(`- ${s.label}: ${s.calls} calls, ${s.errors} errors, ${s.avgMs} ms avg`)
110  lines.push('', `### Gap candidates (${found.length} on this machine, all sessions)`)
111  if (found.length === 0) lines.push('- none yet')
112  found.slice(-40).forEach((g, i) => {
113    lines.push(`${i + 1}. [${g.kind}] ${stamp(g.at)}`)
114    if (g.prompt) lines.push(`   - asked: "${g.prompt}"`)
115    lines.push(`   - evidence: ${g.evidence}`)
116    if (g.tools.length > 0) lines.push(`   - MCP tools in that turn: ${g.tools.join(', ')}`)
117  })
118  return lines.join('\n')
119}
120
121export const register: Register = on => {
122  on('session.start', async ($, e, next) => {
123    const stored = await $.store.get(STORE_KEY)
124    if (Array.isArray(stored)) await update($, gaps, () => stored as GapCandidate[])
125    try {
126      await $.command.register({
127        name: 'mcp-gaps',
128        description: 'MCP gap log: open the pane, or `report` / `clear`',
129        argumentHint: '[report|clear]',
130      })
131    } catch {
132      // The name was taken; the observer still runs without its command.
133    }
134    return next(e)
135  })
136
137  on('prompt.submit', ($, e, next) => {
138    // A prompt queued into a running turn keeps that turn's counters.
139    if (e.turnId !== undefined) return next(e)
140    turnPrompt = e.text.slice(0, 160)
141    turnAdroll = []
142    turnBrowser = 0
143    turnErrors = []
144    return next(e)
145  })
146
147  on('tool.call', { tool: /^mcp__/ }, async ($, e, next) => {
148    const parsed = parseTool(e.tool)
149    if (!parsed) return next(e)
150    const family = familyOf(parsed.server, parsed.name)
151    const startedAt = await $.clock.now()
152    const ran = await next(e)
153    const ms = (await $.clock.now()) - startedAt
154    const status: McpCall['status'] = ran.deny !== undefined ? 'denied' : ran.isError ? 'error' : 'ok'
155    const detail = ran.deny ?? (ran.isError ? (ran.text ?? '').slice(0, 240) : undefined)
156    const call: McpCall = { id: e.tool_use_id, server: parsed.server, tool: parsed.name, family, ms, status, at: startedAt }
157    if (detail) call.detail = detail
158    await update($, calls, list => [...list, call].slice(-500))
159    if (family === 'adroll') turnAdroll.push(parsed.name)
160    if (family === 'browser') turnBrowser += 1
161    if (family === 'adroll' && status !== 'ok') turnErrors.push(call)
162    await refreshStatus($)
163    return ran
164  })
165
166  on('turn.complete', async ($, e, next) => {
167    const res = await next(e)
168    if (e.agentId) return res
169    const now = await $.clock.now()
170    const found: GapCandidate[] = []
171    if (turnAdroll.length > 0 && turnBrowser > 0) {
172      found.push({
173        id: `${e.turnId}-browser`,
174        at: now,
175        kind: 'browser-fallback',
176        prompt: turnPrompt,
177        evidence: `${turnBrowser} browser-automation call(s) in a turn that also made ${turnAdroll.length} AdRoll MCP call(s)`,
178        tools: unique(turnAdroll),
179      })
180    }
181    const stated = statedGap(e.answer)
182    if (stated) {
183      found.push({ id: `${e.turnId}-stated`, at: now, kind: 'stated-gap', prompt: turnPrompt, evidence: stated, tools: unique(turnAdroll) })
184    }
185    for (const err of turnErrors) {
186      found.push({
187        id: `${err.id}-error`,
188        at: err.at,
189        kind: 'tool-error',
190        prompt: turnPrompt,
191        evidence: `${err.tool} → ${err.status}${err.detail ? `: ${err.detail}` : ''}`,
192        tools: [err.tool],
193      })
194    }
195    if (found.length === 0) return res
196    await update($, gaps, list => [...list, ...found].slice(-200))
197    const all = await read($, gaps)
198    await $.store.set(STORE_KEY, all)
199    await refreshStatus($)
200    const kinds = unique(found.map(g => g.kind)).join(', ')
201    const text = `${found.length} possible MCP gap(s) recorded (${kinds}). /mcp-gaps to review · /mcp-gaps report to export`
202    return e.usage ? { text, usage: e.usage } : { text }
203  })
204
205  on('command.run', { command: 'mcp-gaps' }, async ($, e) => {
206    const arg = e.args.trim().toLowerCase()
207    if (arg === 'clear') {
208      await update($, gaps, () => [])
209      await $.store.delete(STORE_KEY)
210      await refreshStatus($)
211      return { text: 'MCP gap log cleared.' }
212    }
213    if (arg === 'report') return { text: await report($) }
214    const opened = await $.ui.open({ id: PANE, title: 'MCP gaps' })
215    return { text: opened.isPlaced ? 'MCP gaps pane opened.' : `MCP gaps pane is waiting: ${opened.reason}` }
216  })
217
218  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
219    const { Box, Text } = $.ui.resolve(e)
220    const list = await read($, calls)
221    const found = await read($, gaps)
222    const stats = summarize(list)
223    const room = Math.max(3, Math.floor(((e.viewport?.rows ?? 24) - 8 - stats.length) / 2))
224
225    return (
226      <Box flexDirection="column">
227        <Text bold>MCP calls this session</Text>
228        {stats.length === 0 && <Text dimColor>No MCP tool calls yet. Use an MCP tool and this fills in.</Text>}
229        {stats.map(s => (
230          <Text wrap="truncate-end">{s.label}: {s.calls} calls · {s.errors} errors · {s.avgMs} ms avg</Text>
231        ))}
232        <Text> </Text>
233        <Text bold>Gap candidates ({found.length})</Text>
234        {found.length === 0 && (
235          <Text dimColor>None yet. A browser fallback, a tool error, or an answer saying the MCP can't do something lands here.</Text>
236        )}
237        {found.slice(-room).map(g => (
238          <Box flexDirection="column">
239            <Text wrap="truncate-end">• [{g.kind}] {g.prompt || '(no prompt)'}</Text>
240            <Text dimColor wrap="truncate-end">  {g.evidence}</Text>
241          </Box>
242        ))}
243        <Text> </Text>
244        <Text dimColor>/mcp-gaps report → paste into #mcp-team · /mcp-gaps clear</Text>
245      </Box>
246    )
247  })
248}
249

1export type ToolFamily = 'adroll' | 'browser' | 'other'
2
3export type McpCall = {
4  id: string
5  server: string
6  tool: string
7  family: ToolFamily
8  ms: number
9  status: 'ok' | 'error' | 'denied'
10  detail?: string
11  at: number
12}
13
14export type GapKind = 'browser-fallback' | 'stated-gap' | 'tool-error'
15
16export type GapCandidate = {
17  id: string
18  at: number
19  kind: GapKind
20  prompt: string
21  evidence: string
22  tools: string[]
23}
24
25declare module 'claude-code' {
26  interface PluginState {
27    'mcp-gaps': { calls: McpCall[]; gaps: GapCandidate[] }
28  }
29}
30