internet-sources

1import { atom, read, update } from 'claude-code'
2import type { Register } from 'claude-code'
3
4import type { SourcePage } from '../types'
5
6const PANE = 'internet-sources'
7const DIR = '.cc-internet-sources'
8const pages = atom({ plugin: 'internet-sources', key: 'pages' } as const, [] as SourcePage[])
9const view = atom({ plugin: 'internet-sources', key: 'view' } as const, {} as { domain?: string; note?: string })
10
11const tokensOf = (text: string) => Math.ceil(text.length / 4)
12
13const domainOf = (url: string) => {
14  const m = /^[a-z][a-z0-9+.-]*:\/\/([^/?#:]+)/i.exec(url)
15  return (m ? m[1] : url).toLowerCase().replace(/^www\./, '')
16}
17
18/** Merge a page in: a repeat of the same url and route adds its tokens and a hit. */
19const addPage = (list: SourcePage[], p: Omit<SourcePage, 'hits' | 'at'>): SourcePage[] => {
20  const i = list.findIndex(x => x.url === p.url && x.via === p.via)
21  if (i < 0) return [...list, { ...p, hits: 1, at: Date.now() }]
22  return list.map((x, j) =>
23    j === i ? { ...x, tokens: x.tokens + p.tokens, hits: x.hits + 1, at: Date.now() } : x,
24  )
25}
26
27const byDomain = (list: SourcePage[]) => {
28  const map = new Map<string, SourcePage[]>()
29  for (const p of list) map.set(p.domain, [...(map.get(p.domain) ?? []), p])
30  return [...map.entries()]
31    .map(([domain, items]) => ({
32      domain,
33      items,
34      tokens: items.reduce((n, p) => n + p.tokens, 0),
35    }))
36    .sort((a, b) => b.items.length - a.items.length || b.tokens - a.tokens)
37}
38
39const toMarkdown = (id: string, list: SourcePage[]) => {
40  const groups = byDomain(list)
41  const total = list.reduce((n, p) => n + p.tokens, 0)
42  const lines = [
43    `# Internet sources — session ${id}`,
44    '',
45    `${list.length} pages across ${groups.length} domains · ~${total} tokens (estimated, chars/4)`,
46    '',
47  ]
48  for (const g of groups) {
49    lines.push(`## ${g.domain} (${g.items.length} pages, ~${g.tokens} tokens)`, '')
50    for (const p of g.items) {
51      const label = p.title ?? p.url
52      lines.push(`- [${label}](${p.url}) — ${p.via === 'fetch' ? 'fetched' : 'search hit'}, ~${p.tokens} tokens${p.hits > 1 ? `, ×${p.hits}` : ''}`)
53    }
54    lines.push('')
55  }
56  return lines.join('\n')
57}
58
59async function save($: any) {
60  const id = await $.session.id()
61  const list = await read($, pages)
62  const path = `${DIR}/${id}.md`
63  await $.fs.write(path, toMarkdown(id, list))
64  await update($, view, v => ({ ...v, note: `Saved ${list.length} pages to ${path}` }))
65  return path
66}
67
68export const register: Register = on => {
69  on('session.start', async ($, e, next) => {
70    await $.command.register({ name: 'internet-sources', description: 'Open the internet sources pane' })
71    await $.command.register({ name: 'internet-sources-save', description: `Save the sources list to ${DIR}/<session>.md` })
72    return next(e)
73  })
74
75  on('session.end', async ($, e, next) => {
76    if (e.reason === 'clear') {
77      await update($, pages, () => [])
78      await update($, view, () => ({}))
79    }
80    return next(e)
81  })
82
83  on('command.run', { command: 'internet-sources' }, async $ => {
84    await $.ui.open({ id: PANE, title: 'Internet sources' })
85    return { text: 'Internet sources pane opened.' }
86  })
87
88  on('command.run', { command: 'internet-sources-save' }, async $ => {
89    const path = await save($)
90    return { text: `Saved to ${path}` }
91  })
92
93  on('tool.call', { tool: 'WebFetch' }, async ($, e, next) => {
94    const ran: any = await next(e)
95    if (ran.deny === undefined && ran.isError !== true && ran.result) {
96      const url: string = ran.result.url ?? e.url
97      const text: string = typeof ran.result.result === 'string' ? ran.result.result : ''
98      await update($, pages, list =>
99        addPage(list, { url, domain: domainOf(url), via: 'fetch', tokens: tokensOf(text) }),
100      )
101    }
102    return ran
103  }).catch(($, e, next) => next(e))
104
105  on('tool.call', { tool: 'WebSearch' }, async ($, e, next) => {
106    const ran: any = await next(e)
107    if (ran.deny === undefined && ran.isError !== true && ran.result) {
108      const hits: { title: string; url: string }[] = []
109      for (const r of ran.result.results ?? []) {
110        if (typeof r !== 'string') hits.push(...r.content)
111      }
112      await update($, pages, list =>
113        hits.reduce(
114          (acc, h) =>
115            addPage(acc, {
116              url: h.url,
117              domain: domainOf(h.url),
118              via: 'search',
119              title: h.title,
120              tokens: tokensOf(`${h.title} ${h.url}`),
121            }),
122          list,
123        ),
124      )
125    }
126    return ran
127  }).catch(($, e, next) => next(e))
128
129  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
130    const { Box, Text, Button } = $.ui.resolve(e)
131    const list = await read($, pages)
132    const v = await read($, view)
133    const groups = byDomain(list)
134    const total = list.reduce((n, p) => n + p.tokens, 0)
135    const room = Math.max(1, (e.viewport?.rows ?? 24) - 8)
136    const group = v.domain === undefined ? undefined : groups.find(g => g.domain === v.domain)
137
138    const footer = (
139      <Box flexDirection="column" marginTop={1}>
140        {v.note !== undefined && <Text color="success">{v.note}</Text>}
141        <Box>
142          <Button variant="primary" hotkey="s" label="Save .md" onPress={() => save($)} />
143          {group && (
144            <Button hotkey="b" label="Back" onPress={() => update($, view, () => ({}))} />
145          )}
146        </Box>
147      </Box>
148    )
149
150    if (group) {
151      return (
152        <Box flexDirection="column">
153          <Text bold>{group.domain}</Text>
154          <Text dimColor>{group.items.length} pages · ~{group.tokens} tokens</Text>
155          {group.items.slice(0, room).map(p => (
156            <Box flexDirection="column" marginTop={1}>
157              <Text wrap="truncate-end">{p.title ?? p.url}</Text>
158              <Text dimColor wrap="truncate-end">
159                {p.via === 'fetch' ? 'fetched' : 'search hit'} · ~{p.tokens} tok{p.hits > 1 ? ` · ×${p.hits}` : ''}
160              </Text>
161              <Text dimColor wrap="truncate-end">{p.url}</Text>
162            </Box>
163          ))}
164          {footer}
165        </Box>
166      )
167    }
168
169    return (
170      <Box flexDirection="column">
171        <Text bold>Internet sources</Text>
172        <Text dimColor>{list.length} pages · {groups.length} domains · ~{total} tokens (est.)</Text>
173        {groups.length === 0 && <Text dimColor>No web pages accessed yet.</Text>}
174        {groups.slice(0, room).map(g => (
175          <Button
176            plain
177            key={`d:${g.domain}`}
178            label={`${g.domain} · ${g.items.length} · ~${g.tokens} tok`}
179            onPress={() => update($, view, () => ({ domain: g.domain }))}
180          />
181        ))}
182        {footer}
183      </Box>
184    )
185  })
186}
187

1export type SourcePage = {
2  url: string
3  domain: string
4  /** WebFetch: page content read into context. WebSearch: surfaced as a search hit. */
5  via: 'fetch' | 'search'
6  title?: string
7  /** Estimated tokens added to the context (chars / 4). */
8  tokens: number
9  hits: number
10  at: number
11}
12export type SourcesView = { domain?: string; note?: string }
13
14declare module 'claude-code' {
15  interface PluginState {
16    'internet-sources': { pages: SourcePage[]; view: SourcesView }
17  }
18}
19