memory-lens

1/**
2 * Hook registration for memory-lens.
3 *
4 * Events:
5 * - session.start: find this project's memory folder (a worktree maps to its
6 *   parent repo), then build the index in the background so the session is
7 *   never held up. Also reads two optional local files: a client list and a
8 *   mirror file of remote titles written by a private job outside this mod.
9 * - prompt.submit: the top 3 memories above the score floor go to the model as
10 *   one context entry and to the human as one transcript line. The 2.1.283
11 *   prompt.submit contract: an answer carries { text } and an optional
12 *   context that is a list of non-empty strings (binary checkArgument).
13 * - tool.call (Read, Edit, Write, Grep, Glob): the path is a new query; at
14 *   most one memory not yet recalled this session is added as tool context.
15 *   A Write or Edit under the memory folder re-indexes that file.
16 * - command.run{command=memory-lens}: stats, reload, show <n>.
17 *
18 * Negative pins: no process.run, no http.fetch, no store. Nothing on the hot
19 * path touches the network; every file read happens at session start or after
20 * a memory write. Hooks module format: exports register(on, options).
21 */
22
23import { contextText, percentiles, screenLines } from '../src/format.js';
24import { isIndexFile, parentProjectKey, parseMemory, parseMirror, projectKey } from '../src/memory.js';
25import { LensIndex } from '../src/rank.js';
26import type { Hit, LensPrivate, MemoryDoc } from '../src/types.js';
27
28/** Minimal $ facade for the calls this module makes. */
29type Lens$ = {
30  env: { get: (name: string) => Promise<string | undefined> };
31  fs: {
32    read: (path: string) => Promise<string>;
33    list: (path: string) => Promise<Array<{ name: string; kind: 'file' | 'dir' | 'other' }> | null>;
34  };
35  ui: { log: (text: string, options?: { to?: 'transcript' | 'debug' }) => Promise<void> };
36  command: { register: (spec: { name: string; description: string }) => Promise<unknown> };
37  clock: { after: (ms: number, fn: () => void) => { cancel?: () => void } };
38};
39
40type NextFn<E> = (ev: E) => Promise<unknown>;
41type SessionStartEvent = { cwd?: string };
42type PromptSubmitEvent = { text?: string; [k: string]: unknown };
43type ToolCallEvent = { tool: string; tool_use_id?: string; [argument: string]: unknown };
44type CommandEvent = { command?: string; args?: string; text?: string };
45
46/** Tools whose path argument is a useful mid-task query. */
47const PATH_TOOLS = new Set(['Read', 'Edit', 'Write', 'Grep', 'Glob']);
48
49// Module-scope state (per session)
50let home = '';
51let memoryDir = '';
52let mirrorPath = '';
53let priv: LensPrivate = { clientTerms: [] };
54let docs = new Map<string, MemoryDoc>();
55let index: LensIndex | null = null;
56let building: Promise<void> | null = null;
57const recalled = new Set<string>();
58const queryMs: number[] = [];
59let lastHits: Hit[] = [];
60
61function join(...parts: string[]): string {
62  return parts.join('/').replace(/\/+/g, '/');
63}
64
65/** Names in ~/.claude/memory-lens, listed first so a missing optional file is not read (and not logged as an error). */
66let lensFiles = new Set<string>();
67
68async function readPrivate($: Lens$): Promise<LensPrivate> {
69  const entries = (await $.fs.list(join(home, '.claude', 'memory-lens')).catch(() => null)) ?? [];
70  lensFiles = new Set(entries.map((x) => x.name));
71  if (!lensFiles.has('private.json')) return { clientTerms: [] };
72  try {
73    const raw = JSON.parse(await $.fs.read(join(home, '.claude', 'memory-lens', 'private.json'))) as { clientTerms?: unknown };
74    const terms = Array.isArray(raw.clientTerms) ? raw.clientTerms.filter((t): t is string => typeof t === 'string') : [];
75    return { clientTerms: terms };
76  } catch {
77    return { clientTerms: [] };
78  }
79}
80
81async function loadDocs($: Lens$): Promise<Map<string, MemoryDoc>> {
82  const out = new Map<string, MemoryDoc>();
83  const entries = (await $.fs.list(memoryDir).catch(() => null)) ?? [];
84  for (const e of entries) {
85    if (e.kind !== 'file' || !e.name.endsWith('.md') || isIndexFile(e.name)) continue;
86    const path = join(memoryDir, e.name);
87    try {
88      out.set(path, parseMemory(path, await $.fs.read(path)));
89    } catch {
90      // unreadable file: skip it, never fail the session
91    }
92  }
93  if (lensFiles.has(mirrorPath.split('/').pop() ?? '')) {
94    try {
95      for (const d of parseMirror(await $.fs.read(mirrorPath), 'mirror')) out.set(`mirror:${d.id}`, d);
96    } catch {
97      // unreadable mirror file: local memory only
98    }
99  }
100  return out;
101}
102
103async function build($: Lens$): Promise<void> {
104  const t0 = Date.now();
105  priv = await readPrivate($);
106  docs = await loadDocs($);
107  index = new LensIndex([...docs.values()]);
108  await $.ui.log(`memory-lens: indexed ${index.size} memories in ${Date.now() - t0} ms (${memoryDir})`, { to: 'debug' }).catch(() => undefined);
109}
110
111/** Longest the first prompt waits for the index; later prompts never wait. */
112export const FIRST_WAIT_MS = 2000;
113
114/** Await p, or give up after ms using the engine clock (a mod has no ambient timers). */
115async function waitFor($: Lens$, p: Promise<void>, ms: number): Promise<void> {
116  let timer: { cancel?: () => void } | undefined;
117  const expired = new Promise<void>((resolve) => {
118    try {
119      timer = $.clock.after(ms, () => resolve());
120    } catch {
121      resolve();
122    }
123  });
124  try {
125    await Promise.race([p, expired]);
126  } finally {
127    timer?.cancel?.();
128  }
129}
130
131function timedQuery(text: string, k: number): Hit[] {
132  if (!index) return [];
133  const t0 = Date.now();
134  const hits = index.query(text, k, recalled);
135  queryMs.push(Date.now() - t0);
136  if (queryMs.length > 500) queryMs.shift();
137  return hits;
138}
139
140function withContext(result: unknown, entry: string): unknown {
141  if (!result || typeof result !== 'object') return result;
142  const r = result as Record<string, unknown>;
143  const prior = Array.isArray(r.context) ? (r.context as unknown[]).filter((c): c is string => typeof c === 'string' && c !== '') : [];
144  return { ...r, context: [...prior, entry] };
145}
146
147function pathQuery(e: ToolCallEvent): string {
148  const raw = String(e.file_path ?? e.path ?? e.pattern ?? '');
149  // the last three path segments carry the topic; the home prefix carries none
150  return raw.split('/').slice(-3).join(' ').replace(/\.[a-z0-9]+$/i, '');
151}
152
153export function register(on: (event: string, matcherOrHook: unknown, hook?: unknown) => void, _options?: unknown): void {
154  on('session.start', async ($: Lens$, e: SessionStartEvent, next: NextFn<SessionStartEvent>) => {
155    home = (await $.env.get('HOME').catch(() => undefined)) ?? '';
156    const key = parentProjectKey(projectKey(e?.cwd ?? ''));
157    memoryDir = join(home, '.claude', 'projects', key, 'memory');
158    mirrorPath = join(home, '.claude', 'memory-lens', `mirror-${key}.json`);
159    index = null;
160    recalled.clear();
161    lastHits = [];
162    building = build($).catch(() => undefined);
163    await $.command.register({ name: 'memory-lens', description: 'Memory lens: stats, reload, show <n>' }).catch(() => undefined);
164    return next(e);
165  });
166
167  on('prompt.submit', async ($: Lens$, e: PromptSubmitEvent, next: NextFn<PromptSubmitEvent>) => {
168    // Query BEFORE next(e) and hand the context down the chain. Measured on
169    // 2.1.283 (2026-09-28): the classic UserPromptSubmit hooks run inside
170    // next(e) for about 560 ms, and the turn started before an answer built
171    // after next(e) settled, so that context never reached the model.
172    let ev = e;
173    let ctx: string | null = null;
174    let hits: Hit[] = [];
175    // The first prompt can arrive while the index is still building (headless
176    // -p sends it at once): wait for the build, at most FIRST_WAIT_MS.
177    if (!index && building) await waitFor($, building, FIRST_WAIT_MS);
178    try {
179      if (index && typeof e?.text === 'string') {
180        hits = timedQuery(e.text, 3);
181        ctx = hits.length ? contextText(hits) : null;
182        if (ctx) ev = withContext(e, ctx) as PromptSubmitEvent;
183      }
184    } catch {
185      ctx = null;
186    }
187    const result = (await next(ev)) ?? ev;
188    try {
189      const r = result as PromptSubmitEvent & { drop?: unknown };
190      if (!ctx || typeof r.text !== 'string' || r.drop !== undefined) return result;
191      lastHits = hits;
192      for (const h of hits) recalled.add(h.doc.id);
193      // One call, rows joined on one line: measured in the 2.1.283 TUI, a
194      // newline inside a log call renders as a replacement glyph, and a second
195      // $.ui.log in the same hook call never drew (the hook settled at once).
196      try {
197        await $.ui.log(screenLines(hits, priv, home).join('  │  '));
198      } catch {
199        // no log surface: the context still went to the model
200      }
201      const has = Array.isArray(r.context) && (r.context as unknown[]).includes(ctx);
202      return has ? result : withContext(result, ctx);
203    } catch {
204      return result;
205    }
206  });
207
208  on('tool.call', async ($: Lens$, e: ToolCallEvent, next?: NextFn<ToolCallEvent>) => {
209    const result = next ? ((await next(e)) ?? {}) : {};
210    try {
211      if (!PATH_TOOLS.has(e.tool)) return result;
212      const path = String(e.file_path ?? '');
213      if ((e.tool === 'Write' || e.tool === 'Edit') && memoryDir && path.startsWith(memoryDir + '/') && path.endsWith('.md')) {
214        // a memory was written: re-index that one file so the next prompt can find it
215        const name = path.split('/').pop() ?? '';
216        if (!isIndexFile(name)) {
217          docs.set(path, parseMemory(path, await $.fs.read(path)));
218          index = new LensIndex([...docs.values()]);
219        }
220        return result;
221      }
222      if (!index) return result;
223      const hits = timedQuery(pathQuery(e), 1);
224      const ctx = contextText(hits);
225      if (!hits.length || !ctx) return result;
226      recalled.add(hits[0].doc.id);
227      return withContext(result, ctx);
228    } catch {
229      return result;
230    }
231  });
232
233  on('command.run', { command: 'memory-lens' }, async ($: Lens$, e: CommandEvent) => {
234    const args = String(e?.args ?? e?.text ?? '').trim().split(/\s+/);
235    if (args[0] === 'reload') {
236      building = build($).catch(() => undefined);
237      await building;
238      return { text: `memory-lens: reloaded ${index?.size ?? 0} memories` };
239    }
240    if (args[0] === 'show') {
241      const n = Number(args[1]) - 1;
242      const h = lastHits[n];
243      if (!h) return { text: `memory-lens: no hit ${args[1] ?? ''} on the last prompt` };
244      return { text: `${n + 1}. ${h.doc.name}\n   ${h.doc.desc}\n   ${h.doc.id}` };
245    }
246    await building;
247    const p = percentiles(queryMs);
248    return {
249      text: `memory-lens: ${index?.size ?? 0} memories from ${memoryDir || '(no project)'} · ${recalled.size} recalled this session · query p50 ${p.p50} ms p95 ${p.p95} ms over ${p.n}`,
250    };
251  });
252}
253

1/**
2 * What the human sees and what the model gets.
3 *
4 * Screen line: title, why it matched, where it lives. A client memory shows as
5 * "client memory (title hidden)": screens get shared and recorded. Model context:
6 * the same hits with their descriptions, capped. Secret-shaped text reaches
7 * neither.
8 */
9
10import type { Hit, LensPrivate } from './types.js';
11
12/** Hard cap on hits per prompt. */
13export const MAX_HITS = 3;
14/** About 600 tokens at 4 characters per token. */
15export const MAX_CONTEXT_CHARS = 2400;
16const DESC_CHARS = 240;
17
18const SECRET = /(sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|xox[abpr]-[A-Za-z0-9-]{8,}|AKIA[0-9A-Z]{16}|ops_[A-Za-z0-9]{20,}|-----BEGIN [A-Z ]*PRIVATE KEY)/;
19
20/** True when text carries something shaped like a credential. */
21export function looksSecret(text: string): boolean {
22  return SECRET.test(text);
23}
24
25/**
26 * True when any field either surface prints is secret-shaped: name and
27 * description, and also the id, source and image path, which the screen line
28 * and the context print for mirrored and local memories alike.
29 */
30export function docLooksSecret(doc: Hit['doc']): boolean {
31  return looksSecret([doc.name, doc.desc, doc.id, doc.source, doc.image ?? ''].join(' '));
32}
33
34/** True when a hit must not show its title on screen. */
35export function isClient(hit: Hit, priv: LensPrivate): boolean {
36  const hay = `${hit.doc.id} ${hit.doc.name} ${hit.doc.desc}`.toLowerCase();
37  if (/(^|[/-])clients?-/.test(hit.doc.id.toLowerCase())) return true;
38  return priv.clientTerms.some((t) => t.length > 1 && hay.includes(t.toLowerCase()));
39}
40
41function shortPath(id: string, home: string): string {
42  if (!id.startsWith('/')) return id;
43  const m = /\/\.claude\/projects\/([^/]+)\/memory\/(.+)$/.exec(id);
44  if (!m) return home && id.startsWith(home) ? '~' + id.slice(home.length) : id;
45  const proj = m[1].replace(/^-Users-[^-]+-coding-/, '').replace(/^-/, '');
46  return `${proj}/memory/${m[2]}`;
47}
48
49function title(hit: Hit): string {
50  const d = hit.doc.desc.trim();
51  const t = d && d.length <= 90 ? d : hit.doc.name.replace(/[_-]+/g, ' ');
52  return t.length > 90 ? t.slice(0, 87) + '...' : t;
53}
54
55/**
56 * The rows the human sees under the prompt, one per memory. Each row is one
57 * $.ui.log call: measured in the 2.1.283 TUI (2026-09-28), a newline inside a
58 * single log call renders as a replacement glyph, and Claude Code already
59 * prefixes every row with the plugin name, so rows carry neither.
60 */
61export function screenLines(hits: Hit[], priv: LensPrivate, home = ''): string[] {
62  return hits.slice(0, MAX_HITS).map((h, i) => {
63    const n = `${i + 1}/${Math.min(hits.length, MAX_HITS)}`;
64    if (isClient(h, priv) || docLooksSecret(h.doc)) return `${n} [locked] client memory (title hidden)`;
65    // the file name on screen, the full path in the model's context
66    const where = h.doc.source === 'local' ? (shortPath(h.doc.id, home).split('/').pop() ?? h.doc.id) : `${h.doc.source}: ${h.doc.id}`;
67    const image = h.doc.image ? ` · image ${h.doc.image}` : '';
68    const why = h.why.length ? ` · ${h.why.slice(0, 2).join(', ')}` : '';
69    return `${n} ${title(h)} (${where}${image})${why}`;
70  });
71}
72
73/** The context entry the model gets, or null when nothing is safe to send. */
74export function contextText(hits: Hit[]): string | null {
75  const lines: string[] = [];
76  for (const h of hits.slice(0, MAX_HITS)) {
77    if (docLooksSecret(h.doc)) continue;
78    const desc = h.doc.desc.length > DESC_CHARS ? h.doc.desc.slice(0, DESC_CHARS - 3) + '...' : h.doc.desc;
79    const where = h.doc.source === 'local' ? h.doc.id : `${h.doc.source} id ${h.doc.id} (fetch the body only if needed)`;
80    lines.push(`- ${h.doc.name}: ${desc || '(no description)'} [${where}]`);
81  }
82  if (!lines.length) return null;
83  let text = 'Related memories (memory-lens; weigh them, they may be stale or disagree with the prompt):\n' + lines.join('\n');
84  if (text.length > MAX_CONTEXT_CHARS) text = text.slice(0, MAX_CONTEXT_CHARS - 3) + '...';
85  return text;
86}
87
88/** p50 and p95 of recorded query times, in ms. */
89export function percentiles(ms: number[]): { p50: number; p95: number; n: number } {
90  const s = [...ms].sort((a, b) => a - b);
91  if (!s.length) return { p50: 0, p95: 0, n: 0 };
92  return { p50: s[Math.floor(s.length / 2)], p95: s[Math.floor(0.95 * (s.length - 1))], n: s.length };
93}
94

1/**
2 * Reading memory: frontmatter parsing, the project folder for a cwd, and mirror files.
3 * Pure over strings; the hooks module owns every file read.
4 */
5
6import { looksSecret } from './format.js';
7import type { MemoryDoc, MirrorItem } from './types.js';
8
9/** Body characters kept for matching. Past this, a memory is detail, not topic. */
10export const BODY_CHARS = 1200;
11
12const IMAGE = /[~/A-Za-z0-9._-]+\.(?:png|jpe?g|svg|webp)\b/;
13
14/** Parse one memory file. Frontmatter keys may be top level or under metadata. */
15export function parseMemory(id: string, text: string): MemoryDoc {
16  const fm: Record<string, string> = {};
17  let body = text;
18  if (text.startsWith('---')) {
19    const end = text.indexOf('\n---', 3);
20    if (end > 0) {
21      for (const line of text.slice(3, end).split('\n')) {
22        const m = /^\s*(name|description|type):\s*(.*)$/.exec(line);
23        if (m && !(m[1] in fm)) fm[m[1]] = m[2].trim().replace(/^["']|["']$/g, '');
24      }
25      body = text.slice(end + 4);
26    }
27  }
28  const base = id.split('/').pop() ?? id;
29  const image = IMAGE.exec(text)?.[0];
30  return {
31    id,
32    name: fm.name || base.replace(/\.md$/, ''),
33    desc: fm.description ?? '',
34    type: fm.type ?? '',
35    body: body.slice(0, BODY_CHARS),
36    ...(image ? { image } : {}),
37    source: 'local',
38  };
39}
40
41/** Claude Code's project folder name for a cwd: every non-alphanumeric becomes "-". */
42export function projectKey(cwd: string): string {
43  return cwd.replace(/[^A-Za-z0-9]/g, '-');
44}
45
46/**
47 * A worktree under <repo>/.worktrees/<name> gets its own, usually empty, project
48 * folder; its lessons live under the parent repo's folder.
49 */
50export function parentProjectKey(key: string): string {
51  return key.split('--worktrees-')[0];
52}
53
54/** Index files that only point at other memories; never recalled themselves. */
55export function isIndexFile(name: string): boolean {
56  return /^MEMORY(-archive|-ARCHIVE)?\.md$/.test(name);
57}
58
59/** Parse a mirror file. Anything malformed yields no items, never a throw. */
60export function parseMirror(text: string, fallbackSource: string): MemoryDoc[] {
61  let parsed: unknown;
62  try {
63    parsed = JSON.parse(text);
64  } catch {
65    return [];
66  }
67  const items = (parsed as { items?: unknown })?.items;
68  if (!Array.isArray(items)) return [];
69  const out: MemoryDoc[] = [];
70  for (const raw of items as MirrorItem[]) {
71    if (!raw || typeof raw.id !== 'string' || typeof raw.title !== 'string' || !raw.title) continue;
72    const source = typeof raw.source === 'string' && raw.source ? raw.source : fallbackSource;
73    // the id and source are printed on screen and sent to the model: a
74    // credential-shaped one drops the whole item (review of #4530)
75    if (looksSecret(raw.id) || looksSecret(source)) continue;
76    out.push({
77      id: raw.id,
78      name: raw.title,
79      desc: typeof raw.summary === 'string' ? raw.summary : '',
80      type: 'remote',
81      body: '',
82      source,
83    });
84  }
85  return out;
86}
87

1/**
2 * BM25F over name, description and body head, with a score floor.
3 *
4 * Measured on 33 real prompts (2026-09-28): 46 % of shown memories useful at
5 * query p95 21.6 ms in Python over 2,035 memories. The floor makes a weak
6 * prompt show nothing instead of noise.
7 */
8
9import { tokenize } from './tokenize.js';
10import type { Hit, MemoryDoc } from './types.js';
11
12const FIELDS = { name: 3.0, desc: 2.0, body: 1.0 } as const;
13type Field = keyof typeof FIELDS;
14const K1 = 1.2;
15const B = 0.75;
16
17/** Below this a match is noise. */
18export const FLOOR = 7.0;
19/** A match needs this many query words, or one rare word. */
20export const MIN_TERMS = 2;
21/** idf at or above this makes one word enough. */
22export const RARE = 4.0;
23/** Session digests and index hubs match everything; they count half. */
24const PENALTY = /(handoff|hub_)/;
25
26export class LensIndex {
27  readonly size: number;
28  private readonly docs: MemoryDoc[];
29  private readonly tf: Array<Record<Field, Map<string, number>>> = [];
30  private readonly len: Record<Field, number[]> = { name: [], desc: [], body: [] };
31  private readonly avg: Record<Field, number> = { name: 1, desc: 1, body: 1 };
32  private readonly df = new Map<string, number>();
33
34  constructor(docs: MemoryDoc[]) {
35    this.docs = docs;
36    this.size = docs.length;
37    for (const doc of docs) {
38      const seen = new Set<string>();
39      const per = { name: new Map(), desc: new Map(), body: new Map() } as Record<Field, Map<string, number>>;
40      for (const f of Object.keys(FIELDS) as Field[]) {
41        const text = f === 'name' ? doc.name.replace(/[_-]/g, ' ') + ' ' + doc.name : doc[f];
42        const toks = tokenize(text);
43        this.len[f].push(toks.length);
44        for (const t of toks) {
45          per[f].set(t, (per[f].get(t) ?? 0) + 1);
46          seen.add(t);
47        }
48      }
49      for (const t of seen) this.df.set(t, (this.df.get(t) ?? 0) + 1);
50      this.tf.push(per);
51    }
52    for (const f of Object.keys(FIELDS) as Field[]) {
53      const v = this.len[f];
54      this.avg[f] = v.length ? v.reduce((a, b) => a + b, 0) / v.length || 1 : 1;
55    }
56  }
57
58  /** Top k hits above the floor, best first. `skip` ids are never returned. */
59  query(text: string, k = 3, skip: ReadonlySet<string> = new Set()): Hit[] {
60    const q = [...new Set(tokenize(text))];
61    if (!q.length || !this.size) return [];
62    const idf = new Map<string, number>();
63    for (const t of q) {
64      const df = this.df.get(t);
65      if (df) idf.set(t, Math.log(1 + (this.size - df + 0.5) / (df + 0.5)));
66    }
67    if (!idf.size) return [];
68    const hits: Hit[] = [];
69    this.tf.forEach((per, i) => {
70      const doc = this.docs[i];
71      if (skip.has(doc.id)) return;
72      let score = 0;
73      let rare = false;
74      const why: Array<[string, number]> = [];
75      for (const [t, w0] of idf) {
76        let w = 0;
77        for (const f of Object.keys(FIELDS) as Field[]) {
78          const tf = per[f].get(t);
79          if (tf) w += (FIELDS[f] * tf) / (1 - B + (B * this.len[f][i]) / this.avg[f]);
80        }
81        if (!w) continue;
82        const part = (w0 * w * (K1 + 1)) / (w + K1);
83        score += part;
84        why.push([t, part]);
85        if (w0 >= RARE) rare = true;
86      }
87      if (PENALTY.test(doc.id.split('/').pop() ?? '')) score *= 0.5;
88      if (score >= FLOOR && (why.length >= MIN_TERMS || rare)) {
89        why.sort((a, b) => b[1] - a[1]);
90        hits.push({ doc, score, why: why.map(([t]) => t) });
91      }
92    });
93    hits.sort((a, b) => b.score - a.score);
94    return hits.slice(0, k);
95  }
96}
97

1// memory-lens: types.ts - shared type definitions
2
3/** One memory the lens can recall: a local memory file or a mirrored remote title. */
4export interface MemoryDoc {
5  /** Stable id: the file path for a local memory, the store id for a mirrored one. */
6  id: string;
7  /** Frontmatter name, or the file name without .md. */
8  name: string;
9  /** Frontmatter description (one line). */
10  desc: string;
11  /** Frontmatter type (feedback, project, reference, user), or "" when untyped. */
12  type: string;
13  /** The first part of the body, used for matching only, never shown. */
14  body: string;
15  /** First image path the memory mentions, if any. */
16  image?: string;
17  /** Where it came from: "local" for a memory file, else the mirror's store label. */
18  source: string;
19}
20
21/** One recalled memory with why it matched. */
22export interface Hit {
23  doc: MemoryDoc;
24  score: number;
25  /** The query words it matched, strongest first. */
26  why: string[];
27}
28
29/** Local, never-committed settings read at session start. */
30export interface LensPrivate {
31  /** Words that mark a memory as client data: its title is hidden on screen. */
32  clientTerms: string[];
33}
34
35/** One entry in a mirror file written by a private job outside this mod. */
36export interface MirrorItem {
37  id: string;
38  title: string;
39  summary?: string;
40  source?: string;
41}
42

1/**
2 * Tokenizer for memory-lens: lower-case words, compound slugs split, filler dropped.
3 *
4 * The stop list carries conversational filler and profanity as well as grammar
5 * words: measured on real prompts, "fix all" or "this is ugly" otherwise matched
6 * whichever long memory repeated those words most.
7 */
8
9const WORD = /[a-z0-9][a-z0-9_.+-]*[a-z0-9]|[a-z0-9]/g;
10
11export const STOP = new Set(
12  (
13    'a an the and or but if then so of to in on at for with from by as is are was were be been it its this that ' +
14    'these those i you we he she they me my our your us them do does did done not no yes can could should would will ' +
15    'just also very more most much many some any all what which who whom why how when where there here into out up ' +
16    'down over under again about than too only own same other such each both few new now get got make made go going ' +
17    'want need like see show let lets please pls ok okay im dont doesnt didnt cant wont thats whats btw etc ' +
18    'one two image seems still fuck fucked fucking wtf shit shitty ugly damn think maybe better good well part ' +
19    'first last right have give everything never continue really missing fix ahead tell him her use find alot lot ' +
20    'thing things stuff way work because since yet even though already always something anything nothing doing ' +
21    'said say says theres wasnt isnt arent explain'
22  ).split(' '),
23);
24
25/** Tokens for matching. A slug like "og-card" also yields "og" and "card". */
26export function tokenize(text: string): string[] {
27  const out: string[] = [];
28  for (const raw of text.toLowerCase().match(WORD) ?? []) {
29    const t = raw.replace(/^[.+_-]+|[.+_-]+$/g, '');
30    if (t.length < 2 || STOP.has(t)) continue;
31    out.push(t);
32    if (/[-_.]/.test(t)) {
33      for (const part of t.split(/[-_.]/)) {
34        if (part.length > 1 && !STOP.has(part)) out.push(part);
35      }
36    }
37  }
38  return out;
39}
40