lossless-compact

1import type { On, PluginOptions, Register, SessionCompactResult, SessionMessage, TurnCompleteInput } from 'claude-code';
2
3import { FileArchive, type TextFs } from '../src/archive/file-store.js';
4import type { ArchiveRecord, ArchiveStore } from '../src/archive/types.js';
5import type { ActionDecision } from '../src/core/actions.js';
6import { RulesetClassifier } from '../src/classifiers/ruleset.js';
7import { JevClassifier, type JevQuestionStyle } from '../src/classifiers/jev.js';
8import type { Classifier } from '../src/classifiers/types.js';
9import { reductionRatio } from '../src/compact.js';
10import { buildLedger, eventTokens } from '../src/core/events.js';
11import { findConstraints } from '../src/core/rules.js';
12import { optimize, type OptimizeReport, type OptimizeResult } from '../src/engine/optimize.js';
13import { REHYDRATION_PREFACE, rehydrateForPrompt } from '../src/engine/rehydrate.js';
14import { estimateTokens } from '../src/state.js';
15import {
16  classifierWithKeeps,
17  parseReview,
18  reviewCandidates,
19  reviewQuestion,
20  reviewQuestionWithContext,
21  type ReviewVerdict,
22} from '../src/engine/review.js';
23import type { Message } from '../src/types.js';
24import {
25  decisionLogLines,
26  jevAsker,
27  resolveHookConfig,
28  summarize,
29  toSessionMessages,
30  type HookConfig,
31  type HookFetch,
32} from './fast-jev.js';
33
34/**
35 * The Claude Code adapter for the context optimizer. Compaction goes through
36 * `optimize` (rules, classifier, archive); the archive lives under the
37 * project in `.lossless-compact/`; `/lossless` inspects and restores it.
38 */
39
40export type ClassifierChoice = 'auto' | 'jev' | 'ruleset';
41
42export type LosslessCompactConfig = HookConfig & {
43  classifier: ClassifierChoice;
44  questionStyle: JevQuestionStyle;
45  archiveDir: string;
46  safetyMargin: number;
47  sketches: boolean;
48  redact: boolean;
49  markRemovedCalls: boolean;
50  /** Retrieve relevant archived content into each prompt automatically. */
51  autoRetrieve: boolean;
52  /** Characters of retrieved content per prompt. */
53  retrieveBudgetChars: number;
54  /**
55   * Compact once the live context holds this many tokens (0 = off). A count,
56   * not a share of the model's window: "big" does not change when the window
57   * does. Default 120000.
58   */
59  compactAtTokens: number;
60  /** Write the exact pre-compaction transcript under `.lossless-compact/snapshots/`. */
61  snapshot: boolean;
62  /** Insert a note into the compacted transcript saying where the full history is. */
63  noteRemoved: boolean;
64  /**
65   * Log the full compaction report and every per-call classifier decision to
66   * the transcript. Off: one plain line per compaction or retrieval.
67   */
68  verbose: boolean;
69};
70
71const DEFAULTS = {
72  classifier: 'auto' as ClassifierChoice,
73  questionStyle: 'useful' as JevQuestionStyle,
74  archiveDir: '.lossless-compact',
75  safetyMargin: 0,
76  sketches: true,
77  redact: true,
78  markRemovedCalls: true,
79  autoRetrieve: true,
80  retrieveBudgetChars: 6000,
81  compactAtTokens: 120_000,
82  /** The percent trigger is off unless set; `compactAtTokens` is the default trigger. */
83  compactAtPercent: 0,
84  snapshot: true,
85  noteRemoved: true,
86  verbose: false,
87};
88
89function optionBoolean(options: PluginOptions, key: string, fallback: boolean): boolean {
90  const value = options[key];
91  return typeof value === 'boolean' ? value : fallback;
92}
93
94export function resolveLosslessCompactConfig(options: PluginOptions): LosslessCompactConfig {
95  const base = resolveHookConfig(options);
96  const classifier = options['classifier'];
97  const style = options['questionStyle'];
98  const dir = options['archiveDir'];
99  const margin = options['safetyMargin'];
100  const config: LosslessCompactConfig = {
101    ...base,
102    classifier:
103      classifier === 'jev' || classifier === 'ruleset' || classifier === 'auto'
104        ? classifier
105        : classifier === 'heuristic' // the ruleset's name up to 0.5.0
106          ? 'ruleset'
107          : DEFAULTS.classifier,
108    questionStyle: style === 'upstream' || style === 'useful' ? style : DEFAULTS.questionStyle,
109    archiveDir: typeof dir === 'string' && dir.length > 0 ? dir : DEFAULTS.archiveDir,
110    safetyMargin: typeof margin === 'number' && Number.isFinite(margin) ? margin : DEFAULTS.safetyMargin,
111    sketches: optionBoolean(options, 'sketches', DEFAULTS.sketches),
112    redact: optionBoolean(options, 'redact', DEFAULTS.redact),
113    markRemovedCalls: optionBoolean(options, 'markRemovedCalls', DEFAULTS.markRemovedCalls),
114    autoRetrieve: optionBoolean(options, 'autoRetrieve', DEFAULTS.autoRetrieve),
115    retrieveBudgetChars:
116      typeof options['retrieveBudgetChars'] === 'number' && Number.isFinite(options['retrieveBudgetChars'])
117        ? Math.max(0, Math.min(30_000, options['retrieveBudgetChars']))
118        : DEFAULTS.retrieveBudgetChars,
119    snapshot: optionBoolean(options, 'snapshot', DEFAULTS.snapshot),
120    noteRemoved: optionBoolean(options, 'noteRemoved', DEFAULTS.noteRemoved),
121    verbose: optionBoolean(options, 'verbose', DEFAULTS.verbose),
122    compactAtTokens:
123      typeof options['compactAtTokens'] === 'number' && Number.isFinite(options['compactAtTokens'])
124        ? Math.max(0, options['compactAtTokens'])
125        : DEFAULTS.compactAtTokens,
126    compactAtPercent:
127      typeof options['compactAtPercent'] === 'number' && Number.isFinite(options['compactAtPercent'])
128        ? Math.max(0, options['compactAtPercent'])
129        : DEFAULTS.compactAtPercent,
130  };
131  // Only an explicit threshold overrides the classifier's own calibration.
132  if (typeof options['keepThreshold'] !== 'number') delete config.keepThreshold;
133  return config;
134}
135
136/**
137 * Whether the live context is big enough to compact: by token count (the
138 * default trigger) or, when enabled, by share of the window. When the host
139 * reports no absolute count, it is derived from the percent and the window.
140 */
141export function shouldCompact(
142  context: { tokens?: number; percent?: number; window?: number },
143  config: Pick<LosslessCompactConfig, 'compactAtTokens' | 'compactAtPercent'>,
144): boolean {
145  const percent = context.percent ?? 0;
146  const tokens = contextTokens(context);
147  if (config.compactAtTokens > 0 && tokens !== undefined && tokens >= config.compactAtTokens) return true;
148  if (config.compactAtPercent > 0 && percent >= config.compactAtPercent) return true;
149  return false;
150}
151
152/** The live context in tokens: the host's count, or its percent of the window. */
153export function contextTokens(context: { tokens?: number; percent?: number; window?: number }): number | undefined {
154  return context.tokens ?? (context.window && context.percent !== undefined ? (context.percent / 100) * context.window : undefined);
155}
156
157/**
158 * When the auto-compaction fires. A crossing of the threshold fires once: a
159 * compaction that was skipped or fell back leaves the context where it was,
160 * and firing again after every turn would loop. Re-armed once the context has
161 * dropped below the threshold (the compaction worked) or grown by a quarter
162 * since the attempt (there is new material to remove).
163 */
164export class AutoCompactTrigger {
165  private attemptedAt: number | undefined;
166
167  constructor(private readonly config: Pick<LosslessCompactConfig, 'compactAtTokens' | 'compactAtPercent'>) {}
168
169  due(context: { tokens?: number; percent?: number; window?: number }): boolean {
170    if (!shouldCompact(context, this.config)) {
171      this.attemptedAt = undefined;
172      return false;
173    }
174    const tokens = contextTokens(context) ?? 0;
175    if (this.attemptedAt !== undefined && tokens < this.attemptedAt * 1.25) return false;
176    this.attemptedAt = tokens;
177    return true;
178  }
179}
180
181/** The engine's `$.fs` as the archive's file system. */
182export function engineFs($: {
183  fs: {
184    read: (path: string) => Promise<string>;
185    write: (path: string, text: string) => Promise<void>;
186    exists: (path: string) => Promise<boolean>;
187    list: (path?: string) => Promise<readonly { name: string }[]>;
188  };
189}): TextFs {
190  return {
191    read: (path) => $.fs.read(path),
192    write: (path, text) => $.fs.write(path, text),
193    exists: (path) => $.fs.exists(path),
194    async list(dir) {
195      if (!(await $.fs.exists(dir))) return [];
196      return (await $.fs.list(dir)).map((entry) => entry.name);
197    },
198  };
199}
200
201/** Resolves `auto` to the classifier this session can actually use. */
202export function resolvedClassifierName(
203  config: Pick<LosslessCompactConfig, 'classifier'>,
204  apiKey: string | undefined,
205): 'jev' | 'ruleset' {
206  return config.classifier === 'auto' ? (apiKey ? 'jev' : 'ruleset') : config.classifier;
207}
208
209/**
210 * One line for a compaction that ran on the ruleset only because no TypeSafe
211 * key was found (`classifier: auto`); a chosen `ruleset` gets none. The
212 * figures are docs/evals.md's: must-keep results left in place verbatim, 3 of
213 * 6 against Jev's 6 of 6, every probe recoverable either way.
214 */
215export function noKeyNotice(
216  config: Pick<LosslessCompactConfig, 'classifier'>,
217  apiKey: string | undefined,
218): string | undefined {
219  if (config.classifier !== 'auto' || apiKey) return undefined;
220  return (
221    'No TypeSafe key found (TYPESAFE_API_KEY, or the plugin\'s apiKey option), so compaction runs the local ruleset: ' +
222    'same archive, everything restorable, about half as good as Jev at keeping must-keep results in place verbatim ' +
223    '(3 of 6 vs 6 of 6 on the eval). Add a key to switch.'
224  );
225}
226
227/**
228 * One line for Jev requests that go somewhere other than TypeSafe (the
229 * `baseUrl` option). That address receives the key and the conversation, so
230 * it is named rather than silent; only scheme and host are shown, so a token
231 * or password in the URL never lands in a log or the compaction note.
232 */
233export function endpointNotice(
234  config: Pick<LosslessCompactConfig, 'classifier' | 'baseUrl'>,
235  apiKey: string | undefined,
236): string | undefined {
237  if (!config.baseUrl || resolvedClassifierName(config, apiKey) !== 'jev') return undefined;
238  let where: string;
239  try {
240    const url = new URL(config.baseUrl);
241    where = `${url.protocol}//${url.host}`;
242  } catch {
243    where = 'an address that is not a valid URL';
244  }
245  return `Jev requests go to ${where} (the plugin's baseUrl option), not TypeSafe: the key and the conversation are sent there.`;
246}
247
248/** Picks the classifier from the config and whether a key is at hand. */
249export function chooseClassifier(
250  config: LosslessCompactConfig,
251  fetchFn: HookFetch,
252  apiKey: string | undefined,
253): Classifier {
254  if (resolvedClassifierName(config, apiKey) === 'jev') {
255    if (!apiKey) throw new Error('TYPESAFE_API_KEY is not configured');
256    return new JevClassifier(jevAsker(fetchFn, apiKey, config.model, config.baseUrl), {
257      questionStyle: config.questionStyle,
258    });
259  }
260  return new RulesetClassifier();
261}
262
263export interface SessionOptimization {
264  result: OptimizeResult;
265  messages: SessionMessage[];
266}
267
268/** Runs the optimizer over a session transcript; throws when the classifier cannot run. */
269export async function optimizeSession(
270  messages: readonly SessionMessage[],
271  config: LosslessCompactConfig,
272  classifier: Classifier,
273  archive: ArchiveStore,
274  sessionId: string,
275): Promise<SessionOptimization> {
276  const result = await optimize(messages, {
277    ...config,
278    sessionId,
279    archive,
280    classifier,
281    policy: { safetyMargin: config.safetyMargin },
282    sketches: config.sketches,
283    redact: config.redact,
284    markRemovedCalls: config.markRemovedCalls,
285  });
286  return { result, messages: toSessionMessages(messages, result.messages) };
287}
288
289/** The tool interactions a compaction took out of the context (each one a stub or a trimmed result now). */
290function archivedUnits(report: OptimizeReport): number {
291  const { actions } = report;
292  return actions.ARCHIVE_ONLY + actions.KEEP_HEAD_TAIL + actions.RERUN_ON_DEMAND + actions.DROP_REDUNDANT;
293}
294
295/** One line per non-trivial protection or action count, for the verbose log. */
296export function reportLines(report: OptimizeReport): string[] {
297  const actions = Object.entries(report.actions)
298    .filter(([, n]) => n > 0)
299    .map(([action, n]) => `${action}=${n}`)
300    .join(' ');
301  const protections = Object.entries(report.protections)
302    .map(([rule, n]) => `${rule}=${n}`)
303    .join(' ');
304  return [
305    `lossless-compact ${report.compactionId}: ${report.classifier}; ~${report.tokens.before}→${report.tokens.after} tokens; archived ${report.tokens.archived} tokens in ${archivedUnits(report)} units`,
306    `actions: ${actions || '(none)'}`,
307    `protected: ${protections || '(none)'}; constraints ${report.constraints}; duplicates ${report.duplicates}; unscored ${report.unscored}; secrets redacted ${report.redactedSecrets}`,
308  ];
309}
310
311/* ------------------------------------------------------------- one-liners */
312
313/** A token count the way a person reads it: 850, 8.4k, 242k, 1.2M. */
314export function shortCount(n: number): string {
315  if (n < 1000) return String(Math.round(n));
316  if (n < 9_950) return `${(n / 1000).toFixed(1).replace(/\.0$/, '')}k`;
317  if (n < 999_500) return `${Math.round(n / 1000)}k`;
318  return `${(n / 1_000_000).toFixed(1).replace(/\.0$/, '')}M`;
319}
320
321/**
322 * The line a finished compaction leaves in the transcript and the toast:
323 * how much smaller the context got, what left it, and that nothing was
324 * summarized. The full report is behind the `verbose` option and `/lossless`.
325 */
326export function compactedLine(report: OptimizeReport, reviewed = false): string {
327  const { before, after } = report.tokens;
328  const smaller = before > 0 ? Math.round((1 - after / before) * 100) : 0;
329  const units = archivedUnits(report);
330  return (
331    `lossless-compact: ~${shortCount(before)} → ${shortCount(after)} tokens (${smaller}% smaller); ` +
332    `archived ${units} old tool result${units === 1 ? '' : 's'}, nothing summarized${reviewed ? ' (reviewed)' : ''} · /lossless to browse or restore`
333  );
334}
335
336/** What fell back to Claude Code's summary, and why, in one line (an error message can be long). */
337export function fallbackLine(reason: string, maxReason = 160): string {
338  return `lossless-compact: used Claude's built-in summary instead (${clip(reason, maxReason)})`;
339}
340
341/** Why there was too little to remove, for the skip and fallback lines. */
342export function tooLittleReason(ratio: number, minimum: number): string {
343  return `only ${percent(ratio)} of the context was removable, the minimum is ${percent(minimum)}`;
344}
345
346/** `text` on one line, at most `max` characters; the end gives way. */
347function clip(text: string, max: number): string {
348  const line = text.replace(/\s+/g, ' ').trim();
349  return line.length <= max ? line : `${line.slice(0, max - 1)}…`;
350}
351
352/** A path short enough to read in a line: its last two segments when it is long. */
353function clipPath(path: string, max = 40): string {
354  const line = path.trim();
355  if (line.length <= max) return line;
356  const tail = line.split(/[\\/]+/).filter(Boolean).slice(-2).join('/');
357  return clip(`…/${tail}`, max);
358}
359
360/** An archived record in a few words: `Read src/a.ts`, `Bash "npm test"`. */
361export function describeRecord(record: Pick<ArchiveRecord, 'kind' | 'toolName' | 'metadata'>): string {
362  const what =
363    record.toolName ??
364    (record.kind === 'user_text' ? 'user message' : record.kind === 'assistant_text' ? 'assistant message' : record.kind);
365  for (const key of ['file_path', 'path', 'url', 'pattern']) {
366    const value = record.metadata[key];
367    if (typeof value === 'string' && value.trim()) return `${what} ${key === 'pattern' ? `"${clip(value, 30)}"` : clipPath(value)}`;
368  }
369  const command = record.metadata['command'];
370  if (typeof command === 'string' && command.trim()) return `${what} "${clip(command, 30)}"`;
371  return what;
372}
373
374/** The line automatic retrieval leaves: what came back into this prompt, and its size. */
375export function retrievedLine(
376  records: readonly Pick<ArchiveRecord, 'id' | 'kind' | 'toolName' | 'metadata'>[],
377  tokens: number,
378  withIds = false,
379): string {
380  const n = records.length;
381  const what = records.map((record) => describeRecord(record) + (withIds ? ` [${record.id}]` : '')).join(', ');
382  return `lossless-compact: recalled ${n} archived result${n === 1 ? '' : 's'} into this prompt: ${what} (~${shortCount(tokens)} tokens)`;
383}
384
385/* ------------------------------------------------------------- snapshot & note */
386
387const SNAPSHOT_MAX_CHARS = 3.5 * 1024 * 1024;
388
389/** A session message without its engine handle, as plain library data. */
390function plainMessage(message: SessionMessage): Message {
391  const copy: Message = {
392    role: message.role,
393    text: message.text,
394    toolUses: message.toolUses.map((tool) => {
395      const use: Message['toolUses'][number] = { tool_use_id: tool.tool_use_id, tool: tool.tool, input: tool.input };
396      if (tool.text !== undefined) use.text = tool.text;
397      if (tool.isError) use.isError = true;
398      return use;
399    }),
400  };
401  if (message.toolResults && message.toolResults.length > 0) {
402    copy.toolResults = message.toolResults.map((result) => ({
403      tool_use_id: result.tool_use_id,
404      text: result.text,
405      isError: result.isError,
406    }));
407  }
408  return copy;
409}
410
411/**
412 * Writes the exact transcript that was about to be compacted:
413 * `<root>/snapshots/<session>/<compaction>.json`, split into
414 * `<compaction>-<n>.json` parts when it would exceed the host's 4 MiB write
415 * cap. Returns the paths written, the manifest first.
416 */
417export async function writeSnapshot(
418  fs: TextFs,
419  root: string,
420  sessionId: string,
421  compactionId: string,
422  messages: readonly SessionMessage[],
423  at: string,
424): Promise<string[]> {
425  const dir = `${root.replace(/[\\/]+$/, '')}/snapshots/${encodeURIComponent(sessionId)}`;
426  const serialised = messages.map((message) => JSON.stringify(plainMessage(message)));
427  const parts: string[][] = [[]];
428  let chars = 0;
429  for (const item of serialised) {
430    if (parts[parts.length - 1]!.length > 0 && chars + item.length + 2 > SNAPSHOT_MAX_CHARS) {
431      parts.push([]);
432      chars = 0;
433    }
434    parts[parts.length - 1]!.push(item);
435    chars += item.length + 2;
436  }
437  const manifest = `${dir}/${compactionId}.json`;
438  if (parts.length === 1) {
439    await fs.write(
440      manifest,
441      `{"version":1,"sessionId":${JSON.stringify(sessionId)},"compactionId":${JSON.stringify(compactionId)},"at":${JSON.stringify(at)},"messages":[${parts[0]!.join(',')}]}`,
442    );
443    return [manifest];
444  }
445  const written: string[] = [];
446  for (let i = 0; i < parts.length; i++) {
447    const path = `${dir}/${compactionId}-${i + 1}.json`;
448    await fs.write(path, `{"version":1,"part":${i + 1},"of":${parts.length},"messages":[${parts[i]!.join(',')}]}`);
449    written.push(path);
450  }
451  await fs.write(
452    manifest,
453    JSON.stringify({
454      version: 1,
455      sessionId,
456      compactionId,
457      at,
458      messages: messages.length,
459      parts: written.map((p) => p.slice(dir.length + 1)),
460    }),
461  );
462  return [manifest, ...written];
463}
464
465/** Where Claude Code keeps this session's raw log, if it can be found from the sandbox. */
466export async function rawSessionLogPath(
467  fs: Pick<TextFs, 'exists'>,
468  home: string | undefined,
469  cwd: string,
470  sessionId: string,
471): Promise<string | undefined> {
472  if (!home) return undefined;
473  const base = `${home.replace(/[\\/]+$/, '')}/.claude/projects`;
474  const encoded = (dir: string): string => dir.replace(/[^A-Za-z0-9]/g, '-');
475  const candidates = new Set(
476    [cwd, cwd.toLowerCase(), cwd.replace(/^([A-Za-z]):/, (m) => m.toLowerCase())].map(encoded),
477  );
478  for (const name of candidates) {
479    const path = `${base}/${name}/${sessionId}.jsonl`;
480    try {
481      if (await fs.exists(path)) return path;
482    } catch {
483      // not findable from here
484    }
485  }
486  return undefined;
487}
488
489/**
490 * The one message inserted into a compacted transcript: what was removed and
491 * exactly where the full history is. Nothing in it claims the removed
492 * content is still present.
493 */
494export function compactionNote(details: {
495  at: string;
496  compactionId: string;
497  report: OptimizeReport;
498  snapshotPath?: string;
499  archiveDir: string;
500  rawLogPath?: string;
501  /** True when Claude Code's built-in summary replaced the transcript after all (a fallback). */
502  summarized?: boolean;
503  /** `noKeyNotice`: why the ruleset ran, when it ran for want of a key. */
504  classifierNote?: string;
505}): string {
506  const { tokens, messages, classifier, ms } = details.report;
507  const removed = archivedUnits(details.report);
508  const units = `${removed} tool interaction${removed === 1 ? '' : 's'} (~${tokens.archived.toLocaleString('en-US')} tokens)`;
509  // The toast never renders in the VS Code extension (the host runs it as a headless
510  // session), so the note is the one place the user can see what ran and how it went.
511  const fewer = tokens.before > 0 ? Math.round((1 - tokens.after / tokens.before) * 100) : 0;
512  const outcome = details.summarized
513    ? `Classifier: ${classifier} · ${ms} ms (its ~${tokens.before.toLocaleString('en-US')}→${tokens.after.toLocaleString('en-US')} token result was replaced by the summary).`
514    : `Classifier: ${classifier} · ~${tokens.before.toLocaleString('en-US')}→${tokens.after.toLocaleString('en-US')} tokens (${fewer}% fewer) · ${messages.before}→${messages.after} messages · ${ms} ms.`;
515  const lines = [
516    details.summarized
517      ? `[lossless-compact] This conversation was compacted at ${details.at} by Claude Code's built-in summary; the message above is a paraphrase, not the original text. Before the summary was written, lossless-compact (compaction ${details.compactionId}) archived ${units} verbatim${
518          details.snapshotPath ? ' and saved the exact pre-compaction transcript' : ''
519        }. If you need anything the summary lost, the full history is on disk:`
520      : `[lossless-compact] This conversation was compacted at ${details.at} (compaction ${details.compactionId}): ${units} were removed from the active context and archived verbatim. Nothing was summarized or paraphrased; user and assistant messages are untouched. If you need any removed message, the full history is on disk:`,
521    outcome,
522  ];
523  if (details.classifierNote) lines.push(details.classifierNote);
524  if (details.snapshotPath) {
525    lines.push(`- Exact pre-compaction transcript (JSON array of messages): ${details.snapshotPath} — grep it, or read a slice.`);
526  }
527  lines.push(
528    details.summarized
529      ? `- Every archived item, with the reason it was removed: ${details.archiveDir}/archive/ — grep for a file path, an error line or a value there to find the record and its id (e_…).`
530      : `- Every archived item, with the reason it was removed: ${details.archiveDir}/archive/ — a stub in this transcript names its id (e_…); grep for that id under ${details.archiveDir}/archive to find the record.`,
531  );
532  if (details.rawLogPath) {
533    lines.push(`- Raw Claude Code session log, never modified by compaction: ${details.rawLogPath}`);
534  }
535  lines.push('The user can also run /lossless why <id>, /lossless show <id> or /lossless restore <id>.');
536  return lines.join('\n');
537}
538
539/** The compacted transcript with the note inserted after the pinned first message. */
540export function withCompactionNote(messages: readonly Message[], note: string): Message[] {
541  const noteMessage: Message = { role: 'user', text: note, toolUses: [] };
542  if (messages.length === 0) return [noteMessage];
543  return [messages[0]!, noteMessage, ...messages.slice(1)];
544}
545
546/**
547 * The compacted transcript with every engine handle removed. A message handed
548 * back with its handle "stands as the engine has it" — its own record, with
549 * its original parent link — and on `--resume` Claude Code 2.1.278 rebuilds
550 * the conversation by walking those links from the newest message: the first
551 * kept message after an evicted one leads straight back into the pre-boundary
552 * log, and the whole compaction is undone (seen live: the second /compact of
553 * a resumed session saw every archived read again, twice). Handle-less
554 * messages are built afresh by the host and chained after the boundary. The
555 * price is the kept assistant messages' thinking blocks, which a summary loses
556 * too.
557 */
558export function rechain(messages: readonly SessionMessage[]): SessionMessage[] {
559  const out: SessionMessage[] = [];
560  for (const message of messages) {
561    const { handle: _handle, ...rest } = message;
562    // A thinking-only assistant message is nothing without its handle; the
563    // host would store it as "(no content)".
564    if (rest.text.trim().length === 0 && rest.toolUses.length === 0 && (rest.toolResults ?? []).length === 0) continue;
565    out.push(rest);
566  }
567  return out;
568}
569
570/** The same insertion over the host's own compacted transcript (its summary comes first). */
571export function withCompactionNoteSession(messages: readonly SessionMessage[], note: string): SessionMessage[] {
572  const noteMessage: SessionMessage = { role: 'user', text: note, toolUses: [] };
573  if (messages.length === 0) return [noteMessage];
574  return [messages[0]!, noteMessage, ...messages.slice(1)];
575}
576
577/**
578 * Upstream #53: a classifier that keeps none of the results it scored is
579 * either miscalibrated (that issue: everything under 0.3 with a 0.5
580 * threshold, on 16 real sessions) or right about a stretch whose tool output
581 * was all disposable (a session that only read files nobody referred to
582 * again). The scores cannot tell the two apart; `reviewKeepNothing` asks a
583 * model that can see the conversation, and failing that the user. Fewer than
584 * five scored results is too few to judge. The best result score comes back
585 * so the log can say how far off it was.
586 */
587export function suspectCalibration(
588  actions: readonly ActionDecision[],
589  classified: number,
590): { suspect: boolean; best: number } {
591  let best = 0;
592  let kept = 0;
593  for (const decision of actions) {
594    if (!decision.scores) continue;
595    if (decision.action === 'KEEP_VERBATIM') kept += 1;
596    best = Math.max(best, decision.scores.keepResult);
597  }
598  return { suspect: classified >= 5 && kept === 0, best };
599}
600
601export type KeepNothingReview = {
602  decision: 'proceed' | 'keep' | 'fallback';
603  /** Classifier call ids (`t3`) to keep verbatim on the re-run; only with `keep`. */
604  keep: string[];
605  /** One sentence for the log and the toast: what was found, who reviewed it, what was decided. */
606  why: string;
607};
608
609const trustKey = (sessionId: string): string => `lossless-compact:trust:${sessionId}`;
610
611/** Whether the user already said "remove and don't ask again" this session. */
612export async function trustedForSession(
613  $: { store: { get: (key: string) => Promise<unknown> } },
614  sessionId: string,
615): Promise<boolean> {
616  try {
617    return (await $.store.get(trustKey(sessionId))) === true;
618  } catch {
619    return false;
620  }
621}
622
623export const REVIEW_ANSWERS = {
624  remove: 'Remove them (archived, restorable)',
625  trust: "Remove, and don't ask again this session",
626  summary: "Use Claude's summary instead",
627} as const;
628
629type ReviewHost = {
630  model: {
631    fork: (request: { prompt: string }) => Promise<{ text: string; usage: { input_tokens: number; output_tokens: number } } | null>;
632    complete: (request: { model: string; prompt: string; maxTokens?: number }) => Promise<string>;
633  };
634  ui: {
635    ask: (question: string, options?: { options?: readonly string[]; header?: string }) => Promise<string>;
636    log: (text: string) => void;
637  };
638  store: { get: (key: string) => Promise<unknown>; set: (key: string, value: unknown) => Promise<void> };
639};
640
641/**
642 * The second opinion when a classifier kept nothing. In order: the session's
643 * own model over its own transcript (`$.model.fork`, cache-shared, so it has
644 * read the conversation), then a small model with the user's turns quoted
645 * (`$.model.complete`), then the user (`$.ui.ask`). Only a "drop_all" from a
646 * model proceeds without asking; "keep_some" re-runs with those kept; anything
647 * else asks. With no one to ask (headless) the built-in summary — with the
648 * note — is the safe answer.
649 */
650export async function reviewKeepNothing(
651  $: ReviewHost,
652  sessionId: string,
653  messages: readonly Message[],
654  config: Pick<LosslessCompactConfig, 'preserveRecentMessages'>,
655  result: OptimizeResult,
656  threshold: number,
657  best: number,
658): Promise<KeepNothingReview> {
659  const candidates = reviewCandidates(messages, result.decisions, config.preserveRecentMessages ?? 6);
660  const ids = new Set(candidates.map((c) => c.id));
661  const facts = `the classifier kept none of the ${candidates.length} results it scored (best ${best.toFixed(2)}, threshold ${threshold})`;
662  let verdict: ReviewVerdict | undefined;
663  let reviewer = '';
664  try {
665    const forked = await $.model.fork({ prompt: reviewQuestion(candidates, threshold) });
666    if (forked) {
667      verdict = parseReview(forked.text, ids);
668      reviewer = `the session's model (${forked.usage.input_tokens + forked.usage.output_tokens} tokens)`;
669    }
670  } catch (error) {
671    $.ui.log(`lossless-compact review: fork unavailable (${error instanceof Error ? error.message : String(error)})`);
672  }
673  if (!verdict) {
674    try {
675      const text = await $.model.complete({
676        model: 'haiku',
677        prompt: reviewQuestionWithContext(messages, candidates, threshold),
678        maxTokens: 400,
679      });
680      verdict = parseReview(text, ids);
681      reviewer = 'haiku (user turns only)';
682    } catch (error) {
683      $.ui.log(`lossless-compact review: completion unavailable (${error instanceof Error ? error.message : String(error)})`);
684    }
685  }
686  if (verdict?.verdict === 'drop_all') {
687    return { decision: 'proceed', keep: [], why: `${facts}; ${reviewer} read the conversation and agreed they are disposable${verdict.reason ? `: ${verdict.reason}` : ''}` };
688  }
689  if (verdict?.verdict === 'keep_some') {
690    return { decision: 'keep', keep: verdict.keep, why: `${facts}; ${reviewer} asked to keep ${verdict.keep.join(', ')}${verdict.reason ? `: ${verdict.reason}` : ''}` };
691  }
692  const doubt = verdict ? `${reviewer} could not confirm that is right${verdict.reason ? ` (${verdict.reason})` : ''}` : 'no model could review it';
693  try {
694    const answer = await $.ui.ask(
695      `lossless-compact: ${facts}, and ${doubt}. Every removed result stays in the archive and can be restored by id. Remove them, or use Claude's summary instead?`,
696      { header: 'lossless-compact', options: [REVIEW_ANSWERS.remove, REVIEW_ANSWERS.trust, REVIEW_ANSWERS.summary] },
697    );
698    if (answer === REVIEW_ANSWERS.trust) {
699      try {
700        await $.store.set(trustKey(sessionId), true);
701      } catch {
702        // then it asks again next time; harmless
703      }
704      return { decision: 'proceed', keep: [], why: `${facts}; ${doubt}; the user chose to remove them and not be asked again this session` };
705    }
706    if (answer === REVIEW_ANSWERS.remove) return { decision: 'proceed', keep: [], why: `${facts}; ${doubt}; the user chose to remove them` };
707    return { decision: 'fallback', keep: [], why: `${facts}; ${doubt}; the user chose the built-in summary` };
708  } catch {
709    return { decision: 'fallback', keep: [], why: `${facts}; ${doubt}; no one to ask (headless), so the built-in summary with the note` };
710  }
711}
712
713/* ------------------------------------------------------------ /lossless */
714
715/** The plugin's slash command; every subcommand hangs off it. */
716export const COMMAND = 'lossless';
717
718export interface LastCompaction {
719  at: string;
720  report: OptimizeReport;
721  summary: string;
722}
723
724function lastKey(sessionId: string): string {
725  return `lossless-compact:last:${sessionId}`;
726}
727
728function formatTokens(n: number): string {
729  return n.toLocaleString('en-US');
730}
731
732export async function statusText(
733  sessionId: string,
734  messages: readonly Message[],
735  archive: ArchiveStore,
736  last: LastCompaction | undefined,
737  classifier: string,
738  classifierNote?: string,
739): Promise<string> {
740  const ledger = buildLedger(messages);
741  const active = eventTokens(ledger.events);
742  const stats = await archive.stats(sessionId);
743  const constraints = findConstraints(messages, ledger);
744  const lines = [
745    `lossless-compact — session ${sessionId}`,
746    `Active context:        ~${formatTokens(active)} tokens (est.), ${messages.length} messages, ${ledger.interactions.size} tool interactions`,
747    `Archived this session: ~${formatTokens(stats.tokens)} tokens in ${stats.records} records`,
748    `Classifier:            ${last?.report.classifier ?? classifier}${last ? '' : ' (configured)'}`,
749    ...(classifierNote ? [`                       ${classifierNote}`] : []),
750    last
751      ? `Last compaction:       ${last.at} — ${last.summary}`
752      : 'Last compaction:       none yet',
753    '',
754    'Protection:',
755    `  ${constraints.length > 0 ? '✓' : '·'} ${constraints.length} explicit user constraint${constraints.length === 1 ? '' : 's'} found (never evicted)`,
756  ];
757  if (last) {
758    for (const [rule, n] of Object.entries(last.report.protections)) {
759      lines.push(`  ✓ ${n} result${n === 1 ? '' : 's'} kept by rule ${rule}`);
760    }
761  }
762  lines.push('', 'Commands: /lossless list [n] · /lossless why <id> · /lossless show <id> · /lossless restore <id> · /lossless retrieve <query>');
763  return lines.join('\n');
764}
765
766export async function whyText(archive: ArchiveStore, id: string): Promise<string> {
767  const record = await archive.get(id);
768  if (!record) return `No archive record ${id}.`;
769  const lines = [
770    `${record.id} — ${record.kind}${record.toolName ? ` (${record.toolName})` : ''}, seq ${record.seq}, ~${record.tokenEstimate} tokens`,
771    `Action: ${record.action} (compaction ${record.compactionId}, ${record.archivedAt})`,
772    '',
773    'Reasons:',
774    ...record.reasons.map((reason) => `- ${reason.detail}${reason.refs?.length ? ` [${reason.refs.join(', ')}]` : ''}`),
775  ];
776  if (record.related.length > 0) {
777    lines.push('', 'Related:', ...record.related.map((rel) => `- ${rel.relation}: ${rel.id}`));
778  }
779  lines.push('', 'The original content is still stored; /lossless restore ' + record.id + ' brings it back.');
780  return lines.join('\n');
781}
782
783export function restoreBlock(record: ArchiveRecord): string {
784  const head = `<retrieved_context id="${record.id}" kind="${record.kind}"${
785    record.toolName ? ` tool="${record.toolName}"` : ''
786  } seq="${record.seq}" archived="${record.archivedAt}">`;
787  return `${head}\n${record.content}\n</retrieved_context>`;
788}
789
790export async function listText(archive: ArchiveStore, sessionId: string, limit: number): Promise<string> {
791  const summaries = await archive.list({ sessionId });
792  if (summaries.length === 0) return 'Nothing archived in this session yet.';
793  const shown = summaries.slice(-limit);
794  return [
795    `${summaries.length} archived record${summaries.length === 1 ? '' : 's'}${
796      shown.length < summaries.length ? ` (newest ${shown.length})` : ''
797    }:`,
798    ...shown.map(
799      (s) =>
800        `${s.id}  ${s.kind}${s.toolName ? `/${s.toolName}` : ''}  ~${s.tokenEstimate}t  ${s.action}  ${s.preview}`,
801    ),
802  ].join('\n');
803}
804
805export async function retrieveText(archive: ArchiveStore, sessionId: string, query: string): Promise<string> {
806  const found = await archive.search(query, { sessionId, limit: 10 });
807  if (found.length === 0) return `Nothing in the archive matches "${query}".`;
808  return [
809    `Best matches for "${query}":`,
810    ...found.map(
811      (s) =>
812        `${s.id}  ${s.kind}${s.toolName ? `/${s.toolName}` : ''}  ~${s.tokenEstimate}t  ${s.preview}`,
813    ),
814    '',
815    '/lossless restore <id> to bring one back verbatim.',
816  ].join('\n');
817}
818
819/** Dispatches `/lossless <sub> ...`; `context` carries what the model should read. */
820export async function runContextCommand(
821  args: string,
822  deps: {
823    sessionId: string;
824    messages: () => Promise<readonly Message[]>;
825    archive: ArchiveStore;
826    last: () => Promise<LastCompaction | undefined>;
827    classifier: string;
828    classifierNote?: string;
829  },
830): Promise<{ text: string; context?: string[] }> {
831  const [sub = 'status', ...rest] = args.trim().split(/\s+/).filter(Boolean);
832  const arg = rest.join(' ');
833  switch (sub) {
834    case 'status':
835      return {
836        text: await statusText(deps.sessionId, await deps.messages(), deps.archive, await deps.last(), deps.classifier, deps.classifierNote),
837      };
838    case 'list':
839      return { text: await listText(deps.archive, deps.sessionId, Math.max(1, Number.parseInt(arg, 10) || 20)) };
840    case 'why':
841      if (!arg) return { text: 'Usage: /lossless why <id>' };
842      return { text: await whyText(deps.archive, arg) };
843    case 'show':
844    case 'restore': {
845      if (!arg) return { text: `Usage: /lossless ${sub} <id>` };
846      const record = await deps.archive.get(arg);
847      if (!record) return { text: `No archive record ${arg}.` };
848      const block = restoreBlock(record);
849      if (sub === 'show') return { text: block };
850      return {
851        text: `Restored ${record.id} (${record.kind}${record.toolName ? `/${record.toolName}` : ''}, ~${record.tokenEstimate} tokens) into the model's context for this turn.`,
852        context: [block],
853      };
854    }
855    case 'retrieve':
856    case 'search':
857      if (!arg) return { text: `Usage: /lossless ${sub} <query>` };
858      return { text: await retrieveText(deps.archive, deps.sessionId, arg) };
859    default:
860      return {
861        text: [
862          `Unknown subcommand "${sub}".`,
863          'Usage: /lossless [status] · list [n] · why <id> · show <id> · restore <id> · retrieve <query>',
864        ].join('\n'),
865      };
866  }
867}
868
869/* ------------------------------------------------------ slash-menu entry */
870
871/**
872 * The VS Code and Cursor extensions fill their slash menu once at startup
873 * from the markdown commands on disk, so `$.command.register`'s `/lossless`
874 * is never in it; `commands/lossless.md` is, as `/lossless-compact:lossless`.
875 * Picking it is a prompt, not a command: `command.run` never fires and the
876 * model reads the file. `skill.prompt` fires as the engine expands it, and
877 * that is where the hook puts the command's real answer in the model's hands.
878 * Typing `/lossless` in full still runs the command directly, no model turn.
879 */
880export const MENU_SKILL = `lossless-compact:${COMMAND}`;
881
882/** Whether a `skill.prompt` is the static menu entry (plugin-qualified, or bare where a host folds the prefix). */
883export function isMenuSkill(skill: string): boolean {
884  return skill === MENU_SKILL || skill === COMMAND;
885}
886
887/**
888 * The arguments the menu entry carried: its body opens with
889 * `/lossless $ARGUMENTS`, which the host substitutes ("" when nothing was
890 * typed; the marker itself on a host that does not substitute).
891 */
892export function menuArgs(text: string): string {
893  const first = (text.split('\n', 1)[0] ?? '').trim();
894  const match = new RegExp(`^/${COMMAND}\\b\\s*(.*)$`).exec(first);
895  const args = (match?.[1] ?? '').trim();
896  return args === '$ARGUMENTS' ? '' : args;
897}
898
899/**
900 * What the model reads in the menu entry's place: the command's answer, to
901 * show verbatim, and after it whatever the command put in the model's context
902 * (a restored record).
903 */
904export function menuPrompt(args: string, answer: { text: string; context?: readonly string[] }): string {
905  const command = `/${COMMAND}${args ? ` ${args}` : ''}`;
906  const lines = [
907    `lossless-compact answered \`${command}\` for this prompt. Show the user the answer between the markers exactly as it is, in one code block, and add nothing else: no tools, no commentary. Anything after the markers is context for you, not for the reply.`,
908    '',
909    '<lossless_answer>',
910    answer.text,
911    '</lossless_answer>',
912  ];
913  if (answer.context && answer.context.length > 0) lines.push('', ...answer.context);
914  return lines.join('\n');
915}
916
917/* ------------------------------------------------------------- register */
918
919/** What `/lossless` needs from the session beyond the engine: the archive's root and the classifier as resolved so far. */
920type CommandState = { archiveDir: string; classifier: string; classifierNote?: string };
921
922/**
923 * `/lossless <args>` answered, for the typed command and for the menu entry
924 * alike. A top-level function: the engine admits `$` only into one of those.
925 */
926async function answerCommand(
927  $: {
928    session: { id: () => Promise<string>; messages: () => Promise<readonly Message[]> };
929    store: { get: (key: string) => Promise<unknown> };
930  } & Parameters<typeof engineFs>[0],
931  args: string,
932  state: CommandState,
933): Promise<{ text: string; context?: string[] }> {
934  const sessionId = await $.session.id();
935  const archive = new FileArchive(engineFs($), { root: state.archiveDir });
936  return runContextCommand(args, {
937    sessionId,
938    messages: () => $.session.messages(),
939    archive,
940    last: async () => (await $.store.get(lastKey(sessionId))) as LastCompaction | undefined,
941    classifier: state.classifier,
942    ...(state.classifierNote ? { classifierNote: state.classifierNote } : {}),
943  });
944}
945
946/**
947 * Starts a compaction the way this host allows: `$.session.compact()` between
948 * turns, or where the host refuses that — the -p/SDK path, which is what the
949 * VS Code and Cursor extensions run a session on ("compaction here runs
950 * inside a turn (a /compact prompt)") — the `/compact` command, queued for
951 * when the session is idle. The same `session.compact` event either way, so
952 * neither summarizes.
953 */
954async function startCompaction(
955  $: {
956    session: { compact: () => Promise<unknown> };
957    command: { run: (args: { command: string }) => Promise<unknown> };
958    ui: { log: (text: string) => void };
959  },
960  verbose: boolean,
961): Promise<void> {
962  try {
963    await $.session.compact();
964  } catch (error) {
965    // Routine on the SDK path (every VS Code chat), so a diagnostic only.
966    if (verbose) $.ui.log(`lossless-compact: compacting through /compact instead (${error instanceof Error ? error.message : String(error)})`);
967    await $.command.run({ command: 'compact' });
968  }
969}
970
971async function getApiKey(
972  $: {
973    env: { get: (name: string) => Promise<string | undefined> };
974    settings: { read: () => Promise<Readonly<Record<string, unknown>>> };
975  },
976  config: HookConfig,
977): Promise<string | undefined> {
978  if (config.apiKey) return config.apiKey;
979  const fromEnv = await $.env.get('TYPESAFE_API_KEY');
980  if (fromEnv) return fromEnv;
981  const settings = await $.settings.read();
982  const env = settings['env'];
983  if (env && typeof env === 'object') {
984    const value = (env as Record<string, unknown>)['TYPESAFE_API_KEY'];
985    if (typeof value === 'string' && value) return value;
986  }
987  return undefined;
988}
989
990function safeNotify(
991  $: {
992    ui: {
993      log: (text: string) => void;
994      toast: (text: string, options?: { timeoutMs?: number }) => void;
995    };
996  },
997  text: string,
998): void {
999  try {
1000    $.ui.log(text);
1001    $.ui.toast(text, { timeoutMs: 15_000 });
1002  } catch {
1003    // A failing UI must never turn a compaction into a failure (upstream #36).
1004  }
1005}
1006
1007function percent(ratio: number): string {
1008  return `${Math.round(ratio * 100)}%`;
1009}
1010
1011export const register: Register = (on: On, options: PluginOptions) => {
1012  const configured = resolveLosslessCompactConfig(options);
1013  let compacting = false;
1014  const trigger = new AutoCompactTrigger(configured);
1015  let classifierName: string = configured.classifier;
1016  let classifierNote: string | undefined;
1017  const commandState = (): CommandState => ({
1018    archiveDir: configured.archiveDir,
1019    classifier: classifierName,
1020    ...(classifierNote ? { classifierNote } : {}),
1021  });
1022
1023  on('session.start', async ($, event, next) => {
1024    // Resolve `auto` up front so a fresh session can prove whether it will use
1025    // Jev or the local ruleset before the first compaction has happened.
1026    try {
1027      const apiKey = await getApiKey($, configured);
1028      classifierName = resolvedClassifierName(configured, apiKey);
1029      // At most one applies: no key means the ruleset, a custom endpoint means Jev.
1030      classifierNote = noKeyNotice(configured, apiKey) ?? endpointNotice(configured, apiKey);
1031      // One dim line in the terminal transcript (the debug log elsewhere); the
1032      // compaction note and `/lossless` carry the same line where it matters.
1033      if (classifierNote) $.ui.log(`lossless-compact: ${classifierNote}`);
1034    } catch (error) {
1035      try {
1036        $.ui.log(`lossless-compact: could not resolve the configured classifier (${error instanceof Error ? error.message : String(error)})`);
1037      } catch {
1038        // The compaction path will retry and fail less if this was transient.
1039      }
1040    }
1041    try {
1042      // Our own name: Claude Code refuses a built-in's (`/context`), and a
1043      // hijacked built-in never shows in the typeahead.
1044      await $.command.register({
1045        name: COMMAND,
1046        description: 'What lossless-compact archived: status, list, why, show, restore, retrieve',
1047        argumentHint: '[status|list [n]|why <id>|show <id>|restore <id>|retrieve <query>]',
1048      });
1049    } catch (error) {
1050      try {
1051        $.ui.log(`lossless-compact: /${COMMAND} not registered (${error instanceof Error ? error.message : String(error)})`);
1052      } catch {
1053        // ignore
1054      }
1055    }
1056    return next(event);
1057  });
1058
1059  on('command.run', { command: COMMAND }, async ($, event) => answerCommand($, event.args, commandState()));
1060
1061  on('skill.prompt', async ($, event, next) => {
1062    if (!isMenuSkill(event.skill)) return next(event);
1063    try {
1064      const args = menuArgs(event.text);
1065      const answered = await answerCommand($, args, commandState());
1066      if (configured.verbose) {
1067        $.ui.log(`lossless-compact: /${COMMAND} ${args} answered from the slash menu (one model turn relays it; typed in full it needs none)`);
1068      }
1069      return { text: menuPrompt(args, answered) };
1070    } catch (error) {
1071      try {
1072        $.ui.log(`lossless-compact: menu entry not answered (${error instanceof Error ? error.message : String(error)})`);
1073      } catch {
1074        // ignore
1075      }
1076      return next(event);
1077    }
1078  });
1079
1080  on('session.compact', async ($, event, next) => {
1081    // The archive and snapshot are on disk before any decision to fall back,
1082    // so the built-in summary still gets a note saying where they are.
1083    let fallbackNote: string | undefined;
1084    const fallback = async (reason: string): Promise<SessionCompactResult> => {
1085      safeNotify($, fallbackLine(reason, configured.verbose ? Number.POSITIVE_INFINITY : undefined));
1086      const built = await next(event);
1087      if (!fallbackNote || built.skip !== undefined || !built.messages?.length) return built;
1088      return { ...built, messages: withCompactionNoteSession(built.messages, fallbackNote) };
1089    };
1090    try {
1091      const sessionId = await $.session.id();
1092      const apiKey = await getApiKey($, configured);
1093      const fetchFn: HookFetch = async (url, init) => {
1094        const response = await $.http.fetch(url, init);
1095        return { status: response.status, ok: response.ok, text: response.text };
1096      };
1097      const classifier = chooseClassifier(configured, fetchFn, apiKey);
1098      classifierName = classifier.name;
1099      classifierNote = noKeyNotice(configured, apiKey) ?? endpointNotice(configured, apiKey);
1100      const archive = new FileArchive(engineFs($), { root: configured.archiveDir });
1101      let optimized = await optimizeSession(event.messages, configured, classifier, archive, sessionId);
1102      const threshold = configured.keepThreshold ?? classifier.defaultThreshold ?? 0.5;
1103      // Upstream #53: a classifier that keeps nothing it scored is either
1104      // miscalibrated or right about a disposable stretch; a model that has
1105      // seen the conversation (or the user) decides which, see reviewKeepNothing.
1106      let distrust: string | undefined;
1107      let reviewed = false;
1108      const calibration = suspectCalibration(optimized.result.actions, optimized.result.report.classified);
1109      if (calibration.suspect && !(await trustedForSession($, sessionId))) {
1110        const review = await reviewKeepNothing($, sessionId, event.messages, configured, optimized.result, threshold, calibration.best);
1111        reviewed = true;
1112        $.ui.log(`lossless-compact: ${review.why}`);
1113        if (review.decision === 'keep') {
1114          const withKeeps = classifierWithKeeps(optimized.result.decisions, new Set(review.keep), `${classifier.name}+review`);
1115          optimized = await optimizeSession(event.messages, configured, withKeeps, archive, sessionId);
1116        } else if (review.decision === 'fallback') {
1117          distrust = review.why;
1118        }
1119      }
1120      const { result } = optimized;
1121      let { messages } = optimized;
1122      const ratio = reductionRatio(result);
1123      const summary = summarize(result);
1124      // The full report, the summary's figures and every per-call score are
1125      // diagnostics: behind `verbose`; the user gets one line below.
1126      if (configured.verbose) for (const line of [...reportLines(result.report), `summary: ${summary}`]) $.ui.log(line);
1127      if (result.archived.length > 0 && (configured.snapshot || configured.noteRemoved)) {
1128        const at = new Date().toISOString();
1129        const cwd = await $.session.cwd();
1130        const root = configured.archiveDir.replace(/[\\/]+$/, '');
1131        const absolute = (relative: string): string => `${cwd.replace(/[\\/]+$/, '')}/${relative}`;
1132        let snapshotPath: string | undefined;
1133        if (configured.snapshot) {
1134          try {
1135            const [manifest] = await writeSnapshot(engineFs($), root, sessionId, result.report.compactionId, event.messages, at);
1136            snapshotPath = manifest ? absolute(manifest) : undefined;
1137          } catch (error) {
1138            $.ui.log(`lossless-compact snapshot skipped (${error instanceof Error ? error.message : String(error)})`);
1139          }
1140        }
1141        if (configured.noteRemoved) {
1142          const home = (await $.env.get('HOME')) ?? (await $.env.get('USERPROFILE'));
1143          const rawLogPath = await rawSessionLogPath(engineFs($), home, cwd, sessionId);
1144          const details = {
1145            at,
1146            compactionId: result.report.compactionId,
1147            report: result.report,
1148            ...(snapshotPath ? { snapshotPath } : {}),
1149            archiveDir: absolute(root),
1150            ...(rawLogPath ? { rawLogPath } : {}),
1151            ...(classifierNote ? { classifierNote } : {}),
1152          };
1153          messages = toSessionMessages(event.messages, withCompactionNote(result.messages, compactionNote(details)));
1154          fallbackNote = compactionNote({ ...details, summarized: true });
1155        }
1156      }
1157      if (configured.verbose) for (const line of decisionLogLines(result)) $.ui.log(line);
1158      const tooLittle = ratio < configured.minReductionRatio;
1159      // The plugin's own trigger (`plugin` in the terminal; `/compact` run
1160      // from turn.complete on the SDK path, where `compacting` is held) asked
1161      // for room the user did not: too little to remove is a reason to wait
1162      // for the context to grow, never to summarize what was protected. A
1163      // typed /compact and the host's own near-limit run still fall back —
1164      // those need the room now.
1165      const own = event.trigger === 'plugin' || compacting;
1166      try {
1167        await $.store.set(lastKey(sessionId), {
1168          at: new Date().toISOString(),
1169          report: result.report,
1170          summary: tooLittle && own ? `left as is (below ${percent(configured.minReductionRatio)} minimum): ${summary}` : summary,
1171        } satisfies LastCompaction);
1172      } catch {
1173        // The status line is a nicety; never fail the compaction over it.
1174      }
1175      if (tooLittle) {
1176        const reason = tooLittleReason(ratio, configured.minReductionRatio);
1177        if (own) {
1178          safeNotify($, `lossless-compact: nothing to compact yet (${reason}); tries again once the context has grown by a quarter`);
1179          return { skip: reason };
1180        }
1181        return fallback(reason);
1182      }
1183      // The review's own line above says what was found and who decided.
1184      if (distrust) return fallback('the classifier kept nothing; see the review above');
1185      safeNotify($, compactedLine(result.report, reviewed));
1186      return { messages: rechain(messages) };
1187    } catch (error) {
1188      return fallback(error instanceof Error ? error.message : String(error));
1189    }
1190  });
1191
1192  on('prompt.submit', async ($, event, next) => {
1193    if (!configured.autoRetrieve || configured.retrieveBudgetChars === 0) return next(event);
1194    try {
1195      const sessionId = await $.session.id();
1196      const archive = new FileArchive(engineFs($), { root: configured.archiveDir });
1197      const existing = (event.context ?? []).reduce((sum, block) => sum + block.length, 0);
1198      const room = Math.min(configured.retrieveBudgetChars, 32_000 - existing - REHYDRATION_PREFACE.length - 64);
1199      if (room < 200) return next(event);
1200      const found = await rehydrateForPrompt(archive, sessionId, event.text, { budgetChars: room });

1import { MemoryArchive, matchesFilter, rankRecords } from './memory-store.js';
2import {
3  summaryOf,
4  type ArchiveRecord,
5  type ArchiveSearchOptions,
6  type ArchiveStats,
7  type ArchiveStore,
8  type ArchiveSummary,
9} from './types.js';
10
11/**
12 * The little of a file system the archive needs, so one store runs on Node
13 * (`src/node/fs.ts`) and inside the Claude Code hook (`$.fs`, whole-file
14 * reads and writes of at most 4 MiB, no append, no delete).
15 */
16export interface TextFs {
17  /** Rejects with an error whose `code` is `ENOENT` when the file is missing. */
18  read(path: string): Promise<string>;
19  /** Creates parent directories as needed. */
20  write(path: string, text: string): Promise<void>;
21  exists(path: string): Promise<boolean>;
22  /** Entry names of a directory; `[]` when it does not exist. */
23  list(dir: string): Promise<string[]>;
24}
25
26export interface FileArchiveOptions {
27  /** Directory the archive lives under; `.lossless-compact` by default (relative to the project). */
28  root?: string;
29  /** Largest shard file, in characters of JSON. Default 3.5 MiB, under the hook's 4 MiB cap. */
30  maxShardChars?: number;
31}
32
33interface IndexEntry extends ArchiveSummary {
34  shard: string;
35  contentHash: string;
36}
37
38interface IndexFile {
39  version: 1;
40  sessionId: string;
41  entries: IndexEntry[];
42}
43
44interface ShardFile {
45  version: 1;
46  records: ArchiveRecord[];
47}
48
49const INDEX = 'index.json';
50
51function isMissing(error: unknown): boolean {
52  if (typeof error !== 'object' || error === null) return false;
53  const { code, message } = error as { code?: string; message?: string };
54  // The plugin host forwards its errno in the message only (`… failed: ENOENT`).
55  return code === 'ENOENT' || /\bENOENT\b/.test(message ?? '');
56}
57
58/**
59 * Records sharded into JSON files per session and compaction, with one index
60 * per session listing every record's summary and shard:
61 *
62 * ```
63 * <root>/archive/<sessionId>/index.json
64 * <root>/archive/<sessionId>/<compactionId>[-<n>].json
65 * ```
66 *
67 * Append-only: a shard is written once and never rewritten; `purge` rewrites
68 * the affected shards and the index. Search loads shards lazily and caches
69 * them for the life of the store.
70 */
71export class FileArchive implements ArchiveStore {
72  private readonly root: string;
73  private readonly maxShardChars: number;
74  private readonly indexes = new Map<string, IndexFile>();
75  private readonly shards = new Map<string, ShardFile>();
76
77  constructor(
78    private readonly fs: TextFs,
79    options: FileArchiveOptions = {},
80  ) {
81    this.root = (options.root ?? '.lossless-compact').replace(/[\\/]+$/, '');
82    this.maxShardChars = options.maxShardChars ?? 3.5 * 1024 * 1024;
83  }
84
85  private dir(sessionId: string): string {
86    return `${this.root}/archive/${encodeURIComponent(sessionId)}`;
87  }
88
89  private async readJson<T>(path: string): Promise<T | undefined> {
90    // A missing index or shard is the normal state before the first compaction;
91    // ask first so the host does not log a failed read.
92    if (!(await this.fs.exists(path))) return undefined;
93    try {
94      return JSON.parse(await this.fs.read(path)) as T;
95    } catch (error) {
96      if (isMissing(error)) return undefined;
97      throw error;
98    }
99  }
100
101  private async index(sessionId: string): Promise<IndexFile> {
102    const cached = this.indexes.get(sessionId);
103    if (cached) return cached;
104    const loaded = (await this.readJson<IndexFile>(`${this.dir(sessionId)}/${INDEX}`)) ?? {
105      version: 1,
106      sessionId,
107      entries: [],
108    };
109    this.indexes.set(sessionId, loaded);
110    return loaded;
111  }
112
113  private async shard(sessionId: string, name: string): Promise<ShardFile> {
114    const path = `${this.dir(sessionId)}/${name}`;
115    const cached = this.shards.get(path);
116    if (cached) return cached;
117    const loaded = (await this.readJson<ShardFile>(path)) ?? { version: 1, records: [] };
118    this.shards.set(path, loaded);
119    return loaded;
120  }
121
122  /** Sessions with an archive directory. */
123  async sessions(): Promise<string[]> {
124    return (await this.fs.list(`${this.root}/archive`)).map((name) => {
125      try {
126        return decodeURIComponent(name);
127      } catch {
128        return name;
129      }
130    });
131  }
132
133  async put(records: readonly ArchiveRecord[]): Promise<void> {
134    const bySession = new Map<string, ArchiveRecord[]>();
135    for (const record of records) {
136      const list = bySession.get(record.sessionId) ?? [];
137      list.push(record);
138      bySession.set(record.sessionId, list);
139    }
140    for (const [sessionId, list] of bySession) {
141      const index = await this.index(sessionId);
142      const known = new Map(index.entries.map((entry) => [entry.id, entry]));
143      const fresh: ArchiveRecord[] = [];
144      for (const record of list) {
145        const existing = known.get(record.id);
146        if (existing) {
147          if (existing.contentHash !== record.contentHash) {
148            throw new Error(`archive record ${record.id} exists with different content`);
149          }
150          continue;
151        }
152        known.set(record.id, { ...summaryOf(record), shard: '', contentHash: record.contentHash });
153        fresh.push(record);
154      }
155      if (fresh.length === 0) continue;
156      const compactionId = fresh[0]!.compactionId.replace(/[^\w.-]+/g, '_');
157      const taken = new Set(index.entries.map((entry) => entry.shard));
158      let part = 0;
159      let batch: ArchiveRecord[] = [];
160      let chars = 0;
161      const flush = async (): Promise<void> => {
162        if (batch.length === 0) return;
163        let name = part === 0 ? `${compactionId}.json` : `${compactionId}-${part}.json`;
164        while (taken.has(name)) {
165          part++;
166          name = `${compactionId}-${part}.json`;
167        }
168        taken.add(name);
169        part++;
170        const file: ShardFile = { version: 1, records: batch };
171        await this.fs.write(`${this.dir(sessionId)}/${name}`, JSON.stringify(file));
172        this.shards.set(`${this.dir(sessionId)}/${name}`, file);
173        for (const record of batch) {
174          index.entries.push({ ...summaryOf(record), shard: name, contentHash: record.contentHash });
175        }
176        batch = [];
177        chars = 0;
178      };
179      for (const record of fresh) {
180        const size = JSON.stringify(record).length + 2;
181        if (batch.length > 0 && chars + size > this.maxShardChars) await flush();
182        batch.push(record);
183        chars += size;
184      }
185      await flush();
186      await this.fs.write(`${this.dir(sessionId)}/${INDEX}`, JSON.stringify(index));
187    }
188  }
189
190  private async locate(id: string): Promise<{ sessionId: string; entry: IndexEntry } | undefined> {
191    for (const [sessionId, index] of this.indexes) {
192      const entry = index.entries.find((e) => e.id === id);
193      if (entry) return { sessionId, entry };
194    }
195    for (const sessionId of await this.sessions()) {
196      if (this.indexes.has(sessionId)) continue;
197      const index = await this.index(sessionId);
198      const entry = index.entries.find((e) => e.id === id);
199      if (entry) return { sessionId, entry };
200    }
201    return undefined;
202  }
203
204  async get(id: string): Promise<ArchiveRecord | undefined> {
205    const located = await this.locate(id);
206    if (!located) return undefined;
207    const shard = await this.shard(located.sessionId, located.entry.shard);
208    return shard.records.find((record) => record.id === id);
209  }
210
211  async getMany(ids: readonly string[]): Promise<ArchiveRecord[]> {
212    const found: ArchiveRecord[] = [];
213    for (const id of ids) {
214      const record = await this.get(id);
215      if (record) found.push(record);
216    }
217    return found;
218  }
219
220  private async entries(options: ArchiveSearchOptions): Promise<{ sessionId: string; entry: IndexEntry }[]> {
221    const sessions = options.sessionId ? [options.sessionId] : await this.sessions();
222    const all: { sessionId: string; entry: IndexEntry }[] = [];
223    for (const sessionId of sessions) {
224      const index = await this.index(sessionId);
225      for (const entry of index.entries) {
226        if (matchesFilter({ sessionId, kind: entry.kind, toolName: entry.toolName }, options)) {
227          all.push({ sessionId, entry });
228        }
229      }
230    }
231    return all.sort((a, b) => a.entry.seq - b.entry.seq || a.entry.archivedAt.localeCompare(b.entry.archivedAt));
232  }
233
234  async list(options: ArchiveSearchOptions = {}): Promise<ArchiveSummary[]> {
235    const limit = options.limit ?? Number.POSITIVE_INFINITY;
236    return (await this.entries(options)).slice(0, limit).map(({ entry }) => {
237      const { shard: _shard, contentHash: _hash, ...summary } = entry;
238      return summary;
239    });
240  }
241
242  async search(query: string, options: ArchiveSearchOptions = {}): Promise<ArchiveSummary[]> {
243    const limit = options.limit ?? 20;
244    const records: ArchiveRecord[] = [];
245    for (const { sessionId, entry } of await this.entries(options)) {
246      const shard = await this.shard(sessionId, entry.shard);
247      const record = shard.records.find((r) => r.id === entry.id);
248      if (record) records.push(record);
249    }
250    return rankRecords(query, records)
251      .slice(0, limit)
252      .map((entry) => ({ ...summaryOf(entry.record), score: entry.score }));
253  }
254
255  async stats(sessionId?: string): Promise<ArchiveStats> {
256    const stats: ArchiveStats = { records: 0, tokens: 0, bySession: {} };
257    for (const { sessionId: id, entry } of await this.entries(sessionId ? { sessionId } : {})) {
258      stats.records++;
259      stats.tokens += entry.tokenEstimate;
260      const session = (stats.bySession[id] ??= { records: 0, tokens: 0 });
261      session.records++;
262      session.tokens += entry.tokenEstimate;
263    }
264    return stats;
265  }
266
267  async purge(ids: readonly string[]): Promise<number> {
268    let removed = 0;
269    const touched = new Map<string, Set<string>>();
270    for (const id of ids) {
271      const located = await this.locate(id);
272      if (!located) continue;
273      const shards = touched.get(located.sessionId) ?? new Set<string>();
274      shards.add(located.entry.shard);
275      touched.set(located.sessionId, shards);
276    }
277    const gone = new Set(ids);
278    for (const [sessionId, shards] of touched) {
279      for (const name of shards) {
280        const shard = await this.shard(sessionId, name);
281        const before = shard.records.length;
282        shard.records = shard.records.filter((record) => !gone.has(record.id));
283        removed += before - shard.records.length;
284        await this.fs.write(`${this.dir(sessionId)}/${name}`, JSON.stringify(shard));
285      }
286      const index = await this.index(sessionId);
287      index.entries = index.entries.filter((entry) => !gone.has(entry.id));
288      await this.fs.write(`${this.dir(sessionId)}/${INDEX}`, JSON.stringify(index));
289    }
290    return removed;
291  }
292
293  /** Everything in one session, loaded into memory (for tests and inspection). */
294  async toMemory(sessionId: string): Promise<MemoryArchive> {
295    const records: ArchiveRecord[] = [];
296    for (const { entry } of await this.entries({ sessionId })) {
297      const shard = await this.shard(sessionId, entry.shard);
298      const record = shard.records.find((r) => r.id === entry.id);
299      if (record) records.push(record);
300    }
301    return new MemoryArchive(records);
302  }
303}
304

1import type { ContextAction, DecisionReason } from '../core/actions.js';
2import type { EventKind, EventRole } from '../core/events.js';
3
4/**
5 * One exact unit of context that left the active window. The archive is the
6 * slow tier of the memory hierarchy: nothing in it is ever paraphrased, and
7 * nothing leaves it unless the user purges it.
8 */
9export interface ArchiveRecord {
10  /** The event id (see `eventId`); the same content archived twice keeps one record. */
11  id: string;
12  sessionId: string;
13  /** Position in the transcript at the time it was archived. */
14  seq: number;
15  role: EventRole;
16  kind: EventKind;
17  toolName?: string;
18  toolUseId?: string;
19  /** The exact original content. */
20  content: string;
21  contentHash: string;
22  tokenEstimate: number;
23  isError?: boolean;
24  /** ISO-8601 time the record was written. */
25  archivedAt: string;
26  /** Which compaction wrote it (a counter or timestamp the engine chooses). */
27  compactionId: string;
28  /** The action that evicted it and why, for `/lossless why`. */
29  action: ContextAction;
30  reasons: DecisionReason[];
31  /** Ids of related records: the call of a result, the result of a call, a duplicate, a dependant. */
32  related: { relation: 'call' | 'result' | 'duplicate_of' | 'referenced_by' | 'supersedes'; id: string }[];
33  /** Tool-specific facts that make the record findable or reproducible (path, command, exit code, file hash …). */
34  metadata: Record<string, unknown>;
35}
36
37export interface ArchiveSummary {
38  id: string;
39  sessionId: string;
40  seq: number;
41  kind: EventKind;
42  toolName?: string;
43  tokenEstimate: number;
44  action: ContextAction;
45  archivedAt: string;
46  /** The first line or so of the content. */
47  preview: string;
48  /** Relevance in [0, 1] when the summary came from `search`. */
49  score?: number;
50}
51
52export interface ArchiveSearchOptions {
53  sessionId?: string;
54  kinds?: EventKind[];
55  toolNames?: string[];
56  limit?: number;
57}
58
59export interface ArchiveStats {
60  records: number;
61  tokens: number;
62  bySession: Record<string, { records: number; tokens: number }>;
63}
64
65/**
66 * Where evicted context lives. Implementations must be append-only from the
67 * engine's point of view: `put` never overwrites a record with different
68 * content, and only `purge` removes anything.
69 */
70export interface ArchiveStore {
71  put(records: readonly ArchiveRecord[]): Promise<void>;
72  get(id: string): Promise<ArchiveRecord | undefined>;
73  getMany(ids: readonly string[]): Promise<ArchiveRecord[]>;
74  list(options?: ArchiveSearchOptions): Promise<ArchiveSummary[]>;
75  /** Lexical search over content and metadata; ranked, best first. */
76  search(query: string, options?: ArchiveSearchOptions): Promise<ArchiveSummary[]>;
77  stats(sessionId?: string): Promise<ArchiveStats>;
78  /** Removes records for good; the only destructive operation. */
79  purge(ids: readonly string[]): Promise<number>;
80}
81
82/** The first line of some content, bounded, for listings. */
83export function preview(content: string, maxChars = 120): string {
84  const line = content.replace(/\s+/g, ' ').trim();
85  return line.length <= maxChars ? line : `${line.slice(0, maxChars - 1)}…`;
86}
87
88export function summaryOf(record: ArchiveRecord): ArchiveSummary {
89  const summary: ArchiveSummary = {
90    id: record.id,
91    sessionId: record.sessionId,
92    seq: record.seq,
93    kind: record.kind,
94    tokenEstimate: record.tokenEstimate,
95    action: record.action,
96    archivedAt: record.archivedAt,
97    preview: preview(record.content),
98  };
99  if (record.toolName) summary.toolName = record.toolName;
100  return summary;
101}
102

1/**
2 * What can happen to one unit of context. Replaces the baseline's
3 * keep / drop_result / drop_call trichotomy with an explicit taxonomy; every
4 * action except the first two moves the exact original into the archive, so
5 * nothing is unrecoverable.
6 */
7export type ContextAction =
8  /** Verbatim, and no policy may evict it (constraints, current request, pinned by user). */
9  | 'PIN_VERBATIM'
10  /** Verbatim for now; a later pass may reconsider. */
11  | 'KEEP_VERBATIM'
12  /** Head (and tail) kept in place, the full text archived. */
13  | 'KEEP_HEAD_TAIL'
14  /** Durable facts extracted into memory, the full text archived. */
15  | 'EXTRACT_MEMORY_AND_ARCHIVE'
16  /** Removed from the active window, retrievable from the archive. */
17  | 'ARCHIVE_ONLY'
18  /** Replaced in place by a one-line stub naming the archive record. */
19  | 'REPLACE_WITH_REFERENCE'
20  /** Archived together with a recipe to reproduce it (tool + input + freshness). */
21  | 'RERUN_ON_DEMAND'
22  /** Duplicate of something retained; archived for provenance, no stub. */
23  | 'DROP_REDUNDANT';
24
25export const CONTEXT_ACTIONS: readonly ContextAction[] = [
26  'PIN_VERBATIM',
27  'KEEP_VERBATIM',
28  'KEEP_HEAD_TAIL',
29  'EXTRACT_MEMORY_AND_ARCHIVE',
30  'ARCHIVE_ONLY',
31  'REPLACE_WITH_REFERENCE',
32  'RERUN_ON_DEMAND',
33  'DROP_REDUNDANT',
34];
35
36/** Actions that leave the unit verbatim in the active window. */
37export function keepsVerbatim(action: ContextAction): boolean {
38  return action === 'PIN_VERBATIM' || action === 'KEEP_VERBATIM';
39}
40
41/** Actions that remove some or all of the unit from the active window. */
42export function evicts(action: ContextAction): boolean {
43  return !keepsVerbatim(action);
44}
45
46/**
47 * The classifier's view of one unit. Not every classifier fills every field;
48 * `keepCall` / `keepResult` are the baseline Jev questions and always present
49 * when a classifier ran. Values are probabilities in [0, 1].
50 */
51export interface ClassifierScores {
52  keepCall: number;
53  keepResult: number;
54  currentRelevance?: number;
55  futureRelevance?: number;
56  constraintImportance?: number;
57  exactnessRequired?: number;
58  superseded?: number;
59  redundant?: number;
60  /** The classifier's own confidence in this row, when it reports one. */
61  confidence?: number;
62}
63
64/**
65 * Why a decision came out the way it did, one entry per contributing rule or
66 * signal; shown verbatim by `/lossless why`.
67 */
68export interface DecisionReason {
69  /** Machine-readable, e.g. `explicit_constraint`, `recent`, `dependency`, `jev_result`. */
70  code: string;
71  /** Human-readable sentence. */
72  detail: string;
73  /** Ids of other events this reason refers to (the duplicate, the dependant …). */
74  refs?: string[];
75}
76
77/** The unit a decision is about: a paired tool interaction or a single text event. */
78export type DecisionUnit = 'tool_interaction' | 'text';
79
80export interface ActionDecision {
81  /** For a tool interaction, the tool_use event id; for text, the text event id. */
82  eventId: string;
83  /** For a tool interaction, the tool_result event id (when the result exists). */
84  resultId?: string;
85  unit: DecisionUnit;
86  toolName?: string;
87  action: ContextAction;
88  scores?: ClassifierScores;
89  reasons: DecisionReason[];
90  /** Names of rules that forbade eviction, when any did. */
91  protectedBy: string[];
92  /** Overall confidence in the action in [0, 1]; low confidence keeps. */
93  confidence: number;
94  /** Where the evicted content went, filled in once it is archived. */
95  archiveIds?: string[];
96}
97
98/** Maps the baseline's per-call action onto the taxonomy. */
99export function fromBaselineAction(action: 'keep' | 'drop_result' | 'drop_call', pinned: boolean): ContextAction {
100  if (action === 'keep') return pinned ? 'PIN_VERBATIM' : 'KEEP_VERBATIM';
101  if (action === 'drop_result') return 'KEEP_HEAD_TAIL';
102  return 'ARCHIVE_ONLY';
103}
104
105/** Maps a taxonomy action onto the baseline's rebuild behaviour. */
106export function toBaselineAction(action: ContextAction): 'keep' | 'drop_result' | 'drop_call' {
107  switch (action) {
108    case 'PIN_VERBATIM':
109    case 'KEEP_VERBATIM':
110      return 'keep';
111    case 'KEEP_HEAD_TAIL':
112    case 'REPLACE_WITH_REFERENCE':
113    case 'RERUN_ON_DEMAND':
114      return 'drop_result';
115    case 'EXTRACT_MEMORY_AND_ARCHIVE':
116    case 'ARCHIVE_ONLY':
117    case 'DROP_REDUNDANT':
118      return 'drop_call';
119  }
120}
121

1import { pathsIn } from '../core/rules.js';
2import type { ClassifierScores } from '../core/actions.js';
3import type { Message, ToolCall } from '../types.js';
4import type { Classifier, ClassifierContext, ClassifierRun } from './types.js';
5
6export interface RulesetOptions {
7  /** Messages after which a result is considered stale. Default 20. */
8  staleAfterMessages?: number;
9  /** Tools whose results are cheap to re-run. Default ['Read','Grep','Glob','LS']. */
10  rerunnableTools?: string[];
11}
12
13const EDITING_TOOLS = new Set(['Edit', 'Write', 'MultiEdit']);
14const NON_REPRODUCIBLE_TOOLS = new Set(['WebFetch', 'WebSearch', 'Agent', 'Task']);
15const FAILURE_WORDS = /\b(?:fail(?:ed|ure|ing)?|error|exception|traceback|panic)\b/i;
16const LARGE_RESULT_CHARS = 20_000;
17
18function clamp01(value: number): number {
19  return Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0;
20}
21
22/** Base keepCall/keepResult before recency scaling and per-call adjustments. */
23function categoryDefaults(
24  tool: string,
25  rerunnableTools: readonly string[],
26): { keepCall: number; keepResult: number } {
27  if (rerunnableTools.includes(tool)) return { keepCall: 0.4, keepResult: 0.25 };
28  if (EDITING_TOOLS.has(tool)) return { keepCall: 0.7, keepResult: 0.3 };
29  if (tool === 'Bash') return { keepCall: 0.5, keepResult: 0.45 };
30  if (NON_REPRODUCIBLE_TOOLS.has(tool)) return { keepCall: 0.5, keepResult: 0.7 };
31  // No signal for this tool; a neutral middle ground rather than a guess.
32  return { keepCall: 0.5, keepResult: 0.4 };
33}
34
35function inputPathOf(input: Record<string, unknown>): string | undefined {
36  for (const key of ['file_path', 'path', 'notebook_path', 'filePath']) {
37    const value = input[key];
38    if (typeof value === 'string' && value.length > 0) return value;
39  }
40  return undefined;
41}
42
43/** Whether a later call of the same tool on the same target already succeeded. */
44function laterSuccessResolved(call: ToolCall, calls: readonly ToolCall[]): boolean {
45  const path = inputPathOf(call.input);
46  const command = typeof call.input['command'] === 'string' ? call.input['command'] : undefined;
47  for (const later of calls) {
48    if (later.callIndex <= call.callIndex || later.tool !== call.tool || later.isError) continue;
49    if (command !== undefined) {
50      if (later.input['command'] === command) return true;
51      continue;
52    }
53    if (path !== undefined) {
54      if (inputPathOf(later.input) === path) return true;
55      continue;
56    }
57    return true;
58  }
59  return false;
60}
61
62/** Recursively sorted-key JSON, so input order never affects the comparison. */
63function canonicalInput(input: Record<string, unknown>): string {
64  const sort = (value: unknown): unknown => {
65    if (Array.isArray(value)) return value.map(sort);
66    if (value !== null && typeof value === 'object') {
67      const out: Record<string, unknown> = {};
68      for (const key of Object.keys(value as Record<string, unknown>).sort()) {
69        out[key] = sort((value as Record<string, unknown>)[key]);
70      }
71      return out;
72    }
73    return value;
74  };
75  try {
76    return JSON.stringify(sort(input));
77  } catch {
78    return '[unserializable]';
79  }
80}
81
82/** An identical later call (same tool, same normalised input) makes this one's copy stale. */
83function isSuperseded(call: ToolCall, calls: readonly ToolCall[]): boolean {
84  const key = canonicalInput(call.input);
85  return calls.some((later) => later.callIndex > call.callIndex && later.tool === call.tool && canonicalInput(later.input) === key);
86}
87
88function resultTextFor(call: ToolCall, messages: readonly Message[]): string {
89  const message = messages[call.resultIndex];
90  const result = message?.toolResults?.find((r) => r.tool_use_id === call.tool_use_id);
91  return result?.text ?? '';
92}
93
94/** Bound on the text handed to the path-scanning regex: a mention worth a
95 * bonus is almost always near the top of a result, and `pathsIn`'s pattern
96 * degrades badly on long separator-free text (this classifier must stay
97 * fast even on a huge, pathological tool result). */
98const PATH_SCAN_CHARS = 4000;
99
100/** Whether the result mentions a file path also mentioned in the last 6 messages. */
101function mentionsRecentPath(resultText: string, messages: readonly Message[]): boolean {
102  if (resultText.length === 0) return false;
103  const resultPaths = pathsIn(resultText.slice(0, PATH_SCAN_CHARS));
104  if (resultPaths.size === 0) return false;
105  const recentText = messages
106    .slice(-6)
107    .map((m) => m.text.slice(0, PATH_SCAN_CHARS))
108    .join('\n');
109  const recentPaths = pathsIn(recentText);
110  for (const path of resultPaths) if (recentPaths.has(path)) return true;
111  return false;
112}
113
114/**
115 * The no-network degraded-mode classifier, a hand-written ruleset: deterministic signals only (age,
116 * tool type, error status, duplication, size), so it stays available when
117 * Jev is unavailable or when a sensitive repo's local policy mode forbids
118 * sending anything off the machine. Fast and O(n) — no state is built and no
119 * request is made.
120 */
121export const RULESET_DEFAULT_THRESHOLD = 0.4;
122
123export class RulesetClassifier implements Classifier {
124  readonly name = 'ruleset';
125  /**
126   * From the 2026-09-20 sweep (docs/evals.md): on 12 real sessions 0.3 → 19 %
127   * reduction, 0.4 → 49 %, 0.5 → 67 %; labelled false drops did not move with
128   * the threshold. The middle setting keeps more in degraded mode.
129   */
130  readonly defaultThreshold = RULESET_DEFAULT_THRESHOLD;
131  private readonly staleAfterMessages: number;
132  private readonly rerunnableTools: readonly string[];
133
134  constructor(options: RulesetOptions = {}) {
135    this.staleAfterMessages = Math.max(1, options.staleAfterMessages ?? 20);
136    this.rerunnableTools = options.rerunnableTools ?? ['Read', 'Grep', 'Glob', 'LS'];
137  }
138
139  async score(candidates: readonly ToolCall[], context: ClassifierContext): Promise<ClassifierRun> {
140    const started = Date.now();
141    const scores = new Map<string, ClassifierScores>();
142    const total = context.messages.length;
143
144    for (const call of candidates) {
145      let { keepCall, keepResult } = categoryDefaults(call.tool, this.rerunnableTools);
146
147      if (call.tool === 'Bash') {
148        const resultText = resultTextFor(call, context.messages);
149        if (FAILURE_WORDS.test(resultText)) keepResult = 0.7;
150        else if (call.resultChars < 200 && !call.isError) keepResult = 0.2;
151      }
152
153      if (call.isError) {
154        keepResult = laterSuccessResolved(call, context.calls) ? 0.2 : 0.75;
155      }
156
157      const age = Math.max(0, total - 1 - call.resultIndex);
158      const recency = clamp01(1 - age / this.staleAfterMessages);
159      const scale = 0.5 + 0.5 * recency;
160      keepCall = keepCall * scale;
161      keepResult = keepResult * scale;
162
163      if (isSuperseded(call, context.calls)) {
164        keepResult *= 0.3;
165        keepCall *= 0.5;
166      }
167
168      const resultText = resultTextFor(call, context.messages);
169      if (mentionsRecentPath(resultText, context.messages)) {
170        keepResult += 0.2;
171      }
172
173      if (call.resultChars > LARGE_RESULT_CHARS) {
174        keepResult = Math.max(0.05, keepResult - 0.1);
175      }
176
177      scores.set(call.id, { keepCall: clamp01(keepCall), keepResult: clamp01(keepResult) });
178    }
179
180    return {
181      scores,
182      stats: {
183        requests: 0,
184        stateTokens: 0,
185        stateStage: 'ruleset',
186        ms: Date.now() - started,
187        unscored: [],
188      },
189    };
190  }
191}
192

1import { batchCalls, questionsFor as upstreamQuestionsFor } from '../compact.js';
2import { noulAnswer } from '../request.js';
3import { fitState } from '../state.js';
4import type { ClassifierScores } from '../core/actions.js';
5import type { JevAsker, JevQuestions, ToolCall } from '../types.js';
6import type { Classifier, ClassifierContext, ClassifierRun } from './types.js';
7
8/**
9 * How the two questions are worded. `upstream` is the baseline's "must stay
10 * verbatim / re-running would not do", which measures irrecoverability and
11 * scores nearly every result below 0.3 on real sessions (upstream issues #26,
12 * #52). `useful` asks whether keeping is useful for the remaining work and
13 * spells out the yes/no criteria (upstream PRs #55, #61).
14 */
15export type JevQuestionStyle = 'upstream' | 'useful';
16
17/**
18 * From the 2026-09-20 cassette sweep against jev-1.13.0 (docs/evals.md): with
19 * the `useful` wording and sketches, every labelled must-keep survives up to
20 * 0.45 and the first false drop appears at 0.50; on 12 real sessions the
21 * score mass sits at 0.3–0.4, so 0.15 (upstream PR #55's suggestion) removes
22 * only 5 % there. 0.35 keeps 0.15 of headroom below the cliff and removes
23 * ~52 % of real-session tokens.
24 */
25export const JEV_DEFAULT_THRESHOLD = 0.35;
26
27/** The two `noul` questions about one call in the `useful` wording. */
28export function usefulQuestionsFor(call: ToolCall): JevQuestions {
29  return {
30    [`call_${call.id}`]: {
31      type: 'noul',
32      instructions: `Keeping tool call ${call.id} (${call.tool}) in the history is useful for the assistant's remaining work on the goal: knowing this call was made, with its input, still matters`,
33      criteria: {
34        true: 'The call records something the assistant may need again: a file it changed, a command whose effect matters, a search it should not repeat, a step of the current task.',
35        false: 'The call was exploratory or has been superseded; nothing later depends on knowing it happened.',
36      },
37    },
38    [`result_${call.id}`]: {
39      type: 'noul',
40      instructions: `Keeping the full output of tool call ${call.id} (${call.tool}, ${call.resultChars} chars) verbatim in the history is useful for the assistant's remaining work on the goal`,
41      criteria: {
42        true: 'The output holds details the assistant may still need exactly — an error, file contents it is still working with, a fact the current task depends on — that would be lost or costly to reproduce.',
43        false: 'The output has served its purpose, is reproducible by re-running the tool, or is superseded by a later result.',
44      },
45    },
46  };
47}
48
49/** A `noul` answer that is a probability; anything outside [0, 1] is malformed (upstream #29). */
50function probability(answers: Parameters<typeof noulAnswer>[0], name: string): number {
51  const value = noulAnswer(answers, name);
52  if (value < 0 || value > 1) throw new Error(`Invalid Jev answer for ${name}: ${value} is not a probability`);
53  return value;
54}
55
56export interface JevClassifierOptions {
57  /** Question wording. Default `useful`. */
58  questionStyle?: JevQuestionStyle;
59  /**
60   * When a batch fails, score the others and report the failed calls as
61   * unscored (the policy keeps them) instead of failing the whole run.
62   * Default true.
63   */
64  salvagePartialBatches?: boolean;
65  /** Retries per failed batch before giving it up. Default 1. */
66  retries?: number;
67  /** Batches in flight at once (upstream #33: everything at once trips rate limits). Default 4. */
68  concurrency?: number;
69}
70
71/** Runs `task` over `items` with at most `limit` in flight, preserving order. */
72export async function mapConcurrent<T, R>(
73  items: readonly T[],
74  limit: number,
75  task: (item: T) => Promise<R>,
76): Promise<R[]> {
77  const results: R[] = new Array(items.length);
78  let next = 0;
79  const worker = async (): Promise<void> => {
80    while (next < items.length) {
81      const index = next++;
82      results[index] = await task(items[index]!);
83    }
84  };
85  await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
86  return results;
87}
88
89/**
90 * The baseline classifier: the upstream two-question Jev protocol (keep the
91 * call? keep the result verbatim?) over the fitted whole-conversation state,
92 * plus result sketches and partial-batch salvage.
93 */
94export class JevClassifier implements Classifier {
95  readonly name: string;
96  readonly defaultThreshold = JEV_DEFAULT_THRESHOLD;
97  private readonly salvage: boolean;
98  private readonly retries: number;
99  private readonly concurrency: number;
100  private readonly questionsFor: (call: ToolCall) => JevQuestions;
101
102  constructor(
103    private readonly asker: JevAsker,
104    options: JevClassifierOptions = {},
105  ) {
106    const style = options.questionStyle ?? 'useful';
107    this.name = style === 'upstream' ? 'jev-upstream' : 'jev';
108    this.questionsFor = style === 'upstream' ? upstreamQuestionsFor : usefulQuestionsFor;
109    this.salvage = options.salvagePartialBatches ?? true;
110    this.retries = Math.max(0, options.retries ?? 1);
111    this.concurrency = Math.max(1, options.concurrency ?? 4);
112  }
113
114  async score(candidates: readonly ToolCall[], context: ClassifierContext): Promise<ClassifierRun> {
115    const started = Date.now();
116    const scores = new Map<string, ClassifierScores>();
117    const unscored: string[] = [];
118    if (candidates.length === 0) {
119      return { scores, stats: { requests: 0, stateTokens: 0, stateStage: '', ms: 0, unscored } };
120    }
121    const fitted = fitState(context.messages, context.calls, context.options, context.sketches);
122    const batches = batchCalls(candidates, fitted.tokens, context.options, this.questionsFor);
123    let requests = 0;
124    const outcomes = await mapConcurrent(batches, this.concurrency, async (batch) => {
125        let lastError: unknown;
126        for (let attempt = 0; attempt <= this.retries; attempt++) {
127          requests++;
128          try {
129            return { batch, answers: await this.askBatch(fitted.state, batch) };
130          } catch (error) {
131            lastError = error;
132          }
133        }
134        if (!this.salvage) throw lastError;
135        return { batch, answers: undefined, error: lastError };
136      });
137    for (const outcome of outcomes) {
138      if (!outcome.answers) {
139        for (const call of outcome.batch) unscored.push(call.id);
140        continue;
141      }
142      for (const [id, answer] of outcome.answers) scores.set(id, answer);
143    }
144    if (scores.size === 0 && unscored.length > 0) {
145      const first = outcomes.find((outcome) => 'error' in outcome && outcome.error)?.error;
146      throw first instanceof Error ? first : new Error('every Jev batch failed');
147    }
148    return {
149      scores,
150      stats: {
151        requests,
152        stateTokens: fitted.tokens,
153        stateStage: fitted.stage,
154        ms: Date.now() - started,
155        unscored,
156      },
157    };
158  }
159
160  private async askBatch(
161    state: Parameters<JevAsker['ask']>[0],
162    batch: readonly ToolCall[],
163  ): Promise<Map<string, ClassifierScores>> {
164    const questions: JevQuestions = Object.assign({}, ...batch.map(this.questionsFor));
165    const { answers } = await this.asker.ask(state, questions);
166    return new Map(
167      batch.map((call) => [
168        call.id,
169        {
170          keepCall: probability(answers, `call_${call.id}`),
171          keepResult: probability(answers, `result_${call.id}`),
172        },
173      ]),
174    );
175  }
176}
177

1import type { ClassifierScores } from '../core/actions.js';
2import type { Ledger } from '../core/events.js';
3import type { Message, ResolvedCompactOptions, ToolCall } from '../types.js';
4
5/** What a classifier is shown besides the candidates. */
6export interface ClassifierContext {
7  messages: readonly Message[];
8  ledger: Ledger;
9  /** Every paired tool call, pinned ones included, in transcript order. */
10  calls: readonly ToolCall[];
11  options: ResolvedCompactOptions;
12  /** Per call id (`t1`, …): a short sketch of the result to show instead of a bare size note. */
13  sketches?: ReadonlyMap<string, string>;
14}
15
16export interface ClassifierRun {
17  /** Scores per call id (`t1`, …) for the candidates it was asked about. */
18  scores: Map<string, ClassifierScores>;
19  /** Diagnostics for the report. */
20  stats: {
21    requests: number;
22    stateTokens: number;
23    stateStage: string;
24    ms: number;
25    /** Calls the classifier could not score (a failed batch); the policy keeps them. */
26    unscored: string[];
27  };
28}
29
30/**
31 * Scores the non-pinned tool interactions. A classifier may be remote (Jev),
32 * local (the ruleset), or a recording of an earlier run (replay). It must not
33 * throw for a partial failure; it reports what it could not score instead and
34 * throws only when it cannot run at all.
35 */
36export interface Classifier {
37  readonly name: string;
38  /**
39   * The keep threshold this classifier's scores are calibrated for, used when
40   * the caller sets none. Jev's `noul` puts "unsure" at 0.5, so its threshold
41   * sits well below that; the ruleset centres on 0.5.
42   */
43  readonly defaultThreshold?: number;
44  score(candidates: readonly ToolCall[], context: ClassifierContext): Promise<ClassifierRun>;
45}
46

1import { noulAnswer } from './request.js';
2import { collectToolCalls, estimateTokens, fitState } from './state.js';
3import type {
4  CallAnswer,
5  CallDecision,
6  CompactOptions,
7  CompactResult,
8  CompactionState,
9  JevAsker,
10  JevQuestions,
11  Message,
12  ResolvedCompactOptions,
13  ToolCall,
14  ToolUse,
15} from './types.js';
16
17export const DEFAULT_OPTIONS: ResolvedCompactOptions = {
18  goal: '',
19  keepThreshold: 0.5,
20  preserveRecentMessages: 6,
21  maxStateTokens: 25_000,
22  maxRequestTokens: 30_000,
23  truncateHeadChars: 300,
24};
25
26/** Tokens the request envelope (`model`, key names) adds around state and questions. */
27const REQUEST_OVERHEAD_TOKENS = 20;
28
29function finite(value: number | undefined, fallback: number): number {
30  return typeof value === 'number' && Number.isFinite(value) ? value : fallback;
31}
32
33export function resolveOptions(options: CompactOptions = {}): ResolvedCompactOptions {
34  return {
35    goal: options.goal ?? DEFAULT_OPTIONS.goal,
36    keepThreshold: finite(options.keepThreshold, DEFAULT_OPTIONS.keepThreshold),
37    preserveRecentMessages: Math.max(
38      0,
39      Math.floor(
40        finite(options.preserveRecentMessages, DEFAULT_OPTIONS.preserveRecentMessages),
41      ),
42    ),
43    maxStateTokens: Math.max(1, finite(options.maxStateTokens, DEFAULT_OPTIONS.maxStateTokens)),
44    maxRequestTokens: Math.max(
45      1,
46      finite(options.maxRequestTokens, DEFAULT_OPTIONS.maxRequestTokens),
47    ),
48    truncateHeadChars: Math.max(
49      0,
50      Math.floor(finite(options.truncateHeadChars, DEFAULT_OPTIONS.truncateHeadChars)),
51    ),
52  };
53}
54
55/** The two `noul` questions asked about one call: keep the call, keep its result. */
56export function questionsFor(call: ToolCall): JevQuestions {
57  return {
58    [`call_${call.id}`]: {
59      type: 'noul',
60      instructions: `Tool call ${call.id} (${call.tool}) should stay in the history: knowing this call was made, with its input, still matters for what the assistant does next`,
61    },
62    [`result_${call.id}`]: {
63      type: 'noul',
64      instructions: `The full output of tool call ${call.id} (${call.tool}, ${call.resultChars} chars) should stay in the history verbatim: the assistant still needs its contents and re-running the tool would not do`,
65    },
66  };
67}
68
69/**
70 * Splits the candidate calls into batches whose questions, together with the
71 * (always complete) state, fit one request.
72 */
73export function batchCalls(
74  calls: readonly ToolCall[],
75  stateTokens: number,
76  options: Pick<ResolvedCompactOptions, 'maxRequestTokens'>,
77  questions: (call: ToolCall) => JevQuestions = questionsFor,
78): ToolCall[][] {
79  const budget = options.maxRequestTokens - stateTokens - REQUEST_OVERHEAD_TOKENS;
80  const batches: ToolCall[][] = [];
81  let current: ToolCall[] = [];
82  let currentTokens = 0;
83  for (const call of calls) {
84    const tokens = estimateTokens(JSON.stringify(questions(call)));
85    if (current.length > 0 && currentTokens + tokens > budget) {
86      batches.push(current);
87      current = [];
88      currentTokens = 0;
89    }
90    if (current.length === 0 && tokens > budget) {
91      throw new Error(
92        `state leaves no room for questions (~${stateTokens} of ${options.maxRequestTokens} tokens)`,
93      );
94    }
95    current.push(call);
96    currentTokens += tokens;
97  }
98  if (current.length > 0) batches.push(current);
99  return batches;
100}
101
102export function decideCall(
103  call: Pick<ToolCall, 'id' | 'tool' | 'pinned'>,
104  answer: CallAnswer,
105  options: Pick<ResolvedCompactOptions, 'keepThreshold'>,
106): CallDecision {
107  const base = { id: call.id, tool: call.tool, ...answer };
108  if (call.pinned) return { ...base, action: 'keep', reason: 'pinned' };
109  if (answer.keepResult >= options.keepThreshold) {
110    return { ...base, action: 'keep', reason: 'kept' };
111  }
112  if (answer.keepCall >= options.keepThreshold) {
113    return { ...base, action: 'drop_result', reason: 'result_dropped' };
114  }
115  return { ...base, action: 'drop_call', reason: 'call_dropped' };
116}
117
118async function askBatch(
119  asker: JevAsker,
120  state: CompactionState,
121  batch: readonly ToolCall[],
122): Promise<Map<string, CallAnswer>> {
123  const questions: JevQuestions = Object.assign({}, ...batch.map(questionsFor));
124  const { answers } = await asker.ask(state, questions);
125  return new Map(
126    batch.map((call) => [
127      call.id,
128      {
129        keepCall: noulAnswer(answers, `call_${call.id}`),
130        keepResult: noulAnswer(answers, `result_${call.id}`),
131      },
132    ]),
133  );
134}
135
136function truncatedResultText(text: string, isError: boolean, headChars: number): string {
137  if (text.length <= headChars + 120) return text;
138  const head = headChars > 0 ? `${text.slice(0, headChars)}\n` : '';
139  return `${head}[fast-jev-compaction truncated ${text.length - headChars} chars of this tool result${
140    isError ? ' (error)' : ''
141  }; re-run the tool if needed]`;
142}
143
144/**
145 * Rebuilds the conversation from the decisions. A dropped call disappears
146 * together with its result; a dropped result keeps a bounded head and note.
147 * Messages that lose all their content are removed; untouched messages are
148 * returned as the same objects they came in as.
149 */
150export function applyDecisions(
151  messages: readonly Message[],
152  decisions: readonly CallDecision[],
153  calls: readonly ToolCall[],
154  headChars: number,
155): Message[] {
156  const byId = new Map(calls.map((call) => [call.id, call]));
157  const actions = new Map<string, CallDecision['action']>();
158  for (const decision of decisions) {
159    const call = byId.get(decision.id);
160    if (call && decision.action !== 'keep') actions.set(call.tool_use_id, decision.action);
161  }
162  const kept: Message[] = [];
163  for (const message of messages) {
164    const touched =
165      message.toolUses.some((tool) => actions.has(tool.tool_use_id)) ||
166      (message.toolResults ?? []).some((result) => actions.has(result.tool_use_id));
167    if (!touched) {
168      kept.push(message);
169      continue;
170    }
171    const toolUses = message.toolUses
172      .filter((tool) => actions.get(tool.tool_use_id) !== 'drop_call')
173      .map((tool) => {
174        if (actions.get(tool.tool_use_id) !== 'drop_result') return tool;
175        const text = truncatedResultText(
176          tool.text ?? '',
177          tool.isError ?? false,
178          headChars,
179        );
180        if ((tool.text ?? '') === text) return tool;
181        const copy: ToolUse = {
182          tool_use_id: tool.tool_use_id,
183          tool: tool.tool,
184          input: tool.input,
185          text,
186        };
187        if (tool.isError) copy.isError = true;
188        return copy;
189      });
190    const toolResults = (message.toolResults ?? [])
191      .filter((result) => actions.get(result.tool_use_id) !== 'drop_call')
192      .map((result) => {
193        if (actions.get(result.tool_use_id) !== 'drop_result') return result;
194        const text = truncatedResultText(result.text, result.isError ?? false, headChars);
195        return text === result.text
196          ? result
197          : {
198              tool_use_id: result.tool_use_id,
199              text,
200              isError: result.isError,
201            };
202      });
203    if (
204      !message.toolUses.some(
205        (tool) => actions.get(tool.tool_use_id) === 'drop_call',
206      ) &&
207      !(message.toolResults ?? []).some(
208        (result) => actions.get(result.tool_use_id) === 'drop_call',
209      ) &&
210      toolUses.every((tool, index) => tool === message.toolUses[index]) &&
211      toolResults.every(
212        (result, index) => result === message.toolResults?.[index],
213      )
214    ) {
215      kept.push(message);
216      continue;
217    }
218    if (message.text.trim().length === 0 && toolUses.length === 0 && toolResults.length === 0) {
219      continue;
220    }
221    const rebuilt: Message = { role: message.role, text: message.text, toolUses };
222    if (toolResults.length > 0) rebuilt.toolResults = toolResults;
223    kept.push(rebuilt);
224  }
225  return kept;
226}
227
228/** Characters of text, tool input and tool output a message holds. */
229export function messageChars(message: Message): number {
230  let total = message.text.length;
231  for (const tool of message.toolUses) {
232    try {
233      total += JSON.stringify(tool.input).length;
234    } catch {
235      total += 20;
236    }
237  }
238  for (const result of message.toolResults ?? []) total += result.text.length;
239  return total;
240}
241
242export function reductionRatio(result: Pick<CompactResult, 'stats'>): number {
243  const { charsBefore, charsAfter } = result.stats;
244  return charsBefore === 0 ? 0 : (charsBefore - charsAfter) / charsBefore;
245}
246
247function count(decisions: readonly CallDecision[], reason: CallDecision['reason']): number {
248  return decisions.filter((decision) => decision.reason === reason).length;
249}
250
251/**
252 * Compacts a transcript by asking Jev, for every tool call outside the pinned
253 * first and newest messages, whether the call and whether its result must
254 * stay. The whole history (results omitted, fitted into `maxStateTokens`) is
255 * sent as state with every batch of questions. Throws when Jev fails or the
256 * history cannot be fitted; the caller decides whether to fall back.
257 */
258export async function compact(
259  messages: readonly Message[],
260  asker: JevAsker,
261  options: CompactOptions = {},
262): Promise<CompactResult> {
263  const started = Date.now();
264  const resolved = resolveOptions(options);
265  const calls = collectToolCalls(messages, resolved.preserveRecentMessages);
266  const candidates = calls.filter((call) => !call.pinned);
267  const charsBefore = messages.reduce((sum, message) => sum + messageChars(message), 0);
268
269  let fitted: { tokens: number; stage: string } = { tokens: 0, stage: '' };
270  let batches: ToolCall[][] = [];
271  const answers = new Map<string, CallAnswer>();
272  if (candidates.length > 0) {
273    const state = fitState(messages, calls, resolved);
274    fitted = state;
275    batches = batchCalls(candidates, state.tokens, resolved);
276    const answered = await Promise.all(
277      batches.map((batch) => askBatch(asker, state.state, batch)),
278    );
279    for (const map of answered) for (const [id, answer] of map) answers.set(id, answer);
280  }
281
282  const decisions = calls.map((call) =>
283    decideCall(call, answers.get(call.id) ?? { keepCall: 1, keepResult: 1 }, resolved),
284  );
285  const kept = applyDecisions(
286    messages,
287    decisions,
288    calls,
289    resolved.truncateHeadChars,
290  );
291  return {
292    messages: kept,
293    decisions,
294    stats: {
295      messagesBefore: messages.length,
296      messagesAfter: kept.length,
297      charsBefore,
298      charsAfter: kept.reduce((sum, message) => sum + messageChars(message), 0),
299      calls: calls.length,
300      kept: count(decisions, 'kept'),
301      resultsDropped: count(decisions, 'result_dropped'),
302      callsDropped: count(decisions, 'call_dropped'),
303      pinned: count(decisions, 'pinned'),
304      stateTokens: fitted.tokens,
305      stateStage: fitted.stage,
306      requests: batches.length,
307      ms: Date.now() - started,
308    },
309  };
310}
311

1import { hashJson, hashText } from './hash.js';
2import { estimateTokens } from '../state.js';
3import type { Message } from '../types.js';
4
5/**
6 * The canonical, host-independent view of a conversation: a flat stream of
7 * events. A host adapter turns its transcript into events (and back); every
8 * policy, archive record and memory refers to events by their stable ids.
9 */
10export type EventRole = 'user' | 'assistant' | 'tool';
11
12export type EventKind = 'user_text' | 'assistant_text' | 'tool_use' | 'tool_result';
13
14export interface NormalizedEvent {
15  /** Stable across compactions of the same session: derived from content, not position. */
16  id: string;
17  /** Position in the stream this ledger was built from (0-based). */
18  seq: number;
19  role: EventRole;
20  kind: EventKind;
21  /** Exact content: the text, or the serialised tool input, or the tool result. */
22  content: string;
23  toolName?: string;
24  toolUseId?: string;
25  /** For a tool_use: the parsed input, kept so tools can be re-run. */
26  toolInput?: Record<string, unknown>;
27  isError?: boolean;
28  /** Index of the `Message` this event came from. */
29  messageIndex: number;
30  /** Fingerprint of `content` (plus tool name and input for a tool_use). */
31  contentHash: string;
32  tokenEstimate: number;
33  metadata: Record<string, unknown>;
34}
35
36/** Events grouped back by their source message, in order. */
37export interface Ledger {
38  events: NormalizedEvent[];
39  byId: Map<string, NormalizedEvent>;
40  /** tool_use_id → [tool_use event, tool_result event or undefined]. */
41  interactions: Map<string, { use: NormalizedEvent; result?: NormalizedEvent }>;
42}
43
44function serialiseInput(input: Record<string, unknown>): string {
45  try {
46    return JSON.stringify(input) ?? '{}';
47  } catch {
48    return '[unserializable input]';
49  }
50}
51
52/**
53 * The stable id of an event: a fingerprint of what it is and says, so the same
54 * message gets the same id no matter how many messages before it were removed.
55 * Identical repeated texts get a `~2`, `~3` … suffix in stream order.
56 */
57export function eventId(
58  kind: EventKind,
59  content: string,
60  toolUseId: string | undefined,
61  taken: Set<string>,
62): string {
63  const base = `e_${hashText(`${kind}\u0000${toolUseId ?? ''}\u0000${content}`)}`;
64  let id = base;
65  for (let n = 2; taken.has(id); n++) id = `${base}~${n}`;
66  taken.add(id);
67  return id;
68}
69
70/** Turns a transcript into its event stream, pairing tool uses with results. */
71export function buildLedger(messages: readonly Message[]): Ledger {
72  const events: NormalizedEvent[] = [];
73  const taken = new Set<string>();
74  const push = (event: Omit<NormalizedEvent, 'id' | 'seq' | 'tokenEstimate' | 'contentHash'>): void => {
75    const hashed =
76      event.kind === 'tool_use'
77        ? hashText(`${event.toolName}\u0000${event.content}`)
78        : hashText(event.content);
79    events.push({
80      ...event,
81      id: eventId(event.kind, event.content, event.toolUseId, taken),
82      seq: events.length,
83      contentHash: hashed,
84      tokenEstimate: estimateTokens(event.content),
85    });
86  };
87  messages.forEach((message, messageIndex) => {
88    if (message.text.trim().length > 0) {
89      push({
90        role: message.role,
91        kind: message.role === 'user' ? 'user_text' : 'assistant_text',
92        content: message.text,
93        messageIndex,
94        metadata: {},
95      });
96    }
97    for (const tool of message.toolUses) {
98      push({
99        role: 'assistant',
100        kind: 'tool_use',
101        content: serialiseInput(tool.input),
102        toolName: tool.tool,
103        toolUseId: tool.tool_use_id,
104        toolInput: tool.input,
105        messageIndex,
106        metadata: {},
107      });
108    }
109    for (const result of message.toolResults ?? []) {
110      push({
111        role: 'tool',
112        kind: 'tool_result',
113        content: result.text,
114        toolUseId: result.tool_use_id,
115        isError: result.isError ?? false,
116        messageIndex,
117        metadata: {},
118      });
119    }
120  });
121  const byId = new Map(events.map((event) => [event.id, event]));
122  const interactions = new Map<string, { use: NormalizedEvent; result?: NormalizedEvent }>();
123  for (const event of events) {
124    if (event.kind === 'tool_use' && event.toolUseId) {
125      interactions.set(event.toolUseId, { use: event });
126    }
127  }
128  for (const event of events) {
129    if (event.kind === 'tool_result' && event.toolUseId) {
130      const pair = interactions.get(event.toolUseId);
131      if (pair) {
132        pair.result = event;
133        pair.use.metadata['resultId'] = event.id;
134      }
135    }
136  }
137  for (const [toolUseId, pair] of interactions) {
138    if (pair.result) pair.result.toolName = pair.use.toolName;
139    pair.use.metadata['toolUseId'] = toolUseId;
140  }
141  return { events, byId, interactions };
142}
143
144/** Sum of the estimated tokens of a set of events. */
145export function eventTokens(events: Iterable<NormalizedEvent>): number {
146  let total = 0;
147  for (const event of events) total += event.tokenEstimate;
148  return total;
149}
150
151/** A fingerprint of a whole transcript (for cassettes, reports and change detection). */
152export function transcriptHash(messages: readonly Message[]): string {
153  return hashJson(
154    messages.map((message) => [
155      message.role,
156      message.text,
157      message.toolUses.map((tool) => [tool.tool_use_id, tool.tool, serialiseInput(tool.input)]),
158      (message.toolResults ?? []).map((result) => [result.tool_use_id, result.text, result.isError ?? false]),
159    ]),
160  );
161}
162

1import type { Ledger, NormalizedEvent } from './events.js';
2import type { Message, ToolCall } from '../types.js';
3
4/**
5 * Deterministic rules run before and around the classifier. They never evict;
6 * they only forbid eviction (a protection) or flag content the rest of the
7 * system must treat as durable (a constraint). Rules are cheap, local and
8 * explainable, and they are what keeps the product safe when the classifier
9 * is wrong, slow, or absent.
10 */
11
12export interface Protection {
13  /** Rule name, e.g. `unresolved_error`, `referenced_later`, `non_reproducible`, `current_file`. */
14  rule: string;
15  detail: string;
16  refs?: string[];
17}
18
19export interface ConstraintHit {
20  /** The user text event holding the constraint. */
21  eventId: string;
22  messageIndex: number;
23  /** The sentence (bounded) that carries it, verbatim. */
24  text: string;
25  /** Which cue matched: `never`, `must not`, `do not`, `always`, `must`, `required`, `only` … */
26  cue: string;
27}
28
29export interface RuleOptions {
30  /** Keep an error result while no later call of the same tool succeeded. Default true. */
31  protectUnresolvedErrors: boolean;
32  /** Keep a result a later message references by path, symbol or error string. Default true. */
33  protectReferenced: boolean;
34  /** Keep results of tools whose output cannot be reproduced by re-running. */
35  nonReproducibleTools: readonly string[];
36  /** Keep results that mention a file edited in the recent window. Default true. */
37  protectCurrentFiles: boolean;
38  /** How many newest messages count as "current" for file protection. Default 12. */
39  currentWindow: number;
40}
41
42export const DEFAULT_RULE_OPTIONS: RuleOptions = {
43  protectUnresolvedErrors: true,
44  protectReferenced: true,
45  nonReproducibleTools: ['WebFetch', 'WebSearch', 'AskUserQuestion'],
46  protectCurrentFiles: true,
47  currentWindow: 12,
48};
49
50export function resolveRuleOptions(options: Partial<RuleOptions> = {}): RuleOptions {
51  return { ...DEFAULT_RULE_OPTIONS, ...options };
52}
53
54/* ---------------------------------------------------------------- constraints */
55
56const CONSTRAINT_CUES = [
57  'never',
58  'must not',
59  'mustn\'t',
60  'do not',
61  'don\'t',
62  'not allowed',
63  'forbidden',
64  'prohibited',
65  'always',
66  'must ',
67  'required',
68  'only ',
69  'make sure',
70  'be careful',
71  'under no circumstances',
72] as const;
73
74const SENTENCE_SPLIT = /(?<=[.!?\n])\s+/;
75
76/**
77 * Sentences of user text that read as explicit instructions or constraints.
78 * Lexical cues only: the classifier and memory extractor refine them, but a
79 * sentence flagged here is never evicted by any policy in this build.
80 */
81export function findConstraints(messages: readonly Message[], ledger?: Ledger): ConstraintHit[] {
82  const hits: ConstraintHit[] = [];
83  messages.forEach((message, messageIndex) => {
84    if (message.role !== 'user' || message.text.trim().length === 0) return;
85    if ((message.toolResults ?? []).length > 0 && message.text.trim().length === 0) return;
86    const event = ledger?.events.find(
87      (e) => e.messageIndex === messageIndex && e.kind === 'user_text',
88    );
89    for (const sentence of message.text.split(SENTENCE_SPLIT)) {
90      const lower = sentence.toLowerCase();
91      const cue = CONSTRAINT_CUES.find((c) => lower.includes(c));
92      if (!cue) continue;
93      // Skip sentences that are clearly quoting a tool result or code.
94      if (sentence.trim().startsWith('```') || sentence.trim().startsWith('>')) continue;
95      hits.push({
96        eventId: event?.id ?? `msg_${messageIndex}`,
97        messageIndex,
98        text: sentence.trim().slice(0, 500),
99        cue: cue.trim(),
100      });
101    }
102  });
103  return hits;
104}
105
106/* ---------------------------------------------------------------- helpers */
107
108const PATH_LIKE = /(?:[A-Za-z]:)?(?:[\w.-]+[\\/])+[\w.-]+\.[A-Za-z0-9]{1,8}/g;
109
110/** File paths mentioned in some text (relative or absolute, with an extension). */
111export function pathsIn(text: string): Set<string> {
112  const paths = new Set<string>();
113  for (const match of text.matchAll(PATH_LIKE)) paths.add(normalisePath(match[0]));
114  return paths;
115}
116
117export function normalisePath(path: string): string {
118  return path.replace(/\\/g, '/').replace(/^\.\//, '').toLowerCase();
119}
120
121function inputPath(input: Record<string, unknown>): string | undefined {
122  for (const key of ['file_path', 'path', 'notebook_path', 'filePath']) {
123    const value = input[key];
124    if (typeof value === 'string' && value.length > 0) return normalisePath(value);
125  }
126  return undefined;
127}
128
129/* ---------------------------------------------------------------- protections */
130
131const EDITING_TOOLS = new Set(['Edit', 'Write', 'MultiEdit', 'NotebookEdit']);
132
133/**
134 * Files touched by an editing tool in the newest `window` messages: the
135 * "current files" of the task.
136 */
137export function currentFiles(messages: readonly Message[], window: number): Set<string> {
138  const files = new Set<string>();
139  const from = Math.max(0, messages.length - window);
140  for (let i = from; i < messages.length; i++) {
141    for (const tool of messages[i]!.toolUses) {
142      if (!EDITING_TOOLS.has(tool.tool)) continue;
143      const path = inputPath(tool.input);
144      if (path) files.add(path);
145    }
146  }
147  return files;
148}
149
150/**
151 * Paths edited by a later call than `call`: a read of one of these shows
152 * content that re-running would not reproduce (plan §16).
153 */
154export function targetChangedLater(call: ToolCall, calls: readonly ToolCall[]): boolean {
155  const path = inputPath(call.input);
156  if (!path) return false;
157  return calls.some(
158    (later) =>
159      later.callIndex > call.callIndex && EDITING_TOOLS.has(later.tool) && inputPath(later.input) === path,
160  );
161}
162
163/**
164 * Whether an error result is still unresolved: no later call of the same tool
165 * (for Bash, the same command; for file tools, the same path) succeeded.
166 */
167function isUnresolvedError(call: ToolCall, calls: readonly ToolCall[]): boolean {
168  if (!call.isError) return false;
169  const path = inputPath(call.input);
170  const command = typeof call.input['command'] === 'string' ? call.input['command'] : undefined;
171  for (const later of calls) {
172    if (later.callIndex <= call.callIndex || later.tool !== call.tool || later.isError) continue;
173    if (command !== undefined) {
174      if (later.input['command'] === command) return false;
175      continue;
176    }
177    if (path !== undefined) {
178      if (inputPath(later.input) === path) return false;
179      continue;
180    }
181    return false;
182  }
183  return true;
184}
185
186export interface ProtectionContext {
187  messages: readonly Message[];
188  ledger: Ledger;
189  calls: readonly ToolCall[];
190  options: RuleOptions;
191  /** result event id → ids of later events that reference it (from the dependency graph). */
192  referencedBy?: ReadonlyMap<string, readonly string[]>;
193}
194
195function resultEvent(call: ToolCall, ledger: Ledger): NormalizedEvent | undefined {
196  return ledger.interactions.get(call.tool_use_id)?.result;
197}
198
199/** Every reason this call's result must stay verbatim; empty when the classifier may decide. */
200export function protectionsFor(call: ToolCall, context: ProtectionContext): Protection[] {
201  const { options } = context;
202  const found: Protection[] = [];
203  const result = resultEvent(call, context.ledger);
204
205  if (options.protectUnresolvedErrors && isUnresolvedError(call, context.calls)) {
206    found.push({
207      rule: 'unresolved_error',
208      detail: `${call.tool} failed and no later ${call.tool} call on the same target succeeded`,
209    });
210  }
211
212  if (
213    options.nonReproducibleTools.some(
214      (name) => name === call.tool || (name.endsWith('*') && call.tool.startsWith(name.slice(0, -1))),
215    )
216  ) {
217    found.push({
218      rule: 'non_reproducible',
219      detail: `${call.tool} output cannot be reproduced by re-running the tool`,
220    });
221  }
222
223  if (options.protectReferenced && result) {
224    const refs = context.referencedBy?.get(result.id);
225    if (refs && refs.length > 0) {
226      found.push({
227        rule: 'referenced_later',
228        detail: `a later message refers to this result (${refs.length} reference${refs.length === 1 ? '' : 's'})`,
229        refs: [...refs],
230      });
231    }
232  }
233
234  if (options.protectCurrentFiles) {
235    const files = currentFiles(context.messages, options.currentWindow);
236    const path = inputPath(call.input);
237    if (path && files.has(path) && !EDITING_TOOLS.has(call.tool)) {
238      found.push({
239        rule: 'current_file',
240        detail: `${path} was edited in the newest ${options.currentWindow} messages`,
241      });
242    }
243  }
244
245  return found;
246}
247

1import { MemoryArchive } from '../archive/memory-store.js';
2import type { ArchiveRecord, ArchiveStore } from '../archive/types.js';
3import { RulesetClassifier } from '../classifiers/ruleset.js';
4import { JevClassifier } from '../classifiers/jev.js';
5import type { Classifier, ClassifierRun } from '../classifiers/types.js';
6import { messageChars, resolveOptions } from '../compact.js';
7import { evicts, toBaselineAction, type ActionDecision, type ContextAction } from '../core/actions.js';
8import { buildDependencyGraph, type DependencyGraph } from '../core/dependencies.js';
9import { buildLedger, eventTokens, type Ledger, type NormalizedEvent } from '../core/events.js';
10import { decideInteraction, resolvePolicyOptions, type PolicyOptions } from '../core/policy.js';
11import { Redactor } from '../core/redact.js';
12import {
13  findConstraints,
14  protectionsFor,
15  resolveRuleOptions,
16  targetChangedLater,
17  type ConstraintHit,
18  type Protection,
19  type RuleOptions,
20} from '../core/rules.js';
21import { sketchResult } from '../core/sketch.js';
22import { collectToolCalls } from '../state.js';
23import type {
24  CallDecision,
25  CompactOptions,
26  CompactResult,
27  JevAsker,
28  Message,
29  ResolvedCompactOptions,
30  ToolCall,
31  ToolResult,
32  ToolUse,
33} from '../types.js';
34
35export interface OptimizeOptions extends CompactOptions {
36  /** Identifies the conversation in the archive. Default `session`. */
37  sessionId?: string;
38  /** Identifies this compaction in the archive. Default a time-based id. */
39  compactionId?: string;
40  /** Where evicted content goes. Default a fresh in-memory archive (returned in the result). */
41  archive?: ArchiveStore;
42  /** Scores the candidates. Default: Jev over `asker` when given, otherwise the local ruleset. */
43  classifier?: Classifier;
44  asker?: JevAsker;
45  rules?: Partial<RuleOptions>;
46  policy?: Partial<Omit<PolicyOptions, 'keepThreshold'>>;
47  /** Show the classifier a sketch of each result instead of only its size. Default true. */
48  sketches?: boolean;
49  /** Redact secrets from everything shown to the classifier. Default true. */
50  redact?: boolean | Redactor;
51  /** Note in an assistant message when its tool calls were removed (upstream #65). Default true. */
52  markRemovedCalls?: boolean;
53  now?: () => Date;
54}
55
56export interface OptimizeReport {
57  sessionId: string;
58  compactionId: string;
59  classifier: string;
60  messages: { before: number; after: number };
61  tokens: { before: number; after: number; archived: number };
62  chars: { before: number; after: number };
63  actions: Record<ContextAction, number>;
64  protections: Record<string, number>;
65  constraints: number;
66  duplicates: number;
67  candidates: number;
68  classified: number;
69  unscored: number;
70  redactedSecrets: number;
71  classifierStats: ClassifierRun['stats'];
72  ms: number;
73}
74
75export interface OptimizeResult extends CompactResult {
76  actions: ActionDecision[];
77  archived: ArchiveRecord[];
78  archive: ArchiveStore;
79  constraints: ConstraintHit[];
80  report: OptimizeReport;
81}
82
83function emptyActionCounts(): Record<ContextAction, number> {
84  return {
85    PIN_VERBATIM: 0,
86    KEEP_VERBATIM: 0,
87    KEEP_HEAD_TAIL: 0,
88    EXTRACT_MEMORY_AND_ARCHIVE: 0,
89    ARCHIVE_ONLY: 0,
90    REPLACE_WITH_REFERENCE: 0,
91    RERUN_ON_DEMAND: 0,
92    DROP_REDUNDANT: 0,
93  };
94}
95
96function defaultCompactionId(now: Date): string {
97  return `c_${now.getTime().toString(36)}`;
98}
99
100/** Copies of the messages with every text, tool input and result redacted. */
101function redactMessages(messages: readonly Message[], redactor: Redactor): Message[] {
102  return messages.map((message) => {
103    const copy: Message = {
104      role: message.role,
105      text: redactor.redact(message.text),
106      toolUses: message.toolUses.map((tool) => {
107        const use: ToolUse = {
108          tool_use_id: tool.tool_use_id,
109          tool: tool.tool,
110          input: redactor.redactValue(tool.input),
111        };
112        if (tool.text !== undefined) use.text = redactor.redact(tool.text);
113        if (tool.isError) use.isError = true;
114        return use;
115      }),
116    };
117    if (message.toolResults) {
118      copy.toolResults = message.toolResults.map((result) => {
119        const out: ToolResult = { tool_use_id: result.tool_use_id, text: redactor.redact(result.text) };
120        if (result.isError !== undefined) out.isError = result.isError;
121        return out;
122      });
123    }
124    return copy;
125  });
126}
127
128function inputSummary(call: ToolCall): string {
129  for (const key of ['file_path', 'path', 'command', 'pattern', 'url', 'query', 'prompt']) {
130    const value = call.input[key];
131    if (typeof value === 'string' && value.length > 0) {
132      const flat = value.replace(/\s+/g, ' ');
133      const head = `${key}=${flat.length > 80 ? `${flat.slice(0, 79)}…` : flat}`;
134      // A chunked read is only identifiable by its offset.
135      const offset = call.input['offset'];
136      return typeof offset === 'number' && offset > 1 ? `${head} offset=${offset}` : head;
137    }
138  }
139  return '';
140}
141
142/**
143 * The text left in place of an evicted result: a bounded head, then a note
144 * naming the archive record so the assistant (or the user) can bring the
145 * exact content back.
146 */
147export function stubResultText(
148  text: string,
149  call: ToolCall,
150  action: ContextAction,
151  archiveId: string,
152  headChars: number,
153): string {
154  if (text.length <= headChars + 120) return text;
155  const head = headChars > 0 ? `${text.slice(0, headChars)}\n` : '';
156  const what = `${text.length - headChars} more chars of this ${call.tool} result${call.isError ? ' (error)' : ''}`;
157  const how =
158    action === 'RERUN_ON_DEMAND'
159      ? `re-run ${call.tool} with the same input to reproduce it, or /lossless restore ${archiveId}`
160      : `/lossless restore ${archiveId} brings it back verbatim, or re-run the tool`;
161  const about = inputSummary(call);
162  return `${head}[lossless-compact archived ${archiveId}: ${what}${about ? ` (${about})` : ''}; ${how}]`;
163}
164
165/** The note appended to an assistant message whose tool calls were removed. */
166export function removedCallsMarker(removed: readonly { call: ToolCall; archiveId: string }[]): string {
167  // Terse on purpose: one of these stands for every removed call in a live
168  // session (Claude Code gives each call its own message), and the
169  // compaction note explains the mechanism once.
170  const items = removed.map((r) => `${`${r.call.tool} ${inputSummary(r.call)}`.trim()} → ${r.archiveId}`).join('; ');
171  return `[lossless-compact archived ${removed.length} tool call${removed.length === 1 ? '' : 's'} made here: ${items} — out of context; /lossless restore <id>]`;
172}
173
174interface Applied {
175  messages: Message[];
176  archived: ArchiveRecord[];
177}
178
179function archiveRecord(
180  event: NormalizedEvent,
181  decision: ActionDecision,
182  options: { sessionId: string; compactionId: string; archivedAt: string },
183  related: ArchiveRecord['related'],
184  metadata: Record<string, unknown>,
185): ArchiveRecord {
186  const record: ArchiveRecord = {
187    id: event.id,
188    sessionId: options.sessionId,
189    seq: event.seq,
190    role: event.role,
191    kind: event.kind,
192    content: event.content,
193    contentHash: event.contentHash,
194    tokenEstimate: event.tokenEstimate,
195    archivedAt: options.archivedAt,
196    compactionId: options.compactionId,
197    action: decision.action,
198    reasons: decision.reasons,
199    related,
200    metadata,
201  };
202  if (event.toolName) record.toolName = event.toolName;
203  if (event.toolUseId) record.toolUseId = event.toolUseId;
204  if (event.isError) record.isError = true;
205  return record;
206}
207
208/**
209 * Rebuilds the transcript from the decisions and produces the archive records
210 * of everything that left it. Untouched messages are returned as the same
211 * objects; no result is ever left without its call; a message that loses all
212 * its content is removed.
213 */
214export function applyActions(
215  messages: readonly Message[],
216  decisions: readonly ActionDecision[],
217  calls: readonly ToolCall[],
218  ledger: Ledger,
219  options: {
220    sessionId: string;
221    compactionId: string;
222    archivedAt: string;
223    headChars: number;
224    markRemovedCalls: boolean;
225  },
226): Applied {
227  const byUseId = new Map(calls.map((call) => [call.tool_use_id, call]));
228  const byEventId = new Map(decisions.map((decision) => [decision.eventId, decision]));
229  const decisionOf = new Map<string, ActionDecision>();
230  for (const call of calls) {
231    const pair = ledger.interactions.get(call.tool_use_id);
232    const decision = pair && byEventId.get(pair.use.id);
233    if (decision && evicts(decision.action)) decisionOf.set(call.tool_use_id, decision);
234  }
235
236  const archived: ArchiveRecord[] = [];
237  const archivedIds = new Set<string>();
238  const archiveInteraction = (call: ToolCall, decision: ActionDecision): string => {
239    const pair = ledger.interactions.get(call.tool_use_id)!;
240    // A tool_use and its tool_result normally live in different messages, so
241    // this runs once from each message's loop below; only the first call must
242    // do the archiving, or the second one's empty `ids` clobbers `archiveIds`.
243    if (decision.archiveIds) return pair.result?.id ?? pair.use.id;
244    const baseline = toBaselineAction(decision.action);
245    const meta: Record<string, unknown> = { tool: call.tool, input: call.input };
246    for (const key of ['file_path', 'path', 'command', 'pattern', 'url']) {
247      if (typeof call.input[key] === 'string') meta[key] = call.input[key];
248    }
249    if (decision.action === 'RERUN_ON_DEMAND') meta['rerun'] = { tool: call.tool, input: call.input };
250    const duplicate = decision.reasons.find((r) => r.code === 'duplicate')?.refs?.[0];
251    if (duplicate) meta['duplicateOf'] = duplicate;
252    const ids: string[] = [];
253    if (pair.result && !archivedIds.has(pair.result.id)) {
254      archived.push(
255        archiveRecord(
256          pair.result,
257          decision,
258          options,
259          [{ relation: 'call', id: pair.use.id }],
260          { ...meta, resultChars: pair.result.content.length, isError: call.isError },
261        ),
262      );
263      archivedIds.add(pair.result.id);
264      ids.push(pair.result.id);
265    }
266    if (baseline === 'drop_call' && !archivedIds.has(pair.use.id)) {
267      archived.push(
268        archiveRecord(
269          pair.use,
270          decision,
271          options,
272          pair.result ? [{ relation: 'result', id: pair.result.id }] : [],
273          meta,
274        ),
275      );
276      archivedIds.add(pair.use.id);
277      ids.push(pair.use.id);
278    }
279    decision.archiveIds = ids;
280    return pair.result?.id ?? pair.use.id;
281  };
282
283  const kept: Message[] = [];
284  /** Marker-only messages by the calls they stand for, so adjacent ones merge into one line. */
285  const markerRuns = new Map<Message, { call: ToolCall; archiveId: string }[]>();
286  for (const message of messages) {
287    const touched =
288      message.toolUses.some((tool) => decisionOf.has(tool.tool_use_id)) ||
289      (message.toolResults ?? []).some((result) => decisionOf.has(result.tool_use_id));
290    if (!touched) {
291      kept.push(message);
292      continue;
293    }
294    const removed: { call: ToolCall; archiveId: string }[] = [];
295    const toolUses: ToolUse[] = [];
296    for (const tool of message.toolUses) {
297      const decision = decisionOf.get(tool.tool_use_id);
298      const call = byUseId.get(tool.tool_use_id);
299      if (!decision || !call) {
300        toolUses.push(tool);
301        continue;
302      }
303      const baseline = toBaselineAction(decision.action);
304      const archiveId = archiveInteraction(call, decision);
305      if (baseline === 'drop_call') {
306        removed.push({ call, archiveId });
307        continue;
308      }
309      const text = stubResultText(tool.text ?? '', call, decision.action, archiveId, options.headChars);
310      if ((tool.text ?? '') === text) {
311        toolUses.push(tool);
312        continue;
313      }
314      const copy: ToolUse = { tool_use_id: tool.tool_use_id, tool: tool.tool, input: tool.input, text };
315      if (tool.isError) copy.isError = true;
316      toolUses.push(copy);
317    }
318    const toolResults: ToolResult[] = [];
319    for (const result of message.toolResults ?? []) {
320      const decision = decisionOf.get(result.tool_use_id);
321      const call = byUseId.get(result.tool_use_id);
322      if (!decision || !call) {
323        toolResults.push(result);
324        continue;
325      }
326      const baseline = toBaselineAction(decision.action);
327      const archiveId = archiveInteraction(call, decision);
328      if (baseline === 'drop_call') continue;
329      const text = stubResultText(result.text, call, decision.action, archiveId, options.headChars);
330      if (text === result.text) {
331        toolResults.push(result);
332        continue;
333      }
334      const copy: ToolResult = { tool_use_id: result.tool_use_id, text };
335      if (result.isError !== undefined) copy.isError = result.isError;
336      toolResults.push(copy);
337    }
338    let text = message.text;
339    if (removed.length > 0 && options.markRemovedCalls) {
340      // Claude Code puts each tool call in its own text-less assistant
341      // message, so a message often has nothing left once its call goes:
342      // it becomes the marker alone, and a run of them merges below.
343      text = message.text.trim().length > 0 ? `${message.text}\n\n${removedCallsMarker(removed)}` : removedCallsMarker(removed);
344    }
345    const originalResults = message.toolResults ?? [];
346    if (
347      removed.length === 0 &&
348      toolUses.length === message.toolUses.length &&
349      toolUses.every((tool, index) => tool === message.toolUses[index]) &&
350      toolResults.length === originalResults.length &&
351      toolResults.every((result, index) => result === originalResults[index])
352    ) {
353      kept.push(message);
354      continue;
355    }
356    if (text.trim().length === 0 && toolUses.length === 0 && toolResults.length === 0) continue;
357    const rebuilt: Message = { role: message.role, text, toolUses };
358    if (toolResults.length > 0) rebuilt.toolResults = toolResults;
359    const markerOnly = message.text.trim().length === 0 && toolUses.length === 0 && toolResults.length === 0;
360    const previous = kept[kept.length - 1];
361    const previousRemoved = previous ? markerRuns.get(previous) : undefined;
362    if (markerOnly && previous && previousRemoved) {
363      const run = [...previousRemoved, ...removed];
364      const merged: Message = { role: 'assistant', text: removedCallsMarker(run), toolUses: [] };
365      kept[kept.length - 1] = merged;
366      markerRuns.set(merged, run);
367      continue;
368    }
369    if (markerOnly) markerRuns.set(rebuilt, removed);
370    kept.push(rebuilt);
371  }
372  return { messages: kept, archived };
373}
374
375/**
376 * A later reference to a result is strong when it quotes it, repeats an error
377 * from it, names its tool id, or comes from the user; a path or symbol the
378 * assistant mentions is weak. Strong references protect, weak ones only raise
379 * the keep probability (plan §11.C).
380 */
381export function splitReferences(
382  graph: DependencyGraph,
383  ledger: Ledger,
384): { strong: Map<string, string[]>; weak: Map<string, string[]> } {
385  const strong = new Map<string, string[]>();
386  const weak = new Map<string, string[]>();
387  for (const edge of graph.edges) {
388    const from = ledger.byId.get(edge.from);
389    const isStrong =
390      edge.via === 'error' || edge.via === 'quote' || edge.via === 'tool_id' || from?.kind === 'user_text';
391    const target = isStrong ? strong : weak;
392    const list = target.get(edge.to) ?? [];
393    if (!list.includes(edge.from)) list.push(edge.from);
394    target.set(edge.to, list);
395  }
396  for (const [id] of strong) weak.delete(id);
397  return { strong, weak };
398}
399
400/** Candidates whose exact interaction (tool, input and result) recurs later; the later one survives. */
401function findDuplicates(
402  calls: readonly ToolCall[],
403  ledger: Ledger,
404): Map<string, string> {
405  const latest = new Map<string, { call: ToolCall; resultId: string }>();
406  const duplicateOf = new Map<string, string>();
407  const key = (call: ToolCall, result: NormalizedEvent): string =>
408    `${call.tool}\u0000${ledger.interactions.get(call.tool_use_id)!.use.contentHash}\u0000${result.contentHash}`;
409  for (let i = calls.length - 1; i >= 0; i--) {
410    const call = calls[i]!;
411    const result = ledger.interactions.get(call.tool_use_id)?.result;
412    if (!result || result.content.length < 40) continue;
413    const k = key(call, result);
414    const later = latest.get(k);
415    if (later) {
416      if (!call.pinned) duplicateOf.set(call.id, later.resultId);
417    } else {
418      latest.set(k, { call, resultId: result.id });
419    }
420  }
421  return duplicateOf;
422}
423
424function toCallDecision(call: ToolCall, decision: ActionDecision): CallDecision {
425  const action = toBaselineAction(decision.action);
426  return {
427    id: call.id,
428    tool: call.tool,
429    keepCall: decision.scores?.keepCall ?? 1,
430    keepResult: decision.scores?.keepResult ?? 1,
431    action,
432    reason:
433      decision.action === 'PIN_VERBATIM'
434        ? 'pinned'
435        : action === 'keep'
436          ? 'kept'
437          : action === 'drop_result'
438            ? 'result_dropped'
439            : 'call_dropped',
440  };
441}
442
443/**
444 * The context optimizer: pins, protects, de-duplicates, classifies, decides,
445 * archives and rebuilds. Everything that leaves the transcript is in the
446 * archive before the new transcript is returned. Throws only when the
447 * classifier cannot run at all (the caller decides whether to fall back);
448 * a partially failed classification keeps whatever it could not score.
449 */
450export async function optimize(
451  messages: readonly Message[],
452  options: OptimizeOptions = {},
453): Promise<OptimizeResult> {
454  const started = Date.now();
455  const now = options.now ?? (() => new Date());
456  const classifier: Classifier =
457    options.classifier ?? (options.asker ? new JevClassifier(options.asker) : new RulesetClassifier());
458  const resolved: ResolvedCompactOptions = resolveOptions({
459    ...options,
460    keepThreshold: options.keepThreshold ?? classifier.defaultThreshold,
461  });
462  const ruleOptions = resolveRuleOptions(options.rules);
463  const policyOptions = resolvePolicyOptions({ ...options.policy, keepThreshold: resolved.keepThreshold });
464  const sessionId = options.sessionId ?? 'session';
465  const compactionId = options.compactionId ?? defaultCompactionId(now());
466  const archive = options.archive ?? new MemoryArchive();
467  const redactor =
468    options.redact === false ? undefined : options.redact instanceof Redactor ? options.redact : new Redactor();
469
470  const ledger = buildLedger(messages);
471  const calls = collectToolCalls(messages, resolved.preserveRecentMessages);
472  const seen = new Map<string, number>();
473  for (const call of calls) seen.set(call.tool_use_id, (seen.get(call.tool_use_id) ?? 0) + 1);
474  const graph = buildDependencyGraph(ledger);
475  const { strong: strongRefs, weak: weakRefs } = splitReferences(graph, ledger);
476  const constraints = findConstraints(messages, ledger);
477  const duplicates = findDuplicates(calls, ledger);
478
479  const candidates = calls.filter((call) => !call.pinned);
480  const protections = new Map<string, Protection[]>();
481  for (const call of candidates) {
482    const found = protectionsFor(call, {
483      messages,
484      ledger,
485      calls,
486      options: ruleOptions,
487      referencedBy: strongRefs,
488    });
489    if ((seen.get(call.tool_use_id) ?? 0) > 1) {
490      found.push({ rule: 'ambiguous_structure', detail: 'tool_use_id occurs more than once in the transcript' });
491    }
492    protections.set(call.id, found);
493  }
494
495  const toClassify = candidates.filter(
496    (call) => (protections.get(call.id) ?? []).length === 0 && !duplicates.has(call.id),
497  );
498  let run: ClassifierRun = {
499    scores: new Map(),
500    stats: { requests: 0, stateTokens: 0, stateStage: '', ms: 0, unscored: [] },
501  };
502  if (toClassify.length > 0) {
503    const shown = redactor ? redactMessages(messages, redactor) : messages;
504    const shownCalls = redactor ? collectToolCalls(shown, resolved.preserveRecentMessages) : calls;
505    let sketches: Map<string, string> | undefined;
506    if (options.sketches !== false) {
507      sketches = new Map();
508      for (const call of shownCalls) {
509        const result = shown[call.resultIndex]?.toolResults?.find((r) => r.tool_use_id === call.tool_use_id);
510        if (result) sketches.set(call.id, sketchResult(call.tool, call.input, result.text, call.isError));
511      }
512    }
513    const shownById = new Map(shownCalls.map((call) => [call.id, call]));
514    const context = {
515      messages: shown,
516      ledger: redactor ? buildLedger(shown) : ledger,
517      calls: shownCalls,
518      options: resolved,
519      ...(sketches ? { sketches } : {}),
520    };
521    run = await classifier.score(
522      toClassify.map((call) => shownById.get(call.id) ?? call),
523      context,
524    );
525  }
526  const unscored = new Set(run.stats.unscored);
527
528  const decisions: ActionDecision[] = [];
529  const baseline: CallDecision[] = [];
530  const actionCounts = emptyActionCounts();
531  const protectionCounts: Record<string, number> = {};
532  for (const call of calls) {
533    const pair = ledger.interactions.get(call.tool_use_id)!;
534    const found = protections.get(call.id) ?? [];
535    const decision = decideInteraction(
536      {
537        call,
538        use: pair.use,
539        ...(pair.result ? { result: pair.result } : {}),
540        protections: found,
541        ...(duplicates.has(call.id) ? { duplicateOf: duplicates.get(call.id) } : {}),
542        ...(run.scores.has(call.id) ? { scores: run.scores.get(call.id) } : {}),
543        unscored: unscored.has(call.id),
544        targetUnchanged: !targetChangedLater(call, calls),
545        ...(pair.result && weakRefs.has(pair.result.id) ? { softReferences: weakRefs.get(pair.result.id) } : {}),
546      },
547      policyOptions,
548    );
549    decisions.push(decision);
550    baseline.push(toCallDecision(call, decision));
551    actionCounts[decision.action]++;
552    for (const rule of decision.protectedBy) protectionCounts[rule] = (protectionCounts[rule] ?? 0) + 1;
553  }
554
555  const applied = applyActions(messages, decisions, calls, ledger, {
556    sessionId,
557    compactionId,
558    archivedAt: now().toISOString(),
559    headChars: resolved.truncateHeadChars,
560    markRemovedCalls: options.markRemovedCalls !== false,
561  });
562  if (applied.archived.length > 0) await archive.put(applied.archived);
563
564  const charsBefore = messages.reduce((sum, message) => sum + messageChars(message), 0);
565  const charsAfter = applied.messages.reduce((sum, message) => sum + messageChars(message), 0);
566  const tokensBefore = eventTokens(ledger.events);
567  const tokensAfter = eventTokens(buildLedger(applied.messages).events);
568  const ms = Date.now() - started;
569  const count = (reason: CallDecision['reason']): number => baseline.filter((d) => d.reason === reason).length;
570
571  const report: OptimizeReport = {
572    sessionId,
573    compactionId,
574    classifier: classifier.name,
575    messages: { before: messages.length, after: applied.messages.length },
576    tokens: {
577      before: tokensBefore,
578      after: tokensAfter,
579      archived: applied.archived.reduce((sum, record) => sum + record.tokenEstimate, 0),
580    },
581    chars: { before: charsBefore, after: charsAfter },
582    actions: actionCounts,
583    protections: protectionCounts,
584    constraints: constraints.length,
585    duplicates: duplicates.size,
586    candidates: candidates.length,
587    classified: run.scores.size,
588    unscored: unscored.size,
589    redactedSecrets: redactor?.count ?? 0,
590    classifierStats: run.stats,
591    ms,
592  };
593
594  return {
595    messages: applied.messages,
596    decisions: baseline,
597    actions: decisions,
598    archived: applied.archived,
599    archive,
600    constraints,
601    report,
602    stats: {
603      messagesBefore: messages.length,
604      messagesAfter: applied.messages.length,
605      charsBefore,
606      charsAfter,
607      calls: calls.length,
608      kept: count('kept'),
609      resultsDropped: count('result_dropped'),
610      callsDropped: count('call_dropped'),
611      pinned: count('pinned'),
612      stateTokens: run.stats.stateTokens,
613      stateStage: run.stats.stateStage,
614      requests: run.stats.requests,
615      ms,
616    },
617  };
618}
619

1import { queryTerms } from '../archive/memory-store.js';
2import type { ArchiveRecord, ArchiveStore } from '../archive/types.js';
3
4/**
5 * Automatic retrieval: before a turn, the archive is searched with the
6 * prompt and the best exact records are handed to the model as
7 * `<retrieved_context>` blocks. Nothing pretends the content was there all
8 * along; every block names its archive id and when it was archived.
9 */
10export interface RehydrateOptions {
11  /** Characters of retrieved content per turn. Default 6000. */
12  budgetChars?: number;
13  /** Least lexical relevance to consider (share of query terms found). Default 0.34. */
14  minScore?: number;
15  /** Most records per turn. Default 3. */
16  limit?: number;
17  /** Prompts shorter than this (in characters) are not searched. Default 12. */
18  minPromptChars?: number;
19}
20
21export interface Rehydration {
22  records: ArchiveRecord[];
23  blocks: string[];
24  chars: number;
25}
26
27/**
28 * The `maxChars` slice of `content` with the most query-term hits, aligned to
29 * line boundaries: a 40k-char file read whose relevant type sits 12k in must
30 * not be handed over as its first 6k chars. Distinct terms count more than
31 * repeats of one term. Without terms, or when it all fits, the head.
32 */
33export function bestWindow(
34  content: string,
35  terms: ReadonlySet<string>,
36  maxChars: number,
37): { start: number; end: number } {
38  if (content.length <= maxChars) return { start: 0, end: content.length };
39  const lower = content.toLowerCase();
40  const hits: { at: number; term: string }[] = [];
41  for (const term of terms) {
42    if (term.length < 3) continue;
43    let at = lower.indexOf(term);
44    for (let n = 0; at !== -1 && n < 200; n++) {
45      hits.push({ at, term });
46      at = lower.indexOf(term, at + term.length);
47    }
48  }
49  if (hits.length === 0) return alignToLines(content, 0, maxChars);
50  hits.sort((a, b) => a.at - b.at);
51  const lead = Math.floor(maxChars * 0.25);
52  let best = { start: 0, score: -1 };
53  for (const hit of hits) {
54    const start = Math.max(0, Math.min(hit.at - lead, content.length - maxChars));
55    const end = start + maxChars;
56    const seen = new Set<string>();
57    let count = 0;
58    for (const other of hits) {
59      if (other.at < start) continue;
60      if (other.at >= end) break;
61      seen.add(other.term);
62      count++;
63    }
64    const score = seen.size * 10 + Math.min(count, 50);
65    if (score > best.score) best = { start, score };
66  }
67  return alignToLines(content, best.start, best.start + maxChars);
68}
69
70function alignToLines(content: string, start: number, end: number): { start: number; end: number } {
71  // Both edges move inward, so the window never grows past what was asked.
72  let from = start;
73  if (from > 0 && content[from - 1] !== '\n') {
74    const newline = content.indexOf('\n', from);
75    if (newline !== -1 && newline - from < 200) from = newline + 1;
76  }
77  let to = Math.min(end, content.length);
78  if (to < content.length) {
79    const newline = content.lastIndexOf('\n', to);
80    if (newline > from && to - newline < 200) to = newline;
81  }
82  return { start: from, end: to };
83}
84
85export function retrievedBlock(record: ArchiveRecord, maxChars?: number, terms?: ReadonlySet<string>): string {
86  let content = record.content;
87  if (maxChars !== undefined && record.content.length > maxChars) {
88    const { start, end } = bestWindow(record.content, terms ?? new Set(), maxChars);
89    const before = start > 0 ? `[… ${start} chars before this excerpt; /lossless show ${record.id} for all of it]\n` : '';
90    const after =
91      end < record.content.length
92        ? `\n[… ${record.content.length - end} more chars; /lossless show ${record.id} for all of it]`
93        : '';
94    content = `${before}${record.content.slice(start, end)}${after}`;
95  }
96  const attrs = [
97    `id="${record.id}"`,
98    `kind="${record.kind}"`,
99    record.toolName ? `tool="${record.toolName}"` : '',
100    `seq="${record.seq}"`,
101    `archived="${record.archivedAt}"`,
102  ]
103    .filter(Boolean)
104    .join(' ');
105  return `<retrieved_context ${attrs}>\n${content}\n</retrieved_context>`;
106}
107
108export const REHYDRATION_PREFACE =
109  'lossless-compact retrieved the following exact archived content because it looks relevant to this prompt. It was removed from the active context earlier and was not continuously present; /lossless why <id> explains why, /lossless restore <id> brings back the whole record.';
110
111/** Picks archived records for a prompt within the budget; empty when nothing is relevant enough. */
112export async function rehydrateForPrompt(
113  archive: ArchiveStore,
114  sessionId: string,
115  prompt: string,
116  options: RehydrateOptions = {},
117): Promise<Rehydration> {
118  const budget = options.budgetChars ?? 6000;
119  const minScore = options.minScore ?? 0.34;
120  const limit = options.limit ?? 3;
121  const text = prompt.trim();
122  if (text.length < (options.minPromptChars ?? 12) || text.startsWith('/')) {
123    return { records: [], blocks: [], chars: 0 };
124  }
125  const found = await archive.search(text, { sessionId, limit: limit * 3 });
126  const terms = queryTerms(text);
127  const chosen: ArchiveRecord[] = [];
128  const blocks: string[] = [];
129  let chars = 0;
130  for (const summary of found) {
131    if ((summary.score ?? 0) < minScore) continue;
132    if (chosen.length >= limit) break;
133    const record = await archive.get(summary.id);
134    if (!record) continue;
135    const room = budget - chars;
136    if (room < 200) break;
137    // The wrapper and the excerpt markers are not content; leave them room.
138    const block = retrievedBlock(record, Math.min(Math.max(100, room - 320), record.content.length), terms);
139    chosen.push(record);
140    blocks.push(block);
141    chars += block.length;
142  }
143  return { records: chosen, blocks, chars };
144}
145