transcript-redactor

1import type { Register, PromptContextBlock } from 'claude-code'
2import {
3  makeConfig,
4  scrubSecrets,
5  scrubPii,
6  restore,
7  mint,
8  walk,
9  collect,
10  escapeLiteralTags,
11  type WalkState,
12} from './detect.ts'
13
14// transcript-redactor: keeps secrets (and, if asked, email addresses) out of
15// the session transcript. Everything that decides what a secret is lives in
16// ./detect.ts; this file only wires it to the events where text enters the
17// conversation, and to the tools where a placeholder must become real again.
18//
19// Every hook fails closed. A hook that throws or runs past its budget is
20// skipped by the engine, and for a redactor that would mean the raw text goes
21// through. So each one has a `.catch` that withholds instead.
22
23// The scan is linear, but a string past this size could still run past the
24// hook's 10 s budget. It is withheld rather than passed on unscanned.
25const SCAN_CAP = 16_000_000
26
27const WITHHELD =
28  'transcript-redactor could not scan this result in time, so it was withheld. Narrow the output (head, grep, offset) and retry.'
29const BLOCK_WITHHELD = '[transcript-redactor withheld this text: the scan failed]'
30
31// Tools whose answer carries live placeholders, so they are not turned into
32// quoted (LITERAL) ones: the subagent tools, whose answer holds placeholders
33// the subagent used. A mod built on this one can add its own tool here.
34export const LIVE_TAG_TOOLS = new Set(['Agent', 'Task'])
35
36// The name on the toast and the tool row. A mod built on this one sets its own.
37export const brand = { name: 'transcript-redactor' }
38
39// Attachments that carry text from outside the engine's own prose: a file,
40// a diff of a file, a queued prompt. These get the full scrub, emails too.
41const OUTSIDE_TEXT = new Set(['file', 'edited_text_file', 'queued_command'])
42
43type Budget = { budget: { remainingMs: number } }
44
45// Throws before a scan that cannot finish: the `.catch` then withholds.
46function guard(text: string, next: Budget): void {
47  if (text.length > SCAN_CAP) throw new Error('transcript-redactor: text too large to scan')
48  const needMs = 300 + text.length / 4000
49  if (next.budget.remainingMs < needMs) throw new Error('transcript-redactor: out of time')
50}
51
52// A Bash command that writes text into git history or posts it to GitHub.
53// Its placeholders stay placeholders: a commit message or an issue body that
54// quotes one must not carry the real key off the machine. The whole command
55// is left as written, so a step that needs the key goes in its own call.
56const PUBLISHES =
57  /\bgit\b(?:[ \t]+-[cC][ \t]+\S+|[ \t]+--?[\w-]+(?:=\S+)?)*[ \t]+(?:commit|tag|notes|send-email)\b|\bgh[ \t]+(?:issue|pr|release|gist|discussion|api|repo[ \t]+(?:create|edit))\b/
58
59// A restored value goes into Bash as shell source. One holding any of these
60// (quotes, $, backticks, separators, globs) could run as code there, so it is
61// never put into a command. Most keys are letters, digits and -_.+/=:
62const SHELL_ACTIVE = /[\s$`"'\\;&|<>()*?[\]{}#~]|[\x00-\x1f\x7f]/
63
64// @-mentioned files the engine attaches as media or a page reference, never
65// as text. When one is too large for $.fs.read it holds no text to check.
66const MEDIA_FILE = /\.(?:png|jpe?g|gif|webp|bmp|tiff?|heic|pdf)$/i
67
68function looksBinary(text: string): boolean {
69  const head = text.slice(0, 8192)
70  return head.includes('\u0000') || (head.match(/�/g)?.length ?? 0) > 8
71}
72
73export const register: Register = (on, options) => {
74  const cfg = makeConfig(options as Record<string, unknown>)
75
76  const scrub = (text: string): string => {
77    let out = text
78    if (cfg.secrets) out = scrubSecrets(out, cfg)
79    if (cfg.pii) out = scrubPii(out, cfg)
80    return out
81  }
82  const secretsOnly = (text: string): string => (cfg.secrets ? scrubSecrets(text, cfg) : text)
83  // An `allowUrlHosts` entry that is not a host name is said once, not dropped silently.
84  let warnedHosts = cfg.allowUrlRejected.length === 0
85
86  // Counts the values hidden, not the placeholders that stand for quoted text.
87  const countHidden = (tags: Set<string>): number => [...tags].filter((t) => !t.startsWith('[REDACTED-LITERAL-')).length
88
89  // 1. What the user types. A pasted key never reaches the model as itself,
90  //    and neither does the context a hook above attached to the prompt.
91  on('prompt.submit', async ($, e, next) => {
92    const { value, tags } = collect(() => {
93      guard(e.text, next)
94      return { text: scrub(e.text), context: e.context?.map((c) => (guard(c, next), scrub(c))) }
95    })
96    const n = countHidden(tags)
97    if (!warnedHosts) {
98      warnedHosts = true
99      $.ui.toast(`${brand.name}: allowUrlHosts ignored ${cfg.allowUrlRejected.join(', ')} (not a host name)`)
100    }
101    if (n > 0 && cfg.notify) $.ui.toast(`${brand.name}: hid ${n} value(s) from the prompt`)
102    if (tags.size === 0) return next(e)
103    return next({ ...e, text: value.text, ...(value.context ? { context: value.context } : {}) })
104  }).catch(($, e, next) => (next.called ? next(e) : { drop: 'transcript-redactor failed to scan this prompt, so it was not sent.' }))
105
106  // 2. Every tool call. On the way in, a placeholder becomes its real value,
107  //    but only for a tool that runs on this machine. On the way out, the
108  //    result, the text the model reads and any attached context are scanned.
109  on('tool.call', async ($, e, next) => {
110    const command = (e as { command?: unknown }).command
111    const publishes = e.tool === 'Bash' && typeof command === 'string' && PUBLISHES.test(command)
112    const isLocal = cfg.restore && cfg.restoreTools.has(e.tool) && !publishes
113    // The real values this call was handed. Whatever the tool prints of them
114    // comes back hidden, even bare and short, where no detector would fire.
115    const handed = new Set<string>()
116    const restoreTracked = (s: string): string =>
117      restore(s) === s
118        ? s
119        : s.replace(/\[REDACTED-SECRET-[0-9a-f]{8}\]/g, (t) => {
120            const v = restore(t)
121            if (v !== t) handed.add(v)
122            return v
123          })
124    const rehideHanded = (text: string): string => {
125      let out = text
126      for (const v of [...handed].sort((a, b) => b.length - a.length)) if (v.length >= 4 && out.includes(v)) out = out.split(v).join(mint('SECRET', v))
127      return out
128    }
129    const input = isLocal ? (walk({ ...e }, restoreTracked) as typeof e) : e
130    if (e.tool === 'Bash' && [...handed].some((v) => SHELL_ACTIVE.test(v))) {
131      return {
132        deny: `${brand.name}: a secret in this command holds shell characters, so its real value was not put into Bash, where it could run as code. Write it to a file with the Write tool and have the command read that file instead.`,
133      }
134    }
135    const r = await next(input)
136    if (r.deny !== undefined) {
137      // A refusal from beneath may quote the input it refused, real values
138      // and all, and the model reads it as the tool's answer.
139      const d = collect(() => (guard(r.deny, next), rehideHanded(scrub(escapeLiteralTags(r.deny)))))
140      return d.tags.size === 0 ? r : { deny: d.value }
141    }
142
143    const escape = !LIVE_TAG_TOOLS.has(e.tool)
144    const st: WalkState = { incomplete: false }
145    // The result and the text often hold the same string: scan it once.
146    const memo = new Map<string, string>()
147    const clean = (s: string): string => {
148      const done = memo.get(s)
149      if (done !== undefined) return done
150      guard(s, next)
151      const out = rehideHanded(scrub(escape ? escapeLiteralTags(s) : s))
152      memo.set(s, out)
153      return out
154    }
155    const { value, tags } = collect(() => ({
156      result: walk(r.result, clean, st),
157      text: r.text === undefined ? undefined : clean(r.text),
158      context: r.context?.map(clean),
159    }))
160    if (tags.size === 0) return r
161    // Something was hidden, and part of the result could not be rewritten:
162    // that part may hold the same secret, so the whole result is withheld.
163    if (st.incomplete) return { deny: 'transcript-redactor: a secret sits in a part of this result it cannot rewrite (too deep, or not plain data). Narrow the call and retry.' }
164
165    const n = countHidden(tags)
166    if (cfg.notify && n > 0) $.ui.notice(e.tool_use_id, `${brand.name}: hid ${n} value(s)`)
167    const context = value.context ? { context: value.context } : {}
168    if (r.isError) return { isError: true as const, result: value.result, text: value.text, ...context }
169    return { result: value.result, ...context } as typeof r
170  }).catch(async ($, e, next) => {
171    // Nothing ran yet: refuse the call. The tool answered: withhold it.
172    if (!next.called) return { deny: WITHHELD }
173    const r = await next(e)
174    if (r.deny !== undefined) return { deny: WITHHELD }
175    if (r.isError) return { isError: true as const, result: WITHHELD, text: WITHHELD }
176    return { deny: WITHHELD }
177  })
178
179  // 3. The blocks attached to the first message (CLAUDE.md and friends).
180  //    Secrets only: the email in there is the user's own, and the engine
181  //    puts it there so the model knows who it is talking to.
182  on('prompt.context', async ($, e, next) => {
183    const r = await next(e)
184    if (!cfg.secrets) return r
185    const blocks: PromptContextBlock[] = r.blocks.map((b) => {
186      guard(b.text, next)
187      return { ...b, text: secretsOnly(b.text) }
188    })
189    // The raw files behind a rewritten CLAUDE.md would carry the key on to
190    // the hooks above, so they are left out once any block changed.
191    if (blocks.every((b, i) => b.text === r.blocks[i].text)) return r
192    return { blocks }
193  }).catch(async ($, e, next) => {
194    // Keep the blocks, so the model sees that something was held back.
195    const r = await next(e)
196    return { blocks: r.blocks.map((b) => ({ ...b, text: BLOCK_WITHHELD })) }
197  })
198
199  // 4. What the engine injects on its own after the first message: a file
200  //    named with @, a diff of a file that changed on disk, a nested
201  //    CLAUDE.md, a settings hook's context. None of it passes tool.call.
202  on('prompt.attachment', async ($, e, next) => {
203    const r = await next(e)
204    if (r.text == null) return r
205    const text = r.text
206    guard(text, next)
207    const isOutside = e.origin.kind !== 'engine' || OUTSIDE_TEXT.has(e.type)
208    const isFile = e.type === 'file' || e.type === 'edited_text_file'
209    const out = isOutside ? scrub(isFile ? escapeLiteralTags(text) : text) : secretsOnly(text)
210    return out === text ? r : { text: out }
211  }).catch(() => ({ text: BLOCK_WITHHELD }))
212
213  // 5. A file named with @ is read with no tool call, and its raw text is
214  //    stored in the transcript whatever the model is shown. So a file that
215  //    holds a secret is refused here, before it is read, and the model is
216  //    told to use the Read tool, whose result is scrubbed.
217  on('prompt.mention', async ($, e, next) => {
218    if (!cfg.secrets) return next(e)
219    let text: string
220    try {
221      const read = await $.fs.read(e.path)
222      if (typeof read !== 'string') return next(e)
223      text = read
224    } catch {
225      // Missing: the engine attaches nothing. An image or a PDF too large to
226      // read is attached as media, not text. A text file that is present
227      // but unreadable (too large, say) cannot be checked, so it is refused.
228      if (!(await $.fs.exists(e.path))) return next(e)
229      if (MEDIA_FILE.test(e.path)) return next(e)
230      return { deny: 'transcript-redactor: could not check this file for secrets; ask Claude to Read it instead' }
231    }
232    if (looksBinary(text)) return next(e)
233    guard(text, next)
234    const { tags } = collect(() => scrubSecrets(text, cfg))
235    if (tags.size > 0) return { deny: 'transcript-redactor: this file holds a secret; ask Claude to Read it instead, so the value is replaced by a placeholder' }
236    return next(e)
237  }).catch(($, e, next) => (next.called ? next(e) : { deny: 'transcript-redactor: could not check this file for secrets' }))
238
239  // 6. A subagent's task. The model only ever holds placeholders, and the
240  //    Agent tool gets no restore, so this is a second layer: a value that
241  //    slipped into the prompt some other way still reaches the subagent as
242  //    a placeholder. A workflow agent's prompt cannot be rewritten, only
243  //    refused, so one that holds a secret is not started.
244  on('agent.spawn', async ($, e, next) => {
245    if (!cfg.secrets) return next(e)
246    if (e.workflow) {
247      guard(e.prompt, next)
248      const { tags } = collect(() => secretsOnly(e.prompt))
249      if (tags.size === 0) return next(e)
250      return { deny: "transcript-redactor: this workflow agent's prompt holds a secret, and a workflow prompt cannot be rewritten. Pass the value by a file or an env var instead." }
251    }
252    guard(e.prompt, next)
253    const prompt = secretsOnly(e.prompt)
254    const description = secretsOnly(e.description)
255    return prompt === e.prompt && description === e.description ? next(e) : next({ ...e, prompt, description })
256  }).catch(($, e, next) => (next.called ? next(e) : { deny: 'transcript-redactor failed to scan this task, so the subagent was not started.' }))
257}
258

1// The pure half of transcript-redactor: the config, the vault, the detectors,
2// and the walkers. Nothing in here touches `$` or registers a hook, so another
3// mod can vendor this file as it is and the tests can run it under plain node.
4//
5// The value is not deleted. It is put in a vault that lives in this module,
6// in memory, for the session, and the transcript gets a placeholder in its
7// place: `[REDACTED-SECRET-1a2b3c4d]`. Within one session the same value always
8// mints the same placeholder, so the model can still tell one customer from
9// another, and the placeholder is swapped back for the real value on the way
10// into a tool that runs on this machine, so a `Bash` command or a `Write` still
11// carries the real key.
12//
13// Nothing is written to disk. The vault dies with the session.
14
15export type Kind = 'SECRET' | 'EMAIL' | 'IP' | 'LITERAL'
16
17export type Config = {
18  secrets: boolean
19  pii: boolean
20  emails: boolean
21  privateIps: boolean
22  restore: boolean
23  restoreTools: ReadonlySet<string>
24  notify: boolean
25  minEntropy: number
26  minLength: number
27  contextMinEntropy: number
28  allow: ReadonlySet<string>
29  allowEmails: readonly string[]
30  allowPrefix: readonly string[]
31  denyPrefix: readonly string[]
32  allowUrl: AllowUrl
33  // Entries of `allowUrlHosts` that are not a host name, shown once.
34  allowUrlRejected: readonly string[]
35}
36
37// The tools that run on this machine. A placeholder is only swapped back for
38// its real value in these: a tool that hands its input to another model or to
39// the network (Agent, WebFetch, WebSearch, SendMessage, an MCP tool) gets the
40// placeholder as written.
41export const DEFAULT_RESTORE_TOOLS = 'Bash,Write,Edit,MultiEdit,NotebookEdit'
42
43// ---------------------------------------------------------------- the vault
44
45const byValue = new Map<string, string>()
46const byTag = new Map<string, string>()
47let hidden = 0
48let minted: Set<string> | null = null
49
50const TAG = /\[REDACTED-(?:SECRET|EMAIL|IP|LITERAL)-[0-9a-f]{8}\]/g
51const WHOLE_TAG = /^\[REDACTED-(?:SECRET|EMAIL|IP|LITERAL)-[0-9a-f]{8}\]$/
52
53function randomHex8(): string {
54  const bytes = new Uint8Array(4)
55  const crypto = (globalThis as { crypto?: { getRandomValues?: (values: Uint8Array) => Uint8Array } }).crypto
56  if (crypto?.getRandomValues) crypto.getRandomValues(bytes)
57  else for (let index = 0; index < bytes.length; index++) bytes[index] = Math.floor(Math.random() * 256)
58  return Array.from(bytes, byte => byte.toString(16).padStart(2, '0')).join('')
59}
60
61// Swaps a value for its placeholder and remembers both. A value that already
62// holds a placeholder from an earlier pass (a key inside a URL password) is
63// stored with that placeholder put back, so one restore gives the real text.
64export function mint(kind: Kind, raw: string): string {
65  hidden++
66  const value = kind === 'LITERAL' ? raw : restore(unpark(raw))
67  const seen = byValue.get(value)
68  if (seen) {
69    if (kind === 'SECRET') remember(value)
70    minted?.add(seen)
71    return seen
72  }
73  let tag: string
74  do tag = `[REDACTED-${kind}-${randomHex8()}]`
75  while (byTag.has(tag))
76  byValue.set(value, tag)
77  byTag.set(tag, value)
78  if (kind === 'SECRET') remember(value)
79  minted?.add(tag)
80  return tag
81}
82
83// A secret is hidden again wherever it turns up later: across the session
84// when it is distinctive enough (echoable), and within the text being
85// scanned right now when it is shorter but still not a plain word.
86function remember(value: string): void {
87  const echoes = echoable(value)
88  if (echoes && !echo.has(value)) {
89    echo.add(value)
90    echoIndex = null
91  }
92  if (local && (echoes || localWorthy(value))) local.add(value)
93}
94
95// One pass, on purpose: a LITERAL placeholder gives back the placeholder text
96// it stands for, and that text must not then turn into the secret.
97export function restore(text: string): string {
98  if (byTag.size === 0 || !text.includes('[REDACTED-')) return text
99  return text.replace(TAG, (tag) => byTag.get(tag) ?? tag)
100}
101
102// Hides a value the caller knows is secret wherever it turns up for the rest
103// of the session, even one too short or too plain for the echo rule (a word
104// password from .env). Values under PREFIX characters can't be indexed.
105export function keepSecret(value: string): string {
106  const tag = mint('SECRET', value)
107  if (value.length >= PREFIX && !echo.has(value)) {
108    echo.add(value)
109    echoIndex = null
110  }
111  return tag
112}
113
114// How many times a value has been hidden this session. A hook reads it before
115// and after a scrub to learn whether anything changed.
116export function hiddenCount(): number {
117  return hidden
118}
119
120// Runs `fn` and hands back every placeholder it minted or reused, so a hook
121// can count the values it hid without counting one value twice.
122export function collect<T>(fn: () => T): { value: T; tags: Set<string> } {
123  const outer = minted
124  const tags = new Set<string>()
125  minted = tags
126  try {
127    return { value: fn(), tags }
128  } finally {
129    minted = outer
130    if (outer) for (const t of tags) outer.add(t)
131  }
132}
133
134// A placeholder read back from a file or a log is quoted text, not a handle.
135// Left alone it would turn into the real secret the next time the model
136// copied that text into a command. So a live placeholder found in a tool's
137// output is swapped for a LITERAL one, which restores to the placeholder text
138// and nothing more. Run this before the detectors, so a key the tool printed
139// still comes back as its live placeholder.
140export function escapeLiteralTags(text: string): string {
141  if (byTag.size === 0 || !text.includes('[REDACTED-')) return text
142  return text.replace(TAG, (t) => (byTag.has(t) && !t.startsWith('[REDACTED-LITERAL-') ? mint('LITERAL', t) : t))
143}
144
145// ------------------------------------------------------- known value echoes
146
147// A value hidden once because of where it sat (after `API_KEY=`, inside a
148// URL) is hidden again wherever it turns up, even bare, where no detector
149// would fire on it: a tool that echoes a restored key must not print it.
150const echo = new Set<string>()
151let echoIndex: EchoIndex | null = null
152// The values minted during the current scan, to hide at their other spots.
153let local: Set<string> | null = null
154
155// Short or all-letter values (`postgres`) are left out: hiding every later
156// `postgres` in unrelated output would make the session unreadable.
157function echoable(value: string): boolean {
158  if (countsUp(value)) return false
159  // A slug or a dated name hidden once beside a key name is not worth hiding
160  // everywhere: it would turn every path that holds it into a placeholder.
161  if (value.split(/[^A-Za-z0-9]+/).every((p) => p === '' || isSlugPart(p))) return false
162  if (value.includes('\n')) return value.length >= 40
163  return value.length >= 20 || (value.length >= 10 && /[0-9]/.test(value) && /[A-Za-z]/.test(value))
164}
165
166// A value too short to hide across the session is still hidden at its other
167// spots in the same text, unless it is a plain word or slug (`postgres`
168// hidden once in a URL must not turn every `postgres://` into a placeholder).
169function localWorthy(value: string): boolean {
170  if (value.length < PREFIX || value.length > 400 || value.includes('\n') || countsUp(value)) return false
171  if (value.split(/[^A-Za-z0-9]+/).every((p) => p === '' || isSlugPart(p))) return false
172  return /[0-9]/.test(value) || /[^A-Za-z0-9]/.test(value) || value.length >= 12
173}
174
175// The known values are found with a rolling hash over their first PREFIX
176// characters, so the cost of the scan does not grow with the vault. A regex
177// of every value joined by `|` took 40 s on 200 KB once a long session had
178// hidden a few thousand values.
179const PREFIX = 8
180// Under each prefix the values are kept by length, longest first, so a
181// prefix that thousands of values share costs one set lookup per length,
182// not a walk of the whole list.
183type ByLength = { lengths: number[]; values: Set<string> }
184type EchoIndex = { hashes: Set<number>; byPrefix: Map<string, ByLength>; top: number }
185
186function prefixHash(text: string, at: number): number {
187  let h = 0
188  for (let i = 0; i < PREFIX; i++) h = (Math.imul(h, 31) + text.charCodeAt(at + i)) >>> 0
189  return h
190}
191
192function buildEchoIndex(values: Iterable<string>): EchoIndex {
193  const byPrefix = new Map<string, ByLength>()
194  for (const v of values) {
195    const p = v.slice(0, PREFIX)
196    let entry = byPrefix.get(p)
197    if (!entry) byPrefix.set(p, (entry = { lengths: [], values: new Set() }))
198    if (!entry.lengths.includes(v.length)) entry.lengths.push(v.length)
199    entry.values.add(v)
200  }
201  for (const entry of byPrefix.values()) entry.lengths.sort((a, b) => b - a)
202  const hashes = new Set<number>()
203  for (const p of byPrefix.keys()) hashes.add(prefixHash(p, 0))
204  // 31 to the power PREFIX - 1, to take the oldest character out of the hash.
205  let top = 1
206  for (let i = 0; i < PREFIX - 1; i++) top = Math.imul(top, 31) >>> 0
207  return { hashes, byPrefix, top }
208}
209
210function rehideKnown(text: string): string {
211  if (echo.size === 0 || text.length < PREFIX) return text
212  return rehideWith(text, (echoIndex ??= buildEchoIndex(echo)))
213}
214
215// The values minted in one scan, hidden at their other spots in it. A few
216// values are found faster one by one than by the rolling hash.
217function rehideLocal(text: string, values: Set<string>): string {
218  if (values.size > 64) return rehideWith(text, buildEchoIndex(values))
219  let out = text
220  for (const v of [...values].sort((a, b) => b.length - a.length)) if (out.includes(v)) out = out.split(v).join(mint('SECRET', v))
221  return out
222}
223
224function rehideWith(text: string, ix: EchoIndex): string {
225  if (text.length < PREFIX) return text
226  const parts: string[] = []
227  let last = 0
228  let h = prefixHash(text, 0)
229  for (let i = 0; i + PREFIX <= text.length; ) {
230    if (ix.hashes.has(h)) {
231      const entry = ix.byPrefix.get(text.slice(i, i + PREFIX))
232      let found: string | undefined
233      if (entry) {
234        for (const n of entry.lengths) {
235          const v = text.slice(i, i + n)
236          if (v.length === n && entry.values.has(v)) {
237            found = v
238            break
239          }
240        }
241      }
242      if (found) {
243        parts.push(text.slice(last, i), mint('SECRET', found))
244        i += found.length
245        last = i
246        if (i + PREFIX <= text.length) h = prefixHash(text, i)
247        continue
248      }
249    }
250    if (i + PREFIX >= text.length) break
251    h = (Math.imul((h - Math.imul(text.charCodeAt(i), ix.top)) >>> 0, 31) + text.charCodeAt(i + PREFIX)) >>> 0
252    i++
253  }
254  if (last === 0) return text
255  parts.push(text.slice(last))
256  return parts.join('')
257}
258
259// ------------------------------------------------------------------ parking
260
261// Some spans are known not to be secrets and are big enough to drown the
262// transcript in placeholders: an image in a data: URI, a lockfile hash, a
263// thinking-block signature, a base64 attachment. They are parked behind a
264// marker for the scan and put back after it. The marker is a character the
265// text does not contain; if every candidate is taken, nothing is parked and
266// the scan simply hides more.
267type Parked = { mark: string; spans: string[]; re: RegExp }
268let parked: Parked | null = null
269const MARKS = ['\u0000', '\uE000', '\uE001', '\uFDD0']
270
271function startParking(text: string): Parked | null {
272  const mark = MARKS.find((m) => !text.includes(m))
273  return mark ? { mark, spans: [], re: new RegExp(`${mark}(\\d+)${mark}`, 'g') } : null
274}
275
276function park(span: string): string {
277  if (!parked) return span
278  parked.spans.push(span)
279  return `${parked.mark}${parked.spans.length - 1}${parked.mark}`
280}
281
282function unpark(text: string): string {
283  const p = parked
284  if (!p || !text.includes(p.mark)) return text
285  let out = text
286  for (let i = 0; i < 3 && out.includes(p.mark); i++) out = out.replace(p.re, (_m, n: string) => p.spans[Number(n)] ?? '')
287  return out
288}
289
290// ------------------------------------------------------------- small tools
291
292// Bits per character. Counted in a reused array for plain ASCII, since the
293// scan calls this for every candidate token.
294const COUNTS = new Int32Array(128)
295
296function entropy(text: string): number {
297  let ascii = true
298  for (let i = 0; i < text.length; i++) {
299    const c = text.charCodeAt(i)
300    if (c >= 128) {
301      ascii = false
302      break
303    }
304  }
305  if (!ascii) {
306    const counts = new Map<string, number>()
307    for (const ch of text) counts.set(ch, (counts.get(ch) ?? 0) + 1)
308    let h = 0
309    for (const n of counts.values()) h -= (n / text.length) * Math.log2(n / text.length)
310    return h
311  }
312  COUNTS.fill(0)
313  for (let i = 0; i < text.length; i++) COUNTS[text.charCodeAt(i)]!++
314  let h = 0
315  for (let c = 0; c < 128; c++) {
316    const n = COUNTS[c]!
317    if (n > 0) h -= (n / text.length) * Math.log2(n / text.length)
318  }
319  return h
320}
321
322const B64_TABLE = (() => {
323  const t = new Int16Array(128).fill(-1)
324  const abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
325  for (let i = 0; i < abc.length; i++) t[abc.charCodeAt(i)] = i
326  t['-'.charCodeAt(0)] = 62
327  t['_'.charCodeAt(0)] = 63
328  return t
329})()
330
331// Decodes the start of a base64 text into one character per byte. The hooks
332// run with no Buffer and no atob, so this is done by hand.
333function decodeBase64Head(text: string, max = 8192): string {
334  let out = ''
335  let acc = 0
336  let bits = 0
337  for (let i = 0; i < text.length && out.length < max; i++) {
338    const c = text.charCodeAt(i)
339    if (c === 61) break
340    const v = c < 128 ? B64_TABLE[c]! : -1
341    if (v < 0) continue
342    acc = ((acc << 6) | v) & 0xffffff
343    bits += 6
344    if (bits >= 8) {
345      bits -= 8
346      out += String.fromCharCode((acc >> bits) & 0xff)
347    }
348  }
349  return out
350}
351
352// Ranges of the text that the token scan skips. A `soft` range is skipped
353// only when no key name stands in front of the token.
354type Span = [start: number, end: number, soft: boolean]
355
356class Spans {
357  private list: Span[] = []
358  private i = 0
359  add(start: number, end: number, soft = false): void {
360    if (end > start) this.list.push([start, end, soft])
361  }
362  seal(): this {
363    this.list.sort((a, b) => a[0] - b[0])
364    return this
365  }
366  // Offsets are asked in ascending order, so a cursor keeps this linear.
367  at(offset: number): 'hard' | 'soft' | null {
368    const l = this.list
369    while (this.i < l.length && l[this.i]![1] <= offset) this.i++
370    let found: 'hard' | 'soft' | null = null
371    for (let j = this.i; j < l.length && l[j]![0] <= offset; j++) {
372      if (l[j]![1] > offset) {
373        if (!l[j]![2]) return 'hard'
374        found = 'soft'
375      }
376    }
377    return found
378  }
379}
380
381// ------------------------------------------------------------ secret tests
382
383// Escapes that end in a letter or digit and may sit right before a key:
384// quoted-printable `=3D`, URL `%3D`, and the `\n` or `"` of escaped JSON.
385const ESC = String.raw`[%=]3D|\\[nrtfbv]|\\u[0-9A-Fa-f]{4}`
386const UUID = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
387
388// A vendor prefix only counts at the start of a word, or right after one of
389// those escapes. Without this, `task-runner-for-the-build` reads as `sk-`.
390// The prefix is written first and the boundary checked behind it, so the
391// regex engine can jump between occurrences of the prefix instead of
392// testing the boundary at every character of the text.
393function word(prefix: string, body: string, edge = 'A-Za-z0-9'): string {
394  return String.raw`${prefix}(?<=(?:^|[^${edge}]|${ESC})${prefix})${body}`
395}
396
397// Shapes that belong to one vendor and mean one thing. These are hidden on
398// sight, whatever their entropy. Each has a hint, a cheap test that the text
399// could hold one, so a 10 MB log pays only for the shapes it might contain.
400// Every quantifier is bounded: an open `{n,}` over a run of millions of
401// characters overflows the regex stack.
402type Shape = { hint: string | RegExp; re: string }
403const VENDOR_SHAPES: readonly Shape[] = [
404  { hint: 'sk-ant-', re: word('sk-ant-', '[A-Za-z0-9_-]{16,4096}') },
405  // A real OpenAI key has a digit or a capital early on; a kebab word does not.
406  { hint: 'sk-', re: word('sk-', '(?=[a-z_-]{0,200}[0-9A-Z])[A-Za-z0-9_-]{20,4096}') },
407  { hint: /gh[pousr]_/, re: word('gh[pousr]_', '[A-Za-z0-9]{20,4096}') },
408  { hint: 'github_pat_', re: word('github_pat_', '[A-Za-z0-9_]{20,4096}') },
409  { hint: 'xox', re: word('xox[baprse]-', '[A-Za-z0-9-]{10,4096}') },
410  { hint: 'xapp-', re: word('xapp-', '[0-9]-[A-Za-z0-9-]{10,4096}') },
411  { hint: /A[KS]IA/, re: word('A(?:KIA|SIA)', '[0-9A-Z]{16}(?![A-Za-z0-9])') },
412  { hint: /[sr]k_(?:live|test)_/, re: word('[sr]k_(?:live|test)_', '[A-Za-z0-9]{16,4096}') },
413  { hint: 'whsec_', re: word('whsec_', '[A-Za-z0-9]{16,4096}') },
414  { hint: 'AIza', re: word('AIza', '[0-9A-Za-z_-]{30,4096}') },
415  { hint: 'ya29.', re: word(String.raw`ya29\.`, '[A-Za-z0-9_-]{20,4096}') },
416  { hint: 'npm_', re: word('npm_', '[A-Za-z0-9]{30,4096}') },
417  { hint: 'dop_v1_', re: word('dop_v1_', '[a-f0-9]{40,4096}') },
418  { hint: 'glpat-', re: word('glpat-', '[A-Za-z0-9_-]{16,4096}') },
419  { hint: 'shpat_', re: word('shpat_', '[a-f0-9]{32,4096}') },
420  { hint: 'SG.', re: word(String.raw`SG\.`, String.raw`[A-Za-z0-9_-]{16,4096}\.[A-Za-z0-9_-]{16,4096}`) },
421  { hint: 'hooks.slack.com', re: String.raw`https://hooks\.slack\.com/services/[A-Za-z0-9/+_-]{16,4096}` },
422  { hint: '/api/webhooks/', re: String.raw`https://discord(?:app)?\.com/api/webhooks/[0-9]{1,30}/[A-Za-z0-9_-]{16,4096}` },
423  // Prefixes built from words, which the name test would otherwise excuse.
424  { hint: 'vercel_blob_rw_', re: word('vercel_blob_rw_', '[A-Za-z0-9]{16}_[A-Za-z0-9]{30}') },
425  { hint: 'tskey-', re: word('tskey-', '[a-z]{1,20}-[A-Za-z0-9_-]{20,4096}') },
426  { hint: 'pscale_', re: word('pscale_', '(?:tkn|pw|oauth)_[A-Za-z0-9_-]{32,4096}') },
427  { hint: 'pdl_', re: word('pdl_', '(?:live|sdbx)_apikey_[A-Za-z0-9_-]{20,4096}') },
428  { hint: 'pypi-AgE', re: word('pypi-AgE', '[A-Za-z0-9_-]{50,4096}') },
429  { hint: 'sntrys_eyJ', re: word('sntrys_eyJ', '[A-Za-z0-9+/=_-]{50,4096}') },
430  { hint: 'pcsk_', re: word('pcsk_', '[A-Za-z0-9_-]{40,4096}') },
431  { hint: 'HRKU-AA', re: word('HRKU-AA', '[A-Za-z0-9_-]{40,4096}') },
432  { hint: 'dckr_', re: word('dckr_', '(?:pat|oat)_[A-Za-z0-9_-]{20,4096}') },
433  { hint: 'figd_', re: word('figd_', '[A-Za-z0-9_-]{30,4096}') },
434  { hint: 'GOCSPX-', re: word('GOCSPX-', '[A-Za-z0-9_-]{20,4096}') },
435  { hint: /gl(?:rt|dt|ptt|cbt|imt|ft|soat|oas|agent)-/, re: word('gl(?:rt|dt|ptt|cbt|imt|ft|soat|oas|agent)-', '[A-Za-z0-9_-]{20,4096}') },
436  { hint: 'bb_live_', re: word('bb_live_', '[A-Za-z0-9_-]{20,4096}') },
437  { hint: 'sb_secret_', re: word('sb_secret_', '[A-Za-z0-9_-]{20,4096}') },
438  { hint: 'ATATT3', re: word('ATATT3', '[A-Za-z0-9_=-]{100,4096}') },
439  // Keys whose secret half sits after a `.`, `:` or `/`.
440  { hint: 'pat', re: word('pat', String.raw`[A-Za-z0-9]{14}\.[a-f0-9]{64}`) },
441  { hint: /[st]k\.eyJ/, re: word(String.raw`[st]k\.eyJ`, String.raw`[A-Za-z0-9_-]{10,4096}\.[A-Za-z0-9_-]{20,4096}`) },
442  { hint: 'st.', re: word(String.raw`st\.`, String.raw`(?:[0-9a-f]{24}|${UUID})\.[0-9a-f]{32}\.[0-9a-f]{32}`) },
443  { hint: /\/[0-9]{8}[0-9/]*:[0-9a-f]{32}/, re: String.raw`(?<![0-9])[0-9]/[0-9]{8,30}(?:/[0-9]{8,30})?:[0-9a-f]{32}` },
444  { hint: /-[0-9a-f]{12}:[0-9a-f]{32}/, re: String.raw`(?<![0-9A-Fa-f])${UUID}:[0-9a-f]{32}` },
445  // Hugging Face tokens are letters only, so no entropy rule can see them.
446  { hint: /(?:hf|api_org)_/, re: word('(?:hf|api_org)_', '[A-Za-z0-9]{34}(?![A-Za-z0-9_])') },
447  { hint: 'ops_eyJ', re: word('ops_eyJ', '[A-Za-z0-9+/=_-]{50,4096}') },
448  { hint: /fm[12][ar]?_/, re: word('fm[12][ar]?_', '[A-Za-z0-9+/=,_-]{50,4096}') },
449  { hint: 'sig=', re: String.raw`(?<=[?&]sig=)[A-Za-z0-9%+/]{30,4096}` },
450  { hint: 'AAAAAAAAAAAAAAAAAAAAA', re: word('AAAAAAAAAAAAAAAAAAAAA', '[A-Za-z0-9]{0,200}%[A-Za-z0-9%]{30,4096}') },
451  { hint: 'Q~', re: String.raw`[A-Za-z0-9_~.-]{3}[0-9][A-Za-z0-9_~.-]Q~[A-Za-z0-9_~.-]{31,34}` },
452  // Lowercase and hex shapes that one vendor owns.
453  { hint: 'e2b_', re: word('e2b_', String.raw`[a-f0-9]{40}(?![\w-])`) },
454  { hint: 'lsv2_', re: word('lsv2_', String.raw`(?:pt|sk)_[a-f0-9]{32}_[a-f0-9]{10}(?![\w-])`) },
455  { hint: /[0-9a-f]-us[0-9]/, re: String.raw`(?:(?<=${ESC})|(?<![A-Za-z0-9]))[a-f0-9]{32}-us[0-9]{1,2}(?![\w-])` },
456  { hint: /-[a-f0-9]{8}-[a-f0-9]{8}(?![\w-])/, re: String.raw`(?:(?<=${ESC})|(?<![A-Za-z0-9]))[a-f0-9]{32}-[a-f0-9]{8}-[a-f0-9]{8}(?![\w-])` },
457  { hint: /xa[ap]t-/, re: word('xa[ap]t-', String.raw`${UUID}(?![\w-])`) },
458  { hint: 'SK', re: word('SK', String.raw`[0-9a-f]{32}(?![\w-])`) },
459  { hint: 'tvly-', re: word('tvly-', '(?:dev-|prod-)?[A-Za-z0-9]{32,4096}') },
460  { hint: 'tr_', re: word('tr_', '(?:dev|prod|stg|pat)_[A-Za-z0-9]{20,4096}', 'A-Za-z0-9_') },
461  { hint: ':AA', re: String.raw`(?<![0-9])[0-9]{8,10}:AA[A-Za-z0-9_-]{33}(?![A-Za-z0-9_-])` },
462  { hint: /a[ks]-/, re: word('a[ks]-', '[A-Za-z0-9]{22}(?![A-Za-z0-9_-])', 'A-Za-z0-9_-') },
463  // Keys that reuse a Stripe id prefix: Resend `re_`, Stripe client secrets.
464  { hint: 're_', re: word('re_', '[A-Za-z0-9]{8}_[A-Za-z0-9]{24}(?![A-Za-z0-9_])', 'A-Za-z0-9_') },
465  { hint: '_secret_', re: word('(?:pi|seti)_', '[A-Za-z0-9]{14,64}_secret_[A-Za-z0-9]{16,4096}', 'A-Za-z0-9_') },
466  // A Klaviyo private key is `pk_` and 34 alnum. Stripe and Clerk publishable
467  // keys always carry `live_` or `test_` after `pk_`, so they do not match.
468  { hint: 'pk_', re: word('pk_', '[A-Za-z0-9]{34}(?![A-Za-z0-9_])', 'A-Za-z0-9_') },
469  // An age identity (SOPS reads it from keys.txt) is upper case and digits.
470  { hint: 'AGE-SECRET-KEY-1', re: word('AGE-SECRET-KEY-1', '[0-9A-Z]{58}') },
471  { hint: 'AGE-PLUGIN-', re: word('AGE-PLUGIN-[A-Z0-9]{1,40}-1', '[0-9A-Z]{20,4096}') },
472  // Tokens that are not mixed case, so no entropy rule sees them alone.
473  { hint: /NR(?:AK|II|JS|AA|RA)-/, re: word('NR(?:AK|II|JS|AA|RA)-', '[A-Za-z0-9_-]{27,40}') },
474  { hint: 'dt0c01.', re: word(String.raw`dt0c01\.`, String.raw`[A-Z0-9]{24}\.[A-Z0-9]{64}`) },
475  { hint: /pat-(?:na|eu|ap)[0-9]-/, re: word('pat-(?:na|eu|ap)[0-9]-', String.raw`${UUID}(?![\w-])`) },
476  { hint: /(?:sdk|api)-[0-9a-f]{8}-/, re: word('(?:sdk|api)-', String.raw`${UUID}(?![\w-])`) },
477  { hint: 'lin_oauth_', re: word('lin_oauth_', '[a-f0-9]{64}') },
478  { hint: 'shippo_', re: word('shippo_(?:live|test)_', '[a-f0-9]{40}') },
479  { hint: 'dapi', re: word('dapi', String.raw`[a-f0-9]{32}(?:-[0-9]{1,2})?(?![\w-])`) },
480  { hint: ':fx', re: String.raw`(?<![0-9a-f-])${UUID}:fx(?!\w)` },
481  { hint: 'access_token$', re: String.raw`access_token\$(?:production|sandbox)\$[a-z0-9]{16}\$[a-f0-9]{32}` },
482  // A Twilio auth token is plain hex; it is known by the account SID beside it.
483  { hint: /AC[0-9a-f]{32}["']?[ \t]*[,:]/, re: String.raw`(?<=\bAC[0-9a-f]{32}["']?[ \t]*[,:][ \t]*["']?)[0-9a-f]{32}(?![0-9A-Za-z])` },
484]
485
486// Each shape also takes in the rest of its token, so a match that stops at a
487// `-` or `_` inside a key cannot leave the tail of the key on screen.
488const VENDOR = VENDOR_SHAPES.map((s) => ({ hint: s.hint, re: new RegExp(`(?:${s.re})[A-Za-z0-9_-]{0,4096}`, 'g') }))
489
490function scrubVendors(text: string, cfg: Config): string {
491  let out = text
492  for (const { hint, re } of VENDOR) {
493    if (typeof hint === 'string' ? !out.includes(hint) : !hint.test(out)) continue
494    out = out.replace(re, (m) => (cfg.allow.has(m) || m.includes('[REDACTED-') ? m : mint('SECRET', m)))
495  }
496  return out
497}
498
499// A private key block, up to its END line (never past the next BEGIN, so a
500// file of BEGIN lines is not scanned again for each one). When the END line
501// is missing (`head key.pem`), the base64 lines that follow the BEGIN line
502// are still taken, and only those.
503const PRIVATE_KEY =
504  /-----BEGIN[^\n-]{0,40}PRIVATE KEY(?: BLOCK)?-----(?:(?:(?!-----BEGIN)[\s\S]){0,65536}?-----END[^\n-]{0,40}PRIVATE KEY(?: BLOCK)?-----|(?:\r?\n(?:[A-Za-z][A-Za-z-]{0,40}: [^\n]*|[ \t]*))*(?:\r?\n[A-Za-z0-9+/=]{4}[A-Za-z0-9+/=]*(?=\r?\n|$))+)/g
505const PUTTY_PRIVATE = /(Private-Lines: \d+\r?\n)((?:[A-Za-z0-9+/=]{1,80}(?:\r?\n|$))+)/g
506
507// A run of dot-separated parts, where a JWT may hide. It has one starting
508// point per run, so a long run is scanned once. A JWT regex of its own went
509// quadratic on base64 JSON, where every row starts with `eyJ`.
510const DOTTED = /(?<![A-Za-z0-9_-])[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)(?:\.[A-Za-z0-9_-]+)+/g
511
512// The password inside a connection string: postgres://user:REDACTED@host.
513// The user may be empty (redis://:pw@host), and the password may hold `/` or
514// `@`: it ends at the `@` that is followed by a host.
515const URL_PASSWORD =
516  /\b([a-z][a-z0-9+.-]{1,20}:\/\/[^\s/@:[\]]{0,80}):(?!\d{1,5}(?:[/?#]|$))([^\s"'`<>]{3,200}?)@(?=[A-Za-z0-9.-]{1,253}(?::\d{1,5})?(?:[/?#\s"'`<>]|$))/gi
517
518// A token used alone as the userinfo: `https://<token>@dev.azure.com/...`,
519// the documented clone form for Azure DevOps, GitHub and Bitbucket tokens.
520const URL_USERINFO = /\b([a-z][a-z0-9+.-]{1,20}:\/\/)([A-Za-z0-9_-]{20,200})@(?=[A-Za-z0-9.-]{1,253}(?::\d{1,5})?[/:])/g
521
522// A credential in a URL query: ?api_key=, &sig=, ?accessToken=. The value may
523// be percent-encoded, so it is decoded before it is judged.
524const URL_KEYWORD =
525  '(?:[Aa]pi[_-]?[Kk]ey|API[_-]?KEY|[Aa]pikey|[Kk]ey|KEY|[Tt]oken|TOKEN|[Ss]ecret|SECRET|[Ss]ig|SIG|[Ss]ignature|SIGNATURE|[Pp]assword|PASSWORD|[Pp]asswd|[Pp]wd|PWD|[Aa]uth|AUTH|[Cc]redential|CREDENTIAL)'
526const URL_PARAM = new RegExp(
527  String.raw`([?&;](?:[A-Za-z0-9._-]*[_.-]|[a-z0-9]+(?=[A-Z]))?${URL_KEYWORD}=)((?:[A-Za-z0-9+_~-]|%[0-9A-Fa-f]{2}){8,400}={0,2})(?=[&#\s"'<>)\]]|$)`,
528  'g',
529)
530
531// Values after a password-like key, judged by the key and not by entropy:
532// a human password (`Summer2024!`, `hunter2hunter2`) has too little.
533const PW_WORD = String.raw`(?:password|passwd|passphrase|secret|secret[_-]?key|client[_-]?secret|api[_-]?key|apikey|private[_-]?key|secret[_-]?access[_-]?key|access[_-]?key|auth[_-]?token)`
534// Short password names (`pass`, `pw`, `pwd`, `passcode`). They count only as
535// the whole key or after a separator, so `compass` and `bypass` are left.
536const SHORT_PW = String.raw`(?:pass(?:[_-]?code)?|pwd?)`
537const SHORT_KEY = new RegExp(String.raw`(?:^|[_.-])${SHORT_PW}$`, 'i')
538// A short key (`pass:`) with a number or a short word after it is a count
539// or a status, not a password.
540const shortKeyStatus = (key: string, v: string): boolean => SHORT_KEY.test(key) && (/^[0-9]+$/.test(v) || /^[a-z]{1,8}$/.test(v))
541const STRONG_QUOTED = new RegExp(
542  String.raw`((?:^|[\s{,;(\[])(\\?["']?)((?:[A-Za-z0-9._-]*[_.-]|pg)?${PW_WORD}|(?:[A-Za-z0-9._-]*[_.-])?${SHORT_PW})\2[ \t]*[:=][ \t]*)(\\?["'])([^\n]{3,200}?)\4`,
543  'gim',
544)
545// A YAML list item (`- password: x`, compose `- DB_PASS=x`) counts too.
546const STRONG_LINE = new RegExp(
547  String.raw`^([ \t]*(?:-[ \t]+)?(?:export[ \t]+)?([A-Za-z0-9._-]*${PW_WORD}|(?:[A-Za-z0-9._-]*[_.-])?${SHORT_PW})[ \t]*[:=][ \t]*)([^\s"'#;][^\s"';]{2,199})[ \t]*$`,
548  'gim',
549)
550// `DB_PASS=`: upper case with a separator only, so prose `Pass:` is left.
551const ENV_PASS = /^([ \t]*(?:-[ \t]+)?(?:export[ \t]+)?()[A-Z0-9_]*_PASS[ \t]*[:=][ \t]*)([^\s"'#;][^\s"';]{2,199})[ \t]*$/gm
552// A Gmail app password written unquoted: four groups of four letters.
553const APP_PASSWORD = new RegExp(
554  String.raw`^([ \t]*(?:-[ \t]+)?(?:export[ \t]+)?[A-Za-z0-9._-]*${PW_WORD}[ \t]*[:=][ \t]*)([a-z]{4}(?: [a-z]{4}){3})[ \t]*$`,
555  'gim',
556)
557// The line after a YAML block indicator: `password: |` then the value.
558const BLOCK_PASSWORD = new RegExp(
559  String.raw`^([ \t]*(?:-[ \t]+)?["']?[A-Za-z0-9._-]*${PW_WORD}["']?[ \t]*:[ \t]*[|>][-+]?[0-9]?[ \t]*\r?\n[ \t]+)(\S[^\n]{2,199}?)[ \t]*(?=\r?$)`,
560  'gim',
561)
562// SQL: `CREATE USER x WITH PASSWORD '...'`, MySQL `IDENTIFIED BY '...'`.
563const SQL_PASSWORD =
564  /(\b(?:(?:with|encrypted|unencrypted)[ \t]+password|identified[ \t]+(?:with[ \t]+\S+[ \t]+)?by(?:[ \t]+password)?)[ \t]+)'([^'\n]{3,200})'/gi
565// Terraform: the default of a variable whose name says it is a password.
566const HCL_DEFAULT = new RegExp(String.raw`(\bvariable[ \t]+"[\w-]*${PW_WORD}"[ \t]*\{[^{}]{0,400}?\bdefault[ \t]*=[ \t]*")([^"\n]{3,200})"`, 'gi')
567// Terraform state: the result of a random_password (usually a database
568// master password), and an output that sits under a password-like name.
569const TF_RESULT = /("result"\s*:\s*")((?:[^"\\\n]|\\.){3,400})"/g
570const TF_OUTPUT = new RegExp(
571  String.raw`("([\w.-]{1,100})"\s*:\s*\{\s*(?:"(?:sensitive|type)"\s*:\s*[^,{}]{1,40},\s*){0,4}"value"\s*:\s*")((?:[^"\\\n]|\\.){3,400})"`,
572  'g',
573)
574// A name and its value as sibling fields: a HAR header or form field, a
575// Kubernetes env entry. The name decides; the value sits apart from it.
576const JSON_NAME_VALUE = /("name"\s*:\s*"([^"\n]{1,100})"\s*,\s*"value"\s*:\s*")((?:[^"\\\n]|\\.){3,400})"/g
577const JSON_VALUE_NAME = /("value"\s*:\s*")((?:[^"\\\n]|\\.){3,400})("\s*,\s*"name"\s*:\s*"([^"\n]{1,100})")/g
578const YAML_NAME_VALUE =
579  /^([ \t]*)(-[ \t]+name:[ \t]*["']?([\w.-]{1,100})["']?[ \t]*\r?\n\1[ \t]+value:[ \t]*(["']?))([^\n"']{3,200}?)\4[ \t]*(?=\r?$)/gm
580const PW_NAME = new RegExp(String.raw`^[\w.-]*${PW_WORD}$|^(?:[\w.-]*[_.-])?${SHORT_PW}$`, 'i')
581// SQL: an INSERT whose column list names a key or a password.
582const SQL_INSERT = /\bINSERT[ \t]+INTO[ \t]+[\w."`[\]]{1,100}[ \t]*\(([^()\n]{1,1000})\)\s*VALUES\s*/gi
583// `PGPASSWORD=x psql` is the one common env name with no separator.
584const KV_PASSWORD = /((?:^|[\s;&?,])-{0,2}(?:[A-Za-z0-9._-]*[_.-]|pg)?(?:password|passwd)=|[;&?]pwd=)([^\s"'&;,]{3,200})/gim
585
586// A generated password in quotes, with no name beside it: upper, lower,
587// digit and a symbol from the Terraform and password-manager sets, no space,
588// no path or regex characters. The token scan cuts such a value at each
589// symbol, so it is judged whole here.
590// The opening quote must follow a space, `=`, `:`, `(`, `,`, `[` or `{`, so
591// the text between two strings (`"0"=20cellpadding=3D"`) is not taken.
592const QUOTED_VALUE = /(?<=^|[\s:=(,[{])(["'])([^\s"'\\/.,;`]{12,128})\1/gm
593const PW_SYMBOL = /[!#$%&*()=+[\]{}<>:?@^~]/
594
595function looksGenerated(v: string): boolean {
596  if (!/[A-Z]/.test(v) || !/[a-z]/.test(v) || !/[0-9]/.test(v) || !PW_SYMBOL.test(v)) return false
597  if (/\$\{|\{\{|%\(|^\^|\$$|[A-Za-z]-[A-Za-z]\]|=>|::|\(\)|<\/?[a-z]|=[0-9A-F]{2}|[0-9]{6}/.test(v)) return false
598  // Real words in it: code, a format, a shortcut (`Ctrl+Alt+F12`), a payee
599  // with an order number (`Merchant#a12345`).
600  const words = v.match(/[A-Za-z]{4,}/g)?.filter((w) => WORDY.test(w)) ?? []
601  if (words.length >= 2 || words.some((w) => w.length >= 6)) return false
602  // Mostly letters and digits: a run of symbols is binary or markup.
603  if ((v.match(/[A-Za-z0-9]/g)?.length ?? 0) * 5 < v.length * 3) return false
604  return entropy(v) >= 3.5
605}
606
607// Things that stand in for a value without being one: a variable, a call, a
608// template, a placeholder, a type, the key's own name, a path.
609// A call has to close (`getSecret("x")`), and a YAML tag is a word
610// (`!Ref DbPassword`), so a generated password that starts `ab(` or `!` is
611// not taken for one.
612const REFERENCE =
613  /^[A-Za-z_$][\w$.]*\(.*\)[,;]?$|^\$|^\{\{|^<.*>$|^%\(|^!!?[A-Za-z][\w:]*(?:[ \t]|$)|^\*+$|^(?:true|false|null|none|nil|undefined|required|optional|string|str|bool|int|number)$|^(?:process\.env|os\.environ|env\.|config\.|settings\.)|^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+$/i
614const SELF_REF =
615  /^[{`(]?(?:[a-z][A-Za-z0-9]*)?(?:Password|Passwd|Secret|Key|Token|Pwd)[A-Za-z0-9]*[)},`]*$|^(?:password|passwd|secret|apikey|api_key|pwd|token)[),}]*,?$/
616const PLACEHOLDER = /^(?:x{3,}|\.{3,}|…|(?:your|my|<)[_ -]?[a-z_ -]*(?:key|password|secret|token)[a-z_ -]*>?)$/i
617
618function isStandIn(v: string): boolean {
619  return (
620    REFERENCE.test(v) ||
621    SELF_REF.test(v) ||
622    PLACEHOLDER.test(v) ||
623    /^(?:[/~]|\.\.?\/)/.test(v) ||
624    /^\{[\w$.]+\}$/.test(v) ||
625    // A test fixture made of words: `test-internal-key`, `fake_secret`.
626    (/^(?:[a-z]+[_-])*(?:test|fake|dummy|mock|sample|example|placeholder|changeme)(?:[_-][a-z]+)*$/.test(v) && /[_-]/.test(v)) ||
627    // `password: pw,` in an object literal is code, not a value.
628    /^[A-Za-z_$][\w$]*,$/.test(v)
629  )
630}
631
632// Credentials on a command line: `--password x`, `curl -u user:pw`,
633// `mysql -pPW`, a .pgpass line, a .netrc entry, `gh secret set --body`.
634const CLI_FLAG =
635  /((?:^|\s)--?(?:password|passwd|pass|api-key|apikey|api_key|token|client-secret|auth-token|access-token)(?:=|[ \t]+)(["']?))(?![$<{%])([^\s"'=-][^\s"']{2,200})\2/g
636const CLI_USERPASS =
637  /(\b(?:curl|wget|http|https|xh|httpie|lftp|ftp)\b[^\n|;&]*?\s(?:-u|-U|--user|--proxy-user)(?:[ \t]+|=)["']?[^\s:"'$]+:)(?!\$)([^\s"'`]{3,200})/g
638const MYSQL_P =
639  /(\b(?:mysql(?:dump|admin|import|sh|check|slap|binlog)?|mariadb(?:-dump|-admin)?)\b[^\n|;&]*?\s-p)(?!\$)([^\s`-][^\s`]{2,200})/g
640const PGPASS =
641  /^((?:\*|[A-Za-z0-9.-]+|\[[0-9A-Fa-f:]+\]):(?:\d{1,5}|\*):(?![0-9]+:)[A-Za-z0-9_$*.-]+:[A-Za-z0-9_$*.@-]+:)(?!\/\/)(\S{3,200})$/gm
642const NETRC =
643  /((?:^|\n)[ \t]*(?:machine[ \t]+\S+|default)(?:\s+(?:login|account|port)[ \t]+\S+)*\s+password[ \t]+)(\S{3,200})(?=[ \t]*(?:\r?\n|$)|\s+(?:machine|default|login|account|macdef)\b)/g
644const GH_SECRET = /(\bgh[ \t]+secret[ \t]+set\b[^\n;&|]*?[ \t](?:-b|--body)(?:[ \t]+|=)(["']?))([^\s"']{3,400})(?=\2)/g
645// Short password flags that only mean a password for one command:
646// `sshpass -p`, `docker login -p`, `mongosh -p`, `7z -pPW`, `sqlcmd -P`,
647// `zip -P`, `redis-cli -a`, `ldapsearch -w`. Each is anchored at its command.
648const CMD_PW_TOOLS: [cmd: string, flag: string][] = [
649  [String.raw`sshpass|(?:docker|podman|nerdctl|helm[ \t]+registry)[ \t]+login|mongo(?:sh|dump|restore|import|export)?`, String.raw`-p[ \t]*`],
650  [String.raw`7za?|7zz`, '-p'],
651  [String.raw`sqlcmd|zip`, String.raw`-P[ \t]+`],
652  [String.raw`redis-cli|valkey-cli`, String.raw`-a[ \t]+`],
653  [String.raw`ldap[a-z]+`, String.raw`-w[ \t]+`],
654]
655const CMD_PW = CMD_PW_TOOLS.map(
656  ([cmd, flag]) => new RegExp(String.raw`(\b(?:${cmd})\b[^\n|;&]*?[ \t]${flag}(["']?))(?![$-])([^\s"']{3,200})\2`, 'g'),
657)
658const CMD_PW_GATE = /sshpass|login|mongo|7z|sqlcmd|zip|redis-cli|valkey-cli|ldap/
659// `openssl ... -passout pass:PW`, `smbclient -U user%PW`, `htpasswd -b file
660// user PW`, `rabbitmqctl add_user user PW`.
661const OPENSSL_PASS = /(-pass(?:in|out)?[ \t]+pass:)([^\s"']{3,200})/g
662const SMB_USER = /(\bsmbclient\b[^\n|;&]*?[ \t]-U[ \t]+["']?[^\s%"']+%)([^\s"']{3,200})/g
663const HTPASSWD = /(\bhtpasswd[ \t]+-[a-zA-Z]*b[a-zA-Z]*[ \t]+(?:-[a-zA-Z]+[ \t]+)*\S+[ \t]+\S+[ \t]+)(["']?)([^\s"']{3,200})\2/g
664const NET_USER = /(\bnet[ \t]+user[ \t]+(?:"[^"\n]{1,64}"|\S+)[ \t]+)(["']?)((?!\/)[^\s"'*]{3,200})\2/gi
665const RABBIT_USER = /(\brabbitmqctl[ \t]+(?:add_user|change_password)[ \t]+\S+[ \t]+)(["']?)([^\s"']{3,200})\2/g
666
667// Cookies: every value in a Cookie header, curl -b, or cookies.txt.
668const COOKIE = /((?:^|["'\s])(?:(?:set-)?cookie["']?[ \t]*:|-b|--cookie)[ \t]*["']?)([^"'\n]*)/gim
669const COOKIE_PAIR = /(^|;[ \t]*)([^=;\s]+=)([^;\s]{16}[^;\s]*)/g
670const COOKIE_ATTR = /^(?:path|domain|expires|max-age|samesite|priority)=$/i
671const COOKIE_VALUE = /^(?:s%3A)?([A-Za-z0-9_+%-]{16}[A-Za-z0-9_+%-]*)(?:\.[A-Za-z0-9_+%/-]{16}[A-Za-z0-9_+%/-]*)?={0,2}$/i
672const COOKIES_TXT =
673  /^((?:#HttpOnly_)?\.?[\w.-]+\t(?:TRUE|FALSE)\t\S+\t(?:TRUE|FALSE)\t\d+\t[^\t\n]+\t)([^\t\r\n]{16}[^\t\r\n]*)$/gm
674const HAR_COOKIE = /("name"\s*:\s*"[^"]*(?:sess|sid|token|auth)[^"]*"\s*,\s*"value"\s*:\s*")([^"]{16}[^"]*)"/gi
675
676// Basic auth and Docker `auth`: base64 of `user:password`, often short.
677const BASIC_AUTH = /((?:["']auth["'][ \t]*:[ \t]*["']|\b[Bb]asic[ \t]+))([A-Za-z0-9+/]{8,200}={0,2})(?![A-Za-z0-9+/=])/g
678// A Kubernetes bootstrap token: `abcdef.0123456789abcdef`.
679const BOOTSTRAP = /\b([a-z0-9]{6})\.([a-z0-9]{16})\b/g
680
681// Inline media in a data: URI. Only media types: a key put in a JSON or text
682// data URI is still scanned. The length floor stops a short key placed after
683// a fake image prefix from being excused.
684const DATA_URI =
685  /data:(?:(?:image|font|audio|video)\/[\w.+-]+|application\/(?:x-)?font-[\w.+-]+|application\/vnd\.ms-fontobject)(?:;[\w-]+=[\w.-]+)*;base64,[A-Za-z0-9+/]{64}[A-Za-z0-9+/]*={0,2}/g
686
687// A base64 blob: one long run, or three or more wrapped lines (a PEM bundle,
688// a MIME part). A trailing short line is only taken when a blank line, a
689// `--boundary` or the end of the text follows it.
690const B64_LINE = String.raw`[A-Za-z0-9+/]{40}[A-Za-z0-9+/]*={0,2}\r?\n`
691const BLOB = new RegExp(
692  String.raw`(?<![A-Za-z0-9+/=])(?:(?:${B64_LINE}){3}(?:${B64_LINE})*(?:[A-Za-z0-9+/]+={0,2}(?=\r?\n[\r\n-]|\r?\n?$))?|[A-Za-z0-9+/]{256}[A-Za-z0-9+/]*={0,2})`,
693  'g',
694)
695const BLOB_NAME =
696  /(?:key|token|secret|password|passwd|credential|auth|private|signature|session|cookie|dsn|cert[_-]?key)[\w.-]*["'\]\s]{0,4}[:=]{1,2}\s*["'`]?\s*$/i
697const BLOB_INNER = /PRIVATE KEY|OPENSSH|"private_key"|"client_secret"|"refresh_token"|"password"|"auths"/
698
699// A Claude thinking-block signature, and a redacted thinking block: opaque
700// to everyone and useless as a credential. Only the value is parked; the
701// thinking text beside it is still scanned.
702const THINKING_SIG =
703  /("type"\s*:\s*"thinking"\s*,\s*"thinking"\s*:\s*"[^"\\]*(?:\\.[^"\\]*)*"\s*,\s*"signature"\s*:\s*")([A-Za-z0-9+/]{200}[A-Za-z0-9+/]*={0,2})(?=")/g
704const REDACTED_THINKING = /("type"\s*:\s*"redacted_thinking"\s*,\s*"data"\s*:\s*")([A-Za-z0-9+/]{100}[A-Za-z0-9+/]*={0,2})(?=")/g
705
706// A lockfile or SRI integrity hash: exact digest lengths only, so a key
707// glued after `sha256-` is still judged.
708const SRI = /\bsha(?:1-[A-Za-z0-9+/]{27}=|256-[A-Za-z0-9+/]{43}=|384-[A-Za-z0-9+/]{64}|512-[A-Za-z0-9+/]{86}==)(?![A-Za-z0-9+/=])/g
709
710// Links whose path is kept as written: public font files (the file name is
711// a hash, not a secret) and the hosts of the `allowUrlHosts` option (a task
712// link like `/app/task/<slug>-<id>`). The host has to follow the scheme and be
713// followed by a port or `/`, so `evil.com/<host>/`, `<host>.evil.com` and
714// `<host>@evil.com` don't match. The slashes may be JSON-escaped (`\/`). Path
715// characters only, so a query string or fragment is still scanned.
716type AllowUrl = { hint: RegExp; re: RegExp }
717const BUILT_IN_URL_HOSTS = ['fonts.gstatic.com']
718const HOST_NAME = /^(?:\*\.)?[a-z0-9_](?:[a-z0-9_.-]*[a-z0-9])?$/
719const IPV6_HOST = /^\[[0-9a-f:.]+\]$/
720
721// `https://Host:8080/path` becomes `host`. An IPv6 literal keeps its brackets.
722function cleanHost(entry: string): string {
723  const h = entry.toLowerCase().replace(/^[a-z][a-z0-9+.-]*:\/\//, '').replace(/[/?#].*$/, '').replace(/^[^@]*@/, '')
724  return h.startsWith('[') ? h.replace(/\](?::\d+)?$/, ']') : h.replace(/:\d*$/, '')
725}
726
727function allowUrlHosts(entries: string[]): { hosts: string[]; rejected: string[] } {
728  const hosts: string[] = []
729  const rejected: string[] = []
730  for (const e of entries) {
731    const h = cleanHost(e)
732    if (HOST_NAME.test(h) || IPV6_HOST.test(h)) hosts.push(h)
733    else rejected.push(e)
734  }
735  return { hosts, rejected }
736}
737
738const reEscape = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
739
740// `*.example.com` stands for any subdomain of example.com, not example.com.
741function allowUrlPattern(hosts: string[]): AllowUrl {
742  const all = [...new Set([...BUILT_IN_URL_HOSTS, ...hosts])]
743  const alt = all.map((h) => (h.startsWith('*.') ? String.raw`(?:[a-z0-9_-]+\.)+${reEscape(h.slice(2))}` : reEscape(h))).join('|')
744  const hint = new RegExp(all.map((h) => reEscape(h.replace(/^\*\./, ''))).join('|'), 'i')
745  const slash = String.raw`(?:\/|\\\/)`
746  return { hint, re: new RegExp(String.raw`https?:${slash}${slash}(?:${alt})(?::\d{1,5})?${slash}(?:[A-Za-z0-9_.~%-]|${slash})*`, 'gi') }
747}
748
749// Parks the path of each allowed link before anything else runs, so neither
750// the token scan nor a value hidden elsewhere in the text can touch it. A path
751// holding a vendor-shaped key is not parked; the passes judge it as usual.
752// Parking also ends the path cleanly: in `/x/ab+<key>` only `/x/ab` is
753// parked, and `<key>` is judged on its own.
754function parkAllowedUrls(text: string, cfg: Config): string {
755  if (!text.includes(':') || !cfg.allowUrl.hint.test(text)) return text
756  return text.replace(cfg.allowUrl.re, (m) => (scrubVendors(m, cfg) === m ? park(m) : m))
757}
758
759// Mail: an optional `cat -n`, Read or `grep -n` prefix on each line.
760const PFX = String.raw`(?:[ \t]*\d+\t|[^\s:]+:\d+[:-])?`
761const PFX_RE = new RegExp(`^${PFX}`)
762const CTE = new RegExp(String.raw`^${PFX}Content-Transfer-Encoding:[ \t]*base64[ \t]*\r?$`, 'gim')
763const CONTENT_TYPE = new RegExp(String.raw`^${PFX}Content-Type:[ \t]*(?:\r?\n${PFX}[ \t]+)?([^;\s]+)`, 'im')
764const MAIL_HEADER_NAMES =
765  'DKIM-Signature|X-Google-DKIM-Signature|ARC-Seal|ARC-Message-Signature|ARC-Authentication-Results|Authentication-Results|Message-I[Dd]|References|In-Reply-To|Feedback-ID|X-SG-EID|X-SG-ID|X-MSFBL|X-Gm-Message-State|X-Google-Smtp-Source|X-Gm-Gg|X-Gm-Features|X-Received|Received-SPF|Return-Path|Received'
766const MAIL_HEADER = new RegExp(String.raw`^${PFX}(?:${MAIL_HEADER_NAMES}):[^\n]*(?:\r?\n${PFX}[ \t][^\n]*){0,12}`, 'gm')
767const HEADER_LINE = new RegExp(String.raw`^${PFX}[A-Z][A-Za-z0-9-]*:`)
768const ENCODED_WORD = /=\?[\w-]+\?[Bb]\?[A-Za-z0-9+/=]{4}[A-Za-z0-9+/=]*\?=/g
769const BOUNDARY = /Content-Type:[ \t]*multipart\/[^\n]*?(?:\r?\n[ \t][^\n]*?)?\bboundary="?([^"\s;]+)"?/gi
770
771// The generic scan. A slash-bearing base64 run (an AWS secret, `openssl rand
772// -base64`) is judged whole first; anything else is cut into candidate
773// tokens. `/` and `.` are left out of a candidate on purpose: with them in,
774// every long file path becomes one. A token never starts inside a `%XX`
775// escape, and `=` only counts as padding when nothing follows it.
776const CAND_BODY = String.raw`[A-Za-z0-9+_-]{12,200}(?:={1,2}(?![A-Za-z0-9=]))?`
777const CANDIDATE_SRC = String.raw`(?<=%[0-9A-Fa-f]{2})(?<!%25)${CAND_BODY}|(?<=%25[0-9A-Fa-f]{2})${CAND_BODY}|(?<!%[0-9A-Fa-f]?|%25[0-9A-Fa-f]?)${CAND_BODY}`
778const SLASHED_SRC = String.raw`(?<![A-Za-z0-9+/_.%~-])[A-Za-z0-9+/]{20,300}={0,2}(?![A-Za-z0-9+/=_.%~-])`
779const CANDIDATE = new RegExp(CANDIDATE_SRC, 'g')
780const SCAN = new RegExp(`(${SLASHED_SRC})|${CANDIDATE_SRC}`, 'g')
781
782// A name to the left of the candidate that says the candidate is a key:
783// `API_KEY=`, `"token":`, an escaped `\"token\":`, `&quot;token&quot;:`,
784// `Authorization: Bearer `, `Authorization: <any scheme> `, `--api-key `, a
785// table cell, a YAML block, a Go header map `X-Api-Key:[`, a Python header
786// tuple `('X-Api-Key', '`. `PAT` is a personal access token, not `PATH`.
787const NAME_WORDS =
788  'key|token|secret|password|passwd|pwd|credential|auth|bearer|private|signature|session|cookie|dsn|salt|nonce|otp|sess(?:ion)?[_-]?id|(?<![A-Za-z])sid|(?<![A-Za-z])sig|(?<![A-Za-z])pat(?![a-z])'
789const Q = String.raw`(?:\\{0,3}["'${'`'}]|&(?:quot|apos|#0*3[49]|#x0*2[27]);|\\u00(?:22|27))`
790const NAMED = new RegExp(
791  '(?:' +
792    String.raw`(?:${NAME_WORDS})(?:(?<=key)[_-]?id)?(?:${Q}|\]|\s){0,6}[:=]{1,2}(?:${Q}|\s){0,4}\[?` +
793    String.raw`|(?:${NAME_WORDS})[\w-]{0,20}${Q}[ \t]*,[ \t]*${Q}` +
794    String.raw`|\bbearer[ \t]+["']?` +
795    String.raw`|authorization["']?[ \t]*[:=][ \t]*["']?(?:[A-Za-z][\w-]{0,31}[ \t]+)?` +
796    String.raw`|(?:^|\s)--?(?![a-z_-]*(?:public|pub|cache|idempotency|signing|recv)[_-]?key)[a-z_-]*(?:key|token|secret|password|passwd|pw|auth)[ \t]+["']?` +
797    String.raw`|\|[ \t]*[\w.-]*(?:key|token|secret|password)[ \t]*\|[ \t]*` +
798    String.raw`|(?:key|token|secret|password|passwd|pwd|credential|auth)[\w-]*["']?[ \t]*:[ \t]*[>|][-+1-9]{0,2}[ \t]*\r?\n[ \t]+` +
799    ')$',
800  'i',
801)
802
803// A sibling field name that makes its value a key: the value is then
804// judged as if the name stood right before it.
805const KEY_NAME = new RegExp(String.raw`(?:${NAME_WORDS}|authorization)`, 'i')
806
807const HEX_ONLY = /^[0-9a-f]+$/i
808const PREFIXED_KEY = /^[a-z]{2,12}(?:_v[0-9])?[_-]([a-z0-9]{32,})$/
809const DIGITS_ONLY = /^[0-9]+$/
810const LETTERS_ONLY = /^[A-Za-z]+$/
811
812// Public ids with a shape rather than a prefix: a YouTube channel or
813// playlist, a Monzo transaction, a Vercel deployment, team or project, an
814// OpenAI org, a Claude Code session, environment or server tool call. Exact
815// lengths only, and the Anthropic ids start `01`, so any other token after
816// the same prefix is still judged.
817const PUBLIC_SHAPE =
818  /^(?:UC[A-Za-z0-9_-]{22}|PL[A-Za-z0-9_-]{16,32}|(?:tx|mm|acc|pot|merch|user)_0000[A-Za-z0-9]{18}|dpl_[A-Za-z0-9]{24,32}|org-[A-Za-z0-9]{24}|team_[A-Za-z0-9]{24}|prj_[A-Za-z0-9]{28}|(?:session|env|srvtoolu)_01[A-Za-z0-9]{22})$/
819
820// Prefixes that name a PUBLIC object id, not a key. Stripe hands these out in
821// dashboards, invoices and source code; hiding them makes a session useless
822// and protects nothing. `pk_live_` is Stripe's publishable key, also public.
823const PUBLIC_PREFIX =
824  /^(?:price|prod|cus|sub|sched|in|ch|pi|cs|py|re|txn|il|si|seti|evt|acct|promo|coupon|plan|card|ba|src|dp|du|iv|ii|rcpt|file|link|pm|tok|pk|test|toolu|msg|req|run|wf)_/i
825// A public id is the prefix, maybe `live_` or `test_`, then one alnum body.
826// `prod_api_key_<random>` is not that shape, so it is judged like any token.
827const PUBLIC_ID_BODY = /^(?:(?:live|test)_)?[A-Za-z0-9]+$/
828// A WooCommerce consumer secret reuses Stripe's `cs_`; Stripe's never is
829// 40 lowercase hex.
830// A Klaviyo private key reuses `pk_`, with no `live_` or `test_` after it.
831const NOT_PUBLIC = /^cs_[a-f0-9]{40}$|^pk_(?!live_|test_)[A-Za-z0-9]{34}$/
832// Generic prefixes a key name in front may override (`API_KEY=test_...`).
833const SOFT_PUBLIC = /^(?:test|tok)_/i
834
835// A name written in code (`archived-modules-marker`, `handleCheckoutSession`,
836// `lesson_article_outline_open_v1`) reads as high entropy but is a word list.
837// A key is not built out of words.
838const WORD_PART = /^(?:[a-z]+[0-9]{0,3}|[A-Z][a-z]+[0-9]{0,3}|[A-Z]{2,})$/
839const ALNUM_ONLY = /^[A-Za-z0-9]+$/
840const SEGMENTS = /[A-Z]+(?![a-z])|[A-Z]?[a-z]+|[0-9]+/g
841const WORDY = /^[A-Za-z]?[a-z]{2,}$/
842const SLASH_WORD = /^(?:[a-z]+[0-9]{0,3}|[A-Z][a-z]+[0-9]{0,3})$/
843const STRICT_CAMEL = /^(?:[a-z]{2,}|[A-Z][a-z]{2,})(?:[A-Z][a-z]{2,})+[0-9]{0,3}$/
844
845// `playerEventBatchV1Schema` splits into six segments of which four are
846// words; a random key splits into many segments of which almost none are.
847// A camel name has a capital in it: an all-lowercase token with a digit run
848// in the middle is a random value more often than a name.
849function isCamelName(token: string): boolean {
850  if (!ALNUM_ONLY.test(token) || !/[A-Z]/.test(token)) return false
851  const segs = token.match(SEGMENTS)
852  if (!segs || segs.join('') !== token) return false
853  const words = segs.filter((seg) => WORDY.test(seg)).length
854  return words >= 2 && words >= segs.length - 2
855}
856
857// Every character used once AND mostly in alphabet order: a charset constant.
858// The order test matters: a random 24-character key has all distinct
859// characters about 1% of the time.
860function isAlphabet(token: string): boolean {
861  if (token.length < 20) return false
862  let runs = 0
863  for (let i = 1; i < token.length; i++) if (token.charCodeAt(i) === token.charCodeAt(i - 1) + 1) runs++
864  return runs * 2 >= token.length && new Set(token).size === token.length
865}
866
867// Half or more of the characters follow the one before in code order:
868// `0123456789abcdef`, the base64 alphabet. A sample in docs, never a key.
869function countsUp(s: string): boolean {
870  let runs = 0
871  for (let i = 1; i < s.length; i++) if (s.charCodeAt(i) === s.charCodeAt(i - 1) + 1) runs++
872  return runs * 2 >= s.length
873}
874
875function isMixed(s: string): boolean {
876  return /[a-z]/.test(s) && /[A-Z]/.test(s) && /[0-9]/.test(s)
877}
878
879// One word, or a number of up to four digits (a year, a month, a day).
880function isSlugPart(part: string): boolean {
881  return WORD_PART.test(part) || /^[0-9]{1,4}$/.test(part)
882}
883
884// A part made only of words, short acronyms and short numbers: `expenseID`,
885// `2026`, `v2Final`. It names something; it is not a random remainder.
886function isWordish(part: string): boolean {
887  const segs = part.match(SEGMENTS)
888  if (!segs || segs.join('') !== part) return false
889  return segs.every((seg) => WORDY.test(seg) || /^[A-Z]{2,4}$/.test(seg) || /^[0-9]{1,4}$/.test(seg))
890}
891
892function looksLikeName(token: string, named: boolean): boolean {
893  if (isCamelName(token)) return true
894  // A name beside the token outweighs the alphabet rule.
895  if (!named && isAlphabet(token)) return true
896  const parts = token.split(/[_-]/)
897  if (parts.length < 2) return false
898  // Beside a key's name, one key-like part makes the whole token a key:
899  // `django-insecure-<random>`, `live-secret-<random>`.
900  if (named && parts.some((p) => p.length >= 16 && /[0-9]/.test(p) && /[A-Za-z]/.test(p) && entropy(p) >= 3.0)) return false
901  // A long random remainder outweighs any words in front of it
902  // (`vercel_blob_rw_<random>`, `dckr_pat_<random>`). Mixed case is asked
903  // for, so a hex cache key or a timestamp is not a remainder.
904  const residue = parts.filter((p) => !WORD_PART.test(p) && !isCamelName(p) && !isWordish(p)).join('')
905  // Words plus one long number (a LinkedIn post slug, a Zoom recording, an
906  // Amazon ref tag): the number is an id, not part of a random remainder.
907  // Only the parts that are neither words, numbers nor date stamps count,
908  // and camel parts are not excused here, since a random string can pass
909  // the camel test.
910  const longNumber = !named && parts.some((p) => /^[A-Z]{0,4}[0-9]{6,}$/.test(p))
911  const randomPart = longNumber ? parts.filter((p) => !WORD_PART.test(p) && !/^[0-9]{1,4}$/.test(p) && !DIGITS_ONLY.test(p) && !DATE_STAMP.test(p)).join('') : ''
912  if ((longNumber ? (randomPart.length >= 8 ? 16 : 0) : residue.length) >= 16 && isMixed(residue)) return false
913  // Words, dates and short numbers only (`2025-08-26-lesson`): a slug.
914  if (parts.every(isSlugPart)) return true
915  // Three or more separated parts, two of them plain words: a naming
916  // convention (`META_Conv_Lookalike-Customers_FreeTrial_2024Q1`), not a key.
917  if (parts.length >= 3 && parts.filter((p) => /^[A-Za-z]{4,}$/.test(p)).length >= 2) return true
918  return longNumber && randomPart.length < 8
919}
920const DATE_STAMP = /^[A-Z]{0,4}(?:19|20)[0-9]{6}$/
921
922function isPublicId(token: string, named: boolean, cfg: Config): boolean {
923  if (PUBLIC_SHAPE.test(token)) return true
924  if (cfg.allowPrefix.some((p) => token.startsWith(p))) return true
925  if (!PUBLIC_PREFIX.test(token) || NOT_PUBLIC.test(token)) return false
926  if (cfg.denyPrefix.some((p) => token.startsWith(p))) return false
927  if (named && SOFT_PUBLIC.test(token)) return false
928  return PUBLIC_ID_BODY.test(token.replace(PUBLIC_PREFIX, ''))
929}
930
931// A letters-only key (no digit at all) splits into many short camel
932// segments; a camel name of the same length splits into a few long ones.
933function lettersLookRandom(token: string, named: boolean, h: number): boolean {
934  if (!LETTERS_ONLY.test(token) || !/[a-z]/.test(token) || !/[A-Z]/.test(token)) return false
935  const segs = token.match(SEGMENTS) ?? []
936  if (named && token.length >= 20 && h >= 3.6 && segs.length * 4 >= token.length) return true
937  return token.length >= 24 && h >= 4.0 && segs.length * 3 >= token.length
938}
939
940// Claude Code transcripts and most analytics put a `sessionId` UUID on every
941// line. A UUID beside that name is an identifier, not a credential; a real
942// session cookie is random text, and the cookie rules hide it on their own.
943const SESSION_UUID = new RegExp(`^${UUID}$`, 'i')
944const SESSION_ID_NAME = new RegExp(String.raw`sess(?:ion)?[_-]?id(?:${Q}|\s){0,6}[:=]{1,2}(?:${Q}|\s){0,4}$`, 'i')
945
946function looksSecret(token: string, before: string, cfg: Config): boolean {
947  if (token.startsWith('REDACTED-')) return false
948  if (cfg.allow.has(token) || DIGITS_ONLY.test(token)) return false
949  const named = NAMED.test(before)
950  if (isPublicId(token, named, cfg)) return false
951  // `invoice+statements+acct_<id>`: a mail sender address made of words and
952  // a public id. Each `+` part has to be one, so base64 is still judged.
953  if (token.includes('+') && token.split('+').every((p) => p === '' || WORD_PART.test(p) || isPublicId(p, named, cfg))) return false
954  if (named && SESSION_UUID.test(token) && SESSION_ID_NAME.test(before)) return false
955  if (looksLikeName(token, named)) return false
956
957  const h = entropy(token)
958
959  // A bare hex run is a git SHA or a checksum far more often than it is a
960  // key, so hex needs a name beside it before it is hidden.
961  if (HEX_ONLY.test(token)) return named && token.length >= 24 && h >= cfg.contextMinEntropy
962
963  // A short lowercase prefix and a long random body (imp_ and 40 hex): a
964  // vendor key no rule names. A git SHA has no prefix, so it is not caught here.
965  const prefixed = token.match(PREFIXED_KEY)
966  if (prefixed && /[0-9]/.test(prefixed[1]!) && /[a-z]/.test(prefixed[1]!) && entropy(prefixed[1]!) >= cfg.contextMinEntropy) return true
967
968  if (isMixed(token) && token.length >= cfg.minLength && h >= cfg.minEntropy) return true
969  if (lettersLookRandom(token, named, h)) return true
970  // Next to a key's name the bar is lower, but the token still has to look
971  // like a key: letters and digits together, not a word.
972  const alnum = /[0-9]/.test(token) && /[A-Za-z]/.test(token)
973  return named && alnum && token.length >= 16 && h >= cfg.contextMinEntropy
974}
975
976// A slash-bearing base64 run, judged on the parts that are not words, so a
977// path like `uploads/2026/10/12/Report2Final` stays a path.
978function slashedSecret(run: string, named: boolean, cfg: Config): boolean {
979  if (cfg.allow.has(run)) return false
980  const body = run.replace(/=+$/, '').replace(/^\/+/, '')
981  if (!body.includes('/') || !isMixed(body) || countsUp(body)) return false
982  const parts = body.split('/').filter(Boolean)
983  const isWord = (p: string): boolean => SLASH_WORD.test(p) || STRICT_CAMEL.test(p) || DIGITS_ONLY.test(p)
984  const words = parts.filter(isWord).length
985  if (words * 2 >= parts.length) return false
986  if (!isMixed(parts.filter((p) => !isWord(p)).join(''))) return false
987  const h = entropy(body)
988  return (body.length >= 32 && h >= 4.3) || (named && body.length >= 16 && h >= 3.6)
989}
990
991// -------------------------------------------------------------- the passes
992
993// Mints `value` unless it is already a placeholder, allowed, or a stand-in.
994function pairOut(cfg: Config, head: string, value: string, tail = ''): string {
995  if (WHOLE_TAG.test(value) || cfg.allow.has(value) || isStandIn(value)) return head + value + tail
996  return head + mint('SECRET', value) + tail
997}
998
999function parkOpaque(text: string): string {
1000  let out = text
1001  if (out.includes('"signature"')) out = out.replace(THINKING_SIG, (_m, head: string, sig: string) => head + park(sig))
1002  if (out.includes('redacted_thinking')) out = out.replace(REDACTED_THINKING, (_m, head: string, data: string) => head + park(data))
1003  if (out.includes('sha')) out = out.replace(SRI, (m) => park(m))
1004  return out
1005}
1006
1007function scrubKeyBlocks(text: string, cfg: Config): string {
1008  let out = text
1009  if (out.includes('PRIVATE KEY')) out = out.replace(PRIVATE_KEY, (m) => (cfg.allow.has(m) ? m : mint('SECRET', m)))
1010  if (out.includes('Private-Lines:')) out = out.replace(PUTTY_PRIVATE, (_m, head: string, body: string) => head + mint('SECRET', body))
1011  return out
1012}
1013
1014// Walks the dot-separated parts of a run: a part holding `eyJ` and the two
1015// parts after it are a JWT, when each is long enough.
1016function scrubJwtChain(chain: string, cfg: Config): string {
1017  if (!chain.includes('eyJ')) return chain
1018  const segs = chain.split('.')
1019  const out: string[] = []
1020  let k = 0
1021  while (k < segs.length) {
1022    const i = k + 2 < segs.length ? segs[k]!.indexOf('eyJ') : -1
1023    if (i >= 0 && segs[k]!.length - i >= 11 && segs[k + 1]!.length >= 8 && segs[k + 2]!.length >= 8) {
1024      const jwt = `${segs[k]!.slice(i)}.${segs[k + 1]}.${segs[k + 2]}`
1025      out.push(segs[k]!.slice(0, i) + (cfg.allow.has(jwt) ? jwt : mint('SECRET', jwt)))
1026      k += 3
1027    } else {
1028      out.push(segs[k]!)
1029      k += 1
1030    }
1031  }
1032  return out.join('.')
1033}
1034
1035// `postgres://user:REDACTED@host` in docs: the word itself stands in for one.
1036const URL_PASSWORD_WORD = /^(?:password|passwd|pass|pw|pwd|secret|<[^>]*>)$/i
1037
1038function scrubUrls(text: string, cfg: Config): string {
1039  let out = text
1040  if (out.includes('://')) {
1041    out = out.replace(URL_PASSWORD, (m, head: string, password: string) =>
1042      WHOLE_TAG.test(password) || cfg.allow.has(password) || URL_PASSWORD_WORD.test(password) ? m : `${head}:${mint('SECRET', password)}@`,
1043    )
1044    out = out.replace(URL_USERINFO, (m, head: string, user: string) => {
1045      if (cfg.allow.has(user) || !/[0-9]/.test(user) || !/[A-Za-z]/.test(user) || looksLikeName(user, true)) return m
1046      return entropy(user) >= cfg.contextMinEntropy ? `${head}${mint('SECRET', user)}@` : m
1047    })
1048  }
1049  if (out.includes('=')) {
1050    out = out.replace(URL_PARAM, (m, head: string, value: string) => (urlParamSecret(value, cfg) ? head + mint('SECRET', value) : m))
1051  }
1052  return out
1053}
1054
1055function urlParamSecret(raw: string, cfg: Config): boolean {
1056  if (cfg.allow.has(raw)) return false
1057  let v = raw
1058  try {
1059    v = decodeURIComponent(raw)
1060  } catch {
1061    // A broken escape: judge it as it stands.
1062  }
1063  if (cfg.allow.has(v) || v.length < 12) return false
1064  if ([/[a-z]/, /[A-Z]/, /[0-9]/, /[^A-Za-z0-9]/].filter((r) => r.test(v)).length < 2) return false
1065  if (looksLikeName(v, true)) return false
1066  return entropy(v) >= cfg.contextMinEntropy
1067}
1068
1069function scrubCommandLines(text: string, cfg: Config): string {
1070  let out = text
1071  if (/-(?:pass|api|token|client|auth|access)/.test(out)) {
1072    out = out.replace(CLI_FLAG, (m, head: string, quote: string, v: string) => {
1073      if (/^[A-Z][A-Z0-9_]*$/.test(v) || /^[A-Za-z_]+=/.test(v) || /^[A-Za-z]+(?:[-_][A-Za-z]+)+$/.test(v)) return m
1074      return pairOut(cfg, head, v, quote)
1075    })
1076  }
1077  if (/\s-(?:u|U|-user|-proxy-user)\b/.test(out)) {
1078    out = out.replace(CLI_USERPASS, (m, head: string, v: string) => (DIGITS_ONLY.test(v) ? m : pairOut(cfg, head, v)))
1079  }
1080  if (/mysql|mariadb/.test(out)) out = out.replace(MYSQL_P, (_m, head: string, v: string) => pairOut(cfg, head, v))
1081  if (out.includes(':')) out = out.replace(PGPASS, (_m, head: string, v: string) => pairOut(cfg, head, v))
1082  if (out.includes('password')) out = out.replace(NETRC, (_m, head: string, v: string) => pairOut(cfg, head, v))
1083  if (out.includes('gh')) out = out.replace(GH_SECRET, (_m, head: string, _q: string, v: string) => pairOut(cfg, head, v))
1084  if (CMD_PW_GATE.test(out)) {
1085    for (const re of CMD_PW) out = out.replace(re, (_m, head: string, q: string, v: string) => pairOut(cfg, head, v, q))
1086  }
1087  if (out.includes('pass:')) out = out.replace(OPENSSL_PASS, (_m, head: string, v: string) => pairOut(cfg, head, v))
1088  if (out.includes('smbclient')) out = out.replace(SMB_USER, (_m, head: string, v: string) => pairOut(cfg, head, v))
1089  if (out.includes('htpasswd')) out = out.replace(HTPASSWD, (_m, head: string, q: string, v: string) => pairOut(cfg, head, v, q))
1090  if (/net[ \t]+user/i.test(out)) out = out.replace(NET_USER, (_m, head: string, q: string, v: string) => pairOut(cfg, head, v, q))
1091  if (out.includes('rabbitmqctl')) out = out.replace(RABBIT_USER, (_m, head: string, q: string, v: string) => pairOut(cfg, head, v, q))
1092  return out
1093}
1094
1095function scrubAssignments(text: string, cfg: Config): string {
1096  let out = text
1097  // A value under a key that says it is public is left: a publishable key.
1098  const value = (head: string, key: string, v: string, tail = ''): string => {
1099    if (/public|publishable/i.test(key)) return head + v + tail
1100    if (PUBLIC_PREFIX.test(v) && !/secret|private/i.test(key) && isPublicId(v, false, cfg)) return head + v + tail
1101    if (shortKeyStatus(key, v)) return head + v + tail
1102    return pairOut(cfg, head, v, tail)
1103  }
1104  // A value whose name sits in a sibling field.
1105  const sibling = (head: string, name: string, v: string, tail = ''): string => {
1106    if (PW_NAME.test(name)) return value(head, name, v, tail)
1107    if (KEY_NAME.test(name) && !/public|publishable/i.test(name) && looksSecret(v, 'key=', cfg)) return head + mint('SECRET', v) + tail
1108    return head + v + tail
1109  }
1110  if (/pass|secret|key|token|pw/i.test(out)) {
1111    out = out.replace(STRONG_QUOTED, (_m, head: string, _q: string, key: string, quote: string, v: string) => value(head + quote, key, v, quote))
1112    out = out.replace(APP_PASSWORD, (_m, head: string, v: string) => pairOut(cfg, head, v))
1113    out = out.replace(STRONG_LINE, (_m, head: string, key: string, v: string) => value(head, key, v))
1114    out = out.replace(ENV_PASS, (_m, head: string, _k: string, v: string) => value(head, '', v))
1115    out = out.replace(BLOCK_PASSWORD, (_m, head: string, v: string) => pairOut(cfg, head, v))
1116    out = out.replace(KV_PASSWORD, (_m, head: string, v: string) => pairOut(cfg, head, v))
1117  }
1118  if (PW_SYMBOL.test(out)) {
1119    out = out.replace(QUOTED_VALUE, (m, q: string, v: string, offset: number, whole: string) => {
1120      if (!looksGenerated(v) || isStandIn(v) || cfg.allow.has(v)) return m
1121      // A MIME boundary is random by design and public.
1122      if (/boundary=$/i.test(whole.slice(Math.max(0, offset - 9), offset))) return m
1123      return q + mint('SECRET', v) + q
1124    })
1125  }
1126  if (/password|identified/i.test(out)) out = out.replace(SQL_PASSWORD, (_m, head: string, v: string) => pairOut(cfg, head + "'", v, "'"))
1127  if (out.includes('variable')) out = out.replace(HCL_DEFAULT, (_m, head: string, v: string) => pairOut(cfg, head, v, '"'))
1128  if (out.includes('"value"')) {
1129    out = out.replace(JSON_NAME_VALUE, (_m, head: string, name: string, v: string) => sibling(head, name, v, '"'))
1130    out = out.replace(JSON_VALUE_NAME, (_m, head: string, v: string, tail: string, name: string) => sibling(head, name, v, tail))
1131    out = out.replace(TF_OUTPUT, (_m, head: string, name: string, v: string) => (PW_NAME.test(name) ? value(head, name, v, '"') : _m))
1132  }
1133  if (/"random_(?:password|string)"/.test(out)) {
1134    out = out.replace(TF_RESULT, (m, head: string, v: string) => (WHOLE_TAG.test(v) || cfg.allow.has(v) ? m : `${head}${mint('SECRET', v)}"`))
1135  }
1136  if (/-[ \t]+name:/.test(out)) {
1137    out = out.replace(YAML_NAME_VALUE, (_m, indent: string, head: string, name: string, q: string, v: string) => sibling(indent + head, name, v, q))
1138  }
1139  if (/INSERT[ \t]+INTO/i.test(out)) out = scrubSqlInserts(out, cfg)
1140  return out
1141}
1142
1143// The VALUES rows after an INSERT whose columns name a key or a password:
1144// the value in that column is hidden. Bounded to 100 rows and 20,000
1145// characters per INSERT. Quoted values only; a number is never a key.
1146function scrubSqlInserts(text: string, cfg: Config): string {
1147  const ranges: [number, number][] = []
1148  for (const m of text.matchAll(SQL_INSERT)) {
1149    const names = m[1]!.split(',').map((c) => c.trim().replace(/^[`"[]|[`"\]]$/g, ''))
1150    const kinds = names.map((n) => (PW_NAME.test(n) ? 'pw' : KEY_NAME.test(n) && !/(?:^|_)id$|public/i.test(n) ? 'key' : ''))
1151    if (!kinds.some(Boolean)) continue
1152    const limit = Math.min(text.length, m.index! + m[0].length + 20_000)
1153    let i = m.index! + m[0].length
1154    const skipSpace = (): void => {
1155      while (i < limit && /\s/.test(text[i]!)) i++
1156    }
1157    for (let row = 0; row < 100 && text[i] === '('; row++) {
1158      i++
1159      for (let col = 0; i < limit && text[i] !== ')'; col++) {
1160        skipSpace()
1161        if (text[i] === "'") {
1162          const from = ++i
1163          while (i < limit && !(text[i] === "'" && text[i + 1] !== "'")) i += text[i] === "'" ? 2 : 1
1164          const v = text.slice(from, i)
1165          i++
1166          const kind = kinds[col]
1167          if (kind && v.length >= 3 && !cfg.allow.has(v) && (kind === 'pw' ? !isStandIn(v) : looksSecret(v, 'key=', cfg))) ranges.push([from, from + v.length])
1168        } else {
1169          while (i < limit && text[i] !== ',' && text[i] !== ')') i++
1170        }
1171        skipSpace()
1172        if (text[i] === ',') i++
1173        else break
1174      }
1175      if (text[i] !== ')') break
1176      i++
1177      skipSpace()
1178      if (text[i] !== ',') break
1179      i++
1180      skipSpace()
1181    }
1182  }
1183  if (ranges.length === 0) return text
1184  const parts: string[] = []
1185  let last = 0
1186  for (const [from, to] of ranges) {
1187    if (from < last) continue
1188    parts.push(text.slice(last, from), mint('SECRET', text.slice(from, to)))
1189    last = to
1190  }
1191  parts.push(text.slice(last))
1192  return parts.join('')
1193}
1194
1195function cookieSecret(raw: string, cfg: Config): boolean {
1196  if (cfg.allow.has(raw) || raw.includes('[REDACTED-')) return false
1197  const m = raw.match(COOKIE_VALUE)
1198  if (!m) return false
1199  const body = m[1]!
1200  if (looksLikeName(body, true)) return false