pace-line

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register, Timer } from 'claude-code'
3
4import type { Gpu, JobPanel, Limit, Snapshot } from '../types'
5import { COLORS, PLAIN, URGENT_SECONDS, WARN_SECONDS, bar, cellWidth, fmtDur, jobLeft, line1, line2, slurmFrom, usageColor } from './format'
6import type { Seg } from './format'
7import { APP_QUERY, GPU_QUERY, parseNvidiaSmi, parseScontrol } from './slurm'
8import type { Alert, BatchJob, Ckpt, NodeGpus } from '../types'
9import { TERMINAL, alertsFor, label } from './jobs/alerts'
10import { EMPTY_METRICS, addChunk, newestCkpt } from './jobs/metrics'
11import { SACCT_FIELDS, SQUEUE_FORMAT, parseDf, parseJobDetail, parseLogTail, parseSacct, parseSqueue, resumeArgv, slurmTime, submittedId, zoneMinutes } from './jobs/parse'
12import { CONVENTIONS, EMPTY_JOBS, TEMPLATES, configFrom } from './jobs/state'
13import type { WatchConfig } from './jobs/state'
14import { jobsLine, jobsTab } from './jobs/view'
15
16const EMPTY: Snapshot = {
17  model: null,
18  effort: null,
19  folder: null,
20  branch: null,
21  contextPercent: null,
22  costUsd: null,
23  startedAt: null,
24  limits: [],
25  slurm: null,
26}
27
28const NO_PANEL: JobPanel = { job: null, gpus: null, gpuVia: null, isLoading: false, error: null, updatedAt: null }
29
30const snapshot = atom({ plugin: 'pace-line', key: 'snapshot' } as const, EMPTY)
31const now = atom({ plugin: 'pace-line', key: 'now' } as const, 0)
32const panel = atom({ plugin: 'pace-line', key: 'panel' } as const, NO_PANEL)
33// The job the 10-minute warning was shown for, so a reload doesn't repeat it.
34const warnedJob = atom({ plugin: 'pace-line', key: 'warnedJob' } as const, null)
35// Which tab of the panel shows: the session's own allocation, or the batch jobs.
36const tab = atom({ plugin: 'pace-line', key: 'tab' } as const, 'node')
37const jobsAtom = atom({ plugin: 'pace-line', key: 'jobs' } as const, EMPTY_JOBS)
38
39// Countdowns and the session timer move on their own; usage figures are pushed.
40const TICK_MS = 15_000
41// The panel's GPU cards refresh this often while it is open, when nvidia-smi
42// runs here. Over ssh, or for scontrol, only on open and Refresh.
43const GPU_TICK_MS = 5_000
44const PANE = 'slurm-job'
45
46let gpuTimer: Timer | null = null
47
48function basename(path: string): string {
49  return path.replace(/\/+$/, '').split('/').pop() || path
50}
51
52async function gitBranch($: EngineInterface, cwd: string): Promise<string | null> {
53  try {
54    const r = await $.process.run(
55      ['git', '-C', cwd, '--no-optional-locks', 'symbolic-ref', '--short', 'HEAD'],
56      { timeoutMs: 3000 },
57    )
58    return r.exitCode === 0 ? r.stdout.trim() || null : null
59  } catch {
60    return null
61  }
62}
63
64const LEVELS = ['low', 'medium', 'high', 'xhigh', 'max']
65
66// /effort saves the level per model (modelSettings["claude-opus-5-5"]), over
67// the top-level effortLevel. Read straight after a change, before any turn.
68async function effortFromSettings($: EngineInterface, model: string): Promise<string | null> {
69  try {
70    const s = (await $.settings.read()) as {
71      effortLevel?: unknown
72      modelSettings?: Record<string, { effortLevel?: unknown } | undefined>
73    }
74    const level = s.modelSettings?.[model]?.effortLevel ?? s.effortLevel
75    return typeof level === 'string' ? level : null
76  } catch {
77    return null
78  }
79}
80
81function limitsOf(rateLimits: readonly Limit[]): Limit[] {
82  return rateLimits.map(l => ({ kind: l.kind, percentUsed: l.percentUsed, resetsAt: l.resetsAt }))
83}
84
85async function refreshPlace($: EngineInterface): Promise<void> {
86  const cwd = await $.session.cwd()
87  const branch = await gitBranch($, cwd)
88  await update($, snapshot, s => ({ ...s, folder: basename(cwd), branch }))
89}
90
91// One toast per job, when 10 minutes are left.
92async function warnIfEnding($: EngineInterface, t: number): Promise<void> {
93  const { slurm } = await read($, snapshot)
94  if (!slurm) return
95  const left = jobLeft(slurm, t)
96  if (left === null || left > WARN_SECONDS || left === 0) return
97  if ((await read($, warnedJob)) === slurm.job) return
98  await update($, warnedJob, () => slurm.job)
99  $.ui.toast(`⌛ job ${slurm.job} ends in ${fmtDur(left)}: save your work`, { timeoutMs: 20_000 })
100}
101
102async function fetchJobInfo($: EngineInterface, jobId: string) {
103  try {
104    const r = await $.process.run(['scontrol', 'show', 'job', jobId], { timeoutMs: 10_000 })
105    return r.exitCode === 0 ? parseScontrol(r.stdout) : null
106  } catch {
107    return null
108  }
109}
110
111/**
112 * The job's GPUs: nvidia-smi here when Claude Code runs on the node (the job's
113 * cgroup shows only its GPUs), else over ssh to the node, which Slurm adopts
114 * into the job. Null when neither answers.
115 */
116async function fetchGpus($: EngineInterface, node: string | null): Promise<{ gpus: Gpu[]; via: 'local' | 'ssh' } | null> {
117  const query = [`--query-gpu=${GPU_QUERY}`, '--format=csv,noheader,nounits']
118  const apps = [`--query-compute-apps=${APP_QUERY}`, '--format=csv,noheader,nounits']
119  const ways: { via: 'local' | 'ssh'; prefix: string[] }[] = [{ via: 'local', prefix: [] }]
120  if (node) ways.push({ via: 'ssh', prefix: ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=5', node] })
121  for (const { via, prefix } of ways) {
122    try {
123      const g = await $.process.run([...prefix, 'nvidia-smi', ...query], { timeoutMs: 10_000 })
124      if (g.exitCode !== 0 || !g.stdout.trim()) continue
125      const a = await $.process.run([...prefix, 'nvidia-smi', ...apps], { timeoutMs: 10_000 })
126      return { gpus: parseNvidiaSmi(g.stdout, a.exitCode === 0 ? a.stdout : ''), via }
127    } catch {
128      // not installed here, or ssh refused: try the next way
129    }
130  }
131  return null
132}
133
134/** Refetches the GPUs, and with `withJob` the scontrol details too. */
135async function refreshPanel($: EngineInterface, withJob: boolean): Promise<void> {
136  const { slurm } = await read($, snapshot)
137  if (!slurm) return
138  await update($, panel, p => ({ ...p, isLoading: true }))
139  const job = withJob ? await fetchJobInfo($, slurm.job) : (await read($, panel)).job
140  const gpus = await fetchGpus($, job?.node || null)
141  const t = await $.clock.now()
142  await update($, panel, () => ({
143    job,
144    gpus: gpus?.gpus ?? null,
145    gpuVia: gpus?.via ?? null,
146    isLoading: false,
147    error: job ? null : `scontrol show job ${slurm.job} gave nothing`,
148    updatedAt: t,
149  }))
150  // The live end time, in case the limit changed since the job started.
151  if (job && job.runSeconds !== null && job.limitSeconds !== null) {
152    const endsAt = t + (job.limitSeconds - job.runSeconds) * 1000
153    await update($, snapshot, s => (s.slurm ? { ...s, slurm: { ...s.slurm, endsAt } } : s))
154  }
155  if (gpus?.via === 'local' && gpuTimer === null) {
156    gpuTimer = $.clock.every(GPU_TICK_MS, () => refreshPanel($, false))
157  }
158}
159
160async function openPanel($: EngineInterface, which: 'node' | 'jobs' = 'node'): Promise<void> {
161  const { slurm } = await read($, snapshot)
162  if (!slurm && which === 'node') {
163    $.ui.toast('Not inside a Slurm job')
164    return
165  }
166  await update($, tab, () => which)
167  const opened = await $.ui.open({ id: PANE, title: 'Slurm' })
168  if (!opened.isPlaced) $.ui.toast(`Type /${which === 'node' ? 'job' : 'jobs'} to open the panel here`)
169  paneOpen = true
170  if (which === 'node') await refreshPanel($, true)
171  else {
172    const s = await read($, jobsAtom)
173    if (s.selected) void refreshGpus($, [s.selected])
174  }
175}
176
177
178// ── Batch jobs ──────────────────────────────────────────────────────────
179// One squeue a minute for all of the user's jobs, then each running job's log
180// tail, metrics.jsonl, checkpoints and GPUs. Alerts go to a toast, Discord and
181// Claude; a TIMEOUT with a fresh checkpoint is resubmitted. All of it lives
182// here: a hooks module hands $ only to functions declared in its own file.
183
184/** Ended jobs stay listed this long. */
185const KEEP_ENDED_MS = 12 * 3600_000
186/** GPUs of jobs not on screen: often enough for the idle alarm. */
187const BACKGROUND_GPU_MS = 120_000
188const DISK_MS = 10 * 60_000
189const LOG_TAIL_BYTES = 65_536
190const RESUMABLE = new Set(['TIMEOUT', 'NODE_FAIL', 'PREEMPTED'])
191/** The Discord plugin's server, as its tools are named (mcp__plugin_discord_discord__reply). */
192const DISCORD_SERVER = 'plugin:discord:discord'
193
194type Ctx = { user: string; ownJob: string | null; zone: number; cfg: WatchConfig }
195let ctx: Ctx | null = null
196let polling = false
197let paneOpen = false
198let disksAt = 0
199/** Interactive jobs (srun --pty, salloc) squeue lists but there is nothing to watch in. */
200const ignored = new Set<string>()
201const hostsOf = new Map<string, string[]>()
202
203async function run($: EngineInterface, argv: string[], opts: { timeoutMs?: number; cwd?: string } = {}): Promise<string | null> {
204  try {
205    const r = await $.process.run(argv, { timeoutMs: opts.timeoutMs ?? 15_000, ...(opts.cwd ? { cwd: opts.cwd } : {}) })
206    return r.exitCode === 0 ? r.stdout : null
207  } catch {
208    return null
209  }
210}
211
212async function statOf($: EngineInterface, path: string) {
213  try {
214    return await $.fs.stat(path)
215  } catch {
216    return null
217  }
218}
219
220const runDir = (j: BatchJob) => (j.workDir ? `${j.workDir.replace(/\/$/, '')}/runs/${j.jobId}` : null)
221
222function newJob(id: string): BatchJob {
223  return {
224    id,
225    jobId: id,
226    arrayId: null,
227    name: '',
228    state: 'PENDING',
229    reason: null,
230    elapsedS: null,
231    limitS: null,
232    nodes: 1,
233    nodeList: '',
234    gpus: null,
235    gpuType: null,
236    submittedAt: null,
237    startedAt: null,
238    workDir: null,
239    stdout: null,
240    command: null,
241    submitLine: null,
242    endedAt: null,
243    exitCode: null,
244    logSize: null,
245    logChangedAt: null,
246    lastLines: [],
247    progress: null,
248    flags: { oom: null, traceback: null, nccl: null, srun: null },
249    metrics: null,
250    ckpt: null,
251    gpuReadings: null,
252    gpusAt: null,
253    idleSince: null,
254    resumedFrom: null,
255    resumedAs: null,
256  }
257}
258
259/** Starts the watcher; false off a Slurm cluster. */
260async function startWatch($: EngineInterface, cfg: WatchConfig, ownJob: string | null): Promise<boolean> {
261  const user = await $.env.get('USER')
262  if (!user || (await run($, ['squeue', '--version'], { timeoutMs: 5_000 })) === null) return false
263  const z = await run($, ['date', '+%z'], { timeoutMs: 5_000 })
264  ctx = { user, ownJob, zone: zoneMinutes(z ?? '+0000'), cfg }
265  await poll($)
266  $.clock.every(cfg.pollSeconds * 1000, () => poll($))
267  $.clock.every(cfg.gpuSeconds * 1000, async () => {
268    if (!paneOpen) return
269    const s = await read($, jobsAtom)
270    const sel = s.jobs.find(j => j.id === s.selected && j.state === 'RUNNING')
271    if (sel) await refreshGpus($, [sel.id])
272  })
273  return true
274}
275
276/** The log's tail, metrics.jsonl and the newest checkpoint, each read only when it changed. */
277async function readFiles($: EngineInterface, j: BatchJob): Promise<BatchJob> {
278  let out = j
279  if (j.stdout) {
280    const st = await statOf($, j.stdout)
281    if (st && st.size !== j.logSize) {
282      const tail = await run($, ['tail', '-c', String(LOG_TAIL_BYTES), j.stdout])
283      if (tail !== null) {
284        const t = parseLogTail(tail)
285        out = {
286          ...out,
287          lastLines: t.lines,
288          progress: t.progress ?? out.progress,
289          // Sticky: an error scrolled out of the tail still happened.
290          flags: {
291            oom: t.flags.oom ?? out.flags.oom,
292            traceback: t.flags.traceback ?? out.flags.traceback,
293            nccl: t.flags.nccl ?? out.flags.nccl,
294            srun: t.flags.srun ?? out.flags.srun,
295          },
296        }
297      }
298      out = { ...out, logSize: st.size, logChangedAt: st.mtimeMs }
299    }
300  }
301  const dir = runDir(j)
302  if (!dir) return out
303  const mpath = `${dir}/metrics.jsonl`
304  const ms = await statOf($, mpath)
305  if (ms) {
306    let m = out.metrics ?? EMPTY_METRICS
307    if (ms.size < m.offset) m = EMPTY_METRICS // rewritten from scratch
308    if (ms.size > m.offset) {
309      const chunk = await run($, ['tail', '-c', `+${m.offset + 1}`, mpath], { timeoutMs: 20_000 })
310      if (chunk !== null) m = addChunk(m, chunk)
311    }
312    out = { ...out, metrics: m }
313  }
314  return { ...out, ckpt: await findCkpt($, dir, out.ckpt) }
315}
316
317/** The newest checkpoint in runs/<id>/checkpoints, else in runs/<id> itself. */
318async function findCkpt($: EngineInterface, dir: string, prev: Ckpt | null): Promise<Ckpt | null> {
319  for (const where of [`${dir}/checkpoints`, dir]) {
320    let entries: { name: string; kind: string; size: number; mtimeMs: number }[]
321    try {
322      entries = await $.fs.list(where)
323    } catch {
324      continue
325    }
326    // A directory's mtime isn't listed: HF's checkpoint-1200/ needs a stat.
327    const dated = await Promise.all(entries.slice(-40).map(async e => {
328      if (e.kind !== 'dir') return e
329      const st = await statOf($, `${where}/${e.name}`)
330      return { ...e, mtimeMs: st?.mtimeMs ?? 0 }
331    }))
332    const top = newestCkpt(where, dated)
333    if (!top) continue
334    if (prev && prev.path === top.path && prev.mtimeMs === top.mtimeMs) return prev
335    if (top.sizeBytes === null) {
336      const du = await run($, ['du', '-sb', top.path], { timeoutMs: 20_000 })
337      const n = Number(du?.split(/\s/)[0])
338      return { ...top, sizeBytes: Number.isFinite(n) && n > 0 ? n : null }
339    }
340    return top
341  }
342  return null
343}
344
345async function hostsFor($: EngineInterface, j: BatchJob): Promise<string[]> {
346  const key = `${j.jobId}:${j.nodeList}`
347  const known = hostsOf.get(key)
348  if (known) return known
349  const out = await run($, ['scontrol', 'show', 'hostnames', j.nodeList], { timeoutMs: 5_000 })
350  const list = out?.split('\n').map(s => s.trim()).filter(Boolean) ?? []
351  if (list.length) hostsOf.set(key, list)
352  return list
353}
354
355/** nvidia-smi over ssh on each of the job's nodes: ssh lands in the job's cgroup there. */
356async function gpusOf($: EngineInterface, j: BatchJob): Promise<NodeGpus[] | null> {
357  const nodes = await hostsFor($, j)
358  if (!nodes.length) return null
359  const readings = await Promise.all(nodes.map(async node => {
360    const ssh = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=5', node, 'nvidia-smi']
361    const g = await run($, [...ssh, `--query-gpu=${GPU_QUERY}`, '--format=csv,noheader,nounits'], { timeoutMs: 12_000 })
362    if (g === null) return null
363    const a = await run($, [...ssh, `--query-compute-apps=${APP_QUERY}`, '--format=csv,noheader,nounits'], { timeoutMs: 12_000 })
364    return { node, gpus: parseNvidiaSmi(g, a ?? '') }
365  }))
366  const ok = readings.filter((r): r is NodeGpus => r !== null)
367  return ok.length ? ok : null
368}
369
370const anyIdle = (r: NodeGpus[] | null) => (r ?? []).some(n => n.gpus.some(g => (g.util ?? 0) === 0 && g.procs.length === 0))
371
372async function refreshGpus($: EngineInterface, ids: readonly string[]): Promise<void> {
373  const s = await read($, jobsAtom)
374  const t = await $.clock.now()
375  const fresh = new Map<string, Pick<BatchJob, 'gpuReadings' | 'gpusAt' | 'idleSince'>>()
376  for (const j of s.jobs.filter(j => ids.includes(j.id) && j.state === 'RUNNING' && (j.gpus ?? 0) > 0)) {
377    const r = await gpusOf($, j)
378    fresh.set(j.id, { gpuReadings: r ?? j.gpuReadings, gpusAt: t, idleSince: r === null ? j.idleSince : anyIdle(r) ? j.idleSince ?? t : null })
379  }
380  if (fresh.size) await update($, jobsAtom, x => ({ ...x, jobs: x.jobs.map(j => (fresh.has(j.id) ? { ...j, ...fresh.get(j.id)! } : j)) }))
381}
382
383/** One pass: squeue, details of new jobs, files of running ones, the end of vanished ones. */
384async function poll($: EngineInterface): Promise<void> {
385  if (!ctx || polling) return
386  polling = true
387  try {
388    const { user, ownJob, zone, cfg } = ctx
389    const t = await $.clock.now()
390    const out = await run($, ['squeue', '-u', user, '-h', '-r', '-o', SQUEUE_FORMAT])
391    if (out === null) {
392      await update($, jobsAtom, s => ({ ...s, error: 'squeue did not answer', updatedAt: t }))
393      return
394    }
395    const rows = parseSqueue(out).filter(r => !cfg.excludeNames.includes(r.name) && r.id !== ownJob && !ignored.has(r.id))
396    const prev = (await read($, jobsAtom)).jobs
397    const byId = new Map(prev.map(j => [j.id, j]))
398    const next: BatchJob[] = []
399
400    for (const r of rows) {
401      const known = byId.get(r.id)
402      let j: BatchJob = {
403        ...(known ?? newJob(r.id)),
404        arrayId: r.arrayId,
405        name: r.name,
406        state: r.state,
407        reason: r.reason,
408        elapsedS: r.elapsedS,
409        limitS: r.limitS,
410        nodes: r.nodes,
411        nodeList: r.nodeList || known?.nodeList || '',
412        gpus: r.gpus ?? known?.gpus ?? null,
413        gpuType: r.gpuType ?? known?.gpuType ?? null,
414        submittedAt: known?.submittedAt ?? slurmTime(r.submitTime, zone),
415      }
416      if (j.state === 'RUNNING' && j.startedAt === null) j.startedAt = t - (r.elapsedS ?? 0) * 1000
417      if (!known || (known.state !== 'RUNNING' && r.state === 'RUNNING')) {
418        const d = parseJobDetail((await run($, ['scontrol', 'show', 'job', r.id])) ?? '')
419        if (d && !d.isBatch) {
420          ignored.add(r.id)
421          continue
422        }
423        if (d) j = { ...j, jobId: d.jobId, workDir: d.workDir, stdout: d.stdout, command: d.command, nodeList: d.nodeList || j.nodeList }
424      }
425      if (j.state === 'RUNNING') j = await readFiles($, j)
426      next.push(j)
427    }
428
429    const ended: BatchJob[] = []
430    for (const j of prev) {
431      if (rows.some(r => r.id === j.id)) continue
432      if (j.endedAt !== null) {
433        if (t - j.endedAt < KEEP_ENDED_MS) next.push(j)
434        continue
435      }
436      const acct = parseSacct((await run($, ['sacct', '-j', j.jobId, '-X', '-n', '-P', '-o', SACCT_FIELDS])) ?? '')
437      if (!acct || !TERMINAL.has(acct.state)) {
438        next.push(j) // accounting lags squeue: ask again next pass
439        continue
440      }
441      const done = await readFiles($, {
442        ...j,
443        state: acct.state,
444        exitCode: acct.exitCode,
445        elapsedS: acct.elapsedS ?? j.elapsedS,
446        submitLine: acct.submitLine,
447        workDir: j.workDir ?? acct.workDir,
448        endedAt: t,
449      })
450      next.push(done)
451      ended.push(done)
452    }
453
454    // Every 2 minutes, the GPUs of running jobs not on screen, for the idle alarm.
455    const due = next.filter(j => j.state === 'RUNNING' && (j.gpus ?? 0) > 0 && t - (j.gpusAt ?? 0) >= BACKGROUND_GPU_MS).map(j => j.id)
456
457    let disks = (await read($, jobsAtom)).disks
458    if (t - disksAt >= DISK_MS) {
459      disksAt = t
460      const paths = [...new Set([`/scratch/${user}`, ...next.map(j => j.workDir).filter((w): w is string => !!w)])]
461      const df = await run($, ['df', '-B1', '--output=target,size,used,avail', ...paths])
462      if (df !== null) disks = parseDf(df).filter((d, i, all) => all.findIndex(x => x.mount === d.mount) === i)
463    }
464
465    await update($, jobsAtom, s => ({
466      ...s,
467      jobs: next,
468      selected: s.selected ?? next[0]?.id ?? null,
469      updatedAt: t,
470      error: null,
471      disks,
472    }))
473    if (due.length) await refreshGpus($, due)
474
475    const fresh: Alert[] = []
476    for (const j of (await read($, jobsAtom)).jobs) fresh.push(...alertsFor(j, t, cfg))
477    for (const j of ended) {
478      const r = await maybeResume($, j, false)
479      if (r) fresh.push(r)
480    }
481    await deliver($, await unseen($, fresh))
482  } finally {
483    polling = false
484  }
485}
486
487/** Alerts not yet raised for that job and kind, by any session (the store is shared). */
488async function unseen($: EngineInterface, alerts: Alert[]): Promise<Alert[]> {
489  if (!alerts.length) return []
490  const t = await $.clock.now()
491  const seen = ((await $.store.get('alerted')) ?? {}) as Record<string, number>
492  const out = alerts.filter(a => seen[`${a.job}:${a.kind}`] === undefined)
493  if (!out.length) return []
494  const kept = Object.fromEntries(Object.entries(seen).filter(([, at]) => t - at < 7 * 86400_000))
495  for (const a of out) kept[`${a.job}:${a.kind}`] = t
496  await $.store.set('alerted', kept)
497  return out
498}
499
500/** Toasts each alert, then one Discord message and one message to Claude for the batch. */
501async function deliver($: EngineInterface, alerts: Alert[]): Promise<void> {
502  if (!alerts.length || !ctx) return
503  const { cfg } = ctx
504  for (const a of alerts) $.ui.toast(a.text, { timeoutMs: a.level === 'error' ? 30_000 : 15_000 })
505  await update($, jobsAtom, s => ({ ...s, alerts: [...alerts, ...s.alerts].slice(0, 20) }))
506
507  const text = alerts.map(a => a.text).join('\n')
508  if (cfg.discordChatId) {
509    try {
510      await $.mcp.call(DISCORD_SERVER, 'reply', { chat_id: cfg.discordChatId, text })
511    } catch {
512      // The Discord server isn't connected in this session: try the webhook.
513      if (cfg.discordWebhook) await webhook($, cfg.discordWebhook, text)
514    }
515  } else if (cfg.discordWebhook) {
516    await webhook($, cfg.discordWebhook, text)
517  }
518
519  const jobs = (await read($, jobsAtom)).jobs
520  const logs = [...new Set(alerts.map(a => jobs.find(j => j.id === a.job)?.stdout).filter(Boolean))]
521  try {
522    await $.prompt.submit({
523      text: [
524        '[pace-line: batch-job alerts, from the job watcher, not typed by the user]',
525        text,
526        logs.length ? `Logs: ${logs.join(', ')}` : '',
527        'For a failure, read the end of its log and propose a fix. Do not resubmit unless the user asks: pace-line already resubmits TIMEOUTs from their latest checkpoint. For an idle GPU, a silent log or a long pending job, check it and say what you would do. For a resubmission, update the project EXPERIMENTS.md row if there is one. Keep it short.',
528      ].filter(Boolean).join('\n'),
529    })
530  } catch {
531    // No session to tell (claude -p): the toasts stand.
532  }
533}
534
535async function webhook($: EngineInterface, url: string, text: string): Promise<void> {
536  try {
537    await $.http.fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ content: text.slice(0, 1900) }) })
538  } catch {
539    // Best effort: the toast already showed it.
540  }
541}
542
543type Chain = { root: string; attempts: number; lastCkpt: string | null; jobs: string[] }
544
545/**
546 * Resubmits a job that hit its time limit (or lost its node) from the newest
547 * checkpoint it wrote, with RESUME=<path> added to its --export. `force` is
548 * /jobs resume: any ended state, past the cap.
549 */
550async function maybeResume($: EngineInterface, j: BatchJob, force: boolean): Promise<Alert | null> {
551  if (!ctx) return null
552  const t = await $.clock.now()
553  const stop = (text: string): Alert => ({ job: j.id, kind: 'resumeStopped', level: 'error', text: `↻ ${label(j)}: not resubmitted, ${text}`, at: t })
554  if (!force && !RESUMABLE.has(j.state)) return null
555  if (j.arrayId) return stop('it is an array task; resubmit the sweep by hand')
556  const ckpt = j.ckpt
557  if (!ckpt || (j.startedAt !== null && ckpt.mtimeMs < j.startedAt)) return stop(`no checkpoint written by this run in ${runDir(j) ?? 'its run dir'}`)
558  if (!j.submitLine) return stop('sacct gave no submit line')
559
560  const chains = ((await $.store.get('chains')) ?? {}) as Record<string, Chain>
561  const chainOf = ((await $.store.get('chainOf')) ?? {}) as Record<string, string>
562  const root = chainOf[j.jobId] ?? j.jobId
563  const chain = chains[root] ?? { root, attempts: 0, lastCkpt: null, jobs: [root] }
564  if (!force && chain.attempts >= ctx.cfg.maxResumes) return stop(`${chain.attempts} resubmits already (limit ${ctx.cfg.maxResumes})`)
565  if (!force && chain.lastCkpt === ckpt.path) return stop('the last attempt wrote no new checkpoint')
566  const claim = `claim:${j.jobId}`
567  if (!force && (await $.store.get(claim))) return null // another session resubmitted it
568  await $.store.set(claim, t)
569
570  const argv = resumeArgv(j.submitLine, ckpt.path)
571  if (!argv) return stop(`its submit line can't be replayed: ${j.submitLine.slice(0, 120)}`)
572  const out = await run($, argv, { cwd: j.workDir ?? undefined, timeoutMs: 30_000 })
573  const id = out === null ? null : submittedId(out)
574  if (!id) return stop(`sbatch refused: ${argv.join(' ').slice(0, 160)}`)
575
576  chains[root] = { root, attempts: chain.attempts + 1, lastCkpt: ckpt.path, jobs: [...chain.jobs, id] }
577  chainOf[id] = root
578  await $.store.set('chains', chains)
579  await $.store.set('chainOf', chainOf)
580  await update($, jobsAtom, s => ({ ...s, jobs: s.jobs.map(x => (x.id === j.id ? { ...x, resumedAs: id } : x)) }))
581  return {
582    job: j.id,
583    kind: 'resumed',
584    level: 'warn',
585    text: `↻ ${label(j)} ${j.state}: resubmitted as ${id} from ${ckpt.name} (resume ${chain.attempts + 1}/${ctx.cfg.maxResumes})`,
586    at: t,
587  }
588}
589
590/** /jobs resume <id>: the same resubmission, asked for by the user. */
591async function resumeByHand($: EngineInterface, id: string): Promise<string> {
592  const j = (await read($, jobsAtom)).jobs.find(x => x.id === id || x.jobId === id)
593  if (!j) return `No job ${id} in the list.`
594  if (j.endedAt === null) return `Job ${id} is still ${j.state}.`
595  const a = await maybeResume($, j, true)
596  if (a) await deliver($, [a])
597  return a?.text ?? `Job ${id} was not resubmitted.`
598}
599
600/** Cancels one job by id; never the session's own allocation. */
601async function cancelJob($: EngineInterface, j: BatchJob): Promise<string> {
602  if (ctx && (j.id === ctx.ownJob || ctx.cfg.excludeNames.includes(j.name))) return `Refusing to cancel ${j.id}: it is this session's allocation.`
603  const out = await run($, ['scancel', j.jobId])
604  return out === null ? `scancel ${j.jobId} failed.` : `Cancelled ${label(j)}.`
605}
606
607/** /jobs init: copies the helpers into the session's directory, never over a file. */
608async function initProject($: EngineInterface): Promise<string> {
609  const cwd = await $.session.cwd()
610  const done: string[] = []
611  for (const t of TEMPLATES) {
612    const dest = `${cwd.replace(/\/$/, '')}/${t.to}`
613    if (await $.fs.exists(dest)) {
614      done.push(`${t.to} (exists, kept)`)
615      continue
616    }
617    await $.fs.write(dest, await $.fs.read(`${$.plugin.root}/${t.from}`))
618    done.push(t.to)
619  }
620  return `pace-line conventions in ${cwd}: ${done.join(', ')}. Logs → runs/slurm-%j.out, metrics → runs/<jobid>/metrics.jsonl, checkpoints → runs/<jobid>/checkpoints/, resume from $RESUME.`
621}
622
623const STATE_COLOR: Record<string, string> = {
624  RUNNING: COLORS.GREEN,
625  PENDING: COLORS.YELLOW,
626  COMPLETING: COLORS.YELLOW,
627}
628
629function tempColor(c: number): string {
630  if (c >= 85) return COLORS.RED
631  if (c >= 70) return COLORS.ORANGE
632  if (c >= 55) return COLORS.YELLOW
633  return COLORS.GREEN
634}
635
636const gib = (mib: number) => (mib / 1024).toFixed(1)
637const pct = (used: number | null, total: number | null) => (used !== null && total ? (used / total) * 100 : null)
638
639export const register: Register = (on, options) => {
640  const cfg = configFrom(options)
641  let isWatching = false
642
643  on('session.start', async ($, e, next) => {
644    const started = await next(e)
645    const [model, usage, t] = await Promise.all([$.session.model(), $.session.usage(), $.clock.now()])
646    // The session's effective level (a --effort flag included) as it hands
647    // it to Bash; then the saved one. A turn's request corrects either.
648    const prior = (await read($, snapshot)).effort
649    const env = await $.env.get('CLAUDE_EFFORT')
650    // Set once when the job starts; Claude Code inherits it from the shell.
651    const slurm = slurmFrom({
652      SLURM_JOB_ID: await $.env.get('SLURM_JOB_ID'),
653      SLURM_JOB_GPUS: await $.env.get('SLURM_JOB_GPUS'),
654      SLURM_STEP_GPUS: await $.env.get('SLURM_STEP_GPUS'),
655      CUDA_VISIBLE_DEVICES: await $.env.get('CUDA_VISIBLE_DEVICES'),
656      SLURM_GPUS_ON_NODE: await $.env.get('SLURM_GPUS_ON_NODE'),
657      SLURM_JOB_END_TIME: await $.env.get('SLURM_JOB_END_TIME'),
658    })
659    const effort = prior ?? (env && LEVELS.includes(env) ? env : await effortFromSettings($, model))
660    await update($, snapshot, s => ({
661      ...s,
662      model,
663      effort,
664      // Keep an end time the panel refreshed, for the same job.
665      slurm: slurm && s.slurm?.job === slurm.job ? { ...slurm, endsAt: s.slurm.endsAt ?? slurm.endsAt } : slurm,
666      contextPercent: usage.context.percent ?? null,
667      costUsd: usage.cost?.usd ?? null,
668      startedAt: usage.startedAt,
669      limits: limitsOf(usage.rateLimits),
670    }))
671    await update($, now, () => t)
672    await refreshPlace($)
673    if (slurm) {
674      await $.command.register({ name: 'job', description: `Show Slurm job ${slurm.job}: time, node, GPUs` })
675      await warnIfEnding($, t)
676    }
677    // Not awaited: the first squeue shouldn't hold the session's start.
678    void startWatch($, cfg, slurm?.job ?? null).then(async ok => {
679      isWatching = ok
680      if (ok) {
681        await $.command.register({
682          name: 'jobs',
683          description: 'Batch jobs: progress, loss, GPUs, checkpoints. /jobs init adds the run helpers here; /jobs resume <id>',
684          argumentHint: '[init | resume <id>]',
685        })
686      }
687    })
688    $.clock.every(TICK_MS, async () => {
689      const t = await $.clock.now()
690      await update($, now, () => t)
691      await warnIfEnding($, t)
692    })
693    return started
694  })
695
696  on('session.measure', async ($, e, next) => {
697    await update($, snapshot, s => ({
698      ...s,
699      contextPercent: e.context.percent ?? s.contextPercent,
700      costUsd: e.cost?.usd ?? s.costUsd,
701      limits: limitsOf(e.rateLimits),
702    }))
703    const t = await $.clock.now()
704    await update($, now, () => t)
705    return next(e)
706  })
707
708  // The model and effort the main thread actually sends, after /model or
709  // /effort changes and any silent downgrade. Subagents' requests are skipped.
710  on('turn.step', async function* ($, e, next) {
711    if (!e.agentId) {
712      const effort = typeof e.effort === 'string' ? e.effort : null
713      await update($, snapshot, s => ({ ...s, model: e.model, effort: effort ?? s.effort }))
714    }
715    return yield* next(e)
716  })
717
718  on('command.run', { command: 'job' }, async $ => {
719    await openPanel($)
720    return { text: 'Opened the job panel.' }
721  })
722
723  on('command.run', { command: 'jobs' }, async ($, e, next) => {
724    const [sub, arg] = e.args.trim().split(/\s+/)
725    if (sub === 'init') return { text: await initProject($) }
726    // Off a cluster pace-line never registered /jobs: it's another plugin's
727    // (job-watch, watching the cluster from a laptop).
728    if (!isWatching) return next(e)
729    if (sub === 'resume' && arg) return { text: await resumeByHand($, arg) }
730    await openPanel($, 'jobs')
731    return { text: 'Opened the batch jobs panel.' }
732  })
733
734  // The run conventions, so jobs Claude writes in any project report themselves.
735  on('prompt.compose', async ($, e, next) => {
736    const composed = await next(e)
737    if (!isWatching) return composed
738    return { sections: [...composed.sections, { id: 'pace-line:jobs', text: CONVENTIONS, scope: 'session' }] }
739  })
740
741  // A change from /model or /effort (typed, or from the band's buttons) shows
742  // at once rather than on the next turn.
743  on('command.run', async ($, e, next) => {
744    const result = await next(e)
745    if (e.command === 'model' || e.command === 'effort') {
746      const model = await $.session.model()
747      const arg = e.args.trim().toLowerCase()
748      const effort = e.command === 'effort' && LEVELS.includes(arg) ? arg : await effortFromSettings($, model)
749      await update($, snapshot, s => ({ ...s, model, effort: effort ?? s.effort }))
750    }
751    return result
752  }).catch(($, e, next) => next(e))
753
754  // A click (or m / e / j / Enter) on line 1: the model or effort opens the
755  // built-in picker, the job its panel.
756  on('ui.message', async ($, e, next) => {
757    const open = (e.data as { open?: unknown } | null)?.open
758    if (open === 'model' || open === 'effort') {
759      void $.command.run({ command: open }).catch(() => $.ui.toast(`Couldn't open /${open}`))
760    }
761    if (open === 'job') await openPanel($)
762    if (open === 'jobs') await openPanel($, 'jobs')
763    return next(e)
764  })
765
766  on('ui.close', { id: PANE }, ($, e, next) => {
767    gpuTimer?.cancel()
768    gpuTimer = null
769    paneOpen = false
770    return next(e)
771  }).catch(($, e, next) => next(e))
772
773  // A turn may have switched branches or moved the session's directory.
774  on('turn.complete', async ($, e, next) => {
775    const result = await next(e)
776    await refreshPlace($)
777    return result
778  })
779
780  // Drawn as the hint row under the prompt, where a status line sits, with
781  // the engine's own hint (? for shortcuts, esc to interrupt) kept beneath.
782  on('ui.render', { component: 'PromptHint' }, async ($, e, next) => {
783    const s = await read($, snapshot)
784    if (s.model === null) return next(e)
785    const hint = await next(e)
786    const t = (await read($, now)) || (await $.clock.now())
787
788    const elements = $.ui.resolve(e)
789    const { Box, Text } = elements
790    const segs1 = line1(s, t)
791    const jobs = jobsLine(await read($, jobsAtom), t)
792    const row = (key: string, segs: Seg[]) => (
793      <Box key={key} flexDirection="row">
794        <Text wrap="truncate-end">
795          {segs.map((seg, i) => <Text key={String(i)} color={seg.color ?? PLAIN}>{seg.text}</Text>)}
796        </Text>
797      </Box>
798    )
799
800    return (
801      <Box flexDirection="column">
802        {(e.surface === 'terminal' || e.surface === 'desktop') && 'Client' in elements
803          ? (
804            // Terminal and desktop: a region that draws its own colours and takes clicks.
805            <elements.Client
806              key="line1"
807              module="./line1.tsx"
808              props={{ segs: segs1, plain: PLAIN }}
809              width={Math.min(e.viewport?.columns ?? 200, segs1.reduce((n, seg) => n + cellWidth(seg.text), 0))}
810            />
811          )
812          : row('l1', segs1)}
813        {row('l2', line2(s, t))}
814        {jobs.length > 0 && row('l3', jobs)}
815        {hint}
816      </Box>
817    )
818  })
819
820  // The job panel: the job at a glance, then one card per GPU.
821  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
822    const { Box, Button, Text } = $.ui.resolve(e)
823    const which = await read($, tab)
824    const tabs = (
825      <Box key="tabs" flexDirection="row" gap={2} marginBottom={1}>
826        {(['node', 'jobs'] as const).map(k => (
827          <Button
828            key={k}
829            label={`${which === k ? '●' : '○'} ${k === 'node' ? 'This node' : 'Batch jobs'}`}
830            onPress={async () => {
831              await update($, tab, () => k)
832              if (k === 'node') await refreshPanel($, true)
833            }}
834          />
835        ))}
836      </Box>
837    )
838    if (which === 'jobs') {
839      const s = await read($, jobsAtom)
840      const t = (await read($, now)) || (await $.clock.now())
841      return (
842        <Box flexDirection="column">
843          {tabs}
844          {jobsTab({ Box, Text, Button }, s, t, e.props.bodyColumns, {
845            select: key => void update($, jobsAtom, x => ({ ...x, selected: key })).then(() => refreshGpus($, [key])),
846            copy: async text => {
847              const r = await $.ui.copy({ text, surface: e.surface })
848              $.ui.toast(r.isCopied ? `Copied: ${text}` : `Couldn't copy: ${r.reason}`)
849            },
850            resume: async j => $.ui.toast(await resumeByHand($, j.id), { timeoutMs: 15_000 }),
851            cancel: async j => {
852              let answer: string | null = null
853              try {
854                answer = await $.ui.ask(`Cancel ${j.name} (${j.id})?`, { header: 'Cancel', options: ['Cancel the job', 'Keep it'] })
855              } catch {
856                answer = null
857              }
858              if (answer === 'Cancel the job') {
859                $.ui.toast(await cancelJob($, j))
860                await poll($)
861              }
862            },
863            refresh: () => void poll($),
864          })}
865        </Box>
866      )
867    }
868    const p = await read($, panel)
869    const { slurm } = await read($, snapshot)
870    const t = (await read($, now)) || (await $.clock.now())
871    const cols = e.props.bodyColumns
872    const barW = Math.max(10, Math.min(30, cols - 34))
873    const job = p.job
874
875    const meter = (key: string, label: string, value: number | null, detail: string) => (
876      <Box key={key} flexDirection="row">
877        <Text color={PLAIN}>{label.padEnd(6)}</Text>
878        {value === null
879          ? <Text color={COLORS.GRAY}>{'·'.repeat(barW)}  n/a</Text>
880          : <Text color={usageColor(value)}>{bar(value, barW)}</Text>}
881        {value !== null && <Text color={PLAIN}>{`  ${detail}`}</Text>}
882      </Box>
883    )
884
885    const left = slurm ? jobLeft(slurm, t) : null
886    const used = job?.runSeconds ?? null
887    const limit = job?.limitSeconds ?? null
888
889    const card = (g: Gpu) => {
890      const mem = pct(g.memUsedMiB, g.memTotalMiB)
891      const power = pct(g.powerW, g.powerLimitW)
892      // Allocated but doing nothing: worth knowing on a shared cluster.
893      const isIdle = (g.util ?? 0) === 0 && g.procs.length === 0
894      return (
895        <Box key={`gpu${g.index}`} flexDirection="column" borderStyle="round" borderColor={isIdle ? COLORS.GRAY : COLORS.CYAN} paddingX={1}>
896          <Box flexDirection="row" justifyContent="space-between">
897            <Text>
898              <Text color={COLORS.CYAN} bold>{`GPU ${g.index}`}</Text>
899              <Text color={PLAIN}>{`  ${g.name}`}</Text>
900              {isIdle && <Text color={COLORS.GRAY} inverse>{' idle '}</Text>}
901            </Text>
902            {g.tempC !== null && <Text color={tempColor(g.tempC)}>{`${g.tempC}°C`}</Text>}
903          </Box>
904          {meter('util', 'util', g.util, `${g.util ?? 0}%`)}
905          {meter('mem', 'mem', mem, g.memUsedMiB !== null && g.memTotalMiB !== null ? `${gib(g.memUsedMiB)} / ${gib(g.memTotalMiB)} GiB` : '')}
906          {meter('power', 'power', power, g.powerW !== null && g.powerLimitW !== null ? `${Math.round(g.powerW)} / ${Math.round(g.powerLimitW)} W` : '')}
907          <Text wrap="truncate-end">
908            <Text color={PLAIN}>{'procs '}</Text>
909            {g.procs.length === 0
910              ? <Text color={COLORS.GRAY}>none</Text>
911              : g.procs.map((pr, i) => (
912                <Text key={pr.pid}>
913                  {i > 0 && <Text color={COLORS.GRAY}>{' · '}</Text>}
914                  <Text color={COLORS.MAGENTA}>{pr.name}</Text>
915                  <Text color={COLORS.GRAY}>{` ${pr.pid}${pr.memMiB !== null ? ` ${gib(pr.memMiB)} GiB` : ''}`}</Text>
916                </Text>
917              ))}
918          </Text>
919        </Box>
920      )
921    }
922
923    return (
924      <Box flexDirection="column">
925        {tabs}
926        <Box flexDirection="row" justifyContent="space-between">
927          <Text>
928            <Text color={COLORS.MAGENTA} bold>{`job ${job?.id ?? slurm?.job ?? '?'}`}</Text>
929            {job?.name ? <Text color={PLAIN}>{`  ${job.name}`}</Text> : null}
930          </Text>
931          {job && <Text color={STATE_COLOR[job.state] ?? COLORS.RED}>{`● ${job.state}`}</Text>}
932        </Box>
933        {job && (
934          <Text color={PLAIN} wrap="truncate-end">
935            {[job.partition, job.node, job.account].filter(Boolean).join(' · ')}
936          </Text>
937        )}
938        {left !== null && (
939          <Box flexDirection="row">
940            <Text color={left < URGENT_SECONDS ? COLORS.RED : COLORS.CYAN}>{`⌛ ${left > 0 ? `${fmtDur(left)} left` : 'ending'}  `}</Text>
941            {used !== null && limit ? (
942              <Text>
943                <Text color={usageColor((used / limit) * 100)}>{bar((used / limit) * 100, barW)}</Text>
944                <Text color={COLORS.GRAY}>{`  ${fmtDur(used)} of ${fmtDur(limit)}`}</Text>
945              </Text>
946            ) : null}
947          </Box>
948        )}
949        {job && (
950          <Text color={COLORS.GRAY}>
951            {[job.cpus && `${job.cpus} cpu`, job.mem && `${job.mem} mem`, job.gpus && `${job.gpus} gpu`].filter(Boolean).join(' · ')}
952          </Text>
953        )}
954        {p.error && <Text color={COLORS.RED}>{p.error}</Text>}
955
956        <Box flexDirection="column" marginTop={1}>
957          {p.gpus === null
958            ? <Text color={COLORS.GRAY}>{p.isLoading ? 'Reading the GPUs…' : 'No GPU readings: nvidia-smi answered neither here nor over ssh.'}</Text>
959            : p.gpus.length === 0
960              ? <Text color={COLORS.GRAY}>This job has no GPUs.</Text>
961              : p.gpus.map(card)}
962        </Box>
963
964        <Box flexDirection="row" gap={2} marginTop={1}>
965          {job?.node && (
966            <Button
967              key="copy-ssh"
968              label={`Copy ssh ${job.node}`}
969              onPress={async press => {
970                const r = await $.ui.copy({ text: `ssh ${job.node}`, surface: press.surface })
971                $.ui.toast(r.isCopied ? `Copied: ssh ${job.node}` : `Couldn't copy: ${r.reason}`)
972              }}
973            />
974          )}
975          <Button key="refresh" label={p.isLoading ? 'Refreshing…' : 'Refresh'} onPress={() => void refreshPanel($, true)} />
976          <Text color={COLORS.GRAY}>
977            {p.gpuVia === 'local' ? `GPUs live every ${GPU_TICK_MS / 1000}s` : p.gpuVia === 'ssh' ? `GPUs over ssh ${job?.node ?? ''}` : ''}
978          </Text>
979        </Box>
980      </Box>
981    )
982  })
983}
984

1// Pure layout for the band: a port of ~/.claude/statusline-command.sh.
2// Each line is a list of coloured segments; register.tsx turns them into Text.
3
4import type { Limit, Slurm, Snapshot } from '../types'
5
6/** `action` marks a segment drawn as a button that opens that picker. */
7export type Seg = { text: string; color?: string; action?: 'model' | 'effort' | 'job' | 'jobs' }
8
9// The script's ANSI colours as Ghostty's palette draws them (its defaults:
10// no theme or palette is set). A mod can't emit palette codes: names and
11// theme keys come out as the engine's own RGB. If you change Ghostty's
12// theme, update these from `ghostty +show-config --default | grep palette`.
13const CYAN = '#8abeb7' // \033[36m, palette 6
14const GREEN = '#b5bd68' // \033[32m, palette 2
15const YELLOW = '#f0c674' // \033[33m, palette 3
16const ORANGE = '#ff8700' // \033[38;5;208m
17const RED = '#cc6666' // \033[31m, palette 1
18const GRAY = '#666666' // \033[90m, palette 8
19const MAGENTA = '#b294bb' // \033[35m, palette 5
20/** The palette, for the job panel. */
21export const COLORS = { CYAN, GREEN, YELLOW, ORANGE, RED, GRAY, MAGENTA } as const
22
23// Under this much time left the countdown turns red; at WARN_SECONDS a toast warns.
24export const URGENT_SECONDS = 15 * 60
25export const WARN_SECONDS = 10 * 60
26
27// Text the script leaves uncoloured: Claude Code draws a status line's
28// default-coloured text in this gray, so the band does too.
29export const PLAIN = '#999999'
30const SEP: Seg = { text: ' | ' }
31
32// Window length and the minimum elapsed time before the pace arrow shows,
33// in seconds. Without the minimum, a burst right after a reset divides by a
34// tiny elapsed time and reads as a huge overshoot.
35const WINDOWS: Record<string, { label: string; length: number; minElapsed: number }> = {
36  five_hour: { label: '5h', length: 18000, minElapsed: 900 },
37  seven_day: { label: '7d', length: 604800, minElapsed: 21600 },
38}
39
40/**
41 * Terminal cells a text takes: emoji-presentation symbols such as ⌛ take two.
42 * Line 1's clickable region is sized and hit-tested with this.
43 */
44export function cellWidth(text: string): number {
45  let n = 0
46  for (const ch of text) n += /\p{Emoji_Presentation}/u.test(ch) ? 2 : 1
47  return n
48}
49
50/** 3d22h / 2h5m / 48m / 44s */
51export function fmtDur(seconds: number): string {
52  const s = Math.max(0, Math.floor(seconds))
53  if (s >= 86400) return `${Math.floor(s / 86400)}d${Math.floor((s % 86400) / 3600)}h`
54  if (s >= 3600) return `${Math.floor(s / 3600)}h${Math.floor((s % 3600) / 60)}m`
55  if (s >= 60) return `${Math.floor(s / 60)}m`
56  return `${s}s`
57}
58
59/** "Opus 5.5" from "Claude Opus 5.5 (1M context)", "claude-opus-5-5[1m]" or "Opus 5.5". */
60export function modelLabel(model: string): string {
61  const m = /(opus|sonnet|haiku|fable)[\s-]*(\d+(?:[.-]\d+)?)?/i.exec(model)
62  if (!m || !m[1]) return model
63  const family = m[1][0]!.toUpperCase() + m[1].slice(1).toLowerCase()
64  // An id's trailing date (claude-haiku-4-5-20251001) is never matched: the
65  // version takes at most one separator.
66  const version = m[2]?.replace('-', '.')
67  return version ? `${family} ${version}` : family
68}
69
70function effortColor(effort: string): string | undefined {
71  switch (effort) {
72    case 'low': return GREEN
73    case 'medium': return YELLOW
74    case 'high': return ORANGE
75    case 'xhigh':
76    case 'max': return RED
77    default: return undefined
78  }
79}
80
81export function bar(percent: number, width: number): string {
82  const filled = Math.min(width, Math.max(0, Math.round((percent * width) / 100)))
83  return '█'.repeat(filled) + '▒'.repeat(width - filled)
84}
85
86export function usageColor(percent: number): string {
87  const p = Math.round(percent)
88  if (p >= 90) return RED
89  if (p >= 65) return ORANGE
90  if (p >= 40) return YELLOW
91  return GREEN
92}
93
94/**
95 * Relative gap from pro-rata usage, in whole percent: positive means burning
96 * faster than the window allows (⇡), negative means headroom (⇣). Null while
97 * the window has not run long enough, or the reset time is unknown.
98 */
99export function paceDelta(limit: Limit, nowMs: number): number | null {
100  const w = WINDOWS[limit.kind]
101  if (!w || !limit.resetsAt) return null
102  const remaining = (Date.parse(limit.resetsAt) - nowMs) / 1000
103  if (!(remaining > 0 && remaining <= w.length)) return null
104  const elapsed = w.length - remaining
105  if (elapsed < w.minElapsed) return null
106  const delta = Math.round((limit.percentUsed * w.length) / elapsed - 100)
107  return Math.max(-999, Math.min(999, delta))
108}
109
110function limitSegs(limit: Limit, nowMs: number, width: number, showReset: boolean): Seg[] {
111  const w = WINDOWS[limit.kind]
112  const segs: Seg[] = [
113    { text: w?.label ?? limit.kind, color: CYAN },
114    { text: ' ' },
115    { text: bar(limit.percentUsed, width), color: usageColor(limit.percentUsed) },
116    { text: ` ${Math.round(limit.percentUsed)}%` },
117  ]
118  const delta = paceDelta(limit, nowMs)
119  if (delta !== null) {
120    segs.push(delta > 0
121      ? { text: ` ⇡${delta}%`, color: RED }
122      : { text: ` ⇣${-delta}%`, color: GREEN })
123  }
124  if (showReset && w && limit.resetsAt) {
125    const remaining = (Date.parse(limit.resetsAt) - nowMs) / 1000
126    if (remaining > 0 && remaining <= w.length) {
127      segs.push({ text: ` ${fmtDur(remaining)}`, color: GRAY })
128    }
129  }
130  return segs
131}
132
133function join(groups: Seg[][], sep: Seg): Seg[] {
134  const out: Seg[] = []
135  for (const g of groups.filter(g => g.length > 0)) {
136    if (out.length > 0) out.push(sep)
137    out.push(...g)
138  }
139  return out
140}
141
142/**
143 * The allocation from Slurm's environment, as the explorer script reads it:
144 * never squeue or scontrol, so a redraw puts no load on the controller.
145 * The GPU ids come from whichever is set first, which depends on how the GPUs
146 * were asked for (--gres or --gpus) and on job versus step.
147 */
148export function slurmFrom(env: {
149  SLURM_JOB_ID?: string
150  SLURM_JOB_GPUS?: string
151  SLURM_STEP_GPUS?: string
152  CUDA_VISIBLE_DEVICES?: string
153  SLURM_GPUS_ON_NODE?: string
154  SLURM_JOB_END_TIME?: string
155}): Slurm | null {
156  if (!env.SLURM_JOB_ID) return null
157  const gpus = env.SLURM_JOB_GPUS || env.SLURM_STEP_GPUS || env.CUDA_VISIBLE_DEVICES || null
158  // Epoch seconds, set by Slurm 23.02+ when the job starts.
159  const end = Number(env.SLURM_JOB_END_TIME)
160  return {
161    job: env.SLURM_JOB_ID,
162    gpus,
163    gpusOnNode: env.SLURM_GPUS_ON_NODE || null,
164    endsAt: end > 0 ? end * 1000 : null,
165  }
166}
167
168/** Seconds until the job's end; null when unknown. */
169export function jobLeft(slurm: Slurm, nowMs: number): number | null {
170  return slurm.endsAt === null ? null : Math.max(0, Math.floor((slurm.endsAt - nowMs) / 1000))
171}
172
173/** job 123456 · 2×gpu[0,1] · ⌛ 1h12m; the job opens the panel. */
174function slurmSegs(slurm: Slurm, nowMs: number): Seg[] {
175  const segs: Seg[] = [{ text: `job ${slurm.job}`, color: MAGENTA, action: 'job' }]
176  if (slurm.gpus) {
177    const count = slurm.gpus.split(',').length
178    segs.push({ text: ' · ' }, { text: `${count}×gpu[${slurm.gpus}]`, color: GRAY })
179  } else if (slurm.gpusOnNode) {
180    segs.push({ text: ' · ' }, { text: `${slurm.gpusOnNode}×gpu`, color: GRAY })
181  }
182  const left = jobLeft(slurm, nowMs)
183  if (left !== null) {
184    segs.push({ text: ' · ' }, {
185      text: left > 0 ? `⌛ ${fmtDur(left)}` : '⌛ ending',
186      color: left < URGENT_SECONDS ? RED : GRAY,
187    })
188  }
189  return segs
190}
191
192/** [Model · effort] · folder | branch | job 123456 · 2×gpu[0,1] · ⌛ 1h12m */
193export function line1(s: Snapshot, nowMs = Date.now()): Seg[] {
194  const model: Seg[] = []
195  if (s.model) {
196    model.push({ text: '[', color: CYAN }, { text: modelLabel(s.model), color: CYAN, action: 'model' })
197    if (s.effort) {
198      model.push({ text: ' · ', color: CYAN }, { text: s.effort, color: effortColor(s.effort), action: 'effort' })
199    }
200    model.push({ text: ']', color: CYAN })
201  }
202  const head = join([model, s.folder ? [{ text: s.folder }] : []], { text: ' · ' })
203  return join([head, s.branch ? [{ text: s.branch }] : [], s.slurm ? slurmSegs(s.slurm, nowMs) : []], SEP)
204}
205
206/** context bar · pct% | $cost | ⏱ time | 5h bar | 7d bar */
207export function line2(s: Snapshot, nowMs: number): Seg[] {
208  // Only the windows the script draws, in its order: 5h then 7d.
209  const limits = ['five_hour', 'seven_day']
210    .map(kind => s.limits.find(l => l.kind === kind))
211    .filter((l): l is Limit => l !== undefined)
212  // Widths adapt to how many windows there are, so the line never wraps.
213  const both = limits.length >= 2
214  const ctxWidth = both ? 10 : 20
215  const limitWidth = both ? 6 : 10
216
217  const ctx: Seg[] = s.contextPercent === null ? [] : [
218    { text: bar(s.contextPercent, ctxWidth), color: GREEN },
219    { text: ` ${Math.round(s.contextPercent)}%` },
220  ]
221  const cost: Seg[] = s.costUsd ? [{ text: `$${s.costUsd.toFixed(2)}`, color: YELLOW }] : []
222  const time: Seg[] = s.startedAt === null ? [] : [{ text: `⏱ ${fmtDur((nowMs - s.startedAt) / 1000)}` }]
223  const lim = join(
224    limits.map(l => limitSegs(l, nowMs, limitWidth, l.kind === 'five_hour' || !both)),
225    { text: '  ' },
226  )
227  return join([ctx, cost, time, lim], SEP)
228}
229

1// Parsers for the job panel's two sources: `scontrol show job` and the two
2// `nvidia-smi` CSV queries. register.tsx runs the commands.
3
4import type { Gpu, GpuProc, JobInfo } from '../types'
5
6export const GPU_QUERY = 'index,uuid,name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw,power.limit'
7export const APP_QUERY = 'gpu_uuid,pid,process_name,used_memory'
8
9/** "1-02:03:04", "02:03:04", "03:04" → seconds; null for UNLIMITED or junk. */
10export function slurmDuration(text: string): number | null {
11  const m = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/.exec(text.trim())
12  if (!m) return null
13  const [, d, h, min, s] = m
14  return Number(d ?? 0) * 86400 + Number(h ?? 0) * 3600 + Number(min) * 60 + Number(s)
15}
16
17/** `scontrol show job` prints Key=Value pairs separated by spaces and newlines. */
18export function parseScontrol(out: string): JobInfo | null {
19  const kv: Record<string, string> = {}
20  for (const m of out.matchAll(/(\w[\w/:]*)=(\S*)/g)) kv[m[1]!] ??= m[2]!
21  if (!kv.JobId) return null
22  const tres = Object.fromEntries((kv.AllocTRES ?? '').split(',').filter(Boolean).map(p => p.split('=') as [string, string]))
23  return {
24    id: kv.JobId,
25    name: kv.JobName ?? '',
26    state: kv.JobState ?? '',
27    partition: kv.Partition ?? '',
28    account: kv.Account ?? '',
29    node: kv.BatchHost && kv.BatchHost !== '(null)' ? kv.BatchHost : (kv.NodeList ?? ''),
30    nodeList: kv.NodeList ?? '',
31    runSeconds: slurmDuration(kv.RunTime ?? ''),
32    limitSeconds: slurmDuration(kv.TimeLimit ?? ''),
33    cpus: tres.cpu ?? null,
34    mem: tres.mem ?? null,
35    gpus: tres['gres/gpu'] ?? null,
36  }
37}
38
39const num = (v: string | undefined): number | null => {
40  const n = Number((v ?? '').trim())
41  return v === undefined || !/\d/.test(v) || Number.isNaN(n) ? null : n
42}
43
44/** The two `--format=csv,noheader,nounits` queries, joined by GPU uuid. */
45export function parseNvidiaSmi(gpuCsv: string, appCsv: string): Gpu[] {
46  const procs = new Map<string, GpuProc[]>()
47  for (const line of appCsv.split('\n').filter(l => l.trim())) {
48    const [uuid, pid, name, mem] = line.split(',').map(c => c.trim())
49    if (!uuid) continue
50    const list = procs.get(uuid) ?? []
51    list.push({ pid: pid ?? '', name: (name ?? '').split('/').pop() ?? '', memMiB: num(mem) })
52    procs.set(uuid, list)
53  }
54  return gpuCsv.split('\n').filter(l => l.trim()).map(line => {
55    const [index, uuid, name, util, memUsed, memTotal, temp, power, powerLimit] = line.split(',').map(c => c.trim())
56    return {
57      index: index ?? '?',
58      name: (name ?? '').replace(/^(NVIDIA|Tesla) /, ''),
59      util: num(util),
60      memUsedMiB: num(memUsed),
61      memTotalMiB: num(memTotal),
62      tempC: num(temp),
63      powerW: num(power),
64      powerLimitW: num(powerLimit),
65      procs: procs.get(uuid ?? '') ?? [],
66    }
67  })
68}
69

1// The alert rules: pure functions of a job, the time and the thresholds.
2// Each kind fires once per job; watch.ts remembers which already did.
3
4import type { Alert, AlertKind, BatchJob } from '../../types'
5import { fmtDur } from '../format'
6
7export type Thresholds = {
8  idleMinutes: number
9  silentMinutes: number
10  nearLimitMinutes: number
11  staleCkptMinutes: number
12  pendingMinutes: number
13}
14
15export const DEFAULT_THRESHOLDS: Thresholds = {
16  idleMinutes: 10,
17  silentMinutes: 20,
18  nearLimitMinutes: 30,
19  staleCkptMinutes: 60,
20  pendingMinutes: 120,
21}
22
23/** States a job leaves squeue in; COMPLETED is the only good one. */
24export const TERMINAL = new Set(['COMPLETED', 'FAILED', 'TIMEOUT', 'CANCELLED', 'OUT_OF_MEMORY', 'NODE_FAIL', 'PREEMPTED', 'BOOT_FAIL', 'DEADLINE'])
25
26/** "t4-train (123)" */
27export function label(job: BatchJob): string {
28  return `${job.name} (${job.id})`
29}
30
31const ageMin = (now: number, t: number | null) => (t === null ? null : (now - t) / 60_000)
32
33export function alertsFor(job: BatchJob, now: number, th: Thresholds): Alert[] {
34  const out: Alert[] = []
35  const add = (kind: AlertKind, level: Alert['level'], text: string) => out.push({ job: job.id, kind, level, text, at: now })
36  const running = job.state === 'RUNNING'
37  const lastErr = job.flags.oom ?? job.flags.nccl ?? job.flags.traceback ?? job.flags.srun
38
39  if (TERMINAL.has(job.state)) {
40    const took = job.elapsedS !== null ? ` after ${fmtDur(job.elapsedS)}` : ''
41    if (job.state === 'COMPLETED') add('ended', 'info', `✓ ${label(job)} completed${took}`)
42    else add('ended', 'error', `✗ ${label(job)} ${job.state}${took}${job.exitCode && job.exitCode !== '0:0' ? ` (exit ${job.exitCode})` : ''}${lastErr ? `: ${lastErr}` : ''}`)
43    return out
44  }
45
46  if (job.flags.oom) add('oom', 'error', `✗ ${label(job)} ran out of memory: ${job.flags.oom}`)
47  if (job.flags.nccl) add('nccl', 'error', `✗ ${label(job)} NCCL trouble: ${job.flags.nccl}`)
48  if (job.flags.traceback && running) add('traceback', 'error', `✗ ${label(job)} raised ${job.flags.traceback}`)
49  if (job.metrics?.nonFinite) add('nonFinite', 'error', `✗ ${label(job)}: ${job.metrics.nonFinite.replace('@', ' went NaN/inf at step ')}`)
50
51  if (job.state === 'PENDING') {
52    const waited = ageMin(now, job.submittedAt)
53    if (waited !== null && waited >= th.pendingMinutes) {
54      add('pending', 'warn', `◌ ${label(job)} pending ${fmtDur(waited * 60)}${job.reason ? ` (${job.reason})` : ''}`)
55    }
56  }
57
58  if (running) {
59    const ranMin = ageMin(now, job.startedAt)
60    const quiet = ageMin(now, job.logChangedAt ?? job.startedAt)
61    if (ranMin !== null && ranMin >= th.silentMinutes && quiet !== null && quiet >= th.silentMinutes) {
62      add('silent', 'warn', `⚠ ${label(job)}: log silent for ${fmtDur(quiet * 60)} (hung rank?)`)
63    }
64    const idle = ageMin(now, job.idleSince)
65    if (idle !== null && idle >= th.idleMinutes) {
66      add('idle', 'warn', `⚠ ${label(job)}: allocated GPU idle for ${fmtDur(idle * 60)}`)
67    }
68    const ckptAge = job.ckpt ? ageMin(now, job.ckpt.mtimeMs) : null
69    if (job.limitS !== null && job.elapsedS !== null) {
70      const left = job.limitS - job.elapsedS
71      // A quarter of the limit at most: an 8-minute job isn't "near" its end at start.
72      if (left <= Math.min(th.nearLimitMinutes * 60, job.limitS / 4)) {
73        const ck = job.ckpt
74          ? `last checkpoint ${fmtDur((ckptAge ?? 0) * 60)} ago`
75          : 'no checkpoint found'
76        const stale = !job.ckpt || (ckptAge ?? 0) >= th.staleCkptMinutes
77        add('nearLimit', stale ? 'error' : 'warn', `⌛ ${label(job)}: ${fmtDur(left)} left, ${ck}`)
78      }
79    }
80    if (job.ckpt && ckptAge !== null && ckptAge >= th.staleCkptMinutes && ranMin !== null && ranMin >= th.staleCkptMinutes) {
81      add('staleCkpt', 'warn', `⚠ ${label(job)}: no new checkpoint for ${fmtDur(ckptAge * 60)}`)
82    }
83  }
84  return out
85}
86
87/** True when every allocated GPU reads 0% with no process. */
88export function allIdle(job: BatchJob): boolean {
89  const gpus = job.gpuReadings?.flatMap(n => n.gpus) ?? []
90  return gpus.length > 0 && gpus.every(g => (g.util ?? 0) === 0 && g.procs.length === 0)
91}
92
93/** True when some allocated GPU reads 0% while others work: a stuck or missing rank. */
94export function someIdle(job: BatchJob): boolean {
95  const gpus = job.gpuReadings?.flatMap(n => n.gpus) ?? []
96  return gpus.length > 1 && gpus.some(g => (g.util ?? 0) === 0) && gpus.some(g => (g.util ?? 0) > 0)
97}
98

1// runs/<jobid>/metrics.jsonl, read a piece at a time: one JSON object per
2// line, `step` plus any numbers (loss, val_*, lr, samples_per_s...). jobkit.py
3// writes it; Python's json writes NaN and Infinity bare, which JSON.parse
4// refuses, so those are read as non-finite on purpose.
5
6import type { Ckpt, Metrics, Series } from '../../types'
7
8/** Points kept per key; past it every other point is dropped, oldest half first. */
9export const MAX_POINTS = 1200
10
11/** Keys that describe the step rather than measure it. */
12const META = new Set(['step', 'time', 'epoch', 'total_steps', 'total_epochs', 'rank'])
13
14export const EMPTY_METRICS: Metrics = {
15  offset: 0,
16  keys: {},
17  lastStep: null,
18  totalSteps: null,
19  epoch: null,
20  totalEpochs: null,
21  firstTime: null,
22  firstStep: null,
23  lastTime: null,
24  nonFinite: null,
25}
26
27/** Bytes the text takes as UTF-8, so offsets match the file's. */
28export function utf8Length(text: string): number {
29  let n = 0
30  for (const ch of text) {
31    const c = ch.codePointAt(0)!
32    n += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4
33  }
34  return n
35}
36
37function thin(s: Series): Series {
38  if (s.step.length <= MAX_POINTS) return s
39  // Halve the older half: recent detail matters most on a live chart.
40  const half = Math.floor(s.step.length / 2)
41  const keep = (_: number, i: number) => i >= half || i % 2 === 0
42  return { step: s.step.filter(keep), value: s.value.filter(keep) }
43}
44
45/**
46 * Adds a chunk read from `m.offset` on. Only whole lines are consumed; a
47 * line still being written stays for the next read.
48 */
49export function addChunk(m: Metrics, chunk: string): Metrics {
50  const end = chunk.lastIndexOf('\n')
51  if (end < 0) return m
52  const whole = chunk.slice(0, end + 1)
53  const keys: Record<string, Series> = { ...m.keys }
54  let { lastStep, totalSteps, epoch, totalEpochs, firstTime, firstStep, lastTime, nonFinite } = m
55  for (const line of whole.split('\n')) {
56    if (!line.trim()) continue
57    let row: Record<string, unknown>
58    try {
59      row = JSON.parse(line.replace(/\b-?Infinity\b|\bNaN\b/g, '"__nonfinite__"')) as Record<string, unknown>
60    } catch {
61      continue
62    }
63    const step = typeof row.step === 'number' ? row.step : lastStep === null ? 0 : lastStep + 1
64    lastStep = step
65    firstStep ??= step
66    if (typeof row.time === 'number') {
67      firstTime ??= row.time
68      lastTime = row.time
69    }
70    if (typeof row.total_steps === 'number') totalSteps = row.total_steps
71    if (typeof row.epoch === 'number') epoch = row.epoch
72    if (typeof row.total_epochs === 'number') totalEpochs = row.total_epochs
73    for (const [k, v] of Object.entries(row)) {
74      if (META.has(k)) continue
75      if (v === '__nonfinite__') {
76        nonFinite ??= `${k}@${step}`
77        continue
78      }
79      if (typeof v !== 'number') continue
80      const s = keys[k] ?? { step: [], value: [] }
81      keys[k] = thin({ step: [...s.step, step], value: [...s.value, v] })
82    }
83  }
84  return { offset: m.offset + utf8Length(whole), keys, lastStep, totalSteps, epoch, totalEpochs, firstTime, firstStep, lastTime, nonFinite }
85}
86
87/** Exponential moving average, the smoothing TensorBoard's slider applies. */
88export function ema(values: readonly number[], alpha = 0.9): number[] {
89  const out: number[] = []
90  let acc: number | null = null
91  for (const v of values) {
92    acc = acc === null ? v : alpha * acc + (1 - alpha) * v
93    out.push(acc)
94  }
95  return out
96}
97
98/** The train loss key: `loss`, then `train_loss`, then any key ending in loss that isn't val. */
99export function lossKey(m: Metrics): string | null {
100  const keys = Object.keys(m.keys)
101  return keys.find(k => k === 'loss') ?? keys.find(k => k === 'train_loss') ?? keys.find(k => /loss$/.test(k) && !/^val/.test(k)) ?? null
102}
103
104export function valKeys(m: Metrics): string[] {
105  return Object.keys(m.keys).filter(k => /^val[_/]/.test(k))
106}
107
108/** Whether a val metric improves downwards: losses and errors do. */
109export function lowerIsBetter(key: string): boolean {
110  return /loss|err|wer|cer|mae|mse|rmse|perplexity|ppl/i.test(key)
111}
112
113/** Steps per second over the run so far, for the ETA. */
114export function stepRate(m: Metrics): number | null {
115  if (m.firstTime === null || m.lastTime === null || m.firstStep === null || m.lastStep === null) return null
116  const dt = m.lastTime - m.firstTime
117  return dt > 0 && m.lastStep > m.firstStep ? (m.lastStep - m.firstStep) / dt : null
118}
119
120export function etaSeconds(m: Metrics): number | null {
121  const rate = stepRate(m)
122  if (rate === null || m.totalSteps === null || m.lastStep === null) return null
123  return Math.max(0, (m.totalSteps - m.lastStep) / rate)
124}
125
126/** The trend of the smoothed loss over its last stretch: -1 falling, 1 rising, 0 flat. */
127export function trend(values: readonly number[]): -1 | 0 | 1 {
128  if (values.length < 8) return 0
129  const s = ema(values)
130  const a = s[Math.floor(s.length * 0.75)]!
131  const b = s[s.length - 1]!
132  const rel = (b - a) / (Math.abs(a) || 1)
133  return rel < -0.01 ? -1 : rel > 0.01 ? 1 : 0
134}
135
136const CKPT_FILE = /\.(pt|pth|ckpt|safetensors|bin)$|^checkpoint-\d+$|^(epoch|step)[=_-]?\d+/
137
138/** step and epoch from the names Lightning, HF Trainer and jobkit give checkpoints. */
139export function ckptNumbers(name: string): { step: number | null; epoch: number | null } {
140  const step = /(?:step[=_-]?|checkpoint-)(\d+)/.exec(name)
141  const epoch = /epoch[=_-]?(\d+)/.exec(name)
142  return { step: step ? Number(step[1]) : null, epoch: epoch ? Number(epoch[1]) : null }
143}
144
145/** The newest checkpoint among a directory's entries (best.* and last.* links aside). */
146export function newestCkpt(dir: string, entries: readonly { name: string; kind: string; size: number; mtimeMs: number }[]): Ckpt | null {
147  const cands = entries.filter(e => CKPT_FILE.test(e.name) && !/^(best|last)\b/.test(e.name) && !e.name.endsWith('.tmp'))
148  const top = [...cands].sort((a, b) => b.mtimeMs - a.mtimeMs)[0]
149  if (!top) return null
150  return {
151    name: top.name,
152    path: `${dir.replace(/\/$/, '')}/${top.name}`,
153    mtimeMs: top.mtimeMs,
154    sizeBytes: top.kind === 'file' ? top.size : null,
155    ...ckptNumbers(top.name),
156  }
157}
158

1// Parsers for what the batch-job watcher reads: squeue, scontrol, sacct, df,
2// a log's tail and an sbatch submit line. watch.ts runs the commands.
3
4import type { Disk, LogFlags } from '../../types'
5import { slurmDuration } from '../slurm'
6
7/** The squeue format the watcher asks for, one job per line, `|`-separated. */
8export const SQUEUE_FORMAT = '%i|%j|%T|%M|%l|%D|%R|%b|%N|%V'
9
10export type QueueRow = {
11  id: string
12  arrayId: string | null
13  name: string
14  state: string
15  elapsedS: number | null
16  limitS: number | null
17  nodes: number
18  /** The pending reason; for a running job squeue puts the node list here. */
19  reason: string | null
20  gpus: number | null
21  gpuType: string | null
22  nodeList: string
23  /** As squeue prints it, local time without a zone. */
24  submitTime: string
25}
26
27/** "gres/gpu:h200:4", "gres:gpu:4", "gres/gpu=2" or "N/A" → count and type. */
28export function gpusFromTres(tres: string): { gpus: number | null; gpuType: string | null } {
29  const m = /gpu(?::([a-zA-Z][\w.-]*))?[:=](\d+)/.exec(tres)
30  if (!m) return { gpus: null, gpuType: null }
31  return { gpus: Number(m[2]), gpuType: m[1] ? m[1].toUpperCase() : null }
32}
33
34export function parseSqueue(out: string): QueueRow[] {
35  const rows: QueueRow[] = []
36  for (const line of out.split('\n')) {
37    if (!line.trim()) continue
38    const [id = '', name = '', state = '', elapsed = '', limit = '', nodes = '', reason = '', tres = '', nodeList = '', submit = ''] = line.split('|')
39    const arr = /^(\d+)_(\d+|\[.*\])$/.exec(id)
40    const { gpus, gpuType } = gpusFromTres(tres)
41    rows.push({
42      id,
43      arrayId: arr ? arr[1]! : null,
44      name,
45      state,
46      elapsedS: slurmDuration(elapsed),
47      limitS: slurmDuration(limit),
48      nodes: Number(nodes) || 1,
49      reason: state === 'PENDING' ? reason.replace(/^\(|\)$/g, '') || null : null,
50      gpus,
51      gpuType,
52      nodeList: state === 'PENDING' ? '' : nodeList,
53      submitTime: submit,
54    })
55  }
56  return rows
57}
58
59/** `scontrol show job` as Key=Value; the first value wins (JobId before ArrayJobId's). */
60export function scontrolFields(out: string): Record<string, string> {
61  const kv: Record<string, string> = {}
62  for (const m of out.matchAll(/(?:^|\s)(\w[\w/:]*)=(\S*)/g)) kv[m[1]!] ??= m[2]!
63  return kv
64}
65
66export type JobDetail = {
67  jobId: string
68  isBatch: boolean
69  workDir: string | null
70  stdout: string | null
71  command: string | null
72  batchHost: string | null
73  nodeList: string
74}
75
76export function parseJobDetail(out: string): JobDetail | null {
77  const kv = scontrolFields(out)
78  if (!kv.JobId) return null
79  const val = (v: string | undefined) => (v && v !== '(null)' ? v : null)
80  return {
81    jobId: kv.JobId,
82    isBatch: kv.BatchFlag === '1',
83    workDir: val(kv.WorkDir),
84    stdout: val(kv.StdOut),
85    command: val(kv.Command),
86    batchHost: val(kv.BatchHost),
87    nodeList: val(kv.NodeList) ?? '',
88  }
89}
90
91export const SACCT_FIELDS = 'JobID,JobName,State,ExitCode,Elapsed,WorkDir,SubmitLine'
92
93export type Accounting = {
94  jobId: string
95  state: string
96  exitCode: string
97  elapsedS: number | null
98  workDir: string | null
99  submitLine: string | null
100}
101
102/**
103 * `sacct -X -n -P -o <SACCT_FIELDS>` for one job. SubmitLine is last and may
104 * itself hold `|` or newlines (an --wrap script), so it takes all the rest.
105 * The state can read "CANCELLED by 1000"; only the first word is kept.
106 */
107export function parseSacct(out: string): Accounting | null {
108  const text = out.replace(/\n+$/, '')
109  if (!text.trim()) return null
110  const parts = text.split('|')
111  if (parts.length < 7) return null
112  const [jobId = '', , state = '', exitCode = '', elapsed = '', workDir = ''] = parts
113  const submitLine = parts.slice(6).join('|').trim()
114  return {
115    jobId,
116    state: state.split(' ')[0] ?? state,
117    exitCode,
118    elapsedS: slurmDuration(elapsed),
119    workDir: workDir || null,
120    submitLine: submitLine || null,
121  }
122}
123
124/** `date +%z` → minutes east of UTC: "-0400" → -240. */
125export function zoneMinutes(z: string): number {
126  const m = /^([+-])(\d\d)(\d\d)$/.exec(z.trim())
127  return m ? (m[1] === '-' ? -1 : 1) * (Number(m[2]) * 60 + Number(m[3])) : 0
128}
129
130/** squeue's "2026-10-07T11:54:38" in the cluster's zone → epoch ms; null for N/A. */
131export function slurmTime(text: string, zone: number): number | null {
132  const ms = Date.parse(`${text}Z`)
133  return Number.isNaN(ms) ? null : ms - zone * 60_000
134}
135
136/** `df -B1 --output=target,size,used,avail <paths>`. */
137export function parseDf(out: string): Disk[] {
138  return out.split('\n').slice(1).filter(l => l.trim()).map(l => {
139    const [mount = '', size = '0', used = '0', avail = '0'] = l.trim().split(/\s+/)
140    return { mount, size: Number(size), used: Number(used), avail: Number(avail) }
141  })
142}
143
144/** Lines that say nothing about the job's progress. */
145const NOISE = [
146  /^\s*$/,
147  /Kwargs passed to/,
148  /(Future|User|Deprecation|Runtime)Warning/,
149  /^\s*warnings\.warn/,
150  /^\s*\d+%\|/, // a tqdm bar with no description: kept as progress, not as a line
151]
152
153export type LogTail = { lines: string[]; progress: { k: number; n: number } | null; flags: LogFlags }
154
155/**
156 * The tail of a job's log: tqdm's carriage returns split into lines, noise
157 * dropped, the last `[k/N]` or tqdm `k/N [` as progress, and trouble flagged.
158 */
159export function parseLogTail(text: string, keep = 3): LogTail {
160  const raw = text.replace(/\r\n/g, '\n').split(/[\r\n]/)
161  let progress: { k: number; n: number } | null = null
162  const flags: LogFlags = { oom: null, traceback: null, nccl: null, srun: null }
163  let inTraceback = false
164  for (let i = 0; i < raw.length; i++) {
165    const line = raw[i]!
166    const p = /\[(\d+)\/(\d+)\]/.exec(line) ?? /\b(\d+)\/(\d+) \[/.exec(line)
167    if (p && Number(p[2]) > 0) progress = { k: Number(p[1]), n: Number(p[2]) }
168    if (/CUDA out of memory|OutOfMemoryError|oom-kill|Out Of Memory/i.test(line)) flags.oom = line.trim()
169    if (/NCCL (error|WARN.*[Tt]imeout)|Watchdog caught collective operation timeout|ProcessGroupNCCL.*(timeout|error)/i.test(line)) flags.nccl = line.trim()
170    if (/^srun: error:/.test(line)) flags.srun = line.trim()
171    if (/^Traceback \(most recent call last\)/.test(line)) inTraceback = true
172    else if (inTraceback && /^\S/.test(line) && !/^(  |During handling|The above exception)/.test(line)) {
173      // The first unindented line after the frames: "ValueError: ...".
174      flags.traceback = line.trim()
175      inTraceback = false
176    }
177  }
178  const lines = raw.filter(l => !NOISE.some(r => r.test(l))).map(l => l.trimEnd()).slice(-keep)
179  return { lines, progress, flags }
180}
181
182/** Splits a command line as a POSIX shell would for plain words, quotes and escapes. */
183export function shellWords(line: string): string[] {
184  const words: string[] = []
185  let cur = ''
186  let has = false
187  let quote: '"' | "'" | null = null
188  for (let i = 0; i < line.length; i++) {
189    const c = line[i]!
190    if (quote) {
191      if (c === quote) quote = null
192      else if (c === '\\' && quote === '"' && i + 1 < line.length) cur += line[++i]
193      else cur += c
194    } else if (c === '"' || c === "'") {
195      quote = c
196      has = true
197    } else if (c === '\\' && i + 1 < line.length) {
198      cur += line[++i]
199      has = true
200    } else if (/\s/.test(c)) {
201      if (has || cur) words.push(cur)
202      cur = ''
203      has = false
204    } else {
205      cur += c
206    }
207  }
208  if (has || cur) words.push(cur)
209  return words
210}
211
212/**
213 * The submit line as argv with RESUME=<ckpt> added to its --export (or an
214 * `--export=ALL,RESUME=…` added after `sbatch`), any earlier RESUME replaced.
215 * Null when the line is not a plain `sbatch … script` (a --wrap, an array, a pipe).
216 */
217export function resumeArgv(submitLine: string, ckpt: string): string[] | null {
218  if (/\n|[|;&<>`]|\$\(/.test(submitLine)) return null
219  const argv = shellWords(submitLine)
220  if (argv[0] !== 'sbatch' && !argv[0]?.endsWith('/sbatch')) return null
221  if (argv.some(a => a === '--wrap' || a.startsWith('--wrap=') || a === '-a' || a === '--array' || a.startsWith('--array='))) return null
222  const set = (list: string) => [...list.split(',').filter(v => v && !v.startsWith('RESUME=')), `RESUME=${ckpt}`].join(',')
223  const out = argv.filter(a => a !== '--parsable')
224  for (let i = 1; i < out.length; i++) {
225    const a = out[i]!
226    if (a.startsWith('--export=')) {
227      out[i] = `--export=${set(a.slice('--export='.length))}`
228      return ['sbatch', '--parsable', ...out.slice(1)]
229    }
230    if (a === '--export' && i + 1 < out.length) {
231      out[i + 1] = set(out[i + 1]!)
232      return ['sbatch', '--parsable', ...out.slice(1)]
233    }
234  }
235  return ['sbatch', '--parsable', `--export=ALL,RESUME=${ckpt}`, ...out.slice(1)]
236}
237
238/** "Submitted batch job 123" or a --parsable "123;cluster" → "123". */
239export function submittedId(stdout: string): string | null {
240  const m = /Submitted batch job (\d+)/.exec(stdout) ?? /^(\d+)(?:;\S+)?\s*$/m.exec(stdout)
241  return m ? m[1]! : null
242}
243

1// The batch-job watcher's settings, empty state and conventions text: pure.
2// The atoms themselves are made in register.tsx, where the state scan reads them.
3
4import type { Jobs } from '../../types'
5import { DEFAULT_THRESHOLDS } from './alerts'
6import type { Thresholds } from './alerts'
7
8export type WatchConfig = Thresholds & {
9  pollSeconds: number
10  gpuSeconds: number
11  maxResumes: number
12  excludeNames: string[]
13  discordChatId: string
14  discordWebhook: string
15}
16
17export const EMPTY_JOBS: Jobs = { jobs: [], selected: null, updatedAt: null, error: null, disks: [], alerts: [] }
18
19export function configFrom(options: Readonly<Record<string, unknown>>): WatchConfig {
20  const num = (k: string, d: number) => (typeof options[k] === 'number' && (options[k] as number) > 0 ? (options[k] as number) : d)
21  const str = (k: string) => (typeof options[k] === 'string' ? (options[k] as string).trim() : '')
22  return {
23    pollSeconds: num('pollSeconds', 60),
24    gpuSeconds: num('gpuSeconds', 30),
25    idleMinutes: num('idleMinutes', DEFAULT_THRESHOLDS.idleMinutes),
26    silentMinutes: num('silentMinutes', DEFAULT_THRESHOLDS.silentMinutes),
27    nearLimitMinutes: num('nearLimitMinutes', DEFAULT_THRESHOLDS.nearLimitMinutes),
28    staleCkptMinutes: num('staleCkptMinutes', DEFAULT_THRESHOLDS.staleCkptMinutes),
29    pendingMinutes: num('pendingMinutes', DEFAULT_THRESHOLDS.pendingMinutes),
30    maxResumes: num('maxResumes', 3),
31    excludeNames: (str('excludeNames') || 'dev-shell,interactive').split(',').map(s => s.trim()).filter(Boolean),
32    discordChatId: str('discordChatId'),
33    discordWebhook: str('discordWebhook'),
34  }
35}
36
37/** The run conventions pace-line watches, for every project on the cluster. */
38export const CONVENTIONS = `# Batch jobs (pace-line)
39pace-line watches every batch job of the user's (squeue each minute) and reports progress, GPU use, checkpoints and failures by itself: never add Monitor, sleep or squeue loops to wait for a job. Its alerts arrive as messages headed "[pace-line: batch-job alerts]".
40Write training and eval jobs to its conventions, in any project:
41- sbatch scripts log to \`--output=runs/slurm-%j.out\` and put every artifact under \`runs/$SLURM_JOB_ID/\`.
42- Metrics: one JSON line per logging step in \`runs/$SLURM_JOB_ID/metrics.jsonl\`: \`{"time": <unix s>, "step": N, "total_steps": T, "epoch": E, "total_epochs": TE, "loss": ..., "val_<name>": ..., "lr": ..., "samples_per_s": ...}\`, rank 0 only.
43- Checkpoints: \`runs/$SLURM_JOB_ID/checkpoints/\`, named with the step or epoch (\`epoch=3-step=1200.pt\`, \`checkpoint-1200/\`), written atomically.
44- Resume: training resumes from the checkpoint path in \`$RESUME\` when set. pace-line resubmits a job that hits TIMEOUT with \`RESUME=<latest checkpoint>\` added to its --export, up to its limit (3 by default).
45- \`jobkit.py\` (log_metrics, save_checkpoint, latest_checkpoint, resume_path, load_resume) and \`slurm/train.template.sbatch\` implement this; \`/jobs init\` copies them into a project.`
46
47/** What /jobs init copies from the plugin into a project. */
48export const TEMPLATES = [
49  { from: 'templates/jobkit.py', to: 'jobkit.py' },
50  { from: 'templates/train.template.sbatch', to: 'slurm/train.template.sbatch' },
51] as const
52

1// What the band's third line and the pane's "Batch jobs" tab draw. Pure
2// layout from the jobs state; register.tsx owns the hooks and the handlers.
3
4import type { EngineInterface } from 'claude-code'
5
6import type { BatchJob, Jobs, Metrics } from '../../types'
7import { COLORS, PLAIN, bar, fmtDur, usageColor } from '../format'
8import type { Seg } from '../format'
9import { TERMINAL } from './alerts'
10import { SWEEP_COLORS, braille, fmtNum, rangeOf, sparkline } from './chart'
11import type { Line, Run } from './chart'
12import { ema, etaSeconds, lossKey, lowerIsBetter, stepRate, trend, valKeys } from './metrics'
13
14export type Item =
15  | { kind: 'job'; key: string; job: BatchJob }
16  | { kind: 'array'; key: string; arrayId: string; name: string; tasks: BatchJob[] }
17
18/** Jobs as the list shows them: array tasks folded into one sweep row. */
19export function items(jobs: readonly BatchJob[]): Item[] {
20  const out: Item[] = []
21  const arrays = new Map<string, BatchJob[]>()
22  for (const j of jobs) {
23    if (j.arrayId) {
24      const list = arrays.get(j.arrayId)
25      if (list) list.push(j)
26      else {
27        const tasks = [j]
28        arrays.set(j.arrayId, tasks)
29        out.push({ kind: 'array', key: `array:${j.arrayId}`, arrayId: j.arrayId, name: j.name, tasks })
30      }
31    } else out.push({ kind: 'job', key: j.id, job: j })
32  }
33  return out
34}
35
36export function stateGlyph(state: string): { glyph: string; color: string } {
37  if (state === 'RUNNING') return { glyph: '▶', color: COLORS.GREEN }
38  if (state === 'PENDING' || state === 'CONFIGURING') return { glyph: '◌', color: COLORS.YELLOW }
39  if (state === 'COMPLETING') return { glyph: '◍', color: COLORS.YELLOW }
40  if (state === 'COMPLETED') return { glyph: '✓', color: COLORS.GREEN }
41  if (TERMINAL.has(state)) return { glyph: '✗', color: COLORS.RED }
42  return { glyph: '·', color: COLORS.GRAY }
43}
44
45const gpuText = (j: BatchJob) => {
46  if (!j.gpus) return null
47  const per = `${j.gpus}×${j.gpuType ?? 'gpu'}`
48  return j.nodes > 1 ? `${j.nodes}n·${per}` : per
49}
50
51/** "ep 3/20", "step 1200/5000", "[12/20]" or null. */
52export function progressText(j: BatchJob): string | null {
53  const m = j.metrics
54  if (m?.epoch !== null && m?.epoch !== undefined && m.totalEpochs) return `ep ${m.epoch}/${m.totalEpochs}`
55  if (m?.lastStep !== null && m?.lastStep !== undefined && m.totalSteps) return `${Math.round((m.lastStep / m.totalSteps) * 100)}%`
56  if (j.progress) return `${j.progress.k}/${j.progress.n}`
57  return null
58}
59
60function lossSegs(m: Metrics | null): Seg[] {
61  const k = m && lossKey(m)
62  if (!m || !k) return []
63  const v = m.keys[k]!.value
64  const t = trend(v)
65  return [{ text: ` ${fmtNum(v[v.length - 1]!)}`, color: PLAIN }, { text: t < 0 ? '↓' : t > 0 ? '↑' : '→', color: t < 0 ? COLORS.GREEN : t > 0 ? COLORS.RED : COLORS.GRAY }]
66}
67
68/** The band's third line: active jobs, then the newest unresolved alert. */
69export function jobsLine(s: Jobs, now: number, maxItems = 3): Seg[] {
70  const segs: Seg[] = []
71  const visible = items(s.jobs).filter(it => it.kind === 'array' || !it.job.endedAt || now - it.job.endedAt < 30 * 60_000)
72  if (!visible.length) return []
73  for (const it of visible.slice(0, maxItems)) {
74    if (segs.length) segs.push({ text: ' · ', color: COLORS.GRAY })
75    if (it.kind === 'array') {
76      const c = (f: (j: BatchJob) => boolean) => it.tasks.filter(f).length
77      segs.push(
78        { text: `${it.name}[${it.tasks.length}] `, color: COLORS.CYAN, action: 'jobs' },
79        { text: `${c(j => j.state === 'COMPLETED')}✓`, color: COLORS.GREEN },
80        { text: `${c(j => j.state === 'RUNNING')}▶`, color: COLORS.GREEN },
81        { text: `${c(j => j.state === 'PENDING')}◌`, color: COLORS.YELLOW },
82      )
83      const failed = c(j => TERMINAL.has(j.state) && j.state !== 'COMPLETED')
84      if (failed) segs.push({ text: `${failed}✗`, color: COLORS.RED })
85      continue
86    }
87    const j = it.job
88    const g = stateGlyph(j.state)
89    segs.push({ text: `${g.glyph} `, color: g.color }, { text: j.name, color: COLORS.CYAN, action: 'jobs' })
90    const gpu = gpuText(j)
91    if (gpu) segs.push({ text: ` ${gpu}`, color: COLORS.GRAY })
92    if (j.state === 'PENDING' && j.reason) segs.push({ text: ` (${j.reason})`, color: COLORS.GRAY })
93    const p = progressText(j)
94    if (p) segs.push({ text: ` ${p}`, color: PLAIN })
95    segs.push(...lossSegs(j.metrics))
96    if (j.state === 'RUNNING' && j.limitS !== null && j.elapsedS !== null) {
97      const left = j.limitS - j.elapsedS
98      segs.push({ text: ` ⌛${fmtDur(left)}`, color: left < 30 * 60 ? COLORS.RED : COLORS.GRAY })
99    }
100  }
101  if (visible.length > maxItems) segs.push({ text: ` +${visible.length - maxItems}`, color: COLORS.GRAY })
102  const alert = s.alerts.find(a => a.level !== 'info' && now - a.at < 60 * 60_000)
103  if (alert) segs.push({ text: '  ' }, { text: alert.text.slice(0, 60), color: alert.level === 'error' ? COLORS.RED : COLORS.ORANGE, action: 'jobs' })
104  return segs
105}
106
107type El = ReturnType<EngineInterface['ui']['resolve']>
108type Kit = { Box: El['Box']; Text: El['Text']; Button: El['Button'] }
109
110export type JobsActions = {
111  select: (key: string) => void
112  copy: (text: string) => void
113  resume: (j: BatchJob) => void
114  cancel: (j: BatchJob) => void
115  refresh: () => void
116}
117
118const gib = (b: number) => `${(b / 2 ** 30).toFixed(1)}G`
119
120function runs(kit: Kit, key: string, rs: Run[]) {
121  const { Text } = kit
122  return (
123    <Text key={key}>
124      {rs.map((r, i) => <Text key={String(i)} color={r.color || COLORS.GRAY}>{r.text}</Text>)}
125    </Text>
126  )
127}
128
129/** A braille chart with its top and bottom values on the left and the step range under it. */
130function chart(kit: Kit, key: string, title: string, lines: Line[], width: number, height: number, note: string) {
131  const { Box, Text } = kit
132  const range = rangeOf(lines)
133  if (!range) return null
134  const rows = braille(lines, width, height, range)
135  const top = fmtNum(range.y.max).padStart(8)
136  const bottom = fmtNum(range.y.min).padStart(8)
137  return (
138    <Box key={key} flexDirection="column" marginTop={1}>
139      <Text>
140        <Text color={COLORS.CYAN} bold>{title}</Text>
141        <Text color={PLAIN}>{`  ${note}`}</Text>
142      </Text>
143      {rows.map((r, i) => (
144        <Box key={String(i)} flexDirection="row">
145          <Text color={COLORS.GRAY}>{i === 0 ? top : i === rows.length - 1 ? bottom : ' '.repeat(8)}</Text>
146          <Text color={COLORS.GRAY}>{' ┤'}</Text>
147          {runs(kit, 'r', r)}
148        </Box>
149      ))}
150      <Text color={COLORS.GRAY}>{`${' '.repeat(10)}step ${Math.round(range.x.min)}${' '.repeat(Math.max(1, width - 14 - String(Math.round(range.x.max)).length))}${Math.round(range.x.max)}`}</Text>
151    </Box>
152  )
153}
154
155function metricCharts(kit: Kit, m: Metrics, width: number) {
156  const { Text } = kit
157  const out = []
158  const lk = lossKey(m)
159  if (lk) {
160    const s = m.keys[lk]!
161    const smooth = ema(s.value)
162    const min = Math.min(...s.value)
163    out.push(chart(kit, 'loss', lk, [
164      { xs: s.step, ys: s.value, color: COLORS.GRAY },
165      { xs: s.step, ys: smooth, color: COLORS.CYAN },
166    ], width, 6, `${fmtNum(s.value[s.value.length - 1]!)} (ema ${fmtNum(smooth[smooth.length - 1]!)}, min ${fmtNum(min)})`))
167  }
168  for (const k of valKeys(m).slice(0, 2)) {
169    const s = m.keys[k]!
170    const low = lowerIsBetter(k)
171    const best = s.value.reduce((bi, v, i) => ((low ? v < s.value[bi]! : v > s.value[bi]!) ? i : bi), 0)
172    out.push(chart(kit, k, k, [
173      { xs: s.step, ys: s.value, color: COLORS.MAGENTA },
174      { xs: [s.step[best]!], ys: [s.value[best]!], color: COLORS.GREEN },
175    ], width, 3, `${fmtNum(s.value[s.value.length - 1]!)} · best ${fmtNum(s.value[best]!)} @ step ${s.step[best]}`))
176  }
177  const lr = m.keys.lr ?? m.keys.learning_rate
178  const tp = m.keys.samples_per_s ?? m.keys.it_s ?? m.keys.throughput
179  const sw = Math.max(10, Math.min(60, width - 30))
180  if (lr) out.push(<Text key="lr"><Text color={COLORS.CYAN}>{'lr          '}</Text><Text color={COLORS.YELLOW}>{sparkline(lr.value, sw)}</Text><Text color={PLAIN}>{`  ${fmtNum(lr.value[lr.value.length - 1]!)}`}</Text></Text>)
181  if (tp) out.push(<Text key="tp"><Text color={COLORS.CYAN}>{'throughput  '}</Text><Text color={COLORS.GREEN}>{sparkline(tp.value, sw)}</Text><Text color={PLAIN}>{`  ${fmtNum(tp.value[tp.value.length - 1]!)}/s`}</Text></Text>)
182  return out
183}
184
185function jobDetail(kit: Kit, j: BatchJob, s: Jobs, now: number, cols: number, act: JobsActions) {
186  const { Box, Text, Button } = kit
187  const g = stateGlyph(j.state)
188  const barW = Math.max(10, Math.min(30, cols - 40))
189  const chartW = Math.max(20, Math.min(100, cols - 14))
190  const m = j.metrics
191  const eta = m ? etaSeconds(m) : null
192  const rate = m ? stepRate(m) : null
193  const pct = m?.lastStep !== null && m?.lastStep !== undefined && m.totalSteps ? (m.lastStep / m.totalSteps) * 100 : j.progress ? (j.progress.k / j.progress.n) * 100 : null
194  const ckptAge = j.ckpt ? (now - j.ckpt.mtimeMs) / 1000 : null
195  const node = j.nodeList.split(/[,[]/)[0] || null
196  const disks = s.disks.filter(d => d.mount.startsWith('/scratch') || (j.workDir ?? '').startsWith(d.mount))
197  return (
198    <Box key="detail" flexDirection="column" marginTop={1}>
199      <Box flexDirection="row" justifyContent="space-between">
200        <Text>
201          <Text color={COLORS.MAGENTA} bold>{j.name}</Text>
202          <Text color={PLAIN}>{`  ${j.id}${j.jobId !== j.id ? ` (${j.jobId})` : ''}`}</Text>
203        </Text>
204        <Text color={g.color}>{`${g.glyph} ${j.state}${j.reason ? ` (${j.reason})` : ''}`}</Text>
205      </Box>
206      <Text color={COLORS.GRAY} wrap="truncate-end">
207        {[j.nodeList || null, gpuText(j), j.workDir, j.resumedFrom && `resumed from ${j.resumedFrom}`, j.resumedAs && `resubmitted as ${j.resumedAs}`].filter(Boolean).join(' · ')}
208      </Text>
209      {j.limitS !== null && j.elapsedS !== null && (
210        <Text>
211          <Text color={COLORS.CYAN}>{'time        '}</Text>
212          <Text color={usageColor((j.elapsedS / j.limitS) * 100)}>{bar((j.elapsedS / j.limitS) * 100, barW)}</Text>
213          <Text color={PLAIN}>{`  ${fmtDur(j.elapsedS)} of ${fmtDur(j.limitS)}`}</Text>
214        </Text>
215      )}
216      {pct !== null && (
217        <Text>
218          <Text color={COLORS.CYAN}>{'progress    '}</Text>
219          <Text color={COLORS.GREEN}>{bar(pct, barW)}</Text>
220          <Text color={PLAIN}>{`  ${progressText(j) ?? ''}${m?.lastStep !== null && m?.lastStep !== undefined ? ` · step ${m.lastStep}` : ''}${rate ? ` · ${fmtNum(rate)} step/s` : ''}${eta !== null ? ` · ETA ${fmtDur(eta)}` : ''}`}</Text>
221        </Text>
222      )}
223      {m && metricCharts(kit, m, chartW)}
224      {j.gpuReadings && (
225        <Box flexDirection="column" marginTop={1}>
226          {j.gpuReadings.flatMap(n => n.gpus.map(gp => {
227            const mem = gp.memUsedMiB !== null && gp.memTotalMiB ? (gp.memUsedMiB / gp.memTotalMiB) * 100 : null
228            const idle = (gp.util ?? 0) === 0 && gp.procs.length === 0
229            return (
230              <Text key={`${n.node}-${gp.index}`}>
231                <Text color={COLORS.CYAN}>{`${n.node} gpu${gp.index}`.padEnd(12)}</Text>
232                <Text color={usageColor(gp.util ?? 0)}>{bar(gp.util ?? 0, 10)}</Text>
233                <Text color={PLAIN}>{` ${String(gp.util ?? 0).padStart(3)}%  `}</Text>
234                <Text color={usageColor(mem ?? 0)}>{bar(mem ?? 0, 10)}</Text>
235                <Text color={PLAIN}>{` ${gp.memUsedMiB !== null ? (gp.memUsedMiB / 1024).toFixed(0) : '?'}/${gp.memTotalMiB !== null ? (gp.memTotalMiB / 1024).toFixed(0) : '?'}G`}</Text>
236                <Text color={COLORS.GRAY}>{gp.tempC !== null ? `  ${gp.tempC}°C` : ''}</Text>
237                {idle && <Text color={COLORS.ORANGE} inverse>{' idle '}</Text>}
238              </Text>
239            )
240          }))}
241        </Box>
242      )}
243      {j.ckpt
244        ? (
245          <Text wrap="truncate-end">
246            <Text color={COLORS.CYAN}>{'checkpoint  '}</Text>
247            <Text color={ckptAge !== null && ckptAge > 3600 ? COLORS.ORANGE : PLAIN}>
248              {[j.ckpt.name, j.ckpt.epoch !== null && `epoch ${j.ckpt.epoch}`, j.ckpt.step !== null && `step ${j.ckpt.step}`, ckptAge !== null && `${fmtDur(ckptAge)} ago`, j.ckpt.sizeBytes !== null && gib(j.ckpt.sizeBytes)].filter(Boolean).join(' · ')}
249            </Text>
250          </Text>
251        )
252        : j.state === 'RUNNING' && <Text color={COLORS.GRAY}>{'checkpoint  none in runs/<jobid>/checkpoints yet'}</Text>}
253      {disks.map(d => (
254        <Text key={d.mount}>
255          <Text color={COLORS.CYAN}>{`disk ${d.mount}`.padEnd(12).slice(0, 12)}</Text>
256          <Text color={usageColor((d.used / d.size) * 100)}>{bar((d.used / d.size) * 100, 10)}</Text>
257          <Text color={PLAIN}>{`  ${gib(d.avail)} free of ${gib(d.size)}`}</Text>
258        </Text>
259      ))}
260      {j.lastLines.length > 0 && (
261        <Box flexDirection="column" marginTop={1}>
262          {j.lastLines.map((l, i) => <Text key={String(i)} color={COLORS.GRAY} wrap="truncate-end">{`› ${l}`}</Text>)}
263        </Box>
264      )}
265      <Box flexDirection="row" gap={2} marginTop={1}>
266        {node && j.state === 'RUNNING' && <Button key="ssh" label={`Copy ssh ${node}`} onPress={() => act.copy(`ssh ${node}`)} />}
267        {j.stdout && <Button key="log" label="Copy tail -f" onPress={() => act.copy(`tail -f ${j.stdout}`)} />}
268        {j.endedAt !== null && j.state !== 'COMPLETED' && !j.resumedAs && <Button key="resume" label="Resume from checkpoint" onPress={() => act.resume(j)} />}
269        {j.endedAt === null && <Button key="cancel" label="Cancel job" onPress={() => act.cancel(j)} />}
270      </Box>
271    </Box>
272  )
273}
274
275function arrayDetail(kit: Kit, it: Extract<Item, { kind: 'array' }>, cols: number, act: JobsActions) {
276  const { Box, Text, Button } = kit
277  const width = Math.max(20, Math.min(100, cols - 14))
278  const lines: Line[] = it.tasks.flatMap((t, i) => {
279    const k = t.metrics && lossKey(t.metrics)
280    if (!t.metrics || !k) return []
281    const s = t.metrics.keys[k]!
282    return [{ xs: s.step, ys: ema(s.value), color: SWEEP_COLORS[i % SWEEP_COLORS.length]! }]
283  })
284  return (
285    <Box key="detail" flexDirection="column" marginTop={1}>
286      <Text>
287        <Text color={COLORS.MAGENTA} bold>{it.name}</Text>
288        <Text color={PLAIN}>{`  array ${it.arrayId} · ${it.tasks.length} tasks`}</Text>
289      </Text>
290      {lines.length > 0 && chart(kit, 'sweep', 'loss (ema) per task', lines, width, 8, '')}
291      <Box flexDirection="column" marginTop={1}>
292        {it.tasks.map((t, i) => {
293          const g = stateGlyph(t.state)
294          const k = t.metrics && lossKey(t.metrics)
295          const last = k ? t.metrics!.keys[k]!.value : null
296          return (
297            <Box key={t.id} flexDirection="row" gap={1}>
298              <Button label={t.id} onPress={() => act.select(t.id)} />
299              <Text>
300                <Text color={SWEEP_COLORS[i % SWEEP_COLORS.length]}>{'━━ '}</Text>
301                <Text color={g.color}>{`${g.glyph} ${t.state}`}</Text>
302                <Text color={PLAIN}>{`  ${progressText(t) ?? ''}${last ? `  loss ${fmtNum(last[last.length - 1]!)}` : ''}`}</Text>
303              </Text>
304            </Box>
305          )
306        })}
307      </Box>
308    </Box>
309  )
310}
311
312/** The "Batch jobs" tab: the list, then the selected job or sweep. */
313export function jobsTab(kit: Kit, s: Jobs, now: number, cols: number, act: JobsActions) {
314  const { Box, Text, Button } = kit
315  const list = items(s.jobs)
316  const selected = list.find(it => it.key === s.selected) ?? (s.selected ? list.find(it => it.kind === 'array' && it.tasks.some(t => t.id === s.selected)) : undefined) ?? list[0]
317  const task = selected?.kind === 'array' ? selected.tasks.find(t => t.id === s.selected) : undefined
318  return (
319    <Box flexDirection="column">
320      {s.error && <Text color={COLORS.RED}>{s.error}</Text>}
321      {list.length === 0 && <Text color={COLORS.GRAY}>{s.updatedAt === null ? 'Reading squeue…' : 'No batch jobs. Submitted jobs show here within a minute.'}</Text>}
322      {list.map(it => {
323        const isSel = it === selected
324        if (it.kind === 'array') {
325          const c = (f: (j: BatchJob) => boolean) => it.tasks.filter(f).length
326          return (
327            <Box key={it.key} flexDirection="row" gap={1}>
328              <Button label={`${isSel ? '›' : ' '} ${it.arrayId}`} onPress={() => act.select(it.key)} />
329              <Text>
330                <Text color={COLORS.CYAN}>{it.name.padEnd(16).slice(0, 16)}</Text>
331                <Text color={COLORS.GREEN}>{` ${c(j => j.state === 'COMPLETED')}✓ ${c(j => j.state === 'RUNNING')}▶`}</Text>
332                <Text color={COLORS.YELLOW}>{` ${c(j => j.state === 'PENDING')}◌`}</Text>
333                <Text color={COLORS.RED}>{` ${c(j => TERMINAL.has(j.state) && j.state !== 'COMPLETED')}✗`}</Text>
334              </Text>
335            </Box>
336          )
337        }
338        const j = it.job
339        const g = stateGlyph(j.state)
340        return (
341          <Box key={it.key} flexDirection="row" gap={1}>
342            <Button label={`${isSel ? '›' : ' '} ${j.id}`} onPress={() => act.select(it.key)} />
343            <Text wrap="truncate-end">
344              <Text color={COLORS.CYAN}>{j.name.padEnd(16).slice(0, 16)}</Text>
345              <Text color={g.color}>{` ${g.glyph} ${j.state.padEnd(9)}`}</Text>
346              <Text color={PLAIN}>{` ${j.elapsedS !== null ? fmtDur(j.elapsedS) : ''}${j.limitS !== null ? `/${fmtDur(j.limitS)}` : ''}`}</Text>
347              <Text color={COLORS.GRAY}>{gpuText(j) ? `  ${gpuText(j)}` : ''}</Text>
348              <Text color={PLAIN}>{progressText(j) ? `  ${progressText(j)}` : ''}</Text>
349              <Text color={COLORS.GRAY}>{j.state === 'PENDING' && j.reason ? `  ${j.reason}` : ''}</Text>
350            </Text>
351          </Box>
352        )
353      })}
354      {task
355        ? jobDetail(kit, task, s, now, cols, act)
356        : selected?.kind === 'array'
357          ? arrayDetail(kit, selected, cols, act)
358          : selected && jobDetail(kit, selected.job, s, now, cols, act)}
359      {s.alerts.length > 0 && (
360        <Box flexDirection="column" marginTop={1}>
361          {s.alerts.slice(0, 4).map((a, i) => (
362            <Text key={String(i)} color={a.level === 'error' ? COLORS.RED : a.level === 'warn' ? COLORS.ORANGE : COLORS.GREEN} wrap="truncate-end">{a.text}</Text>
363          ))}
364        </Box>
365      )}
366      <Box flexDirection="row" gap={2} marginTop={1}>
367        <Button key="refresh" label="Refresh" onPress={() => act.refresh()} />
368        <Text color={COLORS.GRAY}>{s.updatedAt !== null ? `squeue ${fmtDur((now - s.updatedAt) / 1000)} ago` : ''}</Text>
369      </Box>
370    </Box>
371  )
372}
373

1// Surface module for line 1: draws the segments in their own colours at rest
2// (a Button can't be coloured until hovered) and turns a click on the model
3// or effort into a post the hooks module answers by opening that picker, and
4// a click on the Slurm job into one that opens the job panel.
5
6import type { ClientModule } from 'claude-code'
7
8import { cellWidth } from './format'
9import type { Seg } from './format'
10
11type Props = { segs: Seg[]; plain: string }
12type State = { hovered: number | null }
13
14// Which segment the column falls in, by the cells each one takes.
15function segAt(segs: Seg[], x: number): number | null {
16  let left = 0
17  for (let i = 0; i < segs.length; i++) {
18    const right = left + cellWidth(segs[i]!.text)
19    if (x >= left && x < right) return i
20    left = right
21  }
22  return null
23}
24
25const Line1: ClientModule<Props, State> = (props, surface) => {
26  const { Box, Text } = surface.elements
27  const { segs, plain } = props
28  const hovered = surface.state?.hovered ?? null
29
30  const open = (i: number | null) => {
31    const action = i === null ? undefined : segs[i]?.action
32    if (action) surface.post({ open: action })
33  }
34
35  surface.onPointer(e => {
36    const i = segAt(segs, e.x)
37    if (e.type === 'up' && e.button === 'left') open(i)
38    const next = e.type === 'leave' || i === null || !segs[i]?.action ? null : i
39    if (next !== hovered) surface.setState({ hovered: next })
40  })
41
42  // While the band has the focus (ctrl+x tab): m or Enter for the model, e for
43  // effort, j for the Slurm job panel.
44  surface.onKey(e => {
45    if (e.key === 'm' || e.key === 'return') surface.post({ open: 'model' })
46    if (e.key === 'e') surface.post({ open: 'effort' })
47    if (e.key === 'j' && segs.some(seg => seg.action === 'job')) surface.post({ open: 'job' })
48  })
49
50  return (
51    <Box flexDirection="row">
52      {segs.map((seg, i) => (
53        <Text key={String(i)} color={seg.color ?? plain} underline={i === hovered}>
54          {seg.text}
55        </Text>
56      ))}
57    </Box>
58  )
59}
60
61export default Line1
62

1// Terminal charts: braille line charts (2×4 dots per cell) and block sparklines.
2// Pure: series in, rows of coloured runs out; the pane turns runs into Text.
3
4export type Run = { text: string; color: string }
5export type Line = { xs: readonly number[]; ys: readonly number[]; color: string }
6
7const BRAILLE = 0x2800
8// Dot bit for (column 0/1, row 0..3) inside one braille cell.
9const DOT = [
10  [0x01, 0x02, 0x04, 0x40],
11  [0x08, 0x10, 0x20, 0x80],
12]
13
14export type Range = { min: number; max: number }
15
16export function rangeOf(lines: readonly Line[]): { x: Range; y: Range } | null {
17  let xMin = Infinity, xMax = -Infinity, yMin = Infinity, yMax = -Infinity
18  for (const l of lines) {
19    for (let i = 0; i < l.xs.length; i++) {
20      const x = l.xs[i]!, y = l.ys[i]!
21      if (!Number.isFinite(x) || !Number.isFinite(y)) continue
22      xMin = Math.min(xMin, x); xMax = Math.max(xMax, x)
23      yMin = Math.min(yMin, y); yMax = Math.max(yMax, y)
24    }
25  }
26  if (xMin === Infinity) return null
27  if (yMax === yMin) { yMax += Math.abs(yMax) * 0.05 || 1; yMin -= Math.abs(yMin) * 0.05 || 1 }
28  if (xMax === xMin) xMax = xMin + 1
29  return { x: { min: xMin, max: xMax }, y: { min: yMin, max: yMax } }
30}
31
32/**
33 * Draws the lines in `width`×`height` cells, joining consecutive points so a
34 * steep drop reads as a line, not dots. Later lines draw over earlier ones and
35 * a cell takes the colour of the last line that touched it.
36 */
37export function braille(lines: readonly Line[], width: number, height: number, range = rangeOf(lines)): Run[][] {
38  const W = width * 2, H = height * 4
39  const bits = Array.from({ length: height }, () => new Array<number>(width).fill(0))
40  const colors = Array.from({ length: height }, () => new Array<string>(width).fill(''))
41  if (!range) return bits.map(() => [{ text: ' '.repeat(width), color: '' }])
42  const px = (x: number) => Math.round(((x - range.x.min) / (range.x.max - range.x.min)) * (W - 1))
43  const py = (y: number) => (H - 1) - Math.round(((Math.min(range.y.max, Math.max(range.y.min, y)) - range.y.min) / (range.y.max - range.y.min)) * (H - 1))
44  const plot = (x: number, y: number, color: string) => {
45    if (x < 0 || x >= W || y < 0 || y >= H) return
46    const cx = x >> 1, cy = y >> 2
47    bits[cy]![cx]! |= DOT[x & 1]![y & 3]!
48    colors[cy]![cx] = color
49  }
50  for (const l of lines) {
51    let prev: [number, number] | null = null
52    for (let i = 0; i < l.xs.length; i++) {
53      const x = l.xs[i]!, y = l.ys[i]!
54      if (!Number.isFinite(x) || !Number.isFinite(y)) { prev = null; continue }
55      const p: [number, number] = [px(x), py(y)]
56      if (prev) {
57        // Bresenham between the two dots.
58        let [x0, y0] = prev
59        const [x1, y1] = p
60        const dx = Math.abs(x1 - x0), dy = -Math.abs(y1 - y0)
61        const sx = x0 < x1 ? 1 : -1, sy = y0 < y1 ? 1 : -1
62        let err = dx + dy
63        for (;;) {
64          plot(x0, y0, l.color)
65          if (x0 === x1 && y0 === y1) break
66          const e2 = 2 * err
67          if (e2 >= dy) { err += dy; x0 += sx }
68          if (e2 <= dx) { err += dx; y0 += sy }
69        }
70      } else {
71        plot(p[0], p[1], l.color)
72      }
73      prev = p
74    }
75  }
76  return bits.map((row, r) => {
77    const runs: Run[] = []
78    for (let c = 0; c < width; c++) {
79      const ch = row[c] ? String.fromCharCode(BRAILLE + row[c]!) : ' '
80      const color = colors[r]![c]!
81      const last = runs[runs.length - 1]
82      if (last && last.color === color) last.text += ch
83      else runs.push({ text: ch, color })
84    }
85    return runs
86  })
87}
88
89const BLOCKS = '▁▂▃▄▅▆▇█'
90
91/** The last `width` values as one-cell blocks, scaled to their own range. */
92export function sparkline(values: readonly number[], width: number): string {
93  const v = values.filter(Number.isFinite).slice(-width)
94  if (v.length === 0) return ''
95  const min = Math.min(...v), max = Math.max(...v)
96  return v.map(x => BLOCKS[max === min ? 3 : Math.round(((x - min) / (max - min)) * 7)]).join('')
97}
98
99/** 0.000412 → "4.12e-4", 1234.5 → "1234", 0.4123 → "0.412". */
100export function fmtNum(v: number): string {
101  if (!Number.isFinite(v)) return String(v)
102  const a = Math.abs(v)
103  if (a !== 0 && (a < 1e-3 || a >= 1e5)) return v.toExponential(2).replace('e-', 'e-').replace('e+', 'e')
104  if (a >= 100) return v.toFixed(0)
105  if (a >= 10) return v.toFixed(1)
106  return v.toFixed(3)
107}
108
109/** One colour per sweep task, cycling the band's palette. */
110export const SWEEP_COLORS = ['#8abeb7', '#f0c674', '#b294bb', '#b5bd68', '#ff8700', '#81a2be', '#cc6666', '#de935f']
111

1export type Limit = { kind: string; percentUsed: number; resetsAt?: string }
2
3/** A Slurm allocation the session runs inside, from its environment. */
4export type Slurm = {
5  job: string
6  gpus: string | null
7  gpusOnNode: string | null
8  /** When the job ends, in ms: SLURM_JOB_END_TIME, then the panel's scontrol. */
9  endsAt: number | null
10}
11
12export type JobInfo = {
13  id: string
14  name: string
15  state: string
16  partition: string
17  account: string
18  /** The batch host: where `ssh` lands in the job. */
19  node: string
20  nodeList: string
21  runSeconds: number | null
22  limitSeconds: number | null
23  cpus: string | null
24  mem: string | null
25  gpus: string | null
26}
27
28export type GpuProc = { pid: string; name: string; memMiB: number | null }
29
30export type Gpu = {
31  index: string
32  name: string
33  util: number | null
34  memUsedMiB: number | null
35  memTotalMiB: number | null
36  tempC: number | null
37  powerW: number | null
38  powerLimitW: number | null
39  procs: GpuProc[]
40}
41
42/** What the job panel draws; null fields are still loading or unavailable. */
43export type JobPanel = {
44  job: JobInfo | null
45  gpus: Gpu[] | null
46  gpuVia: 'local' | 'ssh' | null
47  isLoading: boolean
48  error: string | null
49  updatedAt: number | null
50}
51
52export type Snapshot = {
53  model: string | null
54  effort: string | null
55  folder: string | null
56  branch: string | null
57  contextPercent: number | null
58  costUsd: number | null
59  startedAt: number | null
60  limits: Limit[]
61  slurm: Slurm | null
62}
63
64
65/** One numeric series from metrics.jsonl: parallel step and value arrays. */
66export type Series = { step: number[]; value: number[] }
67
68/** What pace-line has read of runs/<jobid>/metrics.jsonl. */
69export type Metrics = {
70  /** Bytes read so far: the next read starts here. */
71  offset: number
72  keys: Record<string, Series>
73  lastStep: number | null
74  totalSteps: number | null
75  epoch: number | null
76  totalEpochs: number | null
77  firstTime: number | null
78  firstStep: number | null
79  lastTime: number | null
80  /** The first metric that went NaN or infinite, as `key@step`. */
81  nonFinite: string | null
82}
83
84export type Ckpt = {
85  name: string
86  path: string
87  mtimeMs: number
88  sizeBytes: number | null
89  step: number | null
90  epoch: number | null
91}
92
93export type NodeGpus = { node: string; gpus: Gpu[] }
94
95/** Lines from the log that mean trouble, newest first kept. */
96export type LogFlags = { oom: string | null; traceback: string | null; nccl: string | null; srun: string | null }
97
98/** A batch job of the user's, as squeue, scontrol, sacct and its files tell it. */
99export type BatchJob = {
100  /** As squeue shows it: "123" or "123_4" for an array task. */
101  id: string
102  /** The unique id ($SLURM_JOB_ID inside the job), which names runs/<id>/. */
103  jobId: string
104  arrayId: string | null
105  name: string
106  state: string
107  reason: string | null
108  elapsedS: number | null
109  limitS: number | null
110  nodes: number
111  nodeList: string
112  gpus: number | null
113  gpuType: string | null
114  submittedAt: number | null
115  /** When pace-line first saw it running, less its elapsed time. */
116  startedAt: number | null
117  workDir: string | null
118  stdout: string | null
119  command: string | null
120  submitLine: string | null
121  endedAt: number | null
122  exitCode: string | null
123  logSize: number | null
124  logChangedAt: number | null
125  lastLines: string[]
126  progress: { k: number; n: number } | null
127  flags: LogFlags
128  metrics: Metrics | null
129  ckpt: Ckpt | null
130  gpuReadings: NodeGpus[] | null
131  gpusAt: number | null
132  idleSince: number | null
133  resumedFrom: string | null
134  resumedAs: string | null
135}
136
137export type Disk = { mount: string; size: number; used: number; avail: number }
138
139export type AlertKind = 'ended' | 'oom' | 'traceback' | 'nccl' | 'nonFinite' | 'silent' | 'idle' | 'pending' | 'nearLimit' | 'staleCkpt' | 'resumed' | 'resumeStopped'
140
141export type Alert = { job: string; kind: AlertKind; level: 'info' | 'warn' | 'error'; text: string; at: number }
142
143/** Every batch job pane and band draw. */
144export type Jobs = {
145  jobs: BatchJob[]
146  /** A job id, or `array:<id>` for a sweep. */
147  selected: string | null
148  updatedAt: number | null
149  error: string | null
150  disks: Disk[]
151  alerts: Alert[]
152}
153
154declare module 'claude-code' {
155  interface PluginState {
156    'pace-line': { snapshot: Snapshot; now: number; panel: JobPanel; warnedJob: string | null; jobs: Jobs; tab: 'node' | 'jobs' }
157  }
158}
159