job-watch

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { Alert, BatchJob, Conn } from '../types'
5import { DEFAULT_THRESHOLDS, alertsFor, label } from './jobs/alerts'
6import type { Thresholds } from './jobs/alerts'
7import { resumeArgv, submittedId } from './jobs/parse'
8import { EMPTY_JOBS, addHistory, applyFiles, applyGpus, filesWant, fullWant, gpuSample, hasGpus, mergeQueue, runDir } from './merge'
9import { EMPTY_WANT, buildScript, shq, splitSections, sshArgv, sshError } from './remote'
10import type { Section, Want } from './remote'
11import { paneTree, statusText } from './view'
12
13// ── Watching the cluster from here ──────────────────────────────────────
14// Every poll is one `ssh <host> bash -s`: squeue, scontrol for new jobs, sacct
15// for jobs that left the queue, each running job's log tail, new metrics lines,
16// checkpoints and GPUs. A job seen running for the first time gets its files in
17// a second, smaller round trip. Alerts are toasts; nothing is resubmitted on
18// its own (the cluster's pace-line does that), only from the pane's buttons.
19// All of it lives here: a hooks module hands $ only to functions declared in
20// its own file.
21
22const PANE = 'job-watch'
23const NO_CONN: Conn = { host: 'explorer', isOk: null, error: null, lastOkAt: null, latencyMs: null, failures: 0, isPolling: false }
24
25const jobsAtom = atom({ plugin: 'job-watch', key: 'jobs' } as const, EMPTY_JOBS)
26const connAtom = atom({ plugin: 'job-watch', key: 'conn' } as const, NO_CONN)
27const historyAtom = atom({ plugin: 'job-watch', key: 'history' } as const, {})
28const nowAtom = atom({ plugin: 'job-watch', key: 'now' } as const, 0)
29
30type Config = Thresholds & {
31  host: string
32  pollSeconds: number
33  gpuSeconds: number
34  historySamples: number
35  exclude: string[]
36}
37
38function configFrom(options: Readonly<Record<string, unknown>>): Config {
39  const num = (k: string, d: number) => (typeof options[k] === 'number' && (options[k] as number) > 0 ? (options[k] as number) : d)
40  const str = (k: string, d: string) => (typeof options[k] === 'string' && (options[k] as string).trim() ? (options[k] as string).trim() : d)
41  return {
42    host: str('host', 'explorer'),
43    pollSeconds: Math.max(15, num('pollSeconds', 60)),
44    gpuSeconds: Math.max(10, num('gpuSeconds', 30)),
45    historySamples: Math.min(5000, num('historySamples', 480)),
46    idleMinutes: num('idleMinutes', DEFAULT_THRESHOLDS.idleMinutes),
47    silentMinutes: num('silentMinutes', DEFAULT_THRESHOLDS.silentMinutes),
48    nearLimitMinutes: num('nearLimitMinutes', DEFAULT_THRESHOLDS.nearLimitMinutes),
49    staleCkptMinutes: num('staleCkptMinutes', DEFAULT_THRESHOLDS.staleCkptMinutes),
50    pendingMinutes: num('pendingMinutes', DEFAULT_THRESHOLDS.pendingMinutes),
51    exclude: str('excludeNames', 'dev-shell,interactive').split(',').map(s => s.trim()).filter(Boolean),
52  }
53}
54
55const DISK_MS = 10 * 60_000
56const TICK_MS = 15_000
57const MAX_BACKOFF_S = 600
58const JOB_ID = /^\d+(_\d+)?$/
59
60let cfg: Config = configFrom({})
61let polling = false
62let paneOpen = false
63let disksAt = 0
64/** No full poll before this, after failures. */
65let nextFullAt = 0
66/** squeue ids scontrol said are interactive: nothing to watch in them. */
67const ignored = new Set<string>()
68
69const nonce = () => Math.random().toString(36).slice(2, 10)
70
71type Reply = { sections: Section[]; isComplete: boolean; latencyMs: number } | { error: string }
72
73/** One round trip: the script for `want` on stdin, its sections back. */
74async function roundTrip($: EngineInterface, want: Want): Promise<Reply> {
75  const t0 = await $.clock.now()
76  try {
77    const r = await $.process.run(sshArgv(cfg.host, ['bash', '-s']), { stdin: buildScript(want), timeoutMs: 90_000 })
78    if (!r.stdout.includes(`@@JW:${want.nonce}:`)) return { error: sshError(r.stderr, r.exitCode) }
79    return { ...splitSections(r.stdout, want.nonce), latencyMs: (await $.clock.now()) - t0 }
80  } catch (err) {
81    return { error: err instanceof Error ? err.message.slice(0, 200) : 'ssh did not finish' }
82  }
83}
84
85async function failed($: EngineInterface, error: string): Promise<void> {
86  const t = await $.clock.now()
87  let failures = 0
88  await update($, connAtom, c => {
89    failures = c.failures + 1
90    return { ...c, host: cfg.host, isOk: false, error, failures }
91  })
92  nextFullAt = t + Math.min(MAX_BACKOFF_S, cfg.pollSeconds * 2 ** Math.min(failures, 6)) * 1000
93}
94
95async function reached($: EngineInterface, latencyMs: number): Promise<void> {
96  const t = await $.clock.now()
97  nextFullAt = 0
98  await update($, connAtom, c => ({ ...c, host: cfg.host, isOk: true, error: null, lastOkAt: t, latencyMs, failures: 0 }))
99}
100
101async function setPolling($: EngineInterface, isPolling: boolean): Promise<void> {
102  await update($, connAtom, c => ({ ...c, isPolling }))
103}
104
105/** One pass: queue and details, files of running jobs, GPUs; then files of jobs first seen running. */
106async function poll($: EngineInterface, isForced = false): Promise<void> {
107  if (polling) return
108  const t = await $.clock.now()
109  if (!isForced && t < nextFullAt) return
110  polling = true
111  await setPolling($, true)
112  try {
113    const s = await read($, jobsAtom)
114    const isDfDue = t - disksAt >= DISK_MS
115    const want = fullWant(s, { nonce: nonce(), ignored: [...ignored], exclude: cfg.exclude, gpuIds: s.jobs.filter(hasGpus).map(j => j.id), isDfDue })
116    const res = await roundTrip($, want)
117    if ('error' in res) {
118      await failed($, res.error)
119      return
120    }
121    await reached($, res.latencyMs)
122    if (isDfDue) disksAt = t
123
124    const q = mergeQueue(s, res.sections, t, { ignored: [...ignored], exclude: cfg.exclude })
125    for (const id of q.ignored) ignored.add(id)
126    const read1 = new Set([...want.logs, ...want.metrics, ...want.ckpts].map(x => x.id))
127    const gpus1 = new Set(want.gpus.map(g => g.id))
128    let jobs = q.jobs.map(j => (read1.has(j.id) ? applyFiles(j, res.sections) : j)).map(j => (gpus1.has(j.id) ? applyGpus(j, res.sections, want.nonce, t) : j))
129
130    // Jobs scontrol just described as running, or that ended unread: their files now, not a poll later.
131    const follow = jobs.filter(j => (q.described.includes(j.id) || (q.ended.includes(j.id) && !read1.has(j.id))) && runDir(j) !== null)
132    if (follow.length) {
133      const w2: Want = { ...EMPTY_WANT, nonce: nonce(), ...filesWant(follow), gpus: follow.filter(hasGpus).map(j => ({ id: j.id, nodeList: j.nodeList })) }
134      const r2 = await roundTrip($, w2)
135      if (!('error' in r2)) {
136        const ids = new Set(follow.map(j => j.id))
137        const g2 = new Set(w2.gpus.map(g => g.id))
138        jobs = jobs.map(j => (ids.has(j.id) ? applyFiles(j, r2.sections) : j)).map(j => (g2.has(j.id) ? applyGpus(j, r2.sections, w2.nonce, t) : j))
139      }
140    }
141
142    await update($, jobsAtom, x => ({
143      ...x,
144      jobs,
145      selected: x.selected && jobs.some(j => j.id === x.selected || `array:${j.arrayId}` === x.selected) ? x.selected : jobs.find(j => j.endedAt === null)?.id ?? jobs[0]?.id ?? null,
146      updatedAt: t,
147      error: q.error ?? (res.isComplete ? null : 'The reply was cut short; the rest comes next poll.'),
148      disks: q.disks ?? x.disks,
149    }))
150    await record($, jobs)
151    await deliver($, await unseen($, jobs.flatMap(j => alertsFor(j, t, cfg))))
152  } finally {
153    polling = false
154    await setPolling($, false)
155    await showStatus($)
156  }
157}
158
159/** The selected job's GPUs only, while the pane is open. */
160async function pollGpus($: EngineInterface): Promise<void> {
161  if (polling || !paneOpen) return
162  const s = await read($, jobsAtom)
163  const sel = s.jobs.find(j => j.id === s.selected && hasGpus(j))
164  if (!sel || ((await $.clock.now()) - (sel.gpusAt ?? 0)) < cfg.gpuSeconds * 500) return
165  polling = true
166  try {
167    const want: Want = { ...EMPTY_WANT, nonce: nonce(), gpus: [{ id: sel.id, nodeList: sel.nodeList }] }
168    const res = await roundTrip($, want)
169    if ('error' in res) return
170    const t = await $.clock.now()
171    const fresh = applyGpus(sel, res.sections, want.nonce, t)
172    const patch = { gpuReadings: fresh.gpuReadings, gpusAt: fresh.gpusAt, idleSince: fresh.idleSince }
173    await update($, jobsAtom, x => ({ ...x, jobs: x.jobs.map(j => (j.id === sel.id ? { ...j, ...patch } : j)) }))
174    await record($, [fresh])
175  } finally {
176    polling = false
177  }
178}
179
180/** Adds each job's newest GPU reading to its history, kept in the store across sessions. */
181async function record($: EngineInterface, jobs: readonly BatchJob[]): Promise<void> {
182  const adds = jobs.flatMap(j => {
183    const sample = gpuSample(j)
184    return sample ? [{ id: j.id, sample }] : []
185  })
186  const keep = new Set((await read($, jobsAtom)).jobs.map(j => j.id))
187  const h = addHistory(await read($, historyAtom), adds, keep, cfg.historySamples)
188  await update($, historyAtom, () => h)
189  try {
190    await $.store.set('history', h)
191  } catch {
192    // Too big for the store, or none here: the session's copy stands.
193  }
194}
195
196/** Alerts not yet raised for that job and kind (the store remembers a week). */
197async function unseen($: EngineInterface, alerts: Alert[]): Promise<Alert[]> {
198  if (!alerts.length) return []
199  const t = await $.clock.now()
200  const seen = ((await $.store.get('alerted')) ?? {}) as Record<string, number>
201  const out = alerts.filter(a => seen[`${a.job}:${a.kind}`] === undefined)
202  if (!out.length) return []
203  const kept = Object.fromEntries(Object.entries(seen).filter(([, at]) => t - at < 7 * 86400_000))
204  for (const a of out) kept[`${a.job}:${a.kind}`] = t
205  await $.store.set('alerted', kept)
206  return out
207}
208
209/** A toast per alert, and the pane's list. No Discord, no message to Claude: the cluster's pace-line sends those. */
210async function deliver($: EngineInterface, alerts: Alert[]): Promise<void> {
211  if (!alerts.length) return
212  for (const a of alerts) $.ui.toast(a.text, { timeoutMs: a.level === 'error' ? 30_000 : 15_000 })
213  await update($, jobsAtom, s => ({ ...s, alerts: [...alerts, ...s.alerts].slice(0, 20) }))
214}
215
216async function showStatus($: EngineInterface): Promise<void> {
217  const t = await $.clock.now()
218  $.ui.status(statusText(await read($, jobsAtom), await read($, connAtom), t))
219}
220
221async function confirm($: EngineInterface, question: string, header: string, yes: string): Promise<boolean> {
222  try {
223    return (await $.ui.ask(question, { header, options: [yes, 'Keep it'] })) === yes
224  } catch {
225    return false
226  }
227}
228
229/** One command on the login node; its stdout, or null with the reason. */
230async function remote($: EngineInterface, command: string): Promise<{ out: string | null; error: string }> {
231  try {
232    const r = await $.process.run(sshArgv(cfg.host, [command]), { timeoutMs: 60_000 })
233    return r.exitCode === 0 ? { out: r.stdout, error: '' } : { out: null, error: sshError(r.stderr, r.exitCode) }
234  } catch (err) {
235    return { out: null, error: err instanceof Error ? err.message : 'ssh failed' }
236  }
237}
238
239async function cancelJob($: EngineInterface, j: BatchJob): Promise<void> {
240  if (!JOB_ID.test(j.jobId)) return void $.ui.toast(`Not a job id: ${j.jobId}`)
241  if (!(await confirm($, `Cancel ${label(j)} on ${cfg.host}?`, 'Cancel job', 'Cancel the job'))) return
242  const r = await remote($, `scancel ${j.jobId}`)
243  $.ui.toast(r.out !== null ? `Cancelled ${label(j)}.` : `scancel ${j.jobId} failed: ${r.error}`, { timeoutMs: 15_000 })
244  await poll($, true)
245}
246
247/** Resubmits an ended job from its newest checkpoint with RESUME=<path>, as pace-line's /jobs resume does. */
248async function resumeJob($: EngineInterface, j: BatchJob): Promise<void> {
249  const no = (why: string) => void $.ui.toast(`↻ ${label(j)}: not resubmitted, ${why}`, { timeoutMs: 15_000 })
250  if (j.arrayId) return no('it is an array task; resubmit the sweep by hand')
251  const ckpt = j.ckpt
252  if (!ckpt || (j.startedAt !== null && ckpt.mtimeMs < j.startedAt)) return no(`no checkpoint written by this run in ${runDir(j) ?? 'its run dir'}`)
253  if (!j.submitLine) return no('sacct gave no submit line')
254  if (!j.workDir) return no('its work dir is unknown')
255  const argv = resumeArgv(j.submitLine, ckpt.path)
256  if (!argv) return no(`its submit line can't be replayed: ${j.submitLine.slice(0, 120)}`)
257  if (!(await confirm($, `Resubmit ${label(j)} on ${cfg.host} from ${ckpt.name}?\n\n${argv.join(' ')}`, 'Resume', 'Resubmit'))) return
258  const r = await remote($, `cd ${shq(j.workDir)} && ${argv.map(shq).join(' ')}`)
259  const id = r.out === null ? null : submittedId(r.out)
260  if (!id) return no(`sbatch refused: ${r.error || argv.join(' ').slice(0, 160)}`)
261  await update($, jobsAtom, s => ({ ...s, jobs: s.jobs.map(x => (x.id === j.id ? { ...x, resumedAs: id } : x)) }))
262  $.ui.toast(`↻ ${label(j)}: resubmitted as ${id} from ${ckpt.name}`, { timeoutMs: 15_000 })
263  await poll($, true)
264}
265
266async function openPane($: EngineInterface): Promise<void> {
267  const opened = await $.ui.open({ id: PANE, title: `Jobs · ${cfg.host}` })
268  if (!opened.isPlaced) $.ui.toast('Type /jobs to open the pane here')
269  paneOpen = true
270  const t = await $.clock.now()
271  await update($, nowAtom, () => t)
272  void pollGpus($)
273}
274
275export const register: Register = (on, options) => {
276  cfg = configFrom(options)
277
278  on('session.start', async ($, e, next) => {
279    const started = await next(e)
280    await update($, connAtom, c => ({ ...c, host: cfg.host }))
281    try {
282      const h = await $.store.get('history')
283      if (h && typeof h === 'object') await update($, historyAtom, () => h as Record<string, never[]>)
284    } catch {
285      // No store: history starts empty.
286    }
287    await $.command.register({
288      name: 'jobs',
289      description: `Your Slurm jobs on ${cfg.host}: progress, loss and val curves, GPU history, sweeps`,
290      argumentHint: '[job id]',
291    })
292    // Not awaited: the first ssh shouldn't hold the session's start.
293    void poll($, true)
294    $.clock.every(cfg.pollSeconds * 1000, () => poll($))
295    $.clock.every(cfg.gpuSeconds * 1000, () => pollGpus($))
296    $.clock.every(TICK_MS, async () => {
297      if (!paneOpen) return
298      const t = await $.clock.now()
299      await update($, nowAtom, () => t)
300    })
301    return started
302  })
303
304  on('command.run', { command: 'jobs' }, async ($, e) => {
305    const id = e.args.trim()
306    if (id) {
307      const s = await read($, jobsAtom)
308      const j = s.jobs.find(x => x.id === id || x.jobId === id || x.arrayId === id)
309      if (!j) return { text: `No job ${id} among your jobs on ${cfg.host}.` }
310      await update($, jobsAtom, x => ({ ...x, selected: j.arrayId === id ? `array:${id}` : j.id }))
311    }
312    await openPane($)
313    const c = await read($, connAtom)
314    return { text: c.isOk === false ? `Opened the jobs pane; ${cfg.host} is unreachable: ${c.error ?? ''}` : 'Opened the jobs pane.' }
315  })
316
317  on('ui.close', { id: PANE }, ($, e, next) => {
318    paneOpen = false
319    return next(e)
320  }).catch(($, e, next) => next(e))
321
322  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
323    const el = $.ui.resolve(e)
324    // The terminal draws no vectors: braille charts there.
325    const kit = { Box: el.Box, Text: el.Text, Button: el.Button, Svg: e.surface !== 'terminal' && 'Svg' in el ? el.Svg : undefined }
326    const d = {
327      s: await read($, jobsAtom),
328      conn: await read($, connAtom),
329      history: await read($, historyAtom),
330      now: (await read($, nowAtom)) || (await $.clock.now()),
331    }
332    return paneTree(kit, d, e.props.bodyColumns, {
333      select: key => {
334        void update($, jobsAtom, x => ({ ...x, selected: key })).then(() => pollGpus($))
335      },
336      copy: async text => {
337        const r = await $.ui.copy({ text, surface: e.surface })
338        $.ui.toast(r.isCopied ? `Copied: ${text}` : `Couldn't copy: ${r.reason}`)
339      },
340      resume: j => void resumeJob($, j),
341      cancel: j => void cancelJob($, j),
342      refresh: () => void poll($, true),
343    })
344  })
345}
346

1// Copied from plugins/pace-line/hooks/jobs/alerts.ts by scripts/sync-shared.sh: edit it there.
2// The alert rules: pure functions of a job, the time and the thresholds.
3// Each kind fires once per job; watch.ts remembers which already did.
4
5import type { Alert, AlertKind, BatchJob } from '../../types'
6import { fmtDur } from '../format'
7
8export type Thresholds = {
9  idleMinutes: number
10  silentMinutes: number
11  nearLimitMinutes: number
12  staleCkptMinutes: number
13  pendingMinutes: number
14}
15
16export const DEFAULT_THRESHOLDS: Thresholds = {
17  idleMinutes: 10,
18  silentMinutes: 20,
19  nearLimitMinutes: 30,
20  staleCkptMinutes: 60,
21  pendingMinutes: 120,
22}
23
24/** States a job leaves squeue in; COMPLETED is the only good one. */
25export const TERMINAL = new Set(['COMPLETED', 'FAILED', 'TIMEOUT', 'CANCELLED', 'OUT_OF_MEMORY', 'NODE_FAIL', 'PREEMPTED', 'BOOT_FAIL', 'DEADLINE'])
26
27/** "t4-train (123)" */
28export function label(job: BatchJob): string {
29  return `${job.name} (${job.id})`
30}
31
32const ageMin = (now: number, t: number | null) => (t === null ? null : (now - t) / 60_000)
33
34export function alertsFor(job: BatchJob, now: number, th: Thresholds): Alert[] {
35  const out: Alert[] = []
36  const add = (kind: AlertKind, level: Alert['level'], text: string) => out.push({ job: job.id, kind, level, text, at: now })
37  const running = job.state === 'RUNNING'
38  const lastErr = job.flags.oom ?? job.flags.nccl ?? job.flags.traceback ?? job.flags.srun
39
40  if (TERMINAL.has(job.state)) {
41    const took = job.elapsedS !== null ? ` after ${fmtDur(job.elapsedS)}` : ''
42    if (job.state === 'COMPLETED') add('ended', 'info', `✓ ${label(job)} completed${took}`)
43    else add('ended', 'error', `✗ ${label(job)} ${job.state}${took}${job.exitCode && job.exitCode !== '0:0' ? ` (exit ${job.exitCode})` : ''}${lastErr ? `: ${lastErr}` : ''}`)
44    return out
45  }
46
47  if (job.flags.oom) add('oom', 'error', `✗ ${label(job)} ran out of memory: ${job.flags.oom}`)
48  if (job.flags.nccl) add('nccl', 'error', `✗ ${label(job)} NCCL trouble: ${job.flags.nccl}`)
49  if (job.flags.traceback && running) add('traceback', 'error', `✗ ${label(job)} raised ${job.flags.traceback}`)
50  if (job.metrics?.nonFinite) add('nonFinite', 'error', `✗ ${label(job)}: ${job.metrics.nonFinite.replace('@', ' went NaN/inf at step ')}`)
51
52  if (job.state === 'PENDING') {
53    const waited = ageMin(now, job.submittedAt)
54    if (waited !== null && waited >= th.pendingMinutes) {
55      add('pending', 'warn', `◌ ${label(job)} pending ${fmtDur(waited * 60)}${job.reason ? ` (${job.reason})` : ''}`)
56    }
57  }
58
59  if (running) {
60    const ranMin = ageMin(now, job.startedAt)
61    const quiet = ageMin(now, job.logChangedAt ?? job.startedAt)
62    if (ranMin !== null && ranMin >= th.silentMinutes && quiet !== null && quiet >= th.silentMinutes) {
63      add('silent', 'warn', `⚠ ${label(job)}: log silent for ${fmtDur(quiet * 60)} (hung rank?)`)
64    }
65    const idle = ageMin(now, job.idleSince)
66    if (idle !== null && idle >= th.idleMinutes) {
67      add('idle', 'warn', `⚠ ${label(job)}: allocated GPU idle for ${fmtDur(idle * 60)}`)
68    }
69    const ckptAge = job.ckpt ? ageMin(now, job.ckpt.mtimeMs) : null
70    if (job.limitS !== null && job.elapsedS !== null) {
71      const left = job.limitS - job.elapsedS
72      // A quarter of the limit at most: an 8-minute job isn't "near" its end at start.
73      if (left <= Math.min(th.nearLimitMinutes * 60, job.limitS / 4)) {
74        const ck = job.ckpt
75          ? `last checkpoint ${fmtDur((ckptAge ?? 0) * 60)} ago`
76          : 'no checkpoint found'
77        const stale = !job.ckpt || (ckptAge ?? 0) >= th.staleCkptMinutes
78        add('nearLimit', stale ? 'error' : 'warn', `⌛ ${label(job)}: ${fmtDur(left)} left, ${ck}`)
79      }
80    }
81    if (job.ckpt && ckptAge !== null && ckptAge >= th.staleCkptMinutes && ranMin !== null && ranMin >= th.staleCkptMinutes) {
82      add('staleCkpt', 'warn', `⚠ ${label(job)}: no new checkpoint for ${fmtDur(ckptAge * 60)}`)
83    }
84  }
85  return out
86}
87
88/** True when every allocated GPU reads 0% with no process. */
89export function allIdle(job: BatchJob): boolean {
90  const gpus = job.gpuReadings?.flatMap(n => n.gpus) ?? []
91  return gpus.length > 0 && gpus.every(g => (g.util ?? 0) === 0 && g.procs.length === 0)
92}
93
94/** True when some allocated GPU reads 0% while others work: a stuck or missing rank. */
95export function someIdle(job: BatchJob): boolean {
96  const gpus = job.gpuReadings?.flatMap(n => n.gpus) ?? []
97  return gpus.length > 1 && gpus.some(g => (g.util ?? 0) === 0) && gpus.some(g => (g.util ?? 0) > 0)
98}
99

1// Copied from plugins/pace-line/hooks/jobs/parse.ts by scripts/sync-shared.sh: edit it there.
2// Parsers for what the batch-job watcher reads: squeue, scontrol, sacct, df,
3// a log's tail and an sbatch submit line. watch.ts runs the commands.
4
5import type { Disk, LogFlags } from '../../types'
6import { slurmDuration } from '../slurm'
7
8/** The squeue format the watcher asks for, one job per line, `|`-separated. */
9export const SQUEUE_FORMAT = '%i|%j|%T|%M|%l|%D|%R|%b|%N|%V'
10
11export type QueueRow = {
12  id: string
13  arrayId: string | null
14  name: string
15  state: string
16  elapsedS: number | null
17  limitS: number | null
18  nodes: number
19  /** The pending reason; for a running job squeue puts the node list here. */
20  reason: string | null
21  gpus: number | null
22  gpuType: string | null
23  nodeList: string
24  /** As squeue prints it, local time without a zone. */
25  submitTime: string
26}
27
28/** "gres/gpu:h200:4", "gres:gpu:4", "gres/gpu=2" or "N/A" → count and type. */
29export function gpusFromTres(tres: string): { gpus: number | null; gpuType: string | null } {
30  const m = /gpu(?::([a-zA-Z][\w.-]*))?[:=](\d+)/.exec(tres)
31  if (!m) return { gpus: null, gpuType: null }
32  return { gpus: Number(m[2]), gpuType: m[1] ? m[1].toUpperCase() : null }
33}
34
35export function parseSqueue(out: string): QueueRow[] {
36  const rows: QueueRow[] = []
37  for (const line of out.split('\n')) {
38    if (!line.trim()) continue
39    const [id = '', name = '', state = '', elapsed = '', limit = '', nodes = '', reason = '', tres = '', nodeList = '', submit = ''] = line.split('|')
40    const arr = /^(\d+)_(\d+|\[.*\])$/.exec(id)
41    const { gpus, gpuType } = gpusFromTres(tres)
42    rows.push({
43      id,
44      arrayId: arr ? arr[1]! : null,
45      name,
46      state,
47      elapsedS: slurmDuration(elapsed),
48      limitS: slurmDuration(limit),
49      nodes: Number(nodes) || 1,
50      reason: state === 'PENDING' ? reason.replace(/^\(|\)$/g, '') || null : null,
51      gpus,
52      gpuType,
53      nodeList: state === 'PENDING' ? '' : nodeList,
54      submitTime: submit,
55    })
56  }
57  return rows
58}
59
60/** `scontrol show job` as Key=Value; the first value wins (JobId before ArrayJobId's). */
61export function scontrolFields(out: string): Record<string, string> {
62  const kv: Record<string, string> = {}
63  for (const m of out.matchAll(/(?:^|\s)(\w[\w/:]*)=(\S*)/g)) kv[m[1]!] ??= m[2]!
64  return kv
65}
66
67export type JobDetail = {
68  jobId: string
69  isBatch: boolean
70  workDir: string | null
71  stdout: string | null
72  command: string | null
73  batchHost: string | null
74  nodeList: string
75}
76
77export function parseJobDetail(out: string): JobDetail | null {
78  const kv = scontrolFields(out)
79  if (!kv.JobId) return null
80  const val = (v: string | undefined) => (v && v !== '(null)' ? v : null)
81  return {
82    jobId: kv.JobId,
83    isBatch: kv.BatchFlag === '1',
84    workDir: val(kv.WorkDir),
85    stdout: val(kv.StdOut),
86    command: val(kv.Command),
87    batchHost: val(kv.BatchHost),
88    nodeList: val(kv.NodeList) ?? '',
89  }
90}
91
92export const SACCT_FIELDS = 'JobID,JobName,State,ExitCode,Elapsed,WorkDir,SubmitLine'
93
94export type Accounting = {
95  jobId: string
96  state: string
97  exitCode: string
98  elapsedS: number | null
99  workDir: string | null
100  submitLine: string | null
101}
102
103/**
104 * `sacct -X -n -P -o <SACCT_FIELDS>` for one job. SubmitLine is last and may
105 * itself hold `|` or newlines (an --wrap script), so it takes all the rest.
106 * The state can read "CANCELLED by 1000"; only the first word is kept.
107 */
108export function parseSacct(out: string): Accounting | null {
109  const text = out.replace(/\n+$/, '')
110  if (!text.trim()) return null
111  const parts = text.split('|')
112  if (parts.length < 7) return null
113  const [jobId = '', , state = '', exitCode = '', elapsed = '', workDir = ''] = parts
114  const submitLine = parts.slice(6).join('|').trim()
115  return {
116    jobId,
117    state: state.split(' ')[0] ?? state,
118    exitCode,
119    elapsedS: slurmDuration(elapsed),
120    workDir: workDir || null,
121    submitLine: submitLine || null,
122  }
123}
124
125/** `date +%z` → minutes east of UTC: "-0400" → -240. */
126export function zoneMinutes(z: string): number {
127  const m = /^([+-])(\d\d)(\d\d)$/.exec(z.trim())
128  return m ? (m[1] === '-' ? -1 : 1) * (Number(m[2]) * 60 + Number(m[3])) : 0
129}
130
131/** squeue's "2026-10-07T11:54:38" in the cluster's zone → epoch ms; null for N/A. */
132export function slurmTime(text: string, zone: number): number | null {
133  const ms = Date.parse(`${text}Z`)
134  return Number.isNaN(ms) ? null : ms - zone * 60_000
135}
136
137/** `df -B1 --output=target,size,used,avail <paths>`. */
138export function parseDf(out: string): Disk[] {
139  return out.split('\n').slice(1).filter(l => l.trim()).map(l => {
140    const [mount = '', size = '0', used = '0', avail = '0'] = l.trim().split(/\s+/)
141    return { mount, size: Number(size), used: Number(used), avail: Number(avail) }
142  })
143}
144
145/** Lines that say nothing about the job's progress. */
146const NOISE = [
147  /^\s*$/,
148  /Kwargs passed to/,
149  /(Future|User|Deprecation|Runtime)Warning/,
150  /^\s*warnings\.warn/,
151  /^\s*\d+%\|/, // a tqdm bar with no description: kept as progress, not as a line
152]
153
154export type LogTail = { lines: string[]; progress: { k: number; n: number } | null; flags: LogFlags }
155
156/**
157 * The tail of a job's log: tqdm's carriage returns split into lines, noise
158 * dropped, the last `[k/N]` or tqdm `k/N [` as progress, and trouble flagged.
159 */
160export function parseLogTail(text: string, keep = 3): LogTail {
161  const raw = text.replace(/\r\n/g, '\n').split(/[\r\n]/)
162  let progress: { k: number; n: number } | null = null
163  const flags: LogFlags = { oom: null, traceback: null, nccl: null, srun: null }
164  let inTraceback = false
165  for (let i = 0; i < raw.length; i++) {
166    const line = raw[i]!
167    const p = /\[(\d+)\/(\d+)\]/.exec(line) ?? /\b(\d+)\/(\d+) \[/.exec(line)
168    if (p && Number(p[2]) > 0) progress = { k: Number(p[1]), n: Number(p[2]) }
169    if (/CUDA out of memory|OutOfMemoryError|oom-kill|Out Of Memory/i.test(line)) flags.oom = line.trim()
170    if (/NCCL (error|WARN.*[Tt]imeout)|Watchdog caught collective operation timeout|ProcessGroupNCCL.*(timeout|error)/i.test(line)) flags.nccl = line.trim()
171    if (/^srun: error:/.test(line)) flags.srun = line.trim()
172    if (/^Traceback \(most recent call last\)/.test(line)) inTraceback = true
173    else if (inTraceback && /^\S/.test(line) && !/^(  |During handling|The above exception)/.test(line)) {
174      // The first unindented line after the frames: "ValueError: ...".
175      flags.traceback = line.trim()
176      inTraceback = false
177    }
178  }
179  const lines = raw.filter(l => !NOISE.some(r => r.test(l))).map(l => l.trimEnd()).slice(-keep)
180  return { lines, progress, flags }
181}
182
183/** Splits a command line as a POSIX shell would for plain words, quotes and escapes. */
184export function shellWords(line: string): string[] {
185  const words: string[] = []
186  let cur = ''
187  let has = false
188  let quote: '"' | "'" | null = null
189  for (let i = 0; i < line.length; i++) {
190    const c = line[i]!
191    if (quote) {
192      if (c === quote) quote = null
193      else if (c === '\\' && quote === '"' && i + 1 < line.length) cur += line[++i]
194      else cur += c
195    } else if (c === '"' || c === "'") {
196      quote = c
197      has = true
198    } else if (c === '\\' && i + 1 < line.length) {
199      cur += line[++i]
200      has = true
201    } else if (/\s/.test(c)) {
202      if (has || cur) words.push(cur)
203      cur = ''
204      has = false
205    } else {
206      cur += c
207    }
208  }
209  if (has || cur) words.push(cur)
210  return words
211}
212
213/**
214 * The submit line as argv with RESUME=<ckpt> added to its --export (or an
215 * `--export=ALL,RESUME=…` added after `sbatch`), any earlier RESUME replaced.
216 * Null when the line is not a plain `sbatch … script` (a --wrap, an array, a pipe).
217 */
218export function resumeArgv(submitLine: string, ckpt: string): string[] | null {
219  if (/\n|[|;&<>`]|\$\(/.test(submitLine)) return null
220  const argv = shellWords(submitLine)
221  if (argv[0] !== 'sbatch' && !argv[0]?.endsWith('/sbatch')) return null
222  if (argv.some(a => a === '--wrap' || a.startsWith('--wrap=') || a === '-a' || a === '--array' || a.startsWith('--array='))) return null
223  const set = (list: string) => [...list.split(',').filter(v => v && !v.startsWith('RESUME=')), `RESUME=${ckpt}`].join(',')
224  const out = argv.filter(a => a !== '--parsable')
225  for (let i = 1; i < out.length; i++) {
226    const a = out[i]!
227    if (a.startsWith('--export=')) {
228      out[i] = `--export=${set(a.slice('--export='.length))}`
229      return ['sbatch', '--parsable', ...out.slice(1)]
230    }
231    if (a === '--export' && i + 1 < out.length) {
232      out[i + 1] = set(out[i + 1]!)
233      return ['sbatch', '--parsable', ...out.slice(1)]
234    }
235  }
236  return ['sbatch', '--parsable', `--export=ALL,RESUME=${ckpt}`, ...out.slice(1)]
237}
238
239/** "Submitted batch job 123" or a --parsable "123;cluster" → "123". */
240export function submittedId(stdout: string): string | null {
241  const m = /Submitted batch job (\d+)/.exec(stdout) ?? /^(\d+)(?:;\S+)?\s*$/m.exec(stdout)
242  return m ? m[1]! : null
243}
244

1// From what the jobs state knows to what a poll asks for, and from the reply's
2// sections back to jobs, with pace-line's parsers doing the reading. Pure: the
3// same steps as pace-line's poll, minus the commands, so they test without ssh.
4
5import type { BatchJob, Ckpt, Disk, GpuSample, Jobs, NodeGpus } from '../types'
6import { TERMINAL } from './jobs/alerts'
7import { EMPTY_METRICS, addChunk, newestCkpt } from './jobs/metrics'
8import { parseDf, parseJobDetail, parseLogTail, parseSacct, parseSqueue, slurmTime, zoneMinutes } from './jobs/parse'
9import { byKey, parseFind, splitGpu } from './remote'
10import type { Section, Want } from './remote'
11import { parseNvidiaSmi } from './slurm'
12
13/** Ended jobs stay listed this long. */
14export const KEEP_ENDED_MS = 12 * 3600_000
15
16export const EMPTY_JOBS: Jobs = { jobs: [], selected: null, updatedAt: null, error: null, disks: [], alerts: [] }
17
18export const runDir = (j: BatchJob) => (j.workDir ? `${j.workDir.replace(/\/$/, '')}/runs/${j.jobId}` : null)
19
20export function newJob(id: string): BatchJob {
21  return {
22    id,
23    jobId: id,
24    arrayId: null,
25    name: '',
26    state: 'PENDING',
27    reason: null,
28    elapsedS: null,
29    limitS: null,
30    nodes: 1,
31    nodeList: '',
32    gpus: null,
33    gpuType: null,
34    submittedAt: null,
35    startedAt: null,
36    workDir: null,
37    stdout: null,
38    command: null,
39    submitLine: null,
40    endedAt: null,
41    exitCode: null,
42    logSize: null,
43    logChangedAt: null,
44    lastLines: [],
45    progress: null,
46    flags: { oom: null, traceback: null, nccl: null, srun: null },
47    metrics: null,
48    ckpt: null,
49    gpuReadings: null,
50    gpusAt: null,
51    idleSince: null,
52    resumedFrom: null,
53    resumedAs: null,
54  }
55}
56
57/** Running jobs with GPUs, which the GPU sections and history are for. */
58export const hasGpus = (j: BatchJob) => j.state === 'RUNNING' && (j.gpus ?? 0) > 0 && !!j.nodeList
59
60/** The file reads of `jobs`: log tail, new metrics lines, checkpoints, a directory checkpoint's size. */
61export function filesWant(jobs: readonly BatchJob[]): Pick<Want, 'logs' | 'metrics' | 'ckpts' | 'du'> {
62  const w: Pick<Want, 'logs' | 'metrics' | 'ckpts' | 'du'> = { logs: [], metrics: [], ckpts: [], du: [] }
63  for (const j of jobs) {
64    if (j.stdout) w.logs.push({ id: j.id, path: j.stdout, knownSize: j.logSize })
65    const dir = runDir(j)
66    if (!dir) continue
67    w.metrics.push({ id: j.id, path: `${dir}/metrics.jsonl`, offset: j.metrics?.offset ?? 0 })
68    w.ckpts.push({ id: j.id, dir })
69    if (j.ckpt && j.ckpt.sizeBytes === null) w.du.push({ id: j.id, path: j.ckpt.path })
70  }
71  return w
72}
73
74export type FullOpts = {
75  nonce: string
76  /** squeue ids whose scontrol said interactive: nothing to watch in them. */
77  ignored: readonly string[]
78  exclude: readonly string[]
79  /** Jobs whose GPUs this pass reads. */
80  gpuIds: readonly string[]
81  isDfDue: boolean
82}
83
84/** A full poll: the queue, the files of running jobs already described, GPUs, maybe df. */
85export function fullWant(s: Jobs, o: FullOpts): Want {
86  const listed = s.jobs.filter(j => j.endedAt === null)
87  return {
88    nonce: o.nonce,
89    queue: {
90      knownRunning: [...listed.filter(j => j.state !== 'PENDING').map(j => j.id), ...o.ignored],
91      knownPending: listed.filter(j => j.state === 'PENDING').map(j => j.id),
92      active: listed.map(j => ({ id: j.id, jobId: j.jobId })),
93      exclude: [...o.exclude],
94    },
95    ...filesWant(listed.filter(j => j.state === 'RUNNING' && (j.stdout || j.workDir))),
96    gpus: s.jobs.filter(j => o.gpuIds.includes(j.id) && hasGpus(j)).map(j => ({ id: j.id, nodeList: j.nodeList })),
97    df: o.isDfDue ? [...new Set(s.jobs.map(j => j.workDir).filter((w): w is string => !!w))] : null,
98  }
99}
100
101export type QueueResult = {
102  jobs: BatchJob[]
103  /** Ids sacct showed ended in this pass. */
104  ended: string[]
105  /** Ids scontrol described in this pass while running: their files come next. */
106  described: string[]
107  /** Ids scontrol showed are interactive. */
108  ignored: string[]
109  zone: number | null
110  disks: Disk[] | null
111  error: string | null
112}
113
114/** A full reply merged into the jobs: squeue's rows, new details, the end of jobs it no longer lists. */
115export function mergeQueue(prev: Jobs, sections: readonly Section[], t: number, o: { ignored: readonly string[]; exclude: readonly string[] }): QueueResult {
116  const one = (name: string) => sections.find(s => s.name === name)?.body ?? null
117  const zoneText = one('zone')
118  const zone = zoneText ? zoneMinutes(zoneText.trim()) : null
119  const df = one('df')
120  const disks = df === null ? null : parseDf(df).filter((d, i, all) => all.findIndex(x => x.mount === d.mount) === i)
121  const base = { ended: [], described: [], ignored: [], zone, disks }
122
123  const failed = one('squeue-error')
124  if (failed !== null) return { ...base, jobs: prev.jobs, error: `squeue: ${failed.trim().split('\n')[0] || 'failed'}` }
125  const queue = one('squeue')
126  if (queue === null) return { ...base, jobs: prev.jobs, error: 'the cluster sent no squeue' }
127
128  const details = byKey(sections, 'detail')
129  const sacct = byKey(sections, 'sacct')
130  const ignore = new Set(o.ignored)
131  const rows = parseSqueue(queue).filter(r => !o.exclude.includes(r.name) && !ignore.has(r.id))
132  const byId = new Map(prev.jobs.map(j => [j.id, j]))
133  const jobs: BatchJob[] = []
134  const ended: string[] = []
135  const described: string[] = []
136  const ignored: string[] = []
137
138  for (const r of rows) {
139    const known = byId.get(r.id)
140    let j: BatchJob = {
141      ...(known ?? newJob(r.id)),
142      arrayId: r.arrayId,
143      name: r.name,
144      state: r.state,
145      reason: r.reason,
146      elapsedS: r.elapsedS,
147      limitS: r.limitS,
148      nodes: r.nodes,
149      nodeList: r.nodeList || known?.nodeList || '',
150      gpus: r.gpus ?? known?.gpus ?? null,
151      gpuType: r.gpuType ?? known?.gpuType ?? null,
152      submittedAt: known?.submittedAt ?? slurmTime(r.submitTime, zone ?? 0),
153    }
154    if (j.state === 'RUNNING' && j.startedAt === null) j.startedAt = t - (r.elapsedS ?? 0) * 1000
155    const text = details.get(r.id)
156    if (text !== undefined) {
157      const d = parseJobDetail(text)
158      if (d && !d.isBatch) {
159        ignored.push(r.id)
160        continue
161      }
162      if (d) {
163        j = { ...j, jobId: d.jobId, workDir: d.workDir, stdout: d.stdout, command: d.command, nodeList: d.nodeList || j.nodeList }
164        if (j.state === 'RUNNING') described.push(j.id)
165      }
166    }
167    jobs.push(j)
168  }
169
170  const listed = new Set(rows.map(r => r.id))
171  for (const j of prev.jobs) {
172    if (listed.has(j.id) || ignore.has(j.id)) continue
173    if (j.endedAt !== null) {
174      if (t - j.endedAt < KEEP_ENDED_MS) jobs.push(j)
175      continue
176    }
177    const acct = parseSacct(sacct.get(j.id) ?? '')
178    if (!acct || !TERMINAL.has(acct.state)) {
179      jobs.push(j) // accounting lags squeue: ask again next pass
180      continue
181    }
182    jobs.push({
183      ...j,
184      state: acct.state,
185      exitCode: acct.exitCode,
186      elapsedS: acct.elapsedS ?? j.elapsedS,
187      submitLine: acct.submitLine,
188      workDir: j.workDir ?? acct.workDir,
189      endedAt: t,
190    })
191    ended.push(j.id)
192  }
193  return { jobs, ended, described, ignored, zone, disks, error: null }
194}
195
196/** One job's file sections, where the pass asked for them: log, metrics, checkpoint. */
197export function applyFiles(j: BatchJob, sections: readonly Section[]): BatchJob {
198  const get = (name: string, key = j.id) => {
199    for (let i = sections.length - 1; i >= 0; i--) if (sections[i]!.name === name && sections[i]!.key === key) return sections[i]!.body
200    return null
201  }
202  let out = j
203
204  const logStat = get('logstat')
205  if (logStat !== null) {
206    const [size, mtime] = logStat.trim().split(/\s+/).map(Number)
207    if (Number.isFinite(size) && size !== j.logSize) {
208      const tail = get('log')
209      if (tail !== null) {
210        const t = parseLogTail(tail)
211        out = {
212          ...out,
213          lastLines: t.lines,
214          progress: t.progress ?? out.progress,
215          // Sticky: an error scrolled out of the tail still happened.
216          flags: {
217            oom: t.flags.oom ?? out.flags.oom,
218            traceback: t.flags.traceback ?? out.flags.traceback,
219            nccl: t.flags.nccl ?? out.flags.nccl,
220            srun: t.flags.srun ?? out.flags.srun,
221          },
222        }
223      }
224      out = { ...out, logSize: size!, logChangedAt: Number.isFinite(mtime) ? mtime! * 1000 : out.logChangedAt }
225    }
226  }
227
228  const mStat = get('mstat')
229  if (mStat !== null) {
230    const from = Number(mStat.trim().split(/\s+/)[1])
231    let m = out.metrics ?? EMPTY_METRICS
232    if (from === 0 && m.offset > 0) m = EMPTY_METRICS // rewritten from scratch
233    const chunk = get('metrics')
234    if (chunk !== null && from === m.offset) m = addChunk(m, chunk)
235    out = { ...out, metrics: m }
236  }
237
238  // Called only for jobs the pass read: no ckpt section means no such directory.
239  const dir = runDir(j)
240  if (dir) out = { ...out, ckpt: pickCkpt(dir, j.ckpt, [get('ckpt', `${j.id}|0`), get('ckpt', `${j.id}|1`)], get('du')) }
241  return out
242}
243
244/** The newest checkpoint in runs/<id>/checkpoints, else in runs/<id> itself. */
245function pickCkpt(dir: string, prev: Ckpt | null, bodies: readonly (string | null)[], du: string | null): Ckpt | null {
246  const wheres = [`${dir}/checkpoints`, dir]
247  for (let i = 0; i < wheres.length; i++) {
248    const body = bodies[i]
249    if (body === null || body === undefined) continue
250    const top = newestCkpt(wheres[i]!, parseFind(body))
251    if (!top) continue
252    if (prev && prev.path === top.path && prev.mtimeMs === top.mtimeMs) {
253      if (prev.sizeBytes !== null || du === null) return prev
254      const n = Number(du.trim())
255      return { ...prev, sizeBytes: Number.isFinite(n) && n > 0 ? n : null }
256    }
257    return top
258  }
259  return null
260}
261
262const anyIdle = (r: NodeGpus[] | null) => (r ?? []).some(n => n.gpus.some(g => (g.util ?? 0) === 0 && g.procs.length === 0))
263
264/** A job's GPU sections, one per node, as pace-line reads them over ssh. */
265export function applyGpus(j: BatchJob, sections: readonly Section[], nonce: string, t: number): BatchJob {
266  const prefix = `${j.id}|`
267  const readings: NodeGpus[] = []
268  for (const s of sections) {
269    if (s.name !== 'gpu' || !s.key.startsWith(prefix)) continue
270    const { gpus, apps } = splitGpu(s.body, nonce)
271    if (!gpus.trim()) continue
272    readings.push({ node: s.key.slice(prefix.length), gpus: parseNvidiaSmi(gpus, apps) })
273  }
274  const r = readings.length ? readings : null
275  return { ...j, gpuReadings: r ?? j.gpuReadings, gpusAt: t, idleSince: r === null ? j.idleSince : anyIdle(r) ? j.idleSince ?? t : null }
276}
277
278const pctOf = (used: number | null, total: number | null) => (used !== null && total ? Math.round((used / total) * 1000) / 10 : null)
279
280/** The job's current GPU readings as one history sample, every GPU in percent. */
281export function gpuSample(j: BatchJob): GpuSample | null {
282  if (!j.gpuReadings || j.gpusAt === null) return null
283  const gpus: GpuSample['gpus'] = {}
284  for (const n of j.gpuReadings) {
285    for (const g of n.gpus) gpus[`${n.node}:${g.index}`] = { util: g.util, mem: pctOf(g.memUsedMiB, g.memTotalMiB), power: pctOf(g.powerW, g.powerLimitW) }
286  }
287  return Object.keys(gpus).length ? { t: j.gpusAt, gpus } : null
288}
289
290/** History with `sample` added for `id` (once per reading), at most `max` samples, jobs no longer listed dropped. */
291export function addHistory(h: Record<string, GpuSample[]>, adds: readonly { id: string; sample: GpuSample }[], keep: ReadonlySet<string>, max: number): Record<string, GpuSample[]> {
292  const out: Record<string, GpuSample[]> = {}
293  for (const [id, list] of Object.entries(h)) if (keep.has(id)) out[id] = list
294  for (const { id, sample } of adds) {
295    const list = out[id] ?? []
296    if (list.length && list[list.length - 1]!.t >= sample.t) continue
297    out[id] = [...list, sample].slice(-Math.max(1, max))
298  }
299  return out
300}
301

1// One poll is one ssh round trip: a bash script, sent on stdin to the login
2// node, runs what pace-line runs on the cluster and prints each output under a
3// marker line. Pure: the script is built from what the poll wants, and the
4// reply is split back into sections for pace-line's parsers.
5
6import { SACCT_FIELDS, SQUEUE_FORMAT } from './jobs/parse'
7import { APP_QUERY, GPU_QUERY } from './slurm'
8
9/** The bytes of a log's end the pane reads, as pace-line does. */
10export const LOG_TAIL_BYTES = 65_536
11/** process.run keeps 4 MiB of stdout: what one poll may spend on metrics. */
12export const METRICS_BUDGET = 3_000_000
13/** A cluster file system that hangs shouldn't hang the poll. */
14const FS_TIMEOUT = 'timeout 20'
15
16/** What one poll asks the cluster for. */
17export type Want = {
18  /** Marks this reply's sections; a log line can't fake one. */
19  nonce: string
20  /**
21   * squeue, then scontrol for jobs not yet described, then sacct for active
22   * jobs squeue no longer lists. Null for a files- or GPU-only pass.
23   */
24  queue: {
25    /** squeue ids described while running (or ignored): never asked again. */
26    knownRunning: string[]
27    /** squeue ids described while pending: asked again once they run. */
28    knownPending: string[]
29    /** Listed jobs not yet ended, as `id` and the jobId sacct takes. */
30    active: { id: string; jobId: string }[]
31    /** Job names never described (excludeNames). */
32    exclude: string[]
33  } | null
34  logs: { id: string; path: string; knownSize: number | null }[]
35  metrics: { id: string; path: string; offset: number }[]
36  ckpts: { id: string; dir: string }[]
37  du: { id: string; path: string }[]
38  gpus: { id: string; nodeList: string }[]
39  /** Work dirs for df, besides /scratch/$USER; null when not due. */
40  df: string[] | null
41}
42
43export const EMPTY_WANT: Omit<Want, 'nonce'> = { queue: null, logs: [], metrics: [], ckpts: [], du: [], gpus: [], df: null }
44
45/** A single-quoted shell word. */
46export function shq(s: string): string {
47  return `'${s.replace(/'/g, `'\\''`)}'`
48}
49
50/** The ssh argv one round trip runs; the script goes on stdin. */
51export function sshArgv(host: string, remote: readonly string[]): string[] {
52  return ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', '-o', 'ServerAliveInterval=10', host, ...remote]
53}
54
55const ID = /^[\w.[\]-]+$/
56
57/** The bash script for `w`. Every path and id is quoted; each command may fail alone. */
58export function buildScript(w: Want): string {
59  const n = w.nonce
60  const out: string[] = [
61    'set +e',
62    'export LC_ALL=C',
63    `S() { printf '\\n@@JW:${n}:%s:%s\\n' "$1" "$2"; }`,
64    'S zone -; date +%z',
65  ]
66
67  if (w.queue) {
68    const set = (ids: readonly string[]) => shq(` ${ids.filter(i => ID.test(i)).join(' ')} `)
69    out.push(
70      `q=$(squeue -u "$USER" -h -r -o ${shq(SQUEUE_FORMAT)} 2>&1); rc=$?`,
71      'if [ $rc -eq 0 ]; then',
72      '  S squeue -; printf \'%s\' "$q"',
73      `  kr=${set(w.queue.knownRunning)}; kp=${set(w.queue.knownPending)}; xn=${shq(`|${w.queue.exclude.join('|')}|`)}`,
74      `  printf '%s\\n' "$q" | while IFS='|' read -r id name state rest; do`,
75      '    [ -z "$id" ] && continue',
76      '    case "$xn" in *"|$name|"*) continue;; esac',
77      '    case "$kr" in *" $id "*) continue;; esac',
78      '    case "$kp" in *" $id "*) [ "$state" = RUNNING ] || continue;; esac',
79      '    S detail "$id"; scontrol show job "$id" 2>/dev/null',
80      '  done',
81      `  ids=$(printf '%s\\n' "$q" | cut -d'|' -f1)`,
82    )
83    for (const a of w.queue.active) {
84      if (!ID.test(a.id) || !ID.test(a.jobId)) continue
85      out.push(`  printf '%s\\n' "$ids" | grep -qxF -- ${shq(a.id)} || { S sacct ${shq(a.id)}; sacct -j ${shq(a.jobId)} -X -n -P -o ${shq(SACCT_FIELDS)} 2>/dev/null; }`)
86    }
87    out.push('else', '  S squeue-error -; printf \'%s\' "$q"', 'fi')
88  }
89
90  for (const l of w.logs) {
91    const known = l.knownSize === null ? '-1' : String(l.knownSize)
92    out.push(`f=${shq(l.path)}; st=$(${FS_TIMEOUT} stat -c '%s %Y' -- "$f" 2>/dev/null) && { S logstat ${shq(l.id)}; printf '%s' "$st"; read -r sz mt <<<"$st"; [ "$sz" != ${shq(known)} ] && { S log ${shq(l.id)}; ${FS_TIMEOUT} tail -c ${LOG_TAIL_BYTES} -- "$f"; }; }`)
93  }
94
95  // A new job's whole file could pass the stdout cap: each read gets a share,
96  // and addChunk keeps only whole lines, so the rest comes next poll.
97  const cap = Math.max(200_000, Math.floor(METRICS_BUDGET / Math.max(1, w.metrics.length)))
98  for (const m of w.metrics) {
99    out.push(`m=${shq(m.path)}; off=${Math.max(0, Math.floor(m.offset))}; sz=$(${FS_TIMEOUT} stat -c %s -- "$m" 2>/dev/null) && { [ "$sz" -lt "$off" ] && off=0; S mstat ${shq(m.id)}; printf '%s %s' "$sz" "$off"; [ "$sz" -gt "$off" ] && { S metrics ${shq(m.id)}; ${FS_TIMEOUT} tail -c +$((off+1)) -- "$m" | head -c ${cap}; }; }`)
100  }
101
102  for (const c of w.ckpts) {
103    const dir = c.dir.replace(/\/+$/, '')
104    ;[`${dir}/checkpoints`, dir].forEach((d, i) => {
105      out.push(`d=${shq(d)}; [ -d "$d" ] && { S ckpt ${shq(`${c.id}|${i}`)}; ${FS_TIMEOUT} find "$d" -mindepth 1 -maxdepth 1 -printf '%f|%y|%s|%T@\\n' 2>/dev/null | tail -n 200; }`)
106    })
107  }
108
109  for (const d of w.du) out.push(`S du ${shq(d.id)}; ${FS_TIMEOUT} du -sb -- ${shq(d.path)} 2>/dev/null | cut -f1`)
110
111  for (const g of w.gpus) {
112    const remote = `nvidia-smi --query-gpu=${GPU_QUERY} --format=csv,noheader,nounits; echo @@JWAPPS:${n}; nvidia-smi --query-compute-apps=${APP_QUERY} --format=csv,noheader,nounits`
113    out.push(`for h in $(scontrol show hostnames ${shq(g.nodeList)} 2>/dev/null); do S gpu ${shq(`${g.id}|`)}"$h"; timeout 15 ssh -n -o BatchMode=yes -o ConnectTimeout=5 "$h" ${shq(remote)} 2>/dev/null; done`)
114  }
115
116  if (w.df) out.push(`S df -; df -B1 --output=target,size,used,avail "/scratch/$USER" ${w.df.map(shq).join(' ')} 2>/dev/null`)
117
118  out.push('S end -')
119  // bash reads the script from stdin as it goes: wrapped in a function, it is
120  // read whole before anything runs, and nothing inside can eat the rest.
121  return ['main() {', ...out.map(l => `  ${l}`), '}', 'main </dev/null', ''].join('\n')
122}
123
124export type Section = { name: string; key: string; body: string }
125
126/**
127 * The reply's sections in order, each body byte for byte (the newline before
128 * each marker belongs to the marker). `isComplete` is false when the reply
129 * stopped before the end marker: ssh dropped, or stdout passed its cap.
130 */
131export function splitSections(stdout: string, nonce: string): { sections: Section[]; isComplete: boolean } {
132  const mark = `\n@@JW:${nonce}:`
133  const parts = stdout.split(mark)
134  const sections: Section[] = []
135  let isComplete = false
136  for (const part of parts.slice(1)) {
137    const nl = part.indexOf('\n')
138    const head = nl === -1 ? part : part.slice(0, nl)
139    const body = nl === -1 ? '' : part.slice(nl + 1)
140    const colon = head.indexOf(':')
141    const name = colon === -1 ? head : head.slice(0, colon)
142    const key = colon === -1 ? '' : head.slice(colon + 1)
143    if (name === 'end') {
144      isComplete = true
145      break
146    }
147    sections.push({ name, key, body })
148  }
149  return { sections, isComplete }
150}
151
152/** The sections of one name, by key; a key given twice keeps its last body. */
153export function byKey(sections: readonly Section[], name: string): Map<string, string> {
154  const m = new Map<string, string>()
155  for (const s of sections) if (s.name === name) m.set(s.key, s.body)
156  return m
157}
158
159export type Entry = { name: string; kind: 'file' | 'dir'; size: number; mtimeMs: number }
160
161/** `find -printf '%f|%y|%s|%T@'` lines, as $.fs.list gives them to newestCkpt. */
162export function parseFind(body: string): Entry[] {
163  const out: Entry[] = []
164  for (const line of body.split('\n')) {
165    const parts = line.split('|')
166    if (parts.length < 4) continue
167    const t = Number(parts.pop())
168    const size = Number(parts.pop())
169    const y = parts.pop()
170    const name = parts.join('|')
171    if (!name || !Number.isFinite(t)) continue
172    out.push({ name, kind: y === 'd' ? 'dir' : 'file', size: Number.isFinite(size) ? size : 0, mtimeMs: Math.round(t * 1000) })
173  }
174  return out
175}
176
177/** A gpu section's two nvidia-smi CSVs. */
178export function splitGpu(body: string, nonce: string): { gpus: string; apps: string } {
179  const mark = `@@JWAPPS:${nonce}\n`
180  const i = body.indexOf(mark)
181  return i === -1 ? { gpus: body, apps: '' } : { gpus: body.slice(0, i), apps: body.slice(i + mark.length) }
182}
183
184/** The first line of ssh's complaint, for the pane. */
185export function sshError(stderr: string, exitCode: number): string {
186  const line = stderr.split('\n').map(s => s.trim()).find(s => s && !/^Warning: Permanently added/.test(s))
187  return line ? line.slice(0, 200) : `ssh exited with ${exitCode}`
188}
189

1// The /jobs pane: a header with the connection, one card per job or sweep,
2// then the selected one in detail. Desktop draws SVG charts; the terminal
3// draws pace-line's braille ones. Pure layout; register.tsx owns the handlers.
4
5import type { EngineInterface, Elements } from 'claude-code'
6
7import type { BatchJob, Conn, GpuSample, Jobs, Metrics } from '../types'
8import { TERMINAL } from './jobs/alerts'
9import { braille, rangeOf } from './jobs/chart'
10import type { Line, Run } from './jobs/chart'
11import { ema, etaSeconds, lossKey, lowerIsBetter, stepRate, trend, valKeys } from './jobs/metrics'
12import { fmtDur } from './format'
13import { SLOTS, fmt, lineChart, progressBar, sparkline } from './svg'
14import type { Series } from './svg'
15
16type El = ReturnType<EngineInterface['ui']['resolve']>
17export type Kit = {
18  Box: El['Box']
19  Text: El['Text']
20  Button: El['Button']
21  /** Present where the surface draws vectors (desktop, the editor, mobile). */
22  Svg?: Elements['desktop']['Svg']
23}
24
25export type Actions = {
26  select: (key: string) => void
27  copy: (text: string) => void
28  resume: (j: BatchJob) => void
29  cancel: (j: BatchJob) => void
30  refresh: () => void
31}
32
33export type ViewData = { s: Jobs; conn: Conn; history: Record<string, GpuSample[]>; now: number }
34
35export type Item =
36  | { kind: 'job'; key: string; job: BatchJob }
37  | { kind: 'array'; key: string; arrayId: string; name: string; tasks: BatchJob[] }
38
39/** Jobs as the pane lists them: array tasks folded into one sweep, active before ended. */
40export function items(jobs: readonly BatchJob[]): Item[] {
41  const out: Item[] = []
42  const arrays = new Map<string, BatchJob[]>()
43  const order = [...jobs].sort((a, b) => Number(a.endedAt !== null) - Number(b.endedAt !== null))
44  for (const j of order) {
45    if (j.arrayId) {
46      const list = arrays.get(j.arrayId)
47      if (list) list.push(j)
48      else {
49        const tasks = [j]
50        arrays.set(j.arrayId, tasks)
51        out.push({ kind: 'array', key: `array:${j.arrayId}`, arrayId: j.arrayId, name: j.name, tasks })
52      }
53    } else out.push({ kind: 'job', key: j.id, job: j })
54  }
55  return out
56}
57
58export function stateMark(state: string): { glyph: string; color: string } {
59  if (state === 'RUNNING') return { glyph: '▶', color: 'success' }
60  if (state === 'PENDING' || state === 'CONFIGURING') return { glyph: '◌', color: 'warning' }
61  if (state === 'COMPLETING') return { glyph: '◍', color: 'warning' }
62  if (state === 'COMPLETED') return { glyph: '✓', color: 'success' }
63  if (TERMINAL.has(state)) return { glyph: '✗', color: 'error' }
64  return { glyph: '·', color: 'subtle' }
65}
66
67/** How far along the job is, 0–1: epochs, then steps, then the log's k/N. */
68export function progressOf(j: BatchJob): number | null {
69  const m = j.metrics
70  if (m?.lastStep !== null && m?.lastStep !== undefined && m.totalSteps) return Math.min(1, m.lastStep / m.totalSteps)
71  if (m?.epoch !== null && m?.epoch !== undefined && m.totalEpochs) return Math.min(1, m.epoch / m.totalEpochs)
72  if (j.progress && j.progress.n) return Math.min(1, j.progress.k / j.progress.n)
73  return null
74}
75
76/** "ep 3/20 · step 1.2k/5k · ETA 1h20m", what is known of it. */
77export function progressText(j: BatchJob): string {
78  const m = j.metrics
79  const bits: string[] = []
80  if (m?.epoch !== null && m?.epoch !== undefined && m.totalEpochs) bits.push(`ep ${m.epoch}/${m.totalEpochs}`)
81  if (m?.lastStep !== null && m?.lastStep !== undefined) bits.push(m.totalSteps ? `step ${fmt(m.lastStep)}/${fmt(m.totalSteps)}` : `step ${fmt(m.lastStep)}`)
82  else if (j.progress) bits.push(`${j.progress.k}/${j.progress.n}`)
83  const eta = m && j.endedAt === null ? etaSeconds(m) : null
84  if (eta !== null) bits.push(`ETA ${fmtDur(eta)}`)
85  return bits.join(' · ')
86}
87
88const gpuText = (j: BatchJob) => (j.gpus ? `${j.nodes > 1 ? `${j.nodes}n·` : ''}${j.gpus}×${j.gpuType ?? 'GPU'}` : null)
89const leftS = (j: BatchJob) => (j.state === 'RUNNING' && j.limitS !== null && j.elapsedS !== null ? j.limitS - j.elapsedS : null)
90
91/** The status line: counts, and a mark when something needs a look. */
92export function statusText(s: Jobs, conn: Conn, now: number): string | undefined {
93  if (conn.isOk === false && conn.lastOkAt === null) return `⚙ ${conn.host}: unreachable`
94  const live = s.jobs.filter(j => j.endedAt === null)
95  if (!live.length) return undefined
96  const run = live.filter(j => j.state === 'RUNNING').length
97  const pend = live.filter(j => j.state === 'PENDING').length
98  const parts = [run && `${run} running`, pend && `${pend} pending`].filter(Boolean)
99  const alert = s.alerts.some(a => a.level !== 'info' && now - a.at < 60 * 60_000)
100  return `⚙ ${parts.join(' · ') || `${live.length} jobs`}${alert ? ' ⚠' : ''}${conn.isOk === false ? ' (offline)' : ''}`
101}
102
103// ── SVG charts (desktop) ─────────────────────────────────────────────────
104
105const stepAxis = { xName: 'step', xFormat: fmt }
106
107export function lossSvg(m: Metrics): string | null {
108  const k = lossKey(m)
109  if (!k) return null
110  const s = m.keys[k]!
111  return lineChart({
112    series: [
113      { label: k, xs: s.step, ys: s.value, slot: 'muted', isRaw: true },
114      { label: 'EMA', xs: s.step, ys: ema(s.value), slot: 0 },
115    ],
116    height: 210,
117    endLabel: true,
118    allowLog: true,
119    ...stepAxis,
120  })
121}
122
123/** One chart per val metric (three at most), its best point ringed. */
124export function valSvgs(m: Metrics): { key: string; svg: string; best: string }[] {
125  const out: { key: string; svg: string; best: string }[] = []
126  for (const k of valKeys(m).slice(0, 3)) {
127    const s = m.keys[k]!
128    if (!s.value.length) continue
129    const low = lowerIsBetter(k)
130    const b = s.value.reduce((bi, v, i) => ((low ? v < s.value[bi]! : v > s.value[bi]!) ? i : bi), 0)
131    const best = `best ${fmt(s.value[b]!)} @ ${fmt(s.step[b]!)}`
132    const svg = lineChart({
133      series: [{ label: k, xs: s.step, ys: s.value, slot: 1 }],
134      height: 150,
135      endLabel: true,
136      marker: s.value.length > 1 ? { x: s.step[b]!, y: s.value[b]!, slot: 1, label: best } : undefined,
137      ...stepAxis,
138    })
139    if (svg) out.push({ key: k, svg, best })
140  }
141  return out
142}
143
144/** Learning rate and throughput, small. */
145export function sideSvgs(m: Metrics): { key: string; title: string; svg: string }[] {
146  const out: { key: string; title: string; svg: string }[] = []
147  const lr = m.keys.lr ?? m.keys.learning_rate
148  const tp = m.keys.samples_per_s ?? m.keys.it_s ?? m.keys.throughput
149  const add = (key: string, title: string, s: { step: number[]; value: number[] }, slot: number, unit?: string) => {
150    const svg = lineChart({ series: [{ label: title, xs: s.step, ys: s.value, slot }], height: 110, endLabel: true, unit, ...stepAxis })
151    if (svg) out.push({ key, title, svg })
152  }
153  if (lr) add('lr', 'learning rate', lr, 6)
154  if (tp) add('tp', 'throughput', tp, 2, '/s')
155  return out
156}
157
158const GPU_FIELDS = [
159  { field: 'util', title: 'GPU utilization' },
160  { field: 'mem', title: 'GPU memory' },
161  { field: 'power', title: 'GPU power (of limit)' },
162] as const
163
164/** Utilization, memory and power over the kept history, one line per GPU on a shared 0–100% axis. */
165export function gpuSvgs(history: readonly GpuSample[], now: number): { key: string; title: string; svg: string }[] {
166  if (history.length < 2) return []
167  const keys = [...new Set(history.flatMap(h => Object.keys(h.gpus)))].sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
168  const xs = history.map(h => (h.t - now) / 60_000)
169  const shown = keys.slice(0, SLOTS)
170  const out: { key: string; title: string; svg: string }[] = []
171  for (const { field, title } of GPU_FIELDS) {
172    const series: Series[] = shown.map((k, i) => ({
173      label: k.replace(':', '·'),
174      xs,
175      ys: history.map(h => h.gpus[k]?.[field] ?? NaN),
176      slot: i,
177    }))
178    if (keys.length > SLOTS) {
179      // Past eight GPUs: the rest as one muted mean, not new hues.
180      const rest = keys.slice(SLOTS)
181      series.push({
182        label: `${rest.length} more (mean)`,
183        xs,
184        ys: history.map(h => {
185          const v = rest.map(k => h.gpus[k]?.[field]).filter((x): x is number => x !== null && x !== undefined)
186          return v.length ? v.reduce((a, b) => a + b, 0) / v.length : NaN
187        }),
188        slot: 'muted',
189      })
190    }
191    if (!series.some(s => s.ys.some(Number.isFinite))) continue
192    const last = (s: Series) => {
193      for (let i = s.ys.length - 1; i >= 0; i--) if (Number.isFinite(s.ys[i])) return `${Math.round(s.ys[i]!)}%`
194      return null
195    }
196    const svg = lineChart({
197      series,
198      height: 140,
199      yDomain: [0, 100],
200      unit: '%',
201      xName: 'time',
202      xFormat: x => (x > -0.5 ? 'now' : `${fmtDur(-x * 60)} ago`),
203      legendValues: series.map(last),
204    })
205    if (svg) out.push({ key: field, title, svg })
206  }
207  return out
208}
209
210/** The loss EMA of each task of a sweep, the lowest final one ringed. */
211export function sweepSvg(tasks: readonly BatchJob[]): string | null {
212  const series: Series[] = []
213  tasks.forEach((t, i) => {
214    const k = t.metrics && lossKey(t.metrics)
215    if (!t.metrics || !k) return
216    const s = t.metrics.keys[k]!
217    series.push({ label: t.id, xs: s.step, ys: ema(s.value), slot: i < SLOTS ? i : 'muted' })
218  })
219  if (!series.length) return null
220  const finals = series.map(s => s.ys[s.ys.length - 1] ?? NaN)
221  const b = finals.reduce((bi, v, i) => (v < finals[bi]! ? i : bi), 0)
222  const best = series[b]!
223  return lineChart({
224    series,
225    height: 230,
226    allowLog: true,
227    legendValues: finals.map(v => (Number.isFinite(v) ? fmt(v) : null)),
228    marker: series.length > 1 && best.slot !== 'muted'
229      ? { x: best.xs[best.xs.length - 1]!, y: finals[b]!, slot: best.slot, label: `lowest: ${best.label}` }
230      : undefined,
231    ...stepAxis,
232  })
233}
234
235// ── The pane ─────────────────────────────────────────────────────────────
236
237const ago = (now: number, t: number | null) => (t === null ? null : `${fmtDur(Math.max(0, (now - t) / 1000))} ago`)
238
239function header(kit: Kit, d: ViewData, act: Actions) {
240  const { Box, Text, Button } = kit
241  const { conn, s, now } = d
242  const live = s.jobs.filter(j => j.endedAt === null).length
243  const dot = conn.isOk === null ? { c: 'subtle', t: 'connecting…' } : conn.isOk ? { c: 'success', t: 'connected' } : { c: 'error', t: 'unreachable' }
244  return (
245    <Box key="head" flexDirection="column">
246      <Box flexDirection="row" justifyContent="space-between">
247        <Text wrap="truncate-end">
248          <Text color={dot.c}>{'● '}</Text>
249          <Text bold>{conn.host}</Text>
250          <Text color="subtle">
251            {`  ${dot.t} · ${live} active job${live === 1 ? '' : 's'}`}
252            {conn.lastOkAt !== null ? ` · read ${ago(now, conn.lastOkAt)}` : ''}
253            {conn.latencyMs !== null ? ` in ${(conn.latencyMs / 1000).toFixed(1)}s` : ''}
254          </Text>
255        </Text>
256        <Button key="refresh" label={conn.isPolling ? 'Reading…' : 'Refresh'} onPress={() => act.refresh()} />
257      </Box>
258      {conn.isOk === false && conn.error && (
259        <Text color="error" wrap="wrap">{`ssh ${conn.host}: ${conn.error}${conn.failures > 1 ? ` (retrying, ${conn.failures} failures)` : ''}`}</Text>
260      )}
261      {conn.isOk === false && conn.lastOkAt === null && (
262        <Text color="subtle" wrap="wrap">{`job-watch runs \`ssh -o BatchMode=yes ${conn.host}\`: the alias must log in without a prompt. Set another host in /config → job-watch.`}</Text>
263      )}
264      {s.error && <Text color="error">{s.error}</Text>}
265    </Box>
266  )
267}
268
269function svgEl(kit: Kit, key: string, source: string | null, alt: string, interactive = true) {
270  const { Svg } = kit
271  if (!Svg || !source) return null
272  return <Svg key={key} source={source} alt={alt} isInteractive={interactive || undefined} />
273}
274
275function card(kit: Kit, it: Item, isSel: boolean, now: number, act: Actions) {
276  const { Box, Text, Button } = kit
277  if (it.kind === 'array') {
278    const c = (f: (j: BatchJob) => boolean) => it.tasks.filter(f).length
279    const failed = c(j => TERMINAL.has(j.state) && j.state !== 'COMPLETED')
280    const done = c(j => j.state === 'COMPLETED')
281    return (
282      <Box key={it.key} flexDirection="column" borderStyle="round" borderColor={isSel ? 'claude' : 'subtle'} paddingX={1}>
283        <Box flexDirection="row" justifyContent="space-between">
284          <Button plain label={`▦ ${it.name}`} onPress={() => act.select(it.key)} />
285          <Text color="subtle">{`sweep ${it.arrayId} · ${it.tasks.length} tasks`}</Text>
286        </Box>
287        <Text>
288          <Text color="success">{`${done}✓  ${c(j => j.state === 'RUNNING')}▶  `}</Text>
289          <Text color="warning">{`${c(j => j.state === 'PENDING')}◌  `}</Text>
290          {failed > 0 && <Text color="error">{`${failed}✗`}</Text>}
291        </Text>
292        {svgEl(kit, 'bar', progressBar(it.tasks.length ? done / it.tasks.length : 0, 220), `${done} of ${it.tasks.length} tasks done`, false)}
293      </Box>
294    )
295  }
296  const j = it.job
297  const mk = stateMark(j.state)
298  const p = progressOf(j)
299  const left = leftS(j)
300  const k = j.metrics && lossKey(j.metrics)
301  const loss = k ? j.metrics!.keys[k]!.value : null
302  const tr = loss ? trend(loss) : 0
303  const meta = [j.nodeList || null, gpuText(j), j.ckpt ? `ckpt ${ago(now, j.ckpt.mtimeMs)}` : null].filter(Boolean).join(' · ')
304  return (
305    <Box key={it.key} flexDirection="column" borderStyle="round" borderColor={isSel ? 'claude' : 'subtle'} paddingX={1}>
306      <Box flexDirection="row" justifyContent="space-between">
307        <Button plain dimColor={j.endedAt !== null} label={`${mk.glyph} ${j.name}`} onPress={() => act.select(it.key)} />
308        <Text>
309          <Text color={mk.color}>{j.state === 'PENDING' && j.reason ? `${j.state} (${j.reason})` : j.state}</Text>
310          {left !== null && <Text color={left < 30 * 60 ? 'error' : 'subtle'}>{`  ⌛ ${fmtDur(left)} left`}</Text>}
311          {j.endedAt !== null && <Text color="subtle">{`  ${ago(now, j.endedAt)}`}</Text>}
312        </Text>
313      </Box>
314      {p !== null && (
315        <Box flexDirection="row" gap={1} alignItems="center">
316          {svgEl(kit, 'bar', progressBar(p, 220), `${Math.round(p * 100)}% done`, false)}
317          <Text color="subtle">{`${Math.round(p * 100)}%  ${progressText(j)}`}</Text>
318        </Box>
319      )}
320      {p === null && progressText(j) !== '' && <Text color="subtle">{progressText(j)}</Text>}
321      <Box flexDirection="row" justifyContent="space-between" alignItems="center">
322        <Text color="subtle" wrap="truncate-end">{`${j.id}${meta ? ` · ${meta}` : ''}`}</Text>
323        {loss && (
324          <Box flexDirection="row" gap={1} alignItems="center">
325            {svgEl(kit, 'spark', sparkline(ema(loss)), `loss trend, last ${fmt(loss[loss.length - 1]!)}`, false)}
326            <Text>
327              <Text>{fmt(loss[loss.length - 1]!)}</Text>
328              <Text color={tr < 0 ? 'success' : tr > 0 ? 'error' : 'subtle'}>{tr < 0 ? ' ↓' : tr > 0 ? ' ↑' : ' →'}</Text>
329            </Text>
330          </Box>
331        )}
332      </Box>
333    </Box>
334  )
335}
336
337function section(kit: Kit, key: string, title: string, note: string | null, body: unknown) {
338  const { Box, Text } = kit
339  if (!body) return null
340  return (
341    <Box key={key} flexDirection="column" marginTop={1}>
342      <Text>
343        <Text bold>{title}</Text>
344        {note && <Text color="subtle">{`  ${note}`}</Text>}
345      </Text>
346      {body as never}
347    </Box>
348  )
349}
350
351function jobDetail(kit: Kit, j: BatchJob, d: ViewData, cols: number, act: Actions) {
352  const { Box, Text, Button } = kit
353  const { now, s } = d
354  const mk = stateMark(j.state)
355  const m = j.metrics
356  const rate = m ? stepRate(m) : null
357  const p = progressOf(j)
358  const node = j.nodeList.split(/[,[]/)[0] || null
359  const hasSvg = !!kit.Svg
360  const disks = s.disks.filter(x => x.mount.startsWith('/scratch') || (j.workDir ?? '').startsWith(x.mount))
361  const alerts = s.alerts.filter(a => a.job === j.id).slice(0, 4)
362  const hist = d.history[j.id] ?? []
363
364  const charts: unknown[] = []
365  if (m && hasSvg) {
366    const k = lossKey(m)
367    const loss = lossSvg(m)
368    if (k && loss) {
369      const v = m.keys[k]!.value
370      charts.push(section(kit, 'loss', 'Training loss', `${k} ${fmt(v[v.length - 1]!)} · min ${fmt(Math.min(...v))}`, svgEl(kit, 'svg', loss, `${k} by step, raw and smoothed`)))
371    }
372    for (const v of valSvgs(m)) charts.push(section(kit, `val-${v.key}`, v.key, v.best, svgEl(kit, 'svg', v.svg, `${v.key} by step, ${v.best}`)))
373    for (const x of sideSvgs(m)) charts.push(section(kit, x.key, x.title, null, svgEl(kit, 'svg', x.svg, `${x.title} by step`)))
374  } else if (m) {
375    charts.push(...termCharts(kit, m, Math.max(20, Math.min(100, cols - 12))))
376  }
377  if (hasSvg && hist.length >= 2) {
378    const span = fmtDur((hist[hist.length - 1]!.t - hist[0]!.t) / 1000)
379    for (const g of gpuSvgs(hist, now)) charts.push(section(kit, `gpu-${g.key}`, g.title, `last ${span}`, svgEl(kit, 'svg', g.svg, `${g.title} per GPU over the last ${span}`)))
380  } else if (j.gpuReadings) {
381    charts.push(section(kit, 'gpus', 'GPUs', j.gpusAt !== null ? `read ${ago(now, j.gpusAt)}` : null, (
382      <Box flexDirection="column">
383        {j.gpuReadings.flatMap(n => n.gpus.map(g => {
384          const mem = g.memUsedMiB !== null && g.memTotalMiB ? Math.round((g.memUsedMiB / g.memTotalMiB) * 100) : null
385          const idle = (g.util ?? 0) === 0 && g.procs.length === 0
386          return (
387            <Text key={`${n.node}-${g.index}`}>
388              <Text color="subtle">{`${n.node}·${g.index}  `}</Text>
389              <Text>{`util ${String(g.util ?? '?').padStart(3)}%  mem ${String(mem ?? '?').padStart(3)}%`}</Text>
390              <Text color="subtle">{g.tempC !== null ? `  ${g.tempC}°C` : ''}{g.powerW !== null ? `  ${Math.round(g.powerW)}W` : ''}</Text>
391              {idle && <Text color="warning">{'  idle'}</Text>}
392            </Text>
393          )
394        }))}
395      </Box>
396    )))
397  }
398
399  return (
400    <Box key="detail" flexDirection="column" marginTop={1}>
401      <Box flexDirection="row" justifyContent="space-between">
402        <Text>
403          <Text bold color="claude">{j.name}</Text>
404          <Text color="subtle">{`  ${j.id}${j.jobId !== j.id ? ` (${j.jobId})` : ''}`}</Text>
405        </Text>
406        <Text color={mk.color}>{`${mk.glyph} ${j.state}${j.reason ? ` (${j.reason})` : ''}${j.exitCode && j.endedAt !== null ? ` · exit ${j.exitCode}` : ''}`}</Text>
407      </Box>
408      <Text color="subtle" wrap="truncate-end">
409        {[j.nodeList || null, gpuText(j), j.workDir].filter(Boolean).join(' · ')}
410      </Text>
411      {j.limitS !== null && j.elapsedS !== null && (
412        <Box flexDirection="row" gap={1} alignItems="center">
413          <Text color="subtle">{'time     '}</Text>
414          {svgEl(kit, 'tbar', progressBar(j.elapsedS / j.limitS, 220, (leftS(j) ?? Infinity) < 30 * 60), `${fmtDur(j.elapsedS)} of ${fmtDur(j.limitS)}`, false)}
415          <Text>{`${fmtDur(j.elapsedS)} of ${fmtDur(j.limitS)}`}</Text>
416          {leftS(j) !== null && <Text color={leftS(j)! < 30 * 60 ? 'error' : 'subtle'}>{` · ${fmtDur(leftS(j)!)} left`}</Text>}
417        </Box>
418      )}
419      {(p !== null || progressText(j)) && (
420        <Box flexDirection="row" gap={1} alignItems="center">
421          <Text color="subtle">{'progress '}</Text>
422          {p !== null && svgEl(kit, 'pbar', progressBar(p, 220), `${Math.round(p * 100)}% done`, false)}
423          <Text>{`${p !== null ? `${Math.round(p * 100)}%  ` : ''}${progressText(j)}${rate ? ` · ${fmt(rate)} step/s` : ''}`}</Text>
424        </Box>
425      )}
426      {j.ckpt
427        ? (
428          <Text wrap="truncate-end">
429            <Text color="subtle">{'ckpt     '}</Text>
430            <Text color={now - j.ckpt.mtimeMs > 3600_000 && j.endedAt === null ? 'warning' : undefined}>
431              {[j.ckpt.name, j.ckpt.epoch !== null && `epoch ${j.ckpt.epoch}`, j.ckpt.step !== null && `step ${fmt(j.ckpt.step)}`, ago(now, j.ckpt.mtimeMs), j.ckpt.sizeBytes !== null && `${(j.ckpt.sizeBytes / 2 ** 30).toFixed(1)} GiB`].filter(Boolean).join(' · ')}
432            </Text>
433          </Text>
434        )
435        : j.state === 'RUNNING' && <Text color="subtle">{'ckpt     none in runs/<jobid>/checkpoints yet'}</Text>}
436      {m?.nonFinite && <Text color="error">{`non-finite value: ${m.nonFinite}`}</Text>}
437      {disks.map(x => (
438        <Text key={x.mount} color={x.avail / x.size < 0.1 ? 'warning' : 'subtle'}>{`disk     ${x.mount} · ${(x.avail / 2 ** 40).toFixed(1)} TiB free of ${(x.size / 2 ** 40).toFixed(1)}`}</Text>
439      ))}
440      {charts as never}
441      {j.lastLines.length > 0 && section(kit, 'log', 'Log', j.logChangedAt !== null ? `written ${ago(now, j.logChangedAt)}` : null, (
442        <Box flexDirection="column">
443          {j.lastLines.map((l, i) => <Text key={String(i)} color="subtle" wrap="truncate-end">{`› ${l}`}</Text>)}
444        </Box>
445      ))}
446      {alerts.length > 0 && (
447        <Box flexDirection="column" marginTop={1}>
448          {alerts.map((a, i) => <Text key={String(i)} color={a.level === 'error' ? 'error' : a.level === 'warn' ? 'warning' : 'success'} wrap="wrap">{a.text}</Text>)}
449        </Box>
450      )}
451      <Box flexDirection="row" gap={2} marginTop={1} flexWrap="wrap">
452        {node && j.state === 'RUNNING' && <Button key="ssh" label={`Copy ssh ${node}`} onPress={() => act.copy(`ssh -J ${d.conn.host} ${node}`)} />}
453        {j.stdout && <Button key="log" label="Copy tail -f" onPress={() => act.copy(`ssh ${d.conn.host} tail -f ${j.stdout}`)} />}
454        {j.endedAt !== null && j.state !== 'COMPLETED' && !j.resumedAs && !j.arrayId && <Button key="resume" label="Resume from checkpoint…" onPress={() => act.resume(j)} />}
455        {j.endedAt === null && <Button key="cancel" label="Cancel job…" onPress={() => act.cancel(j)} />}
456      </Box>
457    </Box>
458  )
459}
460
461function arrayDetail(kit: Kit, it: Extract<Item, { kind: 'array' }>, cols: number, act: Actions) {
462  const { Box, Text, Button } = kit
463  const svg = kit.Svg ? sweepSvg(it.tasks) : null
464  const lines: Line[] = kit.Svg ? [] : it.tasks.flatMap((t, i) => {
465    const k = t.metrics && lossKey(t.metrics)
466    if (!t.metrics || !k) return []
467    const s = t.metrics.keys[k]!
468    return [{ xs: s.step, ys: ema(s.value), color: TERM_SWEEP[i % TERM_SWEEP.length]! }]
469  })
470  return (
471    <Box key="detail" flexDirection="column" marginTop={1}>
472      <Text>
473        <Text bold color="claude">{it.name}</Text>
474        <Text color="subtle">{`  sweep ${it.arrayId} · ${it.tasks.length} tasks`}</Text>
475      </Text>
476      {svg && section(kit, 'sweep', 'Loss per task', 'EMA', svgEl(kit, 'svg', svg, `loss EMA of each of the ${it.tasks.length} tasks by step`))}
477      {lines.length > 0 && termChart(kit, 'sweep', 'loss (ema) per task', lines, Math.max(20, Math.min(100, cols - 12)), 8, '')}
478      <Box flexDirection="column" marginTop={1}>
479        {it.tasks.map(t => {
480          const mk = stateMark(t.state)
481          const k = t.metrics && lossKey(t.metrics)
482          const last = k ? t.metrics!.keys[k]!.value : null
483          return (
484            <Box key={t.id} flexDirection="row" gap={1}>
485              <Button plain label={`${mk.glyph} ${t.id}`} onPress={() => act.select(t.id)} />
486              <Text color="subtle">{`${t.state}  ${progressText(t)}${last ? `  loss ${fmt(last[last.length - 1]!)}` : ''}`}</Text>
487            </Box>
488          )
489        })}
490      </Box>
491    </Box>
492  )
493}
494
495/** The pane's tree for any surface. */
496export function paneTree(kit: Kit, d: ViewData, cols: number, act: Actions) {
497  const { Box, Text } = kit
498  const list = items(d.s.jobs)
499  const sel = d.s.selected
500  const selected = list.find(it => it.key === sel) ?? (sel ? list.find(it => it.kind === 'array' && it.tasks.some(t => t.id === sel)) : undefined) ?? list[0]
501  const task = selected?.kind === 'array' ? selected.tasks.find(t => t.id === sel) : undefined
502  return (
503    <Box flexDirection="column">
504      {header(kit, d, act)}
505      {list.length === 0 && d.conn.isOk !== false && (
506        <Text color="subtle">{d.s.updatedAt === null ? 'Reading squeue…' : 'No batch jobs. Submitted jobs show here within a poll.'}</Text>
507      )}
508      {list.length > 0 && (
509        <Box key="cards" flexDirection="column" marginTop={1} gap={0}>
510          {list.map(it => card(kit, it, it === selected, d.now, act))}
511        </Box>
512      )}
513      {task
514        ? jobDetail(kit, task, d, cols, act)
515        : selected?.kind === 'array'
516          ? arrayDetail(kit, selected, cols, act)
517          : selected && jobDetail(kit, selected.job, d, cols, act)}
518    </Box>
519  )
520}
521
522// ── Terminal charts ──────────────────────────────────────────────────────
523
524const TERM_SWEEP = ['suggestion', 'warning', 'claude', 'success', 'permission', 'autoAccept', 'error', 'planMode']
525
526function runs(kit: Kit, key: string, rs: Run[]) {
527  const { Text } = kit
528  return (
529    <Text key={key}>
530      {rs.map((r, i) => <Text key={String(i)} color={r.color || 'subtle'}>{r.text}</Text>)}
531    </Text>
532  )
533}
534
535function termChart(kit: Kit, key: string, title: string, lines: Line[], width: number, height: number, note: string) {
536  const { Box, Text } = kit
537  const range = rangeOf(lines)
538  if (!range) return null
539  const rows = braille(lines, width, height, range)
540  const top = fmt(range.y.max).padStart(7)
541  const bottom = fmt(range.y.min).padStart(7)
542  return (
543    <Box key={key} flexDirection="column" marginTop={1}>
544      <Text>
545        <Text bold>{title}</Text>
546        <Text color="subtle">{note ? `  ${note}` : ''}</Text>
547      </Text>
548      {rows.map((r, i) => (
549        <Box key={String(i)} flexDirection="row">
550          <Text color="subtle">{i === 0 ? top : i === rows.length - 1 ? bottom : ' '.repeat(7)}</Text>
551          <Text color="subtle">{' ┤'}</Text>
552          {runs(kit, 'r', r)}
553        </Box>
554      ))}
555      <Text color="subtle">{`${' '.repeat(9)}step ${fmt(range.x.min)}${' '.repeat(Math.max(1, width - 12 - fmt(range.x.max).length))}${fmt(range.x.max)}`}</Text>
556    </Box>
557  )
558}
559
560function termCharts(kit: Kit, m: Metrics, width: number): unknown[] {
561  const out: unknown[] = []
562  const k = lossKey(m)
563  if (k) {
564    const s = m.keys[k]!
565    const smooth = ema(s.value)
566    out.push(termChart(kit, 'loss', k, [
567      { xs: s.step, ys: s.value, color: 'inactive' },
568      { xs: s.step, ys: smooth, color: 'suggestion' },
569    ], width, 6, `${fmt(s.value[s.value.length - 1]!)} (ema ${fmt(smooth[smooth.length - 1]!)}, min ${fmt(Math.min(...s.value))})`))
570  }
571  for (const v of valKeys(m).slice(0, 2)) {
572    const s = m.keys[v]!
573    const low = lowerIsBetter(v)
574    const b = s.value.reduce((bi, x, i) => ((low ? x < s.value[bi]! : x > s.value[bi]!) ? i : bi), 0)
575    out.push(termChart(kit, v, v, [
576      { xs: s.step, ys: s.value, color: 'warning' },
577      { xs: [s.step[b]!], ys: [s.value[b]!], color: 'success' },
578    ], width, 3, `${fmt(s.value[s.value.length - 1]!)} · best ${fmt(s.value[b]!)} @ ${fmt(s.step[b]!)}`))
579  }
580  return out
581}
582

1// Copied from plugins/pace-line/hooks/format.ts by scripts/sync-shared.sh: edit it there.
2// Pure layout for the band: a port of ~/.claude/statusline-command.sh.
3// Each line is a list of coloured segments; register.tsx turns them into Text.
4
5import type { Limit, Slurm, Snapshot } from '../types'
6
7/** `action` marks a segment drawn as a button that opens that picker. */
8export type Seg = { text: string; color?: string; action?: 'model' | 'effort' | 'job' | 'jobs' }
9
10// The script's ANSI colours as Ghostty's palette draws them (its defaults:
11// no theme or palette is set). A mod can't emit palette codes: names and
12// theme keys come out as the engine's own RGB. If you change Ghostty's
13// theme, update these from `ghostty +show-config --default | grep palette`.
14const CYAN = '#8abeb7' // \033[36m, palette 6
15const GREEN = '#b5bd68' // \033[32m, palette 2
16const YELLOW = '#f0c674' // \033[33m, palette 3
17const ORANGE = '#ff8700' // \033[38;5;208m
18const RED = '#cc6666' // \033[31m, palette 1
19const GRAY = '#666666' // \033[90m, palette 8
20const MAGENTA = '#b294bb' // \033[35m, palette 5
21/** The palette, for the job panel. */
22export const COLORS = { CYAN, GREEN, YELLOW, ORANGE, RED, GRAY, MAGENTA } as const
23
24// Under this much time left the countdown turns red; at WARN_SECONDS a toast warns.
25export const URGENT_SECONDS = 15 * 60
26export const WARN_SECONDS = 10 * 60
27
28// Text the script leaves uncoloured: Claude Code draws a status line's
29// default-coloured text in this gray, so the band does too.
30export const PLAIN = '#999999'
31const SEP: Seg = { text: ' | ' }
32
33// Window length and the minimum elapsed time before the pace arrow shows,
34// in seconds. Without the minimum, a burst right after a reset divides by a
35// tiny elapsed time and reads as a huge overshoot.
36const WINDOWS: Record<string, { label: string; length: number; minElapsed: number }> = {
37  five_hour: { label: '5h', length: 18000, minElapsed: 900 },
38  seven_day: { label: '7d', length: 604800, minElapsed: 21600 },
39}
40
41/**
42 * Terminal cells a text takes: emoji-presentation symbols such as ⌛ take two.
43 * Line 1's clickable region is sized and hit-tested with this.
44 */
45export function cellWidth(text: string): number {
46  let n = 0
47  for (const ch of text) n += /\p{Emoji_Presentation}/u.test(ch) ? 2 : 1
48  return n
49}
50
51/** 3d22h / 2h5m / 48m / 44s */
52export function fmtDur(seconds: number): string {
53  const s = Math.max(0, Math.floor(seconds))
54  if (s >= 86400) return `${Math.floor(s / 86400)}d${Math.floor((s % 86400) / 3600)}h`
55  if (s >= 3600) return `${Math.floor(s / 3600)}h${Math.floor((s % 3600) / 60)}m`
56  if (s >= 60) return `${Math.floor(s / 60)}m`
57  return `${s}s`
58}
59
60/** "Opus 5.5" from "Claude Opus 5.5 (1M context)", "claude-opus-5-5[1m]" or "Opus 5.5". */
61export function modelLabel(model: string): string {
62  const m = /(opus|sonnet|haiku|fable)[\s-]*(\d+(?:[.-]\d+)?)?/i.exec(model)
63  if (!m || !m[1]) return model
64  const family = m[1][0]!.toUpperCase() + m[1].slice(1).toLowerCase()
65  // An id's trailing date (claude-haiku-4-5-20251001) is never matched: the
66  // version takes at most one separator.
67  const version = m[2]?.replace('-', '.')
68  return version ? `${family} ${version}` : family
69}
70
71function effortColor(effort: string): string | undefined {
72  switch (effort) {
73    case 'low': return GREEN
74    case 'medium': return YELLOW
75    case 'high': return ORANGE
76    case 'xhigh':
77    case 'max': return RED
78    default: return undefined
79  }
80}
81
82export function bar(percent: number, width: number): string {
83  const filled = Math.min(width, Math.max(0, Math.round((percent * width) / 100)))
84  return '█'.repeat(filled) + '▒'.repeat(width - filled)
85}
86
87export function usageColor(percent: number): string {
88  const p = Math.round(percent)
89  if (p >= 90) return RED
90  if (p >= 65) return ORANGE
91  if (p >= 40) return YELLOW
92  return GREEN
93}
94
95/**
96 * Relative gap from pro-rata usage, in whole percent: positive means burning
97 * faster than the window allows (⇡), negative means headroom (⇣). Null while
98 * the window has not run long enough, or the reset time is unknown.
99 */
100export function paceDelta(limit: Limit, nowMs: number): number | null {
101  const w = WINDOWS[limit.kind]
102  if (!w || !limit.resetsAt) return null
103  const remaining = (Date.parse(limit.resetsAt) - nowMs) / 1000
104  if (!(remaining > 0 && remaining <= w.length)) return null
105  const elapsed = w.length - remaining
106  if (elapsed < w.minElapsed) return null
107  const delta = Math.round((limit.percentUsed * w.length) / elapsed - 100)
108  return Math.max(-999, Math.min(999, delta))
109}
110
111function limitSegs(limit: Limit, nowMs: number, width: number, showReset: boolean): Seg[] {
112  const w = WINDOWS[limit.kind]
113  const segs: Seg[] = [
114    { text: w?.label ?? limit.kind, color: CYAN },
115    { text: ' ' },
116    { text: bar(limit.percentUsed, width), color: usageColor(limit.percentUsed) },
117    { text: ` ${Math.round(limit.percentUsed)}%` },
118  ]
119  const delta = paceDelta(limit, nowMs)
120  if (delta !== null) {
121    segs.push(delta > 0
122      ? { text: ` ⇡${delta}%`, color: RED }
123      : { text: ` ⇣${-delta}%`, color: GREEN })
124  }
125  if (showReset && w && limit.resetsAt) {
126    const remaining = (Date.parse(limit.resetsAt) - nowMs) / 1000
127    if (remaining > 0 && remaining <= w.length) {
128      segs.push({ text: ` ${fmtDur(remaining)}`, color: GRAY })
129    }
130  }
131  return segs
132}
133
134function join(groups: Seg[][], sep: Seg): Seg[] {
135  const out: Seg[] = []
136  for (const g of groups.filter(g => g.length > 0)) {
137    if (out.length > 0) out.push(sep)
138    out.push(...g)
139  }
140  return out
141}
142
143/**
144 * The allocation from Slurm's environment, as the explorer script reads it:
145 * never squeue or scontrol, so a redraw puts no load on the controller.
146 * The GPU ids come from whichever is set first, which depends on how the GPUs
147 * were asked for (--gres or --gpus) and on job versus step.
148 */
149export function slurmFrom(env: {
150  SLURM_JOB_ID?: string
151  SLURM_JOB_GPUS?: string
152  SLURM_STEP_GPUS?: string
153  CUDA_VISIBLE_DEVICES?: string
154  SLURM_GPUS_ON_NODE?: string
155  SLURM_JOB_END_TIME?: string
156}): Slurm | null {
157  if (!env.SLURM_JOB_ID) return null
158  const gpus = env.SLURM_JOB_GPUS || env.SLURM_STEP_GPUS || env.CUDA_VISIBLE_DEVICES || null
159  // Epoch seconds, set by Slurm 23.02+ when the job starts.
160  const end = Number(env.SLURM_JOB_END_TIME)
161  return {
162    job: env.SLURM_JOB_ID,
163    gpus,
164    gpusOnNode: env.SLURM_GPUS_ON_NODE || null,
165    endsAt: end > 0 ? end * 1000 : null,
166  }
167}
168
169/** Seconds until the job's end; null when unknown. */
170export function jobLeft(slurm: Slurm, nowMs: number): number | null {
171  return slurm.endsAt === null ? null : Math.max(0, Math.floor((slurm.endsAt - nowMs) / 1000))
172}
173
174/** job 123456 · 2×gpu[0,1] · ⌛ 1h12m; the job opens the panel. */
175function slurmSegs(slurm: Slurm, nowMs: number): Seg[] {
176  const segs: Seg[] = [{ text: `job ${slurm.job}`, color: MAGENTA, action: 'job' }]
177  if (slurm.gpus) {
178    const count = slurm.gpus.split(',').length
179    segs.push({ text: ' · ' }, { text: `${count}×gpu[${slurm.gpus}]`, color: GRAY })
180  } else if (slurm.gpusOnNode) {
181    segs.push({ text: ' · ' }, { text: `${slurm.gpusOnNode}×gpu`, color: GRAY })
182  }
183  const left = jobLeft(slurm, nowMs)
184  if (left !== null) {
185    segs.push({ text: ' · ' }, {
186      text: left > 0 ? `⌛ ${fmtDur(left)}` : '⌛ ending',
187      color: left < URGENT_SECONDS ? RED : GRAY,
188    })
189  }
190  return segs
191}
192
193/** [Model · effort] · folder | branch | job 123456 · 2×gpu[0,1] · ⌛ 1h12m */
194export function line1(s: Snapshot, nowMs = Date.now()): Seg[] {
195  const model: Seg[] = []
196  if (s.model) {
197    model.push({ text: '[', color: CYAN }, { text: modelLabel(s.model), color: CYAN, action: 'model' })
198    if (s.effort) {
199      model.push({ text: ' · ', color: CYAN }, { text: s.effort, color: effortColor(s.effort), action: 'effort' })
200    }
201    model.push({ text: ']', color: CYAN })
202  }
203  const head = join([model, s.folder ? [{ text: s.folder }] : []], { text: ' · ' })
204  return join([head, s.branch ? [{ text: s.branch }] : [], s.slurm ? slurmSegs(s.slurm, nowMs) : []], SEP)
205}
206
207/** context bar · pct% | $cost | ⏱ time | 5h bar | 7d bar */
208export function line2(s: Snapshot, nowMs: number): Seg[] {
209  // Only the windows the script draws, in its order: 5h then 7d.
210  const limits = ['five_hour', 'seven_day']
211    .map(kind => s.limits.find(l => l.kind === kind))
212    .filter((l): l is Limit => l !== undefined)
213  // Widths adapt to how many windows there are, so the line never wraps.
214  const both = limits.length >= 2
215  const ctxWidth = both ? 10 : 20
216  const limitWidth = both ? 6 : 10
217
218  const ctx: Seg[] = s.contextPercent === null ? [] : [
219    { text: bar(s.contextPercent, ctxWidth), color: GREEN },
220    { text: ` ${Math.round(s.contextPercent)}%` },
221  ]
222  const cost: Seg[] = s.costUsd ? [{ text: `$${s.costUsd.toFixed(2)}`, color: YELLOW }] : []
223  const time: Seg[] = s.startedAt === null ? [] : [{ text: `⏱ ${fmtDur((nowMs - s.startedAt) / 1000)}` }]
224  const lim = join(
225    limits.map(l => limitSegs(l, nowMs, limitWidth, l.kind === 'five_hour' || !both)),
226    { text: '  ' },
227  )
228  return join([ctx, cost, time, lim], SEP)
229}
230

1// Copied from plugins/pace-line/hooks/slurm.ts by scripts/sync-shared.sh: edit it there.
2// Parsers for the job panel's two sources: `scontrol show job` and the two
3// `nvidia-smi` CSV queries. register.tsx runs the commands.
4
5import type { Gpu, GpuProc, JobInfo } from '../types'
6
7export const GPU_QUERY = 'index,uuid,name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw,power.limit'
8export const APP_QUERY = 'gpu_uuid,pid,process_name,used_memory'
9
10/** "1-02:03:04", "02:03:04", "03:04" → seconds; null for UNLIMITED or junk. */
11export function slurmDuration(text: string): number | null {
12  const m = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/.exec(text.trim())
13  if (!m) return null
14  const [, d, h, min, s] = m
15  return Number(d ?? 0) * 86400 + Number(h ?? 0) * 3600 + Number(min) * 60 + Number(s)
16}
17
18/** `scontrol show job` prints Key=Value pairs separated by spaces and newlines. */
19export function parseScontrol(out: string): JobInfo | null {
20  const kv: Record<string, string> = {}
21  for (const m of out.matchAll(/(\w[\w/:]*)=(\S*)/g)) kv[m[1]!] ??= m[2]!
22  if (!kv.JobId) return null
23  const tres = Object.fromEntries((kv.AllocTRES ?? '').split(',').filter(Boolean).map(p => p.split('=') as [string, string]))
24  return {
25    id: kv.JobId,
26    name: kv.JobName ?? '',
27    state: kv.JobState ?? '',
28    partition: kv.Partition ?? '',
29    account: kv.Account ?? '',
30    node: kv.BatchHost && kv.BatchHost !== '(null)' ? kv.BatchHost : (kv.NodeList ?? ''),
31    nodeList: kv.NodeList ?? '',
32    runSeconds: slurmDuration(kv.RunTime ?? ''),
33    limitSeconds: slurmDuration(kv.TimeLimit ?? ''),
34    cpus: tres.cpu ?? null,
35    mem: tres.mem ?? null,
36    gpus: tres['gres/gpu'] ?? null,
37  }
38}
39
40const num = (v: string | undefined): number | null => {
41  const n = Number((v ?? '').trim())
42  return v === undefined || !/\d/.test(v) || Number.isNaN(n) ? null : n
43}
44
45/** The two `--format=csv,noheader,nounits` queries, joined by GPU uuid. */
46export function parseNvidiaSmi(gpuCsv: string, appCsv: string): Gpu[] {
47  const procs = new Map<string, GpuProc[]>()
48  for (const line of appCsv.split('\n').filter(l => l.trim())) {
49    const [uuid, pid, name, mem] = line.split(',').map(c => c.trim())
50    if (!uuid) continue
51    const list = procs.get(uuid) ?? []
52    list.push({ pid: pid ?? '', name: (name ?? '').split('/').pop() ?? '', memMiB: num(mem) })
53    procs.set(uuid, list)
54  }
55  return gpuCsv.split('\n').filter(l => l.trim()).map(line => {
56    const [index, uuid, name, util, memUsed, memTotal, temp, power, powerLimit] = line.split(',').map(c => c.trim())
57    return {
58      index: index ?? '?',
59      name: (name ?? '').replace(/^(NVIDIA|Tesla) /, ''),
60      util: num(util),
61      memUsedMiB: num(memUsed),
62      memTotalMiB: num(memTotal),
63      tempC: num(temp),
64      powerW: num(power),
65      powerLimitW: num(powerLimit),
66      procs: procs.get(uuid ?? '') ?? [],
67    }
68  })
69}
70

1// Copied from plugins/pace-line/hooks/jobs/metrics.ts by scripts/sync-shared.sh: edit it there.
2// runs/<jobid>/metrics.jsonl, read a piece at a time: one JSON object per
3// line, `step` plus any numbers (loss, val_*, lr, samples_per_s...). jobkit.py
4// writes it; Python's json writes NaN and Infinity bare, which JSON.parse
5// refuses, so those are read as non-finite on purpose.
6
7import type { Ckpt, Metrics, Series } from '../../types'
8
9/** Points kept per key; past it every other point is dropped, oldest half first. */
10export const MAX_POINTS = 1200
11
12/** Keys that describe the step rather than measure it. */
13const META = new Set(['step', 'time', 'epoch', 'total_steps', 'total_epochs', 'rank'])
14
15export const EMPTY_METRICS: Metrics = {
16  offset: 0,
17  keys: {},
18  lastStep: null,
19  totalSteps: null,
20  epoch: null,
21  totalEpochs: null,
22  firstTime: null,
23  firstStep: null,
24  lastTime: null,
25  nonFinite: null,
26}
27
28/** Bytes the text takes as UTF-8, so offsets match the file's. */
29export function utf8Length(text: string): number {
30  let n = 0
31  for (const ch of text) {
32    const c = ch.codePointAt(0)!
33    n += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4
34  }
35  return n
36}
37
38function thin(s: Series): Series {
39  if (s.step.length <= MAX_POINTS) return s
40  // Halve the older half: recent detail matters most on a live chart.
41  const half = Math.floor(s.step.length / 2)
42  const keep = (_: number, i: number) => i >= half || i % 2 === 0
43  return { step: s.step.filter(keep), value: s.value.filter(keep) }
44}
45
46/**
47 * Adds a chunk read from `m.offset` on. Only whole lines are consumed; a
48 * line still being written stays for the next read.
49 */
50export function addChunk(m: Metrics, chunk: string): Metrics {
51  const end = chunk.lastIndexOf('\n')
52  if (end < 0) return m
53  const whole = chunk.slice(0, end + 1)
54  const keys: Record<string, Series> = { ...m.keys }
55  let { lastStep, totalSteps, epoch, totalEpochs, firstTime, firstStep, lastTime, nonFinite } = m
56  for (const line of whole.split('\n')) {
57    if (!line.trim()) continue
58    let row: Record<string, unknown>
59    try {
60      row = JSON.parse(line.replace(/\b-?Infinity\b|\bNaN\b/g, '"__nonfinite__"')) as Record<string, unknown>
61    } catch {
62      continue
63    }
64    const step = typeof row.step === 'number' ? row.step : lastStep === null ? 0 : lastStep + 1
65    lastStep = step
66    firstStep ??= step
67    if (typeof row.time === 'number') {
68      firstTime ??= row.time
69      lastTime = row.time
70    }
71    if (typeof row.total_steps === 'number') totalSteps = row.total_steps
72    if (typeof row.epoch === 'number') epoch = row.epoch
73    if (typeof row.total_epochs === 'number') totalEpochs = row.total_epochs
74    for (const [k, v] of Object.entries(row)) {
75      if (META.has(k)) continue
76      if (v === '__nonfinite__') {
77        nonFinite ??= `${k}@${step}`
78        continue
79      }
80      if (typeof v !== 'number') continue
81      const s = keys[k] ?? { step: [], value: [] }
82      keys[k] = thin({ step: [...s.step, step], value: [...s.value, v] })
83    }
84  }
85  return { offset: m.offset + utf8Length(whole), keys, lastStep, totalSteps, epoch, totalEpochs, firstTime, firstStep, lastTime, nonFinite }
86}
87
88/** Exponential moving average, the smoothing TensorBoard's slider applies. */
89export function ema(values: readonly number[], alpha = 0.9): number[] {
90  const out: number[] = []
91  let acc: number | null = null
92  for (const v of values) {
93    acc = acc === null ? v : alpha * acc + (1 - alpha) * v
94    out.push(acc)
95  }
96  return out
97}
98
99/** The train loss key: `loss`, then `train_loss`, then any key ending in loss that isn't val. */
100export function lossKey(m: Metrics): string | null {
101  const keys = Object.keys(m.keys)
102  return keys.find(k => k === 'loss') ?? keys.find(k => k === 'train_loss') ?? keys.find(k => /loss$/.test(k) && !/^val/.test(k)) ?? null
103}
104
105export function valKeys(m: Metrics): string[] {
106  return Object.keys(m.keys).filter(k => /^val[_/]/.test(k))
107}
108
109/** Whether a val metric improves downwards: losses and errors do. */
110export function lowerIsBetter(key: string): boolean {
111  return /loss|err|wer|cer|mae|mse|rmse|perplexity|ppl/i.test(key)
112}
113
114/** Steps per second over the run so far, for the ETA. */
115export function stepRate(m: Metrics): number | null {
116  if (m.firstTime === null || m.lastTime === null || m.firstStep === null || m.lastStep === null) return null
117  const dt = m.lastTime - m.firstTime
118  return dt > 0 && m.lastStep > m.firstStep ? (m.lastStep - m.firstStep) / dt : null
119}
120
121export function etaSeconds(m: Metrics): number | null {
122  const rate = stepRate(m)
123  if (rate === null || m.totalSteps === null || m.lastStep === null) return null
124  return Math.max(0, (m.totalSteps - m.lastStep) / rate)
125}
126
127/** The trend of the smoothed loss over its last stretch: -1 falling, 1 rising, 0 flat. */
128export function trend(values: readonly number[]): -1 | 0 | 1 {
129  if (values.length < 8) return 0
130  const s = ema(values)
131  const a = s[Math.floor(s.length * 0.75)]!
132  const b = s[s.length - 1]!
133  const rel = (b - a) / (Math.abs(a) || 1)
134  return rel < -0.01 ? -1 : rel > 0.01 ? 1 : 0
135}
136
137const CKPT_FILE = /\.(pt|pth|ckpt|safetensors|bin)$|^checkpoint-\d+$|^(epoch|step)[=_-]?\d+/
138
139/** step and epoch from the names Lightning, HF Trainer and jobkit give checkpoints. */
140export function ckptNumbers(name: string): { step: number | null; epoch: number | null } {
141  const step = /(?:step[=_-]?|checkpoint-)(\d+)/.exec(name)
142  const epoch = /epoch[=_-]?(\d+)/.exec(name)
143  return { step: step ? Number(step[1]) : null, epoch: epoch ? Number(epoch[1]) : null }
144}
145
146/** The newest checkpoint among a directory's entries (best.* and last.* links aside). */
147export function newestCkpt(dir: string, entries: readonly { name: string; kind: string; size: number; mtimeMs: number }[]): Ckpt | null {
148  const cands = entries.filter(e => CKPT_FILE.test(e.name) && !/^(best|last)\b/.test(e.name) && !e.name.endsWith('.tmp'))
149  const top = [...cands].sort((a, b) => b.mtimeMs - a.mtimeMs)[0]
150  if (!top) return null
151  return {
152    name: top.name,
153    path: `${dir.replace(/\/$/, '')}/${top.name}`,
154    mtimeMs: top.mtimeMs,
155    sizeBytes: top.kind === 'file' ? top.size : null,
156    ...ckptNumbers(top.name),
157  }
158}
159

1// Copied from plugins/pace-line/hooks/jobs/chart.ts by scripts/sync-shared.sh: edit it there.
2// Terminal charts: braille line charts (2×4 dots per cell) and block sparklines.
3// Pure: series in, rows of coloured runs out; the pane turns runs into Text.
4
5export type Run = { text: string; color: string }
6export type Line = { xs: readonly number[]; ys: readonly number[]; color: string }
7
8const BRAILLE = 0x2800
9// Dot bit for (column 0/1, row 0..3) inside one braille cell.
10const DOT = [
11  [0x01, 0x02, 0x04, 0x40],
12  [0x08, 0x10, 0x20, 0x80],
13]
14
15export type Range = { min: number; max: number }
16
17export function rangeOf(lines: readonly Line[]): { x: Range; y: Range } | null {
18  let xMin = Infinity, xMax = -Infinity, yMin = Infinity, yMax = -Infinity
19  for (const l of lines) {
20    for (let i = 0; i < l.xs.length; i++) {
21      const x = l.xs[i]!, y = l.ys[i]!
22      if (!Number.isFinite(x) || !Number.isFinite(y)) continue
23      xMin = Math.min(xMin, x); xMax = Math.max(xMax, x)
24      yMin = Math.min(yMin, y); yMax = Math.max(yMax, y)
25    }
26  }
27  if (xMin === Infinity) return null
28  if (yMax === yMin) { yMax += Math.abs(yMax) * 0.05 || 1; yMin -= Math.abs(yMin) * 0.05 || 1 }
29  if (xMax === xMin) xMax = xMin + 1
30  return { x: { min: xMin, max: xMax }, y: { min: yMin, max: yMax } }
31}
32
33/**
34 * Draws the lines in `width`×`height` cells, joining consecutive points so a
35 * steep drop reads as a line, not dots. Later lines draw over earlier ones and
36 * a cell takes the colour of the last line that touched it.
37 */
38export function braille(lines: readonly Line[], width: number, height: number, range = rangeOf(lines)): Run[][] {
39  const W = width * 2, H = height * 4
40  const bits = Array.from({ length: height }, () => new Array<number>(width).fill(0))
41  const colors = Array.from({ length: height }, () => new Array<string>(width).fill(''))
42  if (!range) return bits.map(() => [{ text: ' '.repeat(width), color: '' }])
43  const px = (x: number) => Math.round(((x - range.x.min) / (range.x.max - range.x.min)) * (W - 1))
44  const py = (y: number) => (H - 1) - Math.round(((Math.min(range.y.max, Math.max(range.y.min, y)) - range.y.min) / (range.y.max - range.y.min)) * (H - 1))
45  const plot = (x: number, y: number, color: string) => {
46    if (x < 0 || x >= W || y < 0 || y >= H) return
47    const cx = x >> 1, cy = y >> 2
48    bits[cy]![cx]! |= DOT[x & 1]![y & 3]!
49    colors[cy]![cx] = color
50  }
51  for (const l of lines) {
52    let prev: [number, number] | null = null
53    for (let i = 0; i < l.xs.length; i++) {
54      const x = l.xs[i]!, y = l.ys[i]!
55      if (!Number.isFinite(x) || !Number.isFinite(y)) { prev = null; continue }
56      const p: [number, number] = [px(x), py(y)]
57      if (prev) {
58        // Bresenham between the two dots.
59        let [x0, y0] = prev
60        const [x1, y1] = p
61        const dx = Math.abs(x1 - x0), dy = -Math.abs(y1 - y0)
62        const sx = x0 < x1 ? 1 : -1, sy = y0 < y1 ? 1 : -1
63        let err = dx + dy
64        for (;;) {
65          plot(x0, y0, l.color)
66          if (x0 === x1 && y0 === y1) break
67          const e2 = 2 * err
68          if (e2 >= dy) { err += dy; x0 += sx }
69          if (e2 <= dx) { err += dx; y0 += sy }
70        }
71      } else {
72        plot(p[0], p[1], l.color)
73      }
74      prev = p
75    }
76  }
77  return bits.map((row, r) => {
78    const runs: Run[] = []
79    for (let c = 0; c < width; c++) {
80      const ch = row[c] ? String.fromCharCode(BRAILLE + row[c]!) : ' '
81      const color = colors[r]![c]!
82      const last = runs[runs.length - 1]
83      if (last && last.color === color) last.text += ch
84      else runs.push({ text: ch, color })
85    }
86    return runs
87  })
88}
89
90const BLOCKS = '▁▂▃▄▅▆▇█'
91
92/** The last `width` values as one-cell blocks, scaled to their own range. */
93export function sparkline(values: readonly number[], width: number): string {
94  const v = values.filter(Number.isFinite).slice(-width)
95  if (v.length === 0) return ''
96  const min = Math.min(...v), max = Math.max(...v)
97  return v.map(x => BLOCKS[max === min ? 3 : Math.round(((x - min) / (max - min)) * 7)]).join('')
98}
99
100/** 0.000412 → "4.12e-4", 1234.5 → "1234", 0.4123 → "0.412". */
101export function fmtNum(v: number): string {
102  if (!Number.isFinite(v)) return String(v)
103  const a = Math.abs(v)
104  if (a !== 0 && (a < 1e-3 || a >= 1e5)) return v.toExponential(2).replace('e-', 'e-').replace('e+', 'e')
105  if (a >= 100) return v.toFixed(0)
106  if (a >= 10) return v.toFixed(1)
107  return v.toFixed(3)
108}
109
110/** One colour per sweep task, cycling the band's palette. */
111export const SWEEP_COLORS = ['#8abeb7', '#f0c674', '#b294bb', '#b5bd68', '#ff8700', '#81a2be', '#cc6666', '#de935f']
112

1// SVG charts for the desktop pane: line charts with nice ticks, a hairline
2// grid, direct end labels and a hover layer (crosshair, dots, tooltip) that is
3// CSS alone, since the sandboxed frame runs no script. Colors are the dataviz
4// reference palette, light and dark, picked by prefers-color-scheme.
5// Pure: series in, an `<svg>` string out.
6
7/** The Svg element's cap; charts thin their hover layer to stay under it. */
8export const SVG_MAX = 131_072
9const BUDGET = 120_000
10
11const LIGHT = {
12  ink: '#0b0b0b', ink2: '#52514e', muted: '#898781', grid: '#e1e0d9', axis: '#c3c2b7', surface: '#fcfcfb',
13  s: ['#2a78d6', '#eb6834', '#1baf7a', '#eda100', '#e87ba4', '#008300', '#4a3aa7', '#e34948'],
14  good: '#0ca30c', critical: '#d03b3b',
15}
16const DARK = {
17  ink: '#ffffff', ink2: '#c3c2b7', muted: '#898781', grid: '#2c2c2a', axis: '#383835', surface: '#1a1a19',
18  s: ['#3987e5', '#d95926', '#199e70', '#c98500', '#d55181', '#008300', '#9085e9', '#e66767'],
19  good: '#0ca30c', critical: '#d03b3b',
20}
21
22/** Categorical slots in fixed order; a ninth series folds into the muted "other". */
23export const SLOTS = 8
24
25function vars(p: typeof LIGHT): string {
26  return [
27    `--ink:${p.ink}`, `--ink2:${p.ink2}`, `--muted:${p.muted}`, `--grid:${p.grid}`, `--axis:${p.axis}`, `--surface:${p.surface}`,
28    `--good:${p.good}`, `--critical:${p.critical}`,
29    ...p.s.map((c, i) => `--s${i}:${c}`),
30  ].join(';')
31}
32
33const STYLE = [
34  `svg{${vars(LIGHT)};font-family:system-ui,-apple-system,"Segoe UI",sans-serif}`,
35  `@media (prefers-color-scheme:dark){svg{${vars(DARK)}}}`,
36  '.g{stroke:var(--grid);stroke-width:1}',
37  '.a{stroke:var(--axis);stroke-width:1}',
38  '.t{fill:var(--muted);font-size:11px;font-variant-numeric:tabular-nums}',
39  '.v{fill:var(--ink2);font-size:11px;font-variant-numeric:tabular-nums}',
40  '.k{fill:var(--ink2);font-size:11px}',
41  '.l{fill:none;stroke-width:2;stroke-linejoin:round;stroke-linecap:round}',
42  '.raw{fill:none;stroke:var(--muted);stroke-opacity:.5;stroke-width:1.25;stroke-linejoin:round}',
43  '.d{stroke:var(--surface);stroke-width:2}',
44  '.b .h{opacity:0}',
45  '.b:hover .h{opacity:1}',
46  '.x{stroke:var(--axis);stroke-width:1}',
47  '.tip{fill:var(--surface);stroke:var(--axis);stroke-width:1}',
48  '.tt{fill:var(--ink);font-size:11px;font-variant-numeric:tabular-nums}',
49  '.tm{fill:var(--muted);font-size:11px;font-variant-numeric:tabular-nums}',
50].join('')
51
52const color = (slot: number | 'muted') => (slot === 'muted' ? 'var(--muted)' : `var(--s${slot % SLOTS})`)
53const r1 = (v: number) => Math.round(v * 10) / 10
54const esc = (s: string) => s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;')
55
56/** 0.000412 → "4.1e-4", 1234 → "1.2k", 0.4123 → "0.412". */
57export function fmt(v: number): string {
58  if (!Number.isFinite(v)) return String(v)
59  const a = Math.abs(v)
60  if (a === 0) return '0'
61  if (a >= 1e6) return `${trim(v / 1e6, 1)}M`
62  if (a >= 1e4) return `${trim(v / 1e3, a >= 1e5 ? 0 : 1)}k`
63  if (a >= 1000) return `${trim(v / 1e3, 2)}k`
64  if (a >= 100) return v.toFixed(0)
65  if (a >= 10) return trim(v, 1)
66  if (a >= 0.01) return trim(v, 3)
67  return v.toExponential(1).replace('e-', 'e-')
68}
69const trim = (v: number, d: number) => v.toFixed(d).replace(/\.?0+$/, '')
70
71/** Ticks at 1, 2 or 5 × 10^k covering [min, max], about `count` of them. */
72export function niceTicks(min: number, max: number, count = 4): number[] {
73  if (!(max > min)) return [min]
74  const raw = (max - min) / Math.max(1, count)
75  const mag = 10 ** Math.floor(Math.log10(raw))
76  // The smallest of 1, 2, 2.5, 5, 10 × 10^k near the raw step: a few more ticks rather than too few.
77  const step = [1, 2, 2.5, 5, 10].map(m => m * mag).find(s => s >= raw * 0.75) ?? 10 * mag
78  const out: number[] = []
79  for (let v = Math.floor(min / step) * step; v <= max + step * 1e-9; v += step) out.push(Math.round(v / step) * step)
80  if (out[out.length - 1]! < max - step * 1e-9) out.push(out[out.length - 1]! + step)
81  return out
82}
83
84/**
85 * At most about 2 × `buckets` points, keeping each bucket's first, lowest,
86 * highest and last, so spikes survive and the path stays small.
87 */
88export function decimate(xs: readonly number[], ys: readonly number[], buckets: number): { xs: number[]; ys: number[] } {
89  const pts: [number, number][] = []
90  for (let i = 0; i < xs.length; i++) if (Number.isFinite(xs[i]) && Number.isFinite(ys[i])) pts.push([xs[i]!, ys[i]!])
91  if (pts.length <= buckets * 2) return { xs: pts.map(p => p[0]), ys: pts.map(p => p[1]) }
92  const x0 = pts[0]![0], x1 = pts[pts.length - 1]![0]
93  const span = x1 - x0 || 1
94  const keep = new Set<number>()
95  let b = -1, first = 0, lo = 0, hi = 0
96  const flush = (last: number) => { if (b >= 0) for (const i of [first, lo, hi, last]) keep.add(i) }
97  for (let i = 0; i < pts.length; i++) {
98    const k = Math.min(buckets - 1, Math.floor(((pts[i]![0] - x0) / span) * buckets))
99    if (k !== b) {
100      flush(i - 1)
101      b = k; first = i; lo = i; hi = i
102    } else {
103      if (pts[i]![1] < pts[lo]![1]) lo = i
104      if (pts[i]![1] > pts[hi]![1]) hi = i
105    }
106  }
107  flush(pts.length - 1)
108  const idx = [...keep].sort((a, c) => a - c)
109  return { xs: idx.map(i => pts[i]![0]), ys: idx.map(i => pts[i]![1]) }
110}
111
112export type Series = {
113  label: string
114  xs: readonly number[]
115  ys: readonly number[]
116  /** A categorical slot, or the muted gray of a raw line or "other". */
117  slot: number | 'muted'
118  /** A faint thin line under the others (raw loss under its EMA). */
119  isRaw?: boolean
120  /** Left out of the hover tooltip. */
121  noTip?: boolean
122}
123
124export type LineChartSpec = {
125  series: readonly Series[]
126  width?: number
127  height?: number
128  /** A fixed y domain (GPU percentages); else nice ticks around the data. */
129  yDomain?: [number, number]
130  /** Unit after tooltip values and the end label, as "%". */
131  unit?: string
132  /** x axis name in the tooltip ("step", "min"). */
133  xName?: string
134  xFormat?: (x: number) => string
135  /** Logarithmic y when every value is positive and they span over 2 decades. */
136  allowLog?: boolean
137  /** A point to ring and label: a val metric's best. */
138  marker?: { x: number; y: number; slot: number; label: string }
139  /** The last value of the first non-raw series beside its end. */
140  endLabel?: boolean
141  /** Legend row on top; on by default for two or more labelled series. */
142  legend?: boolean
143  /** Text after each legend label, e.g. its current value. */
144  legendValues?: readonly (string | null)[]
145}
146
147/** A line chart, or null when there is nothing to draw. */
148export function lineChart(spec: LineChartSpec): string | null {
149  const W = spec.width ?? 600
150  const H = spec.height ?? 200
151  const series = spec.series.filter(s => s.xs.some((x, i) => Number.isFinite(x) && Number.isFinite(s.ys[i])))
152  if (!series.length) return null
153  const xf = spec.xFormat ?? fmt
154
155  let xMin = Infinity, xMax = -Infinity, yMin = Infinity, yMax = -Infinity
156  for (const s of series) {
157    for (let i = 0; i < s.xs.length; i++) {
158      const x = s.xs[i]!, y = s.ys[i]!
159      if (!Number.isFinite(x) || !Number.isFinite(y)) continue
160      xMin = Math.min(xMin, x); xMax = Math.max(xMax, x)
161      yMin = Math.min(yMin, y); yMax = Math.max(yMax, y)
162    }
163  }
164  if (xMax === xMin) { xMin -= 1; xMax += 1 }
165  const isLog = !spec.yDomain && !!spec.allowLog && yMin > 0 && yMax / yMin > 100
166  let ticks: number[]
167  let lo: number, hi: number
168  if (spec.yDomain) {
169    ;[lo, hi] = spec.yDomain
170    ticks = niceTicks(lo, hi, 4)
171  } else if (isLog) {
172    lo = 10 ** Math.floor(Math.log10(yMin))
173    hi = 10 ** Math.ceil(Math.log10(yMax))
174    ticks = []
175    for (let v = lo; v <= hi * 1.0001; v *= 10) ticks.push(v)
176  } else {
177    if (yMax === yMin) { const pad = Math.abs(yMax) * 0.05 || 1; yMin -= pad; yMax += pad }
178    ticks = niceTicks(yMin, yMax, 4)
179    lo = ticks[0]!
180    hi = ticks[ticks.length - 1]!
181  }
182
183  const labelled = series.filter(s => !s.isRaw)
184  const ordered = [...series.filter(s => s.isRaw), ...labelled]
185  const showLegend = spec.legend ?? ordered.length >= 2
186  // Legend items laid out in rows that wrap at the chart's width.
187  const legend: { s: Series; text: string; x: number; row: number }[] = []
188  if (showLegend) {
189    let x = 0, row = 0
190    for (const s of ordered) {
191      const value = spec.legendValues?.[spec.series.indexOf(s)] ?? null
192      const text = value ? `${s.label} ${value}` : s.label
193      const w = 19 + text.length * 6.2 + 14
194      if (x > 0 && x + w > W - 60) { x = 0; row++ }
195      legend.push({ s, text, x, row })
196      x += w
197    }
198  }
199  const rows = legend.length ? legend[legend.length - 1]!.row + 1 : 0
200  const top = rows ? 8 + rows * 18 : 10
201  const left = Math.max(...ticks.map(t => fmt(t).length)) * 6.5 + 12
202  const right = spec.endLabel ? 52 : 14
203  const bottom = 22
204  const pw = W - left - right
205  const ph = H - top - bottom
206  const sx = (x: number) => left + ((x - xMin) / (xMax - xMin)) * pw
207  const ty = (y: number) => (isLog ? Math.log10(y) : y)
208  const sy = (y: number) => top + ph - ((ty(Math.min(hi, Math.max(lo, y))) - ty(lo)) / (ty(hi) - ty(lo))) * ph
209
210  const parts: string[] = []
211  // Grid and y labels.
212  for (const t of ticks) {
213    const y = r1(sy(t))
214    parts.push(`<line class="g" x1="${left}" x2="${r1(left + pw)}" y1="${y}" y2="${y}"/>`)
215    parts.push(`<text class="t" x="${r1(left - 6)}" y="${r1(y + 4)}" text-anchor="end">${fmt(t)}${t === ticks[ticks.length - 1] && spec.unit ? esc(spec.unit) : ''}</text>`)
216  }
217  parts.push(`<line class="a" x1="${left}" x2="${r1(left + pw)}" y1="${r1(top + ph)}" y2="${r1(top + ph)}"/>`)
218  // x labels: a few nice ticks inside the range.
219  for (const t of niceTicks(xMin, xMax, Math.max(2, Math.floor(pw / 110)))) {
220    if (t < xMin || t > xMax) continue
221    const x = r1(sx(t))
222    parts.push(`<text class="t" x="${x}" y="${H - 6}" text-anchor="middle">${esc(xf(t))}</text>`)
223  }
224
225  // Lines: raw first, under the rest.
226  for (const s of ordered) {
227    const d = decimate(s.xs, s.ys, Math.round(pw))
228    if (!d.xs.length) continue
229    const path = d.xs.map((x, i) => `${i ? 'L' : 'M'}${r1(sx(x))} ${r1(sy(d.ys[i]!))}`).join('')
230    parts.push(s.isRaw ? `<path class="raw" d="${path}"/>` : `<path class="l" stroke="${color(s.slot)}" d="${path}"/>`)
231    if (d.xs.length === 1) parts.push(`<circle class="d" cx="${r1(sx(d.xs[0]!))}" cy="${r1(sy(d.ys[0]!))}" r="4" fill="${color(s.slot)}"/>`)
232  }
233
234  // The end of the main series: a dot and its value.
235  const main = labelled[0]
236  if (spec.endLabel && main) {
237    const i = lastFinite(main)
238    if (i >= 0) {
239      const x = sx(main.xs[i]!), y = sy(main.ys[i]!)
240      parts.push(`<circle class="d" cx="${r1(x)}" cy="${r1(y)}" r="4" fill="${color(main.slot)}"/>`)
241      parts.push(`<text class="v" x="${r1(x + 8)}" y="${r1(Math.min(top + ph, Math.max(top + 8, y + 4)))}">${esc(fmt(main.ys[i]!) + (spec.unit ?? ''))}</text>`)
242    }
243  }
244
245  if (spec.marker && Number.isFinite(spec.marker.x) && Number.isFinite(spec.marker.y)) {
246    const x = sx(spec.marker.x), y = sy(spec.marker.y)
247    const c = color(spec.marker.slot)
248    parts.push(`<circle cx="${r1(x)}" cy="${r1(y)}" r="8" fill="none" stroke="${c}" stroke-width="1.5"/>`)
249    parts.push(`<circle class="d" cx="${r1(x)}" cy="${r1(y)}" r="4" fill="${c}"/>`)
250    const above = y - top > 22
251    const anchor = x > left + pw * 0.75 ? 'end' : x < left + pw * 0.25 ? 'start' : 'middle'
252    parts.push(`<text class="v" x="${r1(x)}" y="${r1(above ? y - 13 : y + 22)}" text-anchor="${anchor}">${esc(spec.marker.label)}</text>`)
253  }
254
255  for (const it of legend) {
256    const x = left + it.x
257    const y = 10 + it.row * 18
258    const stroke = it.s.isRaw ? 'stroke="var(--muted)" stroke-opacity=".6"' : `stroke="${color(it.s.slot)}"`
259    parts.push(`<line x1="${r1(x)}" x2="${r1(x + 14)}" y1="${y}" y2="${y}" ${stroke} stroke-width="2" stroke-linecap="round"/>`)
260    parts.push(`<text class="k" x="${r1(x + 19)}" y="${y + 4}">${esc(it.text)}</text>`)
261  }
262
263  const head = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${W} ${H}" width="${W}" height="${H}"><style>${STYLE}</style>`
264  const body = parts.join('')
265  // The hover layer, thinned until the whole fits.
266  const tipped = series.filter(s => !s.noTip)
267  // No more bands than the densest series has points: each band snaps to one.
268  const points = Math.max(0, ...tipped.map(s => s.xs.length))
269  for (let bins = Math.min(points, 72, Math.max(12, Math.floor(60_000 / (320 + 150 * tipped.length)))); bins >= 2; bins = Math.floor(bins / 2)) {
270    const hover = hoverLayer(tipped, bins, { left, top, pw, ph, sx, sy, xMin, xMax, xf, xName: spec.xName ?? 'step', unit: spec.unit ?? '', W })
271    const svg = `${head}${body}${hover}</svg>`
272    if (svg.length <= BUDGET) return svg
273  }
274  return `${head}${body}</svg>`
275}
276
277function lastFinite(s: Series): number {
278  for (let i = s.xs.length - 1; i >= 0; i--) if (Number.isFinite(s.xs[i]) && Number.isFinite(s.ys[i])) return i
279  return -1
280}
281
282/** Index of the point of `s` nearest `x` (xs ascending), or -1. */
283function nearest(s: Series, x: number): number {
284  let lo = 0, hi = s.xs.length - 1
285  if (hi < 0) return -1
286  while (lo < hi) {
287    const mid = (lo + hi) >> 1
288    if (s.xs[mid]! < x) lo = mid + 1
289    else hi = mid
290  }
291  const i = lo > 0 && Math.abs(s.xs[lo - 1]! - x) <= Math.abs(s.xs[lo]! - x) ? lo - 1 : lo
292  return Number.isFinite(s.ys[i]) ? i : -1
293}
294
295type Frame = {
296  left: number; top: number; pw: number; ph: number; W: number
297  sx: (x: number) => number; sy: (y: number) => number
298  xMin: number; xMax: number; xf: (x: number) => string; xName: string; unit: string
299}
300
301/** Vertical bands across the plot; hovering one shows its crosshair, the series' dots and a tooltip. */
302function hoverLayer(series: readonly Series[], bins: number, f: Frame): string {
303  const out: string[] = []
304  const bw = f.pw / bins
305  for (let b = 0; b < bins; b++) {
306    const xv = f.xMin + ((b + 0.5) / bins) * (f.xMax - f.xMin)
307    const hits = series.map(s => ({ s, i: nearest(s, xv) })).filter(h => h.i >= 0)
308    if (!hits.length) continue
309    const snap = hits[0]!.s.xs[hits[0]!.i]!
310    const cx = r1(f.sx(snap))
311    const lines = hits.map(h => ({ label: h.s.label, value: fmt(h.s.ys[h.i]!) + f.unit, slot: h.s.slot, y: f.sy(h.s.ys[h.i]!), isRaw: !!h.s.isRaw }))
312    const head = `${f.xName} ${f.xf(snap)}`
313    const tw = Math.max(head.length, ...lines.map(l => l.label.length + l.value.length + 2)) * 6.4 + 26
314    const th = 18 + lines.length * 15
315    const tx = cx + 12 + tw > f.left + f.pw ? cx - 12 - tw : cx + 12
316    const tyTop = f.top + 2
317    const dots = lines.filter(l => !l.isRaw).map(l => `<circle class="d" cx="${cx}" cy="${r1(l.y)}" r="4" fill="${color(l.slot)}"/>`).join('')
318    const rows = lines.map((l, k) => {
319      const y = r1(tyTop + 30 + k * 15)
320      const key = l.isRaw ? 'stroke="var(--muted)"' : `stroke="${color(l.slot)}"`
321      return `<line x1="${r1(tx + 8)}" x2="${r1(tx + 18)}" y1="${r1(y - 4)}" y2="${r1(y - 4)}" ${key} stroke-width="2"/><text class="tt" x="${r1(tx + 22)}" y="${y}">${esc(l.label)} <tspan font-weight="600">${esc(l.value)}</tspan></text>`
322    }).join('')
323    out.push(
324      `<g class="b"><rect x="${r1(f.left + b * bw)}" y="${f.top}" width="${r1(bw + 0.5)}" height="${r1(f.ph)}" fill="#000" fill-opacity="0"/>`
325      + `<g class="h" pointer-events="none"><line class="x" x1="${cx}" x2="${cx}" y1="${f.top}" y2="${r1(f.top + f.ph)}"/>${dots}`
326      + `<rect class="tip" x="${r1(tx)}" y="${r1(tyTop)}" width="${r1(tw)}" height="${th}" rx="6"/>`
327      + `<text class="tm" x="${r1(tx + 8)}" y="${r1(tyTop + 14)}">${esc(head)}</text>${rows}</g></g>`,
328    )
329  }
330  return out.join('')
331}
332
333/** A tiny trend line for a job card: no axes, the last point dotted. */
334export function sparkline(ys: readonly number[], slot = 0, width = 120, height = 28): string | null {
335  const v = ys.filter(Number.isFinite)
336  if (v.length < 2) return null
337  const d = decimate(v.map((_, i) => i), v, width)
338  let lo = Math.min(...d.ys), hi = Math.max(...d.ys)
339  if (hi === lo) { lo -= 1; hi += 1 }
340  const pad = 4
341  const sx = (i: number) => pad + (i / (v.length - 1)) * (width - 2 * pad)
342  const sy = (y: number) => pad + (1 - (y - lo) / (hi - lo)) * (height - 2 * pad)
343  const path = d.xs.map((x, i) => `${i ? 'L' : 'M'}${r1(sx(x))} ${r1(sy(d.ys[i]!))}`).join('')
344  const lx = sx(v.length - 1), ly = sy(v[v.length - 1]!)
345  return `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${width} ${height}" width="${width}" height="${height}"><style>${STYLE}</style>`
346    + `<path class="l" stroke-width="1.5" stroke="${color(slot)}" d="${path}"/><circle class="d" cx="${r1(lx)}" cy="${r1(ly)}" r="3" fill="${color(slot)}"/></svg>`
347}
348
349/** A thin progress bar: the track one step off the surface, the fill in slot 0 (or critical). */
350export function progressBar(fraction: number, width = 160, isCritical = false): string {
351  const f = Math.max(0, Math.min(1, fraction))
352  const h = 6
353  const fill = isCritical ? 'var(--critical)' : 'var(--s0)'
354  return `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${width} ${h}" width="${width}" height="${h}"><style>${STYLE}</style>`
355    + `<rect x="0" y="0" width="${width}" height="${h}" rx="3" fill="var(--grid)"/>`
356    + (f > 0 ? `<rect x="0" y="0" width="${r1(Math.max(h, f * width))}" height="${h}" rx="3" fill="${fill}"/>` : '')
357    + '</svg>'
358}
359

1// Data types from pace-line/types/index.d.ts (the shared hooks/jobs files import them),
2// then job-watch's own state.
3
4export type Limit = { kind: string; percentUsed: number; resetsAt?: string }
5
6/** A Slurm allocation the session runs inside, from its environment. */
7export type Slurm = {
8  job: string
9  gpus: string | null
10  gpusOnNode: string | null
11  /** When the job ends, in ms: SLURM_JOB_END_TIME, then the panel's scontrol. */
12  endsAt: number | null
13}
14
15export type JobInfo = {
16  id: string
17  name: string
18  state: string
19  partition: string
20  account: string
21  /** The batch host: where `ssh` lands in the job. */
22  node: string
23  nodeList: string
24  runSeconds: number | null
25  limitSeconds: number | null
26  cpus: string | null
27  mem: string | null
28  gpus: string | null
29}
30
31export type GpuProc = { pid: string; name: string; memMiB: number | null }
32
33export type Gpu = {
34  index: string
35  name: string
36  util: number | null
37  memUsedMiB: number | null
38  memTotalMiB: number | null
39  tempC: number | null
40  powerW: number | null
41  powerLimitW: number | null
42  procs: GpuProc[]
43}
44
45/** What the job panel draws; null fields are still loading or unavailable. */
46export type JobPanel = {
47  job: JobInfo | null
48  gpus: Gpu[] | null
49  gpuVia: 'local' | 'ssh' | null
50  isLoading: boolean
51  error: string | null
52  updatedAt: number | null
53}
54
55export type Snapshot = {
56  model: string | null
57  effort: string | null
58  folder: string | null
59  branch: string | null
60  contextPercent: number | null
61  costUsd: number | null
62  startedAt: number | null
63  limits: Limit[]
64  slurm: Slurm | null
65}
66
67
68/** One numeric series from metrics.jsonl: parallel step and value arrays. */
69export type Series = { step: number[]; value: number[] }
70
71/** What pace-line has read of runs/<jobid>/metrics.jsonl. */
72export type Metrics = {
73  /** Bytes read so far: the next read starts here. */
74  offset: number
75  keys: Record<string, Series>
76  lastStep: number | null
77  totalSteps: number | null
78  epoch: number | null
79  totalEpochs: number | null
80  firstTime: number | null
81  firstStep: number | null
82  lastTime: number | null
83  /** The first metric that went NaN or infinite, as `key@step`. */
84  nonFinite: string | null
85}
86
87export type Ckpt = {
88  name: string
89  path: string
90  mtimeMs: number
91  sizeBytes: number | null
92  step: number | null
93  epoch: number | null
94}
95
96export type NodeGpus = { node: string; gpus: Gpu[] }
97
98/** Lines from the log that mean trouble, newest first kept. */
99export type LogFlags = { oom: string | null; traceback: string | null; nccl: string | null; srun: string | null }
100
101/** A batch job of the user's, as squeue, scontrol, sacct and its files tell it. */
102export type BatchJob = {
103  /** As squeue shows it: "123" or "123_4" for an array task. */
104  id: string
105  /** The unique id ($SLURM_JOB_ID inside the job), which names runs/<id>/. */
106  jobId: string
107  arrayId: string | null
108  name: string
109  state: string
110  reason: string | null
111  elapsedS: number | null
112  limitS: number | null
113  nodes: number
114  nodeList: string
115  gpus: number | null
116  gpuType: string | null
117  submittedAt: number | null
118  /** When pace-line first saw it running, less its elapsed time. */
119  startedAt: number | null
120  workDir: string | null
121  stdout: string | null
122  command: string | null
123  submitLine: string | null
124  endedAt: number | null
125  exitCode: string | null
126  logSize: number | null
127  logChangedAt: number | null
128  lastLines: string[]
129  progress: { k: number; n: number } | null
130  flags: LogFlags
131  metrics: Metrics | null
132  ckpt: Ckpt | null
133  gpuReadings: NodeGpus[] | null
134  gpusAt: number | null
135  idleSince: number | null
136  resumedFrom: string | null
137  resumedAs: string | null
138}
139
140export type Disk = { mount: string; size: number; used: number; avail: number }
141
142export type AlertKind = 'ended' | 'oom' | 'traceback' | 'nccl' | 'nonFinite' | 'silent' | 'idle' | 'pending' | 'nearLimit' | 'staleCkpt' | 'resumed' | 'resumeStopped'
143
144export type Alert = { job: string; kind: AlertKind; level: 'info' | 'warn' | 'error'; text: string; at: number }
145
146/** Every batch job pane and band draw. */
147export type Jobs = {
148  jobs: BatchJob[]
149  /** A job id, or `array:<id>` for a sweep. */
150  selected: string | null
151  updatedAt: number | null
152  error: string | null
153  disks: Disk[]
154  alerts: Alert[]
155}
156
157
158// ── job-watch ────────────────────────────────────────────────────────────
159
160/** One reading of one GPU, as percentages so every GPU shares the 0–100 axis. */
161export type GpuPoint = { util: number | null; mem: number | null; power: number | null }
162
163/** One poll's GPU readings of a job, keyed `node:index`. */
164export type GpuSample = { t: number; gpus: Record<string, GpuPoint> }
165
166/** How the last ssh round trip to the cluster went. */
167export type Conn = {
168  host: string
169  /** null before the first poll has answered. */
170  isOk: boolean | null
171  error: string | null
172  lastOkAt: number | null
173  latencyMs: number | null
174  failures: number
175  isPolling: boolean
176}
177
178declare module 'claude-code' {
179  interface PluginState {
180    'job-watch': {
181      jobs: Jobs
182      conn: Conn
183      /** GPU history per job id, oldest first. */
184      history: Record<string, GpuSample[]>
185      now: number
186    }
187  }
188}
189