sprc-slurm-mods

1import { atom, read } from 'claude-code'
2import type { EngineInterface, PluginState, Register } from 'claude-code'
3
4import type { Followed, JobReport, Snapshot, View } from '../types'
5import { checkBash, submittedJobs } from './guards'
6import {
7  TERMINAL_STATES,
8  expandNodes,
9  fetchReport,
10  fetchScontrol,
11  fetchSnapshot,
12  gpuTotals,
13  hasJob,
14  isBad,
15  reportLine,
16  tailFile,
17} from './slurm'
18import type { Run } from './slurm'
19import { JobView, Overview, explainReason } from './ui'
20import type { Actions } from './ui'
21
22type $ = EngineInterface
23
24const PANE = 'sprc'
25const PLUGIN = 'sprc-slurm-mods'
26const T = (name: string) => `mcp__${PLUGIN}__${name}`
27
28const REF_snapshot = { plugin: 'sprc-slurm-mods', key: 'snapshot' } as const
29const REF_followed = { plugin: 'sprc-slurm-mods', key: 'followed' } as const
30const REF_finished = { plugin: 'sprc-slurm-mods', key: 'finished' } as const
31const REF_view = { plugin: 'sprc-slurm-mods', key: 'view' } as const
32const REF_detail = { plugin: 'sprc-slurm-mods', key: 'detail' } as const
33const REF_budget = { plugin: 'sprc-slurm-mods', key: 'budget' } as const
34const REF_isEnabled = { plugin: 'sprc-slurm-mods', key: 'isEnabled' } as const
35const REF_hasOpened = { plugin: 'sprc-slurm-mods', key: 'hasOpened' } as const
36const snapshot = atom(REF_snapshot, null)
37const followed = atom(REF_followed, [])
38const finished = atom(REF_finished, [])
39const view = atom(REF_view, { kind: 'overview' } as View)
40const detail = atom(REF_detail, null)
41const budget = atom(REF_budget, null)
42const isEnabled = atom(REF_isEnabled, false)
43const hasOpened = atom(REF_hasOpened, false)
44
45// Slurm's own words and cluster phrases only: "job", "node" or "queue" alone
46// mostly mean CI, Node.js or a web queue.
47const SLURMY =
48  /\b(slurm|sbatch|salloc|srun|squeue|sacct|sinfo|scancel|scontrol|gpus?|h100s?|sprc\w*|sprlab005|my jobs?|the queue|the cluster|compute nodes?|scavenger|expedite)\b/i
49
50// Reads within one dispatch see the state as it stood when the dispatch
51// began, so a value written mid-hook reads stale. This module is the only
52// writer of its values, so it keeps them here too: `get` answers from the
53// cache, `put` writes both. A reload empties the cache; the next `get` loads
54// from the host. Render hooks still `read` the atoms, which subscribes them.
55type S = PluginState['sprc-slurm-mods']
56const mem: { [K in keyof S]?: S[K] } = {}
57async function load<K extends keyof S>($: $, k: K): Promise<S[K]> {
58  switch (k) {
59    case 'snapshot':
60      return (await read($, snapshot)) as S[K]
61    case 'followed':
62      return (await read($, followed)) as S[K]
63    case 'finished':
64      return (await read($, finished)) as S[K]
65    case 'view':
66      return (await read($, view)) as S[K]
67    case 'detail':
68      return (await read($, detail)) as S[K]
69    case 'budget':
70      return (await read($, budget)) as S[K]
71    case 'isEnabled':
72      return (await read($, isEnabled)) as S[K]
73    case 'hasOpened':
74      return (await read($, hasOpened)) as S[K]
75  }
76  throw new Error(`unknown state key ${String(k)}`)
77}
78
79async function get<K extends keyof S>($: $, k: K): Promise<S[K]> {
80  if (!(k in mem)) mem[k] = await load($, k)
81  return mem[k] as S[K]
82}
83
84async function put<K extends keyof S>($: $, k: K, fn: (v: S[K]) => S[K]): Promise<void> {
85  const v = fn(await get($, k))
86  mem[k] = v
87  switch (k) {
88    case 'snapshot':
89      await $.state.set(REF_snapshot, v as S['snapshot'])
90      break
91    case 'followed':
92      await $.state.set(REF_followed, v as S['followed'])
93      break
94    case 'finished':
95      await $.state.set(REF_finished, v as S['finished'])
96      break
97    case 'view':
98      await $.state.set(REF_view, v as S['view'])
99      break
100    case 'detail':
101      await $.state.set(REF_detail, v as S['detail'])
102      break
103    case 'budget':
104      await $.state.set(REF_budget, v as S['budget'])
105      break
106    case 'isEnabled':
107      await $.state.set(REF_isEnabled, v as S['isEnabled'])
108      break
109    case 'hasOpened':
110      await $.state.set(REF_hasOpened, v as S['hasOpened'])
111      break
112  }
113}
114
115// Module-local; a reload starts these over, and the next poll refills them.
116const cfg = { pollMs: 30_000, autoOpen: true, guards: 'enforce', liveContext: true }
117let user = ''
118let isLoginNode = false
119let isPolling = false
120let lastBudgetAt = 0
121let lastPollAt = 0
122// The last snapshot that read cleanly: jobs are diffed against it, so a
123// failed poll in between does not lose the jobs that ended meanwhile.
124let lastGood: Snapshot | null = null
125// What the prompt hook last attached, so an unchanged cluster is not re-sent.
126let lastContext = { text: '', at: 0 }
127// Jobs an sprc_wait call is blocked on: that call reports their end, so the
128// background poll must not post it a second time.
129const awaited = new Set<string>()
130
131function runner($: $): Run {
132  return (argv, init) => $.process.run(argv, { timeoutMs: init?.timeoutMs ?? 20_000 })
133}
134
135// ---- polling ---------------------------------------------------------
136
137async function refreshDetail($: $, id: string): Promise<void> {
138  const run = runner($)
139  const now = await $.clock.now()
140  const fields = (await fetchScontrol(run, id)) ?? {}
141  const state = fields['JobState']
142  let report: JobReport | undefined
143  if (!state || TERMINAL_STATES.has(state)) report = (await fetchReport(run, id, now)) ?? undefined
144  let stdout = fields['StdOut']
145  if (!stdout && report) stdout = report.stdout
146  const log = stdout ? await tailFile(run, stdout, 60) : []
147  await put($, 'detail', () => ({ id, fields, log, report, at: now }))
148}
149
150async function onJobsEnded($: $, ended: readonly string[], prevMine: Snapshot['mine']): Promise<void> {
151  const run = runner($)
152  const now = await $.clock.now()
153  const follows = await get($, 'followed')
154  const done: { report: JobReport; wake: boolean }[] = []
155  for (const id of ended) {
156    const report =
157      (await fetchReport(run, id, now)) ??
158      ({
159        id,
160        name: prevMine.find(j => j.id === id)?.name ?? '',
161        state: 'ENDED',
162        exitCode: '?',
163        elapsed: '?',
164        limit: '?',
165        nodes: '',
166        cpus: 0,
167        gpus: 0,
168        reqMem: '?',
169        maxRss: '?',
170        advice: [],
171        endedAt: now,
172      } satisfies JobReport)
173    await put($, 'finished', list => [report, ...list.filter(r => r.id !== id)].slice(0, 20))
174    const follow = follows.find(f => f.id === id)
175    if (follow && !awaited.has(id)) done.push({ report, wake: follow.wake })
176  }
177  if (follows.some(f => ended.includes(f.id))) await put($, 'followed', list => list.filter(f => !ended.includes(f.id)))
178  if (!done.length) return
179  // One toast, one note and at most one woken turn per poll, however many ended.
180  const bad = done.filter(d => isBad(d.report.state)).length
181  const [first] = done
182  $.ui.toast(
183    done.length === 1
184      ? `${bad ? '✗' : '✓'} job ${first!.report.id} ${first!.report.state} (${first!.report.elapsed})`
185      : `${done.length} followed jobs ended: ${done.length - bad} ✓ ${bad} ✗`,
186    { timeoutMs: 8000 },
187  )
188  const note = done
189    .map(({ report: r }) => `[sprc] Followed ${reportLine(r)}.${r.advice.length ? ' Right-sizing: ' + r.advice.join('; ') + '.' : ''}`)
190    .join('\n')
191  if (done.some(d => d.wake)) {
192    void $.prompt.submit({
193      text: `${note}\n${done.length === 1 ? 'The job you were following has' : 'Jobs you were following have'} ended. Check the result (use sprc_job for the log and details) and report back${bad ? ', diagnosing the failure' : ''}.`,
194    } as Parameters<$['prompt']['submit']>[0])
195  } else {
196    await $.session
197      .append({ message: { type: 'user', content: [{ type: 'text', text: note }] } })
198      .catch(err => $.ui.log(`sprc: job-end note not added: ${String(err)}`, { to: 'debug' }))
199  }
200}
201
202/**
203 * `isTimer`: a background tick. With no jobs of yours and nothing followed,
204 * those run at a quarter of the configured rate; anything that needs fresh
205 * data (a tool, /sprc, a cluster-ish prompt) polls on its own.
206 */
207async function poll($: $, isTimer = false): Promise<void> {
208  if (isPolling || !user) return
209  const now = await $.clock.now()
210  if (isTimer && now - lastPollAt < 4 * cfg.pollMs - 1000) {
211    const snap = await get($, 'snapshot')
212    const isIdle = !!snap && !snap.error && snap.mine.length === 0 && (await get($, 'followed')).length === 0
213    if (isIdle) return
214  }
215  isPolling = true
216  lastPollAt = now
217  try {
218    const snap = await fetchSnapshot(runner($), user, now)
219    await put($, 'snapshot', () => snap)
220    if (!snap.error) {
221      const prev = lastGood
222      lastGood = snap
223      const follows = await get($, 'followed')
224      // Ended: was in the last good read and is gone, or is followed (since
225      // before this read began) and absent: that also catches a job that ended
226      // before it was followed, or an array job whose tasks all finished.
227      const ended = new Set(prev ? prev.mine.filter(j => !hasJob(snap.mine, j.id)).map(j => j.id) : [])
228      for (const f of follows) if ((f.since ?? 0) < now && !hasJob(snap.mine, f.id)) ended.add(f.id)
229      if (ended.size) await onJobsEnded($, [...ended], prev?.mine ?? [])
230      // Started: a followed job went PD -> R.
231      const followIds = new Set(follows.map(f => f.id))
232      for (const j of snap.mine) {
233        const was = prev?.mine.find(p => p.id === j.id)
234        if (was && was.state === 'PD' && j.state === 'R' && followIds.has(j.id)) {
235          $.ui.toast(`▶ job ${j.id} started on ${j.nodes}`)
236        }
237      }
238    }
239    setStatus($, snap)
240    const v = await get($, 'view')
241    if (v.kind === 'job' && (await $.ui.panes()).some(p => p.id === PANE)) await refreshDetail($, v.id)
242    if (now - lastBudgetAt > 5 * 60_000) {
243      lastBudgetAt = now
244      await refreshBudget($, now)
245    }
246  } catch (err) {
247    $.ui.log(`sprc poll failed: ${String(err)}`, { to: 'debug' })
248  } finally {
249    isPolling = false
250  }
251}
252
253async function refreshBudget($: $, now: number): Promise<void> {
254  const r = await $.process.run(['mybudget', '--no-color'], { timeoutMs: 15_000 }).catch(() => null)
255  if (!r || r.exitCode !== 0 || /not enrolled/i.test(r.stdout)) {
256    await put($, 'budget', () => null)
257    return
258  }
259  const lines = r.stdout
260    .split('\n')
261    .map(l => l.trimEnd())
262    .filter(l => l.trim() && !/^\s*-+\s*$/.test(l))
263  await put($, 'budget', () => ({ lines: lines.slice(0, 12), at: now }))
264}
265
266function setStatus($: $, snap: Snapshot): void {
267  if (snap.error) {
268    $.ui.status('sprc ▸ Slurm unreachable')
269    return
270  }
271  const g = gpuTotals(snap.nodes)
272  const r = snap.mine.filter(j => j.state === 'R').length
273  const pd = snap.mine.filter(j => j.state === 'PD').length
274  const mine = snap.mine.length ? ` · you ${r}R ${pd}PD` : ''
275  $.ui.status(`sprc ▸ ${g.usable}/${g.total} GPU free${g.idle > g.usable ? ` (+${g.idle - g.usable} stranded)` : ''}${mine} · queue ${snap.others.pending + pd}`)
276}
277
278function summary(snap: Snapshot | null): string {
279  if (!snap) return 'sprc: no data yet'
280  if (snap.error) return `sprc: Slurm unreachable (${snap.error})`
281  const g = gpuTotals(snap.nodes)
282  const r = snap.mine.filter(j => j.state === 'R')
283  const pd = snap.mine.filter(j => j.state === 'PD')
284  return `[sprc live] ${g.usable}/${g.total} GPUs free for a default GPU job${g.idle > g.usable ? ` (${g.idle - g.usable} more idle on nodes short of the 32 CPU + 375G that go with a GPU by default)` : ''} (${snap.nodes.map(n => `${n.name} ${n.gpus - n.gpusUsed}/${n.gpus} idle, ${n.cpusTotal - n.cpusAlloc} CPU free`).join('; ')}); your jobs: ${r.length} running, ${pd.length} pending; others: ${snap.others.running} running, ${snap.others.pending} pending.`
285}
286
287// ---- actions the pane's buttons call -----------------------------------
288
289function actionsFor($: $): Actions {
290  return {
291    refresh: () => void poll($),
292    open: id => {
293      void (async () => {
294        await put($, 'view', () => ({ kind: 'job', id }) as View)
295        await refreshDetail($, id)
296      })()
297    },
298    back: () => void put($, 'view', () => ({ kind: 'overview' }) as View),
299    cancel: id => {
300      void (async () => {
301        const answer = await $.ui.ask(`Cancel job ${id}?`, ['Keep it', 'scancel it']).catch(() => 'Keep it')
302        if (answer !== 'scancel it') return
303        const r = await $.process.run(['scancel', id])
304        $.ui.toast(r.exitCode === 0 ? `scancel ${id} sent` : `scancel failed: ${r.stderr.trim()}`)
305        await poll($)
306        await refreshDetail($, id)
307      })()
308    },
309    toggleFollow: id => {
310      void (async () => {
311        if ((await get($, 'followed')).some(f => f.id === id)) {
312          await put($, 'followed', list => list.filter(f => f.id !== id))
313          return
314        }
315        const refusal = await followRefusal($, id)
316        if (refusal) $.ui.toast(refusal)
317        else await followJob($, id, false, 'person')
318      })()
319    },
320    clearFinished: () => void put($, 'finished', () => []),
321  }
322}
323
324async function followJob($: $, id: string, wake: boolean, origin: Followed['origin']): Promise<void> {
325  const since = await $.clock.now()
326  await put($, 'followed', list => [...list.filter(f => f.id !== id), { id, wake, origin, since }])
327}
328
329/**
330 * Why job `id` cannot be followed, or null if it can: it must be yours and
331 * still queued or running. An ended one is recorded and its result returned,
332 * so the caller learns it now instead of waiting on an end that never comes.
333 */
334async function followRefusal($: $, id: string): Promise<string | null> {
335  const run = runner($)
336  const q = await run(['squeue', '-h', '-j', id, '-o', '%u'])
337  const owners = new Set(q.exitCode === 0 ? q.stdout.split('\n').map(l => l.trim()).filter(Boolean) : [])
338  if (owners.size) return owners.has(user) ? null : `Job ${id} is not yours; only your own jobs can be followed.`
339  const report = await fetchReport(run, id, await $.clock.now())
340  if (!report) return `Job ${id} is not queued or running, and sacct has no record of it. Check the id.`
341  if (report.user && report.user !== user) return `Job ${id} is not yours; only your own jobs can be followed.`
342  await put($, 'finished', list => [report, ...list.filter(r => r.id !== id)].slice(0, 20))
343  return `Job ${id} has already ended: ${reportLine(report)}${report.advice.length ? `\nRight-sizing: ${report.advice.join('; ')}` : ''}`
344}
345
346async function registerTools($: $): Promise<void> {
347  await $.tool.register({
348    name: 'sprc_status',
349    description:
350      'Live state of the sprc Slurm cluster: free GPUs per node, all of your queued/running jobs with pending reasons, queue depth, and your recently finished jobs with efficiency. Cheaper and more reliable than parsing squeue/sinfo yourself.',
351    inputSchema: { type: 'object', properties: {} },
352  })
353  await $.tool.register({
354    name: 'sprc_job',
355    description:
356      "Full lifecycle of one Slurm job (running, pending or finished): state and pending reason explained, submit/start/end times and queue wait, resources, working dir and log paths, the last lines of its stdout log, and for finished jobs CPU/memory/time efficiency with concrete right-sizing advice for the next submission.",
357    inputSchema: {
358      type: 'object',
359      properties: {
360        job_id: { type: 'string', description: 'The Slurm job id, e.g. "80240".' },
361        log_lines: { type: 'number', description: 'How many trailing log lines to include (default 40, max 200).' },
362        stderr: { type: 'boolean', description: 'Tail StdErr instead of StdOut (when they differ).' },
363      },
364      required: ['job_id'],
365    },
366  })
367  await $.tool.register({
368    name: 'sprc_wait',
369    description:
370      'Block until a Slurm job starts or ends (or changes state), up to 10 minutes per call, then return its state. Use this instead of sleep/squeue polling loops. If it times out, either call again or use sprc_follow and end your turn.',
371    inputSchema: {
372      type: 'object',
373      properties: {
374        job_id: { type: 'string' },
375        until: { type: 'string', enum: ['start', 'end', 'change'], description: 'Default "end".' },
376        timeout_minutes: { type: 'number', description: '1-10, default 10.' },
377      },
378      required: ['job_id'],
379    },
380  })
381  await $.tool.register({
382    name: 'sprc_follow',
383    description:
384      'Follow a Slurm job in the background: when it ends the user gets a notification and you get a note with its result and efficiency. With wake=true a new turn starts automatically when it ends so you can check the result (use when the user asked to be told / to have results checked when a long job finishes). Jobs submitted with sbatch in this session are followed automatically (without wake). Set stop=true to unfollow.',
385    inputSchema: {
386      type: 'object',
387      properties: {
388        job_id: { type: 'string' },
389        wake: { type: 'boolean', description: 'Start a new turn when the job ends. Default false.' },
390        stop: { type: 'boolean' },
391      },
392      required: ['job_id'],
393    },
394  })
395}
396
397export const register: Register = (on, options) => {
398  cfg.pollMs = Math.max(10, Number(options['pollSeconds'] ?? 30)) * 1000
399  cfg.autoOpen = options['autoOpen'] !== false
400  cfg.guards = String(options['guards'] ?? 'enforce')
401  cfg.liveContext = options['liveContext'] !== false
402
403  // ---- session start -----------------------------------------------------
404
405  on('session.start', async ($, e, next) => {
406    const started = await next(e)
407    // Only on the sprc cluster: another site's Slurm, or no Slurm, gets nothing.
408    const probe = await $.process.run(['scontrol', 'show', 'config'], { timeoutMs: 5000 }).catch(() => null)
409    if (!probe || probe.exitCode !== 0 || !/^ClusterName\s*=\s*sprc\s*$/m.test(probe.stdout)) {
410      await put($, 'isEnabled', () => false)
411      return started
412    }
413    user = (await $.env.get('USER')) ?? ''
414    const host = await $.process.run(['hostname', '-s']).catch(() => null)
415    isLoginNode = (host?.stdout.trim() ?? '') === 'sprlab005'
416    await put($, 'isEnabled', () => true)
417
418    await $.command.register({
419      name: 'sprc',
420      description: 'sprc cluster dashboard: /sprc, /sprc job <id>, /sprc follow <id>, /sprc refresh, /sprc close',
421      argumentHint: '[job <id> | follow <id> | refresh | close]',
422      immediate: true,
423    })
424    await registerTools($)
425
426    await poll($)
427    $.clock.every(cfg.pollMs, () => void poll($, true))
428    // session.start runs again on a reload or worker respawn: open the pane
429    // the first time only, so one the person closed stays closed.
430    if (cfg.autoOpen && e.isInteractive && !(await get($, 'hasOpened'))) {
431      await put($, 'hasOpened', () => true)
432      void $.ui.open({ id: PANE, title: 'sprc' })
433    }
434    return started
435  })
436
437  // ---- slash command -------------------------------------------------------
438
439  on('command.run', { command: 'sprc' }, async ($, e) => {
440    if (!(await get($, 'isEnabled'))) return { text: 'sprc: Slurm commands are not available on this machine.' }
441    const [sub, arg] = e.args.trim().split(/\s+/)
442    if (sub === 'close') {
443      await put($, 'view', () => ({ kind: 'overview' }) as View)
444      await $.ui.close({ id: PANE })
445      return { text: 'sprc pane closed.' }
446    }
447    if (sub === 'refresh') {
448      await poll($)
449      return { text: summary(await get($, 'snapshot')) }
450    }
451    if ((sub === 'follow' || sub === 'unfollow') && arg) {
452      if (sub === 'follow') {
453        const refusal = await followRefusal($, arg)
454        if (refusal) return { text: `sprc: ${refusal}` }
455        await followJob($, arg, false, 'person')
456      } else await put($, 'followed', list => list.filter(f => f.id !== arg))
457      return { text: `sprc: ${sub === 'follow' ? 'following' : 'stopped following'} job ${arg}.` }
458    }
459    if ((sub === 'job' || /^\d+$/.test(sub ?? '')) && (arg ?? sub)) {
460      const id = sub === 'job' ? arg! : sub!
461      await put($, 'view', () => ({ kind: 'job', id }) as View)
462      await $.ui.open({ id: PANE, title: 'sprc' })
463      void refreshDetail($, id)
464      return { text: `sprc: showing job ${id}.` }
465    }
466    await put($, 'view', () => ({ kind: 'overview' }) as View)
467    const opened = await $.ui.open({ id: PANE, title: 'sprc' })
468    return { text: opened.isPlaced ? 'sprc dashboard opened.' : summary(await get($, 'snapshot')) }
469  })
470
471  // ---- the pane ------------------------------------------------------------
472
473  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
474    const el = $.ui.resolve(e)
475    const columns = e.props.bodyColumns
476    const v = await read($, view)
477    const snap = await read($, snapshot)
478    const follows = await read($, followed)
479    const now = await $.clock.now()
480    if (!(await read($, isEnabled))) {
481      const { Text } = el
482      return <Text dimColor>Slurm is not available on this machine.</Text>
483    }
484    if (v.kind === 'job') {
485      return JobView(el, {
486        id: v.id,
487        detail: await read($, detail),
488        snap,
489        isFollowed: follows.some(f => f.id === v.id),
490        columns,
491        logRows: Math.max(5, (e.viewport?.rows ?? 40) - 24),
492        actions: actionsFor($),
493      })
494    }
495    return Overview(el, {
496      snap,
497      finished: await read($, finished),
498      followed: follows,
499      budget: await read($, budget),
500      now,
501      columns,
502      actions: actionsFor($),
503    })
504  })
505
506  // ---- model tools -----------------------------------------------------------
507
508  on('tool.call', { tool: 'mcp__sprc-slurm-mods__sprc_status' }, async $ => {
509    await poll($)
510    const snap = await get($, 'snapshot')
511    const fin = await get($, 'finished')
512    const lines = [summary(snap)]
513    // Bounded: a campaign of hundreds of jobs must not become hundreds of lines.
514    const mine = snap?.mine ?? []
515    const running = mine.filter(j => j.state !== 'PD')
516    const pending = mine.filter(j => j.state === 'PD')
517    for (const j of running.slice(0, 12)) {
518      lines.push(`  ${j.id} ${j.state.padEnd(2)} ${j.name} · ${j.elapsed}/${j.limit} on ${j.nodes} · ${j.gpus} GPU ${j.cpus} CPU ${j.mem} · qos ${j.qos}`)
519    }
520    if (running.length > 12) lines.push(`  … ${running.length - 12} more running`)
521    for (const j of pending.slice(0, 8)) {
522      lines.push(
523        `  ${j.id} PD ${j.name} · ${explainReason(j.reason)}${j.start !== 'N/A' ? ` · est. start ${j.start}` : ''} · ${j.gpus} GPU ${j.cpus} CPU ${j.mem} · qos ${j.qos}`,
524      )
525    }
526    if (pending.length > 8) {
527      const why = new Map<string, number>()
528      for (const j of pending.slice(8)) why.set(j.reason, (why.get(j.reason) ?? 0) + 1)
529      lines.push(`  … ${pending.length - 8} more pending (${[...why].map(([k, n]) => `${explainReason(k)} ×${n}`).join(', ')})`)
530    }
531    if (fin.length) {
532      lines.push(`Recently finished (yours, since this session began${fin.length > 5 ? `; newest 5 of ${fin.length}` : ''}):`)
533      for (const r of fin.slice(0, 5)) lines.push(`  ${reportLine(r)}${r.advice.length ? ' → ' + r.advice.join('; ') : ''}`)
534    }
535    const follows = await get($, 'followed')
536    if (follows.length) lines.push(`Following: ${follows.map(f => f.id + (f.wake ? ' (wake)' : '')).join(', ')}`)
537    return { result: lines.join('\n') }
538  })
539
540  on('tool.call', { tool: 'mcp__sprc-slurm-mods__sprc_job' }, async ($, e) => {
541    const args = e as unknown as { job_id: string; log_lines?: number; stderr?: boolean }
542    const id = String(args.job_id ?? '').trim()
543    if (!/^\d+(_\d+)?$/.test(id)) return { deny: `sprc_job: "${id}" is not a job id.` }
544    const run = runner($)
545    const now = await $.clock.now()
546    const fields = await fetchScontrol(run, id)
547    const state = fields?.['JobState']
548    const report = !state || TERMINAL_STATES.has(state) ? await fetchReport(run, id, now) : null
549    if (!fields && !report) return { result: `Job ${id}: unknown to scontrol and sacct.` }
550    const out: string[] = []
551    if (fields) {
552      const f = fields
553      out.push(`Job ${id} "${f['JobName']}" ${f['JobState']}${f['Reason'] && f['Reason'] !== 'None' ? ` (${f['Reason']}: ${explainReason(f['Reason'])})` : ''}`)
554      out.push(`  user ${f['UserId']} · partition ${f['Partition']} · qos ${f['QOS']} · nodes ${f['NodeList'] || '-'}`)
555      out.push(`  TRES ${f['AllocTRES'] || f['ReqTRES'] || '?'} · time ${f['RunTime']} of ${f['TimeLimit']}`)
556      out.push(`  submitted ${f['SubmitTime']} · start ${f['StartTime']} · end ${f['EndTime']} · restarts ${f['Restarts'] ?? 0} · requeue ${f['Requeue'] ?? '?'}`)
557      out.push(`  workdir ${f['WorkDir']}`)
558      out.push(`  command ${f['Command']}`)
559      out.push(`  stdout ${f['StdOut']}${f['StdErr'] !== f['StdOut'] ? `\n  stderr ${f['StdErr']}` : ''}`)
560      if (f['ExitCode']) out.push(`  exit ${f['ExitCode']}`)
561    }
562    if (report) {
563      out.push(`Accounting: ${reportLine(report)}`)
564      if (report.advice.length) out.push(`Right-sizing: ${report.advice.join('; ')}`)
565      else if (report.state === 'COMPLETED') out.push('Right-sizing: request looked about right.')
566    }
567    const path = args.stderr ? fields?.['StdErr'] : fields?.['StdOut']
568    if (path) {
569      const n = Math.max(1, Math.min(200, Number(args.log_lines ?? 40)))
570      const log = await tailFile(run, path, n)
571      out.push(`--- last ${log.length} lines of ${path} ---`, ...log)
572    } else if (!fields) {
573      out.push('(scontrol has forgotten this job, so the log path is unknown; check the job script for --output)')
574    }
575    return { result: out.join('\n') }
576  })
577
578  on('tool.call', { tool: 'mcp__sprc-slurm-mods__sprc_wait' }, async ($, e) => {
579    const args = e as unknown as { job_id: string; until?: string; timeout_minutes?: number }
580    const id = String(args.job_id ?? '').trim()
581    if (!/^\d+(_\d+)?$/.test(id)) return { deny: `sprc_wait: "${id}" is not a job id.` }
582    const until = args.until === 'start' || args.until === 'change' ? args.until : 'end'
583    const minutes = Math.max(1, Math.min(10, Number(args.timeout_minutes ?? 10)))
584    // The loop runs on the host (squeue every 15 s), so the hook's own time
585    // budget is not spent while it waits.
586    const script = `
587id="$1"; until="$2"; deadline=$(( $(date +%s) + $3 ))
588st() { squeue -h -j "$id" -o %T 2>/dev/null | sort -u | paste -sd, -; }
589first=$(st)
590while :; do
591  s=$(st)
592  if [ -z "$s" ]; then echo "ENDED"; exit 0; fi
593  case "$until" in
594    start) [ "$s" != PENDING ] && { echo "$s"; exit 0; } ;;
595    change) [ "$s" != "$first" ] && { echo "$s"; exit 0; } ;;
596  esac
597  [ $(date +%s) -ge $deadline ] && { echo "TIMEOUT $s"; exit 0; }
598  sleep 15
599done`
600    awaited.add(id)
601    let outcome: string
602    try {
603      const r = await $.process.run(['bash', '-c', script, 'sprc_wait', id, until, String(minutes * 60)], {
604        timeoutMs: minutes * 60_000 + 30_000,
605      })
606      outcome = r.stdout.trim()
607      // This call reports the end itself: unfollow first so the poll does not
608      // add the same result again as a note.
609      if (outcome === 'ENDED') await put($, 'followed', list => list.filter(f => f.id !== id))
610      await poll($)
611    } finally {
612      awaited.delete(id)
613    }
614    if (outcome === 'ENDED') {
615      const report = await fetchReport(runner($), id, await $.clock.now())
616      return {
617        result: report
618          ? `Job ${id} has ended: ${reportLine(report)}${report.advice.length ? `\nRight-sizing: ${report.advice.join('; ')}` : ''}\nUse sprc_job for its log.`
619          : `Job ${id} is no longer in the queue (sacct has no record yet).`,
620      }
621    }
622    if (outcome.startsWith('TIMEOUT')) {
623      return {
624        result: `Job ${id} still ${outcome.slice(8) || 'unknown'} after ${minutes} min. Call sprc_wait again, or sprc_follow (wake=true) and end your turn so the user is not blocked.`,
625      }
626    }
627    return { result: `Job ${id} is now ${outcome}.` }
628  })
629
630  on('tool.call', { tool: 'mcp__sprc-slurm-mods__sprc_follow' }, async ($, e) => {
631    const args = e as unknown as { job_id: string; wake?: boolean; stop?: boolean }
632    const id = String(args.job_id ?? '').trim()
633    if (!/^\d+(_\d+)?$/.test(id)) return { deny: `sprc_follow: "${id}" is not a job id.` }
634    if (args.stop) {
635      await put($, 'followed', list => list.filter(f => f.id !== id))
636      return { result: `Stopped following job ${id}.` }
637    }
638    const refusal = await followRefusal($, id)
639    if (refusal) return { result: refusal }
640    await followJob($, id, args.wake === true, 'tool')
641    return {
642      result: args.wake
643        ? `Following job ${id}. When it ends, a new turn will start with its result; you can end this turn now.`
644        : `Following job ${id}. When it ends the user is notified and a note with its result is added to the conversation.`,
645    }
646  })
647
648  // ---- guardrails and sbatch tracking on Bash --------------------------------
649
650  on('tool.call', { tool: 'Bash' }, async ($, e, next) => {
651    if (!(await get($, 'isEnabled'))) return next(e)
652    const command = e.command
653    const isSubmit = /\b(sbatch|salloc)\b/.test(command)
654    const isGuarded =
655      cfg.guards !== 'off' && /\b(sbatch|salloc|srun|ssh|sacct|squeue|scontrol|torchrun|deepspeed|accelerate|vllm|sglang|python[\d.]*)\b/.test(command)
656    if (!isSubmit && !isGuarded) return next(e)
657
658    let allocated: Set<string> | null = null
659    if (isGuarded && /\bssh\b/.test(command) && /\bsprc\d+/.test(command)) {
660      const r = await $.process.run(['squeue', '--me', '-h', '-t', 'R', '-o', '%N']).catch(() => null)
661      if (r && r.exitCode === 0) allocated = new Set(r.stdout.split('\n').flatMap(l => expandNodes(l.trim())))
662    }
663    const findings = isGuarded
664      ? checkBash(command, { cwd: await $.session.cwd(), allocatedNodes: allocated, isLoginNode, waitTool: T('sprc_wait') })
665      : []
666    const blocking = findings.filter(f => f.level !== 'note')
667    if (blocking.length && cfg.guards === 'enforce') {
668      $.ui.toast(`sprc blocked: ${blocking.map(f => f.rule).join(', ')}`)
669      return { deny: `[sprc guard] ${blocking.map(f => f.message).join('\n')}\n(The user can relax this with /config → sprc-slurm-mods guards.)` }
670    }
671
672    const ran = await next(e)
673    const notes = findings.map(f => `[sprc guard] ${f.message}`)
674    if (ran.deny === undefined) {
675      const ids = isSubmit ? submittedJobs(ran.text ?? '', /--parsable\b/.test(command)) : []
676      if (ids.length) {
677        for (const id of ids) await followJob($, id, false, 'sbatch')
678        notes.push(
679          `[sprc] Following job${ids.length > 1 ? 's' : ''} ${ids.join(', ')}: the user sees it in the sprc pane and gets notified when it ends. Use sprc_wait / sprc_job instead of polling squeue.`,
680        )
681        $.ui.toast(`following job ${ids.join(', ')}`)
682      }
683      if (ids.length || /\bscancel\b/.test(command)) void poll($)
684    }
685    if (!notes.length || ran.deny !== undefined) return ran
686    return { ...ran, context: [...(ran.context ?? []), ...notes] }
687  })
688
689  // ---- live context on cluster-ish prompts -----------------------------------
690
691  on('prompt.submit', async ($, e, next) => {
692    if (!cfg.liveContext || e.origin.kind === 'plugin' || !SLURMY.test(e.text)) return next(e)
693    if (!(await get($, 'isEnabled'))) return next(e)
694    const now = await $.clock.now()
695    if (now - lastPollAt > 60_000) await poll($)
696    const snap = await get($, 'snapshot')
697    if (!snap) return next(e)
698    // An unchanged cluster is already in the conversation: re-send it only
699    // after 15 minutes, so a long Slurm-heavy session pays for it once.
700    const text = summary(snap)
701    if (text === lastContext.text && now - lastContext.at < 15 * 60_000) return next(e)
702    lastContext = { text, at: now }
703    return next({ ...e, context: [...(e.context ?? []), text] })
704  })
705}
706

1// Checks on Bash commands before they run. Pure: the hook passes in what it
2// knows about the session and the cluster.
3//
4// `reject` findings are things the cluster itself would refuse or that
5// silently break (so blocking them never costs a working command).
6// `nudge` findings are wasteful patterns with a better path in this mod.
7// `note` findings never block; they ride along as context for the model.
8
9export type Finding = { level: 'reject' | 'nudge' | 'note'; rule: string; message: string }
10
11export type GuardContext = {
12  /** The session's working directory, where a bare command runs; '' = unknown. */
13  cwd: string
14  /** Nodes where this user holds a running allocation right now; null = unknown. */
15  allocatedNodes: ReadonlySet<string> | null
16  /** True when the session runs on the login node (sprlab005). */
17  isLoginNode: boolean
18  /** Name of the wait tool to point at. */
19  waitTool: string
20}
21
22/** Splits a command line into simple commands on ;, &&, ||, |, newlines. Quote-aware. */
23export function segments(command: string): string[][] {
24  const out: string[][] = []
25  let words: string[] = []
26  let word = ''
27  let hasWord = false
28  let quote: '"' | "'" | null = null
29  const endWord = () => {
30    if (hasWord) words.push(word)
31    word = ''
32    hasWord = false
33  }
34  const endSeg = () => {
35    endWord()
36    if (words.length) out.push(words)
37    words = []
38  }
39  for (let i = 0; i < command.length; i++) {
40    const c = command[i]!
41    if (quote) {
42      if (c === quote) quote = null
43      else if (c === '\\' && quote === '"' && i + 1 < command.length) word += command[++i]
44      else word += c
45      continue
46    }
47    if (c === '"' || c === "'") {
48      quote = c
49      hasWord = true
50    } else if (c === '\\' && i + 1 < command.length) {
51      word += command[++i]
52      hasWord = true
53    } else if (c === ' ' || c === '\t') endWord()
54    else if (c === '\n' || c === ';' || c === '&' || c === '|' || c === '(' || c === ')') endSeg()
55    else if (c === '#' && !hasWord) {
56      while (i < command.length && command[i] !== '\n') i++
57      endSeg()
58    } else {
59      word += c
60      hasWord = true
61    }
62  }
63  endSeg()
64  return out
65}
66
67/** Drops leading VAR=value assignments and wrappers like `env`, `time`, `nohup`. */
68function head(words: readonly string[]): string[] {
69  let i = 0
70  while (i < words.length) {
71    const w = words[i]!
72    if (/^[A-Za-z_][A-Za-z0-9_]*=/.test(w)) i++
73    else if (w === 'env' || w === 'time' || w === 'nohup' || w === 'exec' || w === 'command') i++
74    else break
75  }
76  return words.slice(i)
77}
78
79function base(word: string | undefined): string {
80  return (word ?? '').split('/').pop() ?? ''
81}
82
83/** Value of a long/short option in either `--opt=v`, `--opt v`, `-o v` or `-ov` form. */
84function option(words: readonly string[], long: string, short?: string): string | undefined {
85  for (let i = 0; i < words.length; i++) {
86    const w = words[i]!
87    if (w === `--${long}`) return words[i + 1]
88    if (w.startsWith(`--${long}=`)) return w.slice(long.length + 3)
89    if (short && w === `-${short}`) return words[i + 1]
90    if (short && w.startsWith(`-${short}`) && w.length > 2 && !w.startsWith('--')) return w.slice(2)
91  }
92  return undefined
93}
94
95const HEAVY = /^(torchrun|deepspeed|accelerate|vllm|sglang|python[\d.]*)$/
96const SCRATCH_DIRS = /^\/(tmp|var\/tmp|dev\/shm)(\/|$)/
97
98export function checkBash(command: string, ctx: GuardContext): Finding[] {
99  const findings: Finding[] = []
100  let cwd = ctx.cwd
101  const segs = segments(command)
102
103  for (let s = 0; s < segs.length; s++) {
104    const words = head(segs[s]!)
105    const cmd = base(words[0])
106    if (!cmd) continue
107
108    // A relative, ~ or $VAR target is not resolved here: the directory becomes unknown.
109    if (cmd === 'cd' || cmd === 'pushd') cwd = words[1]?.startsWith('/') ? words[1] : ''
110
111    if (cmd === 'salloc' || cmd === 'srun') {
112      const qos = option(words, 'qos', 'q')
113      if (qos === 'scavenger') {
114        findings.push({
115          level: 'reject',
116          rule: 'scavenger-interactive',
117          message: `${cmd} --qos=scavenger is rejected on sprc: scavenger is batch-only. Submit it with sbatch --qos=scavenger (and make it checkpoint/requeue), or use the default QoS for an interactive session.`,
118        })
119      }
120    }
121
122    if (cmd === 'sbatch') {
123      const chdir = option(words, 'chdir', 'D')
124      const workdir = chdir ? (chdir.startsWith('/') ? chdir : '') : cwd
125      if (workdir && SCRATCH_DIRS.test(workdir)) {
126        findings.push({
127          level: 'reject',
128          rule: 'sbatch-from-tmp',
129          message: `sbatch would run with working directory ${workdir}, which is node-local: the compute node cannot see the login node's copy, so the job starts in an empty directory and fails or writes output nowhere you can read. Stage the script and data under /home, /projects or /data and submit from there.`,
130        })
131      }
132    }
133
134    // A note, not a block: admins are exempt from pam_slurm_adopt, and the
135    // node's own refusal already explains itself to everyone else.
136    if (cmd === 'ssh' && ctx.allocatedNodes) {
137      const target = words.slice(1).find(w => !w.startsWith('-') && /^(\S+@)?sprc\d+/.test(w))
138      const node = target?.replace(/^\S+@/, '').replace(/\..*$/, '')
139      if (node && !ctx.allocatedNodes.has(node)) {
140        const held = [...ctx.allocatedNodes]
141        findings.push({
142          level: 'note',
143          rule: 'ssh-without-allocation',
144          message: `If ssh ${node} was refused: compute nodes admit only users with a running job there, and you have none on ${node}${held.length ? ` (you do on ${held.join(', ')})` : ''}. Get an allocation first (salloc), or ssh to a node your job is on.`,
145        })
146      }
147    }
148
149    if (ctx.isLoginNode && HEAVY.test(cmd)) {
150      const isLaunch =
151        cmd === 'torchrun' ||
152        cmd === 'deepspeed' ||
153        (cmd === 'accelerate' && words[1] === 'launch') ||
154        ((cmd === 'vllm' || cmd === 'sglang') && words[1] === 'serve') ||
155        (cmd.startsWith('python') && words.includes('-m') && /^(torch\.distributed\.(run|launch)|vllm\.entrypoints|sglang\.launch_server)/.test(words[words.indexOf('-m') + 1] ?? ''))
156      const wrapped = segs.slice(0, s).some(p => /^(srun|salloc|sbatch)$/.test(base(head(p)[0])))
157      if (isLaunch && !wrapped) {
158        findings.push({
159          level: 'reject',
160          rule: 'gpu-work-on-login-node',
161          message: `\`${words.slice(0, 3).join(' ')}\` would run on the login node sprlab005, which has no GPUs and is shared by everyone. Run it through sbatch, or inside an salloc/srun allocation.`,
162        })
163      }
164    }
165
166    if (cmd === 'sacct' && words.includes('--json')) {
167      findings.push({
168        level: 'note',
169        rule: 'sacct-json-mem',
170        message: 'Note: on this cluster sacct --json omits memory used (MaxRSS). For efficiency numbers use text sacct (-P -o ...MaxRSS...) or the sprc_job tool.',
171      })
172    }
173  }
174
175  // Sleep-and-poll loops burn turns and tokens; the wait tool blocks once.
176  const text = command.replace(/\s+/g, ' ')
177  const polls = /\b(squeue|sacct|scontrol)\b/.test(text)
178  const sleeps = /\bsleep\s+\d+/.test(text) || /\bwatch\b/.test(text)
179  const loops = /\b(while|until|for)\b/.test(text) || /\bwatch\b/.test(text)
180  if (polls && sleeps && loops) {
181    findings.push({
182      level: 'nudge',
183      rule: 'poll-loop',
184      message: `Don't poll Slurm in a sleep loop. Call ${ctx.waitTool} with the job id: it waits (up to 10 min per call) and returns when the job changes state. To be told when a long job finishes, call sprc_follow instead and stop.`,
185    })
186  }
187
188  return findings
189}
190
191/**
192 * Job ids a Bash result announced ("Submitted batch job 123", "Granted job
193 * allocation 123", or a bare "123" / "123;cluster" line from sbatch --parsable).
194 */
195export function submittedJobs(output: string, isParsable = false): string[] {
196  const ids = new Set<string>()
197  if (isParsable) for (const m of output.matchAll(/^(\d+)(?:;\S+)?$/gm)) ids.add(m[1]!)
198  for (const m of output.matchAll(/Submitted batch job (\d+)/g)) ids.add(m[1]!)
199  for (const m of output.matchAll(/Granted job allocation (\d+)/g)) ids.add(m[1]!)
200  return [...ids]
201}
202

1// Slurm queries and parsers. Everything here takes a `run` function so the
2// parsers are testable without a cluster; the hooks module passes
3// `$.process.run`.
4
5import type { JobReport, JobRow, NodeRow, Snapshot } from '../types'
6
7export type RunResult = { exitCode: number; stdout: string; stderr: string }
8export type Run = (argv: readonly string[], init?: { timeoutMs?: number }) => Promise<RunResult>
9
10const SINFO_FIELDS = 'NodeList:|,StateLong:|,Gres:|,GresUsed:|,CPUsState:|,AllocMem:|,Memory:|'
11// Name goes last so a `|` in a job name cannot shift the other columns.
12const SQUEUE_FIELDS =
13  'JobID:|,UserName:|,StateCompact:|,TimeUsed:|,TimeLimit:|,tres-alloc:|,Reason:|,QOS:|,Partition:|,NodeList:|,StartTime:|,Name:|'
14
15export const SACCT_FIELDS =
16  'JobID,JobName,State,ExitCode,Elapsed,Timelimit,NCPUS,TotalCPU,ReqMem,MaxRSS,AllocTRES,End,NodeList,User'
17
18// ---- parsing helpers ---------------------------------------------------
19
20/** "gpu:h100_nvl:2" or "gpu:h100_nvl:1(IDX:0)" -> 2 / 1 */
21export function gresCount(gres: string): number {
22  let total = 0
23  for (const part of gres.split(',')) {
24    const m = /gpu(?::[\w.-]+)?:(\d+)/.exec(part)
25    if (m) total += Number(m[1])
26  }
27  return total
28}
29
30/** "cpu=8,mem=12G,node=1,gres/gpu=1" -> { cpu: "8", mem: "12G", ... } */
31export function parseTres(tres: string): Record<string, string> {
32  const out: Record<string, string> = {}
33  for (const kv of tres.split(',')) {
34    const i = kv.indexOf('=')
35    if (i > 0) out[kv.slice(0, i)] = kv.slice(i + 1)
36  }
37  return out
38}
39
40/** "[D-]HH:MM:SS", "MM:SS", "MM:SS.mmm", "UNLIMITED" -> seconds (NaN if unknown) */
41export function durationSeconds(s: string): number {
42  if (!s || s === 'UNLIMITED' || s === 'Partition_Limit' || s === 'INVALID') return NaN
43  let days = 0
44  let rest = s
45  const dash = s.indexOf('-')
46  if (dash >= 0) {
47    days = Number(s.slice(0, dash))
48    rest = s.slice(dash + 1)
49  }
50  const parts = rest.split(':').map(Number)
51  if (parts.some(Number.isNaN)) return NaN
52  let secs = 0
53  for (const p of parts) secs = secs * 60 + p
54  if (parts.length === 2 && dash < 0) {
55    // MM:SS
56  } else if (parts.length === 1) {
57    secs = parts[0]! * 60 // bare minutes, as sbatch --time takes them
58  }
59  return days * 86400 + secs
60}
61
62export function formatDuration(secs: number): string {
63  if (!Number.isFinite(secs)) return '?'
64  secs = Math.max(0, Math.round(secs))
65  const d = Math.floor(secs / 86400)
66  const h = Math.floor((secs % 86400) / 3600)
67  const m = Math.floor((secs % 3600) / 60)
68  if (d > 0) return `${d}d${h}h`
69  if (h > 0) return `${h}h${String(m).padStart(2, '0')}m`
70  if (m > 0) return `${m}m`
71  return `${secs}s`
72}
73
74/** "12G", "500M", "123456K", "12Gn" -> MiB (NaN if unknown) */
75export function memMb(s: string): number {
76  const m = /^([\d.]+)([KMGT]?)/i.exec(s.trim())
77  if (!m) return NaN
78  const n = Number(m[1])
79  switch ((m[2] ?? '').toUpperCase()) {
80    case 'K':
81      return n / 1024
82    case 'G':
83      return n * 1024
84    case 'T':
85      return n * 1024 * 1024
86    case 'M':
87      return n
88    default:
89      return n / (1024 * 1024) // bytes
90  }
91}
92
93export function formatMem(mb: number): string {
94  if (!Number.isFinite(mb)) return '?'
95  if (mb >= 1024) return `${(mb / 1024).toFixed(mb >= 10240 ? 0 : 1)}G`
96  return mb > 0 && mb < 1 ? '<1M' : `${Math.round(mb)}M`
97}
98
99/** "sprc[00-01,03],sprc05" -> ["sprc00","sprc01","sprc03","sprc05"] */
100export function expandNodes(list: string): string[] {
101  const out: string[] = []
102  const re = /([^,[\]]+)(?:\[([^\]]+)\])?/g
103  let m: RegExpExecArray | null
104  while ((m = re.exec(list)) !== null) {
105    const prefix = m[1]!
106    if (!m[2]) {
107      if (prefix && prefix !== '(null)') out.push(prefix)
108      continue
109    }
110    for (const range of m[2].split(',')) {
111      const [a, b] = range.split('-')
112      if (b === undefined) {
113        out.push(prefix + a)
114        continue
115      }
116      const width = a!.length
117      for (let i = Number(a); i <= Number(b); i++) out.push(prefix + String(i).padStart(width, '0'))
118    }
119  }
120  return out
121}
122
123// ---- cluster snapshot --------------------------------------------------
124
125export function parseSinfo(stdout: string): NodeRow[] {
126  const seen = new Map<string, NodeRow>()
127  for (const line of stdout.split('\n')) {
128    const f = line.split('|')
129    if (f.length < 7 || !f[0]) continue
130    const name = f[0].trim()
131    if (seen.has(name)) continue // one row per partition the node is in
132    const [alloc, , , total] = (f[4] ?? '').split('/').map(Number)
133    seen.set(name, {
134      name,
135      state: (f[1] ?? '').trim(),
136      gpus: gresCount(f[2] ?? ''),
137      gpusUsed: gresCount((f[3] ?? '').replace(/\(IDX:[^)]*\)/g, '')),
138      cpusAlloc: alloc ?? 0,
139      cpusTotal: total ?? 0,
140      memAllocMb: Number(f[5]) || 0,
141      memTotalMb: Number(f[6]) || 0,
142    })
143  }
144  return [...seen.values()].sort((a, b) => a.name.localeCompare(b.name))
145}
146
147export function parseSqueue(stdout: string): JobRow[] {
148  const rows: JobRow[] = []
149  for (const line of stdout.split('\n')) {
150    if (!line.trim()) continue
151    const f = line.split('|')
152    if (f.length < 12) continue
153    const tres = parseTres(f[5] ?? '')
154    rows.push({
155      id: f[0]!.trim(),
156      user: f[1]!.trim(),
157      state: f[2]!.trim(),
158      elapsed: f[3]!.trim(),
159      limit: f[4]!.trim(),
160      gpus: Number(tres['gres/gpu'] ?? 0) || 0,
161      cpus: Number(tres['cpu'] ?? 0) || 0,
162      mem: tres['mem'] ?? '?',
163      reason: f[6]!.trim(),
164      qos: f[7]!.trim(),
165      partition: f[8]!.trim(),
166      nodes: f[9]!.trim(),
167      start: f[10]!.trim(),
168      name: f.slice(11).join('|').replace(/\|$/, '').trim(),
169    })
170  }
171  return rows
172}
173
174export async function fetchSnapshot(run: Run, user: string, now: number): Promise<Snapshot> {
175  const [si, sq] = await Promise.all([
176    run(['sinfo', '-h', '-N', '-O', SINFO_FIELDS]),
177    run(['squeue', '-h', '-a', '-O', SQUEUE_FIELDS]),
178  ])
179  if (si.exitCode !== 0 || sq.exitCode !== 0) {
180    return {
181      at: now,
182      error: (si.stderr || sq.stderr || 'sinfo/squeue failed').trim().split('\n')[0],
183      nodes: [],
184      mine: [],
185      others: { running: 0, pending: 0, gpusRunning: 0, gpusPending: 0 },
186    }
187  }
188  const jobs = parseSqueue(sq.stdout)
189  const mine = jobs.filter(j => j.user === user)
190  const others = { running: 0, pending: 0, gpusRunning: 0, gpusPending: 0 }
191  for (const j of jobs) {
192    if (j.user === user) continue
193    if (j.state === 'R' || j.state === 'CG') {
194      others.running++
195      others.gpusRunning += j.gpus
196    } else if (j.state === 'PD') {
197      others.pending++
198      others.gpusPending += j.gpus
199    }
200  }
201  return { at: now, nodes: parseSinfo(si.stdout), mine, others }
202}
203
204// What a GPU job gets per GPU when it asks for nothing else: slurm.conf's
205// DefCpuPerGPU=32, times DefMemPerCPU=12000.
206export const GPU_SHARE = { cpus: 32, memMb: 32 * 12000 }
207
208/** Idle GPUs on one node that a default 1-GPU job could actually start on. */
209export function nodeUsable(n: NodeRow): number {
210  if (/drain|down|fail|maint|reserved|inval/i.test(n.state)) return 0
211  const byCpu = Math.floor((n.cpusTotal - n.cpusAlloc) / GPU_SHARE.cpus)
212  const byMem = Math.floor((n.memTotalMb - n.memAllocMb) / GPU_SHARE.memMb)
213  return Math.max(0, Math.min(n.gpus - n.gpusUsed, byCpu, byMem))
214}
215
216/**
217 * `idle`: GPUs no job holds. `usable`: idle GPUs a default GPU job could
218 * actually get, which excludes nodes that are down/drained or lack the CPUs
219 * or memory that go with a GPU (a node full of CPU jobs strands its GPUs).
220 */
221export function gpuTotals(nodes: readonly NodeRow[]): { total: number; used: number; idle: number; usable: number } {
222  let total = 0
223  let used = 0
224  let usable = 0
225  for (const n of nodes) {
226    total += n.gpus
227    used += n.gpusUsed
228    usable += nodeUsable(n)
229  }
230  return { total, used, idle: total - used, usable }
231}
232
233/** Whether job `id` (or, for an array or het job, any of its parts) is in `rows`. */
234export function hasJob(rows: readonly { id: string }[], id: string): boolean {
235  return rows.some(j => j.id === id || j.id.startsWith(`${id}_`) || j.id.startsWith(`${id}+`))
236}
237
238// ---- one job -----------------------------------------------------------
239
240/** `scontrol show job -o ID` -> key/value fields (first record). */
241export function parseScontrol(stdout: string): Record<string, string> {
242  const line = stdout.split('\n').find(l => l.includes('JobId=')) ?? ''
243  const out: Record<string, string> = {}
244  // Values may contain spaces only in a few fields (Command, Comment); keys
245  // are CamelCase words followed by '='.
246  const re = /(?:^|\s)([A-Za-z][\w:/]*)=(.*?)(?=\s[A-Za-z][\w:/]*=|$)/g
247  let m: RegExpExecArray | null
248  while ((m = re.exec(line)) !== null) out[m[1]!] = m[2]!
249  return out
250}
251
252export async function fetchScontrol(run: Run, id: string): Promise<Record<string, string> | null> {
253  const r = await run(['scontrol', 'show', 'job', '-o', id])
254  if (r.exitCode !== 0) return null
255  const f = parseScontrol(r.stdout)
256  return f['JobId'] ? f : null
257}
258
259type SacctRow = Record<(typeof SACCT_COLUMNS)[number], string>
260const SACCT_COLUMNS = SACCT_FIELDS.split(',') as readonly string[]
261
262export function parseSacct(stdout: string): SacctRow[] {
263  return stdout
264    .split('\n')
265    .filter(l => l.trim())
266    .map(l => {
267      const f = l.split('|')
268      const row: Record<string, string> = {}
269      SACCT_COLUMNS.forEach((c, i) => (row[c] = f[i] ?? ''))
270      return row as SacctRow
271    })
272}
273
274/** Builds the efficiency report from sacct rows of one job (allocation + steps). */
275export function buildReport(id: string, rows: readonly SacctRow[], endedAt: number): JobReport | null {
276  const main = rows.find(r => r['JobID'] === id) ?? rows.find(r => !r['JobID']!.includes('.'))
277  if (!main) return null
278  const elapsed = durationSeconds(main['Elapsed']!)
279  const limit = durationSeconds(main['Timelimit']!)
280  const cpus = Number(main['NCPUS']) || 0
281  const totalCpu = durationSeconds(main['TotalCPU']!)
282  const reqMemMb = memMb(main['ReqMem']!)
283  let maxRssMb = NaN
284  for (const r of rows) {
285    const v = memMb(r['MaxRSS'] ?? '')
286    // 0 means the 30 s sampler never ran (a short job), not 0 bytes.
287    if (Number.isFinite(v) && v > 0 && !(maxRssMb >= v)) maxRssMb = v
288  }
289  const tres = parseTres(main['AllocTRES'] ?? '')
290  const gpus = Number(tres['gres/gpu'] ?? 0) || 0
291  const state = (main['State'] ?? '').split(' ')[0]!
292  const cpuEff = elapsed > 0 && cpus > 0 && Number.isFinite(totalCpu) ? totalCpu / (elapsed * cpus) : undefined
293  const memEff = reqMemMb > 0 && Number.isFinite(maxRssMb) ? maxRssMb / reqMemMb : undefined
294  const timeEff = limit > 0 && Number.isFinite(elapsed) ? elapsed / limit : undefined
295
296  const advice: string[] = []
297  if (state === 'OUT_OF_MEMORY' || (memEff !== undefined && memEff > 0.92)) {
298    const want = Math.ceil(((Number.isFinite(maxRssMb) ? maxRssMb : reqMemMb) * 1.5) / 1024)
299    advice.push(`memory was the limit: next time request --mem=${want}G`)
300  } else if (memEff !== undefined && memEff < 0.5 && reqMemMb >= 8192) {
301    const want = Math.max(4, Math.ceil((maxRssMb * 1.3) / 1024))
302    advice.push(`peak memory ${formatMem(maxRssMb)} of ${formatMem(reqMemMb)} requested: --mem=${want}G is enough`)
303  }
304  if (state === 'TIMEOUT') {
305    advice.push('hit its walltime: raise --time, or checkpoint and use --requeue so it resumes')
306  } else if (state === 'COMPLETED' && timeEff !== undefined && timeEff < 0.35 && limit >= 3600) {
307    const want = Math.ceil((elapsed * 1.5) / 900) * 15 // round up to 15 min
308    advice.push(`ran ${formatDuration(elapsed)} of a ${formatDuration(limit)} limit: --time=${want} backfills sooner`)
309  }
310  if (cpuEff !== undefined && cpuEff < 0.25 && cpus >= 8 && elapsed > 300) {
311    const want = Math.max(2, Math.ceil(cpus * cpuEff * 1.5))
312    advice.push(`CPU use ${Math.round(cpuEff * 100)}% of ${cpus} cores: --cpus-per-task=${want} would do`)
313  }
314
315  return {
316    id,
317    name: main['JobName'] ?? '',
318    user: main['User'] || undefined,
319    state,
320    exitCode: main['ExitCode'] ?? '',
321    elapsed: main['Elapsed'] ?? '',
322    limit: main['Timelimit'] ?? '',
323    nodes: main['NodeList'] ?? '',
324    cpus,
325    gpus,
326    reqMem: Number.isFinite(reqMemMb) ? formatMem(reqMemMb) : main['ReqMem'] ?? '?',
327    maxRss: Number.isFinite(maxRssMb) ? formatMem(maxRssMb) : '?',
328    cpuEff,
329    memEff,
330    timeEff,
331    advice,
332    endedAt,
333  }
334}
335
336export async function fetchReport(run: Run, id: string, now: number): Promise<JobReport | null> {
337  // Text sacct, not --json: --json omits mem-used on this cluster.
338  const r = await run(['sacct', '-j', id, '-n', '-P', '-o', SACCT_FIELDS])
339  if (r.exitCode !== 0) return null
340  return buildReport(id, parseSacct(r.stdout), now)
341}
342
343export async function tailFile(run: Run, path: string, lines: number): Promise<string[]> {
344  if (!path || path === '(null)') return []
345  const r = await run(['tail', '-n', String(lines), path])
346  if (r.exitCode !== 0) return [`(cannot read ${path}: ${r.stderr.trim().split('\n')[0] ?? 'error'})`]
347  return r.stdout.replace(/\r/g, '').split('\n').filter((l, i, a) => i < a.length - 1 || l !== '')
348}
349
350export const TERMINAL_STATES = new Set([
351  'COMPLETED',
352  'FAILED',
353  'CANCELLED',
354  'TIMEOUT',
355  'OUT_OF_MEMORY',
356  'NODE_FAIL',
357  'PREEMPTED',
358  'BOOT_FAIL',
359  'DEADLINE',
360])
361
362export function isBad(state: string): boolean {
363  return state !== 'COMPLETED' && TERMINAL_STATES.has(state)
364}
365
366export function pct(x: number | undefined): string {
367  return x === undefined ? '–' : `${Math.round(x * 100)}%`
368}
369
370export function reportLine(r: JobReport): string {
371  const eff = [
372    r.cpuEff !== undefined ? `CPU ${pct(r.cpuEff)}` : '',
373    r.memEff !== undefined ? `mem ${r.maxRss}/${r.reqMem} (${pct(r.memEff)})` : '',
374    r.timeEff !== undefined ? `time ${pct(r.timeEff)} of limit` : '',
375  ].filter(Boolean)
376  return `job ${r.id} (${r.name}) ${r.state} exit ${r.exitCode} after ${r.elapsed}${eff.length ? ' · ' + eff.join(' · ') : ''}`
377}
378

1// The dashboard pane's drawing. Only Box, Text and Button, which every
2// surface (terminal, desktop, vscode, mobile) has.
3
4import type { Elements } from 'claude-code'
5
6import type { Budget, Followed, JobDetail, JobReport, JobRow, Snapshot } from '../types'
7import { durationSeconds, formatDuration, gpuTotals, isBad, nodeUsable, pct } from './slurm'
8
9export type El = Pick<Elements['mobile'], 'Box' | 'Text' | 'Button'>
10
11export type Actions = {
12  refresh: () => void
13  open: (id: string) => void
14  back: () => void
15  cancel: (id: string) => void
16  toggleFollow: (id: string) => void
17  clearFinished: () => void
18}
19
20const STATE_COLOR: Record<string, string> = {
21  R: 'green',
22  PD: 'yellow',
23  CG: 'cyan',
24  S: 'magenta',
25}
26
27export function bar(frac: number, width: number): string {
28  const f = Math.max(0, Math.min(1, Number.isFinite(frac) ? frac : 0))
29  const full = Math.round(f * width)
30  return '█'.repeat(full) + '░'.repeat(width - full)
31}
32
33function ago(now: number, at: number): string {
34  const s = Math.max(0, Math.round((now - at) / 1000))
35  return s < 60 ? `${s}s ago` : `${Math.round(s / 60)}m ago`
36}
37
38function clock(iso: string): string {
39  const m = /T(\d\d:\d\d)/.exec(iso)
40  return m ? m[1]! : iso
41}
42
43const REASONS: Record<string, string> = {
44  Priority: 'waiting behind higher-priority jobs',
45  Resources: 'next in line; waiting for resources to free',
46  Dependency: 'waiting on a dependency',
47  DependencyNeverSatisfied: 'dependency failed; will never start (scancel it)',
48  JobArrayTaskLimit: 'array throttle (%N) reached',
49  QOSMaxGRESPerUser: 'at your QoS GPU limit',
50  QOSMaxJobsPerUserLimit: 'at your QoS job-count limit',
51  QOSGrpGRES: 'QoS group GPU limit reached',
52  AssocGrpGRESMinutes: 'GPU-minutes budget exhausted',
53  AssocGrpGRES: 'account GPU limit reached',
54  ReqNodeNotAvail: 'requested node unavailable (drained or reserved)',
55  BeginTime: 'held until its --begin time',
56  JobHeldUser: 'held by you (scontrol release)',
57  JobHeldAdmin: 'held by an admin',
58  PartitionTimeLimit: '--time exceeds the partition limit; will never start',
59  QOSMaxWallDurationPerJobLimit: '--time exceeds the QoS limit; will never start',
60}
61
62export function explainReason(reason: string): string {
63  const key = reason.replace(/^\(|\)$/g, '')
64  return REASONS[key] ?? key
65}
66
67function shorten(s: string, n: number): string {
68  return s.length <= n ? s : s.slice(0, Math.max(1, n - 1)) + '…'
69}
70
71export function Overview(
72  el: El,
73  props: {
74    snap: Snapshot | null
75    finished: readonly JobReport[]
76    followed: readonly Followed[]
77    budget: Budget | null
78    now: number
79    columns: number
80    actions: Actions
81  },
82) {
83  const { Box, Text, Button } = el
84  const { snap, finished, followed, budget, now, columns, actions } = props
85  if (!snap) return <Text dimColor>Reading cluster state…</Text>
86  const followedIds = new Set(followed.map(f => f.id))
87  const gpus = gpuTotals(snap.nodes)
88  const nameW = Math.max(8, columns - 34)
89  const running = snap.mine.filter(j => j.state === 'R' || j.state === 'CG')
90  const pending = snap.mine.filter(j => j.state !== 'R' && j.state !== 'CG')
91  const shown = [...running, ...pending].slice(0, 10)
92
93  return (
94    <Box flexDirection="column">
95      <Box flexDirection="row" justifyContent="space-between">
96        <Text bold>
97          {gpus.usable}/{gpus.total} GPUs free
98          {gpus.idle > gpus.usable ? <Text dimColor> (+{gpus.idle - gpus.usable} stranded)</Text> : null}
99        </Text>
100        <Box flexDirection="row" gap={1}>
101          <Text dimColor>{ago(now, snap.at)}</Text>
102          <Button key="refresh" label="refresh" hotkey="r" plain onPress={actions.refresh} />
103        </Box>
104      </Box>
105      {snap.error && <Text color="red">Slurm: {snap.error}</Text>}
106
107      <Box flexDirection="column" marginTop={1}>
108        <Text dimColor bold>
109          NODES
110        </Text>
111        {snap.nodes.map(n => {
112          const cpu = n.cpusTotal ? n.cpusAlloc / n.cpusTotal : 0
113          const mem = n.memTotalMb ? n.memAllocMb / n.memTotalMb : 0
114          const isOut = /drain|down|fail|maint/i.test(n.state)
115          return (
116            <Box key={`node-${n.name}`} flexDirection="row" gap={1}>
117              <Text>{n.name}</Text>
118              <Text color={n.gpusUsed >= n.gpus ? 'red' : nodeUsable(n) === 0 ? 'yellow' : 'green'}>
119                {'■'.repeat(n.gpusUsed)}
120                {'□'.repeat(Math.max(0, n.gpus - n.gpusUsed))}
121              </Text>
122              <Text dimColor>cpu</Text>
123              <Text color={cpu > 0.9 ? 'red' : undefined}>{bar(cpu, 6)}</Text>
124              <Text dimColor>mem</Text>
125              <Text>{bar(mem, 4)}</Text>
126              <Text color={isOut ? 'red' : undefined} dimColor={!isOut} wrap="truncate-end">
127                {n.state}
128              </Text>
129            </Box>
130          )
131        })}
132      </Box>
133
134      <Box flexDirection="column" marginTop={1}>
135        <Text dimColor bold>
136          YOUR JOBS {snap.mine.length ? `(${running.length} running · ${pending.length} pending)` : ''}
137        </Text>
138        {snap.mine.length === 0 && <Text dimColor>none queued or running</Text>}
139        {shown.map(j => JobLine(el, j, nameW, followedIds.has(j.id), actions))}
140        {snap.mine.length > shown.length && <Text dimColor>+{snap.mine.length - shown.length} more</Text>}
141      </Box>
142
143      <Box flexDirection="column" marginTop={1}>
144        <Text dimColor bold>
145          EVERYONE ELSE
146        </Text>
147        <Text>
148          {snap.others.running} running ({snap.others.gpusRunning} GPU) · {snap.others.pending} pending (
149          {snap.others.gpusPending} GPU)
150        </Text>
151      </Box>
152
153      {finished.length > 0 && (
154        <Box flexDirection="column" marginTop={1}>
155          <Box flexDirection="row" justifyContent="space-between">
156            <Text dimColor bold>
157              RECENTLY FINISHED
158            </Text>
159            <Button key="clear-finished" label="clear" plain onPress={actions.clearFinished} />
160          </Box>
161          {finished.slice(0, 6).map(r => (
162            <Box key={`fin-${r.id}`} flexDirection="column">
163              <Box flexDirection="row" gap={1}>
164                <Text color={isBad(r.state) ? 'red' : 'green'}>{isBad(r.state) ? '✗' : '✓'}</Text>
165                <Button key={`open-fin-${r.id}`} label={r.id} plain onPress={() => actions.open(r.id)} />
166                <Text wrap="truncate-end">
167                  {shorten(r.name, nameW)} {r.state} {r.elapsed}
168                </Text>
169              </Box>
170              {r.advice.slice(0, 1).map(a => (
171                <Text dimColor wrap="truncate-end">
172                  {'   → '}
173                  {a}
174                </Text>
175              ))}
176            </Box>
177          ))}
178        </Box>
179      )}
180
181      {budget && budget.lines.length > 0 && (
182        <Box flexDirection="column" marginTop={1}>
183          <Text dimColor bold>
184            COURSE BUDGET
185          </Text>
186          {budget.lines.map(l => (
187            <Text wrap="truncate-end">{l}</Text>
188          ))}
189        </Box>
190      )}
191    </Box>
192  )
193}
194
195function JobLine(el: El, j: JobRow, nameW: number, isFollowed: boolean, actions: Actions) {
196  const { Box, Text, Button } = el
197  const isRun = j.state === 'R'
198  const used = durationSeconds(j.elapsed)
199  const limit = durationSeconds(j.limit)
200  return (
201    <Box key={`job-${j.id}`} flexDirection="column">
202      <Box flexDirection="row" gap={1}>
203        <Text color={STATE_COLOR[j.state]}>{j.state.padEnd(2)}</Text>
204        <Button key={`open-${j.id}`} label={j.id} plain onPress={() => actions.open(j.id)} />
205        <Text wrap="truncate-end">
206          {isFollowed ? '★ ' : ''}
207          {shorten(j.name, nameW)}
208        </Text>
209      </Box>
210      <Text dimColor wrap="truncate-end">
211        {'      '}
212        {isRun
213          ? `${bar(used / limit, 8)} ${j.elapsed}/${j.limit} on ${j.nodes}${j.gpus ? ` · ${j.gpus} GPU` : ''}`
214          : `${explainReason(j.reason)}${j.start && j.start !== 'N/A' ? ` · est. start ${clock(j.start)}` : ''}`}
215      </Text>
216    </Box>
217  )
218}
219
220const DETAIL_FIELDS: readonly [string, string][] = [
221  ['Partition', 'partition'],
222  ['QOS', 'qos'],
223  ['NumCPUs', 'cpus'],
224  ['AllocTRES', 'tres'],
225  ['ReqTRES', 'requested'],
226  ['TimeLimit', 'limit'],
227  ['WorkDir', 'workdir'],
228  ['StdOut', 'stdout'],
229]
230
231export function JobView(
232  el: El,
233  props: {
234    id: string
235    detail: JobDetail | null
236    snap: Snapshot | null
237    isFollowed: boolean
238    columns: number
239    logRows: number
240    actions: Actions
241  },
242) {
243  const { Box, Text, Button } = el
244  const { id, detail, isFollowed, actions, logRows } = props
245  const f = detail?.id === id ? detail.fields : {}
246  const r = detail?.id === id ? detail.report : undefined
247  const state = f['JobState'] ?? r?.state ?? '…'
248  const isLive = !!f['JobState'] && !/COMPLETED|FAILED|CANCELLED|TIMEOUT|OUT_OF_MEMORY/.test(state)
249  const runTime = durationSeconds(f['RunTime'] ?? '')
250  const limit = durationSeconds(f['TimeLimit'] ?? '')
251  const submit = f['SubmitTime']
252  const start = f['StartTime']
253  const waitSecs =
254    submit && start && start !== 'Unknown' && state !== 'PENDING'
255      ? (Date.parse(start) - Date.parse(submit)) / 1000
256      : NaN
257
258  return (
259    <Box flexDirection="column">
260      <Box flexDirection="row" gap={2}>
261        <Button key="back" label="← back" hotkey="b" plain onPress={actions.back} />
262        <Button key="refresh" label="refresh" hotkey="r" plain onPress={actions.refresh} />
263        <Button
264          key="follow"
265          label={isFollowed ? '★ following' : '☆ follow'}
266          hotkey="f"
267          plain
268          onPress={() => actions.toggleFollow(id)}
269        />
270        {isLive && <Button key="cancel" label="scancel" plain onPress={() => actions.cancel(id)} />}
271      </Box>
272      <Box marginTop={1} flexDirection="row" gap={1}>
273        <Text bold>job {id}</Text>
274        <Text color={isBad(state) ? 'red' : state === 'RUNNING' ? 'green' : 'yellow'}>{state}</Text>
275        <Text wrap="truncate-end">{f['JobName'] ?? r?.name ?? ''}</Text>
276      </Box>
277      {!detail || detail.id !== id ? (
278        <Text dimColor>Loading…</Text>
279      ) : (
280        <Box flexDirection="column">
281          {state === 'PENDING' && f['Reason'] && (
282            <Text color="yellow">
283              pending: {explainReason(f['Reason'])}
284              {start && start !== 'Unknown' ? ` · est. start ${clock(start)}` : ''}
285            </Text>
286          )}
287          {state === 'RUNNING' && Number.isFinite(limit) && (
288            <Text>
289              {bar(runTime / limit, 16)} {formatDuration(runTime)} / {formatDuration(limit)} on {f['NodeList']}
290            </Text>
291          )}
292          <Text dimColor wrap="truncate-end">
293            submitted {clock(submit ?? '?')}
294            {Number.isFinite(waitSecs) ? ` · queued ${formatDuration(waitSecs)} · started ${clock(start!)}` : ''}
295            {f['EndTime'] && f['EndTime'] !== 'Unknown' && !isLive ? ` · ended ${clock(f['EndTime'])}` : ''}
296            {f['Restarts'] && f['Restarts'] !== '0' ? ` · requeued ${f['Restarts']}×` : ''}
297          </Text>
298          {DETAIL_FIELDS.filter(([k]) => f[k]).map(([k, label]) => (
299            <Text wrap="truncate-start">
300              <Text dimColor>{label.padEnd(9)}</Text>
301              {f[k]}
302            </Text>
303          ))}
304          {r && (
305            <Box flexDirection="column" marginTop={1}>
306              <Text dimColor bold>
307                EFFICIENCY
308              </Text>
309              <Text>
310                CPU {pct(r.cpuEff)} of {r.cpus} · mem {r.maxRss}/{r.reqMem} ({pct(r.memEff)}) · time {r.elapsed} of{' '}
311                {r.limit} ({pct(r.timeEff)}) · exit {r.exitCode}
312              </Text>
313              {r.advice.map(a => (
314                <Text color="cyan">→ {a}</Text>
315              ))}
316            </Box>
317          )}
318          <Box flexDirection="column" marginTop={1}>
319            <Text dimColor bold>
320              LOG (last {Math.min(logRows, detail.log.length)} lines)
321            </Text>
322            {detail.log.length === 0 && <Text dimColor>(empty or not yet created)</Text>}
323            {detail.log.slice(-logRows).map(l => (
324              <Text wrap="truncate-end">{l || ' '}</Text>
325            ))}
326          </Box>
327        </Box>
328      )}
329    </Box>
330  )
331}
332

1export type NodeRow = {
2  name: string
3  state: string
4  gpus: number
5  gpusUsed: number
6  cpusAlloc: number
7  cpusTotal: number
8  memAllocMb: number
9  memTotalMb: number
10}
11
12export type JobRow = {
13  id: string
14  user: string
15  name: string
16  /** Compact state: R, PD, CG, ... */
17  state: string
18  elapsed: string
19  limit: string
20  gpus: number
21  cpus: number
22  mem: string
23  reason: string
24  qos: string
25  partition: string
26  nodes: string
27  /** Expected (pending) or actual (running) start, ISO, or N/A. */
28  start: string
29}
30
31export type Snapshot = {
32  at: number
33  error?: string
34  nodes: NodeRow[]
35  mine: JobRow[]
36  others: { running: number; pending: number; gpusRunning: number; gpusPending: number }
37}
38
39export type JobReport = {
40  id: string
41  name: string
42  state: string
43  exitCode: string
44  elapsed: string
45  limit: string
46  nodes: string
47  cpus: number
48  gpus: number
49  reqMem: string
50  maxRss: string
51  /** Fractions 0..1, absent where Slurm had nothing to measure. */
52  cpuEff?: number
53  memEff?: number
54  timeEff?: number
55  advice: string[]
56  user?: string
57  stdout?: string
58  endedAt: number
59}
60
61export type Followed = {
62  id: string
63  wake: boolean
64  origin: 'sbatch' | 'tool' | 'person'
65  /** When following began; a snapshot taken before then cannot say it ended. */
66  since?: number
67}
68
69export type View = { kind: 'overview' } | { kind: 'job'; id: string }
70
71export type JobDetail = {
72  id: string
73  fields: Record<string, string>
74  log: string[]
75  report?: JobReport
76  at: number
77}
78
79export type Budget = { lines: string[]; at: number }
80
81declare module 'claude-code' {
82  interface PluginState {
83    'sprc-slurm-mods': {
84      snapshot: Snapshot | null
85      followed: Followed[]
86      finished: JobReport[]
87      view: View
88      detail: JobDetail | null
89      budget: Budget | null
90      isEnabled: boolean
91      hasOpened: boolean
92    }
93  }
94}
95