jev-judge

1import { atom, read, update } from 'claude-code'
2import type { Register } from 'claude-code'
3
4import type { QuestionVerdict, Verdict } from '../types'
5
6const JEV_URL = 'https://api.typesafe.ai/v1/systemone'
7const KEEP = 20
8const NO_KEY = 'API キーがありません（/plugin configure jev-judge@kuhaku-plugins で設定）'
9
10const verdicts = atom({ plugin: 'jev-judge', key: 'verdicts' } as const, {})
11
12type AskOption = { label: string; description: string; preview?: string }
13type AskQuestion = { question: string; header: string; options: AskOption[]; multiSelect: boolean }
14
15/** fork が返す英訳。options は元の並びのまま。 */
16type English = {
17  context: string
18  questions: { question: string; options: string[] }[]
19}
20
21type JevAnswer =
22  | { type: 'choice'; choice: string; confidence: number; probabilities: Record<string, number> }
23  | { type: 'noul'; noul: number }
24
25const pct = (p: number) => `${Math.round(p * 100)}%`
26
27/** 長さがそろっている前提の 2 配列を組にする。ずれていたら判定を諦める。 */
28const zip = <A, B>(as: readonly A[], bs: readonly B[]): [A, B][] => {
29  if (as.length !== bs.length) throw new Error('判定と質問の数が合いません')
30  return as.map((a, i) => [a, bs[i] as B])
31}
32
33const topIndex = (probs: number[]) => probs.indexOf(Math.max(...probs))
34
35const forkPrompt = (questions: AskQuestion[]) =>
36  [
37    'You are about to show the user the following AskUserQuestion dialog (JSON).',
38    'Translate it into English and summarize, in English, the conversation context',
39    'needed to judge which option is better (goal, constraints, relevant decisions;',
40    'at most 150 words). Reply with ONLY a JSON object of this exact shape:',
41    '{"context": string, "questions": [{"question": string, "options": [string]}]}',
42    'Each option string is "<label>: <description>" in English, in the original order.',
43    '',
44    JSON.stringify(questions.map(({ question, options }) => ({ question, options }))),
45  ].join('\n')
46
47const parseEnglish = (text: string, questions: AskQuestion[]): English => {
48  const body = text.slice(text.indexOf('{'), text.lastIndexOf('}') + 1)
49  const parsed = JSON.parse(body) as English
50  const isAligned =
51    parsed.questions?.length === questions.length &&
52    parsed.questions.every((q, i) => q.options?.length === questions[i]?.options.length)
53  if (!isAligned) throw new Error('英訳の形が元の質問と合いません')
54  return parsed
55}
56
57/** Jev への質問。単一選択は Choice、複数選択は選択肢ごとの Noul。 */
58const jevQuestions = (en: English, questions: AskQuestion[]) => {
59  const out: Record<string, unknown> = {}
60  zip(questions, en.questions).forEach(([q, eq], i) => {
61    if (q.multiSelect) {
62      eq.options.forEach((option, j) => {
63        out[`q${i}_o${j}`] = {
64          type: 'noul',
65          instructions: {
66            question: eq.question,
67            option,
68            ask: 'Given `context`, should the user select `option` for `question`? Several options may be selected.',
69          },
70        }
71      })
72      return
73    }
74    out[`q${i}`] = {
75      type: 'choice',
76      instructions: `Given \`context\`, which option is the better answer to this question: ${eq.question}`,
77      criteria: Object.fromEntries(eq.options.map((option, j) => [`o${j}`, option])),
78    }
79  })
80  return out
81}
82
83const toVerdicts = (answers: Record<string, JevAnswer>, questions: AskQuestion[]): QuestionVerdict[] =>
84  questions.map((q, i) => {
85    if (q.multiSelect) {
86      const probs = q.options.map((_, j) => {
87        const a = answers[`q${i}_o${j}`]
88        return a?.type === 'noul' ? a.noul : 0
89      })
90      return { isMulti: true, probs, confidence: null }
91    }
92    const a = answers[`q${i}`]
93    if (a?.type !== 'choice') throw new Error(`q${i} の回答がありません`)
94    return {
95      isMulti: false,
96      probs: q.options.map((_, j) => a.probabilities[`o${j}`] ?? 0),
97      confidence: a.confidence,
98    }
99  })
100
101const summary = (questions: AskQuestion[], result: QuestionVerdict[]) =>
102  'Jev 推奨: ' +
103  zip(questions, result)
104    .map(([q, v]) => {
105      const options = zip(q.options, v.probs)
106      if (v.isMulti) {
107        const picked = options.filter(([, p]) => p >= 0.5).map(([o]) => o.label)
108        return `${q.header}「${picked.join('・') || 'なし'}」`
109      }
110      const [best, p] = options[topIndex(v.probs)] ?? [q.options[0], 0]
111      return `${q.header}「${best?.label}」${pct(p)}（確信度 ${v.confidence?.toFixed(2)}）`
112    })
113    .join(' / ')
114
115/** ダイアログに描く質問。説明の先頭に判定を足すだけで、label は変えない（回答値になるため）。 */
116const annotate = (questions: AskQuestion[], result: QuestionVerdict[]): AskQuestion[] =>
117  zip(questions, result).map(([q, v]) => {
118    const top = topIndex(v.probs)
119    return {
120      ...q,
121      options: zip(q.options, v.probs).map(([o, p], j) => {
122        const tag = v.isMulti
123          ? `【Jev 選ぶ ${pct(p)}${p >= 0.5 ? '・推奨' : ''}】`
124          : `【Jev ${pct(p)}${j === top ? '・推奨' : ''}】`
125        return { ...o, description: `${tag} ${o.description}` }
126      }),
127    }
128  })
129
130export const register: Register = (on, options) => {
131  const configuredKey = typeof options.api_key === 'string' ? options.api_key : ''
132
133  on('tool.call', { tool: 'AskUserQuestion' }, ($, e, next) => {
134    const id = e.tool_use_id
135    const questions = e.questions as AskQuestion[]
136
137    const setVerdict = (v: Verdict) =>
138      update($, verdicts, all => {
139        const kept = Object.entries(all).filter(([key]) => key !== id).slice(-(KEEP - 1))
140        return { ...Object.fromEntries(kept), [id]: v }
141      })
142    // 回答済みのダイアログには出せない（エンジンが拒否する）ので握りつぶす
143    const notice = (text: string) => {
144      try {
145        $.ui.notice(id, text)
146      } catch {}
147    }
148
149    const judge = async () => {
150      await setVerdict({ status: 'pending' })
151      notice('Jev 判定中…')
152      try {
153        // Desktop アプリはシェルの設定を読まないので、設定（キーチェーン）を優先する
154        const key = configuredKey || (await $.env.get('TYPESAFE_API_KEY'))
155        if (!key) throw new Error(NO_KEY)
156
157        const forked = await $.model.fork({ prompt: forkPrompt(questions) })
158        if (!forked.isAnswered) throw new Error(`英訳に失敗しました（${forked.reason}）`)
159        const en = parseEnglish(forked.text, questions)
160
161        const res = await $.http.fetch(JEV_URL, {
162          method: 'POST',
163          headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
164          body: JSON.stringify({
165            model: 'jev-latest',
166            state: { context: en.context },
167            questions: jevQuestions(en, questions),
168          }),
169        })
170        if (!res.ok) throw new Error(`Jev API が ${res.status} を返しました`)
171        const { answers } = JSON.parse(res.text) as { answers: Record<string, JevAnswer> }
172
173        const result = toVerdicts(answers, questions)
174        await setVerdict({ status: 'done', questions: result })
175        notice(summary(questions, result))
176      } catch (err) {
177        const message = err instanceof Error ? err.message : String(err)
178        await setVerdict({ status: 'error', message })
179        notice(`Jev 判定に失敗: ${message}`)
180        // ダイアログ直下の行を描かない画面もあるので、トーストでも知らせる
181        $.ui.toast(`Jev 判定に失敗: ${message}`)
182      }
183    }
184
185    // ダイアログは待たせずに開き、判定は回答待ちの間に裏で進める
186    const ran = next(e)
187    void judge()
188    return ran
189  })
190
191  on('ui.render', { component: 'AskUserQuestion' }, async ($, e, next) => {
192    const v = (await read($, verdicts))[e.requestId]
193    if (v?.status !== 'done') return next(e)
194    const questions = e.props.questions as AskQuestion[]
195    if (v.questions.length !== questions.length) return next(e)
196    return next({ ...e, props: { ...e.props, questions: annotate(questions, v.questions) } })
197  })
198}
199

1/** 1 問ぶんの判定。probs は options と同じ並び（multiSelect では各選択肢を選ぶ確率）。 */
2export type QuestionVerdict = {
3  isMulti: boolean
4  probs: number[]
5  confidence: number | null
6}
7
8export type Verdict =
9  | { status: 'pending' }
10  | { status: 'error'; message: string }
11  | { status: 'done'; questions: QuestionVerdict[] }
12
13declare module 'claude-code' {
14  interface PluginState {
15    'jev-judge': { verdicts: Record<string, Verdict> }
16  }
17}
18