cca-quiz

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { Current, Score } from '../types'
5import { QUESTIONS } from './questions'
6
7const enabled = atom({ plugin: 'cca-quiz', key: 'enabled' } as const, false)
8const current = atom({ plugin: 'cca-quiz', key: 'current' } as const, null)
9const picked = atom({ plugin: 'cca-quiz', key: 'picked' } as const, null)
10const score = atom({ plugin: 'cca-quiz', key: 'score' } as const, { right: 0, total: 0 })
11const deck = atom({ plugin: 'cca-quiz', key: 'deck' } as const, [])
12
13const REQUEUE_AFTER = 5
14
15function shuffled<T>(list: readonly T[]): T[] {
16  const out = [...list]
17  for (let i = out.length - 1; i > 0; i--) {
18    const j = Math.floor(Math.random() * (i + 1))
19    ;[out[i], out[j]] = [out[j]!, out[i]!]
20  }
21  return out
22}
23
24function questionOf(cur: Current | null) {
25  return cur ? QUESTIONS.find(q => q.id === cur.id) : undefined
26}
27
28async function advance($: EngineInterface) {
29  const previous = (await read($, current))?.id
30  let id = ''
31  await update($, deck, list => {
32    const pool = list.length > 0 ? list : shuffled(QUESTIONS.map(q => q.id))
33    const [first, ...rest] = pool
34    if (first === undefined) return []
35    if (first === previous && rest.length > 0) {
36      id = rest[0]!
37      return [...rest.slice(1), first]
38    }
39    id = first
40    return rest
41  })
42  const next: Current = { id, order: shuffled([0, 1, 2, 3]) }
43  await update($, picked, () => null)
44  await update($, current, () => next)
45}
46
47async function answer($: EngineInterface, choice: number) {
48  const cur = await read($, current)
49  const q = questionOf(cur)
50  if (!cur || !q) return
51  let isFirst = false
52  await update($, picked, p => {
53    isFirst = p === null
54    return p ?? choice
55  })
56  if (!isFirst) return
57  const isRight = cur.order[choice] === q.answer
58  await update($, score, (s: Score) => ({
59    right: s.right + (isRight ? 1 : 0),
60    total: s.total + 1,
61  }))
62  if (!isRight) {
63    await update($, deck, list => [
64      ...list.slice(0, REQUEUE_AFTER),
65      cur.id,
66      ...list.slice(REQUEUE_AFTER),
67    ])
68  }
69}
70
71export const register: Register = on => {
72  on('session.start', async ($, e, next) => {
73    await $.command.register({
74      name: 'quiz',
75      description: 'Toggle the Claude Certified Architect practice quiz shown while Claude works',
76      argumentHint: '[on|off|reset]',
77      immediate: true,
78    })
79
80    return next(e)
81  })
82
83  on('command.run', { command: 'quiz' }, async ($, e) => {
84    const arg = e.args.trim().toLowerCase()
85
86    if (arg === 'reset') {
87      await update($, score, () => ({ right: 0, total: 0 }))
88      await update($, deck, () => [])
89      $.ui.toast('CCA quiz: score and deck reset')
90
91      return {}
92    }
93
94    const isOn = arg === 'on' ? true : arg === 'off' ? false : !(await read($, enabled))
95    await update($, enabled, () => isOn)
96    if (isOn && !questionOf(await read($, current))) {
97      await advance($)
98    }
99    const { right, total } = await read($, score)
100    $.ui.toast(
101      isOn
102        ? `CCA quiz on (${right}/${total}). Questions show above the prompt while Claude works; press 1-4 in the empty prompt to answer.`
103        : 'CCA quiz off',
104    )
105
106    return {}
107  })
108
109  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
110    if (e.props.hasSurvey || !(await read($, enabled))) {
111      return next(e)
112    }
113    const cur = await read($, current)
114    const choice = await read($, picked)
115    const q = questionOf(cur)
116    if (!cur || !q || (!e.props.isWorking && choice === null)) {
117      return next(e)
118    }
119    const { right, total } = await read($, score)
120    const { Box, Button, Text } = $.ui.resolve(e)
121
122    return (
123      <Box flexDirection="column">
124        <Text dimColor wrap="truncate">
125          {`CCA practice · ${q.domain} · ${right}/${total} correct`}
126        </Text>
127        <Text bold>{q.q}</Text>
128        {cur.order.map((option, i) => {
129          const label = q.options[option]
130          if (choice === null) {
131            return (
132              <Button
133                key={`answer-${i + 1}`}
134                plain
135                hotkey={String(i + 1)}
136                label={label}
137                onPress={() => answer($, i)}
138              />
139            )
140          }
141          if (option === q.answer) {
142            return <Text color="success">{`✓ ${i + 1}: ${label}`}</Text>
143          }
144          if (i === choice) {
145            return <Text color="error">{`✗ ${i + 1}: ${label}`}</Text>
146          }
147
148          return <Text dimColor>{`  ${i + 1}: ${label}`}</Text>
149        })}
150        {choice === null ? (
151          <Text dimColor>Press 1-4 in the empty prompt, or click an answer.</Text>
152        ) : (
153          <Box flexDirection="column">
154            <Text>{q.why}</Text>
155            <Button
156              key="next"
157              plain
158              hotkey="0"
159              label="Next question"
160              onPress={() => advance($)}
161            />
162          </Box>
163        )}
164      </Box>
165    )
166  })
167}
168

1export type Question = {
2  id: string
3  domain: string
4  q: string
5  options: [string, string, string, string]
6  answer: number
7  why: string
8}
9
10const AGENTIC = 'Agentic Architecture & Orchestration'
11const CLAUDE_CODE = 'Claude Code Configuration & Workflows'
12const PROMPTING = 'Prompt Engineering & Structured Output'
13const TOOLS = 'Tool Design & MCP Integration'
14const CONTEXT = 'Context Management & Reliability'
15const DESIGN = 'Solution Design & Architecture'
16const EVALS = 'Evaluation, Testing & Optimisation'
17const GOVERNANCE = 'Governance, Safety & Risk'
18const LIFECYCLE = 'Stakeholders & Lifecycle'
19
20export const QUESTIONS: Question[] = [
21  {
22    id: 'ag-workflow-vs-agent',
23    domain: AGENTIC,
24    q: 'A task always runs the same fixed steps: extract fields, translate, then summarize. Which design fits best?',
25    options: [
26      'A prompt chain (workflow) with a gate check between steps',
27      'An autonomous agent with every tool available',
28      'An orchestrator that spawns a subagent per field',
29      'One very long prompt with all steps and a high temperature',
30    ],
31    answer: 0,
32    why: 'Anthropic advises using the simplest pattern that works. Steps that are fixed and known ahead of time suit a predefined workflow. Agents are for open-ended paths you cannot hard-code.',
33  },
34  {
35    id: 'ag-orchestrator-workers',
36    domain: AGENTIC,
37    q: 'A coding task needs changes to an unknown number of files, decided only after looking at the request. Which pattern fits?',
38    options: [
39      'Orchestrator-workers',
40      'Prompt chaining',
41      'Routing',
42      'Parallelization by voting',
43    ],
44    answer: 0,
45    why: 'In orchestrator-workers, a central model breaks the work into subtasks at run time and hands them to workers. That fits when you cannot predict the subtasks in advance.',
46  },
47  {
48    id: 'ag-evaluator-optimizer',
49    domain: AGENTIC,
50    q: 'You have clear quality criteria and iterating on a draft clearly improves it, like a literary translation. Which pattern fits?',
51    options: [
52      'Evaluator-optimizer loop',
53      'Routing to a cheaper model',
54      'Sectioning parallelization',
55      'A single zero-shot call',
56    ],
57    answer: 0,
58    why: 'Evaluator-optimizer pairs a generator with a model that critiques against criteria, looping until the output passes.',
59  },
60  {
61    id: 'ag-guardrail-parallel',
62    domain: AGENTIC,
63    q: 'You want a content-safety screen on every user query without adding latency to the main answer. Best approach?',
64    options: [
65      'Run the screen as a separate parallel call alongside the main response',
66      'Put the screen instructions in the same prompt as the task',
67      'Screen only every tenth query',
68      'Ask the main model to screen itself after answering',
69    ],
70    answer: 0,
71    why: 'Sectioning parallelization: one instance handles the query while another screens it. Separate calls do better than mixing the guardrail and the core task in one prompt.',
72  },
73  {
74    id: 'ag-stop-reason',
75    domain: AGENTIC,
76    q: 'In a hand-written Messages API agent loop, when should the loop run tools and call the model again?',
77    options: [
78      'While stop_reason is "tool_use"',
79      'While stop_reason is "end_turn"',
80      'Until the response has no text blocks',
81      'A fixed number of times, whatever the stop_reason',
82    ],
83    answer: 0,
84    why: '"tool_use" means Claude wants tool results back. Run the tools, send the tool_result blocks, and call again. "end_turn" means it is done. A max-iterations cap is still a good safety net.',
85  },
86  {
87    id: 'ag-subagent-benefit',
88    domain: AGENTIC,
89    q: 'What is the main architectural benefit of delegating a broad search to a subagent?',
90    options: [
91      'It works in its own context window and returns only a condensed result, keeping the main context clean',
92      'Subagents run on a free model tier',
93      'Subagents can skip permission rules',
94      'It guarantees the result is deterministic',
95    ],
96    answer: 0,
97    why: 'Subagents isolate context. They can read many files or pages and hand back a short summary, so the orchestrator\'s context isn\'t filled with raw exploration.',
98  },
99  {
100    id: 'ag-multiagent-cost',
101    domain: AGENTIC,
102    q: "Anthropic's write-up on its multi-agent research system named which main trade-off?",
103    options: [
104      'Much higher token use than single-agent chat, worth it only for high-value, parallelizable tasks',
105      'Lower accuracy than a single agent on breadth-first questions',
106      'It cannot use tools',
107      'It needs fine-tuned models',
108    ],
109    answer: 0,
110    why: 'Multi-agent runs used about 15x the tokens of a chat. They paid off on broad, parallelizable research and are a poor fit for tightly coupled tasks such as most coding.',
111  },
112  {
113    id: 'ag-irreversible',
114    domain: AGENTIC,
115    q: 'An agent can issue refunds and delete customer records. What is the most important control?',
116    options: [
117      'A human confirmation step before irreversible or high-impact actions',
118      'A higher max_tokens',
119      'Telling the agent in the system prompt to be careful',
120      'Running the agent at temperature 0',
121    ],
122    answer: 0,
123    why: 'Prompts alone are not enforcement. Gate destructive actions with human-in-the-loop checkpoints and least-privilege tools.',
124  },
125  {
126    id: 'cc-claudemd-scope',
127    domain: CLAUDE_CODE,
128    q: 'Your team wants coding conventions that every developer\'s Claude Code session picks up in this repo. Where do they go?',
129    options: [
130      'CLAUDE.md at the repo root, committed',
131      '~/.claude/CLAUDE.md on each laptop',
132      'CLAUDE.local.md, committed',
133      'An environment variable in CI',
134    ],
135    answer: 0,
136    why: 'The project CLAUDE.md is checked in and shared. ~/.claude/CLAUDE.md is personal across all projects. CLAUDE.local.md is personal to one project and is meant to be gitignored.',
137  },
138  {
139    id: 'cc-hooks-deterministic',
140    domain: CLAUDE_CODE,
141    q: 'You need a formatter to run after every file Claude edits, every time. What is the right tool?',
142    options: [
143      'A PostToolUse hook matching the edit/write tools',
144      'A line in CLAUDE.md asking Claude to format',
145      'A custom slash command',
146      'A subagent dedicated to formatting',
147    ],
148    answer: 0,
149    why: 'Hooks are deterministic: the harness runs them, not the model. CLAUDE.md is guidance the model may not always follow.',
150  },
151  {
152    id: 'cc-pretooluse-block',
153    domain: CLAUDE_CODE,
154    q: 'A command-type PreToolUse hook should block a dangerous Bash call and tell Claude why. How?',
155    options: [
156      'Exit with code 2 and write the reason to stderr',
157      'Exit 0 and print the reason to stdout',
158      'Exit 1 so Claude Code crashes safely',
159      'Delete the tool from settings.json at run time',
160    ],
161    answer: 0,
162    why: 'Exit code 2 blocks the call and sends stderr back to Claude. Exit 0 allows it. Other codes are non-blocking errors shown to the user. Hooks can also return JSON with a permission decision.',
163  },
164  {
165    id: 'cc-headless-ci',
166    domain: CLAUDE_CODE,
167    q: 'Which invocation fits a CI job that runs Claude Code without interaction and parses the result?',
168    options: [
169      'claude -p "<task>" --output-format json',
170      'claude --interactive "<task>"',
171      'claude /compact',
172      'claude --resume',
173    ],
174    answer: 0,
175    why: '-p (print/headless) mode runs once and exits. --output-format json (or stream-json) gives machine-readable output. Pair it with --allowedTools or permission settings so it doesn\'t stop for prompts.',
176  },
177  {
178    id: 'cc-deny-secrets',
179    domain: CLAUDE_CODE,
180    q: 'How do you reliably stop Claude Code from ever reading .env files in a project?',
181    options: [
182      'A permissions deny rule such as Read(./.env*) in settings.json',
183      'Ask in CLAUDE.md not to open .env',
184      'Add .env to .gitignore',
185      'Rename .env to .env.secret',
186    ],
187    answer: 0,
188    why: 'Permission rules are enforced by the harness, and deny wins over allow. .gitignore affects git, not Claude\'s file access.',
189  },
190  {
191    id: 'cc-mcp-shared',
192    domain: CLAUDE_CODE,
193    q: 'You want an MCP server available to everyone who clones the repo. Where do you configure it?',
194    options: [
195      '.mcp.json at the project root (project scope)',
196      'Each user\'s local MCP scope',
197      'CLAUDE.md',
198      '.claude/commands/',
199    ],
200    answer: 0,
201    why: 'Project-scoped servers live in a checked-in .mcp.json. Claude Code asks each user to approve project servers before using them.',
202  },
203  {
204    id: 'cc-skills',
205    domain: CLAUDE_CODE,
206    q: 'You have a long, specialised procedure (e.g. a release checklist) Claude should use only when relevant, without bloating every session. Best fit?',
207    options: [
208      'A Skill: SKILL.md with a description, loaded on demand',
209      'Paste it into the root CLAUDE.md',
210      'A PreToolUse hook',
211      'Put it in the system prompt with --append-system-prompt every time',
212    ],
213    answer: 0,
214    why: 'Skills use progressive disclosure: only the name and description sit in context. The full instructions load when the task calls for them.',
215  },
216  {
217    id: 'cc-plan-mode',
218    domain: CLAUDE_CODE,
219    q: 'Before a risky multi-file refactor, you want Claude to research and propose an approach without changing anything. Use:',
220    options: [
221      'Plan mode',
222      'Auto-accept edits mode',
223      '--dangerously-skip-permissions',
224      '/clear',
225    ],
226    answer: 0,
227    why: 'Plan mode is read-only: Claude explores and presents a plan for approval before any edits.',
228  },
229  {
230    id: 'cc-managed-settings',
231    domain: CLAUDE_CODE,
232    q: 'Security must enforce a Claude Code policy that developers cannot override in user or project settings. Where?',
233    options: [
234      'Managed (enterprise policy) settings',
235      '.claude/settings.local.json',
236      '~/.claude/settings.json',
237      'The repo CLAUDE.md',
238    ],
239    answer: 0,
240    why: 'Managed policy settings take precedence over everything else in the settings hierarchy.',
241  },
242  {
243    id: 'pe-json-reliable',
244    domain: PROMPTING,
245    q: 'What is the most reliable way to get output that always matches a JSON schema?',
246    options: [
247      'Structured outputs / a strict tool definition with the JSON schema',
248      'Say "respond only in JSON" in the system prompt',
249      'Raise the temperature so it explores formats',
250      'Ask for JSON and fix it with regex afterwards',
251    ],
252    answer: 0,
253    why: 'Constrained decoding against a schema (structured outputs or strict tool use) guarantees valid output. Prompt-only instructions are best effort.',
254  },
255  {
256    id: 'pe-xml-tags',
257    domain: PROMPTING,
258    q: 'Your prompt mixes instructions, a contract, and examples. What helps Claude keep them apart?',
259    options: [
260      'Wrap each part in clear XML tags such as <contract> and <example>',
261      'Write everything in capital letters',
262      'Put it all in one paragraph to save tokens',
263      'Move the instructions into the assistant prefill',
264    ],
265    answer: 0,
266    why: 'XML tags give clear boundaries, cut down on mixing instructions with data, and make the output easy to parse.',
267  },
268  {
269    id: 'pe-long-doc-order',
270    domain: PROMPTING,
271    q: 'With 20k+ tokens of documents in a prompt, where should the question go?',
272    options: [
273      'At the end, after the documents',
274      'At the top, before the documents',
275      'Repeated between every document',
276      'Only in the system prompt',
277    ],
278    answer: 0,
279    why: 'Anthropic\'s guidance: put long documents near the top and the query at the end. Their tests showed up to ~30% better response quality on complex, multi-document inputs.',
280  },
281  {
282    id: 'pe-few-shot',
283    domain: PROMPTING,
284    q: 'Which few-shot practice does Anthropic recommend?',
285    options: [
286      '3-5 relevant, diverse examples wrapped in <example> tags',
287      'One example repeated several times',
288      'As many near-identical examples as fit',
289      'Avoid examples; they always cause overfitting',
290    ],
291    answer: 0,
292    why: 'Diverse, relevant examples show the pattern without Claude copying one example too closely.',
293  },
294  {
295    id: 'pe-hallucination',
296    domain: PROMPTING,
297    q: 'Which technique best reduces made-up claims when answering from provided documents?',
298    options: [
299      'Allow "I don\'t know" and require supporting quotes before the answer',
300      'Raise max_tokens',
301      'Ask for a longer answer',
302      'Remove the documents and rely on training data',
303    ],
304    answer: 0,
305    why: 'Giving Claude permission to say it doesn\'t know and grounding answers in direct quotes (or the Citations feature) cuts hallucinations.',
306  },
307  {
308    id: 'pe-thinking-tools',
309    domain: PROMPTING,
310    q: 'With extended thinking on during a tool-use loop, what must you do with the thinking blocks?',
311    options: [
312      'Pass them back unchanged in the assistant turn alongside the tool_use',
313      'Strip them to save tokens',
314      'Summarise them yourself before sending them back',
315      'Move them into the system prompt',
316    ],
317    answer: 0,
318    why: 'During tool use, the last assistant turn\'s thinking blocks must be returned unmodified (they carry signatures) so Claude can continue its reasoning.',
319  },
320  {
321    id: 'pe-batch',
322    domain: PROMPTING,
323    q: 'You must classify 2 million support tickets overnight and nobody waits on the results. Which API feature fits?',
324    options: [
325      'Message Batches API',
326      'Streaming',
327      'A higher temperature',
328      'Extended thinking with a large budget',
329    ],
330    answer: 0,
331    why: 'Batches are asynchronous, processed within 24 hours, and cost 50% less than standard calls. A good fit for bulk work that isn\'t latency-sensitive.',
332  },
333  {
334    id: 'pe-caching',
335    domain: PROMPTING,
336    q: 'To get the most from prompt caching on a chatbot with a long fixed system prompt, how should you order the request?',
337    options: [
338      'Static content (tools, system prompt) first, with the cache breakpoint after it; changing content last',
339      'User message first, then system prompt',
340      'Put a timestamp at the top so every cache entry is fresh',
341      'Order doesn\'t matter for caching',
342    ],
343    answer: 0,
344    why: 'The cache matches an exact prefix. Anything that changes before the breakpoint breaks the cache. Cache reads cost ~10% of base input price, and the default TTL is 5 minutes (1 hour is an option).',
345  },
346  {
347    id: 'mcp-primitives',
348    domain: TOOLS,
349    q: 'In MCP, which primitive is chosen by the user (e.g. a template picked from a menu), not by the model or the app?',
350    options: ['Prompts', 'Tools', 'Resources', 'Sampling'],
351    answer: 0,
352    why: 'Tools are model-controlled, resources are application-controlled, and prompts are user-controlled templates.',
353  },
354  {
355    id: 'mcp-transport',
356    domain: TOOLS,
357    q: 'Which MCP transport fits a remote, multi-client server hosted by your platform team?',
358    options: [
359      'Streamable HTTP',
360      'stdio',
361      'Unix pipes only',
362      'Shared memory',
363    ],
364    answer: 0,
365    why: 'stdio is for local servers launched as subprocesses. Streamable HTTP is the current remote transport and replaces the older HTTP+SSE transport.',
366  },
367  {
368    id: 'tool-descriptions',
369    domain: TOOLS,
370    q: 'Claude keeps picking the wrong tool among several similar ones. What is the first fix to try?',
371    options: [
372      'Write detailed descriptions: what each tool does, when to use it and when not to, parameter meaning, caveats',
373      'Shorten every description to one line',
374      'Add more similar tools',
375      'Lower max_tokens',
376    ],
377    answer: 0,
378    why: 'Tool descriptions matter most to tool-use performance. Anthropic recommends a thorough description of several sentences per tool, plus merging or namespacing overlapping tools.',
379  },
380  {
381    id: 'tool-consolidate',
382    domain: TOOLS,
383    q: 'An MCP server has list_users, list_events and create_event, and agents chain them clumsily to book meetings. Anthropic\'s tool-writing guidance suggests:',
384    options: [
385      'Consolidate into a higher-level tool such as schedule_event that does the common workflow',
386      'Expose every raw API endpoint as its own tool',
387      'Return full, unfiltered database dumps so the agent has everything',
388      'Remove the descriptions to save context',
389    ],
390    answer: 0,
391    why: 'Agent tools should match real tasks, not wrap APIs one-to-one. They should return focused, token-efficient context (filtering, pagination, truncation).',
392  },
393  {
394    id: 'tool-errors',
395    domain: TOOLS,
396    q: 'Your tool hits a recoverable error (bad date format). What should it return to Claude?',
397    options: [
398      'A tool_result with is_error: true and a message saying what went wrong and how to fix the input',
399      'An empty tool_result',
400      'Nothing; just end the conversation',
401      'A raw stack trace with no explanation',
402    ],
403    answer: 0,
404    why: 'Clear, actionable errors let Claude fix its call and retry instead of guessing.',
405  },
406  {
407    id: 'tool-parallel-results',
408    domain: TOOLS,
409    q: 'Claude returns three tool_use blocks in one response. How do you send back the results?',
410    options: [
411      'All three tool_result blocks in a single user message, matched by tool_use_id',
412      'Three separate user messages, one per result',
413      'Only the first result; Claude will ask again',
414      'Put the results in the system prompt',
415    ],
416    answer: 0,
417    why: 'Parallel tool results belong together in one user message. Splitting them teaches Claude to stop making parallel calls.',
418  },
419  {
420    id: 'tool-choice',
421    domain: TOOLS,
422    q: 'You need Claude to call a specific tool, extract_invoice, on every request. Which tool_choice?',
423    options: [
424      '{"type": "tool", "name": "extract_invoice"}',
425      '{"type": "auto"}',
426      '{"type": "none"}',
427      'Leave tools empty and ask for JSON',
428    ],
429    answer: 0,
430    why: 'tool_choice options: auto (model decides), any (must use some tool), tool (must use the named one), none.',
431  },
432  {
433    id: 'mcp-injection',
434    domain: TOOLS,
435    q: 'A third-party MCP server\'s tool results may contain text crafted to hijack the agent. Best mitigation?',
436    options: [
437      'Treat tool output as untrusted data, use least-privilege tools, and require approval for sensitive actions',
438      'Trust it because MCP is a standard',
439      'Turn off all logging',
440      'Raise the model temperature',
441    ],
442    answer: 0,
443    why: 'Prompt injection via tool results is a known risk. Use defence in depth: limit scopes, vet servers, and keep a human in the loop for high-impact actions.',
444  },
445  {
446    id: 'ctx-context-rot',
447    domain: CONTEXT,
448    q: 'Per Anthropic\'s context-engineering guidance, why not fill a long-running agent\'s context with everything that might help?',
449    options: [
450      'Recall and focus degrade as context grows ("context rot"); aim for the smallest set of high-signal tokens',
451      'The API rejects any request over 10k tokens',
452      'Extra context always makes the model refuse',
453      'Context is free, so this is actually best practice',
454    ],
455    answer: 0,
456    why: 'Attention is a finite budget. Curate context: compaction, structured note-taking, just-in-time retrieval, and subagents.',
457  },
458  {
459    id: 'ctx-just-in-time',
460    domain: CONTEXT,
461    q: 'An agent works over a large codebase. Which context strategy matches the "just-in-time" approach?',
462    options: [
463      'Keep light references (paths, queries) and load content with tools when needed',
464      'Concatenate the whole repo into the first prompt',
465      'Fine-tune the model on the repo',
466      'Pre-summarise every file into the system prompt',
467    ],
468    answer: 0,
469    why: 'Like Claude Code using grep, glob and file reads, the agent pulls in only what the current step needs.',
470  },
471  {
472    id: 'ctx-compact-clear',
473    domain: CONTEXT,
474    q: 'In Claude Code you finished one task and are starting an unrelated one. Best move?',
475    options: [
476      '/clear to start with a fresh context',
477      '/compact so the old task stays summarised',
478      'Keep going; more history always helps',
479      'Restart your computer',
480    ],
481    answer: 0,
482    why: 'Unrelated history is noise. Use /clear between unrelated tasks, and /compact (optionally with instructions) to keep the essentials of an ongoing task.',
483  },
484  {
485    id: 'ctx-retries',
486    domain: CONTEXT,
487    q: 'Production calls sometimes get 429 or 529 (overloaded) errors. Correct handling?',
488    options: [
489      'Retry with exponential backoff and jitter, honouring retry-after',
490      'Retry immediately in a tight loop',
491      'Treat both as permanent failures',
492      'Switch to a random different API key',
493    ],
494    answer: 0,
495    why: 'Both are transient. Back off (the SDKs retry these by default), respect retry-after, and consider queueing or a fallback model for 529s.',
496  },
497  {
498    id: 'ctx-token-count',
499    domain: CONTEXT,
500    q: 'You need to know whether a large request will fit the context window before sending it. Use:',
501    options: [
502      'The token counting endpoint',
503      'Character count divided by 2',
504      'Send it and read the error',
505      'The Batches API',
506    ],
507    answer: 0,
508    why: 'The count_tokens endpoint returns the input token count for the exact messages, tools and system prompt, without running the model.',
509  },
510  {
511    id: 'pro-model-selection',
512    domain: DESIGN,
513    q: 'A high-volume, latency-sensitive intent classifier with simple labels. Which model choice is most defensible?',
514    options: [
515      'The smallest fast tier (Haiku), checked against an eval set',
516      'The largest model, to be safe',
517      'Whichever model is newest',
518      'Two large models voting on every request',
519    ],
520    answer: 0,
521    why: 'Pick the cheapest, fastest model that meets your eval bar. Save the biggest models for hard reasoning, or route to them only when needed.',
522  },
523  {
524    id: 'pro-cloud-boundary',
525    domain: DESIGN,
526    q: 'A bank requires all inference to stay inside its existing AWS account controls and billing. Which option fits?',
527    options: [
528      'Claude on Amazon Bedrock',
529      'The consumer Claude.ai app',
530      'A self-hosted copy of the Claude weights',
531      'Screen-scraping the web UI',
532    ],
533    answer: 0,
534    why: 'Claude is offered through Amazon Bedrock and Google Cloud Vertex AI, so enterprises can use their cloud\'s IAM, networking and billing. The weights are not self-hostable.',
535  },
536  {
537    id: 'pro-citations',
538    domain: DESIGN,
539    q: 'A legal research app must show which sentence of which source document supports each claim. Which feature fits?',
540    options: [
541      'The Citations feature on document content blocks',
542      'Prefilling the answer with "According to"',
543      'A higher temperature',
544      'Extended thinking alone',
545    ],
546    answer: 0,
547    why: 'Citations returns answer text tied to exact passages of the provided documents, which is more reliable than asking for citations in the prompt.',
548  },
549  {
550    id: 'pro-streaming',
551    domain: DESIGN,
552    q: 'Users complain a chat feels slow, though total generation time is fine. Cheapest improvement?',
553    options: [
554      'Stream the response so tokens show as they are produced',
555      'Switch to the Batches API',
556      'Raise max_tokens',
557      'Add extended thinking',
558    ],
559    answer: 0,
560    why: 'Streaming cuts time-to-first-token, which drives how fast the chat feels.',
561  },
562  {
563    id: 'pro-eval-grader',
564    domain: EVALS,
565    q: 'Your eval checks a classifier whose output is one of 12 fixed labels. Best grading method?',
566    options: [
567      'Code-based exact match against labelled answers',
568      'An LLM-as-judge with a free-form rubric',
569      'Manual review of every output',
570      'Count how long the outputs are',
571    ],
572    answer: 0,
573    why: 'Use the simplest reliable grader. Code-based checks are fast, cheap and deterministic. Save LLM judges for open-ended outputs that need nuance.',
574  },
575  {
576    id: 'pro-pass-k',
577    domain: EVALS,
578    q: 'A customer-facing agent must succeed every time it is run. Which metric reflects that?',
579    options: [
580      'pass^k: the chance that all k trials succeed',
581      'pass@k: the chance that at least one of k trials succeeds',
582      'Average output length',
583      'Tokens per second',
584    ],
585    answer: 0,
586    why: 'pass@k rewards succeeding once in k tries, which suits tools where one good answer is enough. pass^k measures consistency across tries, which is what a customer-facing agent needs.',
587  },
588  {
589    id: 'pro-eval-first',
590    domain: EVALS,
591    q: 'A teammate wants to tweak the production system prompt. What should come first?',
592    options: [
593      'An eval set with success criteria, run before and after the change',
594      'Ship it and watch social media',
595      'Ask Claude whether the new prompt is better',
596      'Test it on one example by hand',
597    ],
598    answer: 0,
599    why: 'Define success criteria and build evals before prompt work, so changes are measured rather than judged by feel and regressions get caught.',
600  },
601  {
602    id: 'pro-cost',
603    domain: EVALS,
604    q: 'An assistant sends the same 30k-token policy manual with every request. Biggest cost lever?',
605    options: [
606      'Prompt caching on the manual prefix',
607      'Lowering temperature',
608      'Shortening user messages by 5%',
609      'Switching to streaming',
610    ],
611    answer: 0,
612    why: 'Cached prefix reads cost about 10% of normal input price and also cut latency. Repeated large prefixes are the classic use case.',
613  },
614  {
615    id: 'pro-pinned-models',
616    domain: LIFECYCLE,
617    q: 'How should production code refer to the model to avoid surprise behaviour changes?',
618    options: [
619      'Pin a specific model snapshot ID and migrate deliberately after re-running evals',
620      'Always use a moving "latest" alias',
621      'Let each developer choose at run time',
622      'Hard-code whichever model the docs show first',
623    ],
624    answer: 0,
625    why: 'Aliases can move to new snapshots. Pinned IDs keep behaviour stable. Plan migrations around deprecation notices and validate with your evals.',
626  },
627  {
628    id: 'pro-stakeholder',
629    domain: LIFECYCLE,
630    q: 'Leadership asks, "Is the AI accurate?" The most defensible answer is:',
631    options: [
632      'Measured rates from a representative eval set, with known failure modes and how they are mitigated',
633      '"It is state of the art, so yes"',
634      'A few cherry-picked good transcripts',
635      '"It is 100% accurate after prompt tuning"',
636    ],
637    answer: 0,
638    why: 'Defend decisions with evidence: eval results on realistic data, error analysis, and the human or automated safeguards for the remaining risk.',
639  },
640  {
641    id: 'pro-agent-injection',
642    domain: GOVERNANCE,
643    q: 'A browsing agent with email access reads untrusted web pages. Which control best limits prompt-injection damage?',
644    options: [
645      'Least-privilege tool scopes plus user confirmation before sending email or other sensitive actions',
646      'A system prompt saying "ignore malicious instructions"',
647      'Only browsing pages over HTTPS',
648      'A larger model',
649    ],
650    answer: 0,
651    why: 'No prompt makes injection impossible. Limit what a hijacked agent could do (scopes, sandboxing) and gate consequential actions with confirmation.',
652  },
653  {
654    id: 'pro-pii',
655    domain: GOVERNANCE,
656    q: 'A healthcare workflow sends patient notes to Claude. Which step belongs in the design before launch?',
657    options: [
658      'Confirm the right contractual and data-retention terms (e.g. a BAA where required) and minimise the PHI sent',
659      'Nothing; API traffic is never stored anywhere',
660      'Base64-encode the notes so they are anonymous',
661      'Tell the model to forget the data afterwards',
662    ],
663    answer: 0,
664    why: 'Regulated data needs the right agreements and retention settings, plus data minimisation and redaction where possible. Encoding is not anonymisation.',
665  },
666  {
667    id: 'pro-team-enablement',
668    domain: LIFECYCLE,
669    q: 'Fastest way to give a 40-engineer team a consistent Claude Code setup (rules, MCP servers, permissions)?',
670    options: [
671      'Check in CLAUDE.md, .mcp.json and .claude/settings.json (plus shared skills or plugins) in the repo',
672      'Send a wiki page and hope everyone copies it',
673      'Give everyone --dangerously-skip-permissions',
674      'Have each engineer write their own CLAUDE.md from memory',
675    ],
676    answer: 0,
677    why: 'Project-scoped, version-controlled configuration spreads automatically and changes are reviewed like code. Use managed settings for anything that must be enforced.',
678  },
679]
680

1export type Score = { right: number; total: number }
2export type Current = { id: string; order: number[] }
3
4declare module 'claude-code' {
5  interface PluginState {
6    'cca-quiz': {
7      enabled: boolean
8      current: Current | null
9      picked: number | null
10      score: Score
11      deck: string[]
12    }
13  }
14}
15