From e4abbd7248a5cf6ca6f7204646bb5f5b98c0a7a3 Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 21:16:28 +0100 Subject: [PATCH 01/12] docs(rag): add design and implementation plan for the quiz agent harness Refs #134 --- .../plans/2026-11-05-quiz-agent-harness.md | 1058 +++++++++++++++++ .../2026-11-05-quiz-agent-harness-design.md | 257 ++++ 2 files changed, 1315 insertions(+) create mode 100644 docs/superpowers/plans/2026-11-05-quiz-agent-harness.md create mode 100644 docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md diff --git a/docs/superpowers/plans/2026-11-05-quiz-agent-harness.md b/docs/superpowers/plans/2026-11-05-quiz-agent-harness.md new file mode 100644 index 0000000..854e380 --- /dev/null +++ b/docs/superpowers/plans/2026-11-05-quiz-agent-harness.md @@ -0,0 +1,1058 @@ +# Quiz Agent Harness Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace quiz generation's single schema-validated `/generate_json` call with a bounded +plan → generate → critique → repair loop, so quiz questions are checked against explicit quality +criteria instead of only being well-formed. + +**Architecture:** A new `app/services/quiz_harness.py` module owns the four-stage loop and is +called by `quiz_generation.generate_quiz_questions` (unchanged public signature, delegates +internally). Both existing callers (course builder, study page) feed it through the existing +`context_items` convention, extended with an explicit `chunk_id` key. Persistence +(`quiz_generation.persist_quiz`) is untouched. + +**Tech Stack:** Python 3.11, FastAPI, SQLAlchemy, `qvac_structured.generate_json` (async, schema- +validated JSON over HTTP), pytest + `unittest.mock.AsyncMock`. + +**Spec:** [`docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md`](../specs/2026-11-05-quiz-agent-harness-design.md) + +## Global Constraints + +- Every returned question carries a resolvable `chunk_id` (issue #134 acceptance criterion). +- Hard cap on LLM calls, no unbounded loop — enforced by construction: at most + `1 (plan) + N (generate) + N (critique) + N (repair-generate) + N (repair-critique) = 4N + 1` + calls for `N` targets, with exactly one repair round per rejected question. +- Degrade honestly: on `LlmDisabledError`/`StructuredGenerationError`, return fewer questions (or + none) with a reason in the trace — never raise out of the harness, never return malformed output. +- `generate_json` is always imported and called as `from app.services.qvac_structured import + generate_json` inside the module that calls it, and tests patch it at that module's path (e.g. + `app.services.quiz_harness.generate_json`), per the existing convention in + `quiz_generation.py`/`lesson_service.py`. +- No new background job / `GenerationRun` row for quizzes — stays synchronous, same shape as + today's `POST /courses/{course_id}/quizzes/generate`. +- No frontend changes in this plan — the `trace` field rides in the existing API response only. + +## Review Focus + +- Fewer distinct chunks in `context_items` than the target question count — the plan stage must + reuse chunks (not crash or return fewer targets than requested). Covered in Task 2. +- A critique response missing one of the required boolean fields (worker validated the schema but + a key check in Python code must not `KeyError`) — use `.get(..., False)` defensively. Covered in + Task 4. +- Duplicate questions across the same run — the critique call for question *k* must see the + already-accepted questions 1..k-1 and flag a near-duplicate, not silently accept it. Covered in + Task 4. +- A repair-stage call (generate or critique) itself raising `LlmDisabledError` mid-run — the run + must drop that one target and continue/finish rather than propagating the exception out of + `run()`. Covered in Task 5. +- `target_count` larger than the number of distinct `context_items` entries — round-robin target + assignment must not raise an `IndexError` and must still produce valid, distinct questions. + Covered in Task 2 (planning) and Task 5 (full run). + +--- + +## Task 1: Fix `context_items` so every caller's entries carry a `chunk_id` + +The study-page caller currently builds `context_items` from `result.citations` +(`{label, text}`), which has no chunk id anywhere on it — only `result.evidence_pack.chunks[i] +.chunk_id` does. The course-builder caller already has a chunk id, just under the `label` key. Both +need an explicit `"chunk_id"` key before the harness (Task 2+) can plan against it. + +**Files:** +- Modify: `app/services/lesson_service.py:189-195` (`_load_context`) +- Modify: `app/api/quizzes_api.py:159-172` (`generate_quiz`) +- Test: `tests/unit/test_lesson_service.py:140-151` (`test_load_context_returns_items`) +- Test: `tests/integration/test_quizzes_certificates_api.py:14,38-42,58-105` + +**Interfaces:** +- Produces: every `context_items` entry downstream now has keys `{"label": str, "text": str, + "chunk_id": str}` — this is what Task 2's `_plan_targets` and Task 3's `_generate_one` consume. + +- [ ] **Step 1: Write the failing test for `_load_context`** + +In `tests/unit/test_lesson_service.py`, extend `test_load_context_returns_items` with: +```python + assert items[0]["chunk_id"] == "p1" +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_lesson_service.py::test_load_context_returns_items -v` +Expected: FAIL — `KeyError: 'chunk_id'` + +- [ ] **Step 3: Add `chunk_id` to `_load_context`'s output** + +In `app/services/lesson_service.py:193`, change: +```python + context_items.append({"label": cid, "text": rows[cid].text}) +``` +to also set `"chunk_id": cid` (same value as `label` here — `label` already doubles as the chunk id +for this caller; this just makes the key explicit and uniform with the study-page caller after +Step 6). + +- [ ] **Step 4: Run it to verify it passes** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_lesson_service.py::test_load_context_returns_items -v` +Expected: PASS + +- [ ] **Step 5: Write the failing test for `quizzes_api.generate_quiz`'s context_items** + +In `tests/integration/test_quizzes_certificates_api.py`, the module-level `_RETRIEVE_RESULT` +currently only sets `citations`. Replace it with a `DispatchResult` that also carries an +`evidence_pack`: +```python +from app.schemas.evidence_pack import CitationAnchor, EvidenceChunk, EvidencePack + +_RETRIEVE_RESULT = DispatchResult( + answer="", + citations=[SourceChunk(snippet="A UTXO is an unspent transaction output.", score=0.9, label="p1")], + retrieval_used=True, + evidence_pack=EvidencePack( + query="UTXO model basics", + action="retrieve", + chunks=[ + EvidenceChunk( + chunk_id="chunk-1", + text="A UTXO is an unspent transaction output.", + score=0.9, + anchor=CitationAnchor( + doc_id="doc-1", doc_name="p1", chunk_id="chunk-1", chunk_type="text", + ), + ) + ], + total_candidates=1, + ordering=[0], + deduped_passages=["A UTXO is an unspent transaction output."], + ), +) +``` +Add a new test `test_generate_quiz_uses_evidence_pack_chunk_id` that patches +`app.services.quiz_generation.generate_json` (unchanged patch target for this task — Task 7 +switches this to `quiz_harness.generate_json`), asserts the mock's `context` kwarg on its first +call contains a dict with `"chunk_id": "chunk-1"`. + +- [ ] **Step 6: Run it to verify it fails** + +Run: `cd services/ai && uv run --no-sync pytest tests/integration/test_quizzes_certificates_api.py::TestGenerateQuiz::test_generate_quiz_uses_evidence_pack_chunk_id -v` +Expected: FAIL — context items built from `citations` have no `chunk_id`. + +- [ ] **Step 7: Build `context_items` from `result.evidence_pack.chunks` in `quizzes_api.py`** + +In `app/api/quizzes_api.py:159-172`, replace the `result.citations`-based comprehension: +```python + if result.evidence_pack is None or not result.evidence_pack.chunks: + raise ValidationError_( + "No source material found for this topic in the course. Try a broader query." + ) + + context_items = [ + {"label": c.anchor.doc_name, "text": c.text, "chunk_id": c.chunk_id} + for c in result.evidence_pack.chunks + ] +``` +(drops the old `if not context_items: raise ...` check, replaced by the `evidence_pack` guard above +it, since an empty-chunks pack and a `None` pack are now both "no source material"). + +- [ ] **Step 8: Run it to verify it passes, then run the rest of the quiz integration tests** + +Run: `cd services/ai && uv run --no-sync pytest tests/integration/test_quizzes_certificates_api.py -v` +Expected: all PASS, including `test_generate_quiz_422_when_no_source_material` (its +`DispatchResult` has no `evidence_pack` set, so the `None` guard fires) and +`test_generate_quiz_persists_to_db` (uses the now-fixed `_RETRIEVE_RESULT`). + +- [ ] **Step 9: Commit** + +```bash +git add app/services/lesson_service.py app/api/quizzes_api.py tests/unit/test_lesson_service.py tests/integration/test_quizzes_certificates_api.py +git commit -m "fix(rag): carry chunk_id through quiz context_items for both callers" +``` + +--- + +## Task 2: `quiz_harness.py` — plan stage + +Selects which passages to test and at what cognitive level, from the evidence already in +`context_items` — no new retrieval call. The model is asked to refer to passages by their 1-based +position (matching the `[N]` convention `generate-json.js`'s `buildContextBlock` already uses), not +by `chunk_id` text, so an invented or out-of-range reference can be corrected deterministically +instead of trusted. + +**Files:** +- Create: `app/services/quiz_harness.py` +- Create: `tests/unit/test_quiz_harness.py` +- Modify: `app/services/qvac_router.py:33-42` (`LADDER_ROUTING`) +- Modify: `tests/unit/test_qvac_router.py` + +**Interfaces:** +- Consumes: `app.services.qvac_structured.generate_json(prompt, schema, *, context=None, + system_prompt=None, generation_params=None, task_type=None) -> Dict[str, Any]`; + `app.services.qvac_structured.LlmDisabledError`, `StructuredGenerationError`. +- Produces: `PlanTarget = Dict[str, str]` with keys `{"chunk_id": str, "cognitive_level": str}`; + `async def _plan_targets(context_items: List[Dict[str, str]], target_count: int) -> + List[PlanTarget]` — always returns exactly `target_count` targets (round-robin over + `context_items` filling in for anything missing/invalid from the model), or raises + `LlmDisabledError`/`StructuredGenerationError` if the `generate_json` call itself fails. Consumed + by Task 5's `run()`. + +- [ ] **Step 1: Write the failing tests** + +```python +# tests/unit/test_quiz_harness.py +import pytest +from unittest.mock import AsyncMock, patch + +from app.services.qvac_structured import LlmDisabledError, StructuredGenerationError +from app.services.quiz_harness import _plan_targets + +_CONTEXT = [ + {"label": "p1", "text": "UTXO text", "chunk_id": "c1"}, + {"label": "p2", "text": "PoW text", "chunk_id": "c2"}, +] + + +@pytest.mark.asyncio +async def test_plan_targets_maps_passage_index_to_chunk_id(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"targets": [ + {"passage_index": 2, "cognitive_level": "application"}, + {"passage_index": 1, "cognitive_level": "recall"}, + ]} + targets = await _plan_targets(_CONTEXT, target_count=2) + + assert targets == [ + {"chunk_id": "c2", "cognitive_level": "application"}, + {"chunk_id": "c1", "cognitive_level": "recall"}, + ] + + +@pytest.mark.asyncio +async def test_plan_targets_corrects_invalid_passage_index(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"targets": [{"passage_index": 99, "cognitive_level": "recall"}]} + targets = await _plan_targets(_CONTEXT, target_count=1) + + assert targets[0]["chunk_id"] in {"c1", "c2"} + + +@pytest.mark.asyncio +async def test_plan_targets_pads_when_model_returns_too_few(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"targets": [{"passage_index": 1, "cognitive_level": "recall"}]} + targets = await _plan_targets(_CONTEXT, target_count=3) + + assert len(targets) == 3 + + +@pytest.mark.asyncio +async def test_plan_targets_reuses_chunks_when_target_count_exceeds_available(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"targets": []} + targets = await _plan_targets(_CONTEXT, target_count=5) + + assert len(targets) == 5 + assert {t["chunk_id"] for t in targets} == {"c1", "c2"} + + +@pytest.mark.asyncio +async def test_plan_targets_propagates_llm_disabled(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = LlmDisabledError("disabled") + with pytest.raises(LlmDisabledError): + await _plan_targets(_CONTEXT, target_count=1) +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -v` +Expected: FAIL — `ModuleNotFoundError: No module named 'app.services.quiz_harness'` + +- [ ] **Step 3: Create `app/services/quiz_harness.py` with the plan stage** + +```python +"""Agent harness for quiz generation — plan -> generate -> critique -> repair. + +See docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md. +""" +import logging +from typing import Any, Dict, List + +from app.services.qvac_structured import generate_json + +logger = logging.getLogger(__name__) + +PlanTarget = Dict[str, str] + +_COGNITIVE_LEVELS = ("recall", "application", "analysis") + +PLAN_PROMPT_VERSION = "v1" + +_PLAN_SCHEMA: Dict[str, Any] = { + "type": "object", + "required": ["targets"], + "properties": { + "targets": { + "type": "array", + "items": { + "type": "object", + "required": ["passage_index", "cognitive_level"], + "properties": { + "passage_index": {"type": "integer"}, + "cognitive_level": {"type": "string", "enum": list(_COGNITIVE_LEVELS)}, + }, + }, + } + }, +} + +_PLAN_SYSTEM = ( + "You are planning a quiz. Given numbered source passages, select which passages to " + "test and at what cognitive level, so coverage spans distinct passages and levels " + "rather than clustering on one passage.\n" + "Refer to each passage by the 1-based index shown before it (e.g. passage [2]).\n" + "cognitive_level: 'recall' (states a fact), 'application' (uses a concept in a new " + "situation), or 'analysis' (compares or explains relationships between concepts)." +) + + +async def _plan_targets(context_items: List[Dict[str, str]], target_count: int) -> List[PlanTarget]: + """Select target_count (chunk_id, cognitive_level) pairs from context_items. + + Always returns exactly target_count targets. An invalid or missing + passage_index from the model is corrected by round-robin assignment over + context_items rather than trusted — the model never needs to reproduce a + chunk_id string correctly. + """ + ... +``` + +The implementer writes the body: one `generate_json(prompt, _PLAN_SCHEMA, context=context_items, +system_prompt=_PLAN_SYSTEM, generation_params={"temp": 0.2}, task_type="quiz_plan")` call (prompt +text states `target_count` and lists the available passage count), then builds the +`target_count`-length result by walking `result.get("targets", [])`: for each raw target, resolve +`passage_index` to `context_items[idx - 1]["chunk_id"]` when `1 <= idx <= len(context_items)`, +otherwise assign `context_items[i % len(context_items)]["chunk_id"]` (round-robin by position in +the output list); validate `cognitive_level` against `_COGNITIVE_LEVELS`, defaulting to `"recall"`. +Stop once `target_count` targets are collected; if the model returned fewer, keep round-robin- +filling until the count is reached. Let `LlmDisabledError`/`StructuredGenerationError` from +`generate_json` propagate uncaught — Task 5's `run()` catches them at the call site. + +- [ ] **Step 4: Run to verify they pass** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -v` +Expected: PASS + +- [ ] **Step 5: Register `quiz_plan` in `qvac_router.LADDER_ROUTING`** + +In `app/services/qvac_router.py:33-42`, add `"quiz_plan": ("server", SERVER_MODEL),` to the +`LADDER_ROUTING` dict (alongside `"judge"`/`"content_gen"` — explicit registration rather than +relying on the unknown-task_type fallback). + +- [ ] **Step 6: Add a parametrize case to `test_qvac_router.py`** + +In `tests/unit/test_qvac_router.py`, add `("quiz_plan", qvac_router.SERVER_MODEL)` to the +`test_server_task_types_route_to_server` parametrize list. + +- [ ] **Step 7: Run the router tests to verify they pass** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_qvac_router.py -v` +Expected: PASS + +- [ ] **Step 8: Commit** + +```bash +git add app/services/quiz_harness.py tests/unit/test_quiz_harness.py app/services/qvac_router.py tests/unit/test_qvac_router.py +git commit -m "feat(rag): add plan stage to the quiz agent harness" +``` + +--- + +## Task 3: `quiz_harness.py` — generate stage + +One `generate_json` call per target, pinned to that target's single chunk so the model cannot pull +material from elsewhere in the evidence pack. Reuses the `content_gen` task type (same model as +today's quiz generation — no new routing entry needed). + +**Files:** +- Modify: `app/services/quiz_harness.py` +- Modify: `tests/unit/test_quiz_harness.py` + +**Interfaces:** +- Consumes: `PlanTarget` from Task 2. +- Produces: `async def _generate_one(target: PlanTarget, context_items: List[Dict[str, str]], *, + feedback: Optional[str] = None) -> Optional[Dict[str, Any]]` — returns a question dict with keys + `{"prompt", "options", "correct_key", "concept_tag", "difficulty", "chunk_id"}`, or `None` if + generation failed (`StructuredGenerationError`/`LlmDisabledError` caught here, not propagated — + the caller treats `None` as "this target needs repair or must be dropped"). Consumed by Task 5's + `run()` and by Task 4's critique tests (as critique input). + +- [ ] **Step 1: Write the failing tests** + +```python +_TARGET = {"chunk_id": "c1", "cognitive_level": "recall"} + +_QUESTION = { + "prompt": "What is a UTXO?", + "options": [ + {"key": "A", "text": "An unspent transaction output"}, + {"key": "B", "text": "A mining reward"}, + {"key": "C", "text": "A block header"}, + {"key": "D", "text": "A wallet address"}, + ], + "correct_key": "A", + "concept_tag": "utxo-model", + "difficulty": "beginner", +} + + +@pytest.mark.asyncio +async def test_generate_one_attaches_chunk_id(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = dict(_QUESTION) + question = await _generate_one(_TARGET, _CONTEXT) + + assert question["chunk_id"] == "c1" + assert question["prompt"] == "What is a UTXO?" + + +@pytest.mark.asyncio +async def test_generate_one_scopes_context_to_the_target_chunk(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = dict(_QUESTION) + await _generate_one(_TARGET, _CONTEXT) + + call_context = mock_gj.call_args.kwargs["context"] + assert call_context == [{"label": "p1", "text": "UTXO text", "chunk_id": "c1"}] + + +@pytest.mark.asyncio +async def test_generate_one_includes_feedback_in_prompt_when_repairing(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = dict(_QUESTION) + await _generate_one(_TARGET, _CONTEXT, feedback="rejected: distractors too easy") + + prompt_arg = mock_gj.call_args.args[0] + assert "rejected: distractors too easy" in prompt_arg + + +@pytest.mark.asyncio +async def test_generate_one_returns_none_on_structured_generation_error(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = StructuredGenerationError("bad schema") + question = await _generate_one(_TARGET, _CONTEXT) + + assert question is None +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -k generate_one -v` +Expected: FAIL — `ImportError: cannot import name '_generate_one'` + +- [ ] **Step 3: Implement `_generate_one` in `app/services/quiz_harness.py`** + +Add schema/prompt constants: +```python +GENERATE_PROMPT_VERSION = "v1" + +_GENERATE_SCHEMA: Dict[str, Any] = { + "type": "object", + "required": ["prompt", "options", "correct_key", "concept_tag", "difficulty"], + "properties": { + "prompt": {"type": "string"}, + "options": { + "type": "array", + "minItems": 4, + "maxItems": 4, + "items": { + "type": "object", + "required": ["key", "text"], + "properties": { + "key": {"type": "string", "enum": ["A", "B", "C", "D"]}, + "text": {"type": "string"}, + }, + }, + }, + "correct_key": {"type": "string", "enum": ["A", "B", "C", "D"]}, + "concept_tag": {"type": "string"}, + "difficulty": {"type": "string", "enum": ["beginner", "intermediate", "advanced"]}, + }, +} + +_GENERATE_SYSTEM = ( + "You are an expert educator at BitPolito Academy creating one assessment question " + "from a single source passage.\n" + "RULES:\n" + "1. The question must test the cognitive level given in the task (recall / application " + "/ analysis), based ONLY on the provided passage.\n" + "2. All 4 options (A-D) must be plausible — wrong options should be conceptually close " + "but demonstrably wrong from the passage.\n" + "3. Exactly one option must be correct and directly supported by the passage.\n" + "4. Use exact technical terminology from the passage.\n" + "5. concept_tag: a short kebab-case slug for the concept tested.\n" + "6. difficulty: beginner / intermediate / advanced." +) + + +async def _generate_one( + target: PlanTarget, + context_items: List[Dict[str, str]], + *, + feedback: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + ... +``` + +Implementer writes the body: find the single entry in `context_items` whose `"chunk_id"` matches +`target["chunk_id"]` and pass it as `context=[that_item]` (a list of exactly one); build the prompt +from `target["cognitive_level"]` and, when `feedback` is given, append it verbatim (e.g. `f"Note: +the previous attempt was rejected — {feedback}"`); call `generate_json(prompt, _GENERATE_SCHEMA, +context=[item], system_prompt=_GENERATE_SYSTEM, generation_params={"temp": 0.3}, +task_type="content_gen")`; on success, return the result dict with `"chunk_id": +target["chunk_id"]` merged in; catch `StructuredGenerationError` and return `None` (do not catch +`LlmDisabledError` — let it propagate, matching the plan stage's contract, since a disabled LLM +should abort the run, not silently drop one target). + +- [ ] **Step 4: Run to verify they pass** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -k generate_one -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add app/services/quiz_harness.py tests/unit/test_quiz_harness.py +git commit -m "feat(rag): add generate stage to the quiz agent harness" +``` + +--- + +## Task 4: `quiz_harness.py` — critique stage + +One call per generated question, checking it against the five criteria from issue #134. Fails +closed on an unavailable critic (treated as a failure, not a pass). + +**Files:** +- Modify: `app/services/quiz_harness.py` +- Modify: `tests/unit/test_quiz_harness.py` +- Modify: `app/services/qvac_router.py:33-42` +- Modify: `tests/unit/test_qvac_router.py` + +**Interfaces:** +- Consumes: a question dict from Task 3's `_generate_one`. +- Produces: `CritiqueVerdict = Dict[str, Any]` with keys `{"passed": bool, "failed_criteria": + List[str]}`; `async def _critique_one(question: Dict[str, Any], source_text: str, + accepted_questions: List[Dict[str, Any]]) -> CritiqueVerdict`. Never raises — fails closed. + Consumed by Task 5's `run()`. + +- [ ] **Step 1: Write the failing tests** + +```python +@pytest.mark.asyncio +async def test_critique_one_passes_when_all_criteria_met(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = { + "entailed": True, "distractors_plausible": True, "single_defensible": True, + "requires_course_material": True, "duplicate": False, + } + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict == {"passed": True, "failed_criteria": []} + + +@pytest.mark.asyncio +async def test_critique_one_reports_each_failed_criterion(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = { + "entailed": False, "distractors_plausible": True, "single_defensible": True, + "requires_course_material": True, "duplicate": True, + } + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict["passed"] is False + assert set(verdict["failed_criteria"]) == {"entailed", "duplicate"} + + +@pytest.mark.asyncio +async def test_critique_one_handles_missing_fields_defensively(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"entailed": True} # worker returned a partial object + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict["passed"] is False + assert "distractors_plausible" in verdict["failed_criteria"] + + +@pytest.mark.asyncio +async def test_critique_one_passes_accepted_questions_as_duplicate_context(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = { + "entailed": True, "distractors_plausible": True, "single_defensible": True, + "requires_course_material": True, "duplicate": False, + } + await _critique_one(_QUESTION, "UTXO text", [{"prompt": "Earlier question?"}]) + + prompt_arg = mock_gj.call_args.args[0] + assert "Earlier question?" in prompt_arg + + +@pytest.mark.asyncio +async def test_critique_one_fails_closed_on_llm_disabled(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = LlmDisabledError("disabled") + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict["passed"] is False + assert verdict["failed_criteria"] == ["critic unavailable"] + + +@pytest.mark.asyncio +async def test_critique_one_fails_closed_on_structured_generation_error(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = StructuredGenerationError("bad schema") + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict["passed"] is False + assert verdict["failed_criteria"] == ["critic unavailable"] +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -k critique_one -v` +Expected: FAIL — `ImportError: cannot import name '_critique_one'` + +- [ ] **Step 3: Implement `_critique_one` in `app/services/quiz_harness.py`** + +```python +CRITIQUE_PROMPT_VERSION = "v1" + +_CRITIQUE_CRITERIA = ( + "entailed", "distractors_plausible", "single_defensible", "requires_course_material", +) +# "duplicate" is tracked separately — it's the one criterion where True is the failure, not False. + +_CRITIQUE_SCHEMA: Dict[str, Any] = { + "type": "object", + "required": ["entailed", "distractors_plausible", "single_defensible", + "requires_course_material", "duplicate"], + "properties": { + "entailed": {"type": "boolean"}, + "distractors_plausible": {"type": "boolean"}, + "single_defensible": {"type": "boolean"}, + "requires_course_material": {"type": "boolean"}, + "duplicate": {"type": "boolean"}, + }, +} + +_CRITIQUE_SYSTEM = ( + "You are a critic judging one multiple-choice question against its source passage and " + "the other questions already accepted into this quiz.\n" + "entailed: the correct answer is directly supported by the passage.\n" + "distractors_plausible: every wrong option is plausible but demonstrably wrong from the " + "passage — not absurd, not another correct answer.\n" + "single_defensible: exactly one option is correct; no other option could also be " + "defended as correct.\n" + "requires_course_material: the question cannot be answered from general knowledge alone.\n" + "duplicate: true if this question tests the same fact as one of the other questions shown." +) + + +async def _critique_one( + question: Dict[str, Any], source_text: str, accepted_questions: List[Dict[str, Any]], +) -> Dict[str, Any]: + ... +``` + +Implementer writes the body: build the prompt from `question["prompt"]`/`question["options"]`/ +`question["correct_key"]`, the `source_text`, and a listing of `accepted_questions`' `"prompt"` +values (empty-list-safe); call `generate_json(prompt, _CRITIQUE_SCHEMA, context=[{"label": +"source", "text": source_text}], system_prompt=_CRITIQUE_SYSTEM, generation_params={"temp": 0.1}, +task_type="quiz_critique")`; read each of `_CRITIQUE_CRITERIA` via `result.get(name, False)` +(missing ⇒ failed, matching Step 1's defensive test) and `duplicate = result.get("duplicate", +True)` (missing ⇒ treated as a duplicate, the safer default); `failed_criteria` is every criterion +in `_CRITIQUE_CRITERIA` whose value is falsy, plus `"duplicate"` if `duplicate` is truthy; `passed += not failed_criteria`. On `(LlmDisabledError, StructuredGenerationError)`, return `{"passed": +False, "failed_criteria": ["critic unavailable"]}` — fail closed, mirroring +`lesson_service._judge_groundedness`'s pattern in `process_lesson()`. + +- [ ] **Step 4: Run to verify they pass** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -k critique_one -v` +Expected: PASS + +- [ ] **Step 5: Register `quiz_critique` in `qvac_router.LADDER_ROUTING` and add its router test** + +Same as Task 2 Step 5-7: add `"quiz_critique": ("server", SERVER_MODEL),` to `LADDER_ROUTING`, add +`("quiz_critique", qvac_router.SERVER_MODEL)` to `test_server_task_types_route_to_server`'s +parametrize list, run `tests/unit/test_qvac_router.py` to confirm PASS. + +- [ ] **Step 6: Commit** + +```bash +git add app/services/quiz_harness.py tests/unit/test_quiz_harness.py app/services/qvac_router.py tests/unit/test_qvac_router.py +git commit -m "feat(rag): add critique stage to the quiz agent harness" +``` + +--- + +## Task 5: `quiz_harness.py` — repair loop and `run()` orchestration + +Wires plan → generate → critique together, with exactly one repair attempt per rejected question, +and produces the structured trace. + +**Files:** +- Modify: `app/services/quiz_harness.py` +- Modify: `tests/unit/test_quiz_harness.py` + +**Interfaces:** +- Consumes: `_plan_targets` (Task 2), `_generate_one` (Task 3), `_critique_one` (Task 4). +- Produces: `DEFAULT_TARGET_COUNT: int = 4`; `HarnessResult = Dict[str, Any]` with keys + `{"questions": List[Dict[str, Any]], "trace": Dict[str, Any]}`; `trace` has keys + `{"targets_planned": int, "generated": int, "rejected": List[Dict[str, Any]], "repaired": int, + "final_count": int, "calls_used": int, "calls_budget": int}`; `async def run(topic: str, + context_items: List[Dict[str, str]], target_count: int = DEFAULT_TARGET_COUNT) -> + HarnessResult`. Never raises for `StructuredGenerationError`/per-target `LlmDisabledError`; + re-raises `LlmDisabledError` only if it happens at the plan stage (nothing has been produced yet, + so there is nothing to degrade to — matches `generate_quiz_questions`'s existing contract of + letting `LlmDisabledError` propagate, see `lesson_service.process_lesson`'s `except + LlmDisabledError: raise`). Consumed by Task 7's `quiz_generation.generate_quiz_questions`. + +- [ ] **Step 1: Write the failing tests** + +```python +@pytest.mark.asyncio +async def test_run_all_pass_returns_every_question_with_no_repairs(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] * 2 + mock_gen.return_value = dict(_QUESTION) + mock_crit.return_value = {"passed": True, "failed_criteria": []} + + result = await run("UTXO model", _CONTEXT, target_count=2) + + assert len(result["questions"]) == 2 + assert result["trace"]["repaired"] == 0 + assert result["trace"]["final_count"] == 2 + assert result["trace"]["calls_used"] == 1 + 2 + 2 # plan + 2 generate + 2 critique + + +@pytest.mark.asyncio +async def test_run_rejected_then_repaired_is_recorded_and_kept(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] + mock_gen.return_value = dict(_QUESTION) + mock_crit.side_effect = [ + {"passed": False, "failed_criteria": ["entailed"]}, + {"passed": True, "failed_criteria": []}, + ] + + result = await run("UTXO model", _CONTEXT, target_count=1) + + assert result["trace"]["repaired"] == 1 + assert len(result["questions"]) == 1 + assert result["trace"]["calls_used"] == 1 + 2 + 2 # plan + (generate+critique) x2 + + +@pytest.mark.asyncio +async def test_run_retry_limit_reached_drops_the_question_with_reasons(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] + mock_gen.return_value = dict(_QUESTION) + mock_crit.return_value = {"passed": False, "failed_criteria": ["entailed"]} + + result = await run("UTXO model", _CONTEXT, target_count=1) + + assert result["questions"] == [] + assert result["trace"]["final_count"] == 0 + assert len(result["trace"]["rejected"]) == 1 + assert result["trace"]["rejected"][0]["reasons"] == ["entailed"] + # Exactly one repair attempt — no third generate/critique pass. + assert result["trace"]["calls_used"] == 1 + 2 + 2 + + +@pytest.mark.asyncio +async def test_run_llm_disabled_at_plan_stage_returns_no_questions(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan: + mock_plan.side_effect = LlmDisabledError("disabled") + + result = await run("UTXO model", _CONTEXT, target_count=2) + + assert result["questions"] == [] + assert result["trace"]["final_count"] == 0 + + +@pytest.mark.asyncio +async def test_run_llm_disabled_mid_generate_drops_that_target_and_continues(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] * 2 + mock_gen.side_effect = [dict(_QUESTION), None] # second target's generation failed + mock_crit.return_value = {"passed": True, "failed_criteria": []} + + result = await run("UTXO model", _CONTEXT, target_count=2) + + assert len(result["questions"]) == 1 + assert len(result["trace"]["rejected"]) == 1 +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -k "test_run" -v` +Expected: FAIL — `ImportError: cannot import name 'run'` + +- [ ] **Step 3: Implement `run()` in `app/services/quiz_harness.py`** + +```python +from app.services.qvac_structured import LlmDisabledError, StructuredGenerationError + +DEFAULT_TARGET_COUNT = 4 + + +async def run( + topic: str, context_items: List[Dict[str, str]], target_count: int = DEFAULT_TARGET_COUNT, +) -> Dict[str, Any]: + ... +``` + +Implementer writes the body following the design doc's orchestration exactly: call +`_plan_targets`, letting `LlmDisabledError`/`StructuredGenerationError` from it end the run +immediately with `{"questions": [], "trace": {..., "rejected": [{"stage": "plan", "reason": +str(exc)}]}}`; otherwise iterate the returned targets, for each one calling `_generate_one` then (if +it returned a question) `_critique_one` against the already-`accepted` list; on a failed critique +or a `None` generation result, attempt exactly one repair (`_generate_one(target, context_items, +feedback=...)` built from `failed_criteria`, then `_critique_one` again if the repair generated a +question); append to `accepted` on any pass, otherwise to `trace["rejected"]` with a +`question_preview` (the rejected question's `"prompt"` if one exists, else `None`) and `"reasons"`; +increment `trace["calls_used"]` once per `_plan_targets`/`_generate_one`/`_critique_one` call +actually made (a `None` from a previous repair generate must not trigger an extra critique call — +track this explicitly so the `calls_used == 1 + 2 + 2` assertions in Step 1 hold); set +`trace["calls_budget"] = 4 * target_count + 1` and `trace["final_count"] = len(accepted)` before +returning. `topic` is accepted for interface symmetry with `generate_quiz_questions` and is not +otherwise used inside `run()` yet — the implementer may thread it into `_generate_one`'s prompt +later if useful, but no test here requires it. + +- [ ] **Step 4: Run to verify they pass** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -v` +Expected: PASS (every test in the file, not just this task's) + +- [ ] **Step 5: Commit** + +```bash +git add app/services/quiz_harness.py tests/unit/test_quiz_harness.py +git commit -m "feat(rag): add bounded repair loop and run() orchestrator to the quiz harness" +``` + +--- + +## Task 6: Prompt registry wiring + +Registers the harness's new prompts so `get_prompt`/`list_prompts` can inspect them, and retires +`quiz_generation.py`'s now-dead multi-question schema/prompt constants (superseded by Task 3's +single-question ones). + +**Files:** +- Modify: `app/prompts/registry.py:29-52` +- Modify: `tests/unit/test_prompt_registry.py` + +(`app/services/quiz_generation.py`'s now-dead `QUIZ_PROMPT_VERSION`/`QUIZ_SCHEMA`/`QUIZ_SYSTEM` +are deleted in Task 7, once their last caller is rewritten — see that task's rationale.) + +**Interfaces:** +- Consumes: `quiz_harness.PLAN_PROMPT_VERSION`, `quiz_harness._PLAN_SYSTEM`, + `quiz_harness.GENERATE_PROMPT_VERSION`, `quiz_harness._GENERATE_SYSTEM`, + `quiz_harness.CRITIQUE_PROMPT_VERSION`, `quiz_harness._CRITIQUE_SYSTEM` (all from Tasks 2-4). +- Produces: registry entries `"quiz_plan"`, `"quiz_critique"`; `"quiz_gen"` now mirrors + `quiz_harness.GENERATE_PROMPT_VERSION`/`_GENERATE_SYSTEM` instead of the deleted + `quiz_generation.QUIZ_PROMPT_VERSION`/`QUIZ_SYSTEM`. + +- [ ] **Step 1: Write the failing test** + +In `tests/unit/test_prompt_registry.py`, add `"quiz_plan"` and `"quiz_critique"` to +`EXPECTED_NAMES`, and add: +```python +def test_quiz_plan_version_matches_quiz_harness_constant(): + from app.services import quiz_harness + assert get_prompt("quiz_plan").version == quiz_harness.PLAN_PROMPT_VERSION + assert get_prompt("quiz_plan").system == quiz_harness._PLAN_SYSTEM + + +def test_quiz_critique_version_matches_quiz_harness_constant(): + from app.services import quiz_harness + assert get_prompt("quiz_critique").version == quiz_harness.CRITIQUE_PROMPT_VERSION + assert get_prompt("quiz_critique").system == quiz_harness._CRITIQUE_SYSTEM +``` +Update `test_quiz_gen_version_matches_quiz_generation_constant` to +`test_quiz_gen_version_matches_quiz_harness_constant`, asserting against +`quiz_harness.GENERATE_PROMPT_VERSION`/`_GENERATE_SYSTEM` instead of `quiz_generation`'s deleted +constants. + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_prompt_registry.py -v` +Expected: FAIL — `AssertionError` (names mismatch) and `AttributeError` on the renamed test. + +- [ ] **Step 3: Update `_build_registry()` in `app/prompts/registry.py`** + +Change the import line to `from app.services import lesson_service, outline_service, +quiz_generation, quiz_harness, study_service` (keep `quiz_generation` — still used by +`persist_quiz` elsewhere, just no longer for the `quiz_gen` prompt). Replace the `"quiz_gen"` entry: +```python + "quiz_gen": Prompt( + "quiz_gen", quiz_harness.GENERATE_PROMPT_VERSION, quiz_harness._GENERATE_SYSTEM, + ), + "quiz_plan": Prompt( + "quiz_plan", quiz_harness.PLAN_PROMPT_VERSION, quiz_harness._PLAN_SYSTEM, + ), + "quiz_critique": Prompt( + "quiz_critique", quiz_harness.CRITIQUE_PROMPT_VERSION, quiz_harness._CRITIQUE_SYSTEM, + ), +``` + +Leave `quiz_generation.py`'s `QUIZ_PROMPT_VERSION`/`QUIZ_SCHEMA`/`QUIZ_SYSTEM` in place for now — +`generate_quiz_questions` still calls `generate_json` with them until Task 7 rewires it. Deleting +them here would leave the module referencing nothing until Task 7 lands, breaking the "tests pass +after every task" invariant. Task 7 Step 3 deletes them once their last caller is rewritten. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_prompt_registry.py -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add app/prompts/registry.py tests/unit/test_prompt_registry.py +git commit -m "feat(rag): register quiz harness plan/critique prompts, repoint quiz_gen" +``` + +--- + +## Task 7: Wire the harness into `generate_quiz_questions` and the API response + +The last task: `quiz_generation.generate_quiz_questions` delegates to `quiz_harness.run` instead of +calling `generate_json` directly; `quizzes_api.generate_quiz`'s response carries the new `trace` +field. + +**Files:** +- Modify: `app/services/quiz_generation.py:83-108` (`generate_quiz_questions`) +- Modify: `app/api/quizzes_api.py:141-205` (`generate_quiz`) +- Modify: `tests/unit/test_quiz_generation.py:62-85` +- Modify: `tests/integration/test_quizzes_certificates_api.py` + +**Interfaces:** +- Consumes: `quiz_harness.run` (Task 5). +- Produces: `generate_quiz_questions(topic, context_items) -> Optional[Dict[str, Any]]` unchanged + in signature; now returns `{"questions": [...], "trace": {...}}` or `None` (when `run()` produced + no questions) — same external contract as before, callers that only read `.get("questions")` + (`lesson_service._persist_quiz`) are unaffected. `POST /courses/{course_id}/quizzes/generate`'s + JSON response gains a `"trace"` key alongside the existing quiz fields. + +- [ ] **Step 1: Write the failing tests** + +Replace the two tests in `tests/unit/test_quiz_generation.py` (lines 62-85): +```python +@pytest.mark.asyncio +async def test_generate_quiz_questions_delegates_to_harness(): + with patch("app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock) as mock_run: + mock_run.return_value = {"questions": _QUESTIONS, "trace": {"final_count": 1}} + result = await generate_quiz_questions("UTXO model", [{"label": "p1", "text": "...", "chunk_id": "c1"}]) + + assert result == {"questions": _QUESTIONS, "trace": {"final_count": 1}} + mock_run.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_generate_quiz_questions_returns_none_when_harness_produces_nothing(): + with patch("app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock) as mock_run: + mock_run.return_value = {"questions": [], "trace": {"final_count": 0}} + result = await generate_quiz_questions("UTXO model", [{"label": "p1", "text": "...", "chunk_id": "c1"}]) + + assert result is None +``` + +In `tests/integration/test_quizzes_certificates_api.py`, change +`test_generate_quiz_persists_to_db`'s patch target from +`"app.services.quiz_generation.generate_json"` to `"app.services.quiz_generation.quiz_harness.run"`, +with `mock_gj.return_value = {"questions": _QUESTIONS, "trace": {"final_count": 1}}`, and add: +```python + assert body["trace"] == {"final_count": 1} +``` +Update Task 1's `test_generate_quiz_uses_evidence_pack_chunk_id` the same way: it now asserts the +`context` kwarg on `quiz_harness.run`'s call (its second positional/keyword argument, +`context_items`) rather than on `generate_json`. + +- [ ] **Step 2: Run to verify they fail** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_generation.py tests/integration/test_quizzes_certificates_api.py -v` +Expected: FAIL — `generate_quiz_questions` still calls `generate_json` directly; no `trace` key in +the response. + +- [ ] **Step 3: Rewrite `generate_quiz_questions` in `app/services/quiz_generation.py`** + +```python +from app.services import quiz_harness + +_DEFAULT_TARGET_COUNT = quiz_harness.DEFAULT_TARGET_COUNT + + +async def generate_quiz_questions( + topic: str, + context_items: List[Dict[str, str]], +) -> Optional[Dict[str, Any]]: + """Run the plan/generate/critique/repair harness to produce MCQ quiz data. + + Returns None (rather than raising) when the harness produces zero + questions — quiz generation is treated as best-effort by all current + callers. + """ + result = await quiz_harness.run(topic, context_items, target_count=_DEFAULT_TARGET_COUNT) + if not result["questions"]: + return None + return result +``` +Delete `QUIZ_PROMPT_VERSION`, `QUIZ_SCHEMA`, and `QUIZ_SYSTEM` (the old lines 23-80) — nothing +calls them after this rewrite (Task 6 already repointed the `"quiz_gen"` registry entry away from +them). Remove the now-unused `StructuredGenerationError`/`generate_json` imports from this file if +nothing else in it still uses them (check `persist_quiz` — it doesn't). + +- [ ] **Step 4: Add the `trace` field to the API response in `app/api/quizzes_api.py`** + +In `generate_quiz` (around line 196-205), capture the trace before building the response: +```python + response_body = _to_student_detail(quiz, questions, db).model_dump() + response_body["trace"] = quiz_data.get("trace") + return JSONResponse(status_code=status.HTTP_201_CREATED, content=response_body) +``` + +- [ ] **Step 5: Run to verify they pass** + +Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_generation.py tests/integration/test_quizzes_certificates_api.py tests/unit/test_quiz_harness.py tests/unit/test_prompt_registry.py tests/unit/test_qvac_router.py tests/unit/test_lesson_service.py -v` +Expected: all PASS. + +- [ ] **Step 6: Run the full backend suite and mypy** + +Run: `cd services/ai && uv run --no-sync pytest && uv run --no-sync mypy app` +Expected: PASS (no regressions elsewhere — `lesson_service._generate_quiz_data`/`_persist_quiz` +are unaffected since they only read `.get("questions")` off the same dict shape). + +- [ ] **Step 7: Commit** + +```bash +git add app/services/quiz_generation.py app/api/quizzes_api.py tests/unit/test_quiz_generation.py tests/integration/test_quizzes_certificates_api.py +git commit -m "feat(rag): wire the quiz agent harness into quiz generation and the API response" +``` + +--- + +## After merge (not a coded task) + +Per issue #134's acceptance criteria: run a manual review of 20 generated questions on real +Bitcoin course material against a live QVAC service, and record pass/fail judgments in the issue +before closing it. This requires a running QVAC worker and is not something these tasks can verify. diff --git a/docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md b/docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md new file mode 100644 index 0000000..33bbfb2 --- /dev/null +++ b/docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md @@ -0,0 +1,257 @@ +# Quiz agent harness — design + +Issue: [#134](https://github.com/BitPolito/bitcoin-academy/issues/134) — Agent harness for quiz +generation (plan → generate → critique → repair). + +## Intent and why + +Quiz generation today (`quiz_generation.generate_quiz_questions`) is a single schema-validated +`generate_json` call: it guarantees well-formed questions, not *good* ones. Observed failure modes +(from the issue): guessable distractors, general-knowledge questions, multiple defensible answers, +coverage clustered on one passage, answers not actually grounded in the cited chunk. + +This design replaces that single call, for both callers (course builder per-lesson quizzes and the +study-page ad-hoc quiz action), with a bounded four-stage loop: plan which passages/cognitive +levels to test, generate one question per target, critique each against explicit criteria, repair +(regenerate + re-critique) questions that fail once, then persist through the existing +`persist_quiz` path unchanged. + +## Goals + +- Each returned question passed the critic, or the run honestly reports why it could not reach the + target count. +- Bounded, predictable LLM call count per run — no path that can loop unboundedly. +- Deliberate coverage: targets are planned from the evidence pack rather than incidental to however + the single old prompt happened to sample it. +- A structured trace (targets planned, generated, rejected with reasons, repaired, final count) + returned to the caller, so quality issues are debuggable without re-reading logs. +- Degrade honestbly when generation is unavailable: fewer questions or a clear error, never + malformed output (matches existing `generate_quiz_questions` contract of returning `None`/fewer + items rather than raising on the caller). + +## Non-goals / anti-goals + +- Not touching `persist_quiz` — the loop produces the same `questions: List[Dict]` shape + `generate_quiz_questions` returns today; persistence is unchanged. +- Not building a background job / `GenerationRun` row for quizzes. Confirmed with the repo owner: + stays synchronous, single in-request call, same shape as today's + `POST /courses/{course_id}/quizzes/generate`. +- Not tier-scaling the call budget. `content_gen`/`judge` task types already route to the **server** + model via `qvac_router.LADDER_ROUTING` regardless of the caller's local hardware tier — confirmed + by reading `qvac_router.py`. A tier-scaled cap would be theater: the calls all hit the same shared + server model regardless of the caller's tier. The cap is instead sized to the question count. +- Not spaced repetition or the "open questions" companion issue — explicitly out of scope per #134. +- Not a frontend debug-page change. Reviewed and confirmed: the existing `debug/page.tsx`'s `quiz` + dropdown option only exercises evidence-pack retrieval, never calls `POST /quizzes/generate` — + there is no existing "quiz generation" flow on that page to extend, so adding a trace panel there + would be new UI, not an extension. Decided: the `trace` field is carried in the API response only + (see "Trace surface" below); the debug-page control is left to a follow-up issue. + +## Context-item shape fix (prerequisite, found during review) + +The design assumed both callers' `context_items` already carry a resolvable `chunk_id` the plan +stage can select from. True for the course-builder path — `lesson_service._load_context` already +sets `label` to the chunk id (`context_items.append({"label": cid, "text": rows[cid].text})`) — +but **false for the study-page path**: `quizzes_api.py` builds `context_items` from +`result.citations` (`CitationOut`, only `label`/`doc_id`/`snippet`), which never carries a chunk id. +The real id exists on `result.evidence_pack.chunks[i].chunk_id` (`EvidenceChunk`, already populated +for the `RETRIEVE` action per `study_service.dispatch`) — data already fetched, no new retrieval +call needed. + +Fix, as a prerequisite step before the harness lands: both callers add an explicit `chunk_id` key +to each `context_items` dict (confirmed safe — `generate-json.js`'s `buildContextBlock` only reads +`label`/`text`, extra keys are ignored by the worker): + +- `lesson_service._load_context`: `context_items.append({"label": cid, "text": rows[cid].text, + "chunk_id": cid})` (redundant with `label` today, but makes the key explicit and uniform across + callers rather than relying on the course-builder path's `label`-doubles-as-chunk_id convention). +- `quizzes_api.generate_quiz`: build from `result.evidence_pack.chunks` instead of + `result.citations`: `context_items = [{"label": c.anchor.doc_name, "text": c.text, "chunk_id": + c.chunk_id} for c in result.evidence_pack.chunks]`. `result.evidence_pack` can be `None` + defensively (treat as "no source material", same error path as today's empty-citations check) — + confirm this doesn't happen for a successful `RETRIEVE` dispatch, but guard it anyway since + `DispatchResult.evidence_pack` is typed `Optional`. + +The harness's `_plan_targets`/`_generate_one` read `item["chunk_id"]` uniformly across both callers +after this fix. + +## Constraints (from the issue, confirmed as hard requirements) + +- Every returned question carries a resolvable `chunk_id`. +- Hard cap on LLM calls — no unbounded loop. +- Quality criteria the critic must check, verbatim from the issue: + 1. The answer is entailed by the cited source chunk. + 2. Every distractor is plausible but demonstrably wrong from the source. + 3. Exactly one option is defensible. + 4. The question is not answerable without the course material. + 5. The question does not duplicate another in the set. +- Unit tests with a mocked `generate_json` covering: all pass, some rejected then repaired, retry + limit reached, LLM disabled. +- Manual review of 20 generated questions on real Bitcoin material, results recorded in the issue — + this is a post-implementation activity for whoever runs it against a live QVAC service, not + something the code can satisfy; call it out as a follow-up step in the PR, not a coded check. + +## Call budget (decided, not tier-scaled) + +For a target of `N` questions: + +- **Plan**: 1 call — selects `N` `(chunk_id, cognitive_level)` targets from the evidence pack + already available to the caller (reuses the `context_items` the existing callers already fetch + via `study_service.dispatch(..., action=StudyAction.RETRIEVE)`; no new retrieval call). +- **Generate**: up to `N` calls — one `generate_json` call per target. +- **Critique**: up to `N` calls — one call per generated question (batching all N into one critique + call was considered and rejected: a single malformed critique response would then invalidate + every verdict at once, and the per-lesson judge precedent in `lesson_service._judge_groundedness` + is also one call per judged unit). +- **Repair**: up to `R` additional (generate + critique) pairs, where `R` = number of questions that + failed critique on the first pass, each regenerated and recritiqued **once** (confirmed: 1 repair + round, not more). + +Worst case total: `1 + N + N + 2N = 4N + 1` calls. For the existing default of 3-4 questions +(`QUIZ_SCHEMA` in `quiz_generation.py` currently asks the model for 3-4 at once), that's at most 17 +calls — each still capped by the existing 300s transport timeout, same ceiling `generate_json` +already imposes per call today. This is the hard cap: the loop tracks calls made and refuses to +start a repair pass once the budget is exhausted, returning what it has with a reported reason. + +## The four stages + +### 1. Plan + +New function, e.g. `_plan_targets(context_items, target_count) -> List[PlanTarget]` where +`PlanTarget = {chunk_id, cognitive_level}` (`cognitive_level` ∈ `recall | application | analysis`). +One `generate_json` call with a new JSON schema (list of `{chunk_id, cognitive_level}`, length +`target_count`) and a new system prompt instructing the model to spread targets across distinct +chunks and levels. `chunk_id` values are constrained to the ones present in `context_items` (reject +and fall back to round-robin assignment across available chunks if the model invents one — this is +a correctness guard, not an LLM call). + +### 2. Generate + +Reuses the existing generation call shape but one target at a time instead of one call for the +whole batch: `_generate_one(target, context_items) -> Dict | None`. New schema: a single question +object (same shape as one entry in today's `QUIZ_SCHEMA.items`, i.e. `question, options (A-D), +correct_answer, concept_tag, difficulty, chunk_id`), with the prompt pinned to `target.chunk_id`'s +passage and `target.cognitive_level`. On `StructuredGenerationError`, this target is marked failed +at generation (counts against the budget, not a separate "generation failed" dead end — it can +still be retried in the repair pass like a critique failure). + +### 3. Critique + +New function `_critique(question, source_chunk, other_questions) -> CritiqueVerdict` where +`CritiqueVerdict = {passed: bool, failed_criteria: List[str]}`, one call per question, schema +listing the 5 criteria above as named boolean fields (mirrors `_JUDGE_SCHEMA`'s shape in +`lesson_service.py`) so failures are attributable to a specific criterion in the trace. Duplicate +detection (criterion 5) is checked against the other questions already accepted in this run, passed +in as context — this is the one criterion that needs same-run state, so critique must run after +each generation rather than once over the whole batch in parallel, keeping the loop sequential. +On `(LlmDisabledError, StructuredGenerationError)`, **fail closed** like +`_judge_groundedness` does — treat as `passed=False` with reason `"critic unavailable"`, do not +silently accept. + +### 4. Repair + +For each question that failed critique, exactly one retry: regenerate via the same `_generate_one` +with the failed criteria appended to the prompt as explicit feedback (e.g. "the previous attempt +was rejected because: ..."), then critique again. Whatever the second verdict is, it is final — +pass → kept, fail → dropped and recorded in the trace with its reasons. + +### 5. Persist + +Unchanged: the harness's output is the same `questions: List[Dict]` shape +`generate_quiz_questions` already returns; `persist_quiz` is called by the same callers exactly as +today. + +## Public API + +New module: `services/ai/app/services/quiz_harness.py`. Keeps `quiz_generation.py` as the +persistence + public-entry module (so existing imports in `lesson_service.py` and `quizzes_api.py` +need minimal changes): + +```python +async def generate_quiz_questions( + topic: str, context_items: List[Dict[str, str]] +) -> Optional[Dict[str, Any]]: + ... +``` + +stays the public entry point in `quiz_generation.py`, but its body now delegates to the harness: + +```python +async def generate_quiz_questions(topic, context_items): + result = await quiz_harness.run(topic, context_items, target_count=_DEFAULT_TARGET_COUNT) + if not result.questions: + return None + return {"questions": result.questions, "trace": result.trace} +``` + +`quiz_harness.run(...) -> HarnessResult` where `HarnessResult = {questions: List[Dict], trace: +HarnessTrace}`. `HarnessTrace` is a plain dict (not persisted, not a DB model — see Non-goals): +`{targets_planned: int, generated: int, rejected: [{question_preview, reasons}], repaired: int, +final_count: int, calls_used: int, calls_budget: int}`. + +### API response shape change + +`POST /courses/{course_id}/quizzes/generate` (`quizzes_api.py`) currently returns the persisted +quiz. Add a `trace` field alongside it (additive, not breaking) sourced from the dict +`generate_quiz_questions` now returns. `lesson_service.py`'s course-builder path can log the trace +(e.g. via `_update_run`'s stage/error fields or a log line) rather than surface it in a UI, since the +course-builder flow has no per-call debug surface today — only the study-page path gets the debug +panel (see below). + +## Task-type routing + +Add to `qvac_router.LADDER_ROUTING`: `"quiz_plan": ("server", SERVER_MODEL)`, `"quiz_critique": +("server", SERVER_MODEL)`. Generation calls reuse the existing `"content_gen"` task type (no new +entry needed — same model, same routing). This is a deliberate registration (visible in review) even +though `resolve()` would default unknown task types to `server` anyway — explicit entries document +intent, matching how `"judge"` and `"content_gen"` are already registered rather than left to the +fallback. + +## Prompt registry + +Add `_PLAN_SYSTEM`/`PLAN_PROMPT_VERSION` and `_CRITIQUE_SYSTEM`/`CRITIQUE_PROMPT_VERSION` constants +in `quiz_harness.py`, registered in `prompts/registry.py::_build_registry()` following the existing +`"judge"` entry pattern (import `quiz_harness`, add `"quiz_plan"` and `"quiz_critique"` entries). +Generation keeps using `quiz_generation`'s existing `QUIZ_PROMPT_VERSION`/`QUIZ_SYSTEM` (already +registered as `"quiz_gen"`), adjusted to operate on one target instead of a batch. + +## Debug surface + +Decided (revised after review — see Non-goals): no frontend change in this PR. The `trace` field +rides in the existing `POST /quizzes/generate` JSON response body, inspectable via the API directly +(curl, API client, or browser devtools network tab) without any new backend route. A follow-up +issue should add an actual trigger-and-display control to `debug/page.tsx`, since today's `quiz` +dropdown option there only runs evidence-pack retrieval, not real generation — building that control +is new UI work, not an extension of something that already calls the generate endpoint. + +## Testing + +Follow the existing mocking convention (patch `generate_json` where imported into the calling +module, i.e. `app.services.quiz_harness.generate_json`, with `AsyncMock`): + +- All pass: each stage's mock returns a passing verdict; assert `calls_used` matches the + no-repair-needed budget (`1 + 2N`) and all `N` questions returned. +- Some rejected then repaired: first critique fails for a subset, second (repair) critique passes; + assert those questions appear in `trace.repaired` and the final set, with `calls_used` reflecting + the extra repair calls. +- Retry limit reached: repair critique also fails; assert the question is dropped, appears in + `trace.rejected` with its final reasons, and the loop does not attempt a third pass. +- LLM disabled: `generate_json` raises `LlmDisabledError` at the plan stage; assert the harness + returns no questions with a clear reason rather than raising, matching + `generate_quiz_questions`'s existing `None`-on-failure contract. +- Persistence path (`persist_quiz`) is untested here since it is unchanged — existing + `test_quiz_generation.py` coverage stands. + +## What's left to the builder + +- Exact wording of the plan/critique system prompts (content, not structure — structure is fixed + above). +- Exact JSON schema field names/ordering for `PlanTarget`, the single-question generation schema, + and `CritiqueVerdict` (shape is specified; naming is the builder's call, following existing + snake_case conventions in `quiz_generation.QUIZ_SCHEMA`/`lesson_service._JUDGE_SCHEMA`). +- Whether `_DEFAULT_TARGET_COUNT` stays at today's 3-4 (as a range) or becomes a fixed single number + now that generation is per-target — builder's call, defaulting to keeping the existing range + (e.g. target_count = 4, allow the plan stage to return 3 if the evidence pack has too few distinct + chunks to avoid forcing duplicate-chunk targets). +- Whether `quiz_harness.py` or `quiz_generation.py` owns the `HarnessTrace`/`PlanTarget` type + definitions — builder's call; suggested: define them in `quiz_harness.py` since it owns the loop. From c77a4a2bfb09245d5890c43d596f0d575396b12a Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 21:29:04 +0100 Subject: [PATCH 02/12] fix(rag): carry chunk_id through quiz context_items for both callers The study-page caller built context_items from result.citations, which has no chunk id anywhere on it. Switch it to result.evidence_pack.chunks, which does, and guard on an empty/missing evidence_pack instead of an empty context_items list. The course-builder caller already had the chunk id under label; add it explicitly as chunk_id for uniformity. Both callers now emit {label, text, chunk_id} entries, which the upcoming quiz agent harness (Task 2+) needs to plan questions against specific source chunks. Refs #134 --- services/ai/app/api/quizzes_api.py | 11 ++--- services/ai/app/services/lesson_service.py | 2 +- .../test_quizzes_certificates_api.py | 42 +++++++++++++++++++ services/ai/tests/unit/test_lesson_service.py | 1 + 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/services/ai/app/api/quizzes_api.py b/services/ai/app/api/quizzes_api.py index 2185f98..a434006 100644 --- a/services/ai/app/api/quizzes_api.py +++ b/services/ai/app/api/quizzes_api.py @@ -162,15 +162,16 @@ async def generate_quiz( action=StudyAction.RETRIEVE, ) - context_items = [ - {"label": c.label or c.doc_id or f"ref_{i}", "text": c.snippet} - for i, c in enumerate(result.citations) - ] - if not context_items: + if result.evidence_pack is None or not result.evidence_pack.chunks: raise ValidationError_( "No source material found for this topic in the course. Try a broader query." ) + context_items = [ + {"label": c.anchor.doc_name, "text": c.text, "chunk_id": c.chunk_id} + for c in result.evidence_pack.chunks + ] + quiz_data = await quiz_generation.generate_quiz_questions(body.query, context_items) if not quiz_data or not quiz_data.get("questions"): raise ValidationError_( diff --git a/services/ai/app/services/lesson_service.py b/services/ai/app/services/lesson_service.py index cfcd2c8..13c470c 100644 --- a/services/ai/app/services/lesson_service.py +++ b/services/ai/app/services/lesson_service.py @@ -190,7 +190,7 @@ def _load_context(lesson: Lesson, db: Session) -> Tuple[List[str], List[Dict[str used_ids: List[str] = [] for cid in chunk_ids: if cid in rows: - context_items.append({"label": cid, "text": rows[cid].text}) + context_items.append({"label": cid, "text": rows[cid].text, "chunk_id": cid}) used_ids.append(cid) return used_ids, context_items diff --git a/services/ai/tests/integration/test_quizzes_certificates_api.py b/services/ai/tests/integration/test_quizzes_certificates_api.py index 05c074b..8bf3ba5 100644 --- a/services/ai/tests/integration/test_quizzes_certificates_api.py +++ b/services/ai/tests/integration/test_quizzes_certificates_api.py @@ -10,6 +10,7 @@ from app.core.config import create_access_token from app.db.models import Quiz, QuizScope +from app.schemas.evidence_pack import CitationAnchor, EvidenceChunk, EvidencePack from app.services.quiz_generation import persist_quiz from app.services.study_service import DispatchResult, SourceChunk from tests.conftest import make_course_with_lessons, make_user @@ -39,6 +40,23 @@ def _auth(user_id: str) -> dict: answer="", citations=[SourceChunk(snippet="A UTXO is an unspent transaction output.", score=0.9, label="p1")], retrieval_used=True, + evidence_pack=EvidencePack( + query="UTXO model basics", + action="retrieve", + chunks=[ + EvidenceChunk( + chunk_id="chunk-1", + text="A UTXO is an unspent transaction output.", + score=0.9, + anchor=CitationAnchor( + doc_id="doc-1", doc_name="p1", chunk_id="chunk-1", chunk_type="text", + ), + ) + ], + total_candidates=1, + ordering=[0], + deduped_passages=["A UTXO is an unspent transaction output."], + ), ) @@ -87,6 +105,30 @@ def test_generate_quiz_persists_to_db(self, client, db): assert quiz is not None assert quiz.course_id == course.id + @pytest.mark.integration + def test_generate_quiz_uses_evidence_pack_chunk_id(self, client, db): + user = make_user(db) + course, _ = make_course_with_lessons(db) + + with patch( + "app.services.study_service.dispatch", new_callable=AsyncMock + ) as mock_dispatch, patch( + "app.services.quiz_generation.generate_json", new_callable=AsyncMock + ) as mock_gj: + mock_dispatch.return_value = _RETRIEVE_RESULT + mock_gj.return_value = {"questions": _QUESTIONS} + + resp = client.post( + f"/api/courses/{course.id}/quizzes/generate", + json={"query": "UTXO model basics"}, + headers=_auth(user.id), + ) + + assert resp.status_code == 201 + _, kwargs = mock_gj.call_args_list[0] + context_items = kwargs["context"] + assert any(item.get("chunk_id") == "chunk-1" for item in context_items) + @pytest.mark.integration def test_generate_quiz_422_when_no_source_material(self, client, db): user = make_user(db) diff --git a/services/ai/tests/unit/test_lesson_service.py b/services/ai/tests/unit/test_lesson_service.py index a6a7c65..552e850 100644 --- a/services/ai/tests/unit/test_lesson_service.py +++ b/services/ai/tests/unit/test_lesson_service.py @@ -148,6 +148,7 @@ def test_load_context_returns_items(): assert len(items) == 2 assert items[0]["label"] == "p1" assert "Bitcoin" in items[0]["text"] + assert items[0]["chunk_id"] == "p1" def test_load_context_caps_at_max(): From cae94d62919bb3225a103a7f369fd0772c3ccb76 Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 21:36:15 +0100 Subject: [PATCH 03/12] feat(rag): add plan stage to the quiz agent harness Adds app/services/quiz_harness.py with _plan_targets, the first stage of the plan -> generate -> critique -> repair loop that will replace the single schema-validated quiz generation call. The model selects source passages by 1-based position (matching generate-json.js's [N] convention) rather than by chunk_id text, so an invented or out-of-range reference is corrected deterministically by round-robin assignment instead of trusted. _plan_targets always returns exactly target_count targets. Registers quiz_plan in qvac_router.LADDER_ROUTING as an explicit server-routed task_type, alongside judge/content_gen. Refs #134 --- services/ai/app/services/quiz_harness.py | 89 +++++++++++++++++++++ services/ai/app/services/qvac_router.py | 1 + services/ai/tests/unit/test_quiz_harness.py | 61 ++++++++++++++ services/ai/tests/unit/test_qvac_router.py | 1 + 4 files changed, 152 insertions(+) create mode 100644 services/ai/app/services/quiz_harness.py create mode 100644 services/ai/tests/unit/test_quiz_harness.py diff --git a/services/ai/app/services/quiz_harness.py b/services/ai/app/services/quiz_harness.py new file mode 100644 index 0000000..346a8dc --- /dev/null +++ b/services/ai/app/services/quiz_harness.py @@ -0,0 +1,89 @@ +"""Agent harness for quiz generation — plan -> generate -> critique -> repair. + +See docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md. +""" +import logging +from typing import Any, Dict, List + +from app.services.qvac_structured import generate_json + +logger = logging.getLogger(__name__) + +PlanTarget = Dict[str, str] + +_COGNITIVE_LEVELS = ("recall", "application", "analysis") + +PLAN_PROMPT_VERSION = "v1" + +_PLAN_SCHEMA: Dict[str, Any] = { + "type": "object", + "required": ["targets"], + "properties": { + "targets": { + "type": "array", + "items": { + "type": "object", + "required": ["passage_index", "cognitive_level"], + "properties": { + "passage_index": {"type": "integer"}, + "cognitive_level": {"type": "string", "enum": list(_COGNITIVE_LEVELS)}, + }, + }, + } + }, +} + +_PLAN_SYSTEM = ( + "You are planning a quiz. Given numbered source passages, select which passages to " + "test and at what cognitive level, so coverage spans distinct passages and levels " + "rather than clustering on one passage.\n" + "Refer to each passage by the 1-based index shown before it (e.g. passage [2]).\n" + "cognitive_level: 'recall' (states a fact), 'application' (uses a concept in a new " + "situation), or 'analysis' (compares or explains relationships between concepts)." +) + + +async def _plan_targets(context_items: List[Dict[str, str]], target_count: int) -> List[PlanTarget]: + """Select target_count (chunk_id, cognitive_level) pairs from context_items. + + Always returns exactly target_count targets. An invalid or missing + passage_index from the model is corrected by round-robin assignment over + context_items rather than trusted — the model never needs to reproduce a + chunk_id string correctly. + """ + prompt = ( + f"Select exactly {target_count} target passages for a quiz out of the " + f"{len(context_items)} numbered passages provided, each with a cognitive_level." + ) + result = await generate_json( + prompt, + _PLAN_SCHEMA, + context=context_items, + system_prompt=_PLAN_SYSTEM, + generation_params={"temp": 0.2}, + task_type="quiz_plan", + ) + + raw_targets = result.get("targets", []) + targets: List[PlanTarget] = [] + + for i in range(target_count): + if i < len(raw_targets): + raw = raw_targets[i] + idx = raw.get("passage_index") + level = raw.get("cognitive_level") + else: + idx = None + level = None + + if isinstance(idx, int) and 1 <= idx <= len(context_items): + chunk_id = context_items[idx - 1]["chunk_id"] + else: + chunk_id = context_items[i % len(context_items)]["chunk_id"] + + if level not in _COGNITIVE_LEVELS: + level = "recall" + + targets.append({"chunk_id": chunk_id, "cognitive_level": level}) + + return targets diff --git a/services/ai/app/services/qvac_router.py b/services/ai/app/services/qvac_router.py index 1bf4089..33a55d4 100644 --- a/services/ai/app/services/qvac_router.py +++ b/services/ai/app/services/qvac_router.py @@ -39,6 +39,7 @@ "content_gen": ("server", SERVER_MODEL), "judge": ("server", SERVER_MODEL), "precompute": ("server", SERVER_MODEL), + "quiz_plan": ("server", SERVER_MODEL), } diff --git a/services/ai/tests/unit/test_quiz_harness.py b/services/ai/tests/unit/test_quiz_harness.py new file mode 100644 index 0000000..f2f8f9b --- /dev/null +++ b/services/ai/tests/unit/test_quiz_harness.py @@ -0,0 +1,61 @@ +import pytest +from unittest.mock import AsyncMock, patch + +from app.services.qvac_structured import LlmDisabledError, StructuredGenerationError +from app.services.quiz_harness import _plan_targets + +_CONTEXT = [ + {"label": "p1", "text": "UTXO text", "chunk_id": "c1"}, + {"label": "p2", "text": "PoW text", "chunk_id": "c2"}, +] + + +@pytest.mark.asyncio +async def test_plan_targets_maps_passage_index_to_chunk_id(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"targets": [ + {"passage_index": 2, "cognitive_level": "application"}, + {"passage_index": 1, "cognitive_level": "recall"}, + ]} + targets = await _plan_targets(_CONTEXT, target_count=2) + + assert targets == [ + {"chunk_id": "c2", "cognitive_level": "application"}, + {"chunk_id": "c1", "cognitive_level": "recall"}, + ] + + +@pytest.mark.asyncio +async def test_plan_targets_corrects_invalid_passage_index(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"targets": [{"passage_index": 99, "cognitive_level": "recall"}]} + targets = await _plan_targets(_CONTEXT, target_count=1) + + assert targets[0]["chunk_id"] in {"c1", "c2"} + + +@pytest.mark.asyncio +async def test_plan_targets_pads_when_model_returns_too_few(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"targets": [{"passage_index": 1, "cognitive_level": "recall"}]} + targets = await _plan_targets(_CONTEXT, target_count=3) + + assert len(targets) == 3 + + +@pytest.mark.asyncio +async def test_plan_targets_reuses_chunks_when_target_count_exceeds_available(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"targets": []} + targets = await _plan_targets(_CONTEXT, target_count=5) + + assert len(targets) == 5 + assert {t["chunk_id"] for t in targets} == {"c1", "c2"} + + +@pytest.mark.asyncio +async def test_plan_targets_propagates_llm_disabled(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = LlmDisabledError("disabled") + with pytest.raises(LlmDisabledError): + await _plan_targets(_CONTEXT, target_count=1) diff --git a/services/ai/tests/unit/test_qvac_router.py b/services/ai/tests/unit/test_qvac_router.py index bf6b034..c158bc5 100644 --- a/services/ai/tests/unit/test_qvac_router.py +++ b/services/ai/tests/unit/test_qvac_router.py @@ -13,6 +13,7 @@ ("content_gen", qvac_router.SERVER_MODEL), ("judge", qvac_router.SERVER_MODEL), ("precompute", qvac_router.SERVER_MODEL), + ("quiz_plan", qvac_router.SERVER_MODEL), ]) def test_server_task_types_route_to_server(task_type, expected_model): route = qvac_router.resolve(task_type) From 5cd5d055d5f3a3de428f62817bf0048163cffdcc Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 21:43:06 +0100 Subject: [PATCH 04/12] feat(rag): add generate stage to the quiz agent harness Add _generate_one, which produces one question per planned target via a single generate_json call pinned to that target's own chunk, so the model cannot pull material from elsewhere in the evidence pack. Reuses the content_gen task type since it is the same model as today's quiz generation. StructuredGenerationError is caught and turned into None (the caller treats this as "needs repair or must be dropped"); LlmDisabledError propagates to abort the run, matching the plan stage's contract. Refs #134 --- services/ai/app/services/quiz_harness.py | 85 ++++++++++++++++++++- services/ai/tests/unit/test_quiz_harness.py | 57 +++++++++++++- 2 files changed, 139 insertions(+), 3 deletions(-) diff --git a/services/ai/app/services/quiz_harness.py b/services/ai/app/services/quiz_harness.py index 346a8dc..b5a4a8c 100644 --- a/services/ai/app/services/quiz_harness.py +++ b/services/ai/app/services/quiz_harness.py @@ -3,9 +3,9 @@ See docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md. """ import logging -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional -from app.services.qvac_structured import generate_json +from app.services.qvac_structured import StructuredGenerationError, generate_json logger = logging.getLogger(__name__) @@ -87,3 +87,84 @@ async def _plan_targets(context_items: List[Dict[str, str]], target_count: int) targets.append({"chunk_id": chunk_id, "cognitive_level": level}) return targets + + +GENERATE_PROMPT_VERSION = "v1" + +_GENERATE_SCHEMA: Dict[str, Any] = { + "type": "object", + "required": ["prompt", "options", "correct_key", "concept_tag", "difficulty"], + "properties": { + "prompt": {"type": "string"}, + "options": { + "type": "array", + "minItems": 4, + "maxItems": 4, + "items": { + "type": "object", + "required": ["key", "text"], + "properties": { + "key": {"type": "string", "enum": ["A", "B", "C", "D"]}, + "text": {"type": "string"}, + }, + }, + }, + "correct_key": {"type": "string", "enum": ["A", "B", "C", "D"]}, + "concept_tag": {"type": "string"}, + "difficulty": {"type": "string", "enum": ["beginner", "intermediate", "advanced"]}, + }, +} + +_GENERATE_SYSTEM = ( + "You are an expert educator at BitPolito Academy creating one assessment question " + "from a single source passage.\n" + "RULES:\n" + "1. The question must test the cognitive level given in the task (recall / application " + "/ analysis), based ONLY on the provided passage.\n" + "2. All 4 options (A-D) must be plausible — wrong options should be conceptually close " + "but demonstrably wrong from the passage.\n" + "3. Exactly one option must be correct and directly supported by the passage.\n" + "4. Use exact technical terminology from the passage.\n" + "5. concept_tag: a short kebab-case slug for the concept tested.\n" + "6. difficulty: beginner / intermediate / advanced." +) + + +async def _generate_one( + target: PlanTarget, + context_items: List[Dict[str, str]], + *, + feedback: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """Generate one question for target, scoped to its single source chunk. + + Returns None (rather than propagating) on StructuredGenerationError, so the + caller can treat a failed target as needing repair or being dropped. + LlmDisabledError is not caught here and propagates to abort the run. + """ + item = next( + (ci for ci in context_items if ci["chunk_id"] == target["chunk_id"]), + None, + ) + context = [item] if item is not None else [] + + prompt = ( + f"Write one {target['cognitive_level']}-level multiple-choice question from the " + "provided passage." + ) + if feedback: + prompt += f"\nNote: the previous attempt was rejected — {feedback}" + + try: + result = await generate_json( + prompt, + _GENERATE_SCHEMA, + context=context, + system_prompt=_GENERATE_SYSTEM, + generation_params={"temp": 0.3}, + task_type="content_gen", + ) + except StructuredGenerationError: + return None + + return {**result, "chunk_id": target["chunk_id"]} diff --git a/services/ai/tests/unit/test_quiz_harness.py b/services/ai/tests/unit/test_quiz_harness.py index f2f8f9b..7c61c8a 100644 --- a/services/ai/tests/unit/test_quiz_harness.py +++ b/services/ai/tests/unit/test_quiz_harness.py @@ -2,7 +2,7 @@ from unittest.mock import AsyncMock, patch from app.services.qvac_structured import LlmDisabledError, StructuredGenerationError -from app.services.quiz_harness import _plan_targets +from app.services.quiz_harness import _generate_one, _plan_targets _CONTEXT = [ {"label": "p1", "text": "UTXO text", "chunk_id": "c1"}, @@ -59,3 +59,58 @@ async def test_plan_targets_propagates_llm_disabled(): mock_gj.side_effect = LlmDisabledError("disabled") with pytest.raises(LlmDisabledError): await _plan_targets(_CONTEXT, target_count=1) + + +_TARGET = {"chunk_id": "c1", "cognitive_level": "recall"} + +_QUESTION = { + "prompt": "What is a UTXO?", + "options": [ + {"key": "A", "text": "An unspent transaction output"}, + {"key": "B", "text": "A mining reward"}, + {"key": "C", "text": "A block header"}, + {"key": "D", "text": "A wallet address"}, + ], + "correct_key": "A", + "concept_tag": "utxo-model", + "difficulty": "beginner", +} + + +@pytest.mark.asyncio +async def test_generate_one_attaches_chunk_id(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = dict(_QUESTION) + question = await _generate_one(_TARGET, _CONTEXT) + + assert question["chunk_id"] == "c1" + assert question["prompt"] == "What is a UTXO?" + + +@pytest.mark.asyncio +async def test_generate_one_scopes_context_to_the_target_chunk(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = dict(_QUESTION) + await _generate_one(_TARGET, _CONTEXT) + + call_context = mock_gj.call_args.kwargs["context"] + assert call_context == [{"label": "p1", "text": "UTXO text", "chunk_id": "c1"}] + + +@pytest.mark.asyncio +async def test_generate_one_includes_feedback_in_prompt_when_repairing(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = dict(_QUESTION) + await _generate_one(_TARGET, _CONTEXT, feedback="rejected: distractors too easy") + + prompt_arg = mock_gj.call_args.args[0] + assert "rejected: distractors too easy" in prompt_arg + + +@pytest.mark.asyncio +async def test_generate_one_returns_none_on_structured_generation_error(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = StructuredGenerationError("bad schema") + question = await _generate_one(_TARGET, _CONTEXT) + + assert question is None From 771b4aa7fe65d1479a832eb7503a1646b4a632de Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 21:46:37 +0100 Subject: [PATCH 05/12] fix(rag): guard _generate_one against an unresolvable target chunk_id Refs #134 --- services/ai/app/services/quiz_harness.py | 10 ++++++++-- services/ai/tests/unit/test_quiz_harness.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/services/ai/app/services/quiz_harness.py b/services/ai/app/services/quiz_harness.py index b5a4a8c..1ea0e73 100644 --- a/services/ai/app/services/quiz_harness.py +++ b/services/ai/app/services/quiz_harness.py @@ -139,14 +139,20 @@ async def _generate_one( """Generate one question for target, scoped to its single source chunk. Returns None (rather than propagating) on StructuredGenerationError, so the - caller can treat a failed target as needing repair or being dropped. + caller can treat a failed target as needing repair or being dropped. The + same applies if target["chunk_id"] does not match any context_items entry: + we must not fall back to a context-free generate_json call, since that + would silently produce an ungrounded question while still being tagged + with target["chunk_id"] on the way out. LlmDisabledError is not caught here and propagates to abort the run. """ item = next( (ci for ci in context_items if ci["chunk_id"] == target["chunk_id"]), None, ) - context = [item] if item is not None else [] + if item is None: + return None + context = [item] prompt = ( f"Write one {target['cognitive_level']}-level multiple-choice question from the " diff --git a/services/ai/tests/unit/test_quiz_harness.py b/services/ai/tests/unit/test_quiz_harness.py index 7c61c8a..90d28c1 100644 --- a/services/ai/tests/unit/test_quiz_harness.py +++ b/services/ai/tests/unit/test_quiz_harness.py @@ -114,3 +114,13 @@ async def test_generate_one_returns_none_on_structured_generation_error(): question = await _generate_one(_TARGET, _CONTEXT) assert question is None + + +@pytest.mark.asyncio +async def test_generate_one_returns_none_for_unresolvable_chunk_id(): + unresolvable_target = {"chunk_id": "does-not-exist", "cognitive_level": "recall"} + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + question = await _generate_one(unresolvable_target, _CONTEXT) + + assert question is None + mock_gj.assert_not_called() From 7278a6e2ccfed1fd040d18c5d34c719318e50679 Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 21:56:09 +0100 Subject: [PATCH 06/12] feat(rag): add critique stage to the quiz agent harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add _critique_one, judging a generated question against the five quality criteria from issue #134 (entailed, distractors_plausible, single_defensible, requires_course_material, duplicate). Fails closed on LlmDisabledError/StructuredGenerationError, mirroring lesson_service._judge_groundedness's pattern — an unavailable critic is treated as a failed question, never a silent pass. Register quiz_critique in qvac_router.LADDER_ROUTING alongside the other quiz harness task types. Refs #134 --- services/ai/app/services/quiz_harness.py | 75 ++++++++++++++++++++- services/ai/app/services/qvac_router.py | 1 + services/ai/tests/unit/test_quiz_harness.py | 70 ++++++++++++++++++- services/ai/tests/unit/test_qvac_router.py | 1 + 4 files changed, 145 insertions(+), 2 deletions(-) diff --git a/services/ai/app/services/quiz_harness.py b/services/ai/app/services/quiz_harness.py index 1ea0e73..ad96bfe 100644 --- a/services/ai/app/services/quiz_harness.py +++ b/services/ai/app/services/quiz_harness.py @@ -5,7 +5,7 @@ import logging from typing import Any, Dict, List, Optional -from app.services.qvac_structured import StructuredGenerationError, generate_json +from app.services.qvac_structured import LlmDisabledError, StructuredGenerationError, generate_json logger = logging.getLogger(__name__) @@ -174,3 +174,76 @@ async def _generate_one( return None return {**result, "chunk_id": target["chunk_id"]} + + +CritiqueVerdict = Dict[str, Any] + +CRITIQUE_PROMPT_VERSION = "v1" + +_CRITIQUE_CRITERIA = ( + "entailed", "distractors_plausible", "single_defensible", "requires_course_material", +) +# "duplicate" is tracked separately — it's the one criterion where True is the failure, not False. + +_CRITIQUE_SCHEMA: Dict[str, Any] = { + "type": "object", + "required": ["entailed", "distractors_plausible", "single_defensible", + "requires_course_material", "duplicate"], + "properties": { + "entailed": {"type": "boolean"}, + "distractors_plausible": {"type": "boolean"}, + "single_defensible": {"type": "boolean"}, + "requires_course_material": {"type": "boolean"}, + "duplicate": {"type": "boolean"}, + }, +} + +_CRITIQUE_SYSTEM = ( + "You are a critic judging one multiple-choice question against its source passage and " + "the other questions already accepted into this quiz.\n" + "entailed: the correct answer is directly supported by the passage.\n" + "distractors_plausible: every wrong option is plausible but demonstrably wrong from the " + "passage — not absurd, not another correct answer.\n" + "single_defensible: exactly one option is correct; no other option could also be " + "defended as correct.\n" + "requires_course_material: the question cannot be answered from general knowledge alone.\n" + "duplicate: true if this question tests the same fact as one of the other questions shown." +) + + +async def _critique_one( + question: Dict[str, Any], source_text: str, accepted_questions: List[Dict[str, Any]], +) -> CritiqueVerdict: + """Judge one generated question against the five quality criteria. + + Fails closed: if the critic is unavailable (LlmDisabledError) or returns + an unparsable response (StructuredGenerationError), the question is + treated as a failure, not a pass — it is never silently accepted just + because we could not critique it. Mirrors + lesson_service._judge_groundedness's fail-closed pattern. + """ + accepted_prompts = [q["prompt"] for q in accepted_questions] + prompt = ( + f"Question: {question['prompt']}\n" + f"Options: {question['options']}\n" + f"Correct key: {question['correct_key']}\n" + f"Other questions already accepted into this quiz: {accepted_prompts}" + ) + + try: + result = await generate_json( + prompt, + _CRITIQUE_SCHEMA, + context=[{"label": "source", "text": source_text}], + system_prompt=_CRITIQUE_SYSTEM, + generation_params={"temp": 0.1}, + task_type="quiz_critique", + ) + except (LlmDisabledError, StructuredGenerationError): + return {"passed": False, "failed_criteria": ["critic unavailable"]} + + failed_criteria = [name for name in _CRITIQUE_CRITERIA if not result.get(name, False)] + if result.get("duplicate", True): + failed_criteria.append("duplicate") + + return {"passed": not failed_criteria, "failed_criteria": failed_criteria} diff --git a/services/ai/app/services/qvac_router.py b/services/ai/app/services/qvac_router.py index 33a55d4..425e84b 100644 --- a/services/ai/app/services/qvac_router.py +++ b/services/ai/app/services/qvac_router.py @@ -40,6 +40,7 @@ "judge": ("server", SERVER_MODEL), "precompute": ("server", SERVER_MODEL), "quiz_plan": ("server", SERVER_MODEL), + "quiz_critique": ("server", SERVER_MODEL), } diff --git a/services/ai/tests/unit/test_quiz_harness.py b/services/ai/tests/unit/test_quiz_harness.py index 90d28c1..7cf7d47 100644 --- a/services/ai/tests/unit/test_quiz_harness.py +++ b/services/ai/tests/unit/test_quiz_harness.py @@ -2,7 +2,7 @@ from unittest.mock import AsyncMock, patch from app.services.qvac_structured import LlmDisabledError, StructuredGenerationError -from app.services.quiz_harness import _generate_one, _plan_targets +from app.services.quiz_harness import _critique_one, _generate_one, _plan_targets _CONTEXT = [ {"label": "p1", "text": "UTXO text", "chunk_id": "c1"}, @@ -124,3 +124,71 @@ async def test_generate_one_returns_none_for_unresolvable_chunk_id(): assert question is None mock_gj.assert_not_called() + + +@pytest.mark.asyncio +async def test_critique_one_passes_when_all_criteria_met(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = { + "entailed": True, "distractors_plausible": True, "single_defensible": True, + "requires_course_material": True, "duplicate": False, + } + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict == {"passed": True, "failed_criteria": []} + + +@pytest.mark.asyncio +async def test_critique_one_reports_each_failed_criterion(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = { + "entailed": False, "distractors_plausible": True, "single_defensible": True, + "requires_course_material": True, "duplicate": True, + } + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict["passed"] is False + assert set(verdict["failed_criteria"]) == {"entailed", "duplicate"} + + +@pytest.mark.asyncio +async def test_critique_one_handles_missing_fields_defensively(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"entailed": True} # worker returned a partial object + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict["passed"] is False + assert "distractors_plausible" in verdict["failed_criteria"] + + +@pytest.mark.asyncio +async def test_critique_one_passes_accepted_questions_as_duplicate_context(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = { + "entailed": True, "distractors_plausible": True, "single_defensible": True, + "requires_course_material": True, "duplicate": False, + } + await _critique_one(_QUESTION, "UTXO text", [{"prompt": "Earlier question?"}]) + + prompt_arg = mock_gj.call_args.args[0] + assert "Earlier question?" in prompt_arg + + +@pytest.mark.asyncio +async def test_critique_one_fails_closed_on_llm_disabled(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = LlmDisabledError("disabled") + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict["passed"] is False + assert verdict["failed_criteria"] == ["critic unavailable"] + + +@pytest.mark.asyncio +async def test_critique_one_fails_closed_on_structured_generation_error(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = StructuredGenerationError("bad schema") + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict["passed"] is False + assert verdict["failed_criteria"] == ["critic unavailable"] diff --git a/services/ai/tests/unit/test_qvac_router.py b/services/ai/tests/unit/test_qvac_router.py index c158bc5..48d52e2 100644 --- a/services/ai/tests/unit/test_qvac_router.py +++ b/services/ai/tests/unit/test_qvac_router.py @@ -14,6 +14,7 @@ ("judge", qvac_router.SERVER_MODEL), ("precompute", qvac_router.SERVER_MODEL), ("quiz_plan", qvac_router.SERVER_MODEL), + ("quiz_critique", qvac_router.SERVER_MODEL), ]) def test_server_task_types_route_to_server(task_type, expected_model): route = qvac_router.resolve(task_type) From 8798d72467c9d6607a7634b8d38bbf0c8d52d8e9 Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 22:04:42 +0100 Subject: [PATCH 07/12] feat(rag): add bounded repair loop and run() orchestrator to the quiz harness Wire _plan_targets, _generate_one, and _critique_one into run(), the plan -> generate -> critique -> repair orchestrator that Task 7 will call from quiz_generation.generate_quiz_questions. Each target gets at most one generate+critique round plus exactly one repair round if the first critique fails, bounding the call budget at 4 * target_count + 1 (trace["calls_budget"]) and recorded exactly in trace["calls_used"]. A None generation result (unresolvable chunk_id, schema error, or the LLM going down mid-run) is dropped immediately rather than retried, since repeating the same target against the same context would almost certainly fail identically; this also keeps a failed repair generate from spending an extra, pointless critique call. LlmDisabledError/StructuredGenerationError at the plan stage end the run immediately with no questions produced rather than propagating, since nothing has been generated yet to return instead. Refs #134 --- services/ai/app/services/quiz_harness.py | 113 ++++++++++++++++++++ services/ai/tests/unit/test_quiz_harness.py | 83 +++++++++++++- 2 files changed, 195 insertions(+), 1 deletion(-) diff --git a/services/ai/app/services/quiz_harness.py b/services/ai/app/services/quiz_harness.py index ad96bfe..c738a57 100644 --- a/services/ai/app/services/quiz_harness.py +++ b/services/ai/app/services/quiz_harness.py @@ -247,3 +247,116 @@ async def _critique_one( failed_criteria.append("duplicate") return {"passed": not failed_criteria, "failed_criteria": failed_criteria} + + +HarnessResult = Dict[str, Any] + +DEFAULT_TARGET_COUNT = 4 + + +def _source_text_for(target: PlanTarget, context_items: List[Dict[str, str]]) -> str: + item = next( + (ci for ci in context_items if ci["chunk_id"] == target["chunk_id"]), + None, + ) + return item["text"] if item is not None else "" + + +async def run( + topic: str, + context_items: List[Dict[str, str]], + target_count: int = DEFAULT_TARGET_COUNT, +) -> HarnessResult: + """Run the plan -> generate -> critique -> repair harness. + + Produces at most `target_count` questions, each one passing through at + most two generate+critique rounds (an initial attempt and, on a failed + critique, exactly one repair attempt built from the critique's + `failed_criteria`). The call budget is therefore bounded by + `4 * target_count + 1` (plan, plus up to generate+critique twice per + target) — see `trace["calls_budget"]`. + + A `None` generation result (an unresolvable chunk_id, a schema error, or + the LLM being disabled mid-run) is not retried: the same target and + context would almost certainly fail the same way again, so the target is + dropped immediately without spending a repair attempt or a critique call + on it. + + `LlmDisabledError`/`StructuredGenerationError` at the plan stage end the + run immediately with no questions produced, since there is nothing yet to + degrade to. `topic` is accepted for interface symmetry with + `generate_quiz_questions` and is not otherwise used here yet. + """ + trace: Dict[str, Any] = { + "targets_planned": 0, + "generated": 0, + "rejected": [], + "repaired": 0, + "final_count": 0, + "calls_used": 0, + "calls_budget": 4 * target_count + 1, + } + accepted: List[Dict[str, Any]] = [] + + try: + targets = await _plan_targets(context_items, target_count) + trace["calls_used"] += 1 + except (LlmDisabledError, StructuredGenerationError) as exc: + trace["calls_used"] += 1 + trace["rejected"].append({"stage": "plan", "reason": str(exc)}) + trace["final_count"] = len(accepted) + return {"questions": accepted, "trace": trace} + + trace["targets_planned"] = len(targets) + + for target in targets: + source_text = _source_text_for(target, context_items) + + try: + question = await _generate_one(target, context_items) + except LlmDisabledError: + question = None + trace["calls_used"] += 1 + + if question is None: + trace["rejected"].append({"question_preview": None, "reasons": ["generation failed"]}) + continue + + trace["generated"] += 1 + verdict = await _critique_one(question, source_text, accepted) + trace["calls_used"] += 1 + + if verdict["passed"]: + accepted.append(question) + continue + + # Exactly one repair attempt, informed by the critique's failed_criteria. + feedback = ", ".join(verdict["failed_criteria"]) + try: + repaired_question = await _generate_one(target, context_items, feedback=feedback) + except LlmDisabledError: + repaired_question = None + trace["calls_used"] += 1 + + if repaired_question is None: + # A None from a repair generate must not trigger an extra critique call. + trace["rejected"].append({ + "question_preview": question["prompt"] if question else None, + "reasons": verdict["failed_criteria"], + }) + continue + + repaired_verdict = await _critique_one(repaired_question, source_text, accepted) + trace["calls_used"] += 1 + + if repaired_verdict["passed"]: + accepted.append(repaired_question) + trace["repaired"] += 1 + else: + trace["rejected"].append({ + "question_preview": repaired_question["prompt"], + "reasons": repaired_verdict["failed_criteria"], + }) + + trace["final_count"] = len(accepted) + return {"questions": accepted, "trace": trace} diff --git a/services/ai/tests/unit/test_quiz_harness.py b/services/ai/tests/unit/test_quiz_harness.py index 7cf7d47..3fa9b41 100644 --- a/services/ai/tests/unit/test_quiz_harness.py +++ b/services/ai/tests/unit/test_quiz_harness.py @@ -2,7 +2,7 @@ from unittest.mock import AsyncMock, patch from app.services.qvac_structured import LlmDisabledError, StructuredGenerationError -from app.services.quiz_harness import _critique_one, _generate_one, _plan_targets +from app.services.quiz_harness import _critique_one, _generate_one, _plan_targets, run _CONTEXT = [ {"label": "p1", "text": "UTXO text", "chunk_id": "c1"}, @@ -192,3 +192,84 @@ async def test_critique_one_fails_closed_on_structured_generation_error(): assert verdict["passed"] is False assert verdict["failed_criteria"] == ["critic unavailable"] + + +@pytest.mark.asyncio +async def test_run_all_pass_returns_every_question_with_no_repairs(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] * 2 + mock_gen.return_value = dict(_QUESTION) + mock_crit.return_value = {"passed": True, "failed_criteria": []} + + result = await run("UTXO model", _CONTEXT, target_count=2) + + assert len(result["questions"]) == 2 + assert result["trace"]["repaired"] == 0 + assert result["trace"]["final_count"] == 2 + assert result["trace"]["calls_used"] == 1 + 2 + 2 # plan + 2 generate + 2 critique + + +@pytest.mark.asyncio +async def test_run_rejected_then_repaired_is_recorded_and_kept(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] + mock_gen.return_value = dict(_QUESTION) + mock_crit.side_effect = [ + {"passed": False, "failed_criteria": ["entailed"]}, + {"passed": True, "failed_criteria": []}, + ] + + result = await run("UTXO model", _CONTEXT, target_count=1) + + assert result["trace"]["repaired"] == 1 + assert len(result["questions"]) == 1 + assert result["trace"]["calls_used"] == 1 + 2 + 2 # plan + (generate+critique) x2 + + +@pytest.mark.asyncio +async def test_run_retry_limit_reached_drops_the_question_with_reasons(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] + mock_gen.return_value = dict(_QUESTION) + mock_crit.return_value = {"passed": False, "failed_criteria": ["entailed"]} + + result = await run("UTXO model", _CONTEXT, target_count=1) + + assert result["questions"] == [] + assert result["trace"]["final_count"] == 0 + assert len(result["trace"]["rejected"]) == 1 + assert result["trace"]["rejected"][0]["reasons"] == ["entailed"] + # Exactly one repair attempt — no third generate/critique pass. + assert result["trace"]["calls_used"] == 1 + 2 + 2 + + +@pytest.mark.asyncio +async def test_run_llm_disabled_at_plan_stage_returns_no_questions(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan: + mock_plan.side_effect = LlmDisabledError("disabled") + + result = await run("UTXO model", _CONTEXT, target_count=2) + + assert result["questions"] == [] + assert result["trace"]["final_count"] == 0 + + +@pytest.mark.asyncio +async def test_run_llm_disabled_mid_generate_drops_that_target_and_continues(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] * 2 + mock_gen.side_effect = [dict(_QUESTION), None] # second target's generation failed + mock_crit.return_value = {"passed": True, "failed_criteria": []} + + result = await run("UTXO model", _CONTEXT, target_count=2) + + assert len(result["questions"]) == 1 + assert len(result["trace"]["rejected"]) == 1 From ea92ac6f0ec3056ecfeddeb4647324226c4e0a01 Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 22:13:26 +0100 Subject: [PATCH 08/12] feat(rag): register quiz harness plan/critique prompts, repoint quiz_gen Wires the harness's new plan/critique/generate prompts into the central registry so get_prompt()/list_prompts() can inspect them. The quiz_gen entry now points to quiz_harness.GENERATE_PROMPT_VERSION and _GENERATE_SYSTEM instead of the quiz_generation equivalents that will be deleted once Task 7 rewires their only remaining caller. Refs #134 --- services/ai/app/prompts/registry.py | 10 +++++++-- .../ai/tests/unit/test_prompt_registry.py | 21 +++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/services/ai/app/prompts/registry.py b/services/ai/app/prompts/registry.py index a4ddeec..de6229c 100644 --- a/services/ai/app/prompts/registry.py +++ b/services/ai/app/prompts/registry.py @@ -27,7 +27,7 @@ class Prompt: def _build_registry() -> Dict[str, Prompt]: - from app.services import lesson_service, outline_service, quiz_generation, study_service + from app.services import lesson_service, outline_service, quiz_generation, quiz_harness, study_service entries: Dict[str, Prompt] = { "outline_map": Prompt( @@ -43,7 +43,13 @@ def _build_registry() -> Dict[str, Prompt]: "judge", lesson_service.CONTENT_PROMPT_VERSION, lesson_service._JUDGE_SYSTEM, ), "quiz_gen": Prompt( - "quiz_gen", quiz_generation.QUIZ_PROMPT_VERSION, quiz_generation.QUIZ_SYSTEM, + "quiz_gen", quiz_harness.GENERATE_PROMPT_VERSION, quiz_harness._GENERATE_SYSTEM, + ), + "quiz_plan": Prompt( + "quiz_plan", quiz_harness.PLAN_PROMPT_VERSION, quiz_harness._PLAN_SYSTEM, + ), + "quiz_critique": Prompt( + "quiz_critique", quiz_harness.CRITIQUE_PROMPT_VERSION, quiz_harness._CRITIQUE_SYSTEM, ), } for action, text in study_service._SYSTEM_PROMPTS.items(): diff --git a/services/ai/tests/unit/test_prompt_registry.py b/services/ai/tests/unit/test_prompt_registry.py index 3571534..2cfce64 100644 --- a/services/ai/tests/unit/test_prompt_registry.py +++ b/services/ai/tests/unit/test_prompt_registry.py @@ -11,6 +11,7 @@ EXPECTED_NAMES = { "outline_map", "outline_reduce", "content_gen", "judge", "quiz_gen", + "quiz_plan", "quiz_critique", "study_explain", "study_summarize", "study_open_questions", "study_quiz", "study_oral", "study_derive", "study_compare", } @@ -51,10 +52,22 @@ def test_judge_version_matches_lesson_service_constant(): assert get_prompt("judge").system == lesson_service._JUDGE_SYSTEM -def test_quiz_gen_version_matches_quiz_generation_constant(): - from app.services import quiz_generation - assert get_prompt("quiz_gen").version == quiz_generation.QUIZ_PROMPT_VERSION - assert get_prompt("quiz_gen").system == quiz_generation.QUIZ_SYSTEM +def test_quiz_gen_version_matches_quiz_harness_constant(): + from app.services import quiz_harness + assert get_prompt("quiz_gen").version == quiz_harness.GENERATE_PROMPT_VERSION + assert get_prompt("quiz_gen").system == quiz_harness._GENERATE_SYSTEM + + +def test_quiz_plan_version_matches_quiz_harness_constant(): + from app.services import quiz_harness + assert get_prompt("quiz_plan").version == quiz_harness.PLAN_PROMPT_VERSION + assert get_prompt("quiz_plan").system == quiz_harness._PLAN_SYSTEM + + +def test_quiz_critique_version_matches_quiz_harness_constant(): + from app.services import quiz_harness + assert get_prompt("quiz_critique").version == quiz_harness.CRITIQUE_PROMPT_VERSION + assert get_prompt("quiz_critique").system == quiz_harness._CRITIQUE_SYSTEM def test_outline_prompts_version_match_outline_service_constant(): From 9b957afa9d7396bde3f21cdc1598406b49ed891b Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 22:25:43 +0100 Subject: [PATCH 09/12] feat(rag): wire the quiz agent harness into quiz generation and the API response generate_quiz_questions now delegates to quiz_harness.run (plan -> generate -> critique -> repair) instead of calling generate_json directly, and the POST /courses/{course_id}/quizzes/generate response carries the harness's trace alongside the existing quiz fields. Deleted the now-dead QUIZ_PROMPT_VERSION/QUIZ_SCHEMA/QUIZ_SYSTEM constants, superseded by the harness's own generate-stage prompt (Task 6 already repointed the "quiz_gen" registry entry to it). Also updated two test sites outside the brief's file list (tests/unit/test_lesson_service.py, tests/integration/test_lesson_review_api.py) that patched the now-removed app.services.quiz_generation.generate_json attribute for the course-builder's lesson-processing flow; they now patch quiz_harness.run with an equivalent {"questions": [...]} shaped return value. Refs #134 --- services/ai/app/api/quizzes_api.py | 7 +- services/ai/app/services/quiz_generation.py | 87 ++----------------- .../integration/test_lesson_review_api.py | 4 +- .../test_quizzes_certificates_api.py | 13 +-- services/ai/tests/unit/test_lesson_service.py | 4 +- .../ai/tests/unit/test_quiz_generation.py | 25 +++--- 6 files changed, 33 insertions(+), 107 deletions(-) diff --git a/services/ai/app/api/quizzes_api.py b/services/ai/app/api/quizzes_api.py index a434006..63354ab 100644 --- a/services/ai/app/api/quizzes_api.py +++ b/services/ai/app/api/quizzes_api.py @@ -200,10 +200,9 @@ async def generate_quiz( .order_by(Question.order_index) .all() ) - return JSONResponse( - status_code=status.HTTP_201_CREATED, - content=_to_student_detail(quiz, questions, db).model_dump(), - ) + response_body = _to_student_detail(quiz, questions, db).model_dump() + response_body["trace"] = quiz_data.get("trace") + return JSONResponse(status_code=status.HTTP_201_CREATED, content=response_body) @router.get( diff --git a/services/ai/app/services/quiz_generation.py b/services/ai/app/services/quiz_generation.py index b03c811..7bf5d92 100644 --- a/services/ai/app/services/quiz_generation.py +++ b/services/ai/app/services/quiz_generation.py @@ -16,96 +16,27 @@ from app.core.config import settings from app.db.models import OptionChoice, Question, QuestionType, Quiz, QuizAttempt, QuizScope -from app.services.qvac_structured import StructuredGenerationError, generate_json +from app.services import quiz_harness logger = logging.getLogger(__name__) -QUIZ_PROMPT_VERSION = "v2" # v2 = adds concept_tag/difficulty to the schema - -QUIZ_SCHEMA: Dict[str, Any] = { - "type": "object", - "required": ["questions"], - "properties": { - "questions": { - "type": "array", - "minItems": 3, - "maxItems": 4, - "items": { - "type": "object", - "required": [ - "prompt", "options", "correct_key", "concept_tag", "difficulty", - ], - "properties": { - "prompt": {"type": "string"}, - "options": { - "type": "array", - "minItems": 4, - "maxItems": 4, - "items": { - "type": "object", - "required": ["key", "text"], - "properties": { - "key": {"type": "string", "enum": ["A", "B", "C", "D"]}, - "text": {"type": "string"}, - }, - }, - }, - "correct_key": {"type": "string", "enum": ["A", "B", "C", "D"]}, - "concept_tag": { - "type": "string", - "description": "short slug for the concept being tested, e.g. 'utxo-model'", - }, - "difficulty": { - "type": "string", - "enum": ["beginner", "intermediate", "advanced"], - }, - }, - }, - } - }, -} - -QUIZ_SYSTEM = ( - "You are an expert educator at BitPolito Academy creating assessment questions.\n" - "RULES:\n" - "1. Create 3-4 multiple-choice questions based ONLY on the provided source passages.\n" - "2. Each question tests conceptual understanding, not trivial recall.\n" - "3. All 4 options (A-D) must be plausible — wrong options should be conceptually close.\n" - "4. The correct answer must be directly supported by the source text.\n" - "5. Use exact technical terminology from the document.\n" - "6. concept_tag: a short kebab-case slug identifying the single concept the question " - "tests (e.g. 'utxo-model', 'proof-of-work', 'block-subsidy') — reuse the same tag " - "across questions that test the same concept.\n" - "7. difficulty: rate the question beginner / intermediate / advanced." -) +_DEFAULT_TARGET_COUNT = quiz_harness.DEFAULT_TARGET_COUNT async def generate_quiz_questions( topic: str, context_items: List[Dict[str, str]], ) -> Optional[Dict[str, Any]]: - """Call /generate_json to produce MCQ quiz data with concept tags. + """Run the plan/generate/critique/repair harness to produce MCQ quiz data. - Returns None (rather than raising) on schema-validation failure — quiz - generation is treated as best-effort by all current callers. + Returns None (rather than raising) when the harness produces zero + questions — quiz generation is treated as best-effort by all current + callers. """ - prompt = ( - f'Generate a quiz on: "{topic}". ' - "Create 3-4 multiple-choice questions that test understanding of the key concepts " - "in the provided source passages." - ) - try: - return await generate_json( - prompt, - QUIZ_SCHEMA, - context=context_items, - system_prompt=QUIZ_SYSTEM, - generation_params={"temp": 0.3}, - task_type="content_gen", - ) - except StructuredGenerationError as exc: - logger.warning("Quiz generation failed for topic '%s': %s", topic, exc) + result = await quiz_harness.run(topic, context_items, target_count=_DEFAULT_TARGET_COUNT) + if not result["questions"]: return None + return result def persist_quiz( diff --git a/services/ai/tests/integration/test_lesson_review_api.py b/services/ai/tests/integration/test_lesson_review_api.py index 6f9096b..c433159 100644 --- a/services/ai/tests/integration/test_lesson_review_api.py +++ b/services/ai/tests/integration/test_lesson_review_api.py @@ -293,10 +293,10 @@ def test_regenerate_clears_cache_and_forces_reprocessing(self, db_session): with patch( "app.services.lesson_service.generate_json", new_callable=_AsyncMock ) as mock_gj, patch( - "app.services.quiz_generation.generate_json", new_callable=_AsyncMock + "app.services.quiz_generation.quiz_harness.run", new_callable=_AsyncMock ) as mock_quiz_gj: mock_gj.side_effect = [content_result, judge_result] - mock_quiz_gj.return_value = None + mock_quiz_gj.return_value = {"questions": [], "trace": {"final_count": 0}} asyncio.run( lesson_service.generate_course_content( course_id=course.id, db=db_session, run_id=run.id, lesson_ids=[ls.id], diff --git a/services/ai/tests/integration/test_quizzes_certificates_api.py b/services/ai/tests/integration/test_quizzes_certificates_api.py index 8bf3ba5..62eb05b 100644 --- a/services/ai/tests/integration/test_quizzes_certificates_api.py +++ b/services/ai/tests/integration/test_quizzes_certificates_api.py @@ -81,10 +81,10 @@ def test_generate_quiz_persists_to_db(self, client, db): with patch( "app.services.study_service.dispatch", new_callable=AsyncMock ) as mock_dispatch, patch( - "app.services.quiz_generation.generate_json", new_callable=AsyncMock + "app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock ) as mock_gj: mock_dispatch.return_value = _RETRIEVE_RESULT - mock_gj.return_value = {"questions": _QUESTIONS} + mock_gj.return_value = {"questions": _QUESTIONS, "trace": {"final_count": 1}} resp = client.post( f"/api/courses/{course.id}/quizzes/generate", @@ -97,6 +97,7 @@ def test_generate_quiz_persists_to_db(self, client, db): assert "id" in body assert body["scope"] == QuizScope.COURSE assert len(body["questions"]) == 1 + assert body["trace"] == {"final_count": 1} # No is_correct leak to the student-facing generate response for opt in body["questions"][0]["options"]: assert "is_correct" not in opt @@ -113,10 +114,10 @@ def test_generate_quiz_uses_evidence_pack_chunk_id(self, client, db): with patch( "app.services.study_service.dispatch", new_callable=AsyncMock ) as mock_dispatch, patch( - "app.services.quiz_generation.generate_json", new_callable=AsyncMock + "app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock ) as mock_gj: mock_dispatch.return_value = _RETRIEVE_RESULT - mock_gj.return_value = {"questions": _QUESTIONS} + mock_gj.return_value = {"questions": _QUESTIONS, "trace": {"final_count": 1}} resp = client.post( f"/api/courses/{course.id}/quizzes/generate", @@ -125,8 +126,8 @@ def test_generate_quiz_uses_evidence_pack_chunk_id(self, client, db): ) assert resp.status_code == 201 - _, kwargs = mock_gj.call_args_list[0] - context_items = kwargs["context"] + args, _ = mock_gj.call_args_list[0] + context_items = args[1] assert any(item.get("chunk_id") == "chunk-1" for item in context_items) @pytest.mark.integration diff --git a/services/ai/tests/unit/test_lesson_service.py b/services/ai/tests/unit/test_lesson_service.py index 552e850..df6f9b3 100644 --- a/services/ai/tests/unit/test_lesson_service.py +++ b/services/ai/tests/unit/test_lesson_service.py @@ -350,7 +350,7 @@ async def test_process_lesson_published_when_faithful(): with patch( "app.services.lesson_service.generate_json", new_callable=AsyncMock ) as mock_gj, patch( - "app.services.quiz_generation.generate_json", new_callable=AsyncMock + "app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock ) as mock_quiz_gj: mock_gj.side_effect = [_CONTENT_RESULT, _JUDGE_PASS] mock_quiz_gj.return_value = _QUIZ_RESULT @@ -378,7 +378,7 @@ async def test_process_lesson_needs_review_when_unfaithful(): with patch( "app.services.lesson_service.generate_json", new_callable=AsyncMock ) as mock_gj, patch( - "app.services.quiz_generation.generate_json", new_callable=AsyncMock + "app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock ) as mock_quiz_gj: mock_gj.side_effect = [_CONTENT_RESULT, _JUDGE_FAIL] mock_quiz_gj.return_value = _QUIZ_RESULT diff --git a/services/ai/tests/unit/test_quiz_generation.py b/services/ai/tests/unit/test_quiz_generation.py index 9bd7d32..469893a 100644 --- a/services/ai/tests/unit/test_quiz_generation.py +++ b/services/ai/tests/unit/test_quiz_generation.py @@ -20,7 +20,6 @@ User, UserRole, ) -from app.services.qvac_structured import StructuredGenerationError from app.services.quiz_generation import generate_quiz_questions, persist_quiz @@ -64,24 +63,20 @@ def _make_db(): # --------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_generate_quiz_questions_returns_schema_result(): - with patch( - "app.services.quiz_generation.generate_json", new_callable=AsyncMock - ) as mock_gj: - mock_gj.return_value = {"questions": _QUESTIONS} - result = await generate_quiz_questions("UTXO model", [{"label": "p1", "text": "..."}]) +async def test_generate_quiz_questions_delegates_to_harness(): + with patch("app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock) as mock_run: + mock_run.return_value = {"questions": _QUESTIONS, "trace": {"final_count": 1}} + result = await generate_quiz_questions("UTXO model", [{"label": "p1", "text": "...", "chunk_id": "c1"}]) - assert result is not None - assert result["questions"][0]["concept_tag"] == "utxo-model" + assert result == {"questions": _QUESTIONS, "trace": {"final_count": 1}} + mock_run.assert_awaited_once() @pytest.mark.asyncio -async def test_generate_quiz_questions_returns_none_on_schema_failure(): - with patch( - "app.services.quiz_generation.generate_json", new_callable=AsyncMock - ) as mock_gj: - mock_gj.side_effect = StructuredGenerationError("bad schema") - result = await generate_quiz_questions("UTXO model", [{"label": "p1", "text": "..."}]) +async def test_generate_quiz_questions_returns_none_when_harness_produces_nothing(): + with patch("app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock) as mock_run: + mock_run.return_value = {"questions": [], "trace": {"final_count": 0}} + result = await generate_quiz_questions("UTXO model", [{"label": "p1", "text": "...", "chunk_id": "c1"}]) assert result is None From affa299a54149db667eed76069f283ee8fe23ac5 Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 22:52:34 +0100 Subject: [PATCH 10/12] fix(rag): address final review findings on the quiz agent harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix six Important findings from the whole-branch review before the harness PR goes up: 1. generate_quiz_questions never collapses an empty-questions harness result to None anymore — it always returns the full {questions, trace} dict and logs the trace when empty, so the 422/quiz-step paths that most need to see rejection reasons actually get them. 2. trace["rejected"] entries now share one shape everywhere (stage, chunk_id, cognitive_level, question_preview, reasons) instead of two incompatible ones for plan-stage vs per-target failures. 3. _generate_one and _critique_one now catch httpx.HTTPError the same way they already catch StructuredGenerationError/LlmDisabledError, so a transient transport error drops one target instead of discarding every question already accepted in the run. 4. docs/specifications.md's quiz section now describes the plan/generate/ critique/repair loop, the five critique criteria, and the bounded call cap; two stale comments (one claiming a single /generate_json call, one pointing at dead quiz_generation.py prompt constants) are corrected. 5. Added coverage for previously-unexercised error paths: a per-target LlmDisabledError raised (not returned) from _generate_one, a repair-stage generate returning None, and an end-to-end run() test that mocks generate_json directly to prove the real plan/generate/critique functions are wired together correctly. 6. persist_quiz now writes Question.source_ref from the question's chunk_id, so the chunk_id the harness computed is actually persisted. Refs #134 --- docs/specifications.md | 36 ++++++- services/ai/app/api/quizzes_api.py | 7 +- services/ai/app/services/lesson_service.py | 12 ++- services/ai/app/services/quiz_generation.py | 16 +++- services/ai/app/services/quiz_harness.py | 90 ++++++++++++----- .../ai/tests/unit/test_quiz_generation.py | 33 ++++++- services/ai/tests/unit/test_quiz_harness.py | 96 +++++++++++++++++++ 7 files changed, 248 insertions(+), 42 deletions(-) diff --git a/docs/specifications.md b/docs/specifications.md index 188ea64..7fb47d5 100644 --- a/docs/specifications.md +++ b/docs/specifications.md @@ -376,8 +376,40 @@ the authenticated learner's attempts. The Study source pane links to a saved lesson quiz. Lesson quizzes and chapter tests share one authenticated answer-submission form, which displays the server's passing threshold, score and corrections. A missing saved quiz shows an explicit -unavailable state instead of submitting an ad-hoc Study action. Generated question -quality remains tracked separately in #134. +unavailable state instead of submitting an ad-hoc Study action. + +**Quiz question quality (#134).** Both quiz callers (the course builder's +per-lesson quizzes and the study page's ad-hoc topic quizzes) generate +questions through an agent harness (`quiz_harness.py`) rather than a single +schema-validated model call. For a target of `N` questions, the harness runs: + +1. **Plan** (1 call) — selects `N` `(chunk_id, cognitive_level)` targets from + the available source passages, spread across distinct passages and + cognitive levels (recall / application / analysis). +2. **Generate** (up to `N` calls) — one question per target, scoped to that + target's single source passage. +3. **Critique** (up to `N` calls) — each generated question is checked + against five criteria: the answer is entailed by the cited source chunk; + every distractor is plausible but demonstrably wrong; exactly one option + is defensible; the question is not answerable from general knowledge + alone; the question does not duplicate another already accepted into the + same quiz. A critique failure is recorded by the specific criterion it + failed. +4. **Repair** (up to `2N` calls) — a question that fails critique is + regenerated once, with the failed criteria fed back as explicit feedback, + then re-critiqued; whatever that second verdict is, it is final. + +The loop is bounded by construction at `4*N + 1` calls (17 for the default +`N=4`), never unbounded. Every returned question carries a resolvable source +chunk id, now persisted on the `Question` row as well as in the API payload. +If the harness cannot reach the target count, it returns fewer questions +(or none) rather than malformed output, and reports why in a `trace` object +(targets planned, generated, rejected with reasons, repaired, final count, +calls used/budget). `POST /courses/{course_id}/quizzes/generate` returns this +`trace` as an additive field alongside the generated quiz; the course-builder +path logs it instead of surfacing it in the UI. A manual review of 20 +generated questions against real course material is a follow-up to this +implementation, tracked in #134. --- diff --git a/services/ai/app/api/quizzes_api.py b/services/ai/app/api/quizzes_api.py index 63354ab..f569fd4 100644 --- a/services/ai/app/api/quizzes_api.py +++ b/services/ai/app/api/quizzes_api.py @@ -154,8 +154,11 @@ async def generate_quiz( from app.schemas.study_schemas import StudyAction # noqa: PLC0415 from app.services import study_service # noqa: PLC0415 - # Retrieval-only: no LLM call for the sources, so quiz generation costs - # exactly one /generate_json call instead of a retrieval-answer + reparse. + # Retrieval-only: no LLM call for the sources themselves. Quiz generation + # below goes through the plan -> generate -> critique -> repair harness + # (quiz_harness.run), a bounded multi-call loop of up to 4*N+1 calls (17 + # for the default N=4 target count) rather than a single /generate_json + # call — see docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md. result = await study_service.dispatch( question=body.query, course_id=course_id, diff --git a/services/ai/app/services/lesson_service.py b/services/ai/app/services/lesson_service.py index 13c470c..33ce30a 100644 --- a/services/ai/app/services/lesson_service.py +++ b/services/ai/app/services/lesson_service.py @@ -96,10 +96,11 @@ }, } -# Quiz schema/system-prompt live in quiz_generation.py — shared with the -# study-page ad-hoc quiz endpoint (see quizzes_api.py). Keeping a single -# generator avoids the free-text-parsing/DB-persisted split that used to -# exist between the two call sites. +# Quiz generation is owned by the quiz_harness module (plan -> generate -> +# critique -> repair), reached through quiz_generation.generate_quiz_questions +# — shared with the study-page ad-hoc quiz endpoint (see quizzes_api.py). +# Keeping a single generator avoids the free-text-parsing/DB-persisted split +# that used to exist between the two call sites. # --------------------------------------------------------------------------- # System prompts @@ -269,7 +270,8 @@ async def _judge_groundedness( async def _generate_quiz_data( lesson: Lesson, context_items: List[Dict[str, str]] ) -> Optional[Dict[str, Any]]: - """Call /generate_json (via quiz_generation.py) to produce MCQ quiz data.""" + """Run the quiz_harness plan/generate/critique/repair loop (via + quiz_generation.generate_quiz_questions) to produce MCQ quiz data.""" return await quiz_generation.generate_quiz_questions(lesson.title, context_items) diff --git a/services/ai/app/services/quiz_generation.py b/services/ai/app/services/quiz_generation.py index 7bf5d92..b1037ea 100644 --- a/services/ai/app/services/quiz_generation.py +++ b/services/ai/app/services/quiz_generation.py @@ -26,16 +26,21 @@ async def generate_quiz_questions( topic: str, context_items: List[Dict[str, str]], -) -> Optional[Dict[str, Any]]: +) -> Dict[str, Any]: """Run the plan/generate/critique/repair harness to produce MCQ quiz data. - Returns None (rather than raising) when the harness produces zero - questions — quiz generation is treated as best-effort by all current - callers. + Always returns the full `{"questions": [...], "trace": {...}}` dict from + the harness, even when `questions` is empty — the trace is most valuable + exactly in that case (issue #134's "surface rejected questions" + acceptance criterion), and callers that check `.get("questions")` or do + `if quiz_data:` already treat an empty-questions dict as "no quiz" since + it is still truthy, so this never changes caller behaviour. """ result = await quiz_harness.run(topic, context_items, target_count=_DEFAULT_TARGET_COUNT) if not result["questions"]: - return None + logger.warning( + "Quiz generation for '%s' produced no questions: %s", topic, result["trace"] + ) return result @@ -116,6 +121,7 @@ def persist_quiz( order_index=order_idx, concept_tag=q_data.get("concept_tag"), difficulty=q_data.get("difficulty"), + source_ref=q_data.get("chunk_id"), ) db.add(question) db.flush() diff --git a/services/ai/app/services/quiz_harness.py b/services/ai/app/services/quiz_harness.py index c738a57..f80e886 100644 --- a/services/ai/app/services/quiz_harness.py +++ b/services/ai/app/services/quiz_harness.py @@ -5,6 +5,8 @@ import logging from typing import Any, Dict, List, Optional +import httpx + from app.services.qvac_structured import LlmDisabledError, StructuredGenerationError, generate_json logger = logging.getLogger(__name__) @@ -138,12 +140,14 @@ async def _generate_one( ) -> Optional[Dict[str, Any]]: """Generate one question for target, scoped to its single source chunk. - Returns None (rather than propagating) on StructuredGenerationError, so the - caller can treat a failed target as needing repair or being dropped. The - same applies if target["chunk_id"] does not match any context_items entry: - we must not fall back to a context-free generate_json call, since that - would silently produce an ungrounded question while still being tagged - with target["chunk_id"] on the way out. + Returns None (rather than propagating) on StructuredGenerationError or a + transport-level httpx.HTTPError (e.g. a connection error or a non-2xx + response that survived generate_json's own retries), so the caller can + treat a failed target as needing repair or being dropped. The same + applies if target["chunk_id"] does not match any context_items entry: we + must not fall back to a context-free generate_json call, since that would + silently produce an ungrounded question while still being tagged with + target["chunk_id"] on the way out. LlmDisabledError is not caught here and propagates to abort the run. """ item = next( @@ -170,7 +174,7 @@ async def _generate_one( generation_params={"temp": 0.3}, task_type="content_gen", ) - except StructuredGenerationError: + except (StructuredGenerationError, httpx.HTTPError): return None return {**result, "chunk_id": target["chunk_id"]} @@ -216,11 +220,12 @@ async def _critique_one( ) -> CritiqueVerdict: """Judge one generated question against the five quality criteria. - Fails closed: if the critic is unavailable (LlmDisabledError) or returns - an unparsable response (StructuredGenerationError), the question is - treated as a failure, not a pass — it is never silently accepted just - because we could not critique it. Mirrors - lesson_service._judge_groundedness's fail-closed pattern. + Fails closed: if the critic is unavailable (LlmDisabledError), returns an + unparsable response (StructuredGenerationError), or a transport-level + error occurs (httpx.HTTPError), the question is treated as a failure, not + a pass — it is never silently accepted just because we could not + critique it. Mirrors lesson_service._judge_groundedness's fail-closed + pattern. """ accepted_prompts = [q["prompt"] for q in accepted_questions] prompt = ( @@ -239,7 +244,7 @@ async def _critique_one( generation_params={"temp": 0.1}, task_type="quiz_critique", ) - except (LlmDisabledError, StructuredGenerationError): + except (LlmDisabledError, StructuredGenerationError, httpx.HTTPError): return {"passed": False, "failed_criteria": ["critic unavailable"]} failed_criteria = [name for name in _CRITIQUE_CRITERIA if not result.get(name, False)] @@ -262,6 +267,29 @@ def _source_text_for(target: PlanTarget, context_items: List[Dict[str, str]]) -> return item["text"] if item is not None else "" +def _rejection( + *, + stage: str, + chunk_id: Optional[str], + cognitive_level: Optional[str], + question_preview: Optional[str], + reasons: List[str], +) -> Dict[str, Any]: + """Build one trace["rejected"] entry in the single uniform shape used by + every rejection path (plan-stage failure, generation failure, or a + critique failure on either the initial or repair attempt), so a trace + consumer never has to branch on which keys a rejected entry happens to + carry. + """ + return { + "stage": stage, + "chunk_id": chunk_id, + "cognitive_level": cognitive_level, + "question_preview": question_preview, + "reasons": reasons, + } + + async def run( topic: str, context_items: List[Dict[str, str]], @@ -301,9 +329,12 @@ async def run( try: targets = await _plan_targets(context_items, target_count) trace["calls_used"] += 1 - except (LlmDisabledError, StructuredGenerationError) as exc: + except (LlmDisabledError, StructuredGenerationError, httpx.HTTPError) as exc: trace["calls_used"] += 1 - trace["rejected"].append({"stage": "plan", "reason": str(exc)}) + trace["rejected"].append(_rejection( + stage="plan", chunk_id=None, cognitive_level=None, + question_preview=None, reasons=[str(exc)], + )) trace["final_count"] = len(accepted) return {"questions": accepted, "trace": trace} @@ -314,12 +345,16 @@ async def run( try: question = await _generate_one(target, context_items) - except LlmDisabledError: + except (LlmDisabledError, StructuredGenerationError, httpx.HTTPError): question = None trace["calls_used"] += 1 if question is None: - trace["rejected"].append({"question_preview": None, "reasons": ["generation failed"]}) + trace["rejected"].append(_rejection( + stage="generate", chunk_id=target["chunk_id"], + cognitive_level=target["cognitive_level"], + question_preview=None, reasons=["generation failed"], + )) continue trace["generated"] += 1 @@ -334,16 +369,17 @@ async def run( feedback = ", ".join(verdict["failed_criteria"]) try: repaired_question = await _generate_one(target, context_items, feedback=feedback) - except LlmDisabledError: + except (LlmDisabledError, StructuredGenerationError, httpx.HTTPError): repaired_question = None trace["calls_used"] += 1 if repaired_question is None: # A None from a repair generate must not trigger an extra critique call. - trace["rejected"].append({ - "question_preview": question["prompt"] if question else None, - "reasons": verdict["failed_criteria"], - }) + trace["rejected"].append(_rejection( + stage="generate", chunk_id=target["chunk_id"], + cognitive_level=target["cognitive_level"], + question_preview=None, reasons=["generation failed"], + )) continue repaired_verdict = await _critique_one(repaired_question, source_text, accepted) @@ -353,10 +389,12 @@ async def run( accepted.append(repaired_question) trace["repaired"] += 1 else: - trace["rejected"].append({ - "question_preview": repaired_question["prompt"], - "reasons": repaired_verdict["failed_criteria"], - }) + trace["rejected"].append(_rejection( + stage="critique", chunk_id=target["chunk_id"], + cognitive_level=target["cognitive_level"], + question_preview=repaired_question["prompt"], + reasons=repaired_verdict["failed_criteria"], + )) trace["final_count"] = len(accepted) return {"questions": accepted, "trace": trace} diff --git a/services/ai/tests/unit/test_quiz_generation.py b/services/ai/tests/unit/test_quiz_generation.py index 469893a..d5e3f7a 100644 --- a/services/ai/tests/unit/test_quiz_generation.py +++ b/services/ai/tests/unit/test_quiz_generation.py @@ -73,12 +73,30 @@ async def test_generate_quiz_questions_delegates_to_harness(): @pytest.mark.asyncio -async def test_generate_quiz_questions_returns_none_when_harness_produces_nothing(): +async def test_generate_quiz_questions_returns_empty_dict_when_harness_produces_nothing(): + """The trace matters most exactly when questions is empty (issue #134), so + generate_quiz_questions must never collapse this case to None — callers + need the trace to see *why* generation produced nothing.""" with patch("app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock) as mock_run: mock_run.return_value = {"questions": [], "trace": {"final_count": 0}} result = await generate_quiz_questions("UTXO model", [{"label": "p1", "text": "...", "chunk_id": "c1"}]) - assert result is None + assert result == {"questions": [], "trace": {"final_count": 0}} + + +@pytest.mark.asyncio +async def test_generate_quiz_questions_logs_trace_when_no_questions(): + # Patch the module logger directly rather than relying on the stdlib + # logging/caplog machinery, which other tests in the suite can leave in + # a state (e.g. a disabled logger via alembic's fileConfig) unrelated to + # the behaviour under test here. + with patch("app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock) as mock_run, \ + patch("app.services.quiz_generation.logger") as mock_logger: + mock_run.return_value = {"questions": [], "trace": {"rejected": ["reason"]}} + await generate_quiz_questions("UTXO model", [{"label": "p1", "text": "...", "chunk_id": "c1"}]) + + mock_logger.warning.assert_called_once() + assert "UTXO model" in mock_logger.warning.call_args.args # --------------------------------------------------------------------------- @@ -106,6 +124,17 @@ def test_persist_quiz_creates_rows_with_concept_tags(): assert len(correct) == 1 +def test_persist_quiz_persists_source_ref_from_chunk_id(): + db, course, chapter = _make_db() + questions_with_chunk_id = [{**_QUESTIONS[0], "chunk_id": "chunk-42"}] + quiz = persist_quiz( + db, questions_with_chunk_id, scope=QuizScope.COURSE, title="Quiz: UTXO", course_id=course.id, + ) + + question = db.query(Question).filter(Question.quiz_id == quiz.id).first() + assert question.source_ref == "chunk-42" + + def test_persist_quiz_returns_none_for_empty_questions(): db, course, chapter = _make_db() quiz = persist_quiz(db, [], scope=QuizScope.COURSE, title="Empty", course_id=course.id) diff --git a/services/ai/tests/unit/test_quiz_harness.py b/services/ai/tests/unit/test_quiz_harness.py index 3fa9b41..57a4b8a 100644 --- a/services/ai/tests/unit/test_quiz_harness.py +++ b/services/ai/tests/unit/test_quiz_harness.py @@ -1,3 +1,4 @@ +import httpx import pytest from unittest.mock import AsyncMock, patch @@ -126,6 +127,18 @@ async def test_generate_one_returns_none_for_unresolvable_chunk_id(): mock_gj.assert_not_called() +@pytest.mark.asyncio +async def test_generate_one_returns_none_on_httpx_transport_error(): + """A transient transport error (e.g. a connection error surviving + generate_json's own retries) must drop this one target, not propagate — + same contract as StructuredGenerationError.""" + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = httpx.ConnectError("boom") + question = await _generate_one(_TARGET, _CONTEXT) + + assert question is None + + @pytest.mark.asyncio async def test_critique_one_passes_when_all_criteria_met(): with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: @@ -194,6 +207,18 @@ async def test_critique_one_fails_closed_on_structured_generation_error(): assert verdict["failed_criteria"] == ["critic unavailable"] +@pytest.mark.asyncio +async def test_critique_one_fails_closed_on_httpx_transport_error(): + """Same fail-closed contract as LlmDisabledError/StructuredGenerationError: + a transient transport error must never be silently treated as a pass.""" + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = httpx.ConnectError("boom") + verdict = await _critique_one(_QUESTION, "UTXO text", []) + + assert verdict["passed"] is False + assert verdict["failed_criteria"] == ["critic unavailable"] + + @pytest.mark.asyncio async def test_run_all_pass_returns_every_question_with_no_repairs(): with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ @@ -273,3 +298,74 @@ async def test_run_llm_disabled_mid_generate_drops_that_target_and_continues(): assert len(result["questions"]) == 1 assert len(result["trace"]["rejected"]) == 1 + + +@pytest.mark.asyncio +async def test_run_generate_one_raising_llm_disabled_drops_that_target_and_continues(): + """A per-target LlmDisabledError raised by _generate_one itself (as + opposed to _generate_one returning None, covered above) must be caught at + the call site and treated like any other per-target failure — drop and + continue — never propagate out of run(). Only a plan-stage failure aborts + the whole run, and even that does so gracefully (see + test_run_llm_disabled_at_plan_stage_returns_no_questions).""" + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] * 2 + mock_gen.side_effect = [dict(_QUESTION), LlmDisabledError("disabled mid-run")] + mock_crit.return_value = {"passed": True, "failed_criteria": []} + + result = await run("UTXO model", _CONTEXT, target_count=2) + + assert len(result["questions"]) == 1 + assert len(result["trace"]["rejected"]) == 1 + assert result["trace"]["rejected"][0]["stage"] == "generate" + + +@pytest.mark.asyncio +async def test_run_repair_generate_returns_none_drops_question_without_a_third_pass(): + """When the REPAIR generate call (not the initial one) returns None, the + question must be dropped and no second repair attempted — _generate_one + is called exactly twice (initial + one repair) and _critique_one exactly + once (the initial critique; no critique call for the failed repair).""" + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] + mock_gen.side_effect = [dict(_QUESTION), None] # initial generate ok, repair generate fails + mock_crit.return_value = {"passed": False, "failed_criteria": ["entailed"]} + + result = await run("UTXO model", _CONTEXT, target_count=1) + + assert result["questions"] == [] + assert mock_gen.await_count == 2 + assert mock_crit.await_count == 1 + assert len(result["trace"]["rejected"]) == 1 + assert result["trace"]["rejected"][0]["stage"] == "generate" + assert result["trace"]["rejected"][0]["reasons"] == ["generation failed"] + + +@pytest.mark.asyncio +async def test_run_end_to_end_with_generate_json_mocked_directly(): + """Wires the real _plan_targets/_generate_one/_critique_one together — not + just run()'s own orchestration logic in isolation — by mocking + generate_json itself with one plan response, one generate response, and + one passing critique response, in call order, for a single target.""" + plan_response = {"targets": [{"passage_index": 1, "cognitive_level": "recall"}]} + generate_response = dict(_QUESTION) + critique_response = { + "entailed": True, "distractors_plausible": True, "single_defensible": True, + "requires_course_material": True, "duplicate": False, + } + + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = [plan_response, generate_response, critique_response] + + result = await run("UTXO model", _CONTEXT, target_count=1) + + assert mock_gj.await_count == 3 + assert len(result["questions"]) == 1 + assert result["questions"][0]["chunk_id"] == "c1" + assert result["trace"]["final_count"] == 1 + assert result["trace"]["repaired"] == 0 + assert result["trace"]["rejected"] == [] From 59079fa689c0422fd06d962993d996fdb7f0eb7f Mon Sep 17 00:00:00 2001 From: longobucco Date: Thu, 5 Nov 2026 22:58:10 +0100 Subject: [PATCH 11/12] docs(rag): correct Task 5's plan prose to match its own verbatim tests Two sentences in the Task 5 section described behavior (re-raising LlmDisabledError at the plan stage, repairing a None generation result) that contradicted the plan's own Global Constraints and verbatim test code. Both were authoring errors, ruled on and corrected during SDD execution per the ledger at .superpowers/sdd/2026-11-05-quiz-agent-harness/progress.md. Refs #134 --- .../plans/2026-11-05-quiz-agent-harness.md | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/docs/superpowers/plans/2026-11-05-quiz-agent-harness.md b/docs/superpowers/plans/2026-11-05-quiz-agent-harness.md index 854e380..f604654 100644 --- a/docs/superpowers/plans/2026-11-05-quiz-agent-harness.md +++ b/docs/superpowers/plans/2026-11-05-quiz-agent-harness.md @@ -711,11 +711,16 @@ and produces the structured trace. `{"targets_planned": int, "generated": int, "rejected": List[Dict[str, Any]], "repaired": int, "final_count": int, "calls_used": int, "calls_budget": int}`; `async def run(topic: str, context_items: List[Dict[str, str]], target_count: int = DEFAULT_TARGET_COUNT) -> - HarnessResult`. Never raises for `StructuredGenerationError`/per-target `LlmDisabledError`; - re-raises `LlmDisabledError` only if it happens at the plan stage (nothing has been produced yet, - so there is nothing to degrade to — matches `generate_quiz_questions`'s existing contract of - letting `LlmDisabledError` propagate, see `lesson_service.process_lesson`'s `except - LlmDisabledError: raise`). Consumed by Task 7's `quiz_generation.generate_quiz_questions`. + HarnessResult`. Never raises — `LlmDisabledError` at the plan stage is caught the same as + `StructuredGenerationError` and turned into a graceful empty `HarnessResult` with a reason in + the trace, per this plan's own Global Constraints ("never raise out of the harness"). **Correction + (ruled during Task 5's implementation, see SDD ledger):** an earlier draft of this paragraph said + `run()` should re-raise `LlmDisabledError` at the plan stage to match + `generate_quiz_questions`'s pre-harness contract; that contradicted this task's own verbatim + test (`test_run_llm_disabled_at_plan_stage_returns_no_questions`, which asserts a graceful + return, not `pytest.raises`) and was an authoring error, not a deliberate choice — the graceful + path is what's implemented and is correct. Consumed by Task 7's + `quiz_generation.generate_quiz_questions`. - [ ] **Step 1: Write the failing tests** @@ -821,13 +826,19 @@ async def run( ``` Implementer writes the body following the design doc's orchestration exactly: call -`_plan_targets`, letting `LlmDisabledError`/`StructuredGenerationError` from it end the run +`_plan_targets`, catching `LlmDisabledError`/`StructuredGenerationError` from it to end the run immediately with `{"questions": [], "trace": {..., "rejected": [{"stage": "plan", "reason": -str(exc)}]}}`; otherwise iterate the returned targets, for each one calling `_generate_one` then (if -it returned a question) `_critique_one` against the already-`accepted` list; on a failed critique -or a `None` generation result, attempt exactly one repair (`_generate_one(target, context_items, -feedback=...)` built from `failed_criteria`, then `_critique_one` again if the repair generated a -question); append to `accepted` on any pass, otherwise to `trace["rejected"]` with a +str(exc)}]}}` (**correction, ruled during Task 5** — catch, do not propagate; see the Interfaces +block above); otherwise iterate the returned targets, for each one calling `_generate_one` then (if +it returned a question) `_critique_one` against the already-`accepted` list; on a failed critique, +attempt exactly one repair (`_generate_one(target, context_items, feedback=...)` built from +`failed_criteria`, then `_critique_one` again if the repair generated a +question); a `None` from the *initial* generate call (not the repair attempt) is dropped +immediately with no repair — repair is reserved for a question that generated but failed critique +(**correction, ruled during Task 5** — the verbatim test +`test_run_llm_disabled_mid_generate_drops_that_target_and_continues` requires this: it mocks +exactly 2 `_generate_one` calls for 2 targets, which a repair-on-None path would exceed); append +to `accepted` on any pass, otherwise to `trace["rejected"]` with a `question_preview` (the rejected question's `"prompt"` if one exists, else `None`) and `"reasons"`; increment `trace["calls_used"]` once per `_plan_targets`/`_generate_one`/`_critique_one` call actually made (a `None` from a previous repair generate must not trigger an extra critique call — From b850bf20bf27d30dcea3efb6c0260e6a53d1f358 Mon Sep 17 00:00:00 2001 From: longobucco Date: Fri, 6 Nov 2026 13:56:27 +0100 Subject: [PATCH 12/12] fix(rag): propagate LlmDisabledError and dedupe quiz plan passages LlmDisabledError was caught and swallowed at every stage of the quiz harness, so a disabled LLM silently produced zero questions instead of reporting the outage to the caller. Only StructuredGenerationError and httpx.HTTPError are now caught for per-question fallbacks; LlmDisabledError propagates from planning, generation, repair, and critique. The plan stage could also assign the same passage to multiple targets when the model repeated or omitted a passage_index, triggering the duplicate -> repair -> duplicate cycle noted in the PR's review-watch section. _plan_targets now tracks used indices and fills gaps round-robin over unused passages, reusing one only once all have been assigned. Moved the design doc out of docs/superpowers/ into docs/ alongside the other specs, and dropped the now-stale implementation plan, per review. Refs #134 --- docs/overview.md | 1 + ...design.md => quiz-agent-harness-design.md} | 10 +- .../plans/2026-11-05-quiz-agent-harness.md | 1069 ----------------- services/ai/app/api/quizzes_api.py | 2 +- services/ai/app/services/quiz_generation.py | 5 +- services/ai/app/services/quiz_harness.py | 78 +- services/ai/tests/unit/test_quiz_harness.py | 75 +- 7 files changed, 111 insertions(+), 1129 deletions(-) rename docs/{superpowers/specs/2026-11-05-quiz-agent-harness-design.md => quiz-agent-harness-design.md} (96%) delete mode 100644 docs/superpowers/plans/2026-11-05-quiz-agent-harness.md diff --git a/docs/overview.md b/docs/overview.md index bf9982b..a16f43c 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -191,6 +191,7 @@ follow, and are not strictly ordered relative to each other. |---|---| | [`overview.md`](overview.md) | This document — vision, scope, architecture | | [`specifications.md`](specifications.md) | Functional requirements with implementation status | +| [`quiz-agent-harness-design.md`](quiz-agent-harness-design.md) | Design for bounded, grounded quiz generation | | [`agent-memory-plan.md`](agent-memory-plan.md) | Inference ladder, hardware tiers, and persistent student-memory roadmap | | [`../AGENTS.md`](../AGENTS.md) | Contribution workflow, branching, PR and merge rules | | [`../README.md`](../README.md) | Setup, configuration, testing, troubleshooting | diff --git a/docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md b/docs/quiz-agent-harness-design.md similarity index 96% rename from docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md rename to docs/quiz-agent-harness-design.md index 33bbfb2..f2830b6 100644 --- a/docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md +++ b/docs/quiz-agent-harness-design.md @@ -25,9 +25,8 @@ levels to test, generate one question per target, critique each against explicit the single old prompt happened to sample it. - A structured trace (targets planned, generated, rejected with reasons, repaired, final count) returned to the caller, so quality issues are debuggable without re-reading logs. -- Degrade honestbly when generation is unavailable: fewer questions or a clear error, never - malformed output (matches existing `generate_quiz_questions` contract of returning `None`/fewer - items rather than raising on the caller). +- Degrade honestly when an individual generation or critique request fails: return fewer questions, + never malformed output. A disabled LLM is an operational error and propagates to the caller. ## Non-goals / anti-goals @@ -236,9 +235,8 @@ module, i.e. `app.services.quiz_harness.generate_json`, with `AsyncMock`): the extra repair calls. - Retry limit reached: repair critique also fails; assert the question is dropped, appears in `trace.rejected` with its final reasons, and the loop does not attempt a third pass. -- LLM disabled: `generate_json` raises `LlmDisabledError` at the plan stage; assert the harness - returns no questions with a clear reason rather than raising, matching - `generate_quiz_questions`'s existing `None`-on-failure contract. +- LLM disabled: `generate_json` raises `LlmDisabledError`; assert the harness propagates it from + planning, generation, and critique so the caller can report the disabled service. - Persistence path (`persist_quiz`) is untested here since it is unchanged — existing `test_quiz_generation.py` coverage stands. diff --git a/docs/superpowers/plans/2026-11-05-quiz-agent-harness.md b/docs/superpowers/plans/2026-11-05-quiz-agent-harness.md deleted file mode 100644 index f604654..0000000 --- a/docs/superpowers/plans/2026-11-05-quiz-agent-harness.md +++ /dev/null @@ -1,1069 +0,0 @@ -# Quiz Agent Harness Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace quiz generation's single schema-validated `/generate_json` call with a bounded -plan → generate → critique → repair loop, so quiz questions are checked against explicit quality -criteria instead of only being well-formed. - -**Architecture:** A new `app/services/quiz_harness.py` module owns the four-stage loop and is -called by `quiz_generation.generate_quiz_questions` (unchanged public signature, delegates -internally). Both existing callers (course builder, study page) feed it through the existing -`context_items` convention, extended with an explicit `chunk_id` key. Persistence -(`quiz_generation.persist_quiz`) is untouched. - -**Tech Stack:** Python 3.11, FastAPI, SQLAlchemy, `qvac_structured.generate_json` (async, schema- -validated JSON over HTTP), pytest + `unittest.mock.AsyncMock`. - -**Spec:** [`docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md`](../specs/2026-11-05-quiz-agent-harness-design.md) - -## Global Constraints - -- Every returned question carries a resolvable `chunk_id` (issue #134 acceptance criterion). -- Hard cap on LLM calls, no unbounded loop — enforced by construction: at most - `1 (plan) + N (generate) + N (critique) + N (repair-generate) + N (repair-critique) = 4N + 1` - calls for `N` targets, with exactly one repair round per rejected question. -- Degrade honestly: on `LlmDisabledError`/`StructuredGenerationError`, return fewer questions (or - none) with a reason in the trace — never raise out of the harness, never return malformed output. -- `generate_json` is always imported and called as `from app.services.qvac_structured import - generate_json` inside the module that calls it, and tests patch it at that module's path (e.g. - `app.services.quiz_harness.generate_json`), per the existing convention in - `quiz_generation.py`/`lesson_service.py`. -- No new background job / `GenerationRun` row for quizzes — stays synchronous, same shape as - today's `POST /courses/{course_id}/quizzes/generate`. -- No frontend changes in this plan — the `trace` field rides in the existing API response only. - -## Review Focus - -- Fewer distinct chunks in `context_items` than the target question count — the plan stage must - reuse chunks (not crash or return fewer targets than requested). Covered in Task 2. -- A critique response missing one of the required boolean fields (worker validated the schema but - a key check in Python code must not `KeyError`) — use `.get(..., False)` defensively. Covered in - Task 4. -- Duplicate questions across the same run — the critique call for question *k* must see the - already-accepted questions 1..k-1 and flag a near-duplicate, not silently accept it. Covered in - Task 4. -- A repair-stage call (generate or critique) itself raising `LlmDisabledError` mid-run — the run - must drop that one target and continue/finish rather than propagating the exception out of - `run()`. Covered in Task 5. -- `target_count` larger than the number of distinct `context_items` entries — round-robin target - assignment must not raise an `IndexError` and must still produce valid, distinct questions. - Covered in Task 2 (planning) and Task 5 (full run). - ---- - -## Task 1: Fix `context_items` so every caller's entries carry a `chunk_id` - -The study-page caller currently builds `context_items` from `result.citations` -(`{label, text}`), which has no chunk id anywhere on it — only `result.evidence_pack.chunks[i] -.chunk_id` does. The course-builder caller already has a chunk id, just under the `label` key. Both -need an explicit `"chunk_id"` key before the harness (Task 2+) can plan against it. - -**Files:** -- Modify: `app/services/lesson_service.py:189-195` (`_load_context`) -- Modify: `app/api/quizzes_api.py:159-172` (`generate_quiz`) -- Test: `tests/unit/test_lesson_service.py:140-151` (`test_load_context_returns_items`) -- Test: `tests/integration/test_quizzes_certificates_api.py:14,38-42,58-105` - -**Interfaces:** -- Produces: every `context_items` entry downstream now has keys `{"label": str, "text": str, - "chunk_id": str}` — this is what Task 2's `_plan_targets` and Task 3's `_generate_one` consume. - -- [ ] **Step 1: Write the failing test for `_load_context`** - -In `tests/unit/test_lesson_service.py`, extend `test_load_context_returns_items` with: -```python - assert items[0]["chunk_id"] == "p1" -``` - -- [ ] **Step 2: Run it to verify it fails** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_lesson_service.py::test_load_context_returns_items -v` -Expected: FAIL — `KeyError: 'chunk_id'` - -- [ ] **Step 3: Add `chunk_id` to `_load_context`'s output** - -In `app/services/lesson_service.py:193`, change: -```python - context_items.append({"label": cid, "text": rows[cid].text}) -``` -to also set `"chunk_id": cid` (same value as `label` here — `label` already doubles as the chunk id -for this caller; this just makes the key explicit and uniform with the study-page caller after -Step 6). - -- [ ] **Step 4: Run it to verify it passes** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_lesson_service.py::test_load_context_returns_items -v` -Expected: PASS - -- [ ] **Step 5: Write the failing test for `quizzes_api.generate_quiz`'s context_items** - -In `tests/integration/test_quizzes_certificates_api.py`, the module-level `_RETRIEVE_RESULT` -currently only sets `citations`. Replace it with a `DispatchResult` that also carries an -`evidence_pack`: -```python -from app.schemas.evidence_pack import CitationAnchor, EvidenceChunk, EvidencePack - -_RETRIEVE_RESULT = DispatchResult( - answer="", - citations=[SourceChunk(snippet="A UTXO is an unspent transaction output.", score=0.9, label="p1")], - retrieval_used=True, - evidence_pack=EvidencePack( - query="UTXO model basics", - action="retrieve", - chunks=[ - EvidenceChunk( - chunk_id="chunk-1", - text="A UTXO is an unspent transaction output.", - score=0.9, - anchor=CitationAnchor( - doc_id="doc-1", doc_name="p1", chunk_id="chunk-1", chunk_type="text", - ), - ) - ], - total_candidates=1, - ordering=[0], - deduped_passages=["A UTXO is an unspent transaction output."], - ), -) -``` -Add a new test `test_generate_quiz_uses_evidence_pack_chunk_id` that patches -`app.services.quiz_generation.generate_json` (unchanged patch target for this task — Task 7 -switches this to `quiz_harness.generate_json`), asserts the mock's `context` kwarg on its first -call contains a dict with `"chunk_id": "chunk-1"`. - -- [ ] **Step 6: Run it to verify it fails** - -Run: `cd services/ai && uv run --no-sync pytest tests/integration/test_quizzes_certificates_api.py::TestGenerateQuiz::test_generate_quiz_uses_evidence_pack_chunk_id -v` -Expected: FAIL — context items built from `citations` have no `chunk_id`. - -- [ ] **Step 7: Build `context_items` from `result.evidence_pack.chunks` in `quizzes_api.py`** - -In `app/api/quizzes_api.py:159-172`, replace the `result.citations`-based comprehension: -```python - if result.evidence_pack is None or not result.evidence_pack.chunks: - raise ValidationError_( - "No source material found for this topic in the course. Try a broader query." - ) - - context_items = [ - {"label": c.anchor.doc_name, "text": c.text, "chunk_id": c.chunk_id} - for c in result.evidence_pack.chunks - ] -``` -(drops the old `if not context_items: raise ...` check, replaced by the `evidence_pack` guard above -it, since an empty-chunks pack and a `None` pack are now both "no source material"). - -- [ ] **Step 8: Run it to verify it passes, then run the rest of the quiz integration tests** - -Run: `cd services/ai && uv run --no-sync pytest tests/integration/test_quizzes_certificates_api.py -v` -Expected: all PASS, including `test_generate_quiz_422_when_no_source_material` (its -`DispatchResult` has no `evidence_pack` set, so the `None` guard fires) and -`test_generate_quiz_persists_to_db` (uses the now-fixed `_RETRIEVE_RESULT`). - -- [ ] **Step 9: Commit** - -```bash -git add app/services/lesson_service.py app/api/quizzes_api.py tests/unit/test_lesson_service.py tests/integration/test_quizzes_certificates_api.py -git commit -m "fix(rag): carry chunk_id through quiz context_items for both callers" -``` - ---- - -## Task 2: `quiz_harness.py` — plan stage - -Selects which passages to test and at what cognitive level, from the evidence already in -`context_items` — no new retrieval call. The model is asked to refer to passages by their 1-based -position (matching the `[N]` convention `generate-json.js`'s `buildContextBlock` already uses), not -by `chunk_id` text, so an invented or out-of-range reference can be corrected deterministically -instead of trusted. - -**Files:** -- Create: `app/services/quiz_harness.py` -- Create: `tests/unit/test_quiz_harness.py` -- Modify: `app/services/qvac_router.py:33-42` (`LADDER_ROUTING`) -- Modify: `tests/unit/test_qvac_router.py` - -**Interfaces:** -- Consumes: `app.services.qvac_structured.generate_json(prompt, schema, *, context=None, - system_prompt=None, generation_params=None, task_type=None) -> Dict[str, Any]`; - `app.services.qvac_structured.LlmDisabledError`, `StructuredGenerationError`. -- Produces: `PlanTarget = Dict[str, str]` with keys `{"chunk_id": str, "cognitive_level": str}`; - `async def _plan_targets(context_items: List[Dict[str, str]], target_count: int) -> - List[PlanTarget]` — always returns exactly `target_count` targets (round-robin over - `context_items` filling in for anything missing/invalid from the model), or raises - `LlmDisabledError`/`StructuredGenerationError` if the `generate_json` call itself fails. Consumed - by Task 5's `run()`. - -- [ ] **Step 1: Write the failing tests** - -```python -# tests/unit/test_quiz_harness.py -import pytest -from unittest.mock import AsyncMock, patch - -from app.services.qvac_structured import LlmDisabledError, StructuredGenerationError -from app.services.quiz_harness import _plan_targets - -_CONTEXT = [ - {"label": "p1", "text": "UTXO text", "chunk_id": "c1"}, - {"label": "p2", "text": "PoW text", "chunk_id": "c2"}, -] - - -@pytest.mark.asyncio -async def test_plan_targets_maps_passage_index_to_chunk_id(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = {"targets": [ - {"passage_index": 2, "cognitive_level": "application"}, - {"passage_index": 1, "cognitive_level": "recall"}, - ]} - targets = await _plan_targets(_CONTEXT, target_count=2) - - assert targets == [ - {"chunk_id": "c2", "cognitive_level": "application"}, - {"chunk_id": "c1", "cognitive_level": "recall"}, - ] - - -@pytest.mark.asyncio -async def test_plan_targets_corrects_invalid_passage_index(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = {"targets": [{"passage_index": 99, "cognitive_level": "recall"}]} - targets = await _plan_targets(_CONTEXT, target_count=1) - - assert targets[0]["chunk_id"] in {"c1", "c2"} - - -@pytest.mark.asyncio -async def test_plan_targets_pads_when_model_returns_too_few(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = {"targets": [{"passage_index": 1, "cognitive_level": "recall"}]} - targets = await _plan_targets(_CONTEXT, target_count=3) - - assert len(targets) == 3 - - -@pytest.mark.asyncio -async def test_plan_targets_reuses_chunks_when_target_count_exceeds_available(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = {"targets": []} - targets = await _plan_targets(_CONTEXT, target_count=5) - - assert len(targets) == 5 - assert {t["chunk_id"] for t in targets} == {"c1", "c2"} - - -@pytest.mark.asyncio -async def test_plan_targets_propagates_llm_disabled(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.side_effect = LlmDisabledError("disabled") - with pytest.raises(LlmDisabledError): - await _plan_targets(_CONTEXT, target_count=1) -``` - -- [ ] **Step 2: Run to verify they fail** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -v` -Expected: FAIL — `ModuleNotFoundError: No module named 'app.services.quiz_harness'` - -- [ ] **Step 3: Create `app/services/quiz_harness.py` with the plan stage** - -```python -"""Agent harness for quiz generation — plan -> generate -> critique -> repair. - -See docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md. -""" -import logging -from typing import Any, Dict, List - -from app.services.qvac_structured import generate_json - -logger = logging.getLogger(__name__) - -PlanTarget = Dict[str, str] - -_COGNITIVE_LEVELS = ("recall", "application", "analysis") - -PLAN_PROMPT_VERSION = "v1" - -_PLAN_SCHEMA: Dict[str, Any] = { - "type": "object", - "required": ["targets"], - "properties": { - "targets": { - "type": "array", - "items": { - "type": "object", - "required": ["passage_index", "cognitive_level"], - "properties": { - "passage_index": {"type": "integer"}, - "cognitive_level": {"type": "string", "enum": list(_COGNITIVE_LEVELS)}, - }, - }, - } - }, -} - -_PLAN_SYSTEM = ( - "You are planning a quiz. Given numbered source passages, select which passages to " - "test and at what cognitive level, so coverage spans distinct passages and levels " - "rather than clustering on one passage.\n" - "Refer to each passage by the 1-based index shown before it (e.g. passage [2]).\n" - "cognitive_level: 'recall' (states a fact), 'application' (uses a concept in a new " - "situation), or 'analysis' (compares or explains relationships between concepts)." -) - - -async def _plan_targets(context_items: List[Dict[str, str]], target_count: int) -> List[PlanTarget]: - """Select target_count (chunk_id, cognitive_level) pairs from context_items. - - Always returns exactly target_count targets. An invalid or missing - passage_index from the model is corrected by round-robin assignment over - context_items rather than trusted — the model never needs to reproduce a - chunk_id string correctly. - """ - ... -``` - -The implementer writes the body: one `generate_json(prompt, _PLAN_SCHEMA, context=context_items, -system_prompt=_PLAN_SYSTEM, generation_params={"temp": 0.2}, task_type="quiz_plan")` call (prompt -text states `target_count` and lists the available passage count), then builds the -`target_count`-length result by walking `result.get("targets", [])`: for each raw target, resolve -`passage_index` to `context_items[idx - 1]["chunk_id"]` when `1 <= idx <= len(context_items)`, -otherwise assign `context_items[i % len(context_items)]["chunk_id"]` (round-robin by position in -the output list); validate `cognitive_level` against `_COGNITIVE_LEVELS`, defaulting to `"recall"`. -Stop once `target_count` targets are collected; if the model returned fewer, keep round-robin- -filling until the count is reached. Let `LlmDisabledError`/`StructuredGenerationError` from -`generate_json` propagate uncaught — Task 5's `run()` catches them at the call site. - -- [ ] **Step 4: Run to verify they pass** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -v` -Expected: PASS - -- [ ] **Step 5: Register `quiz_plan` in `qvac_router.LADDER_ROUTING`** - -In `app/services/qvac_router.py:33-42`, add `"quiz_plan": ("server", SERVER_MODEL),` to the -`LADDER_ROUTING` dict (alongside `"judge"`/`"content_gen"` — explicit registration rather than -relying on the unknown-task_type fallback). - -- [ ] **Step 6: Add a parametrize case to `test_qvac_router.py`** - -In `tests/unit/test_qvac_router.py`, add `("quiz_plan", qvac_router.SERVER_MODEL)` to the -`test_server_task_types_route_to_server` parametrize list. - -- [ ] **Step 7: Run the router tests to verify they pass** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_qvac_router.py -v` -Expected: PASS - -- [ ] **Step 8: Commit** - -```bash -git add app/services/quiz_harness.py tests/unit/test_quiz_harness.py app/services/qvac_router.py tests/unit/test_qvac_router.py -git commit -m "feat(rag): add plan stage to the quiz agent harness" -``` - ---- - -## Task 3: `quiz_harness.py` — generate stage - -One `generate_json` call per target, pinned to that target's single chunk so the model cannot pull -material from elsewhere in the evidence pack. Reuses the `content_gen` task type (same model as -today's quiz generation — no new routing entry needed). - -**Files:** -- Modify: `app/services/quiz_harness.py` -- Modify: `tests/unit/test_quiz_harness.py` - -**Interfaces:** -- Consumes: `PlanTarget` from Task 2. -- Produces: `async def _generate_one(target: PlanTarget, context_items: List[Dict[str, str]], *, - feedback: Optional[str] = None) -> Optional[Dict[str, Any]]` — returns a question dict with keys - `{"prompt", "options", "correct_key", "concept_tag", "difficulty", "chunk_id"}`, or `None` if - generation failed (`StructuredGenerationError`/`LlmDisabledError` caught here, not propagated — - the caller treats `None` as "this target needs repair or must be dropped"). Consumed by Task 5's - `run()` and by Task 4's critique tests (as critique input). - -- [ ] **Step 1: Write the failing tests** - -```python -_TARGET = {"chunk_id": "c1", "cognitive_level": "recall"} - -_QUESTION = { - "prompt": "What is a UTXO?", - "options": [ - {"key": "A", "text": "An unspent transaction output"}, - {"key": "B", "text": "A mining reward"}, - {"key": "C", "text": "A block header"}, - {"key": "D", "text": "A wallet address"}, - ], - "correct_key": "A", - "concept_tag": "utxo-model", - "difficulty": "beginner", -} - - -@pytest.mark.asyncio -async def test_generate_one_attaches_chunk_id(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = dict(_QUESTION) - question = await _generate_one(_TARGET, _CONTEXT) - - assert question["chunk_id"] == "c1" - assert question["prompt"] == "What is a UTXO?" - - -@pytest.mark.asyncio -async def test_generate_one_scopes_context_to_the_target_chunk(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = dict(_QUESTION) - await _generate_one(_TARGET, _CONTEXT) - - call_context = mock_gj.call_args.kwargs["context"] - assert call_context == [{"label": "p1", "text": "UTXO text", "chunk_id": "c1"}] - - -@pytest.mark.asyncio -async def test_generate_one_includes_feedback_in_prompt_when_repairing(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = dict(_QUESTION) - await _generate_one(_TARGET, _CONTEXT, feedback="rejected: distractors too easy") - - prompt_arg = mock_gj.call_args.args[0] - assert "rejected: distractors too easy" in prompt_arg - - -@pytest.mark.asyncio -async def test_generate_one_returns_none_on_structured_generation_error(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.side_effect = StructuredGenerationError("bad schema") - question = await _generate_one(_TARGET, _CONTEXT) - - assert question is None -``` - -- [ ] **Step 2: Run to verify they fail** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -k generate_one -v` -Expected: FAIL — `ImportError: cannot import name '_generate_one'` - -- [ ] **Step 3: Implement `_generate_one` in `app/services/quiz_harness.py`** - -Add schema/prompt constants: -```python -GENERATE_PROMPT_VERSION = "v1" - -_GENERATE_SCHEMA: Dict[str, Any] = { - "type": "object", - "required": ["prompt", "options", "correct_key", "concept_tag", "difficulty"], - "properties": { - "prompt": {"type": "string"}, - "options": { - "type": "array", - "minItems": 4, - "maxItems": 4, - "items": { - "type": "object", - "required": ["key", "text"], - "properties": { - "key": {"type": "string", "enum": ["A", "B", "C", "D"]}, - "text": {"type": "string"}, - }, - }, - }, - "correct_key": {"type": "string", "enum": ["A", "B", "C", "D"]}, - "concept_tag": {"type": "string"}, - "difficulty": {"type": "string", "enum": ["beginner", "intermediate", "advanced"]}, - }, -} - -_GENERATE_SYSTEM = ( - "You are an expert educator at BitPolito Academy creating one assessment question " - "from a single source passage.\n" - "RULES:\n" - "1. The question must test the cognitive level given in the task (recall / application " - "/ analysis), based ONLY on the provided passage.\n" - "2. All 4 options (A-D) must be plausible — wrong options should be conceptually close " - "but demonstrably wrong from the passage.\n" - "3. Exactly one option must be correct and directly supported by the passage.\n" - "4. Use exact technical terminology from the passage.\n" - "5. concept_tag: a short kebab-case slug for the concept tested.\n" - "6. difficulty: beginner / intermediate / advanced." -) - - -async def _generate_one( - target: PlanTarget, - context_items: List[Dict[str, str]], - *, - feedback: Optional[str] = None, -) -> Optional[Dict[str, Any]]: - ... -``` - -Implementer writes the body: find the single entry in `context_items` whose `"chunk_id"` matches -`target["chunk_id"]` and pass it as `context=[that_item]` (a list of exactly one); build the prompt -from `target["cognitive_level"]` and, when `feedback` is given, append it verbatim (e.g. `f"Note: -the previous attempt was rejected — {feedback}"`); call `generate_json(prompt, _GENERATE_SCHEMA, -context=[item], system_prompt=_GENERATE_SYSTEM, generation_params={"temp": 0.3}, -task_type="content_gen")`; on success, return the result dict with `"chunk_id": -target["chunk_id"]` merged in; catch `StructuredGenerationError` and return `None` (do not catch -`LlmDisabledError` — let it propagate, matching the plan stage's contract, since a disabled LLM -should abort the run, not silently drop one target). - -- [ ] **Step 4: Run to verify they pass** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -k generate_one -v` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add app/services/quiz_harness.py tests/unit/test_quiz_harness.py -git commit -m "feat(rag): add generate stage to the quiz agent harness" -``` - ---- - -## Task 4: `quiz_harness.py` — critique stage - -One call per generated question, checking it against the five criteria from issue #134. Fails -closed on an unavailable critic (treated as a failure, not a pass). - -**Files:** -- Modify: `app/services/quiz_harness.py` -- Modify: `tests/unit/test_quiz_harness.py` -- Modify: `app/services/qvac_router.py:33-42` -- Modify: `tests/unit/test_qvac_router.py` - -**Interfaces:** -- Consumes: a question dict from Task 3's `_generate_one`. -- Produces: `CritiqueVerdict = Dict[str, Any]` with keys `{"passed": bool, "failed_criteria": - List[str]}`; `async def _critique_one(question: Dict[str, Any], source_text: str, - accepted_questions: List[Dict[str, Any]]) -> CritiqueVerdict`. Never raises — fails closed. - Consumed by Task 5's `run()`. - -- [ ] **Step 1: Write the failing tests** - -```python -@pytest.mark.asyncio -async def test_critique_one_passes_when_all_criteria_met(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = { - "entailed": True, "distractors_plausible": True, "single_defensible": True, - "requires_course_material": True, "duplicate": False, - } - verdict = await _critique_one(_QUESTION, "UTXO text", []) - - assert verdict == {"passed": True, "failed_criteria": []} - - -@pytest.mark.asyncio -async def test_critique_one_reports_each_failed_criterion(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = { - "entailed": False, "distractors_plausible": True, "single_defensible": True, - "requires_course_material": True, "duplicate": True, - } - verdict = await _critique_one(_QUESTION, "UTXO text", []) - - assert verdict["passed"] is False - assert set(verdict["failed_criteria"]) == {"entailed", "duplicate"} - - -@pytest.mark.asyncio -async def test_critique_one_handles_missing_fields_defensively(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = {"entailed": True} # worker returned a partial object - verdict = await _critique_one(_QUESTION, "UTXO text", []) - - assert verdict["passed"] is False - assert "distractors_plausible" in verdict["failed_criteria"] - - -@pytest.mark.asyncio -async def test_critique_one_passes_accepted_questions_as_duplicate_context(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.return_value = { - "entailed": True, "distractors_plausible": True, "single_defensible": True, - "requires_course_material": True, "duplicate": False, - } - await _critique_one(_QUESTION, "UTXO text", [{"prompt": "Earlier question?"}]) - - prompt_arg = mock_gj.call_args.args[0] - assert "Earlier question?" in prompt_arg - - -@pytest.mark.asyncio -async def test_critique_one_fails_closed_on_llm_disabled(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.side_effect = LlmDisabledError("disabled") - verdict = await _critique_one(_QUESTION, "UTXO text", []) - - assert verdict["passed"] is False - assert verdict["failed_criteria"] == ["critic unavailable"] - - -@pytest.mark.asyncio -async def test_critique_one_fails_closed_on_structured_generation_error(): - with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: - mock_gj.side_effect = StructuredGenerationError("bad schema") - verdict = await _critique_one(_QUESTION, "UTXO text", []) - - assert verdict["passed"] is False - assert verdict["failed_criteria"] == ["critic unavailable"] -``` - -- [ ] **Step 2: Run to verify they fail** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -k critique_one -v` -Expected: FAIL — `ImportError: cannot import name '_critique_one'` - -- [ ] **Step 3: Implement `_critique_one` in `app/services/quiz_harness.py`** - -```python -CRITIQUE_PROMPT_VERSION = "v1" - -_CRITIQUE_CRITERIA = ( - "entailed", "distractors_plausible", "single_defensible", "requires_course_material", -) -# "duplicate" is tracked separately — it's the one criterion where True is the failure, not False. - -_CRITIQUE_SCHEMA: Dict[str, Any] = { - "type": "object", - "required": ["entailed", "distractors_plausible", "single_defensible", - "requires_course_material", "duplicate"], - "properties": { - "entailed": {"type": "boolean"}, - "distractors_plausible": {"type": "boolean"}, - "single_defensible": {"type": "boolean"}, - "requires_course_material": {"type": "boolean"}, - "duplicate": {"type": "boolean"}, - }, -} - -_CRITIQUE_SYSTEM = ( - "You are a critic judging one multiple-choice question against its source passage and " - "the other questions already accepted into this quiz.\n" - "entailed: the correct answer is directly supported by the passage.\n" - "distractors_plausible: every wrong option is plausible but demonstrably wrong from the " - "passage — not absurd, not another correct answer.\n" - "single_defensible: exactly one option is correct; no other option could also be " - "defended as correct.\n" - "requires_course_material: the question cannot be answered from general knowledge alone.\n" - "duplicate: true if this question tests the same fact as one of the other questions shown." -) - - -async def _critique_one( - question: Dict[str, Any], source_text: str, accepted_questions: List[Dict[str, Any]], -) -> Dict[str, Any]: - ... -``` - -Implementer writes the body: build the prompt from `question["prompt"]`/`question["options"]`/ -`question["correct_key"]`, the `source_text`, and a listing of `accepted_questions`' `"prompt"` -values (empty-list-safe); call `generate_json(prompt, _CRITIQUE_SCHEMA, context=[{"label": -"source", "text": source_text}], system_prompt=_CRITIQUE_SYSTEM, generation_params={"temp": 0.1}, -task_type="quiz_critique")`; read each of `_CRITIQUE_CRITERIA` via `result.get(name, False)` -(missing ⇒ failed, matching Step 1's defensive test) and `duplicate = result.get("duplicate", -True)` (missing ⇒ treated as a duplicate, the safer default); `failed_criteria` is every criterion -in `_CRITIQUE_CRITERIA` whose value is falsy, plus `"duplicate"` if `duplicate` is truthy; `passed -= not failed_criteria`. On `(LlmDisabledError, StructuredGenerationError)`, return `{"passed": -False, "failed_criteria": ["critic unavailable"]}` — fail closed, mirroring -`lesson_service._judge_groundedness`'s pattern in `process_lesson()`. - -- [ ] **Step 4: Run to verify they pass** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -k critique_one -v` -Expected: PASS - -- [ ] **Step 5: Register `quiz_critique` in `qvac_router.LADDER_ROUTING` and add its router test** - -Same as Task 2 Step 5-7: add `"quiz_critique": ("server", SERVER_MODEL),` to `LADDER_ROUTING`, add -`("quiz_critique", qvac_router.SERVER_MODEL)` to `test_server_task_types_route_to_server`'s -parametrize list, run `tests/unit/test_qvac_router.py` to confirm PASS. - -- [ ] **Step 6: Commit** - -```bash -git add app/services/quiz_harness.py tests/unit/test_quiz_harness.py app/services/qvac_router.py tests/unit/test_qvac_router.py -git commit -m "feat(rag): add critique stage to the quiz agent harness" -``` - ---- - -## Task 5: `quiz_harness.py` — repair loop and `run()` orchestration - -Wires plan → generate → critique together, with exactly one repair attempt per rejected question, -and produces the structured trace. - -**Files:** -- Modify: `app/services/quiz_harness.py` -- Modify: `tests/unit/test_quiz_harness.py` - -**Interfaces:** -- Consumes: `_plan_targets` (Task 2), `_generate_one` (Task 3), `_critique_one` (Task 4). -- Produces: `DEFAULT_TARGET_COUNT: int = 4`; `HarnessResult = Dict[str, Any]` with keys - `{"questions": List[Dict[str, Any]], "trace": Dict[str, Any]}`; `trace` has keys - `{"targets_planned": int, "generated": int, "rejected": List[Dict[str, Any]], "repaired": int, - "final_count": int, "calls_used": int, "calls_budget": int}`; `async def run(topic: str, - context_items: List[Dict[str, str]], target_count: int = DEFAULT_TARGET_COUNT) -> - HarnessResult`. Never raises — `LlmDisabledError` at the plan stage is caught the same as - `StructuredGenerationError` and turned into a graceful empty `HarnessResult` with a reason in - the trace, per this plan's own Global Constraints ("never raise out of the harness"). **Correction - (ruled during Task 5's implementation, see SDD ledger):** an earlier draft of this paragraph said - `run()` should re-raise `LlmDisabledError` at the plan stage to match - `generate_quiz_questions`'s pre-harness contract; that contradicted this task's own verbatim - test (`test_run_llm_disabled_at_plan_stage_returns_no_questions`, which asserts a graceful - return, not `pytest.raises`) and was an authoring error, not a deliberate choice — the graceful - path is what's implemented and is correct. Consumed by Task 7's - `quiz_generation.generate_quiz_questions`. - -- [ ] **Step 1: Write the failing tests** - -```python -@pytest.mark.asyncio -async def test_run_all_pass_returns_every_question_with_no_repairs(): - with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ - patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ - patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: - mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] * 2 - mock_gen.return_value = dict(_QUESTION) - mock_crit.return_value = {"passed": True, "failed_criteria": []} - - result = await run("UTXO model", _CONTEXT, target_count=2) - - assert len(result["questions"]) == 2 - assert result["trace"]["repaired"] == 0 - assert result["trace"]["final_count"] == 2 - assert result["trace"]["calls_used"] == 1 + 2 + 2 # plan + 2 generate + 2 critique - - -@pytest.mark.asyncio -async def test_run_rejected_then_repaired_is_recorded_and_kept(): - with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ - patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ - patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: - mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] - mock_gen.return_value = dict(_QUESTION) - mock_crit.side_effect = [ - {"passed": False, "failed_criteria": ["entailed"]}, - {"passed": True, "failed_criteria": []}, - ] - - result = await run("UTXO model", _CONTEXT, target_count=1) - - assert result["trace"]["repaired"] == 1 - assert len(result["questions"]) == 1 - assert result["trace"]["calls_used"] == 1 + 2 + 2 # plan + (generate+critique) x2 - - -@pytest.mark.asyncio -async def test_run_retry_limit_reached_drops_the_question_with_reasons(): - with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ - patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ - patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: - mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] - mock_gen.return_value = dict(_QUESTION) - mock_crit.return_value = {"passed": False, "failed_criteria": ["entailed"]} - - result = await run("UTXO model", _CONTEXT, target_count=1) - - assert result["questions"] == [] - assert result["trace"]["final_count"] == 0 - assert len(result["trace"]["rejected"]) == 1 - assert result["trace"]["rejected"][0]["reasons"] == ["entailed"] - # Exactly one repair attempt — no third generate/critique pass. - assert result["trace"]["calls_used"] == 1 + 2 + 2 - - -@pytest.mark.asyncio -async def test_run_llm_disabled_at_plan_stage_returns_no_questions(): - with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan: - mock_plan.side_effect = LlmDisabledError("disabled") - - result = await run("UTXO model", _CONTEXT, target_count=2) - - assert result["questions"] == [] - assert result["trace"]["final_count"] == 0 - - -@pytest.mark.asyncio -async def test_run_llm_disabled_mid_generate_drops_that_target_and_continues(): - with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ - patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ - patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: - mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] * 2 - mock_gen.side_effect = [dict(_QUESTION), None] # second target's generation failed - mock_crit.return_value = {"passed": True, "failed_criteria": []} - - result = await run("UTXO model", _CONTEXT, target_count=2) - - assert len(result["questions"]) == 1 - assert len(result["trace"]["rejected"]) == 1 -``` - -- [ ] **Step 2: Run to verify they fail** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -k "test_run" -v` -Expected: FAIL — `ImportError: cannot import name 'run'` - -- [ ] **Step 3: Implement `run()` in `app/services/quiz_harness.py`** - -```python -from app.services.qvac_structured import LlmDisabledError, StructuredGenerationError - -DEFAULT_TARGET_COUNT = 4 - - -async def run( - topic: str, context_items: List[Dict[str, str]], target_count: int = DEFAULT_TARGET_COUNT, -) -> Dict[str, Any]: - ... -``` - -Implementer writes the body following the design doc's orchestration exactly: call -`_plan_targets`, catching `LlmDisabledError`/`StructuredGenerationError` from it to end the run -immediately with `{"questions": [], "trace": {..., "rejected": [{"stage": "plan", "reason": -str(exc)}]}}` (**correction, ruled during Task 5** — catch, do not propagate; see the Interfaces -block above); otherwise iterate the returned targets, for each one calling `_generate_one` then (if -it returned a question) `_critique_one` against the already-`accepted` list; on a failed critique, -attempt exactly one repair (`_generate_one(target, context_items, feedback=...)` built from -`failed_criteria`, then `_critique_one` again if the repair generated a -question); a `None` from the *initial* generate call (not the repair attempt) is dropped -immediately with no repair — repair is reserved for a question that generated but failed critique -(**correction, ruled during Task 5** — the verbatim test -`test_run_llm_disabled_mid_generate_drops_that_target_and_continues` requires this: it mocks -exactly 2 `_generate_one` calls for 2 targets, which a repair-on-None path would exceed); append -to `accepted` on any pass, otherwise to `trace["rejected"]` with a -`question_preview` (the rejected question's `"prompt"` if one exists, else `None`) and `"reasons"`; -increment `trace["calls_used"]` once per `_plan_targets`/`_generate_one`/`_critique_one` call -actually made (a `None` from a previous repair generate must not trigger an extra critique call — -track this explicitly so the `calls_used == 1 + 2 + 2` assertions in Step 1 hold); set -`trace["calls_budget"] = 4 * target_count + 1` and `trace["final_count"] = len(accepted)` before -returning. `topic` is accepted for interface symmetry with `generate_quiz_questions` and is not -otherwise used inside `run()` yet — the implementer may thread it into `_generate_one`'s prompt -later if useful, but no test here requires it. - -- [ ] **Step 4: Run to verify they pass** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_harness.py -v` -Expected: PASS (every test in the file, not just this task's) - -- [ ] **Step 5: Commit** - -```bash -git add app/services/quiz_harness.py tests/unit/test_quiz_harness.py -git commit -m "feat(rag): add bounded repair loop and run() orchestrator to the quiz harness" -``` - ---- - -## Task 6: Prompt registry wiring - -Registers the harness's new prompts so `get_prompt`/`list_prompts` can inspect them, and retires -`quiz_generation.py`'s now-dead multi-question schema/prompt constants (superseded by Task 3's -single-question ones). - -**Files:** -- Modify: `app/prompts/registry.py:29-52` -- Modify: `tests/unit/test_prompt_registry.py` - -(`app/services/quiz_generation.py`'s now-dead `QUIZ_PROMPT_VERSION`/`QUIZ_SCHEMA`/`QUIZ_SYSTEM` -are deleted in Task 7, once their last caller is rewritten — see that task's rationale.) - -**Interfaces:** -- Consumes: `quiz_harness.PLAN_PROMPT_VERSION`, `quiz_harness._PLAN_SYSTEM`, - `quiz_harness.GENERATE_PROMPT_VERSION`, `quiz_harness._GENERATE_SYSTEM`, - `quiz_harness.CRITIQUE_PROMPT_VERSION`, `quiz_harness._CRITIQUE_SYSTEM` (all from Tasks 2-4). -- Produces: registry entries `"quiz_plan"`, `"quiz_critique"`; `"quiz_gen"` now mirrors - `quiz_harness.GENERATE_PROMPT_VERSION`/`_GENERATE_SYSTEM` instead of the deleted - `quiz_generation.QUIZ_PROMPT_VERSION`/`QUIZ_SYSTEM`. - -- [ ] **Step 1: Write the failing test** - -In `tests/unit/test_prompt_registry.py`, add `"quiz_plan"` and `"quiz_critique"` to -`EXPECTED_NAMES`, and add: -```python -def test_quiz_plan_version_matches_quiz_harness_constant(): - from app.services import quiz_harness - assert get_prompt("quiz_plan").version == quiz_harness.PLAN_PROMPT_VERSION - assert get_prompt("quiz_plan").system == quiz_harness._PLAN_SYSTEM - - -def test_quiz_critique_version_matches_quiz_harness_constant(): - from app.services import quiz_harness - assert get_prompt("quiz_critique").version == quiz_harness.CRITIQUE_PROMPT_VERSION - assert get_prompt("quiz_critique").system == quiz_harness._CRITIQUE_SYSTEM -``` -Update `test_quiz_gen_version_matches_quiz_generation_constant` to -`test_quiz_gen_version_matches_quiz_harness_constant`, asserting against -`quiz_harness.GENERATE_PROMPT_VERSION`/`_GENERATE_SYSTEM` instead of `quiz_generation`'s deleted -constants. - -- [ ] **Step 2: Run to verify it fails** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_prompt_registry.py -v` -Expected: FAIL — `AssertionError` (names mismatch) and `AttributeError` on the renamed test. - -- [ ] **Step 3: Update `_build_registry()` in `app/prompts/registry.py`** - -Change the import line to `from app.services import lesson_service, outline_service, -quiz_generation, quiz_harness, study_service` (keep `quiz_generation` — still used by -`persist_quiz` elsewhere, just no longer for the `quiz_gen` prompt). Replace the `"quiz_gen"` entry: -```python - "quiz_gen": Prompt( - "quiz_gen", quiz_harness.GENERATE_PROMPT_VERSION, quiz_harness._GENERATE_SYSTEM, - ), - "quiz_plan": Prompt( - "quiz_plan", quiz_harness.PLAN_PROMPT_VERSION, quiz_harness._PLAN_SYSTEM, - ), - "quiz_critique": Prompt( - "quiz_critique", quiz_harness.CRITIQUE_PROMPT_VERSION, quiz_harness._CRITIQUE_SYSTEM, - ), -``` - -Leave `quiz_generation.py`'s `QUIZ_PROMPT_VERSION`/`QUIZ_SCHEMA`/`QUIZ_SYSTEM` in place for now — -`generate_quiz_questions` still calls `generate_json` with them until Task 7 rewires it. Deleting -them here would leave the module referencing nothing until Task 7 lands, breaking the "tests pass -after every task" invariant. Task 7 Step 3 deletes them once their last caller is rewritten. - -- [ ] **Step 4: Run to verify it passes** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_prompt_registry.py -v` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add app/prompts/registry.py tests/unit/test_prompt_registry.py -git commit -m "feat(rag): register quiz harness plan/critique prompts, repoint quiz_gen" -``` - ---- - -## Task 7: Wire the harness into `generate_quiz_questions` and the API response - -The last task: `quiz_generation.generate_quiz_questions` delegates to `quiz_harness.run` instead of -calling `generate_json` directly; `quizzes_api.generate_quiz`'s response carries the new `trace` -field. - -**Files:** -- Modify: `app/services/quiz_generation.py:83-108` (`generate_quiz_questions`) -- Modify: `app/api/quizzes_api.py:141-205` (`generate_quiz`) -- Modify: `tests/unit/test_quiz_generation.py:62-85` -- Modify: `tests/integration/test_quizzes_certificates_api.py` - -**Interfaces:** -- Consumes: `quiz_harness.run` (Task 5). -- Produces: `generate_quiz_questions(topic, context_items) -> Optional[Dict[str, Any]]` unchanged - in signature; now returns `{"questions": [...], "trace": {...}}` or `None` (when `run()` produced - no questions) — same external contract as before, callers that only read `.get("questions")` - (`lesson_service._persist_quiz`) are unaffected. `POST /courses/{course_id}/quizzes/generate`'s - JSON response gains a `"trace"` key alongside the existing quiz fields. - -- [ ] **Step 1: Write the failing tests** - -Replace the two tests in `tests/unit/test_quiz_generation.py` (lines 62-85): -```python -@pytest.mark.asyncio -async def test_generate_quiz_questions_delegates_to_harness(): - with patch("app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock) as mock_run: - mock_run.return_value = {"questions": _QUESTIONS, "trace": {"final_count": 1}} - result = await generate_quiz_questions("UTXO model", [{"label": "p1", "text": "...", "chunk_id": "c1"}]) - - assert result == {"questions": _QUESTIONS, "trace": {"final_count": 1}} - mock_run.assert_awaited_once() - - -@pytest.mark.asyncio -async def test_generate_quiz_questions_returns_none_when_harness_produces_nothing(): - with patch("app.services.quiz_generation.quiz_harness.run", new_callable=AsyncMock) as mock_run: - mock_run.return_value = {"questions": [], "trace": {"final_count": 0}} - result = await generate_quiz_questions("UTXO model", [{"label": "p1", "text": "...", "chunk_id": "c1"}]) - - assert result is None -``` - -In `tests/integration/test_quizzes_certificates_api.py`, change -`test_generate_quiz_persists_to_db`'s patch target from -`"app.services.quiz_generation.generate_json"` to `"app.services.quiz_generation.quiz_harness.run"`, -with `mock_gj.return_value = {"questions": _QUESTIONS, "trace": {"final_count": 1}}`, and add: -```python - assert body["trace"] == {"final_count": 1} -``` -Update Task 1's `test_generate_quiz_uses_evidence_pack_chunk_id` the same way: it now asserts the -`context` kwarg on `quiz_harness.run`'s call (its second positional/keyword argument, -`context_items`) rather than on `generate_json`. - -- [ ] **Step 2: Run to verify they fail** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_generation.py tests/integration/test_quizzes_certificates_api.py -v` -Expected: FAIL — `generate_quiz_questions` still calls `generate_json` directly; no `trace` key in -the response. - -- [ ] **Step 3: Rewrite `generate_quiz_questions` in `app/services/quiz_generation.py`** - -```python -from app.services import quiz_harness - -_DEFAULT_TARGET_COUNT = quiz_harness.DEFAULT_TARGET_COUNT - - -async def generate_quiz_questions( - topic: str, - context_items: List[Dict[str, str]], -) -> Optional[Dict[str, Any]]: - """Run the plan/generate/critique/repair harness to produce MCQ quiz data. - - Returns None (rather than raising) when the harness produces zero - questions — quiz generation is treated as best-effort by all current - callers. - """ - result = await quiz_harness.run(topic, context_items, target_count=_DEFAULT_TARGET_COUNT) - if not result["questions"]: - return None - return result -``` -Delete `QUIZ_PROMPT_VERSION`, `QUIZ_SCHEMA`, and `QUIZ_SYSTEM` (the old lines 23-80) — nothing -calls them after this rewrite (Task 6 already repointed the `"quiz_gen"` registry entry away from -them). Remove the now-unused `StructuredGenerationError`/`generate_json` imports from this file if -nothing else in it still uses them (check `persist_quiz` — it doesn't). - -- [ ] **Step 4: Add the `trace` field to the API response in `app/api/quizzes_api.py`** - -In `generate_quiz` (around line 196-205), capture the trace before building the response: -```python - response_body = _to_student_detail(quiz, questions, db).model_dump() - response_body["trace"] = quiz_data.get("trace") - return JSONResponse(status_code=status.HTTP_201_CREATED, content=response_body) -``` - -- [ ] **Step 5: Run to verify they pass** - -Run: `cd services/ai && uv run --no-sync pytest tests/unit/test_quiz_generation.py tests/integration/test_quizzes_certificates_api.py tests/unit/test_quiz_harness.py tests/unit/test_prompt_registry.py tests/unit/test_qvac_router.py tests/unit/test_lesson_service.py -v` -Expected: all PASS. - -- [ ] **Step 6: Run the full backend suite and mypy** - -Run: `cd services/ai && uv run --no-sync pytest && uv run --no-sync mypy app` -Expected: PASS (no regressions elsewhere — `lesson_service._generate_quiz_data`/`_persist_quiz` -are unaffected since they only read `.get("questions")` off the same dict shape). - -- [ ] **Step 7: Commit** - -```bash -git add app/services/quiz_generation.py app/api/quizzes_api.py tests/unit/test_quiz_generation.py tests/integration/test_quizzes_certificates_api.py -git commit -m "feat(rag): wire the quiz agent harness into quiz generation and the API response" -``` - ---- - -## After merge (not a coded task) - -Per issue #134's acceptance criteria: run a manual review of 20 generated questions on real -Bitcoin course material against a live QVAC service, and record pass/fail judgments in the issue -before closing it. This requires a running QVAC worker and is not something these tasks can verify. diff --git a/services/ai/app/api/quizzes_api.py b/services/ai/app/api/quizzes_api.py index f569fd4..2cfe296 100644 --- a/services/ai/app/api/quizzes_api.py +++ b/services/ai/app/api/quizzes_api.py @@ -158,7 +158,7 @@ async def generate_quiz( # below goes through the plan -> generate -> critique -> repair harness # (quiz_harness.run), a bounded multi-call loop of up to 4*N+1 calls (17 # for the default N=4 target count) rather than a single /generate_json - # call — see docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md. + # call — see docs/quiz-agent-harness-design.md. result = await study_service.dispatch( question=body.query, course_id=course_id, diff --git a/services/ai/app/services/quiz_generation.py b/services/ai/app/services/quiz_generation.py index b1037ea..81abf7f 100644 --- a/services/ai/app/services/quiz_generation.py +++ b/services/ai/app/services/quiz_generation.py @@ -33,8 +33,9 @@ async def generate_quiz_questions( the harness, even when `questions` is empty — the trace is most valuable exactly in that case (issue #134's "surface rejected questions" acceptance criterion), and callers that check `.get("questions")` or do - `if quiz_data:` already treat an empty-questions dict as "no quiz" since - it is still truthy, so this never changes caller behaviour. + callers must inspect `.get("questions")`; the result dict itself remains + truthy even when it contains no questions. `persist_quiz` safely returns + None for an empty question list. """ result = await quiz_harness.run(topic, context_items, target_count=_DEFAULT_TARGET_COUNT) if not result["questions"]: diff --git a/services/ai/app/services/quiz_harness.py b/services/ai/app/services/quiz_harness.py index f80e886..6fae3a9 100644 --- a/services/ai/app/services/quiz_harness.py +++ b/services/ai/app/services/quiz_harness.py @@ -1,6 +1,6 @@ """Agent harness for quiz generation — plan -> generate -> critique -> repair. -See docs/superpowers/specs/2026-11-05-quiz-agent-harness-design.md. +See docs/quiz-agent-harness-design.md. """ import logging from typing import Any, Dict, List, Optional @@ -48,10 +48,11 @@ async def _plan_targets(context_items: List[Dict[str, str]], target_count: int) -> List[PlanTarget]: """Select target_count (chunk_id, cognitive_level) pairs from context_items. - Always returns exactly target_count targets. An invalid or missing - passage_index from the model is corrected by round-robin assignment over - context_items rather than trusted — the model never needs to reproduce a - chunk_id string correctly. + Always returns exactly target_count targets. Invalid, missing, or repeated + passage indices from the model are corrected by round-robin assignment over + unused context_items rather than trusted — the model never needs to + reproduce a chunk_id string correctly. A passage is reused only after all + available passages have been used once. """ prompt = ( f"Select exactly {target_count} target passages for a quiz out of the " @@ -68,6 +69,22 @@ async def _plan_targets(context_items: List[Dict[str, str]], target_count: int) raw_targets = result.get("targets", []) targets: List[PlanTarget] = [] + used_indices: set[int] = set() + fallback_index = 0 + + def next_unused_index() -> int: + """Choose the next source passage, resetting only after a full cycle.""" + nonlocal fallback_index + if len(used_indices) == len(context_items): + used_indices.clear() + + for offset in range(len(context_items)): + candidate = (fallback_index + offset) % len(context_items) + if candidate not in used_indices: + fallback_index = (candidate + 1) % len(context_items) + return candidate + + raise RuntimeError("No unused quiz passage available") for i in range(target_count): if i < len(raw_targets): @@ -78,10 +95,12 @@ async def _plan_targets(context_items: List[Dict[str, str]], target_count: int) idx = None level = None - if isinstance(idx, int) and 1 <= idx <= len(context_items): - chunk_id = context_items[idx - 1]["chunk_id"] - else: - chunk_id = context_items[i % len(context_items)]["chunk_id"] + planned_index = idx - 1 if isinstance(idx, int) and 1 <= idx <= len(context_items) else None + if planned_index is None or planned_index in used_indices: + planned_index = next_unused_index() + + used_indices.add(planned_index) + chunk_id = context_items[planned_index]["chunk_id"] if level not in _COGNITIVE_LEVELS: level = "recall" @@ -220,12 +239,12 @@ async def _critique_one( ) -> CritiqueVerdict: """Judge one generated question against the five quality criteria. - Fails closed: if the critic is unavailable (LlmDisabledError), returns an - unparsable response (StructuredGenerationError), or a transport-level - error occurs (httpx.HTTPError), the question is treated as a failure, not - a pass — it is never silently accepted just because we could not - critique it. Mirrors lesson_service._judge_groundedness's fail-closed - pattern. + Fails closed: if the critic returns an unparsable response + (StructuredGenerationError), or a transport-level error occurs + (httpx.HTTPError), the question is treated as a failure, not a pass — it + is never silently accepted just because we could not critique it. + LlmDisabledError deliberately propagates so callers can distinguish a + disabled service from an individual failed question. """ accepted_prompts = [q["prompt"] for q in accepted_questions] prompt = ( @@ -244,7 +263,7 @@ async def _critique_one( generation_params={"temp": 0.1}, task_type="quiz_critique", ) - except (LlmDisabledError, StructuredGenerationError, httpx.HTTPError): + except (StructuredGenerationError, httpx.HTTPError): return {"passed": False, "failed_criteria": ["critic unavailable"]} failed_criteria = [name for name in _CRITIQUE_CRITERIA if not result.get(name, False)] @@ -304,16 +323,17 @@ async def run( `4 * target_count + 1` (plan, plus up to generate+critique twice per target) — see `trace["calls_budget"]`. - A `None` generation result (an unresolvable chunk_id, a schema error, or - the LLM being disabled mid-run) is not retried: the same target and - context would almost certainly fail the same way again, so the target is - dropped immediately without spending a repair attempt or a critique call - on it. - - `LlmDisabledError`/`StructuredGenerationError` at the plan stage end the - run immediately with no questions produced, since there is nothing yet to - degrade to. `topic` is accepted for interface symmetry with - `generate_quiz_questions` and is not otherwise used here yet. + A `None` generation result (an unresolvable chunk_id or a schema error) + is not retried: the same target and context would almost certainly fail + the same way again, so the target is dropped immediately without spending + a repair attempt or a critique call on it. + + LlmDisabledError propagates from planning, generation, repair, and + critique so the endpoint can report that the LLM is disabled. Per-question + StructuredGenerationError and httpx.HTTPError failures are recorded in + the trace and allow later targets to proceed. `topic` is accepted for + interface symmetry with `generate_quiz_questions` and is not otherwise + used here yet. """ trace: Dict[str, Any] = { "targets_planned": 0, @@ -329,7 +349,7 @@ async def run( try: targets = await _plan_targets(context_items, target_count) trace["calls_used"] += 1 - except (LlmDisabledError, StructuredGenerationError, httpx.HTTPError) as exc: + except (StructuredGenerationError, httpx.HTTPError) as exc: trace["calls_used"] += 1 trace["rejected"].append(_rejection( stage="plan", chunk_id=None, cognitive_level=None, @@ -345,7 +365,7 @@ async def run( try: question = await _generate_one(target, context_items) - except (LlmDisabledError, StructuredGenerationError, httpx.HTTPError): + except (StructuredGenerationError, httpx.HTTPError): question = None trace["calls_used"] += 1 @@ -369,7 +389,7 @@ async def run( feedback = ", ".join(verdict["failed_criteria"]) try: repaired_question = await _generate_one(target, context_items, feedback=feedback) - except (LlmDisabledError, StructuredGenerationError, httpx.HTTPError): + except (StructuredGenerationError, httpx.HTTPError): repaired_question = None trace["calls_used"] += 1 diff --git a/services/ai/tests/unit/test_quiz_harness.py b/services/ai/tests/unit/test_quiz_harness.py index 57a4b8a..20820e5 100644 --- a/services/ai/tests/unit/test_quiz_harness.py +++ b/services/ai/tests/unit/test_quiz_harness.py @@ -54,6 +54,29 @@ async def test_plan_targets_reuses_chunks_when_target_count_exceeds_available(): assert {t["chunk_id"] for t in targets} == {"c1", "c2"} +@pytest.mark.asyncio +async def test_plan_targets_deduplicates_repeated_model_passages_before_reusing(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.return_value = {"targets": [ + {"passage_index": 1, "cognitive_level": "recall"}, + {"passage_index": 1, "cognitive_level": "application"}, + {"passage_index": 1, "cognitive_level": "analysis"}, + {"passage_index": 1, "cognitive_level": "recall"}, + ]} + targets = await _plan_targets( + _CONTEXT + [ + {"label": "p3", "text": "Script text", "chunk_id": "c3"}, + {"label": "p4", "text": "Node text", "chunk_id": "c4"}, + ], + target_count=4, + ) + + assert [target["chunk_id"] for target in targets] == ["c1", "c2", "c3", "c4"] + assert [target["cognitive_level"] for target in targets] == [ + "recall", "application", "analysis", "recall", + ] + + @pytest.mark.asyncio async def test_plan_targets_propagates_llm_disabled(): with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: @@ -139,6 +162,14 @@ async def test_generate_one_returns_none_on_httpx_transport_error(): assert question is None +@pytest.mark.asyncio +async def test_generate_one_propagates_llm_disabled(): + with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: + mock_gj.side_effect = LlmDisabledError("disabled") + with pytest.raises(LlmDisabledError): + await _generate_one(_TARGET, _CONTEXT) + + @pytest.mark.asyncio async def test_critique_one_passes_when_all_criteria_met(): with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: @@ -188,13 +219,11 @@ async def test_critique_one_passes_accepted_questions_as_duplicate_context(): @pytest.mark.asyncio -async def test_critique_one_fails_closed_on_llm_disabled(): +async def test_critique_one_propagates_llm_disabled(): with patch("app.services.quiz_harness.generate_json", new_callable=AsyncMock) as mock_gj: mock_gj.side_effect = LlmDisabledError("disabled") - verdict = await _critique_one(_QUESTION, "UTXO text", []) - - assert verdict["passed"] is False - assert verdict["failed_criteria"] == ["critic unavailable"] + with pytest.raises(LlmDisabledError): + await _critique_one(_QUESTION, "UTXO text", []) @pytest.mark.asyncio @@ -275,18 +304,16 @@ async def test_run_retry_limit_reached_drops_the_question_with_reasons(): @pytest.mark.asyncio -async def test_run_llm_disabled_at_plan_stage_returns_no_questions(): +async def test_run_propagates_llm_disabled_at_plan_stage(): with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan: mock_plan.side_effect = LlmDisabledError("disabled") - result = await run("UTXO model", _CONTEXT, target_count=2) - - assert result["questions"] == [] - assert result["trace"]["final_count"] == 0 + with pytest.raises(LlmDisabledError): + await run("UTXO model", _CONTEXT, target_count=2) @pytest.mark.asyncio -async def test_run_llm_disabled_mid_generate_drops_that_target_and_continues(): +async def test_run_none_generation_drops_that_target_and_continues(): with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: @@ -301,13 +328,7 @@ async def test_run_llm_disabled_mid_generate_drops_that_target_and_continues(): @pytest.mark.asyncio -async def test_run_generate_one_raising_llm_disabled_drops_that_target_and_continues(): - """A per-target LlmDisabledError raised by _generate_one itself (as - opposed to _generate_one returning None, covered above) must be caught at - the call site and treated like any other per-target failure — drop and - continue — never propagate out of run(). Only a plan-stage failure aborts - the whole run, and even that does so gracefully (see - test_run_llm_disabled_at_plan_stage_returns_no_questions).""" +async def test_run_propagates_llm_disabled_from_generation(): with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: @@ -315,11 +336,21 @@ async def test_run_generate_one_raising_llm_disabled_drops_that_target_and_conti mock_gen.side_effect = [dict(_QUESTION), LlmDisabledError("disabled mid-run")] mock_crit.return_value = {"passed": True, "failed_criteria": []} - result = await run("UTXO model", _CONTEXT, target_count=2) + with pytest.raises(LlmDisabledError): + await run("UTXO model", _CONTEXT, target_count=2) - assert len(result["questions"]) == 1 - assert len(result["trace"]["rejected"]) == 1 - assert result["trace"]["rejected"][0]["stage"] == "generate" + +@pytest.mark.asyncio +async def test_run_propagates_llm_disabled_from_critique(): + with patch("app.services.quiz_harness._plan_targets", new_callable=AsyncMock) as mock_plan, \ + patch("app.services.quiz_harness._generate_one", new_callable=AsyncMock) as mock_gen, \ + patch("app.services.quiz_harness._critique_one", new_callable=AsyncMock) as mock_crit: + mock_plan.return_value = [{"chunk_id": "c1", "cognitive_level": "recall"}] + mock_gen.return_value = dict(_QUESTION) + mock_crit.side_effect = LlmDisabledError("disabled during critique") + + with pytest.raises(LlmDisabledError): + await run("UTXO model", _CONTEXT, target_count=1) @pytest.mark.asyncio