diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 3583dc9..d10262a 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ "name": "data-engineering-knowledge-capture", "source": "./", "description": "Capture data-platform knowledge \u2014 schemas, lineage, medallion, semantic layer, business glossary \u2014 into OKF. Extends PKC; depends on OKF.", - "version": "0.5.1", + "version": "0.5.2", "author": { "name": "Rick Hightower" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index aac5654..efd040f 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "data-engineering-knowledge-capture", - "version": "0.5.1", + "version": "0.5.2", "description": "Data Engineering Knowledge Capture (DEKC) \u2014 extends PKC + OKF for schemas, lineage, medallion layers, SQL/DAX, semantic models, business objects, and an indexed second brain. Agents walk data lakes. Works in Claude Code, Grok Build, Codex, and OpenCode.", "author": { "name": "Rick Hightower", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 58df265..af17f5f 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "data-engineering-knowledge-capture", - "version": "0.5.1", + "version": "0.5.2", "description": "Data Engineering Knowledge Capture (DEKC) \u2014 schemas, lineage, medallion layers, semantic models, business glossary. Extends PKC; depends on OKF. Codex-native skills + hooks port.", "author": { "name": "Rick Hightower", diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index 399c5be..a30822d 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "data-engineering-knowledge-capture", - "version": "0.5.1", + "version": "0.5.2", "description": "Data Engineering Knowledge Capture: schemas, lineage, medallion, semantic models, and glossary into a durable OKF knowledge graph. Multi-host bindings and write isolation.", "author": { "name": "Rick Hightower" diff --git a/.grok-plugin/marketplace.json b/.grok-plugin/marketplace.json index a4a7ddb..9f0341b 100644 --- a/.grok-plugin/marketplace.json +++ b/.grok-plugin/marketplace.json @@ -1,13 +1,13 @@ { "name": "dekc-plugin-marketplace", "description": "Optional native Grok marketplace metadata. Grok Build loads Claude plugins with zero config; this pins identity for listings.", - "version": "0.5.1", + "version": "0.5.2", "plugins": [ { "name": "data-engineering-knowledge-capture", "source": ".", "description": "DEKC \u2014 data engineering knowledge capture on PKC + OKF. Claude-compatible.", - "version": "0.5.1", + "version": "0.5.2", "compatibility": { "claude_plugin": true, "zero_config": true diff --git a/.opencode/plugin/dekc.json b/.opencode/plugin/dekc.json index 838184f..9fc5b17 100644 --- a/.opencode/plugin/dekc.json +++ b/.opencode/plugin/dekc.json @@ -1,6 +1,6 @@ { "name": "data-engineering-knowledge-capture", - "version": "0.5.1", + "version": "0.5.2", "description": "OpenCode port of DEKC \u2014 same skills/agents/scripts as Claude & Codex. Policy via AGENTS.md.", "skills": "../../skills", "agents": "../../agents", diff --git a/AGENTS.md b/AGENTS.md index 0d9dd53..598f61c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,6 +79,16 @@ Plugin root: `${CLAUDE_PLUGIN_ROOT}`. | **data-lake-walker** | Default orchestrator: walk → produce → adversarial grade → index | | **reverse-engineering-orchestrator** | Multi-cloud RE (Fabric/AWS/GCP), strict LoopPolicy + fan-out | +## Query-time retrieval + +Orthogonal to reverse engineering. Search, score, pack, and deepen stay in the sub-agent. The parent sees a **summary card only**. + +| Agent | Role | +|-------|------| +| **data-retriever** | Query-time data-plane context. Spawn via `/dekc-retrieve`. Never dump hit lists or pack markdown into the parent. | + +When the question also needs project decisions or system topology, fan out PKC **knowledge-retriever** and/or SAC **architecture-retriever** in parallel. Do not run `dekc_search.py` / `dekc_pack.py` / `dekc_brain.py` in the parent for Q&A. + ## Producer workers | Agent | Role | @@ -131,7 +141,8 @@ python3 tests/test_dekc.py | medallion health | layer-auditor / dekc-doctor | | wireframe / architecture / ERD / job diagrams | dekc-diagram / dekc_diagram.py | | lake / mart / catalog / stream / DQ / ingestion job | dekc-platform / dekc_platform.py | -| search the second brain | dekc-search / dekc-index | +| retrieve / what's this table / metric / impact Q&A | data-retriever / dekc-retrieve | +| search the second brain (raw hits, not parent Q&A) | dekc-search / dekc-index | | multi-agent loop authoring | okf-agent-graph `/ager-*` + DEKC KnowledgeBind | diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c99225..9a1ab76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ ## Unreleased +## 0.5.2 — 2026-09-12 + +Query-time retrieval sub-agent, matching the PKC/SAC parity pattern: search + +score + pack + deepen stay off the parent; the parent gets a summary card only. + +### Added + +- **`data-retriever`** agent — retrieval-only. No capture, walk, reverse + engineer, or knowledge writes. Returns a retrieval card (seed, fit, engine, + pack stats, lead nodes, lineage note, gaps, next). +- **`/dekc-retrieve`** skill + slash command. Parent must spawn `data-retriever` + and must not run `dekc_search.py` / `dekc_pack.py` in-process for Q&A. + Orthogonal fan-out: PKC `knowledge-retriever`, SAC `architecture-retriever`. +- `dekc_pack.py --summary` — compact card-friendly output, bodies off (root + included), fail-closed token budget. Retriever prefers `--tiny --summary`. + +### Notes + +- Reverse-engineering agents (`data-lake-walker`, `lineage-tracer`, …) are + unchanged. Query-time is a separate section on the RE orchestrator. + ## 0.5.1 — 2026-08-31 Correctness patch for catalog rendering and the rg-backed reverse index. Both diff --git a/README.md b/README.md index a1bf393..704aaa4 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ DEKC **extends [Project Knowledge Capture (PKC)](https://github.com/SpillwaveSol |---|---| | **Plugin name** | `data-engineering-knowledge-capture` | | **Repo** | [SpillwaveSolutions/data-engineering-knowledge-capture](https://github.com/SpillwaveSolutions/data-engineering-knowledge-capture) | -| **Version** | 0.5.1 | +| **Version** | 0.5.2 | | **License** | MIT | | **Hosts** | Claude Code · Grok Build · Codex · OpenCode · Agent Plugins 1.0 · Grok Bot · LangChain Deep Agents | @@ -208,12 +208,14 @@ python3 scripts/dekc_doctor.py --repo . --bundle knowledge python3 scripts/dekc_search.py "revenue" --repo . --bundle knowledge ``` -Slash / skill entry points: `/dekc-init` · `/dekc-walk` · `/dekc-lineage` · `/dekc-business-object` · `/dekc-glossary` · `/dekc-semantic` · `/dekc-context` · `/dekc-search` · `/dekc-index` · `/dekc-doctor` +Slash / skill entry points: `/dekc-init` · `/dekc-walk` · `/dekc-lineage` · `/dekc-business-object` · `/dekc-glossary` · `/dekc-semantic` · `/dekc-retrieve` · `/dekc-context` · `/dekc-search` · `/dekc-index` · `/dekc-doctor` ## Agent loop (AGER-shaped): producers + adversarial judges **Orchestrators:** `data-lake-walker` (default) · `reverse-engineering-orchestrator` (Fabric/AWS/GCP). +**Query-time:** `data-retriever` (`/dekc-retrieve`) — search + pack stay in the sub-agent; the parent gets a summary card only. Not an RE worker. + Producer workers fan out, then **adversarial skeptics grade reverse engineering with rubrics** before index/pack. Lead judge: **re-adversary-judge** (threshold **0.75**). Fail → re-plan or **retract** unproven claims — never invent edges to pass. ```text diff --git a/agents/data-retriever.md b/agents/data-retriever.md new file mode 100644 index 0000000..3b29d68 --- /dev/null +++ b/agents/data-retriever.md @@ -0,0 +1,72 @@ +--- +name: data-retriever +description: Retrieve DEKC data-plane context without contaminating the parent. Use for Table, Metric, LineagePath, IngestionJob, Transformation, Dashboard, DataProduct, GlossaryTerm, BusinessObject queries. Search, score fit, pack, optionally deepen/lineage. Returns a summary card only. +--- + +You are **Data Retriever** — the query-time retrieval sub-agent for DEKC (Data Engineering Knowledge Capture). + +You are **not** a reverse-engineering agent. Do not confuse this role with `data-lake-walker`, `lineage-tracer`, `schema-scout`, or other capture/RE workers. You read the knowledge graph. You do not grow it. + +This is the PKC `knowledge-retriever` / SAC `architecture-retriever` parity role for the data plane. + +## Contract + +- **Retrieval-only.** No capture, walk, reverse-engineer, promote, materialize, index rebuild, or knowledge writes. No `brain_session`. +- **Never** return full hit lists or full pack markdown to the parent. +- Search, score, pack, and deepen stay **inside this sub-agent**. +- The parent receives a **summary card only**. +- Fiction samples only in public trees (Northstar / Lumenfield). Never name a private remote. + +## Card shape (only thing the parent sees) + +```markdown +## Retrieval card +- Query: … +- Seed: `/path` (`Type`) — why chosen +- Fit: high|medium|low — one sentence +- Engine: index|rg|scan +- Pack: hops=N nodes=N tokens=N/budget +- Lead nodes: (5–8 bullets) +- Lineage note: (upstream/downstream one-liner) or none +- Open gaps: or none +- Next: stay|deepen-2hop|lineage|try-alt-seed `/other` +``` + +## Scripts (read-only) + +```bash +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_search.py" "" --repo . --bundle knowledge --limit 5 +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_search.py" "" --type Table,Metric,LineagePath,DataProduct --limit 5 --json +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_pack.py" --repo . --bundle knowledge --tiny --summary +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_pack.py" --repo . --bundle knowledge --hops 2 --summary +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_lineage.py" --repo . --bundle knowledge upstream --hops 2 +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_lineage.py" --repo . --bundle knowledge downstream --hops 2 +``` + +Prefer `--summary` on every pack. Fall back to `--tiny` without `--summary` only if the flag is missing. Do **not** run `dekc_brain.py` for the parent — those packs stay here if you need an intent hint, and they still must not leak. + +Forbidden: `dekc_capture.py`, `dekc_walk.py`, `dekc_lineage.py materialize`, `dekc_business.py`, `dekc_pack.py --write`, `brain_session.py`. + +## Workflow + +1. **Seed or search.** If the caller already named a concept path, use it. Otherwise `dekc_search.py --limit 5`. +2. **Score fit** (no LLM in Python — you score). Prefer **Table / Metric / LineagePath / DataProduct**. Then IngestionJob, Transformation, Dashboard, BusinessObject, GlossaryTerm. + - **high** — preferred type and title/path match the question + - **medium** — preferred type *or* a strong keyword hit + - **low** — glossary-only, wrong layer, or weak lexical match +3. **Pack** `--tiny --summary`. Bodies stay off. Read hops / nodes / tokens / engine / lead nodes from the summary. +4. **Deepen or lineage** only if the card would otherwise be thin, or the question is impact / blast-radius. Use `--hops 2 --summary` or the lineage upstream/downstream helpers. **Max 2 deepen steps.** +5. Return the **card only**. One card. No appendix. + +## Fit and Next + +- `stay` — seed answers the question +- `deepen-2hop` — neighborhood is thin; parent may re-spawn you with a 2-hop ask +- `lineage` — impact / upstream / downstream still open +- `try-alt-seed /other` — better seed exists; name it + +Open gaps: missing gold BO, no landing job, broken link, empty search. Write `none` when the pack is enough. + +## Orthogonal fan-out + +If the question also needs project decisions or system topology, tell the parent to spawn PKC **knowledge-retriever** and/or SAC **architecture-retriever** in parallel. Do not impersonate those agents. Do not pack their nouns. diff --git a/agents/reverse-engineering-orchestrator.md b/agents/reverse-engineering-orchestrator.md index 7ef7dc7..d11581e 100644 --- a/agents/reverse-engineering-orchestrator.md +++ b/agents/reverse-engineering-orchestrator.md @@ -88,3 +88,16 @@ Full topologies: `docs/designs/current_design_doc.md`. - Pass/fail vs RE threshold **0.75** - Explicit revise list if fail - Never claim reverse engineering complete without judge pass + +## Query-time (not reverse engineering) + +**data-retriever** is query-time only. It is **not** an RE sub-agent. Do not put it in producer or skeptic fan-out. + +When someone asks about an existing Table, Metric, LineagePath, IngestionJob, Transformation, Dashboard, DataProduct, GlossaryTerm, or BusinessObject: + +1. Spawn **data-retriever** (via `/dekc-retrieve`). +2. Do **not** run `dekc_search.py`, `dekc_pack.py`, or `dekc_brain.py` in this orchestrator for that Q&A. +3. Keep the **retrieval card** only — never a hit list or full pack. +4. If the question also needs project decisions or system topology, fan out PKC **knowledge-retriever** and/or SAC **architecture-retriever** in parallel. + +See `agents/data-retriever.md` and `skills/dekc-retrieve/SKILL.md`. diff --git a/commands/dekc-retrieve.md b/commands/dekc-retrieve.md new file mode 100644 index 0000000..d8a9e7a --- /dev/null +++ b/commands/dekc-retrieve.md @@ -0,0 +1,10 @@ +--- +name: dekc-retrieve +description: Query-time retrieval. Spawn data-retriever; parent keeps a summary card only. +--- + +Run the **dekc-retrieve** skill. + +User request: `$ARGUMENTS` + +Follow `${CLAUDE_PLUGIN_ROOT}/skills/dekc-retrieve/SKILL.md` completely. Spawn **data-retriever**. Do not run `dekc_search.py` or `dekc_pack.py` in the parent. Return the retrieval card only. diff --git a/docs/GROK_BOT.md b/docs/GROK_BOT.md index 194c962..d748f82 100644 --- a/docs/GROK_BOT.md +++ b/docs/GROK_BOT.md @@ -58,7 +58,9 @@ Writes fail closed without `--author` or `SECOND_BRAIN_IDENTITY`. Successful kno Default ContextPack: **2 hops / ~20 nodes**. -Pack before answering or writing. Do not dump the entire tree. +For Q&A, spawn **data-retriever** (`/dekc-retrieve`). Search, score, pack, and deepen stay in that sub-agent. The parent keeps a **retrieval card** only — never a hit list, full pack, or `dekc_brain.py` dump. Pack `--summary` (bodies off) is the card-friendly path. + +Do not dump the entire tree. When the question also needs project or architecture context, fan out PKC **knowledge-retriever** and/or SAC **architecture-retriever** in parallel. ## Skill binding diff --git a/docs/ONBOARDING.md b/docs/ONBOARDING.md index beaefcd..543ce8f 100644 --- a/docs/ONBOARDING.md +++ b/docs/ONBOARDING.md @@ -37,7 +37,7 @@ See [GROK_BOT.md](GROK_BOT.md) for the binding contract. 1. State your identity: `Grok Bot: Data Engineering Knowledge Capture`. 2. Confirm the knowledge root (`SECOND_BRAIN_ROOT` or the target bundle). -3. Pack the relevant subgraph (2 hops) before answering or writing. +3. For Q&A, spawn **data-retriever** (`/dekc-retrieve`) and keep the retrieval card only. Do not dump a 2-hop pack or `dekc_brain.py` output into the parent. Pack before writing. 4. Persist only through skills + deterministic scripts inside an isolation session when writing a shared brain. 5. Report path + validation result, not a dumped graph. diff --git a/docs/designs/current_design_doc.md b/docs/designs/current_design_doc.md index 1b6be67..221228f 100644 --- a/docs/designs/current_design_doc.md +++ b/docs/designs/current_design_doc.md @@ -314,6 +314,8 @@ Compensation is rare (knowledge is additive); “undo” = revert Git commit of Orchestrator and workers load **2-hop packs** (~20 nodes) via RetrievalBinding, not the full lake graph. Judge sees doctor JSON + gap list only. This matches OKF progressive disclosure and AGER ContextIsolationPolicy for subagents. +Query-time Q&A is a different path: spawn **data-retriever** (`/dekc-retrieve`). Search, score, pack (`--summary`), and deepen stay in that sub-agent; the parent keeps a retrieval card only. Do not confuse this with RE producers. + ### 3.10 Optional formal AGER bundle Teams can author an explicit AGER bundle (via `/ager-init`) that **KnowledgeBinds** the DEKC `knowledge/` root and **uses** DEKC scripts as Tools. DEKC agents remain usable without that bundle; the AGER bundle is the portable runtime config for LangGraph/CrewAI adapters. diff --git a/docs/designs/retrieval-ladder.md b/docs/designs/retrieval-ladder.md index b05f22f..281c66f 100644 --- a/docs/designs/retrieval-ladder.md +++ b/docs/designs/retrieval-ladder.md @@ -55,6 +55,7 @@ the tree. The curate hook does not install rg and does not build the index. |---|---|---| | `dekc_search.py` | yes | candidate prefilter | | `dekc_pack.py` inbound | yes | reverse index via literal path + self SQL | +| `dekc_pack.py --summary` | n/a | card-friendly render; bodies off; fail-closed budget | | `dekc_validate.py` / `dekc-curate.sh` | **no** | must parse frontmatter + resolve links | | orphans / doctor listings | **no** | need the parsed graph | | mermaid (`build_graph`) | **no** | leftover full scan; accepted | diff --git a/docs/user_guide/user-guide.md b/docs/user_guide/user-guide.md index a3636c8..a74074d 100644 --- a/docs/user_guide/user-guide.md +++ b/docs/user_guide/user-guide.md @@ -132,7 +132,9 @@ python3 scripts/dekc_brain.py "silver.orders" --intent impact --repo . --bundle Each intent returns: design checklist, ranked schema-typed concepts, progressive disclosure pack, lineage snippet. -Skills: `dekc-second-brain`, `dekc-design-report`, `dekc-land-data`. +For Q&A, spawn **data-retriever** (`/dekc-retrieve`) instead of dumping a brain pack into the parent. + +Skills: `dekc-retrieve`, `dekc-second-brain`, `dekc-design-report`, `dekc-land-data`. Patterns: `patterns/design-report-from-gold.md`, `patterns/land-stream-to-bronze.md`. @@ -221,6 +223,9 @@ python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_pack.py" tables/gold-order-daily.md # tiny pack for chat focus python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_pack.py" tables/gold-order-daily.md \ --repo . --bundle knowledge --tiny +# card-friendly summary (bodies off) — prefer this from data-retriever +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_pack.py" tables/gold-order-daily.md \ + --repo . --bundle knowledge --tiny --summary ``` If `okf-graph-eng` is available, prefer its pack/impact for the same paths. @@ -237,8 +242,9 @@ If `okf-graph-eng` is available, prefer its pack/impact for the same paths. | `dekc-business-object` | Promote technical → business + glossary | | `dekc-glossary` | Author glossary terms | | `dekc-semantic` | Semantic models, metrics, dashboards | -| `dekc-context` | Context packs | -| `dekc-search` | Second-brain search | +| `dekc-retrieve` | Query-time retrieval — spawn `data-retriever`; card only | +| `dekc-context` | Context packs (retriever internals / known seed) | +| `dekc-search` | Second-brain search (raw hits; not parent Q&A) | | `dekc-index` | Rebuild index | | `dekc-grade` | Rubric grade + adversarial RE protocol | | `dekc-doctor` | Health: coverage, orphans, validation | @@ -247,6 +253,7 @@ If `okf-graph-eng` is available, prefer its pack/impact for the same paths. | Agent | Use when | |-------|----------| +| **data-retriever** | Query-time Q&A — spawn via `/dekc-retrieve`; parent keeps a card only | | **data-lake-walker** | Full reverse-engineer loop (orchestrator) | | **schema-scout** | Only structure (schemas/tables/columns) | | **lineage-tracer** | Only SQL/job lineage and promotions | diff --git a/hosts/grok-bot/SKILL.md b/hosts/grok-bot/SKILL.md index 985a124..1011d80 100644 --- a/hosts/grok-bot/SKILL.md +++ b/hosts/grok-bot/SKILL.md @@ -9,6 +9,6 @@ Read `docs/ONBOARDING.md` first, then follow `docs/GROK_BOT.md`. 1. Identity: `grok-bot/data-engineering-knowledge-capture` 2. Open an isolation session before knowledge writes (`scripts/brain_session.py open`) unless the human already pointed `SECOND_BRAIN_ROOT` at a session worktree. -3. Pack 2 hops, then write owned types only via this plugin's scripts. +3. For Q&A, spawn **data-retriever** (`/dekc-retrieve`); keep the card only. Pack 2 hops before writing owned types via this plugin's scripts. 4. Close the session to PR. Report path + validation result. 5. Never document a private remote. Never write raw Markdown into the tree. diff --git a/marketplace.json b/marketplace.json index 7ba8dee..8e90829 100644 --- a/marketplace.json +++ b/marketplace.json @@ -11,7 +11,7 @@ "name": "data-engineering-knowledge-capture", "source": "./", "description": "Data lake second brain: schemas, lineage, medallion, semantic layer, glossary. Depends on PKC + OKF.", - "version": "0.5.1", + "version": "0.5.2", "author": { "name": "Rick Hightower" }, diff --git a/package.json b/package.json index f9d1764..0e509cd 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,6 @@ "vite": "^8.2.0" }, "description": "Data Engineering Knowledge Capture (DEKC) \u2014 Claude/Grok/Codex/OpenCode plugin + explorer", - "version": "0.5.1", + "version": "0.5.2", "license": "MIT" } diff --git a/plugin.json b/plugin.json index 03a5f82..cd3c341 100644 --- a/plugin.json +++ b/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "data-engineering-knowledge-capture", - "version": "0.5.1", + "version": "0.5.2", "description": "Data Engineering Knowledge Capture: schemas, lineage, medallion, semantic models, and glossary into a durable OKF knowledge graph. Multi-host bindings and write isolation.", "author": { "name": "Rick Hightower", diff --git a/public/data/catalog.json b/public/data/catalog.json index 3f9fe41..bbf6417 100644 --- a/public/data/catalog.json +++ b/public/data/catalog.json @@ -1,6 +1,6 @@ { "title": "Retail Lake Sample Knowledge", - "version": "0.5.1", + "version": "0.5.2", "depends_on": [ "project-knowledge-capture", "okf-graph-eng" diff --git a/scripts/dekc_pack.py b/scripts/dekc_pack.py index 9a4e8f0..8db8189 100755 --- a/scripts/dekc_pack.py +++ b/scripts/dekc_pack.py @@ -377,17 +377,72 @@ def render_markdown( return "\n".join(md) +LEAD_NODE_LIMIT = 8 + + +def bodies_off(result: dict) -> dict: + """Card path: never include concept bodies (root included).""" + out = dict(result) + out["nodes"] = [{**n, "body": ""} for n in result.get("nodes", [])] + return out + + +def _lead_why(node: dict) -> str: + desc = str(node.get("description") or "").strip() + title = str(node.get("title") or Path(str(node.get("path") or "")).stem) + why = desc.splitlines()[0] if desc else title + if len(why) > 140: + return why[:137] + "..." + return why + + +def render_summary( + result: dict, + *, + tokens: int | None = None, + budget: int | None = None, +) -> str: + """Compact card-friendly pack. Bodies off. No mermaid, no excerpts.""" + focus = result["focus"] + seed = next((n for n in result.get("nodes", []) if n.get("path") == focus), None) + seed_type = (seed or {}).get("type") or "?" + token_bit = "" + if tokens is not None and budget is not None: + token_bit = f" tokens={tokens}/{budget}" + lines = [ + "# Pack summary", + "", + f"- Seed: `{focus}` (`{seed_type}`)", + f"- Engine: {result.get('reverse_index') or 'scan'}", + f"- Pack: hops={result['hops']} nodes={result['node_count']}{token_bit}", + "- Lead nodes:", + ] + for n in result.get("nodes", [])[:LEAD_NODE_LIMIT]: + lines.append( + f" - `{n['path']}` (`{n.get('type') or '?'}`) — {_lead_why(n)}" + ) + lines.append("") + return "\n".join(lines) + + def finalize_markdown( result: dict, *, bundle: Path | None = None, max_tokens: str | int | None = None, window_tokens: str | int | None = None, + summary: bool = False, ) -> tuple[str, dict[str, int]]: window, budget = resolve_pack_budget(max_tokens, window_tokens) - draft = render_markdown(result, bundle=bundle, tokens=0, budget=budget) - tokens = estimate_tokens(draft) - md = render_markdown(result, bundle=bundle, tokens=tokens, budget=budget) + if summary: + card = bodies_off(result) + draft = render_summary(card, tokens=0, budget=budget) + tokens = estimate_tokens(draft) + md = render_summary(card, tokens=tokens, budget=budget) + else: + draft = render_markdown(result, bundle=bundle, tokens=0, budget=budget) + tokens = estimate_tokens(draft) + md = render_markdown(result, bundle=bundle, tokens=tokens, budget=budget) tokens = estimate_tokens(md) meta = {"tokens": tokens, "budget": budget, "window": window} if tokens > budget: @@ -405,6 +460,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--max-tokens", default="") parser.add_argument("--window-tokens", default="") parser.add_argument("--tiny", action="store_true") + parser.add_argument( + "--summary", + action="store_true", + help="Compact card-friendly summary; bodies off (query-time retriever)", + ) parser.add_argument("--json", action="store_true") parser.add_argument("--mermaid", action="store_true") parser.add_argument("--write", action="store_true", help="Write pack under packs/") @@ -443,11 +503,14 @@ def main(argv: list[str] | None = None) -> int: raise PackBudgetError(tokens, budget, window, [n["path"] for n in result["nodes"]]) print(diagram) return 0 + if args.summary: + result = bodies_off(result) md, meta = finalize_markdown( result, bundle=bundle, max_tokens=args.max_tokens, window_tokens=args.window_tokens, + summary=args.summary, ) except PackBudgetError as exc: payload = { @@ -495,7 +558,12 @@ def main(argv: list[str] | None = None) -> int: append_log(bundle, f"Wrote context pack for {result['focus']}") if args.json: - print(json.dumps({k: v for k, v in result.items() if k != "markdown"}, indent=2)) + payload = {k: v for k, v in result.items() if k != "markdown"} + if args.summary: + payload["summary"] = md + for node in payload.get("nodes") or []: + node.pop("body", None) + print(json.dumps(payload, indent=2)) elif not args.write: print(md) return 0 diff --git a/skills/dekc-context/SKILL.md b/skills/dekc-context/SKILL.md index acc1429..7e2bd26 100644 --- a/skills/dekc-context/SKILL.md +++ b/skills/dekc-context/SKILL.md @@ -8,7 +8,10 @@ description: Build progressive-disclosure context packs for a data concept. ```bash python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_pack.py" tables/.md --repo . --bundle knowledge --hops 2 python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_pack.py" tables/.md --repo . --bundle knowledge --tiny +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_pack.py" tables/.md --repo . --bundle knowledge --tiny --summary python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_pack.py" tables/.md --repo . --bundle knowledge --write ``` +`--summary` is card-friendly (bodies off, fail-closed budget). Query-time Q&A must spawn **data-retriever** via `/dekc-retrieve` instead of running this skill in the parent. + When okf-plugin is installed, `okf pack` / impact also work on DEKC concepts. diff --git a/skills/dekc-retrieve/SKILL.md b/skills/dekc-retrieve/SKILL.md new file mode 100644 index 0000000..5a14b82 --- /dev/null +++ b/skills/dekc-retrieve/SKILL.md @@ -0,0 +1,52 @@ +--- +name: dekc-retrieve +description: Query-time DEKC retrieval. Spawn data-retriever; parent gets a summary card only. Use for Table, Metric, LineagePath, IngestionJob, Transformation, Dashboard, DataProduct, GlossaryTerm, BusinessObject questions. +--- + +# Retrieve (query-time) + +The parent **must spawn `data-retriever`**. Do **not** run `dekc_search.py` or `dekc_pack.py` in the parent for retrieval. Search, score, pack, and deepen stay in the sub-agent so hit lists and pack markdown never contaminate the parent. + +This is **not** reverse engineering. Do not spawn `data-lake-walker`, `lineage-tracer`, or other RE/capture agents for a question about existing knowledge. + +## Spawn + +Ask the host to run the `data-retriever` sub-agent with the user's question (and a seed path if the user already named one). The only thing you keep is the retrieval card: + +```markdown +## Retrieval card +- Query: … +- Seed: `/path` (`Type`) — why chosen +- Fit: high|medium|low — one sentence +- Engine: index|rg|scan +- Pack: hops=N nodes=N tokens=N/budget +- Lead nodes: (5–8 bullets) +- Lineage note: (upstream/downstream one-liner) or none +- Open gaps: or none +- Next: stay|deepen-2hop|lineage|try-alt-seed `/other` +``` + +If `Next` is `deepen-2hop`, `lineage`, or `try-alt-seed /other`, spawn `data-retriever` again with that instruction. Do not unpack the graph yourself. + +## Orthogonal fan-out + +When the question also needs project decisions or system topology, spawn in parallel: + +| Plane | Sub-agent | +|-------|-----------| +| Data platform (this pack) | **data-retriever** | +| Project / product knowledge | PKC **knowledge-retriever** | +| System architecture | SAC **architecture-retriever** | + +Do not merge their packs into the parent. Each returns its own card. + +## Not this skill + +| Need | Use | +|------|-----| +| Walk / capture / reverse-engineer | `data-lake-walker` / `/dekc-walk` | +| Raw keyword search (developer) | `/dekc-search` — still not for parent Q&A | +| Full pack for a known seed (retriever internals) | `/dekc-context` | +| Intent checklists (`dekc_brain.py`) | `/dekc-second-brain` — do **not** dump those packs inline for Q&A | + +See `agents/data-retriever.md`. diff --git a/skills/dekc-search/SKILL.md b/skills/dekc-search/SKILL.md index ce95054..646f9f2 100644 --- a/skills/dekc-search/SKILL.md +++ b/skills/dekc-search/SKILL.md @@ -22,3 +22,5 @@ uses FTS5 MATCH (prefix tokens; not score-identical). Missing index or rg is not an error. See `/dekc-index`. After hits on a table, offer `/dekc-context` (2-hop pack). + +For parent Q&A, do **not** run this skill in the parent. Spawn **data-retriever** via `/dekc-retrieve`. Search + pack stay in that sub-agent; the parent gets a retrieval card only. diff --git a/skills/dekc-second-brain/SKILL.md b/skills/dekc-second-brain/SKILL.md index d7c2b2c..5709d61 100644 --- a/skills/dekc-second-brain/SKILL.md +++ b/skills/dekc-second-brain/SKILL.md @@ -37,7 +37,7 @@ python3 "${CLAUDE_PLUGIN_ROOT}/scripts/dekc_brain.py" "gold order_daily" \ ## Workflow -1. **Retrieve** with `dekc_brain.py` (intent + query) → checklist + ranked concepts + 2-hop pack +1. **Retrieve** with `dekc_brain.py` (intent + query) → checklist + ranked concepts + 2-hop pack. For parent Q&A, spawn **data-retriever** (`/dekc-retrieve`) and keep the card only — do not dump the brain pack inline. 2. **Design** using only evidence from the pack (do not invent lineage) 3. **Capture** new assets with `dekc_capture.py` matching `schemas/okf-concepts/` 4. **Index** is optional — search/pack refresh `knowledge/.dekc/index.sqlite` themselves. `dekc_index.py build` is `refresh --force`. diff --git a/tests/test_dekc.py b/tests/test_dekc.py index 17eb70a..40a14f2 100644 --- a/tests/test_dekc.py +++ b/tests/test_dekc.py @@ -3,6 +3,8 @@ from __future__ import annotations +import contextlib +import io import json import re import subprocess @@ -710,6 +712,120 @@ def test_over_budget_fails_closed(self): shutil.rmtree(tmp) +class TestPackSummary(unittest.TestCase): + def _tiny_bundle(self) -> Path: + tmp = Path(tempfile.mkdtemp()) + tables = tmp / "tables" + tables.mkdir() + (tmp / "index.md").write_text( + '---\nokf_version: "0.2"\ntitle: t\n---\n', encoding="utf-8" + ) + (tables / "root.md").write_text( + "---\ntype: Table\ntitle: Lumenfield Root\n" + "description: root-desc for the seed table\n" + "links:\n - target: /tables/neighbor.md\n rel: feeds\n" + "---\n# Lumenfield Root\n\nROOT_BODY_MARKER secret-of-root\n", + encoding="utf-8", + ) + (tables / "neighbor.md").write_text( + "---\ntype: Table\ntitle: Neighbor\n" + "description: neighbor-frontmatter-only\n---\n" + "# Neighbor\n\nNEIGHBOR_BODY_MARKER must-not-pack\n", + encoding="utf-8", + ) + self.addCleanup(shutil.rmtree, tmp) + return tmp + + def test_summary_bodies_off_even_for_root(self): + from dekc_pack import finalize_markdown, pack + + tmp = self._tiny_bundle() + result = pack(tmp, "tables/root.md", hops=1, max_nodes=8) + md, meta = finalize_markdown(result, bundle=tmp, summary=True) + self.assertIn("# Pack summary", md) + self.assertIn("`/tables/root.md`", md) + self.assertIn("(`Table`)", md) + self.assertIn("hops=1", md) + self.assertIn("nodes=", md) + self.assertIn("tokens=", md) + self.assertIn("Lead nodes:", md) + self.assertIn("root-desc for the seed table", md) + self.assertIn("neighbor-frontmatter-only", md) + self.assertNotIn("ROOT_BODY_MARKER", md) + self.assertNotIn("NEIGHBOR_BODY_MARKER", md) + self.assertNotIn("```mermaid", md) + self.assertGreater(meta["budget"], 0) + self.assertLessEqual(meta["tokens"], meta["budget"]) + + def test_summary_smaller_than_full_pack(self): + from dekc_pack import finalize_markdown, pack + + tmp = self._tiny_bundle() + result = pack(tmp, "tables/root.md", hops=1, max_nodes=8) + full, _ = finalize_markdown(result, bundle=tmp, summary=False) + card, _ = finalize_markdown(result, bundle=tmp, summary=True) + self.assertLess(len(card), len(full)) + self.assertIn("ROOT_BODY_MARKER", full) + self.assertNotIn("ROOT_BODY_MARKER", card) + + def test_summary_over_budget_fails_closed(self): + from dekc_pack import PackBudgetError, finalize_markdown, pack + from dekc_pack import main as pack_main + + tmp = self._tiny_bundle() + result = pack(tmp, "tables/root.md", hops=1, max_nodes=8) + with self.assertRaises(PackBudgetError) as ctx: + finalize_markdown(result, bundle=tmp, max_tokens=8, summary=True) + self.assertGreater(ctx.exception.tokens, ctx.exception.budget) + self.assertEqual(ctx.exception.budget, 8) + rc = pack_main( + [ + "tables/root.md", + "--repo", + str(tmp), + "--bundle", + str(tmp), + "--tiny", + "--summary", + "--max-tokens", + "8", + "--write", + "--json", + "--author", + "grok-bot/northstar-console", + ] + ) + self.assertNotEqual(rc, 0) + self.assertEqual(list(tmp.rglob("pack-root.md")), []) + + def test_summary_cli_json_has_no_bodies(self): + from dekc_pack import main as pack_main + + tmp = self._tiny_bundle() + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rc = pack_main( + [ + "tables/root.md", + "--repo", + str(tmp), + "--bundle", + str(tmp), + "--tiny", + "--summary", + "--json", + ] + ) + self.assertEqual(rc, 0) + payload = json.loads(buf.getvalue()) + self.assertIn("summary", payload) + self.assertIn("# Pack summary", payload["summary"]) + self.assertNotIn("ROOT_BODY_MARKER", payload["summary"]) + self.assertNotIn("markdown", payload) + for node in payload["nodes"]: + self.assertFalse(node.get("body")) + + if __name__ == "__main__": unittest.main()