Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ontogeny/server/ontogeny/evolve/evalrunner.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,10 @@ async def run(
report["cases"].append({"name": case.name, "kind": "agent", "ok": ok, **details})
report["passed"] = report["passed"] and ok

if not report["cases"]:
report["passed"] = False
report["error"] = "no executable eval cases"

return report

async def _run_agent_case(self, source_session, case, candidate=None) -> tuple[bool, dict[str, Any]]:
Expand Down
20 changes: 20 additions & 0 deletions ontogeny/tests/test_evolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,26 @@ async def test_replay_matches_history_and_guards_double_completion(self, env):
# overall suite green (queries included)
assert report["passed"] is True

async def test_empty_and_coverage_only_suites_do_not_pass(self, env):
from ontogeny.core.models import CoverageCfg

base = env["compiled"].eval_suites["lifecycle-flow-suite"]
cases = []
for coverage in (None, CoverageCfg(objects=["wafer-lot"])):
suite = base.model_copy(deep=True)
suite.spec.queries = []
suite.spec.replays = []
suite.spec.agents = []
suite.spec.coverage = coverage
cases.append(suite)

async with env["maker"]() as s:
reports = [await env["runner"].run(s, suite) for suite in cases]

assert all(report["cases"] == [] for report in reports)
assert all(report["passed"] is False for report in reports)
assert all(report["error"] == "no executable eval cases" for report in reports)

async def test_replay_sees_derived_properties_on_both_revision_states(self, env):
from ontogeny.core.models import PropertyDef, Rule

Expand Down
Loading