diff --git a/ontogeny/server/ontogeny/evolve/evalrunner.py b/ontogeny/server/ontogeny/evolve/evalrunner.py index 3fa0e82c2..f80740dd1 100644 --- a/ontogeny/server/ontogeny/evolve/evalrunner.py +++ b/ontogeny/server/ontogeny/evolve/evalrunner.py @@ -349,6 +349,10 @@ async def run( report["cases"].append({"name": case.name, "kind": "agent", "ok": ok, **details}) report["passed"] = report["passed"] and ok + if not report["cases"]: + report["passed"] = False + report["error"] = "no executable eval cases" + return report async def _run_agent_case(self, source_session, case, candidate=None) -> tuple[bool, dict[str, Any]]: diff --git a/ontogeny/tests/test_evolve.py b/ontogeny/tests/test_evolve.py index e6a2f06d4..f47c96d44 100644 --- a/ontogeny/tests/test_evolve.py +++ b/ontogeny/tests/test_evolve.py @@ -219,6 +219,26 @@ async def test_replay_matches_history_and_guards_double_completion(self, env): # overall suite green (queries included) assert report["passed"] is True + async def test_empty_and_coverage_only_suites_do_not_pass(self, env): + from ontogeny.core.models import CoverageCfg + + base = env["compiled"].eval_suites["lifecycle-flow-suite"] + cases = [] + for coverage in (None, CoverageCfg(objects=["wafer-lot"])): + suite = base.model_copy(deep=True) + suite.spec.queries = [] + suite.spec.replays = [] + suite.spec.agents = [] + suite.spec.coverage = coverage + cases.append(suite) + + async with env["maker"]() as s: + reports = [await env["runner"].run(s, suite) for suite in cases] + + assert all(report["cases"] == [] for report in reports) + assert all(report["passed"] is False for report in reports) + assert all(report["error"] == "no executable eval cases" for report in reports) + async def test_replay_sees_derived_properties_on_both_revision_states(self, env): from ontogeny.core.models import PropertyDef, Rule