import json from wikiskill.datasets import make_demo_dataset, _demo_tools from wikiskill.llm import MockLLM from wikiskill.skills import SkillsLayer from wikiskill.workspace import Workspace def test_gating_math(): """Strict improvement: accept iff score > R_best (paper eq. 4).""" def gate(score, r_best): return score > r_best assert gate(0.5, 0.5) is False # ties are rejected assert gate(0.6, 0.5) is True assert gate(0.4, 0.5) is False def test_e2e_evolution_loop(tmp_path, capsys): from wikiskill.orchestrator import EvolutionOrchestrator root = tmp_path / "ws" ws = Workspace(str(root)) orch = EvolutionOrchestrator(llm=MockLLM(), dataset=make_demo_dataset(), ws=ws, tools=_demo_tools(), K=3) result = orch.run() # Baseline (no skill) fails on val; after skill creation, val should improve. assert result["r_best"] > 0.0 assert "unit-conversion" in result["final_skills"] # Three-layer artifacts exist assert (root / "raw" / "iter_000").is_dir() # baseline val rollouts assert (root / "raw" / "iter_001" / "train").exists() or \ list((root / "raw" / "iter_001").glob("*.json")) assert (root / "wiki" / "patterns" / "rounding-errors.md").is_file() assert (root / "wiki" / "index.md").is_file() assert (root / "wiki" / "logs.md").is_file() assert (root / "wiki" / "skill-impact.md").is_file() assert (root / "skills" / "unit-conversion" / "SKILL.md").is_file() assert (root / "skills" / "unit-conversion" / "PURPOSE.md").is_file() # Harness wrote the audit trail with outcome + diff impact = (root / "wiki" / "skill-impact.md").read_text(encoding="utf-8") assert "Accepted" in impact # run_state.json persisted state = json.loads((root / "run_state.json").read_text(encoding="utf-8")) assert state["final_skills"] == ["unit-conversion"] def test_e2e_early_stop_at_perfect_val(tmp_path): from wikiskill.orchestrator import EvolutionOrchestrator ws = Workspace(str(tmp_path / "ws")) orch = EvolutionOrchestrator(llm=MockLLM(), dataset=make_demo_dataset(), ws=ws, tools=_demo_tools(), K=5) result = orch.run() # With the skill accepted, val = 1.0 -> the loop must early-stop. assert result["r_best"] == 1.0 assert len(result["history"]) <= 2 def test_e2e_no_proposal_iterations_are_audited(tmp_path): """Algorithm 1 line 19 runs every iteration: when the proposer emits nothing, the iteration must still leave a trace in the wiki audit trail.""" from wikiskill.orchestrator import EvolutionOrchestrator def bad_proposer(messages, call_log): return "Sorry, I could not analyze the traces." llm = MockLLM(handlers={"Skill Proposer Agent": bad_proposer}) ws = Workspace(str(tmp_path / "ws")) orch = EvolutionOrchestrator(llm=llm, dataset=make_demo_dataset(), ws=ws, tools=_demo_tools(), K=2) result = orch.run() assert result["final_skills"] == [] # nothing accepted logs = ws.wiki_logs() assert "no valid proposal" in logs # audited in evolution log impact_path = tmp_path / "ws" / "wiki" / "skill-impact.md" assert not impact_path.exists() # skill-impact only logs proposals def test_e2e_invalid_proposal_recorded_in_skill_impact(tmp_path): from wikiskill.orchestrator import EvolutionOrchestrator def bad_create(messages, call_log): return ('{"action": "create", "skill": "bad name!", "content": "x", ' '"purpose": "p"}') # invalid skill name -> apply fails llm = MockLLM(handlers={"Skill Proposer Agent": bad_create}) ws = Workspace(str(tmp_path / "ws")) orch = EvolutionOrchestrator(llm=llm, dataset=make_demo_dataset(), ws=ws, tools=_demo_tools(), K=2) orch.run() from wikiskill.skills import SkillsLayer assert "invalid proposal" in ws.skill_impact() assert SkillsLayer(ws).list_skills() == [] # rolled back / never applied