mirror of
https://github.com/kenhuangus/wikiskill
synced 2026-09-22 23:34:25 +08:00
96 lines
4.0 KiB
Python
96 lines
4.0 KiB
Python
import json
|
|
|
|
from wikiskill.datasets import make_demo_dataset, _demo_tools
|
|
from wikiskill.llm import MockLLM
|
|
from wikiskill.skills import SkillsLayer
|
|
from wikiskill.workspace import Workspace
|
|
|
|
|
|
def test_gating_math():
|
|
"""Strict improvement: accept iff score > R_best (paper eq. 4)."""
|
|
def gate(score, r_best):
|
|
return score > r_best
|
|
assert gate(0.5, 0.5) is False # ties are rejected
|
|
assert gate(0.6, 0.5) is True
|
|
assert gate(0.4, 0.5) is False
|
|
|
|
|
|
def test_e2e_evolution_loop(tmp_path, capsys):
|
|
from wikiskill.orchestrator import EvolutionOrchestrator
|
|
root = tmp_path / "ws"
|
|
ws = Workspace(str(root))
|
|
orch = EvolutionOrchestrator(llm=MockLLM(), dataset=make_demo_dataset(), ws=ws,
|
|
tools=_demo_tools(), K=3)
|
|
result = orch.run()
|
|
|
|
# Baseline (no skill) fails on val; after skill creation, val should improve.
|
|
assert result["r_best"] > 0.0
|
|
assert "unit-conversion" in result["final_skills"]
|
|
|
|
# Three-layer artifacts exist
|
|
assert (root / "raw" / "iter_000").is_dir() # baseline val rollouts
|
|
assert (root / "raw" / "iter_001" / "train").exists() or \
|
|
list((root / "raw" / "iter_001").glob("*.json"))
|
|
assert (root / "wiki" / "patterns" / "rounding-errors.md").is_file()
|
|
assert (root / "wiki" / "index.md").is_file()
|
|
assert (root / "wiki" / "logs.md").is_file()
|
|
assert (root / "wiki" / "skill-impact.md").is_file()
|
|
assert (root / "skills" / "unit-conversion" / "SKILL.md").is_file()
|
|
assert (root / "skills" / "unit-conversion" / "PURPOSE.md").is_file()
|
|
|
|
# Harness wrote the audit trail with outcome + diff
|
|
impact = (root / "wiki" / "skill-impact.md").read_text(encoding="utf-8")
|
|
assert "Accepted" in impact
|
|
|
|
# run_state.json persisted
|
|
state = json.loads((root / "run_state.json").read_text(encoding="utf-8"))
|
|
assert state["final_skills"] == ["unit-conversion"]
|
|
|
|
|
|
def test_e2e_early_stop_at_perfect_val(tmp_path):
|
|
from wikiskill.orchestrator import EvolutionOrchestrator
|
|
ws = Workspace(str(tmp_path / "ws"))
|
|
orch = EvolutionOrchestrator(llm=MockLLM(), dataset=make_demo_dataset(), ws=ws,
|
|
tools=_demo_tools(), K=5)
|
|
result = orch.run()
|
|
# With the skill accepted, val = 1.0 -> the loop must early-stop.
|
|
assert result["r_best"] == 1.0
|
|
assert len(result["history"]) <= 2
|
|
|
|
|
|
def test_e2e_no_proposal_iterations_are_audited(tmp_path):
|
|
"""Algorithm 1 line 19 runs every iteration: when the proposer emits nothing,
|
|
the iteration must still leave a trace in the wiki audit trail."""
|
|
from wikiskill.orchestrator import EvolutionOrchestrator
|
|
|
|
def bad_proposer(messages, call_log):
|
|
return "Sorry, I could not analyze the traces."
|
|
|
|
llm = MockLLM(handlers={"Skill Proposer Agent": bad_proposer})
|
|
ws = Workspace(str(tmp_path / "ws"))
|
|
orch = EvolutionOrchestrator(llm=llm, dataset=make_demo_dataset(), ws=ws,
|
|
tools=_demo_tools(), K=2)
|
|
result = orch.run()
|
|
assert result["final_skills"] == [] # nothing accepted
|
|
logs = ws.wiki_logs()
|
|
assert "no valid proposal" in logs # audited in evolution log
|
|
impact_path = tmp_path / "ws" / "wiki" / "skill-impact.md"
|
|
assert not impact_path.exists() # skill-impact only logs proposals
|
|
|
|
|
|
def test_e2e_invalid_proposal_recorded_in_skill_impact(tmp_path):
|
|
from wikiskill.orchestrator import EvolutionOrchestrator
|
|
|
|
def bad_create(messages, call_log):
|
|
return ('{"action": "create", "skill": "bad name!", "content": "x", '
|
|
'"purpose": "p"}') # invalid skill name -> apply fails
|
|
|
|
llm = MockLLM(handlers={"Skill Proposer Agent": bad_create})
|
|
ws = Workspace(str(tmp_path / "ws"))
|
|
orch = EvolutionOrchestrator(llm=llm, dataset=make_demo_dataset(), ws=ws,
|
|
tools=_demo_tools(), K=2)
|
|
orch.run()
|
|
from wikiskill.skills import SkillsLayer
|
|
assert "invalid proposal" in ws.skill_impact()
|
|
assert SkillsLayer(ws).list_skills() == [] # rolled back / never applied
|