WikiSkill: reference implementation of arXiv:2608.27454 (Google Research) - three-layer workspace, evolutionary agents, validation gating

tests/test_agents.py

@@ -0,0 +1,69 @@
+from wikiskill.datasets import make_demo_dataset, _demo_tools
+from wikiskill.llm import MockLLM, extract_json
+from wikiskill.skills import SkillsLayer
+from wikiskill.workspace import Workspace
+
+
+def _ws(tmp_path):
+    return Workspace(str(tmp_path / "ws"))
+
+
+def test_extract_json():
+    assert extract_json('```json\n{"a": 1}\n```') == {"a": 1}
+    assert extract_json('noise {"a": {"b": 2}} tail') == {"a": {"b": 2}}
+    assert extract_json("no json") is None
+
+
+def test_skills_create_snapshot_restore(tmp_path):
+    skills = SkillsLayer(_ws(tmp_path))
+    skills.apply_proposal({"action": "create", "skill": "s1",
+                           "content": "# S1\nline1\nline2\n", "purpose": "why"})
+    assert skills.list_skills() == ["s1"]
+    snap = skills.snapshot()
+    skills.apply_proposal({"action": "patch", "skill": "s1",
+                           "edits": [{"op": "replace", "target": "line2",
+                                      "content": "LINE2"}], "purpose": "why2"})
+    assert "LINE2" in skills.skill_content("s1")
+    skills.restore(snap)
+    assert "LINE2" not in skills.skill_content("s1")
+    skills.apply_proposal({"action": "create", "skill": "s2", "content": "x",
+                           "purpose": "p"})
+    skills.restore(snap)
+    assert skills.list_skills() == ["s1"]
+
+
+def test_skill_injection_context(tmp_path):
+    skills = SkillsLayer(_ws(tmp_path))
+    assert "(No active skills.)" in skills.active_skills_context()
+    skills.apply_proposal({"action": "create", "skill": "s", "content": "BODY",
+                           "purpose": "p"})
+    assert "BODY" in skills.active_skills_context()
+
+
+def test_mock_inference_without_and_with_skill(tmp_path):
+    from wikiskill.agents import InferenceAgent
+    ws = _ws(tmp_path)
+    skills = SkillsLayer(ws)
+    agent = InferenceAgent(MockLLM(), _demo_tools(), ws, skills)
+    ds = make_demo_dataset()
+    traj = agent.run(ds.val[0])
+    assert traj.correct is False  # truncation error without skill
+    skills.apply_proposal({"action": "create", "skill": "unit-conversion",
+                           "content": "Round ONLY the final result to 2 decimal places.",
+                           "purpose": "p"})
+    agent2 = InferenceAgent(MockLLM(), _demo_tools(), ws, skills)
+    traj2 = agent2.run(ds.val[0])
+    assert traj2.correct is True
+
+
+def test_stratified_sample():
+    from wikiskill.agents import stratified_sample
+    from wikiskill.datasets import Trajectory
+
+    def t(i, ok):
+        return Trajectory(task_id=i, messages=[{"role": "user", "content": "x"}],
+                          tool_calls=[], final_answer="a", correct=ok)
+    traces = [t(f"f{i}", False) for i in range(9)] + [t(f"p{i}", True) for i in range(5)]
+    s = stratified_sample(traces)
+    assert sum(1 for x in s if not x.correct) == 5
+    assert sum(1 for x in s if x.correct) == 3

tests/test_e2e.py

@@ -0,0 +1,58 @@
+import json
+
+from wikiskill.datasets import make_demo_dataset, _demo_tools
+from wikiskill.llm import MockLLM
+from wikiskill.skills import SkillsLayer
+from wikiskill.workspace import Workspace
+
+
+def test_gating_math():
+    """Strict improvement: accept iff score > R_best (paper eq. 4)."""
+    def gate(score, r_best):
+        return score > r_best
+    assert gate(0.5, 0.5) is False   # ties are rejected
+    assert gate(0.6, 0.5) is True
+    assert gate(0.4, 0.5) is False
+
+
+def test_e2e_evolution_loop(tmp_path, capsys):
+    from wikiskill.orchestrator import EvolutionOrchestrator
+    root = tmp_path / "ws"
+    ws = Workspace(str(root))
+    orch = EvolutionOrchestrator(llm=MockLLM(), dataset=make_demo_dataset(), ws=ws,
+                                 tools=_demo_tools(), K=3)
+    result = orch.run()
+
+    # Baseline (no skill) fails on val; after skill creation, val should improve.
+    assert result["r_best"] > 0.0
+    assert "unit-conversion" in result["final_skills"]
+
+    # Three-layer artifacts exist
+    assert (root / "raw" / "iter_000").is_dir()       # baseline val rollouts
+    assert (root / "raw" / "iter_001" / "train").exists() or \
+        list((root / "raw" / "iter_001").glob("*.json"))
+    assert (root / "wiki" / "patterns" / "rounding-errors.md").is_file()
+    assert (root / "wiki" / "index.md").is_file()
+    assert (root / "wiki" / "logs.md").is_file()
+    assert (root / "wiki" / "skill-impact.md").is_file()
+    assert (root / "skills" / "unit-conversion" / "SKILL.md").is_file()
+    assert (root / "skills" / "unit-conversion" / "PURPOSE.md").is_file()
+
+    # Harness wrote the audit trail with outcome + diff
+    impact = (root / "wiki" / "skill-impact.md").read_text(encoding="utf-8")
+    assert "Accepted" in impact
+
+    # run_state.json persisted
+    state = json.loads((root / "run_state.json").read_text(encoding="utf-8"))
+    assert state["final_skills"] == ["unit-conversion"]
+
+
+def test_e2e_early_stop_at_perfect_val(tmp_path):
+    from wikiskill.orchestrator import EvolutionOrchestrator
+    ws = Workspace(str(tmp_path / "ws"))
+    orch = EvolutionOrchestrator(llm=MockLLM(), dataset=make_demo_dataset(), ws=ws,
+                                 tools=_demo_tools(), K=5)
+    result = orch.run()
+    # With the skill accepted, val = 1.0 -> the loop must early-stop.
+    assert result["r_best"] == 1.0
+    assert len(result["history"]) <= 2

tests/test_metrics.py

@@ -0,0 +1,13 @@
+from wikiskill.metrics import accuracy, paired_bootstrap
+
+
+def test_accuracy():
+    assert accuracy([True, False, True, True]) == 0.75
+    assert accuracy([]) == 0.0
+
+
+def test_paired_bootstrap_significant_and_not():
+    a = [True] * 10 + [False] * 0
+    b = [True] * 5 + [False] * 5
+    assert paired_bootstrap(a, b, n=500, seed=1) < 0.05
+    assert paired_bootstrap(a, a, n=200, seed=1) == 1.0

tests/test_workspace.py

@@ -0,0 +1,46 @@
+import pytest
+
+from wikiskill.workspace import PatchError, Workspace, apply_edits
+
+
+def test_patch_ops():
+    c = "alpha\nbeta\n"
+    assert apply_edits(c, [{"op": "append", "content": "gamma"}]) == "alpha\nbeta\ngamma"
+    assert apply_edits(c, [{"op": "replace", "target": "beta", "content": "BETA"}]) \
+        == "alpha\nBETA\n"
+    assert apply_edits(c, [{"op": "insert_after", "target": "alpha", "content": " mid"}]) \
+        == "alpha mid\nbeta\n"
+    with pytest.raises(PatchError):
+        apply_edits(c, [{"op": "replace", "target": "nope", "content": "x"}])
+    with pytest.raises(PatchError):
+        apply_edits(c, [{"op": "bogus", "content": "x"}])
+
+
+def test_raw_layer_immutable(tmp_path):
+    ws = Workspace(str(tmp_path / "ws"))
+    ws.add_raw_trace(1, "t1", {"a": 1})
+    with pytest.raises(FileExistsError):
+        ws.add_raw_trace(1, "t1", {"a": 2})
+
+
+def test_read_file_scoped(tmp_path):
+    ws = Workspace(str(tmp_path / "ws"))
+    assert "ERROR" in ws.read_file("missing.md")
+    with pytest.raises(ValueError):
+        ws.read_file("../outside.txt")
+
+
+def test_maintainer_update(tmp_path):
+    ws = Workspace(str(tmp_path / "ws"))
+    report = ws.apply_maintainer_update({
+        "create_patterns": [{"name": "p.md", "content": "# P\nbody\n"}],
+        "update_patterns": [{"name": "p.md",
+                             "edits": [{"op": "append", "content": "more\n"}]}],
+        "update_index": "- [p](wiki/patterns/p.md): x\n",
+        "append_log": "iter 1 findings",
+    })
+    assert report["created"] == ["p.md"] and report["updated"] == ["p.md"]
+    assert ws.pattern_page("p.md").endswith("more\n")
+    assert "[p]" in ws.wiki_index() and "iter 1" in ws.wiki_logs()
+    ws.record_skill_impact("## entry\n- outcome: Accepted\n")
+    assert "Accepted" in ws.skill_impact()