diff --git a/src/wikiskill/agents.py b/src/wikiskill/agents.py index 6b8a3ba..90d3517 100644 --- a/src/wikiskill/agents.py +++ b/src/wikiskill/agents.py @@ -64,15 +64,26 @@ class InferenceAgent: def stratified_sample(traces: List, max_fail: int = 5, max_pass: int = 3, max_chars: int = 15000) -> List: - """Paper Appendix C: <=5 failing + <=3 passing traces, each capped at 15,000 chars.""" + """Paper Appendix C: <=5 failing + <=3 passing traces. Each individual execution + log is capped at max_chars (15,000 in the paper) prior to prompt injection.""" fails = [t for t in traces if not t.correct][:max_fail] passes = [t for t in traces if t.correct][:max_pass] sampled = fails + passes for t in sampled: - if len(t.messages) and isinstance(t.messages[-1].get("content"), str) and \ - len(t.messages[-1]["content"]) > max_chars: - c = t.messages[-1]["content"][:max_chars] - t.messages[-1] = dict(t.messages[-1], content=c + "\n...[truncated]") + # Cap the WHOLE execution log, not just one message: greedily truncate + # message contents (longest first) until the serialized trace fits. + def total_len(msgs): + return sum(len(str(m.get("content", ""))) for m in msgs) + msgs = t.messages + while total_len(msgs) > max_chars: + longest = max(range(len(msgs)), + key=lambda i: len(str(msgs[i].get("content", "")))) + content = str(msgs[longest].get("content", "")) + keep = max(0, len(content) - (total_len(msgs) - max_chars) - 30) + msgs[longest] = dict(msgs[longest], + content=content[:keep] + "\n...[truncated]") + if len(content) <= 40: # cannot shrink further; avoid infinite loop + break return sampled diff --git a/src/wikiskill/orchestrator.py b/src/wikiskill/orchestrator.py index 02b0d6e..ce3f215 100644 --- a/src/wikiskill/orchestrator.py +++ b/src/wikiskill/orchestrator.py @@ -57,12 +57,14 @@ class EvolutionOrchestrator: def _score(self, trajs: List[Trajectory]) -> float: return self.metric([t.correct for t in trajs]) - def _outcome_summary(self, trajs: List[Trajectory], tasks) -> str: + def _outcome_summary(self, trajs: List[Trajectory], tasks, iteration: int) -> str: by_id = {t.id: t for t in tasks} + # Exact paths: the ReAct proposer only has read_file (no directory listing), + # per paper §3.2.3 -- so it must be able to construct valid trace paths. return "\n".join( f"{tr.task_id}: {'PASS' if tr.correct else 'FAIL'} | " f"predicted={tr.prediction!r} ground_truth={by_id[tr.task_id].y!r} " - f"| trace: raw/iter_*/{tr.task_id}.json" for tr in trajs) + f"| trace: raw/iter_{iteration:03d}/{tr.task_id}.json" for tr in trajs) def _baseline_validation(self) -> None: self.log("Baseline validation (S0 = empty) ...") @@ -88,7 +90,7 @@ class EvolutionOrchestrator: f"errors={report.get('errors')}") # line 11: skill proposal proposal = self.proposer.propose( - self._outcome_summary(train_trajs, self.dataset.train), k) + self._outcome_summary(train_trajs, self.dataset.train, k), k) entry = {"iteration": k, "proposal": None, "val_score": None, "accepted": False} if proposal: @@ -109,6 +111,9 @@ class EvolutionOrchestrator: self.skills.restore(snap) # line 17: skills-only rollback self.log(f"Proposal {applied['action']} '{applied['skill']}' -> " f"val={score:.4f} ({'ACCEPTED' if accepted else 'REJECTED'})") + entry.update({"proposal": applied.get("skill"), + "action": applied["action"], "val_score": score, + "accepted": accepted}) # line 19: harness records audit trail in skill-impact.md diff = applied.get("diff", "(new skill)") self.ws.record_skill_impact( @@ -118,11 +123,23 @@ class EvolutionOrchestrator: f"- outcome: {'Accepted' if accepted else 'Rejected'}\n" f"- purpose: {proposal.get('purpose', '')}\n" f"- diff:\n```diff\n{diff}\n```\n") - entry.update({"proposal": applied.get("skill"), - "action": applied["action"], "val_score": score, - "accepted": accepted}) + else: + # Invalid proposal: record it so later iterations never repeat it + # (paper: skill-impact.md is the objective audit trail of ALL + # past interventions). + self.skills.restore(snap) + self.ws.record_skill_impact( + f"\n## Iteration {k} — invalid proposal (discarded)\n" + f"- outcome: Rejected\n" + f"- reason: proposal could not be applied\n" + f"- purpose: {proposal.get('purpose', '')}\n") else: + # Algorithm 1 line 19 runs every iteration; log no-proposal outcomes + # in the evolution log so future proposals avoid repeating them. self.log("No proposal produced this iteration.") + self.ws.apply_maintainer_update({ + "append_log": f"Iteration {k}: proposer emitted no valid proposal; " + f"no skill change was evaluated."}) self.history.append(entry) final_skills = self.skills.list_skills() result = {"r_best": self.r_best, "history": self.history, diff --git a/src/wikiskill/prompts.py b/src/wikiskill/prompts.py index c3dc6c1..4cd69d8 100644 --- a/src/wikiskill/prompts.py +++ b/src/wikiskill/prompts.py @@ -72,6 +72,8 @@ by the persistent wiki. diagnose root causes before proposing. 3. Propose exactly ONE change: either create a new skill (full content with YAML frontmatter: name, description) or patch one existing skill with minimal edits. + SKILL.md must also include an "## When to use" section stating the applicability + conditions (task types / situations in which the skill applies). 4. In "purpose", map the skill back to the motivating wiki patterns. 5. When ready, emit ONLY the finish tool call with valid JSON. diff --git a/tests/test_agents.py b/tests/test_agents.py index 919444b..00a40c4 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -67,3 +67,17 @@ def test_stratified_sample(): s = stratified_sample(traces) assert sum(1 for x in s if not x.correct) == 5 assert sum(1 for x in s if x.correct) == 3 + + +def test_stratified_sample_caps_whole_log(): + """Paper App. C: the ENTIRE execution log is capped at 15,000 chars, not just + the last message.""" + from wikiskill.agents import stratified_sample + from wikiskill.datasets import Trajectory + big = Trajectory(task_id="big", correct=False, tool_calls=[], final_answer=None, + prediction=None, + messages=[{"role": "system", "content": "s" * 9000}, + {"role": "assistant", "content": "a" * 9000}]) + (sampled,) = stratified_sample([big]) + assert sum(len(str(m.get("content", ""))) for m in sampled.messages) < 16000 + assert any("[truncated]" in str(m.get("content", "")) for m in sampled.messages) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 1e09d5f..72dc4f6 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -56,3 +56,40 @@ def test_e2e_early_stop_at_perfect_val(tmp_path): # With the skill accepted, val = 1.0 -> the loop must early-stop. assert result["r_best"] == 1.0 assert len(result["history"]) <= 2 + + +def test_e2e_no_proposal_iterations_are_audited(tmp_path): + """Algorithm 1 line 19 runs every iteration: when the proposer emits nothing, + the iteration must still leave a trace in the wiki audit trail.""" + from wikiskill.orchestrator import EvolutionOrchestrator + + def bad_proposer(messages, call_log): + return "Sorry, I could not analyze the traces." + + llm = MockLLM(handlers={"Skill Proposer Agent": bad_proposer}) + ws = Workspace(str(tmp_path / "ws")) + orch = EvolutionOrchestrator(llm=llm, dataset=make_demo_dataset(), ws=ws, + tools=_demo_tools(), K=2) + result = orch.run() + assert result["final_skills"] == [] # nothing accepted + logs = ws.wiki_logs() + assert "no valid proposal" in logs # audited in evolution log + impact_path = tmp_path / "ws" / "wiki" / "skill-impact.md" + assert not impact_path.exists() # skill-impact only logs proposals + + +def test_e2e_invalid_proposal_recorded_in_skill_impact(tmp_path): + from wikiskill.orchestrator import EvolutionOrchestrator + + def bad_create(messages, call_log): + return ('{"action": "create", "skill": "bad name!", "content": "x", ' + '"purpose": "p"}') # invalid skill name -> apply fails + + llm = MockLLM(handlers={"Skill Proposer Agent": bad_create}) + ws = Workspace(str(tmp_path / "ws")) + orch = EvolutionOrchestrator(llm=llm, dataset=make_demo_dataset(), ws=ws, + tools=_demo_tools(), K=2) + orch.run() + from wikiskill.skills import SkillsLayer + assert "invalid proposal" in ws.skill_impact() + assert SkillsLayer(ws).list_skills() == [] # rolled back / never applied