Paper-fidelity fixes: whole-log 15k cap (App.C), audit no/invalid proposals (Alg.1 line 19), exact trace paths for ReAct proposer, applicability conditions in SKILL.md

src/wikiskill/agents.py

@@ -64,15 +64,26 @@ class InferenceAgent:
 
 def stratified_sample(traces: List, max_fail: int = 5, max_pass: int = 3,
                       max_chars: int = 15000) -> List:
-    """Paper Appendix C: <=5 failing + <=3 passing traces, each capped at 15,000 chars."""
+    """Paper Appendix C: <=5 failing + <=3 passing traces. Each individual execution
+    log is capped at max_chars (15,000 in the paper) prior to prompt injection."""
     fails = [t for t in traces if not t.correct][:max_fail]
     passes = [t for t in traces if t.correct][:max_pass]
     sampled = fails + passes
     for t in sampled:
-        if len(t.messages) and isinstance(t.messages[-1].get("content"), str) and \
-                len(t.messages[-1]["content"]) > max_chars:
-            c = t.messages[-1]["content"][:max_chars]
-            t.messages[-1] = dict(t.messages[-1], content=c + "\n...[truncated]")
+        # Cap the WHOLE execution log, not just one message: greedily truncate
+        # message contents (longest first) until the serialized trace fits.
+        def total_len(msgs):
+            return sum(len(str(m.get("content", ""))) for m in msgs)
+        msgs = t.messages
+        while total_len(msgs) > max_chars:
+            longest = max(range(len(msgs)),
+                          key=lambda i: len(str(msgs[i].get("content", ""))))
+            content = str(msgs[longest].get("content", ""))
+            keep = max(0, len(content) - (total_len(msgs) - max_chars) - 30)
+            msgs[longest] = dict(msgs[longest],
+                                 content=content[:keep] + "\n...[truncated]")
+            if len(content) <= 40:  # cannot shrink further; avoid infinite loop
+                break
     return sampled
 
 

src/wikiskill/orchestrator.py

@@ -57,12 +57,14 @@ class EvolutionOrchestrator:
     def _score(self, trajs: List[Trajectory]) -> float:
         return self.metric([t.correct for t in trajs])
 
-    def _outcome_summary(self, trajs: List[Trajectory], tasks) -> str:
+    def _outcome_summary(self, trajs: List[Trajectory], tasks, iteration: int) -> str:
         by_id = {t.id: t for t in tasks}
+        # Exact paths: the ReAct proposer only has read_file (no directory listing),
+        # per paper §3.2.3 -- so it must be able to construct valid trace paths.
         return "\n".join(
             f"{tr.task_id}: {'PASS' if tr.correct else 'FAIL'} | "
             f"predicted={tr.prediction!r} ground_truth={by_id[tr.task_id].y!r} "
-            f"| trace: raw/iter_*/{tr.task_id}.json" for tr in trajs)
+            f"| trace: raw/iter_{iteration:03d}/{tr.task_id}.json" for tr in trajs)
 
     def _baseline_validation(self) -> None:
         self.log("Baseline validation (S0 = empty) ...")
@@ -88,7 +90,7 @@ class EvolutionOrchestrator:
                      f"errors={report.get('errors')}")
             # line 11: skill proposal
             proposal = self.proposer.propose(
-                self._outcome_summary(train_trajs, self.dataset.train), k)
+                self._outcome_summary(train_trajs, self.dataset.train, k), k)
             entry = {"iteration": k, "proposal": None, "val_score": None,
                      "accepted": False}
             if proposal:
@@ -109,6 +111,9 @@ class EvolutionOrchestrator:
                         self.skills.restore(snap)      # line 17: skills-only rollback
                     self.log(f"Proposal {applied['action']} '{applied['skill']}' -> "
                              f"val={score:.4f} ({'ACCEPTED' if accepted else 'REJECTED'})")
+                    entry.update({"proposal": applied.get("skill"),
+                                  "action": applied["action"], "val_score": score,
+                                  "accepted": accepted})
                     # line 19: harness records audit trail in skill-impact.md
                     diff = applied.get("diff", "(new skill)")
                     self.ws.record_skill_impact(
@@ -118,11 +123,23 @@ class EvolutionOrchestrator:
                         f"- outcome: {'Accepted' if accepted else 'Rejected'}\n"
                         f"- purpose: {proposal.get('purpose', '')}\n"
                         f"- diff:\n```diff\n{diff}\n```\n")
-                    entry.update({"proposal": applied.get("skill"),
-                                  "action": applied["action"], "val_score": score,
-                                  "accepted": accepted})
+                else:
+                    # Invalid proposal: record it so later iterations never repeat it
+                    # (paper: skill-impact.md is the objective audit trail of ALL
+                    # past interventions).
+                    self.skills.restore(snap)
+                    self.ws.record_skill_impact(
+                        f"\n## Iteration {k} — invalid proposal (discarded)\n"
+                        f"- outcome: Rejected\n"
+                        f"- reason: proposal could not be applied\n"
+                        f"- purpose: {proposal.get('purpose', '')}\n")
             else:
+                # Algorithm 1 line 19 runs every iteration; log no-proposal outcomes
+                # in the evolution log so future proposals avoid repeating them.
                 self.log("No proposal produced this iteration.")
+                self.ws.apply_maintainer_update({
+                    "append_log": f"Iteration {k}: proposer emitted no valid proposal; "
+                                  f"no skill change was evaluated."})
             self.history.append(entry)
         final_skills = self.skills.list_skills()
         result = {"r_best": self.r_best, "history": self.history,

src/wikiskill/prompts.py

@@ -72,6 +72,8 @@ by the persistent wiki.
    diagnose root causes before proposing.
 3. Propose exactly ONE change: either create a new skill (full content with YAML
    frontmatter: name, description) or patch one existing skill with minimal edits.
+   SKILL.md must also include an "## When to use" section stating the applicability
+   conditions (task types / situations in which the skill applies).
 4. In "purpose", map the skill back to the motivating wiki patterns.
 5. When ready, emit ONLY the finish tool call with valid JSON.
 

tests/test_agents.py

@@ -67,3 +67,17 @@ def test_stratified_sample():
     s = stratified_sample(traces)
     assert sum(1 for x in s if not x.correct) == 5
     assert sum(1 for x in s if x.correct) == 3
+
+
+def test_stratified_sample_caps_whole_log():
+    """Paper App. C: the ENTIRE execution log is capped at 15,000 chars, not just
+    the last message."""
+    from wikiskill.agents import stratified_sample
+    from wikiskill.datasets import Trajectory
+    big = Trajectory(task_id="big", correct=False, tool_calls=[], final_answer=None,
+                     prediction=None,
+                     messages=[{"role": "system", "content": "s" * 9000},
+                               {"role": "assistant", "content": "a" * 9000}])
+    (sampled,) = stratified_sample([big])
+    assert sum(len(str(m.get("content", ""))) for m in sampled.messages) < 16000
+    assert any("[truncated]" in str(m.get("content", "")) for m in sampled.messages)

tests/test_e2e.py

@@ -56,3 +56,40 @@ def test_e2e_early_stop_at_perfect_val(tmp_path):
     # With the skill accepted, val = 1.0 -> the loop must early-stop.
     assert result["r_best"] == 1.0
     assert len(result["history"]) <= 2
+
+
+def test_e2e_no_proposal_iterations_are_audited(tmp_path):
+    """Algorithm 1 line 19 runs every iteration: when the proposer emits nothing,
+    the iteration must still leave a trace in the wiki audit trail."""
+    from wikiskill.orchestrator import EvolutionOrchestrator
+
+    def bad_proposer(messages, call_log):
+        return "Sorry, I could not analyze the traces."
+
+    llm = MockLLM(handlers={"Skill Proposer Agent": bad_proposer})
+    ws = Workspace(str(tmp_path / "ws"))
+    orch = EvolutionOrchestrator(llm=llm, dataset=make_demo_dataset(), ws=ws,
+                                 tools=_demo_tools(), K=2)
+    result = orch.run()
+    assert result["final_skills"] == []           # nothing accepted
+    logs = ws.wiki_logs()
+    assert "no valid proposal" in logs            # audited in evolution log
+    impact_path = tmp_path / "ws" / "wiki" / "skill-impact.md"
+    assert not impact_path.exists()               # skill-impact only logs proposals
+
+
+def test_e2e_invalid_proposal_recorded_in_skill_impact(tmp_path):
+    from wikiskill.orchestrator import EvolutionOrchestrator
+
+    def bad_create(messages, call_log):
+        return ('{"action": "create", "skill": "bad name!", "content": "x", '
+                '"purpose": "p"}')  # invalid skill name -> apply fails
+
+    llm = MockLLM(handlers={"Skill Proposer Agent": bad_create})
+    ws = Workspace(str(tmp_path / "ws"))
+    orch = EvolutionOrchestrator(llm=llm, dataset=make_demo_dataset(), ws=ws,
+                                 tools=_demo_tools(), K=2)
+    orch.run()
+    from wikiskill.skills import SkillsLayer
+    assert "invalid proposal" in ws.skill_impact()
+    assert SkillsLayer(ws).list_skills() == []    # rolled back / never applied