From 0f2df89097b848d567282f72e3df5e6b0fd48480 Mon Sep 17 00:00:00 2001 From: "DistributedApps.AI" Date: Sun, 30 Aug 2026 23:07:08 -0400 Subject: [PATCH] WikiSkill: reference implementation of arXiv:2608.27454 (Google Research) - three-layer workspace, evolutionary agents, validation gating --- .gitignore | 10 +++ LICENSE | 29 +++++++ README.md | 99 ++++++++++++++++++++++ design.md | 140 +++++++++++++++++++++++++++++++ plan.md | 45 ++++++++++ prd.md | 93 +++++++++++++++++++++ pyproject.toml | 19 +++++ src/wikiskill/__init__.py | 6 ++ src/wikiskill/agents.py | 134 ++++++++++++++++++++++++++++++ src/wikiskill/cli.py | 71 ++++++++++++++++ src/wikiskill/datasets.py | 106 ++++++++++++++++++++++++ src/wikiskill/demo.py | 32 +++++++ src/wikiskill/llm.py | 139 +++++++++++++++++++++++++++++++ src/wikiskill/metrics.py | 28 +++++++ src/wikiskill/orchestrator.py | 135 ++++++++++++++++++++++++++++++ src/wikiskill/prompts.py | 82 ++++++++++++++++++ src/wikiskill/skills.py | 90 ++++++++++++++++++++ src/wikiskill/workspace.py | 152 ++++++++++++++++++++++++++++++++++ tests/test_agents.py | 69 +++++++++++++++ tests/test_e2e.py | 58 +++++++++++++ tests/test_metrics.py | 13 +++ tests/test_workspace.py | 46 ++++++++++ 22 files changed, 1596 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 design.md create mode 100644 plan.md create mode 100644 prd.md create mode 100644 pyproject.toml create mode 100644 src/wikiskill/__init__.py create mode 100644 src/wikiskill/agents.py create mode 100644 src/wikiskill/cli.py create mode 100644 src/wikiskill/datasets.py create mode 100644 src/wikiskill/demo.py create mode 100644 src/wikiskill/llm.py create mode 100644 src/wikiskill/metrics.py create mode 100644 src/wikiskill/orchestrator.py create mode 100644 src/wikiskill/prompts.py create mode 100644 src/wikiskill/skills.py create mode 100644 src/wikiskill/workspace.py create mode 100644 tests/test_agents.py create mode 100644 tests/test_e2e.py create mode 100644 tests/test_metrics.py create mode 100644 tests/test_workspace.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e0d7601 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +__pycache__/ +*.pyc +.pytest_cache/ +*.egg-info/ +build/ +dist/ +wikiskill_demo_workspace/ +wikiskill_workspace/ +demo_ws/ +.vscode/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b7b785a --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +MIT License + +Copyright (c) 2026 Repository contributors + +Code in this repository is released under the MIT License. + +The WikiSkill method and prompts reimplemented here originate from the paper +"WikiSkill: Compiling Agent Experience into Persistent Knowledge for Skill Evolution" +by Tang et al. (Google Research), arXiv:2608.27454, published under CC BY 4.0. +The paper's prompts (adapted in src/wikiskill/prompts.py) remain subject to CC BY 4.0; +attribute the original authors when reusing this work. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..61ef2fb --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +# WikiSkill — Implementation + +Python implementation of **"WikiSkill: Compiling Agent Experience into Persistent +Knowledge for Skill Evolution"** (arXiv:2608.27454v1, Tang et al., Google Research). + +Docs: see [`prd.md`](prd.md) (requirements), [`design.md`](design.md) (architecture), +[`plan.md`](plan.md) (build plan). + +## Credits & Attribution + +This project is an independent, open-source reimplementation of the research paper: + +> **WikiSkill: Compiling Agent Experience into Persistent Knowledge for Skill Evolution** +> Liyan Tang, Cyrus Rashtchian, Chun-Sung Ferng, Andrew Tomkins, Da-Cheng Juan, Tu Vu — +> **Google Research** (and Virginia Tech). +> arXiv:2608.27454v1 [cs.AI], 27 Aug 2026. License: CC BY 4.0. +> Paper: https://arxiv.org/abs/2608.27454 + +All credit for the WikiSkill **method, algorithm (Algorithm 1), three-layer knowledge +architecture, agent designs, experimental findings, and agent prompts (Appendix E)** +belongs to the Google Research authors. This repository reimplements the framework in +code for study/experimentation purposes and is **not** an official Google product. +Agent system prompts in [`src/wikiskill/prompts.py`](src/wikiskill/prompts.py) are +condensed adaptations of those published in the paper (CC BY 4.0); adaptations in this +repo are released under the same spirit — please cite the original paper if you use +this work: + +```bibtex +@article{tang2026wikiskill, + title = {WikiSkill: Compiling Agent Experience into Persistent Knowledge + for Skill Evolution}, + author = {Tang, Liyan and Rashtchian, Cyrus and Ferng, Chun-Sung and + Tomkins, Andrew and Juan, Da-Cheng and Vu, Tu}, + journal = {arXiv preprint arXiv:2608.27454}, + year = {2026}, + note = {Google Research} +} +``` + + +## What it implements + +- **Three-layer workspace** (paper §3.1): immutable `raw/` traces, persistent `wiki/` + (patterns + index + logs + skill-impact tracker), active `skills/` (SKILL.md + PURPOSE.md). +- **Evolutionary loop** (paper Algorithm 1, §3.2): Inference Agent rollouts with skill + injection but **no wiki access**; Wiki Maintainer consolidation (create/update pattern + pages via `append`/`replace`/`insert_after` patches); a **ReAct Skill Proposer** that + reads the wiki and raw traces on demand and emits one atomic skill proposal. +- **Gating & rollback** (§3.2.4): accept iff validation score strictly improves `R_best`; + otherwise roll back **skills only** — the wiki is never rolled back. The harness + programmatically appends proposal + unified diff + outcome to `wiki/skill-impact.md`. +- **Stratified trace sampling** (App. C): ≤5 failing + ≤3 passing traces, 15,000-char cap. +- **Paired bootstrap significance testing** (App. C) in `metrics.py`. +- Early stop when `R_best == 1.0`. + +## Quickstart + +```bash +pip install -e ".[dev]" +pytest # 14 tests incl. end-to-end loop +python -m wikiskill.demo # offline demo with the deterministic MockLLM +``` + +Demo output: a `wikiskill_demo_workspace/` with `raw/`, `wiki/`, `skills/` populated, +accepted/rejected proposals logged in `wiki/skill-impact.md`, and `run_state.json`. + +## Using a real LLM + +```bash +wikiskill evolve --tasks tasks.jsonl \ + --train-ids train.txt --val-ids val.txt --test-ids test.txt \ + --model --base-url https://your-endpoint/v1 +``` + +API key is read from `--api-key`, `WIKISKILL_API_KEY`, or `OPENAI_API_KEY`. +Any OpenAI-compatible `/chat/completions` endpoint works (including local vLLM, as in +the paper). `tasks.jsonl` lines: `{"id", "x", "y", "meta?"}`. + +## Layout + +``` +src/wikiskill/ +├── workspace.py # three layers + patch engine + scoped read_file +├── skills.py # skills layer, apply_proposal, skills-only snapshot/rollback +├── agents.py # InferenceAgent, WikiMaintainer, ReAct SkillProposer, stratified_sample +├── orchestrator.py # Algorithm 1 with gating/rollback + audit trail +├── prompts.py # agent system prompts (adapted from paper Appendix E) +├── llm.py # LLM protocol, OpenAI-compatible client, MockLLM, extract_json +├── metrics.py # accuracy + paired bootstrap (1,000 iterations) +├── datasets.py # dataclasses + synthetic demo dataset +├── demo.py / cli.py +└── tests/… # unit + end-to-end tests +``` + +## Notes & limitations (mirroring the paper) + +- Skills are injected directly into the prompt (no retrieval), matching the paper's setup. +- Gating requires strict improvement; neutral proposals are rejected (paper Limitations). +- The wiki accumulates without pruning (paper Limitations). diff --git a/design.md b/design.md new file mode 100644 index 0000000..e9e0813 --- /dev/null +++ b/design.md @@ -0,0 +1,140 @@ +# WikiSkill — Design Document + +Implements: arXiv:2608.27454v1 (WikiSkill). See `prd.md` for requirements. + +## 1. Architecture overview + +``` +wikiskill/ +├── pyproject.toml +├── src/wikiskill/ +│ ├── __init__.py +│ ├── llm.py # LLM provider abstraction: MockLLM, OpenAICompatLLM +│ ├── workspace.py # Three-layer workspace: raw/, wiki/, skills/ + patch engine +│ ├── prompts.py # System prompts (Inference / Wiki Maintainer / Skill Proposer) +│ ├── agents.py # InferenceAgent, WikiMaintainer, SkillProposer (ReAct) +│ ├── orchestrator.py # Algorithm 1: evolution loop, gating, rollback, logging +│ ├── metrics.py # accuracy metric + paired bootstrap significance test +│ ├── datasets.py # Task/Trace dataclasses + demo dataset +│ ├── demo.py # End-to-end demo run with mock LLM +│ └── cli.py # `wikiskill evolve` / `wikiskill demo` entry point +└── tests/ + ├── test_workspace.py + ├── test_gating.py + └── test_e2e.py +``` + +## 2. Core data model (`datasets.py`) + +- `Task(x, y, meta)` — task instance + ground truth (paper: 𝒟 = {(x_i, y_i)}). +- `ToolCall / Trajectory` — trace `τ = (o1, a1, ..., oT, aT)` captured as messages + + structured metadata (final answer, correct flag, tool calls and outputs). +- `ToolSpec(name, description, fn)` — environment tools 𝒰 (bash-like demo tools, `read_file` + for the proposer). + +## 3. Workspace (`workspace.py`) — the three layers + +`Workspace(root)` manages: + +## 4. LLM abstraction (`llm.py`) + +```python +class LLM(Protocol): + def chat(self, messages: list[dict], **kw) -> str: ... +``` + +- `OpenAICompatLLM`: calls any OpenAI-compatible `/chat/completions` endpoint + (configurable base_url, api_key via env, model, temperature). Uses `urllib` (no hard deps). +- `MockLLM`: deterministic scripted responder for tests/demo. Routes on content: + - Maintainer JSON → returns a small valid wiki update JSON. + - Proposer JSON → returns a valid create-skill proposal. + - Inference → returns a demo answer (extracted from task text). +- Robust JSON extraction helper (`extract_json`) shared by all agents (strips code fences). + +## 5. Agents (`agents.py`) + +### 5.1 Inference Agent +Runs one task: system prompt = task-type prompt + **injected active skills** +(full SKILL.md contents, no wiki access — paper §3.2/ablation). Loop: +LLM → tool call → observation → ... → final `ANSWER: ...` line or max turns. +Returns `Trajectory` (messages, tool calls, final answer, correct flag). + +### 5.2 Wiki Maintainer (`M_WM`) +One LLM call (full-batch; paper Appendix D: B = N_train). Input: current wiki context +(index + all pattern pages + logs + skill-impact) and the stratified trace sample. +Output JSON parsed into workspace ops: +`create_patterns[]`, `update_patterns[] (edits[])`, `update_index`, `append_log`. +A **stratified sampler** picks ≤5 failing + ≤3 passing traces, each truncated to 15,000 chars. + +### 5.3 Wiki-Informed Skill Proposer (`M_P`) — ReAct +Loop of up to `max_react_turns` (default 20, paper: ~10–20): +- Initial context: wiki index, `skill-impact.md`, pass/fail outcome summary (no pre-sampled traces). +- Tools: `read_file(path)` (workspace-scoped) + `finish(...)` to emit the final proposal. +- Final output JSON: `{"action": "create"|"patch", "skill": name, "content"/"edits": [...], + "purpose": mapping to wiki patterns}` — atomic, single-skill per paper §3.2.3. + +## 6. Orchestrator (`orchestrator.py`) — Algorithm 1 + +```python +EvolutionOrchestrator(llm, tools, dataset, workspace, metric, K, ...).run() +``` +Per iteration k (state S_k, W_k exactly as in the paper): +1. early stop if `R_best == 1.0`; +2. roll out training tasks with S_{k-1} → raw traces (skill injection, wiki hidden); +3. stratified sample T_sample,k; +4. `W'_k = M_WM(W_{k-1}, T_sample,k)`; +5. `P_k = M_P(W'_k, S_{k-1}, T_train,k)`; +6. snapshot skills; `S'_k = Apply(S_{k-1}, P_k)`; +7. validate on D_val → `R(T_val,k)`; +8. accept iff `R > R_best` else restore snapshot (skills-only rollback); +9. `W_k = Update(W'_k, P_k, R, a_k)` → append unified diff + outcome to `skill-impact.md`; +10. persist run state (history JSON: per-iteration scores, accept/reject, best). + +## 7. Metrics & significance (`metrics.py`) + +- `accuracy(traces)` — fraction correct. +- `paired_bootstrap(mine, base, n=1000, seed)` — resample test tasks with replacement, + return p-value that margins ≤ 0; used for "best method" claims (paper Appendix C). + +## 8. Demo dataset (`datasets.py`) + +`make_demo_dataset(seed)`: synthetic "unit-conversion" arithmetic tasks. The no-skill agent +often errs (rounding/format pitfalls baked into the mock); a good skill (the mock proposer +writes one) nudges the inference prompt to compute correctly — so validation gating can +genuinely accept/reject. Also pluggable: any `Dataset` of `Task`s with tools + metric. + +## 9. Error handling & robustness + +- Malformed LLM JSON → retry once, then skip gracefully (maintainer: no-op update with log + entry; proposer: proposal = None, iteration recorded as "no proposal"). +- Missing files, bad patch targets → logged, op skipped (never crashes the loop). +- Path traversal blocked in `read_file`. + +## 10. Testing strategy + +- Unit: patch ops (exact/insert_after/append, invalid target), snapshot/restore, + path allowlist, immutable raw layer. +- Integration: gating math (accept on strict improvement, reject on equal/lower), + skill-impact.md written by harness with unified diff. +- E2E: MockLLM-driven full loop (K=3) — assert artifacts exist, acceptance behavior, + early-stop at R_best=1.0. + + +- **Raw layer**: `add_raw_trace(iter_k, task_id, trace_dict)` writes + `raw/iter_{k}/{task_id}.json`. Immutable: an existing file is never rewritten (enforced). +- **Wiki layer** (`wiki/`): + - `patterns/` pages: markdown files; updates via a patch engine implementing exactly the + three ops from the paper's Maintainer prompt: `append`, `replace` (exact-substring + target), `insert_after` (exact-substring target). + - `index.md`: full-content rewrite (maintainer always supplies complete index). + - `logs.md`: append-only evolution log. + - `skill-impact.md`: append-only; the **orchestrator** (not any agent) writes the + programmatic entry after gating: iteration, proposal metadata, target skill, unified + diff (`difflib.unified_diff` of skill content), validation score, accept/reject. +- **Skills layer**: `skills//SKILL.md` + `PURPOSE.md`. + - `apply_proposal(P)`: create (new dir with SKILL.md + PURPOSE.md) or patch (patch ops on + SKILL.md, mirrored to PURPOSE.md updates). + - `snapshot()` / `restore(snapshot)`: cheap rollback of the Skills layer only (dict of + name → content; rollback deletes/rewrites files accordingly). Wiki/raw untouched. +- `read_file(rel_path)` with a path-allowlist (agents may only read within the workspace) + — used by the ReAct Skill Proposer. diff --git a/plan.md b/plan.md new file mode 100644 index 0000000..f847f48 --- /dev/null +++ b/plan.md @@ -0,0 +1,45 @@ +# WikiSkill — Implementation Plan + +Source: paper + `prd.md` + `design.md`. Ordered milestones with verification gates. + +## Milestone 0 — Project scaffold +- [ ] `wikiskill/pyproject.toml` (no third-party runtime deps; pytest for dev) +- [ ] Package skeleton `src/wikiskill/` + `tests/` +- Verify: `pip install -e .[dev]` works; `pytest` collects. + +## Milestone 1 — Data model + LLM abstraction +- [ ] `datasets.py`: `Task`, `ToolSpec`, `Trajectory`, `Dataset`, demo dataset factory +- [ ] `llm.py`: `LLM` protocol, `OpenAICompatLLM` (urllib), `MockLLM`, `extract_json` +- Verify: unit test for `extract_json` + MockLLM routing. + +## Milestone 2 — Three-layer workspace (paper §3.1) +- [ ] `workspace.py`: raw traces (immutable), wiki (patterns/index/logs/skill-impact), + patch engine (`append`/`replace`/`insert_after`), skills (SKILL.md + PURPOSE.md), + snapshot/restore, scoped `read_file` +- Verify: `test_workspace.py` passes (incl. immutability + path traversal). + +## Milestone 3 — Agents (paper §3.2.1–3.2.3) +- [ ] `prompts.py`: three agent system prompts adapted from paper Appendix E +- [ ] `agents.py`: InferenceAgent (tool loop, skill injection, no wiki), WikiMaintainer + (single call, JSON ops), SkillProposer (ReAct, ≤20 turns, `read_file`, atomic proposal) +- Verify: mock-driven unit tests for each agent's outputs. + +## Milestone 4 — Orchestrator: gating & rollback (paper §3.2.4, Algorithm 1) +- [ ] `orchestrator.py`: full loop, stratified sampling (≤5 fail / ≤3 pass, 15k char cap), + strict-improvement gate, skills-only rollback, programmatic `skill-impact.md` diff + logging, early stop at R_best = 1.0, run history persistence +- Verify: `test_gating.py` passes (accept/reject math, wiki never rolled back). + +## Milestone 5 — Metrics & significance (paper Appendix C) +- [ ] `metrics.py`: accuracy + paired bootstrap (1,000 iterations) +- Verify: unit test with seeded RNG. + +## Milestone 6 — End-to-end validation +- [ ] `demo.py` + `cli.py` +- [ ] `test_e2e.py`: K=3 loop with MockLLM; assert layers populated, gating works, artifacts +- [ ] Live run: `python -m wikiskill.demo` +- Verify: all tests green; demo produces workspace with accepted skill or logged rejections. + +## Milestone 7 — Documentation & wrap-up +- [ ] `README.md` (usage, mapping to paper) +- Final review of all docs vs. implementation. diff --git a/prd.md b/prd.md new file mode 100644 index 0000000..9ca3af8 --- /dev/null +++ b/prd.md @@ -0,0 +1,93 @@ +# WikiSkill — PRD (Product Requirements Document) + +Source paper: **"WikiSkill: Compiling Agent Experience into Persistent Knowledge for Skill Evolution"** +(Tang, Rashtchian, Ferng, Tomkins, Juan, Vu — Google Research / Virginia Tech, arXiv:2608.27454v1) + +## 1. Background + +Agent skills package specialized procedural knowledge into reusable, filesystem-based +resources. Recent methods (Trace2Skill, EvoSkill, SkillOpt) discover skills from agent +execution traces, but the insights that guide skill development remain scattered across +optimization histories, limiting systematic reuse across iterations. + +WikiSkill co-evolves agent skills with a **persistent knowledge base (wiki)**. It separates +raw experience, accumulated knowledge, and executable skills, and continuously consolidates +experience into the wiki so that subsequent skill updates build on it. + +## 2. Goals + +- G1: Implement the full WikiSkill evolution loop (paper Algorithm 1) as a runnable Python framework. +- G2: Faithfully reproduce the paper's three-layer knowledge architecture (`raw/`, `wiki/`, `skills/`). +- G3: Implement the three evolutionary agents: Inference Agent, Wiki Maintainer, and a + ReAct-style Wiki-Informed Skill Proposer. +- G4: Implement validation gating with skill-only rollback (the wiki is never rolled back). +- G5: Make the system testable end-to-end without any LLM API key (mock LLM provider), + while supporting real providers (OpenAI-compatible APIs) for actual use. + +## 3. Non-goals + +- No skill retrieval/triggering: active skills are injected directly into the Inference + Agent's prompt (per the paper's setup, which isolates skill quality). +- No re-implementation of baselines (Trace2Skill / EvoSkill / SkillOpt). +- No benchmark harnesses for the paper's exact datasets (LiveMath, SealQA, SpreadsheetBench, + OfficeQA, ALFWorld); a simple demo dataset with an identical interface is provided instead. + +## 4. Functional requirements + +### FR1 — Three-layer workspace +- **Raw Layer (`raw/`)**: immutable execution traces per iteration (reasoning, tool calls, + tool outputs, final answers, pass/fail, prediction vs. ground truth). Never modified. +- **Wiki Layer (`wiki/`)**: persistent, compounding; contains + - `wiki/patterns/*.md` — pattern pages (failure modes / successful strategies + workarounds), + - `wiki/index.md` — concise catalog of patterns, + - `wiki/logs.md` — evolution log appended by the Wiki Maintainer each iteration, + - `wiki/skill-impact.md` — skill impact tracker updated programmatically by the harness + after gating (proposal metadata, target skill, unified diff, val score, accept/reject). +- **Skills Layer (`skills//`)**: each skill contains `SKILL.md` (frontmatter: name, + description, plus procedural instructions) and `PURPOSE.md` (mapping to motivating wiki patterns). + +### FR2 — Evolution loop (paper Algorithm 1) +1. Initialize `S0 = ∅`, `W0 = ∅`; baseline validation `R_best = R(T_val,0)`. +2. For k = 1..K (early-stop if `R_best == 1.0`): + - Roll out Inference Agent on `D_train` with skills `S_{k-1}` (NO wiki access). + - Sample `T_sample,k ⊂ T_train,k` (stratified: ≤5 failing, ≤3 passing traces; each trace + capped at 15,000 characters). + - Wiki Maintenance: `W'_k ← M_WM(W_{k-1}, T_sample,k)` — create/update pattern pages via + patch operations, update index, append log. + - Skill Proposal: `P_k ← M_P(W'_k, S_{k-1}, T_train,k)` — a ReAct agent that reads the wiki + index, skill-impact tracker, outcome summary, and on-demand reads pattern pages and raw + traces via a `read_file` tool. Produces ONE atomic proposal targeting a single skill + (create new skill OR patch-based edit). + - Apply: `S'_k = Apply(S_{k-1}, P_k)`. + - Validate on `D_val`. + - Gate: accept iff `R(T_val,k) > R_best` (strict improvement); otherwise roll back skills + to `S_{k-1}`. Wiki is never rolled back. + - Update wiki: append proposal metadata + diff + score + outcome to `skill-impact.md`. +3. Return final skill set `S_K` and wiki `W_K`. + +### FR3 — Agents +- **Inference Agent**: LLM agent with environment tools; receives task context + injected + active skill contents; multi-turn trajectory. +- **Wiki Maintainer**: single LLM call per iteration; outputs JSON with + `create_patterns`, `update_patterns` (patch ops: append/replace/insert_after), + `update_index` (full content), `append_log`. +- **Skill Proposer**: multi-turn ReAct agent (paper: ~10–20 turns) with `read_file` and a + final structured proposal (create/patch one skill). + +### FR4 — Gating and persistence +- Strict-improvement acceptance, rollback of skills only, programmatic `skill-impact.md` updates. +- Wiki state persists across all iterations and rollbacks. + +### FR5 — Configurability & evaluation +- Pluggable LLM provider (mock / OpenAI-compatible), configurable K, tools, splits. +- Metric per dataset (accuracy); paired bootstrap significance testing helper (1,000 iterations) + as in paper Appendix C. + +## 5. Acceptance criteria + +1. `pytest` passes: unit tests for workspace/patches/gating, and an end-to-end test of the + full evolution loop using a scripted mock LLM. +2. A demo run (`python -m wikiskill.demo`) executes ≥2 evolution iterations, produces a + workspace with all three layers populated, and accepts/rolls back proposals correctly. +3. Code mirrors paper notation (S_k, W_k, P_k, T_sample, R_best) and cites Algorithm 1 steps + in docstrings. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8d7126d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "wikiskill" +version = "0.1.0" +description = "WikiSkill: Compiling Agent Experience into Persistent Knowledge for Skill Evolution (arXiv:2608.27454v1)" +requires-python = ">=3.9" +dependencies = [] + +[project.optional-dependencies] +dev = ["pytest>=7"] + +[project.scripts] +wikiskill = "wikiskill.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/src/wikiskill/__init__.py b/src/wikiskill/__init__.py new file mode 100644 index 0000000..de63413 --- /dev/null +++ b/src/wikiskill/__init__.py @@ -0,0 +1,6 @@ +"""WikiSkill: co-evolving agent skills with a persistent knowledge base (wiki). + +Implementation of arXiv:2608.27454v1. +""" + +__version__ = "0.1.0" diff --git a/src/wikiskill/agents.py b/src/wikiskill/agents.py new file mode 100644 index 0000000..6b8a3ba --- /dev/null +++ b/src/wikiskill/agents.py @@ -0,0 +1,134 @@ +"""Evolutionary agents (paper §3.2): InferenceAgent, WikiMaintainer, SkillProposer.""" +from __future__ import annotations + +import json +import re +from typing import Callable, Dict, List, Optional + +from .llm import LLM, extract_json +from .prompts import INFERENCE_SYSTEM, SKILL_PROPOSER_SYSTEM, WIKI_MAINTAINER_SYSTEM +from .skills import SkillsLayer +from .workspace import Workspace + +ANSWER_RE = re.compile(r"ANSWER:\s*(.+)", re.IGNORECASE) + + +def _parse_answer(text: str) -> Optional[str]: + m = ANSWER_RE.search(text or "") + return m.group(1).strip() if m else None + + +class InferenceAgent: + """§3.2.1: π(x_i; S). Active skills are injected; wiki access is NOT given.""" + + def __init__(self, llm: LLM, tools: list, ws: Workspace, skills: SkillsLayer, + max_turns: int = 6): + self.llm, self.ws, self.skills = llm, ws, skills + self.tools = {t.name: t for t in tools} + self.max_turns = max_turns + + def run(self, task) -> "object": + from .datasets import Trajectory + system = INFERENCE_SYSTEM.replace("{active_skills}", + self.skills.active_skills_context()) + messages: List[dict] = [{"role": "system", "content": system}, + {"role": "user", "content": task.x}] + tool_calls: List[dict] = [] + final: Optional[str] = None + for _ in range(self.max_turns): + text = self.llm.chat(messages, temperature=0.0) + messages.append({"role": "assistant", "content": text}) + parsed = _parse_answer(text) + if parsed is not None: + final = parsed + break + tm = re.search(r'TOOL:\s*(\S+)\s*(\{.*\})?', text, re.DOTALL) + if not tm or tm.group(1) not in self.tools: + messages.append({"role": "user", "content": + "Please reply with TOOL: {json args} or a final " + "'ANSWER: ...' line."}) + continue + tool = self.tools[tm.group(1)] + try: + args = json.loads(tm.group(2) or "{}") + except json.JSONDecodeError: + args = {} + obs = tool.fn(**args) + tool_calls.append({"tool": tool.name, "args": args, "observation": obs}) + messages.append({"role": "user", "content": f"Observation: {obs}"}) + traj = Trajectory(task_id=task.id, messages=messages, tool_calls=tool_calls, + final_answer=final, correct=False, prediction=final) + traj.correct = (final is not None and str(final).strip() == str(task.y).strip()) + return traj + + +def stratified_sample(traces: List, max_fail: int = 5, max_pass: int = 3, + max_chars: int = 15000) -> List: + """Paper Appendix C: <=5 failing + <=3 passing traces, each capped at 15,000 chars.""" + fails = [t for t in traces if not t.correct][:max_fail] + passes = [t for t in traces if t.correct][:max_pass] + sampled = fails + passes + for t in sampled: + if len(t.messages) and isinstance(t.messages[-1].get("content"), str) and \ + len(t.messages[-1]["content"]) > max_chars: + c = t.messages[-1]["content"][:max_chars] + t.messages[-1] = dict(t.messages[-1], content=c + "\n...[truncated]") + return sampled + + +class WikiMaintainer: + """§3.2.2: W'_k = M_WM(W_{k-1}, T_sample,k). One LLM call per iteration.""" + + def __init__(self, llm: LLM, ws: Workspace): + self.llm, self.ws = llm, ws + + def consolidate(self, sampled_traces: List[dict], iteration: int) -> dict: + trace_dump = json.dumps([t.to_dict() for t in sampled_traces], + indent=1, ensure_ascii=False) + user = (f"## Iteration {iteration}\n\n## Current wiki context\n" + f"{self.ws.wiki_context()}\n\n## Latest execution traces\n{trace_dump}\n\n" + "Produce the wiki update JSON now.") + messages = [{"role": "system", "content": WIKI_MAINTAINER_SYSTEM}, + {"role": "user", "content": user}] + parsed = extract_json(self.llm.chat(messages, temperature=0.2)) + if parsed is None: # one retry, then graceful no-op + parsed = extract_json(self.llm.chat(messages + [{"role": "user", "content": + "Your previous reply was not valid JSON. Return ONLY the JSON object."}], + temperature=0.0)) + return parsed or {} + +class SkillProposer: + """§3.2.3: P_k = M_P(W'_k, S_{k-1}, T_train,k). Multi-turn ReAct agent that + reads the wiki index, skill-impact tracker, and traces on demand via read_file.""" + + def __init__(self, llm: LLM, ws: Workspace, skills: SkillsLayer, max_turns: int = 20): + self.llm, self.ws, self.skills = llm, ws, skills + self.max_turns = max_turns + + def propose(self, outcome_summary: str, iteration: int) -> Optional[dict]: + index = self.ws.wiki_index() or "(empty wiki)" + impact = self.ws.skill_impact() or "(no proposals yet)" + user = (f"## Iteration {iteration}\n\n## Wiki index\n{index}\n\n" + f"## Skill impact history\n{impact}\n\n" + f"## Training task outcomes (pass/fail summary)\n{outcome_summary}\n\n" + "Use read_file to inspect pattern pages (wiki/patterns/...) and raw traces " + "(raw/iter_.../...) as needed, then finish with your atomic proposal JSON.") + messages = [{"role": "system", "content": SKILL_PROPOSER_SYSTEM}, + {"role": "user", "content": user}] + for _ in range(self.max_turns): + text = self.llm.chat(messages, temperature=0.2) + parsed = extract_json(text) + if parsed and parsed.get("action") in ("create", "patch"): + return parsed + fm = re.search(r'"path"\s*:\s*"([^"]+)"', text) + if fm: + obs = self.ws.read_file(fm.group(1)) + messages.append({"role": "assistant", "content": text}) + messages.append({"role": "user", + "content": f"read_file observation:\n{obs}"}) + continue + messages.append({"role": "assistant", "content": text}) + messages.append({"role": "user", "content": + "Reply with a read_file tool call or the finish JSON."}) + return None + diff --git a/src/wikiskill/cli.py b/src/wikiskill/cli.py new file mode 100644 index 0000000..40bb841 --- /dev/null +++ b/src/wikiskill/cli.py @@ -0,0 +1,71 @@ +"""CLI: `wikiskill demo` (offline mock) or `wikiskill evolve` (real LLM endpoint).""" +from __future__ import annotations + +import argparse +import json +import os +import sys + + +def main(argv=None) -> int: + p = argparse.ArgumentParser("wikiskill") + sub = p.add_subparsers(dest="cmd", required=True) + + d = sub.add_parser("demo", help="run the offline demo with the mock LLM") + d.add_argument("--root", default="wikiskill_demo_workspace") + d.add_argument("--iterations", type=int, default=3) + + e = sub.add_parser("evolve", help="evolve skills on a JSONL task file") + e.add_argument("--tasks", required=True, help="JSONL file with {id,x,y} objects") + e.add_argument("--train-ids", required=True) + e.add_argument("--val-ids", required=True) + e.add_argument("--test-ids", required=True) + e.add_argument("--root", default="wikiskill_workspace") + e.add_argument("--model", required=True) + e.add_argument("--base-url", default=None) + e.add_argument("--api-key", default=None) + e.add_argument("--iterations", type=int, default=8) + args = p.parse_args(argv) + + if args.cmd == "demo": + from .demo import run_demo + run_demo(args.root, args.iterations) + return 0 + + from .datasets import Dataset, Task, ToolSpec, load_jsonl + from .llm import OpenAICompatLLM + from .orchestrator import EvolutionOrchestrator + from .workspace import Workspace + + rows = {r["id"]: r for r in load_jsonl(args.tasks)} + + def ids(path): + with open(path, "r", encoding="utf-8") as f: + return [line.strip() for line in f if line.strip()] + + def to_tasks(id_list): + return [Task(id=i, x=rows[i]["x"], y=rows[i]["y"], meta=rows[i].get("meta", {})) + for i in id_list if i in rows] + + dataset = Dataset(train=to_tasks(ids(args.train_ids)), + val=to_tasks(ids(args.val_ids)), + test=to_tasks(ids(args.test_ids))) + + def calculator(expression: str) -> str: + try: + return repr(eval(expression, {"__builtins__": {}}, {})) + except Exception as exc: # noqa: BLE001 + return f"ERROR: {exc}" + + tools = [ToolSpec("calculator", "Evaluate a Python arithmetic expression.", calculator)] + llm = OpenAICompatLLM(model=args.model, base_url=args.base_url, api_key=args.api_key) + orch = EvolutionOrchestrator(llm=llm, dataset=dataset, + ws=Workspace(args.root), tools=tools, + K=args.iterations) + result = orch.run() + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/wikiskill/datasets.py b/src/wikiskill/datasets.py new file mode 100644 index 0000000..d7e38fb --- /dev/null +++ b/src/wikiskill/datasets.py @@ -0,0 +1,106 @@ +"""Data model + demo dataset (design.md §2, §8).""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional + + +@dataclass +class Task: + """One task instance: paper §2 𝒟 = {(x_i, y_i)}.""" + id: str + x: str # task context shown to the agent + y: str # ground-truth answer + meta: Dict = field(default_factory=dict) + + +@dataclass +class ToolSpec: + """An environment tool 𝒰 (paper §2). fn(**kwargs) -> str observation.""" + name: str + description: str + fn: Callable[..., str] + + def schema(self) -> dict: + return {"name": self.name, "description": self.description} + + +@dataclass +class Trajectory: + """Execution trace τ_i = (o1, a1, ..., oT, aT) plus grading metadata.""" + task_id: str + messages: List[dict] + tool_calls: List[dict] + final_answer: Optional[str] + correct: bool + prediction: Optional[str] = None + + def to_dict(self) -> dict: + return { + "task_id": self.task_id, + "messages": self.messages, + "tool_calls": self.tool_calls, + "final_answer": self.final_answer, + "correct": self.correct, + "prediction": self.prediction, + } + + +@dataclass +class Dataset: + """Train/val/test split (paper §2).""" + train: List[Task] + val: List[Task] + test: List[Task] + name: str = "demo" + + +def _demo_tools() -> List[ToolSpec]: + """Simple calculator tool for the demo environment.""" + def calc(expression: str) -> str: + try: + return repr(eval(expression, {"__builtins__": {}}, {})) + except Exception as e: # noqa: BLE001 - demo tool + return f"ERROR: {e}" + return [ToolSpec("calculator", "Evaluate a Python arithmetic expression.", calc)] + + +def make_demo_dataset(seed: int = 0) -> Dataset: + """Synthetic unit-conversion dataset (deterministic). + + Tasks embed `expected=`; the MockLLM truncates decimals without the + evolved skill and answers exactly with it — so gating has real signal. + """ + conversions = [ + ("kilometers to miles", 0.621371, "miles"), + ("miles to kilometers", 1.609344, "kilometers"), + ("kilograms to pounds", 2.20462, "pounds"), + ("pounds to kilograms", 0.453592, "kilograms"), + ("inches to centimeters", 2.54, "centimeters"), + ("gallons to liters", 3.78541, "liters"), + ] + counts = [3, 2, 2, 2, 1, 1] # 11 tasks: 5 train / 3 val / 3 test + tasks: List[Task] = [] + n = 0 + for (name, factor, unit), count in zip(conversions, counts): + for i in range(count): + n += 1 + value = round(1.0 + ((n * 7 + seed * 13) % 50) / 3.0, 4) + answer = round(value * factor, 2) + x = (f"Convert {value} {name.split(' to ')[0]} to {unit}. " + f"The conversion factor is {factor}. " + f"Report the numeric value rounded to exactly 2 decimal places. " + f"(reference: expected={answer})") + tasks.append(Task(id=f"task-{n:03d}", x=x, y=f"{answer}", meta={"unit": unit})) + return Dataset(train=tasks[:5], val=tasks[5:8], test=tasks[8:], name="demo-unitconv") + + +def load_jsonl(path: str) -> List[dict]: + out = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + out.append(json.loads(line)) + return out diff --git a/src/wikiskill/demo.py b/src/wikiskill/demo.py new file mode 100644 index 0000000..9d8ae37 --- /dev/null +++ b/src/wikiskill/demo.py @@ -0,0 +1,32 @@ +"""End-to-end demo run of the WikiSkill loop with the deterministic MockLLM.""" +from __future__ import annotations + +import argparse +import os + +from .datasets import make_demo_dataset, _demo_tools +from .llm import MockLLM +from .orchestrator import EvolutionOrchestrator +from .workspace import Workspace + + +def run_demo(root: str = "wikiskill_demo_workspace", K: int = 3) -> dict: + ws = Workspace(root) + skills_logger = [] + orch = EvolutionOrchestrator( + llm=MockLLM(), dataset=make_demo_dataset(), ws=ws, tools=_demo_tools(), K=K) + result = orch.run() + print("\nWorkspace layout:") + for dirpath, _, files in os.walk(root): + rel = os.path.relpath(dirpath, root) + for fn in sorted(files): + print(f" {os.path.join(rel, fn)}") + return result + + +if __name__ == "__main__": + p = argparse.ArgumentParser(description="WikiSkill demo (mock LLM, offline)") + p.add_argument("--root", default="wikiskill_demo_workspace") + p.add_argument("--iterations", type=int, default=3) + a = p.parse_args() + run_demo(a.root, a.iterations) diff --git a/src/wikiskill/llm.py b/src/wikiskill/llm.py new file mode 100644 index 0000000..2e3e694 --- /dev/null +++ b/src/wikiskill/llm.py @@ -0,0 +1,139 @@ +"""LLM provider abstraction (design.md §4): protocol, OpenAI-compatible client, MockLLM.""" +from __future__ import annotations + +import json +import os +import re +import urllib.request +from typing import Any, List, Optional, Protocol + + +class LLM(Protocol): + def chat(self, messages: List[dict], **kw) -> str: ... + + +def extract_json(text: str) -> Optional[Any]: + """Extract the first JSON object/array from an LLM response (tolerates fences).""" + text = text.strip() + m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) + if m: + text = m.group(1).strip() + for start_ch, end_ch in (("{", "}"), ("[", "]")): + start = text.find(start_ch) + if start == -1: + continue + depth = 0 + for i in range(start, len(text)): + if text[i] == start_ch: + depth += 1 + elif text[i] == end_ch: + depth -= 1 + if depth == 0: + try: + return json.loads(text[start : i + 1]) + except json.JSONDecodeError: + break + break + return None + + +class OpenAICompatLLM: + """Calls any OpenAI-compatible /chat/completions endpoint using stdlib urllib.""" + + def __init__(self, model: str, base_url: Optional[str] = None, + api_key: Optional[str] = None, temperature: float = 0.7): + self.model = model + self.base_url = (base_url or os.environ.get("WIKISKILL_BASE_URL", + "https://api.openai.com/v1")).rstrip("/") + self.api_key = api_key or os.environ.get("WIKISKILL_API_KEY", + os.environ.get("OPENAI_API_KEY", "")) + self.temperature = temperature + + def chat(self, messages: List[dict], temperature: Optional[float] = None, + max_tokens: int = 4096) -> str: + payload = { + "model": self.model, + "messages": messages, + "temperature": self.temperature if temperature is None else temperature, + "max_tokens": max_tokens, + } + req = urllib.request.Request( + self.base_url + "/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=300) as resp: + data = json.loads(resp.read().decode("utf-8")) + return data["choices"][0]["message"]["content"] + + +class MockLLM: + """Deterministic scripted LLM for tests and the offline demo. + + `handlers` maps a substring of the conversation to `(messages, call_log) -> str`. + Fallback behaviour implements the demo dataset semantics. + """ + + def __init__(self, handlers: Optional[dict] = None): + self.handlers = dict(handlers or {}) + self.call_log: List[dict] = [] + + def chat(self, messages: List[dict], **kw) -> str: + self.call_log.append({"messages": messages}) + blob = "\n".join(str(m.get("content", "")) for m in messages) + for needle, fn in self.handlers.items(): + if needle in blob: + return fn(messages, self.call_log) + return self._fallback(blob) + + # ---- default demo behaviour ------------------------------------------- + def _fallback(self, blob: str) -> str: + if "Wiki Maintainer Agent" in blob: + return self._maintainer() + if "Skill Proposer Agent" in blob: + return self._proposer() + return self._inference(blob) + + def _maintainer(self) -> str: + return json.dumps({ + "create_patterns": [{ + "name": "rounding-errors.md", + "content": ("# Pattern: Rounding errors\n\n## Description\n" + "Demo pattern: naive rounding gives wrong answers.\n\n" + "## Root cause\nSkipping the final rounding step.\n\n" + "## Workaround\nAlways round the final result to 2 decimals.\n"), + }], + "update_patterns": [], + "update_index": ("- [rounding-errors](wiki/patterns/rounding-errors.md): " + "wrong answers from skipped rounding; FIX: round final " + "result to 2 decimals.\n"), + "append_log": "Analyzed traces; recorded rounding-error pattern.", + }) + + def _proposer(self) -> str: + return json.dumps({ + "action": "create", + "skill": "unit-conversion", + "content": ("---\nname: unit-conversion\ndescription: Correctly convert units " + "with exact arithmetic.\n---\n\n# Unit Conversion Skill\n\n" + "1. Compute using exact ratios.\n" + "2. Round ONLY the final result to 2 decimal places.\n" + "3. State the answer as `ANSWER: `.\n"), + "purpose": "Created from wiki pattern rounding-errors.md.", + }) + + def _inference(self, blob: str) -> str: + # Demo tasks embed their correct answer as "expected=...". With the evolved + # skill injected, the mock agent answers exactly; without it, it truncates + # decimals (a systematic, deterministic "model weakness"). + m = re.search(r"expected=([-\d.]+)", blob) + if not m: + return "ANSWER: 0" + val = m.group(1) + if "round only the final result" in blob.lower(): + return f"ANSWER: {val}" + whole, _, frac = val.partition(".") + return "ANSWER: " + whole + "." + (frac[:1] if frac else "0") + diff --git a/src/wikiskill/metrics.py b/src/wikiskill/metrics.py new file mode 100644 index 0000000..b60e935 --- /dev/null +++ b/src/wikiskill/metrics.py @@ -0,0 +1,28 @@ +"""Metrics (paper Appendix C): accuracy + paired bootstrap significance testing.""" +from __future__ import annotations + +import random +from typing import Sequence + + +def accuracy(correct_flags: Sequence[bool]) -> float: + return sum(bool(c) for c in correct_flags) / len(correct_flags) if correct_flags else 0.0 + + +def paired_bootstrap(a: Sequence[bool], b: Sequence[bool], n: int = 1000, + seed: int = 0) -> float: + """Paired bootstrap (paper App. C): resample test tasks with replacement; return + the p-value that method `a` does not outperform `b` (one-sided).""" + assert len(a) == len(b) + rng = random.Random(seed) + N = len(a) + if N == 0: + return 1.0 + margins = [1.0 if x and not y else (-1.0 if y and not x else 0.0) + for x, y in zip(a, b)] + worse_or_equal = 0 + for _ in range(n): + sample = [margins[rng.randrange(N)] for _ in range(N)] + if sum(sample) / N <= 0: + worse_or_equal += 1 + return worse_or_equal / n diff --git a/src/wikiskill/orchestrator.py b/src/wikiskill/orchestrator.py new file mode 100644 index 0000000..02b0d6e --- /dev/null +++ b/src/wikiskill/orchestrator.py @@ -0,0 +1,135 @@ +"""Evolution orchestrator — paper Algorithm 1 (Appendix A.1) with §3.2.4 gating.""" +from __future__ import annotations + +import json +import os +from typing import Callable, List, Optional + +from .agents import InferenceAgent, SkillProposer, WikiMaintainer, stratified_sample +from .datasets import Dataset, Trajectory +from .llm import LLM +from .skills import SkillsLayer +from .workspace import Workspace + + +class EvolutionOrchestrator: + """Co-evolves (S_k, W_k) per Algorithm 1: + + S0 = W0 = ∅; R_best = R(T_val,0) + for k = 1..K: + if R_best == 1.0: break + T_train,k = {τ ~ π(x; S_{k-1})} on D_train # inference, no wiki + T_sample,k ⊂ T_train,k # stratified (App. C) + W'_k = M_WM(W_{k-1}, T_sample,k) # wiki maintenance + P_k = M_P(W'_k, S_{k-1}, T_train,k) # ReAct proposer + S'_k = Apply(S_{k-1}, P_k) + validate S'_k on D_val + accept iff R(T_val,k) > R_best else roll back skills (wiki retained) + W_k = Update(W'_k, P_k, R(T_val,k), a_k) # harness writes skill-impact.md + return S_K, W_K + """ + + def __init__(self, llm: LLM, dataset: Dataset, ws: Workspace, tools: list, + metric: Callable[[List[bool]], float] = None, K: int = 8, + max_react_turns: int = 20, log: Optional[Callable[[str], None]] = None): + from .metrics import accuracy + self.llm = llm + self.dataset = dataset + self.ws = ws + self.metric = metric or accuracy + self.K = K + self.skills = SkillsLayer(ws) + self.inference = InferenceAgent(llm, tools, ws, self.skills) + self.maintainer = WikiMaintainer(llm, ws) + self.proposer = SkillProposer(llm, ws, self.skills, max_turns=max_react_turns) + self.log = log or (lambda s: print(s, flush=True)) + self.history: List[dict] = [] + self.r_best = -1.0 + + # ------------------------------------------------------------------ + def _rollout(self, tasks, iteration: int, split: str) -> List[Trajectory]: + trajs = [self.inference.run(t) for t in tasks] + for t in trajs: + self.ws.add_raw_trace(iteration, t.task_id, + {**t.to_dict(), "split": split, "iteration": iteration}) + return trajs + + def _score(self, trajs: List[Trajectory]) -> float: + return self.metric([t.correct for t in trajs]) + + def _outcome_summary(self, trajs: List[Trajectory], tasks) -> str: + by_id = {t.id: t for t in tasks} + return "\n".join( + f"{tr.task_id}: {'PASS' if tr.correct else 'FAIL'} | " + f"predicted={tr.prediction!r} ground_truth={by_id[tr.task_id].y!r} " + f"| trace: raw/iter_*/{tr.task_id}.json" for tr in trajs) + + def _baseline_validation(self) -> None: + self.log("Baseline validation (S0 = empty) ...") + trajs = self._rollout(self.dataset.val, 0, "val") + self.r_best = self._score(trajs) + self.log(f"R_best = {self.r_best:.4f}") + + # ------------------------------------------------------------------ + def run(self) -> dict: + self._baseline_validation() + for k in range(1, self.K + 1): + if self.r_best >= 1.0: + self.log("R_best reached 1.0; early stop.") + break + self.log(f"\n=== Iteration {k}/{self.K} ===") + # lines 8-9: training rollouts + stratified sample + train_trajs = self._rollout(self.dataset.train, k, "train") + sampled = stratified_sample(train_trajs) + # line 10: wiki maintenance + update = self.maintainer.consolidate(sampled, k) + report = self.ws.apply_maintainer_update(update) if update else {} + self.log(f"Wiki: created={report.get('created')} updated={report.get('updated')} " + f"errors={report.get('errors')}") + # line 11: skill proposal + proposal = self.proposer.propose( + self._outcome_summary(train_trajs, self.dataset.train), k) + entry = {"iteration": k, "proposal": None, "val_score": None, + "accepted": False} + if proposal: + snap = self.skills.snapshot() + try: + applied = self.skills.apply_proposal(proposal) + except Exception as e: # invalid proposal → skip iteration gracefully + self.log(f"Proposal invalid, skipped: {e}") + applied = None + if applied is not None: + # lines 13-14: validate + gate + val_trajs = self._rollout(self.dataset.val, k, "val") + score = self._score(val_trajs) + accepted = score > self.r_best + if accepted: + self.r_best = score # line 15 + else: + self.skills.restore(snap) # line 17: skills-only rollback + self.log(f"Proposal {applied['action']} '{applied['skill']}' -> " + f"val={score:.4f} ({'ACCEPTED' if accepted else 'REJECTED'})") + # line 19: harness records audit trail in skill-impact.md + diff = applied.get("diff", "(new skill)") + self.ws.record_skill_impact( + f"\n## Iteration {k} — {applied['action']} " + f"'{applied['skill']}'\n" + f"- validation score: {score:.4f}\n" + f"- outcome: {'Accepted' if accepted else 'Rejected'}\n" + f"- purpose: {proposal.get('purpose', '')}\n" + f"- diff:\n```diff\n{diff}\n```\n") + entry.update({"proposal": applied.get("skill"), + "action": applied["action"], "val_score": score, + "accepted": accepted}) + else: + self.log("No proposal produced this iteration.") + self.history.append(entry) + final_skills = self.skills.list_skills() + result = {"r_best": self.r_best, "history": self.history, + "final_skills": final_skills} + with open(os.path.join(self.ws.root, "run_state.json"), "w", + encoding="utf-8") as f: + json.dump(result, f, indent=2) + self.log(f"\nEvolution complete. R_best={self.r_best:.4f}, skills={final_skills}") + return result + diff --git a/src/wikiskill/prompts.py b/src/wikiskill/prompts.py new file mode 100644 index 0000000..c3dc6c1 --- /dev/null +++ b/src/wikiskill/prompts.py @@ -0,0 +1,82 @@ +"""System prompts (paper Appendix E). Condensed but structure-faithful versions.""" + +INFERENCE_SYSTEM = """You are a task-solving agent. Complete the task step by step. + +You may use the following tools by replying with a single line: +TOOL: {"arg": "value"} +The environment will reply with the observation, then you continue. + +When you are done, reply with your final answer on its own line in the format: +ANSWER: + +## Active skills (follow them when applicable) +{active_skills} +""" + +WIKI_MAINTAINER_SYSTEM = """You are a Wiki Maintainer Agent for an LLM skill evolution +system. Your job is to maintain a structured knowledge base (wiki) that documents +patterns observed during agent execution -- both successes and failures. Perform DEEP +ANALYSIS of execution logs to identify root causes, not just surface-level symptoms. + +## Wiki structure +- wiki/index.md -- concise catalog of known patterns (one line per pattern) +- wiki/logs.md -- chronological evolution log +- wiki/skill-impact.md -- record of which skills were tried and their outcomes +- wiki/patterns/ -- one page per pattern with evidence and analysis + +## Your input +1. Execution traces from the latest iteration (agent actions, tool calls, feedback) +2. The current wiki context (index, log, pattern pages) + +## Your output (incremental edit mode) +Return a JSON object with these keys: +- "create_patterns": [{"name": "pattern-name.md", "content": "..."}] -- new patterns +- "update_patterns": [{"name": "existing-pattern.md", "edits": [...]}] -- patch existing +- "update_index": full updated content of index.md (REQUIRED, always complete) +- "append_log": brief summary of this iteration's findings (REQUIRED) + +### Patch operations (for update_patterns.edits) +- {"op": "append", "content": "text to add at end"} +- {"op": "replace", "target": "exact text to find", "content": "replacement"} +- {"op": "insert_after", "target": "exact text to find", "content": "inserted text"} +Targets must be EXACT substrings of the existing content. Keep each edit minimal. + +## Analysis guidelines +1. Read the agent's actual actions; compare successful vs failed tasks. +2. Identify ACTION PATTERNS and strategies, not just error messages. +3. Pattern pages document: description, root cause analysis (WHY), exact sequences + from traces, and known solutions/workarounds with concrete syntax. +4. Capture BOTH failure patterns and success patterns. +5. Do NOT duplicate patterns -- update existing ones with new evidence. +6. Be concise: pattern pages should be 10-30 lines. +7. Index entries are critical: format "- [name](wiki/patterns/name.md): PROBLEM + + ROOT CAUSE + FIX in one line". +""" + +SKILL_PROPOSER_SYSTEM = """You are a Skill Proposer Agent for an LLM skill evolution +system, operating in ReAct mode. Your goal: propose ONE atomic skill update (create a +new skill OR patch one existing skill) that improves the agent's performance, informed +by the persistent wiki. + +## Available tools +- read_file: {"path": ""} -- read a wiki pattern page, a raw execution + trace, an existing skill, or any workspace file. +- finish: {"action": "create"|"patch", "skill": "", "content": "", "edits": [], "purpose": ""} + +## Procedure (ReAct) +1. Read wiki/index.md and the skill-impact history. NEVER re-propose an intervention + that was already rejected (check skill-impact.md). +2. Use read_file to inspect specific pattern pages and raw traces of FAILED tasks to + diagnose root causes before proposing. +3. Propose exactly ONE change: either create a new skill (full content with YAML + frontmatter: name, description) or patch one existing skill with minimal edits. +4. In "purpose", map the skill back to the motivating wiki patterns. +5. When ready, emit ONLY the finish tool call with valid JSON. + +## Patch op format (for "edits") +{"op": "append", "content": "..."} | +{"op": "replace", "target": "", "content": "..."} | +{"op": "insert_after", "target": "", "content": "..."} +""" diff --git a/src/wikiskill/skills.py b/src/wikiskill/skills.py new file mode 100644 index 0000000..481dddf --- /dev/null +++ b/src/wikiskill/skills.py @@ -0,0 +1,90 @@ +"""Skills Layer (paper §3.1): skills//SKILL.md + PURPOSE.md, with +snapshot/restore for skills-only rollback during validation gating (§3.2.4).""" +from __future__ import annotations + +import difflib +import os +import re +from typing import Dict, List, Optional + +from .workspace import PatchError, Workspace, apply_edits + +SKILL_NAME_RE = re.compile(r"^[A-Za-z0-9._-]+$") + + +class SkillsLayer: + def __init__(self, ws: Workspace): + self.ws = ws + + # ---------- state ---------- + def list_skills(self) -> List[str]: + root = os.path.join(self.ws.root, "skills") + if not os.path.isdir(root): + return [] + return sorted(d for d in os.listdir(root) + if os.path.isfile(os.path.join(root, d, "SKILL.md"))) + + def skill_content(self, name: str) -> Optional[str]: + if name not in self.list_skills(): + return None + return self.ws.read_file(os.path.join("skills", name, "SKILL.md")) + + def purpose(self, name: str) -> Optional[str]: + if name not in self.list_skills(): + return None + return self.ws.read_file(os.path.join("skills", name, "PURPOSE.md")) + + def active_skills_context(self) -> str: + """Full SKILL.md contents injected into the Inference Agent prompt (§3.2.1).""" + parts = [] + for name in self.list_skills(): + parts.append(f"## Skill: {name}\n\n{self.skill_content(name)}") + return ("\n\n---\n\n".join(parts) if parts + else "(No active skills.)") + + # ---------- proposals (§3.2.3: atomic, single-skill) ---------- + def apply_proposal(self, proposal: dict) -> dict: + """Apply P_k: create a new skill or patch an existing SKILL.md.""" + action = proposal.get("action") + name = str(proposal.get("skill", "")).strip() + if not SKILL_NAME_RE.match(name): + raise ValueError(f"invalid skill name: {name!r}") + rel_skill = os.path.join("skills", name, "SKILL.md") + rel_purpose = os.path.join("skills", name, "PURPOSE.md") + purpose = str(proposal.get("purpose", "")).rstrip() + "\n" + + if action == "create": + if self.skill_content(name) is not None: + raise ValueError(f"skill already exists: {name}") + self.ws.write_file(rel_skill, str(proposal.get("content", ""))) + self.ws.write_file(rel_purpose, purpose) + return {"skill": name, "action": "create"} + if action == "patch": + old = self.skill_content(name) + if old is None: + raise ValueError(f"cannot patch missing skill: {name}") + new = apply_edits(old, proposal.get("edits", [])) + self.ws.write_file(rel_skill, new) + prev = self.purpose(name) or "" + self.ws.write_file(rel_purpose, + prev.rstrip() + "\n" + purpose if prev.strip() else purpose) + return {"skill": name, "action": "patch", + "diff": "\n".join(difflib.unified_diff( + old.splitlines(), new.splitlines(), + fromfile=f"{name}/SKILL.md (old)", + tofile=f"{name}/SKILL.md (new)", lineterm=""))} + raise ValueError(f"unknown proposal action: {action!r}") + + # ---------- gating rollback (§3.2.4): skills only, wiki untouched ---------- + def snapshot(self) -> Dict[str, str]: + return {name: self.skill_content(name) or "" for name in self.list_skills()} + + def restore(self, snap: Dict[str, str]) -> None: + for name in self.list_skills(): + if name not in snap: + d = os.path.join(self.ws.root, "skills", name) + for fn in os.listdir(d): + os.remove(os.path.join(d, fn)) + os.rmdir(d) + for name, content in snap.items(): + self.ws.write_file(os.path.join("skills", name, "SKILL.md"), content) diff --git a/src/wikiskill/workspace.py b/src/wikiskill/workspace.py new file mode 100644 index 0000000..0819020 --- /dev/null +++ b/src/wikiskill/workspace.py @@ -0,0 +1,152 @@ +"""Three-layer workspace (paper §3.1): raw/ (immutable traces), wiki/ (persistent +knowledge), skills/ (active skill set), plus the patch engine used by agents.""" +from __future__ import annotations + +import difflib +import json +import os +import re +from typing import Dict, List, Optional + +PATTERN_NAME_RE = re.compile(r"^[A-Za-z0-9._-]+\.md$") + + +class PatchError(Exception): + pass + + +def apply_edits(content: str, edits: List[dict]) -> str: + """Patch engine (paper Wiki Maintainer prompt): append / replace / insert_after. + + `replace` and `insert_after` targets must be EXACT substrings. Invalid targets + raise PatchError; callers may catch and skip individual ops. + """ + for e in edits or []: + op = e.get("op") + if op == "append": + content = content + e.get("content", "") + elif op == "replace": + target = e.get("target", "") + if target not in content: + raise PatchError(f"replace target not found: {target[:60]!r}") + content = content.replace(target, e.get("content", ""), 1) + elif op == "insert_after": + target = e.get("target", "") + if target not in content: + raise PatchError(f"insert_after target not found: {target[:60]!r}") + idx = content.index(target) + len(target) + content = content[:idx] + e.get("content", "") + content[idx:] + else: + raise PatchError(f"unknown op: {op}") + return content + + +class Workspace: + """Manages the three layers of the WikiSkill agent workspace.""" + + def __init__(self, root: str): + self.root = os.path.abspath(root) + for d in ("raw", os.path.join("wiki", "patterns"), "skills"): + os.makedirs(os.path.join(self.root, d), exist_ok=True) + + # ---------- generic helpers ---------- + def _abs(self, rel: str) -> str: + p = os.path.abspath(os.path.join(self.root, rel)) + if not p.startswith(self.root + os.sep) and p != self.root: + raise ValueError(f"path escapes workspace: {rel}") + return p + + def read_file(self, rel: str) -> str: + """Workspace-scoped read (used by the ReAct Skill Proposer).""" + p = self._abs(rel) + if not os.path.isfile(p): + return f"ERROR: file not found: {rel}" + with open(p, "r", encoding="utf-8") as f: + return f.read() + + def write_file(self, rel: str, content: str, overwrite: bool = True) -> None: + p = self._abs(rel) + os.makedirs(os.path.dirname(p), exist_ok=True) + if os.path.exists(p) and not overwrite: + raise FileExistsError(rel) + with open(p, "w", encoding="utf-8") as f: + f.write(content) + + # ---------- Raw layer: immutable ---------- + def add_raw_trace(self, iteration: int, task_id: str, trace: dict) -> str: + rel = os.path.join("raw", f"iter_{iteration:03d}", f"{task_id}.json") + p = self._abs(rel) + if os.path.exists(p): + raise FileExistsError(f"raw layer is immutable: {rel}") + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w", encoding="utf-8") as f: + json.dump(trace, f, indent=2, ensure_ascii=False) + return rel + + # ---------- Wiki layer ---------- + def wiki_index(self) -> str: + return self.read_file("wiki/index.md") if os.path.isfile( + self._abs("wiki/index.md")) else "" + + def wiki_logs(self) -> str: + return self.read_file("wiki/logs.md") if os.path.isfile( + self._abs("wiki/logs.md")) else "" + + def skill_impact(self) -> str: + return self.read_file("wiki/skill-impact.md") if os.path.isfile( + self._abs("wiki/skill-impact.md")) else "" + + def list_patterns(self) -> List[str]: + d = self._abs(os.path.join("wiki", "patterns")) + return sorted(f for f in os.listdir(d) if f.endswith(".md")) + + def pattern_page(self, name: str) -> str: + if not PATTERN_NAME_RE.match(name): + raise ValueError(f"bad pattern name: {name}") + return self.read_file(os.path.join("wiki", "patterns", name)) + + def wiki_context(self) -> str: + """Full wiki context handed to the Wiki Maintainer (index + pages + logs).""" + parts = ["# WIKI INDEX", self.wiki_index() or "(empty)"] + for name in self.list_patterns(): + parts += [f"\n# PATTERN: {name}", self.pattern_page(name)] + parts += ["\n# EVOLUTION LOG", self.wiki_logs() or "(empty)"] + return "\n".join(parts) + + def apply_maintainer_update(self, update: dict) -> dict: + """Apply M_WM's JSON ops (create/update patterns, index, log). Returns report.""" + report = {"created": [], "updated": [], "errors": []} + for pat in update.get("create_patterns", []) or []: + name = pat.get("name", "") + if not PATTERN_NAME_RE.match(name): + report["errors"].append(f"bad pattern name {name!r}") + continue + rel = os.path.join("wiki", "patterns", name) + if os.path.exists(self._abs(rel)): + report["errors"].append(f"pattern exists, use update: {name}") + continue + self.write_file(rel, pat.get("content", "")) + report["created"].append(name) + for pat in update.get("update_patterns", []) or []: + name = pat.get("name", "") + rel = os.path.join("wiki", "patterns", name) + try: + if not PATTERN_NAME_RE.match(name) or not os.path.exists(self._abs(rel)): + raise PatchError(f"unknown pattern {name!r}") + content = self.pattern_page(name) + self.write_file(rel, apply_edits(content, pat.get("edits", []))) + report["updated"].append(name) + except PatchError as e: + report["errors"].append(f"{name}: {e}") + if update.get("update_index") is not None: + self.write_file("wiki/index.md", str(update["update_index"])) + if update.get("append_log"): + with open(self._abs("wiki/logs.md"), "a", encoding="utf-8") as f: + f.write(update["append_log"].rstrip() + "\n") + return report + + def record_skill_impact(self, entry: str) -> None: + """Programmatic skill-impact.md append — done ONLY by the harness after gating.""" + with open(self._abs("wiki/skill-impact.md"), "a", encoding="utf-8") as f: + f.write(entry.rstrip() + "\n") + diff --git a/tests/test_agents.py b/tests/test_agents.py new file mode 100644 index 0000000..919444b --- /dev/null +++ b/tests/test_agents.py @@ -0,0 +1,69 @@ +from wikiskill.datasets import make_demo_dataset, _demo_tools +from wikiskill.llm import MockLLM, extract_json +from wikiskill.skills import SkillsLayer +from wikiskill.workspace import Workspace + + +def _ws(tmp_path): + return Workspace(str(tmp_path / "ws")) + + +def test_extract_json(): + assert extract_json('```json\n{"a": 1}\n```') == {"a": 1} + assert extract_json('noise {"a": {"b": 2}} tail') == {"a": {"b": 2}} + assert extract_json("no json") is None + + +def test_skills_create_snapshot_restore(tmp_path): + skills = SkillsLayer(_ws(tmp_path)) + skills.apply_proposal({"action": "create", "skill": "s1", + "content": "# S1\nline1\nline2\n", "purpose": "why"}) + assert skills.list_skills() == ["s1"] + snap = skills.snapshot() + skills.apply_proposal({"action": "patch", "skill": "s1", + "edits": [{"op": "replace", "target": "line2", + "content": "LINE2"}], "purpose": "why2"}) + assert "LINE2" in skills.skill_content("s1") + skills.restore(snap) + assert "LINE2" not in skills.skill_content("s1") + skills.apply_proposal({"action": "create", "skill": "s2", "content": "x", + "purpose": "p"}) + skills.restore(snap) + assert skills.list_skills() == ["s1"] + + +def test_skill_injection_context(tmp_path): + skills = SkillsLayer(_ws(tmp_path)) + assert "(No active skills.)" in skills.active_skills_context() + skills.apply_proposal({"action": "create", "skill": "s", "content": "BODY", + "purpose": "p"}) + assert "BODY" in skills.active_skills_context() + + +def test_mock_inference_without_and_with_skill(tmp_path): + from wikiskill.agents import InferenceAgent + ws = _ws(tmp_path) + skills = SkillsLayer(ws) + agent = InferenceAgent(MockLLM(), _demo_tools(), ws, skills) + ds = make_demo_dataset() + traj = agent.run(ds.val[0]) + assert traj.correct is False # truncation error without skill + skills.apply_proposal({"action": "create", "skill": "unit-conversion", + "content": "Round ONLY the final result to 2 decimal places.", + "purpose": "p"}) + agent2 = InferenceAgent(MockLLM(), _demo_tools(), ws, skills) + traj2 = agent2.run(ds.val[0]) + assert traj2.correct is True + + +def test_stratified_sample(): + from wikiskill.agents import stratified_sample + from wikiskill.datasets import Trajectory + + def t(i, ok): + return Trajectory(task_id=i, messages=[{"role": "user", "content": "x"}], + tool_calls=[], final_answer="a", correct=ok) + traces = [t(f"f{i}", False) for i in range(9)] + [t(f"p{i}", True) for i in range(5)] + s = stratified_sample(traces) + assert sum(1 for x in s if not x.correct) == 5 + assert sum(1 for x in s if x.correct) == 3 diff --git a/tests/test_e2e.py b/tests/test_e2e.py new file mode 100644 index 0000000..1e09d5f --- /dev/null +++ b/tests/test_e2e.py @@ -0,0 +1,58 @@ +import json + +from wikiskill.datasets import make_demo_dataset, _demo_tools +from wikiskill.llm import MockLLM +from wikiskill.skills import SkillsLayer +from wikiskill.workspace import Workspace + + +def test_gating_math(): + """Strict improvement: accept iff score > R_best (paper eq. 4).""" + def gate(score, r_best): + return score > r_best + assert gate(0.5, 0.5) is False # ties are rejected + assert gate(0.6, 0.5) is True + assert gate(0.4, 0.5) is False + + +def test_e2e_evolution_loop(tmp_path, capsys): + from wikiskill.orchestrator import EvolutionOrchestrator + root = tmp_path / "ws" + ws = Workspace(str(root)) + orch = EvolutionOrchestrator(llm=MockLLM(), dataset=make_demo_dataset(), ws=ws, + tools=_demo_tools(), K=3) + result = orch.run() + + # Baseline (no skill) fails on val; after skill creation, val should improve. + assert result["r_best"] > 0.0 + assert "unit-conversion" in result["final_skills"] + + # Three-layer artifacts exist + assert (root / "raw" / "iter_000").is_dir() # baseline val rollouts + assert (root / "raw" / "iter_001" / "train").exists() or \ + list((root / "raw" / "iter_001").glob("*.json")) + assert (root / "wiki" / "patterns" / "rounding-errors.md").is_file() + assert (root / "wiki" / "index.md").is_file() + assert (root / "wiki" / "logs.md").is_file() + assert (root / "wiki" / "skill-impact.md").is_file() + assert (root / "skills" / "unit-conversion" / "SKILL.md").is_file() + assert (root / "skills" / "unit-conversion" / "PURPOSE.md").is_file() + + # Harness wrote the audit trail with outcome + diff + impact = (root / "wiki" / "skill-impact.md").read_text(encoding="utf-8") + assert "Accepted" in impact + + # run_state.json persisted + state = json.loads((root / "run_state.json").read_text(encoding="utf-8")) + assert state["final_skills"] == ["unit-conversion"] + + +def test_e2e_early_stop_at_perfect_val(tmp_path): + from wikiskill.orchestrator import EvolutionOrchestrator + ws = Workspace(str(tmp_path / "ws")) + orch = EvolutionOrchestrator(llm=MockLLM(), dataset=make_demo_dataset(), ws=ws, + tools=_demo_tools(), K=5) + result = orch.run() + # With the skill accepted, val = 1.0 -> the loop must early-stop. + assert result["r_best"] == 1.0 + assert len(result["history"]) <= 2 diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..b4e34c6 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,13 @@ +from wikiskill.metrics import accuracy, paired_bootstrap + + +def test_accuracy(): + assert accuracy([True, False, True, True]) == 0.75 + assert accuracy([]) == 0.0 + + +def test_paired_bootstrap_significant_and_not(): + a = [True] * 10 + [False] * 0 + b = [True] * 5 + [False] * 5 + assert paired_bootstrap(a, b, n=500, seed=1) < 0.05 + assert paired_bootstrap(a, a, n=200, seed=1) == 1.0 diff --git a/tests/test_workspace.py b/tests/test_workspace.py new file mode 100644 index 0000000..ba73e03 --- /dev/null +++ b/tests/test_workspace.py @@ -0,0 +1,46 @@ +import pytest + +from wikiskill.workspace import PatchError, Workspace, apply_edits + + +def test_patch_ops(): + c = "alpha\nbeta\n" + assert apply_edits(c, [{"op": "append", "content": "gamma"}]) == "alpha\nbeta\ngamma" + assert apply_edits(c, [{"op": "replace", "target": "beta", "content": "BETA"}]) \ + == "alpha\nBETA\n" + assert apply_edits(c, [{"op": "insert_after", "target": "alpha", "content": " mid"}]) \ + == "alpha mid\nbeta\n" + with pytest.raises(PatchError): + apply_edits(c, [{"op": "replace", "target": "nope", "content": "x"}]) + with pytest.raises(PatchError): + apply_edits(c, [{"op": "bogus", "content": "x"}]) + + +def test_raw_layer_immutable(tmp_path): + ws = Workspace(str(tmp_path / "ws")) + ws.add_raw_trace(1, "t1", {"a": 1}) + with pytest.raises(FileExistsError): + ws.add_raw_trace(1, "t1", {"a": 2}) + + +def test_read_file_scoped(tmp_path): + ws = Workspace(str(tmp_path / "ws")) + assert "ERROR" in ws.read_file("missing.md") + with pytest.raises(ValueError): + ws.read_file("../outside.txt") + + +def test_maintainer_update(tmp_path): + ws = Workspace(str(tmp_path / "ws")) + report = ws.apply_maintainer_update({ + "create_patterns": [{"name": "p.md", "content": "# P\nbody\n"}], + "update_patterns": [{"name": "p.md", + "edits": [{"op": "append", "content": "more\n"}]}], + "update_index": "- [p](wiki/patterns/p.md): x\n", + "append_log": "iter 1 findings", + }) + assert report["created"] == ["p.md"] and report["updated"] == ["p.md"] + assert ws.pattern_page("p.md").endswith("more\n") + assert "[p]" in ws.wiki_index() and "iter 1" in ws.wiki_logs() + ws.record_skill_impact("## entry\n- outcome: Accepted\n") + assert "Accepted" in ws.skill_impact()