diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1277b83..1395cd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,4 +24,4 @@ jobs: - name: CLI demo (mock, end-to-end) run: python -m wikiskill.demo --root ci_demo --iterations 3 - name: Live-server smoke test - run: python smoke_test.py + run: python smoke_check.py diff --git a/smoke_test.py b/smoke_check.py similarity index 100% rename from smoke_test.py rename to smoke_check.py diff --git a/smoke_run/raw/eval_baseline/task-009.json b/smoke_run/raw/eval_baseline/task-009.json new file mode 100644 index 0000000..aa2e9dd --- /dev/null +++ b/smoke_run/raw/eval_baseline/task-009.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-009", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 5.3333 pounds to kilograms. The conversion factor is 0.453592. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=2.42)" + }, + { + "role": "assistant", + "content": "ANSWER: 2.4" + } + ], + "tool_calls": [], + "final_answer": "2.4", + "correct": false, + "prediction": "2.4", + "split": "test", + "iteration": "eval_baseline" +} \ No newline at end of file diff --git a/smoke_run/raw/eval_baseline/task-010.json b/smoke_run/raw/eval_baseline/task-010.json new file mode 100644 index 0000000..42e1249 --- /dev/null +++ b/smoke_run/raw/eval_baseline/task-010.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-010", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 7.6667 inches to centimeters. The conversion factor is 2.54. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=19.47)" + }, + { + "role": "assistant", + "content": "ANSWER: 19.4" + } + ], + "tool_calls": [], + "final_answer": "19.4", + "correct": false, + "prediction": "19.4", + "split": "test", + "iteration": "eval_baseline" +} \ No newline at end of file diff --git a/smoke_run/raw/eval_baseline/task-011.json b/smoke_run/raw/eval_baseline/task-011.json new file mode 100644 index 0000000..0dece78 --- /dev/null +++ b/smoke_run/raw/eval_baseline/task-011.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-011", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 10.0 gallons to liters. The conversion factor is 3.78541. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=37.85)" + }, + { + "role": "assistant", + "content": "ANSWER: 37.8" + } + ], + "tool_calls": [], + "final_answer": "37.8", + "correct": false, + "prediction": "37.8", + "split": "test", + "iteration": "eval_baseline" +} \ No newline at end of file diff --git a/smoke_run/raw/eval_skilled/task-009.json b/smoke_run/raw/eval_skilled/task-009.json new file mode 100644 index 0000000..2a66f67 --- /dev/null +++ b/smoke_run/raw/eval_skilled/task-009.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-009", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n## Skill: unit-conversion\n\n---\nname: unit-conversion\ndescription: Correctly convert units with exact arithmetic.\n---\n\n# Unit Conversion Skill\n\n1. Compute using exact ratios.\n2. Round ONLY the final result to 2 decimal places.\n3. State the answer as `ANSWER: `.\n\n" + }, + { + "role": "user", + "content": "Convert 5.3333 pounds to kilograms. The conversion factor is 0.453592. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=2.42)" + }, + { + "role": "assistant", + "content": "ANSWER: 2.42" + } + ], + "tool_calls": [], + "final_answer": "2.42", + "correct": true, + "prediction": "2.42", + "split": "test", + "iteration": "eval_skilled" +} \ No newline at end of file diff --git a/smoke_run/raw/eval_skilled/task-010.json b/smoke_run/raw/eval_skilled/task-010.json new file mode 100644 index 0000000..50ccf12 --- /dev/null +++ b/smoke_run/raw/eval_skilled/task-010.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-010", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n## Skill: unit-conversion\n\n---\nname: unit-conversion\ndescription: Correctly convert units with exact arithmetic.\n---\n\n# Unit Conversion Skill\n\n1. Compute using exact ratios.\n2. Round ONLY the final result to 2 decimal places.\n3. State the answer as `ANSWER: `.\n\n" + }, + { + "role": "user", + "content": "Convert 7.6667 inches to centimeters. The conversion factor is 2.54. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=19.47)" + }, + { + "role": "assistant", + "content": "ANSWER: 19.47" + } + ], + "tool_calls": [], + "final_answer": "19.47", + "correct": true, + "prediction": "19.47", + "split": "test", + "iteration": "eval_skilled" +} \ No newline at end of file diff --git a/smoke_run/raw/eval_skilled/task-011.json b/smoke_run/raw/eval_skilled/task-011.json new file mode 100644 index 0000000..fe3c844 --- /dev/null +++ b/smoke_run/raw/eval_skilled/task-011.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-011", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n## Skill: unit-conversion\n\n---\nname: unit-conversion\ndescription: Correctly convert units with exact arithmetic.\n---\n\n# Unit Conversion Skill\n\n1. Compute using exact ratios.\n2. Round ONLY the final result to 2 decimal places.\n3. State the answer as `ANSWER: `.\n\n" + }, + { + "role": "user", + "content": "Convert 10.0 gallons to liters. The conversion factor is 3.78541. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=37.85)" + }, + { + "role": "assistant", + "content": "ANSWER: 37.85" + } + ], + "tool_calls": [], + "final_answer": "37.85", + "correct": true, + "prediction": "37.85", + "split": "test", + "iteration": "eval_skilled" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_000/task-006.json b/smoke_run/raw/iter_000/task-006.json new file mode 100644 index 0000000..ae8a2ae --- /dev/null +++ b/smoke_run/raw/iter_000/task-006.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-006", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 15.0 kilograms to pounds. The conversion factor is 2.20462. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=33.07)" + }, + { + "role": "assistant", + "content": "ANSWER: 33.0" + } + ], + "tool_calls": [], + "final_answer": "33.0", + "correct": false, + "prediction": "33.0", + "split": "val", + "iteration": "0" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_000/task-007.json b/smoke_run/raw/iter_000/task-007.json new file mode 100644 index 0000000..1fa673c --- /dev/null +++ b/smoke_run/raw/iter_000/task-007.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-007", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 17.3333 kilograms to pounds. The conversion factor is 2.20462. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=38.21)" + }, + { + "role": "assistant", + "content": "ANSWER: 38.2" + } + ], + "tool_calls": [], + "final_answer": "38.2", + "correct": false, + "prediction": "38.2", + "split": "val", + "iteration": "0" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_000/task-008.json b/smoke_run/raw/iter_000/task-008.json new file mode 100644 index 0000000..b0d1857 --- /dev/null +++ b/smoke_run/raw/iter_000/task-008.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-008", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 3.0 pounds to kilograms. The conversion factor is 0.453592. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=1.36)" + }, + { + "role": "assistant", + "content": "ANSWER: 1.3" + } + ], + "tool_calls": [], + "final_answer": "1.3", + "correct": false, + "prediction": "1.3", + "split": "val", + "iteration": "0" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_001/task-001.json b/smoke_run/raw/iter_001/task-001.json new file mode 100644 index 0000000..a8405b3 --- /dev/null +++ b/smoke_run/raw/iter_001/task-001.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-001", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 3.3333 kilometers to miles. The conversion factor is 0.621371. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=2.07)" + }, + { + "role": "assistant", + "content": "ANSWER: 2.0" + } + ], + "tool_calls": [], + "final_answer": "2.0", + "correct": false, + "prediction": "2.0", + "split": "train", + "iteration": "1" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_001/task-002.json b/smoke_run/raw/iter_001/task-002.json new file mode 100644 index 0000000..0a74705 --- /dev/null +++ b/smoke_run/raw/iter_001/task-002.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-002", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 5.6667 kilometers to miles. The conversion factor is 0.621371. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=3.52)" + }, + { + "role": "assistant", + "content": "ANSWER: 3.5" + } + ], + "tool_calls": [], + "final_answer": "3.5", + "correct": false, + "prediction": "3.5", + "split": "train", + "iteration": "1" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_001/task-003.json b/smoke_run/raw/iter_001/task-003.json new file mode 100644 index 0000000..2f25099 --- /dev/null +++ b/smoke_run/raw/iter_001/task-003.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-003", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 8.0 kilometers to miles. The conversion factor is 0.621371. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=4.97)" + }, + { + "role": "assistant", + "content": "ANSWER: 4.9" + } + ], + "tool_calls": [], + "final_answer": "4.9", + "correct": false, + "prediction": "4.9", + "split": "train", + "iteration": "1" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_001/task-004.json b/smoke_run/raw/iter_001/task-004.json new file mode 100644 index 0000000..1632bb0 --- /dev/null +++ b/smoke_run/raw/iter_001/task-004.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-004", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 10.3333 miles to kilometers. The conversion factor is 1.609344. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=16.63)" + }, + { + "role": "assistant", + "content": "ANSWER: 16.6" + } + ], + "tool_calls": [], + "final_answer": "16.6", + "correct": false, + "prediction": "16.6", + "split": "train", + "iteration": "1" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_001/task-005.json b/smoke_run/raw/iter_001/task-005.json new file mode 100644 index 0000000..36c61db --- /dev/null +++ b/smoke_run/raw/iter_001/task-005.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-005", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n(No active skills.)\n" + }, + { + "role": "user", + "content": "Convert 12.6667 miles to kilometers. The conversion factor is 1.609344. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=20.39)" + }, + { + "role": "assistant", + "content": "ANSWER: 20.3" + } + ], + "tool_calls": [], + "final_answer": "20.3", + "correct": false, + "prediction": "20.3", + "split": "train", + "iteration": "1" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_001/task-006.json b/smoke_run/raw/iter_001/task-006.json new file mode 100644 index 0000000..6df7323 --- /dev/null +++ b/smoke_run/raw/iter_001/task-006.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-006", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n## Skill: unit-conversion\n\n---\nname: unit-conversion\ndescription: Correctly convert units with exact arithmetic.\n---\n\n# Unit Conversion Skill\n\n1. Compute using exact ratios.\n2. Round ONLY the final result to 2 decimal places.\n3. State the answer as `ANSWER: `.\n\n" + }, + { + "role": "user", + "content": "Convert 15.0 kilograms to pounds. The conversion factor is 2.20462. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=33.07)" + }, + { + "role": "assistant", + "content": "ANSWER: 33.07" + } + ], + "tool_calls": [], + "final_answer": "33.07", + "correct": true, + "prediction": "33.07", + "split": "val", + "iteration": "1" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_001/task-007.json b/smoke_run/raw/iter_001/task-007.json new file mode 100644 index 0000000..faab2fa --- /dev/null +++ b/smoke_run/raw/iter_001/task-007.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-007", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n## Skill: unit-conversion\n\n---\nname: unit-conversion\ndescription: Correctly convert units with exact arithmetic.\n---\n\n# Unit Conversion Skill\n\n1. Compute using exact ratios.\n2. Round ONLY the final result to 2 decimal places.\n3. State the answer as `ANSWER: `.\n\n" + }, + { + "role": "user", + "content": "Convert 17.3333 kilograms to pounds. The conversion factor is 2.20462. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=38.21)" + }, + { + "role": "assistant", + "content": "ANSWER: 38.21" + } + ], + "tool_calls": [], + "final_answer": "38.21", + "correct": true, + "prediction": "38.21", + "split": "val", + "iteration": "1" +} \ No newline at end of file diff --git a/smoke_run/raw/iter_001/task-008.json b/smoke_run/raw/iter_001/task-008.json new file mode 100644 index 0000000..d5461ba --- /dev/null +++ b/smoke_run/raw/iter_001/task-008.json @@ -0,0 +1,23 @@ +{ + "task_id": "task-008", + "messages": [ + { + "role": "system", + "content": "You are a task-solving agent. Complete the task step by step.\n\nYou may use the following tools by replying with a single line:\nTOOL: {\"arg\": \"value\"}\nThe environment will reply with the observation, then you continue.\n\nWhen you are done, reply with your final answer on its own line in the format:\nANSWER: \n\n## Active skills (follow them when applicable)\n## Skill: unit-conversion\n\n---\nname: unit-conversion\ndescription: Correctly convert units with exact arithmetic.\n---\n\n# Unit Conversion Skill\n\n1. Compute using exact ratios.\n2. Round ONLY the final result to 2 decimal places.\n3. State the answer as `ANSWER: `.\n\n" + }, + { + "role": "user", + "content": "Convert 3.0 pounds to kilograms. The conversion factor is 0.453592. Report the numeric value rounded to exactly 2 decimal places. (reference: expected=1.36)" + }, + { + "role": "assistant", + "content": "ANSWER: 1.36" + } + ], + "tool_calls": [], + "final_answer": "1.36", + "correct": true, + "prediction": "1.36", + "split": "val", + "iteration": "1" +} \ No newline at end of file diff --git a/smoke_run/run_state.json b/smoke_run/run_state.json new file mode 100644 index 0000000..9620e5a --- /dev/null +++ b/smoke_run/run_state.json @@ -0,0 +1,15 @@ +{ + "r_best": 1.0, + "history": [ + { + "iteration": 1, + "proposal": "unit-conversion", + "val_score": 1.0, + "accepted": true, + "action": "create" + } + ], + "final_skills": [ + "unit-conversion" + ] +} \ No newline at end of file diff --git a/smoke_run/skills/unit-conversion/PURPOSE.md b/smoke_run/skills/unit-conversion/PURPOSE.md new file mode 100644 index 0000000..e83b258 --- /dev/null +++ b/smoke_run/skills/unit-conversion/PURPOSE.md @@ -0,0 +1 @@ +Created from wiki pattern rounding-errors.md. diff --git a/smoke_run/skills/unit-conversion/SKILL.md b/smoke_run/skills/unit-conversion/SKILL.md new file mode 100644 index 0000000..95f5738 --- /dev/null +++ b/smoke_run/skills/unit-conversion/SKILL.md @@ -0,0 +1,10 @@ +--- +name: unit-conversion +description: Correctly convert units with exact arithmetic. +--- + +# Unit Conversion Skill + +1. Compute using exact ratios. +2. Round ONLY the final result to 2 decimal places. +3. State the answer as `ANSWER: `. diff --git a/smoke_run/wiki/index.md b/smoke_run/wiki/index.md new file mode 100644 index 0000000..ddd8bb8 --- /dev/null +++ b/smoke_run/wiki/index.md @@ -0,0 +1 @@ +- [rounding-errors](wiki/patterns/rounding-errors.md): wrong answers from skipped rounding; FIX: round final result to 2 decimals. diff --git a/smoke_run/wiki/logs.md b/smoke_run/wiki/logs.md new file mode 100644 index 0000000..1985e8f --- /dev/null +++ b/smoke_run/wiki/logs.md @@ -0,0 +1 @@ +Analyzed traces; recorded rounding-error pattern. diff --git a/smoke_run/wiki/patterns/rounding-errors.md b/smoke_run/wiki/patterns/rounding-errors.md new file mode 100644 index 0000000..2d314c2 --- /dev/null +++ b/smoke_run/wiki/patterns/rounding-errors.md @@ -0,0 +1,10 @@ +# Pattern: Rounding errors + +## Description +Demo pattern: naive rounding gives wrong answers. + +## Root cause +Skipping the final rounding step. + +## Workaround +Always round the final result to 2 decimals. diff --git a/smoke_run/wiki/skill-impact.md b/smoke_run/wiki/skill-impact.md new file mode 100644 index 0000000..4c97110 --- /dev/null +++ b/smoke_run/wiki/skill-impact.md @@ -0,0 +1,9 @@ + +## Iteration 1 — create 'unit-conversion' +- validation score: 1.0000 +- outcome: Accepted +- purpose: Created from wiki pattern rounding-errors.md. +- diff: +```diff +(new skill) +```