LZH-YS1998
321 lines
11 KiB
Python
321 lines
11 KiB
Python
"""Integration tests for NativeAgent upgrades.
|
|
|
|
Tests all 7 upgrade items from NATIVE_AGENT_UPGRADE_PLAN.md:
|
|
1. Role-aware system prompts
|
|
2. TODO tool interception
|
|
3. Probe (read-only exploration sub-loop)
|
|
4. Concurrent tool execution
|
|
5. Transient LLM error retry
|
|
6. Doom loop detection
|
|
7. Context compression
|
|
|
|
Runs through the OPC engine with real LLM calls.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from pathlib import Path
|
|
|
|
# Ensure project root on path
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
|
|
from opc.core.config import OPCConfig
|
|
from opc.engine import OPCEngine
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_logs: list[str] = []
|
|
|
|
|
|
def _progress():
|
|
async def cb(text: str) -> None:
|
|
_logs.append(text)
|
|
# Print tool calls for visibility
|
|
if text.startswith("[Tool:"):
|
|
print(f" {text[:120]}")
|
|
return cb
|
|
|
|
|
|
async def run_task(desc: str, task_msg: str, timeout: int = 300) -> tuple[bool, str]:
|
|
"""Run a single task through the engine. Returns (success, response)."""
|
|
config = OPCConfig.load(Path(__file__).parent.parent / ".opc" / "config")
|
|
engine = OPCEngine(config=config, project_id="test", on_progress=_progress())
|
|
_logs.clear()
|
|
|
|
try:
|
|
await engine.initialize()
|
|
response = await asyncio.wait_for(
|
|
engine.process_message(task_msg, project_id="test"),
|
|
timeout=timeout,
|
|
)
|
|
# Consider it successful if we got a non-error response
|
|
success = bool(response and "Error" not in response[:50] and "failed" not in response[:50].lower())
|
|
return success, response
|
|
except asyncio.TimeoutError:
|
|
return False, f"TIMEOUT after {timeout}s"
|
|
except Exception as e:
|
|
return False, f"EXCEPTION: {e}\n{traceback.format_exc()}"
|
|
finally:
|
|
try:
|
|
await engine.shutdown()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def print_result(idx: int, desc: str, success: bool, response: str, elapsed: float):
|
|
status = "PASS" if success else "FAIL"
|
|
icon = "✓" if success else "✗"
|
|
print(f"\n{'='*60}")
|
|
print(f"[{icon}] Test {idx}: {desc} — {status} ({elapsed:.1f}s)")
|
|
print(f"{'='*60}")
|
|
# Show first 300 chars of response
|
|
preview = response[:300].replace("\n", " ")
|
|
print(f" Response: {preview}...")
|
|
if _logs:
|
|
tool_calls = [l for l in _logs if l.startswith("[Tool:")]
|
|
if tool_calls:
|
|
print(f" Tool calls ({len(tool_calls)}):")
|
|
for tc in tool_calls[:5]:
|
|
print(f" {tc[:100]}")
|
|
if len(tool_calls) > 5:
|
|
print(f" ... and {len(tool_calls) - 5} more")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test definitions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
REGULAR_TASKS = [
|
|
# Test 1: Basic single-agent coding task (executor role)
|
|
(
|
|
"Basic coding task (executor role, CODING_GUIDELINES)",
|
|
"Create a Python function that calculates the Fibonacci sequence up to n terms. "
|
|
"Write it to /tmp/opc_test_fib.py and verify it works by running it."
|
|
),
|
|
# Test 2: Review-style task (should use REVIEW_GUIDELINES)
|
|
(
|
|
"Review/analysis task",
|
|
"Read the file /Users/lizongwei/Desktop/Coding_Project/OPENOPC_BENCH/OpenOPC/opc/__init__.py "
|
|
"and tell me what version of OPC is defined there."
|
|
),
|
|
# Test 3: Planning task (should trigger coordinator)
|
|
(
|
|
"Planning/decomposition task",
|
|
"What are the key components needed to build a simple REST API with FastAPI? "
|
|
"Just list the components, don't write code."
|
|
),
|
|
# Test 4: File operations (tests file_read, file_write tools)
|
|
(
|
|
"File read/write operations",
|
|
"Read the file /Users/lizongwei/Desktop/Coding_Project/OPENOPC_BENCH/OpenOPC/pyproject.toml "
|
|
"and tell me the project name and version."
|
|
),
|
|
# Test 5: Shell execution
|
|
(
|
|
"Shell command execution",
|
|
"Run `python --version` and `pip --version` and report the versions."
|
|
),
|
|
# Test 6: Multi-step task (should benefit from TODO tracking)
|
|
(
|
|
"Multi-step task with TODO tracking",
|
|
"Do these three things: 1) List the files in /tmp 2) Create a file /tmp/opc_test_hello.txt with content 'Hello OPC' "
|
|
"3) Read the file back and confirm its content."
|
|
),
|
|
# Test 7: Code search task (tests file_search/list_dir)
|
|
(
|
|
"Code search and analysis",
|
|
"Search the OPC codebase at /Users/lizongwei/Desktop/Coding_Project/OPENOPC_BENCH/OpenOPC/opc/ "
|
|
"for all files that import 'asyncio'. Just list the file paths."
|
|
),
|
|
# Test 8: Python execution
|
|
(
|
|
"Python execution tool",
|
|
"Use Python to calculate: What is the sum of all prime numbers below 100?"
|
|
),
|
|
# Test 9: Web-related (tests web_search if available)
|
|
(
|
|
"Simple computation task",
|
|
"Calculate the factorial of 20 using Python and tell me the result."
|
|
),
|
|
# Test 10: Git operations
|
|
(
|
|
"Git status check",
|
|
"Check the git status of the repository at /Users/lizongwei/Desktop/Coding_Project/OPENOPC_BENCH/OpenOPC "
|
|
"and report the current branch and any uncommitted changes."
|
|
),
|
|
]
|
|
|
|
RANDOM_TASKS = [
|
|
(
|
|
"Random: JSON processing",
|
|
"Create a Python script at /tmp/opc_test_json.py that reads a JSON string "
|
|
"'{\"users\": [{\"name\": \"Alice\", \"age\": 30}, {\"name\": \"Bob\", \"age\": 25}]}' "
|
|
"and prints each user's name and age. Then run it."
|
|
),
|
|
(
|
|
"Random: Directory listing",
|
|
"List the contents of /Users/lizongwei/Desktop/Coding_Project/OPENOPC_BENCH/OpenOPC/opc/layer3_agent/ "
|
|
"and briefly describe the purpose of each file."
|
|
),
|
|
(
|
|
"Random: Math calculation",
|
|
"What is 2^64? Calculate it precisely using Python."
|
|
),
|
|
(
|
|
"Random: Text processing",
|
|
"Write a Python one-liner that reverses the string 'Hello, OPC World!' and tell me the result."
|
|
),
|
|
(
|
|
"Random: File creation",
|
|
"Create a CSV file at /tmp/opc_test_data.csv with 5 rows of sample data "
|
|
"(name, age, city columns) and then read it back."
|
|
),
|
|
(
|
|
"Random: Code explanation",
|
|
"Read the file /Users/lizongwei/Desktop/Coding_Project/OPENOPC_BENCH/OpenOPC/opc/layer4_tools/todo.py "
|
|
"and explain what it does in 2-3 sentences."
|
|
),
|
|
(
|
|
"Random: System info",
|
|
"Run `uname -a` and `whoami` and report the system information."
|
|
),
|
|
(
|
|
"Random: String manipulation",
|
|
"Use Python to count how many vowels are in the sentence: "
|
|
"'The quick brown fox jumps over the lazy dog'"
|
|
),
|
|
(
|
|
"Random: File analysis",
|
|
"How many lines of Python code are in the file "
|
|
"/Users/lizongwei/Desktop/Coding_Project/OPENOPC_BENCH/OpenOPC/opc/layer3_agent/runtime_v2/runtime.py?"
|
|
),
|
|
(
|
|
"Random: Simple script",
|
|
"Write a Python script at /tmp/opc_test_sort.py that sorts the list [5, 2, 8, 1, 9, 3] "
|
|
"and prints the sorted result. Then run it."
|
|
),
|
|
]
|
|
|
|
STRESS_TASKS = [
|
|
(
|
|
"Stress: Long output handling",
|
|
"List all files recursively in /Users/lizongwei/Desktop/Coding_Project/OPENOPC_BENCH/OpenOPC/opc/ "
|
|
"and count the total number of .py files."
|
|
),
|
|
(
|
|
"Stress: Multiple tool calls",
|
|
"Do all of these: 1) Read /tmp/opc_test_hello.txt 2) Run `date` 3) Run `python -c \"print(42**10)\"` "
|
|
"4) List files in /tmp/opc_test*.* pattern. Report all results."
|
|
),
|
|
(
|
|
"Stress: Complex computation",
|
|
"Write a Python script at /tmp/opc_test_matrix.py that multiplies two 3x3 matrices "
|
|
"[[1,2,3],[4,5,6],[7,8,9]] and [[9,8,7],[6,5,4],[3,2,1]] and prints the result. Run it."
|
|
),
|
|
(
|
|
"Stress: File search + analysis",
|
|
"Find all Python files in /Users/lizongwei/Desktop/Coding_Project/OPENOPC_BENCH/OpenOPC/opc/ "
|
|
"that contain the word 'async' and count them."
|
|
),
|
|
(
|
|
"Stress: Multi-file operations",
|
|
"Create three files: /tmp/opc_stress_a.txt with 'AAA', /tmp/opc_stress_b.txt with 'BBB', "
|
|
"/tmp/opc_stress_c.txt with 'CCC'. Then read all three and confirm contents."
|
|
),
|
|
(
|
|
"Stress: Error recovery",
|
|
"Try to read a file that doesn't exist: /tmp/nonexistent_opc_test_xyz.txt. "
|
|
"Then create it with content 'recovered' and read it again."
|
|
),
|
|
(
|
|
"Stress: Chained operations",
|
|
"Run `echo hello` and capture the output. Then create /tmp/opc_stress_echo.txt with that output. "
|
|
"Then run `cat /tmp/opc_stress_echo.txt` to verify."
|
|
),
|
|
(
|
|
"Stress: Code generation + execution",
|
|
"Write a Python script at /tmp/opc_stress_primes.py that finds all prime numbers between 1 and 200. "
|
|
"Run it and report the count of primes found."
|
|
),
|
|
(
|
|
"Stress: Rapid computation",
|
|
"Calculate these using Python: 1) 123456789 * 987654321 2) 2**100 3) sum(range(1, 10001)). "
|
|
"Report all three results."
|
|
),
|
|
(
|
|
"Stress: File read + summary",
|
|
"Read /Users/lizongwei/Desktop/Coding_Project/OPENOPC_BENCH/OpenOPC/opc/engine.py "
|
|
"and tell me: how many async methods does the OPCEngine class have? List their names."
|
|
),
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Runner
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def run_suite(name: str, tasks: list[tuple[str, str]]) -> tuple[int, int]:
|
|
"""Run a test suite. Returns (passed, total)."""
|
|
print(f"\n{'#'*60}")
|
|
print(f"# {name} ({len(tasks)} tasks)")
|
|
print(f"{'#'*60}")
|
|
|
|
passed = 0
|
|
for i, (desc, msg) in enumerate(tasks, 1):
|
|
print(f"\n>>> Running test {i}/{len(tasks)}: {desc}")
|
|
start = time.time()
|
|
success, response = await run_task(desc, msg, timeout=180)
|
|
elapsed = time.time() - start
|
|
print_result(i, desc, success, response, elapsed)
|
|
if success:
|
|
passed += 1
|
|
|
|
print(f"\n{'='*60}")
|
|
print(f"{name} Results: {passed}/{len(tasks)} passed")
|
|
print(f"{'='*60}")
|
|
return passed, len(tasks)
|
|
|
|
|
|
async def main():
|
|
total_passed = 0
|
|
total_tests = 0
|
|
|
|
# Phase 1: Regular tasks
|
|
p, t = await run_suite("REGULAR TASKS", REGULAR_TASKS)
|
|
total_passed += p
|
|
total_tests += t
|
|
|
|
# Phase 2: Random tasks
|
|
p, t = await run_suite("RANDOM TASKS", RANDOM_TASKS)
|
|
total_passed += p
|
|
total_tests += t
|
|
|
|
# Phase 3: Stress tests
|
|
p, t = await run_suite("STRESS TESTS", STRESS_TASKS)
|
|
total_passed += p
|
|
total_tests += t
|
|
|
|
# Final summary
|
|
print(f"\n{'#'*60}")
|
|
print(f"# FINAL SUMMARY: {total_passed}/{total_tests} passed")
|
|
if total_passed == total_tests:
|
|
print(f"# ALL TESTS PASSED!")
|
|
else:
|
|
print(f"# {total_tests - total_passed} FAILURES — need to fix and re-run")
|
|
print(f"{'#'*60}")
|
|
|
|
return total_passed == total_tests
|
|
|
|
|
|
if __name__ == "__main__":
|
|
success = asyncio.run(main())
|
|
sys.exit(0 if success else 1)
|