Files
OpenOPC/opc/layer4_tools/file_ops.py
T
2026-07-01 17:56:31 +08:00

766 lines
30 KiB
Python

"""File system operation tools with compatibility aliases."""
from __future__ import annotations
import asyncio
import difflib
import re
import shutil
from pathlib import Path
from typing import Any
from opc.layer4_tools.output_budget import TextClip, clip_text
from opc.layer4_tools.registry import ToolDefinition
_MAX_TEXT_CHARS = 12_000
_MAX_LIST_RESULTS = 500
_MAX_SEARCH_MATCH_CHARS = 500
_MAX_DIFF_CHARS = 16_000
class PatchApplyError(RuntimeError):
"""Raised when an apply_patch payload cannot be applied safely."""
def _resolve_task_path(path: str, task: Any | None = None) -> Path:
candidate = Path(path)
if candidate.is_absolute():
return candidate
base_dir = ""
if task is not None:
metadata = getattr(task, "metadata", {}) or {}
candidate_roots = [
str(metadata.get("output_root", "") or "").strip(),
str(metadata.get("target_output_dir", "") or "").strip(),
str(metadata.get("workspace_root", "") or "").strip(),
str(metadata.get("comms_workspace_root", "") or "").strip(),
]
for raw in candidate_roots:
if not raw:
continue
resolved = Path(raw).expanduser()
if resolved.exists():
base_dir = str(resolved)
break
if not base_dir:
base_dir = str(resolved)
if base_dir:
return Path(base_dir) / candidate
return candidate
def _safe_read_text(path: Path) -> str:
return path.read_text(encoding="utf-8", errors="replace")
def _normalize_text(value: str) -> str:
return value.replace("\r\n", "\n").replace("\r", "\n")
def _truncate_text(value: str, *, limit: int = _MAX_TEXT_CHARS) -> str:
return clip_text(value, limit=limit, marker="truncated").text
def _truncate_match_entry(value: str, *, limit: int = _MAX_SEARCH_MATCH_CHARS) -> str:
return clip_text(value, limit=limit, marker="match truncated", prefer_newline=False).text.replace("\n", " ")
def _build_diff_preview(old_text: str, new_text: str, path: str) -> TextClip:
old_lines = _normalize_text(old_text).splitlines()
new_lines = _normalize_text(new_text).splitlines()
diff = difflib.unified_diff(
old_lines,
new_lines,
fromfile=f"{path} (before)",
tofile=f"{path} (after)",
lineterm="",
)
return clip_text("\n".join(diff), limit=_MAX_DIFF_CHARS, marker="diff truncated")
def _diff_fields(old_text: str, new_text: str, path: str) -> dict[str, Any]:
preview = _build_diff_preview(old_text, new_text, path)
return {
"diff_preview": preview.text,
"diff_truncated": preview.truncated,
"diff_omitted_chars": preview.omitted_chars,
"diff_original_chars": preview.original_chars,
}
def _render_file_slice(
lines: list[str],
*,
offset: int,
limit: int | None,
include_line_numbers: bool,
) -> dict[str, Any]:
start = max(0, int(offset or 0))
line_limit = int(limit) if limit and int(limit) > 0 else None
end = start + line_limit if line_limit is not None else len(lines)
selected = lines[start:end]
rendered_lines = [
f"{start + idx + 1}: {line}" if include_line_numbers else line
for idx, line in enumerate(selected)
]
full_rendered = "".join(rendered_lines)
if len(full_rendered) <= _MAX_TEXT_CHARS:
returned_count = len(selected)
return {
"content": full_rendered,
"truncated": False,
"omitted_chars": 0,
"returned_line_count": returned_count,
"next_offset": start + returned_count if start + returned_count < len(lines) else None,
}
kept: list[str] = []
used = 0
for rendered in rendered_lines:
if used + len(rendered) > _MAX_TEXT_CHARS:
if not kept:
first = clip_text(rendered, limit=_MAX_TEXT_CHARS, marker="file_read line truncated")
kept.append(first.text)
break
kept.append(rendered)
used += len(rendered)
returned_count = max(1, len(kept)) if rendered_lines else 0
content = "".join(kept)
if not content.endswith("]"):
omitted = max(0, len(full_rendered) - len(content))
content = content.rstrip() + f"\n[file_read truncated: {omitted} chars omitted]"
return {
"content": content,
"truncated": True,
"omitted_chars": max(0, len(full_rendered) - len("".join(kept))),
"returned_line_count": returned_count,
"next_offset": start + returned_count if start + returned_count < len(lines) else None,
}
def _iter_directory(root: Path, *, recursive: bool, max_depth: int) -> list[Path]:
results: list[Path] = []
def _walk(current: Path, depth: int) -> None:
if depth > max_depth:
return
try:
entries = sorted(current.iterdir(), key=lambda item: (not item.is_dir(), item.name.lower()))
except (FileNotFoundError, PermissionError):
return
for item in entries:
results.append(item)
if recursive and item.is_dir():
_walk(item, depth + 1)
_walk(root, 0)
return results
def _apply_text_patch(current_text: str, patch_lines: list[str], path: str) -> str:
updated = _normalize_text(current_text)
current_pos = 0
chunk: list[str] = []
def _apply_chunk(buffer: list[str], working_text: str, cursor: int) -> tuple[str, int]:
filtered = [line for line in buffer if line and line != "*** End of File"]
if not filtered:
return working_text, cursor
search_lines = [line[1:] for line in filtered if line[:1] in {" ", "-"}]
replace_lines = [line[1:] for line in filtered if line[:1] in {" ", "+"}]
search_text = "\n".join(search_lines)
replacement_text = "\n".join(replace_lines)
if not search_text:
raise PatchApplyError(f"Patch chunk for `{path}` is missing search context.")
start = working_text.find(search_text, cursor)
if start < 0:
start = working_text.find(search_text)
if start < 0:
raise PatchApplyError(f"Unable to match patch chunk in `{path}`.")
end = start + len(search_text)
next_text = working_text[:start] + replacement_text + working_text[end:]
return next_text, start + len(replacement_text)
for line in patch_lines:
if line.startswith("@@"):
updated, current_pos = _apply_chunk(chunk, updated, current_pos)
chunk = []
continue
if line.startswith((" ", "+", "-")) or line == "*** End of File":
chunk.append(line)
continue
raise PatchApplyError(f"Unsupported patch line in `{path}`: {line}")
updated, current_pos = _apply_chunk(chunk, updated, current_pos)
_ = current_pos
return updated
def _parse_patch_operations(patch: str) -> list[dict[str, Any]]:
lines = patch.splitlines()
if not lines or lines[0] != "*** Begin Patch":
raise PatchApplyError("Patch must start with `*** Begin Patch`.")
operations: list[dict[str, Any]] = []
index = 1
while index < len(lines):
line = lines[index]
if line == "*** End Patch":
return operations
if line.startswith("*** Add File: "):
path = line[len("*** Add File: "):].strip()
index += 1
content_lines: list[str] = []
while index < len(lines) and not lines[index].startswith("*** "):
payload = lines[index]
if not payload.startswith("+"):
raise PatchApplyError(f"Add File expects `+` lines only for `{path}`.")
content_lines.append(payload[1:])
index += 1
operations.append({"kind": "add", "path": path, "content": "\n".join(content_lines)})
continue
if line.startswith("*** Delete File: "):
path = line[len("*** Delete File: "):].strip()
operations.append({"kind": "delete", "path": path})
index += 1
continue
if line.startswith("*** Update File: "):
path = line[len("*** Update File: "):].strip()
index += 1
move_to = None
if index < len(lines) and lines[index].startswith("*** Move to: "):
move_to = lines[index][len("*** Move to: "):].strip()
index += 1
patch_lines: list[str] = []
while index < len(lines):
candidate = lines[index]
if candidate == "*** End Patch" or candidate.startswith("*** Add File: ") or candidate.startswith("*** Delete File: ") or candidate.startswith("*** Update File: "):
break
patch_lines.append(candidate)
index += 1
operations.append({"kind": "update", "path": path, "move_to": move_to, "patch_lines": patch_lines})
continue
raise PatchApplyError(f"Unsupported patch operation: {line}")
raise PatchApplyError("Patch is missing `*** End Patch`.")
async def file_read(
path: str,
offset: int = 0,
limit: int | None = None,
include_line_numbers: bool = False,
task: Any | None = None,
) -> dict[str, Any]:
"""Read file contents."""
p = _resolve_task_path(path, task)
if not p.exists():
return {"error": f"File not found: {path}", "success": False}
if not p.is_file():
return {"error": f"Not a file: {path}", "success": False}
try:
text = _safe_read_text(p)
lines = text.splitlines(keepends=True)
rendered = _render_file_slice(
lines,
offset=offset,
limit=limit,
include_line_numbers=include_line_numbers,
)
return {
"content": rendered["content"],
"total_lines": len(text.splitlines()),
"returned_start_line": max(0, int(offset or 0)) + 1 if lines else 0,
"returned_line_count": rendered["returned_line_count"],
"next_offset": rendered["next_offset"],
"truncated": rendered["truncated"],
"omitted_chars": rendered["omitted_chars"],
"path": str(p.resolve()),
"success": True,
}
except Exception as exc:
return {"error": str(exc), "success": False}
async def file_write(path: str, content: str, create_dirs: bool = True, task: Any | None = None) -> dict[str, Any]:
"""Write content to a file."""
p = _resolve_task_path(path, task)
existed = p.exists()
before = _safe_read_text(p) if existed and p.is_file() else ""
try:
if create_dirs:
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(content, encoding="utf-8")
return {
"success": True,
"path": str(p.resolve()),
"bytes_written": len(content.encode("utf-8")),
"created": not existed,
**_diff_fields(before, content, str(p)),
}
except Exception as exc:
return {"error": str(exc), "success": False}
async def file_edit(
path: str,
old_string: str,
new_string: str,
replace_all: bool = False,
task: Any | None = None,
) -> dict[str, Any]:
"""Replace a specific string in a file."""
p = _resolve_task_path(path, task)
if not p.exists():
return {"error": f"File not found: {path}", "success": False}
try:
text = _safe_read_text(p)
count = text.count(old_string)
if count == 0:
return {"error": "old_string not found in file", "success": False}
if count > 1 and not replace_all:
return {"error": f"old_string found {count} times; add more context or use replace_all=true.", "success": False}
updated = text.replace(old_string, new_string) if replace_all else text.replace(old_string, new_string, 1)
p.write_text(updated, encoding="utf-8")
return {
"success": True,
"path": str(p.resolve()),
"replacements": count if replace_all else 1,
**_diff_fields(text, updated, str(p)),
}
except Exception as exc:
return {"error": str(exc), "success": False}
async def apply_patch(patch: str, task: Any | None = None) -> dict[str, Any]:
"""Apply an OpenAI-style patch document to one or more files."""
try:
operations = _parse_patch_operations(patch)
except PatchApplyError as exc:
return {"error": str(exc), "success": False}
changed: list[dict[str, Any]] = []
try:
for operation in operations:
kind = operation["kind"]
raw_path = str(operation["path"])
path = _resolve_task_path(raw_path, task)
if kind == "add":
path.parent.mkdir(parents=True, exist_ok=True)
new_text = str(operation["content"])
path.write_text(new_text, encoding="utf-8")
changed.append({
"kind": "add",
"path": str(path.resolve()),
**_diff_fields("", new_text, str(path)),
})
continue
if kind == "delete":
if not path.exists():
raise PatchApplyError(f"Cannot delete missing file `{raw_path}`.")
before = _safe_read_text(path) if path.is_file() else ""
path.unlink()
changed.append({
"kind": "delete",
"path": str(path.resolve()),
**_diff_fields(before, "", str(path)),
})
continue
if kind == "update":
if not path.exists():
raise PatchApplyError(f"Cannot update missing file `{raw_path}`.")
before = _safe_read_text(path)
updated = _apply_text_patch(before, list(operation.get("patch_lines", [])), raw_path)
target = _resolve_task_path(str(operation.get("move_to") or raw_path), task)
target.parent.mkdir(parents=True, exist_ok=True)
if target != path:
path.unlink()
target.write_text(updated, encoding="utf-8")
changed.append({
"kind": "update",
"path": str(target.resolve()),
"moved_from": str(path.resolve()) if target != path else "",
**_diff_fields(before, updated, str(target)),
})
continue
return {
"success": True,
"changed_files": changed,
"applied_operations": len(changed),
}
except Exception as exc:
return {"error": str(exc), "success": False}
async def glob(
pattern: str,
path: str = ".",
recursive: bool = True,
include_dirs: bool = False,
max_results: int = 200,
task: Any | None = None,
) -> dict[str, Any]:
"""Return files matching a glob pattern."""
root = _resolve_task_path(path, task)
if not root.exists():
return {"error": f"Directory not found: {path}", "success": False}
iterator = root.rglob(pattern) if recursive else root.glob(pattern)
entries: list[str] = []
for item in iterator:
if item.is_dir() and not include_dirs:
continue
try:
entries.append(str(item.relative_to(root)))
except ValueError:
entries.append(str(item))
if len(entries) >= max_results:
break
return {
"success": True,
"path": str(root.resolve()),
"pattern": pattern,
"entries": entries,
"count": len(entries),
}
async def grep(
query: str,
path: str = ".",
file_glob: str = "*",
max_results: int = 200,
offset: int = 0,
head_limit: int | None = None,
output_mode: str = "content",
case_sensitive: bool = False,
context_before: int = 0,
context_after: int = 0,
task: Any | None = None,
) -> dict[str, Any]:
"""Search file contents for a regex or fixed-string query."""
root = _resolve_task_path(path, task)
if not root.exists():
return {"error": f"Directory not found: {path}", "success": False}
applied_offset = max(0, int(offset or 0))
effective_limit = max_results if head_limit is None else int(head_limit)
def _apply_pagination(items: list[str]) -> tuple[list[str], bool, int | None]:
if effective_limit == 0:
paged = items[applied_offset:]
return paged, False, None
limit_value = max(0, effective_limit)
paged = items[applied_offset:applied_offset + limit_value]
truncated = applied_offset + limit_value < len(items)
next_offset = applied_offset + limit_value if truncated else None
return paged, truncated, next_offset
def _format_output(all_matches: list[str]) -> dict[str, Any]:
mode = str(output_mode or "content").strip() or "content"
if mode not in {"content", "files_with_matches", "count"}:
mode = "content"
items = all_matches
if mode == "files_with_matches":
seen: set[str] = set()
files: list[str] = []
for entry in all_matches:
file_part = entry.split(":", 1)[0]
if file_part not in seen:
seen.add(file_part)
files.append(file_part)
items = files
elif mode == "count":
counts: dict[str, int] = {}
for entry in all_matches:
file_part = entry.split(":", 1)[0]
counts[file_part] = counts.get(file_part, 0) + 1
items = [f"{name}: {count}" for name, count in sorted(counts.items())]
paged, truncated, next_offset = _apply_pagination(items)
clipped = [_truncate_match_entry(line) for line in paged]
result: dict[str, Any] = {
"success": True,
"path": str(root.resolve()),
"query": query,
"output_mode": mode,
"matches": clipped,
"count": len(clipped),
"total_count": len(items),
"truncated": truncated,
"next_offset": next_offset,
"applied_offset": applied_offset,
}
if truncated and effective_limit != 0:
result["applied_limit"] = max(0, effective_limit)
if mode == "files_with_matches":
result["files"] = clipped
result["num_files"] = len(items)
if mode == "count":
result["counts"] = clipped
return result
rg = shutil.which("rg")
if rg:
args = [rg, "--no-heading", "--line-number", "--color", "never", "--glob", file_glob]
if case_sensitive:
args.append("--case-sensitive")
else:
args.append("--ignore-case")
if context_before:
args.extend(["-B", str(context_before)])
if context_after:
args.extend(["-A", str(context_after)])
args.extend([query, str(root)])
try:
proc = await asyncio.create_subprocess_exec(
*args,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30)
output = stdout.decode("utf-8", errors="replace").splitlines()
if proc.returncode not in (0, 1):
return {"error": stderr.decode("utf-8", errors="replace"), "success": False}
return _format_output(output)
except Exception:
pass
flags = 0 if case_sensitive else re.IGNORECASE
pattern = re.compile(query, flags=flags)
matches: list[str] = []
for candidate in root.rglob(file_glob):
if not candidate.is_file():
continue
try:
lines = _safe_read_text(candidate).splitlines()
except Exception:
continue
for line_index, line in enumerate(lines):
if not pattern.search(line):
continue
start = max(0, line_index - max(0, context_before))
end = min(len(lines), line_index + max(0, context_after) + 1)
context_lines = []
for idx in range(start, end):
context_lines.append(
_truncate_match_entry(f"{candidate.relative_to(root)}:{idx + 1}:{lines[idx]}")
)
matches.extend(context_lines)
return _format_output(matches)
async def file_search(
pattern: str,
directory: str = ".",
file_glob: str = "*",
max_results: int = 50,
offset: int = 0,
head_limit: int | None = None,
task: Any | None = None,
) -> dict[str, Any]:
"""Compatibility wrapper for grep."""
result = await grep(
query=pattern,
path=directory,
file_glob=file_glob,
max_results=max_results,
offset=offset,
head_limit=head_limit,
task=task,
)
if result.get("success"):
result["matches"] = list(result.get("matches", []))
return result
async def list_dir(path: str = ".", recursive: bool = False, max_depth: int = 3, task: Any | None = None) -> dict[str, Any]:
"""List directory contents."""
root = _resolve_task_path(path, task)
if not root.exists():
return {"error": f"Directory not found: {path}", "success": False}
if not root.is_dir():
return {"error": f"Not a directory: {path}", "success": False}
entries: list[str] = []
items: list[dict[str, Any]] = []
for item in _iter_directory(root, recursive=recursive, max_depth=max_depth):
try:
relative = str(item.relative_to(root))
except ValueError:
relative = str(item)
entry_path = relative + ("/" if item.is_dir() else "")
entries.append(entry_path)
items.append({
"path": relative + ("/" if item.is_dir() else ""),
"is_dir": item.is_dir(),
"size": item.stat().st_size if item.is_file() else 0,
})
if len(items) >= _MAX_LIST_RESULTS:
break
return {
"success": True,
"path": str(root.resolve()),
"entries": entries,
"items": items,
"total": len(items),
}
def create_file_tools() -> list[ToolDefinition]:
return [
ToolDefinition(
name="file_read",
description="Read the contents of a file. Supports line offset, line limit, and optional line numbers.",
parameters={
"type": "object",
"properties": {
"path": {"type": "string", "description": "Path to the file"},
"offset": {"type": "integer", "description": "Line offset (0-based)", "default": 0},
"limit": {"type": "integer", "description": "Maximum lines to read"},
"include_line_numbers": {"type": "boolean", "description": "Prefix output with line numbers", "default": False},
},
"required": ["path"],
},
func=file_read,
category="file",
concurrency_safe=True,
read_only=True,
self_bounded_output=True,
max_result_chars=80_000,
),
ToolDefinition(
name="file_write",
description="Write content to a file and return a diff preview against the previous content.",
parameters={
"type": "object",
"properties": {
"path": {"type": "string", "description": "Path to the file"},
"content": {"type": "string", "description": "Content to write"},
"create_dirs": {"type": "boolean", "description": "Create parent directories if needed", "default": True},
},
"required": ["path", "content"],
},
func=file_write,
category="file",
concurrency_safe=False,
read_only=False,
),
ToolDefinition(
name="file_edit",
description="Replace a specific string in a file and return a diff preview.",
parameters={
"type": "object",
"properties": {
"path": {"type": "string", "description": "Path to the file"},
"old_string": {"type": "string", "description": "Exact string to find"},
"new_string": {"type": "string", "description": "Replacement string"},
"replace_all": {"type": "boolean", "description": "Replace all occurrences instead of requiring uniqueness", "default": False},
},
"required": ["path", "old_string", "new_string"],
},
func=file_edit,
category="file",
concurrency_safe=False,
read_only=False,
),
ToolDefinition(
name="apply_patch",
description="Apply an OpenAI-style patch document to one or more files.",
parameters={
"type": "object",
"properties": {
"patch": {"type": "string", "description": "Patch document beginning with `*** Begin Patch`"},
},
"required": ["patch"],
},
func=apply_patch,
category="file",
concurrency_safe=False,
read_only=False,
),
ToolDefinition(
name="grep",
description="Search file contents using ripgrep when available, with a Python fallback.",
parameters={
"type": "object",
"properties": {
"query": {"type": "string", "description": "Regex or fixed-string query"},
"path": {"type": "string", "description": "Directory to search", "default": "."},
"file_glob": {"type": "string", "description": "File glob pattern", "default": "*"},
"max_results": {"type": "integer", "description": "Maximum returned matches", "default": 200},
"offset": {"type": "integer", "description": "Skip this many results before returning matches", "default": 0},
"head_limit": {"type": "integer", "description": "Maximum returned entries; 0 means unlimited", "default": 200},
"output_mode": {
"type": "string",
"description": "Return matching lines (`content`), unique files (`files_with_matches`), or per-file counts (`count`)",
"default": "content",
},
"case_sensitive": {"type": "boolean", "description": "Use case-sensitive matching", "default": False},
"context_before": {"type": "integer", "description": "Context lines before each hit", "default": 0},
"context_after": {"type": "integer", "description": "Context lines after each hit", "default": 0},
},
"required": ["query"],
},
func=grep,
category="file",
concurrency_safe=True,
read_only=True,
self_bounded_output=True,
max_result_chars=80_000,
),
ToolDefinition(
name="glob",
description="Return files matching a glob pattern.",
parameters={
"type": "object",
"properties": {
"pattern": {"type": "string", "description": "Glob pattern to match"},
"path": {"type": "string", "description": "Directory to scan", "default": "."},
"recursive": {"type": "boolean", "description": "Search recursively", "default": True},
"include_dirs": {"type": "boolean", "description": "Include directories in the result set", "default": False},
"max_results": {"type": "integer", "description": "Maximum entries to return", "default": 200},
},
"required": ["pattern"],
},
func=glob,
category="file",
concurrency_safe=True,
read_only=True,
),
ToolDefinition(
name="file_search",
description="Compatibility alias for `grep`.",
parameters={
"type": "object",
"properties": {
"pattern": {"type": "string", "description": "Regex pattern to search"},
"directory": {"type": "string", "description": "Directory to search in", "default": "."},
"file_glob": {"type": "string", "description": "File glob pattern", "default": "*"},
"max_results": {"type": "integer", "description": "Maximum results", "default": 50},
"offset": {"type": "integer", "description": "Skip this many results before returning matches", "default": 0},
"head_limit": {"type": "integer", "description": "Maximum returned entries; 0 means unlimited"},
},
"required": ["pattern"],
},
func=file_search,
category="file",
concurrency_safe=True,
read_only=True,
),
ToolDefinition(
name="list_dir",
description="List directory contents. Optionally recurse up to a maximum depth.",
parameters={
"type": "object",
"properties": {
"path": {"type": "string", "description": "Directory path", "default": "."},
"recursive": {"type": "boolean", "description": "List recursively", "default": False},
"max_depth": {"type": "integer", "description": "Maximum recursion depth", "default": 3},
},
"required": [],
},
func=list_dir,
category="file",
concurrency_safe=True,
read_only=True,
),
]