feat: deep-agent canvas, live observability, and multi-environment tooling

Self-hosted platform for building, testing, and shipping LangChain/LangGraph agents. Deep-agent sub-agents on the canvas, a live tracing/observability timeline, auto-provisioned built-in tools with import/export, per-environment tool variables, streamed evaluations, and per-user auth token forwarding.

apps/api/tests/conftest.py

@@ -0,0 +1,50 @@
+"""Shared test fixtures.
+
+CRITICAL: tests must NOT touch the dev database. We point every persistence path at
+a throwaway temp dir *before* any `forge` module imports (the engine, settings, secret
+store, and Chroma path are all bound at import time from these env vars).
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from pathlib import Path
+
+_TMP = Path(tempfile.mkdtemp(prefix="forge_tests_"))
+os.environ.setdefault("FORGE_DATABASE_URL", f"sqlite+aiosqlite:///{(_TMP / 'test.db').as_posix()}")
+os.environ.setdefault("FORGE_CHROMA_PATH", (_TMP / "chroma").as_posix())
+os.environ.setdefault("FORGE_CHECKPOINT_DB", "memory")
+os.environ.setdefault("FORGE_SECRET_KEY_FILE", (_TMP / "master.key").as_posix())
+os.environ.setdefault("FORGE_SEED_DEMO", "false")
+# Tests hit MockTransport / fake hosts; don't do real DNS in the SSRF guard. The
+# guard's blocking logic is covered explicitly in test_ssrf.py (explicit policies).
+os.environ.setdefault("FORGE_EGRESS_BLOCK_PRIVATE", "false")
+# Auth defaults to ON in the real app; the test suite mostly calls services directly
+# without tokens, so run permissive. test_auth forces auth_required=True where it matters.
+os.environ.setdefault("FORGE_AUTH_REQUIRED", "false")
+
+import pytest  # noqa: E402
+
+from forge.db.base import init_db  # noqa: E402
+
+
+@pytest.fixture(autouse=True)
+async def _ensure_tables():
+    # create_all is idempotent; cheap to run per-test for an isolated DB state.
+    await init_db()
+    yield
+
+
+@pytest.fixture(autouse=True)
+def _reset_sse_appstatus():
+    # sse_starlette caches a module-level `should_exit_event` bound to the FIRST event loop it
+    # runs in. pytest-asyncio gives each test a fresh loop, so a *second* streaming test in the
+    # process would await that stale event -> "bound to a different event loop". Reset it per
+    # test so each SSE response recreates the event in its own loop.
+    try:
+        from sse_starlette.sse import AppStatus
+    except ImportError:
+        return
+    AppStatus.should_exit = False
+    AppStatus.should_exit_event = None

apps/api/tests/test_advanced_nodes.py

@@ -0,0 +1,54 @@
+"""parallel_fanout (Send map) + join aggregation, and the loop node."""
+
+from __future__ import annotations
+
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.engine.compiler import compile_workflow
+from forge.engine.context import CompileContext
+from forge.nodes.flow import loop_factory
+from forge.services.runtime import make_runtime_ctx
+
+_FANOUT_WF = {
+    "id": "fan", "version": 1,
+    "state": {
+        "messages": {"type": "list[message]", "reducer": "add_messages"},
+        "items": {"type": "list[json]", "reducer": "last"},
+        "results": {"type": "list[str]", "reducer": "add"},
+    },
+    "entry_node": "start",
+    "nodes": [
+        {"id": "start", "type": "start", "config": {}},
+        {"id": "fan", "type": "parallel_fanout", "config": {"over": "items", "child_node": "worker", "item_key": "item"}},
+        {"id": "worker", "type": "transform", "config": {"expression": "[item]", "output_key": "results"}},
+        {"id": "join", "type": "join", "config": {"reducer": "concat"}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [
+        {"source": "start", "target": "fan"},
+        {"source": "worker", "target": "join"},
+        {"source": "join", "target": "end"},
+    ],
+}
+
+
+async def test_parallel_fanout_maps_over_items():
+    ctx = make_runtime_ctx("t_fan", "p_fan")
+    ctx.checkpointer = InMemorySaver()
+    graph = compile_workflow(_FANOUT_WF, ctx)
+    out = await graph.ainvoke({"items": ["a", "b", "c"]}, {"configurable": {"thread_id": "f1"}})
+    assert sorted(out["results"]) == ["a", "b", "c"]  # every item processed in parallel + aggregated
+
+
+def test_loop_node_counts_and_stops():
+    node = loop_factory({"max_iter": 3, "condition": ""}, CompileContext(tenant_id="t", project_id="p"))
+    s1 = node({"_loop_count": 0})
+    assert s1 == {"_loop_count": 1, "_loop": "continue"}
+    s2 = node({"_loop_count": 2})
+    assert s2 == {"_loop_count": 3, "_loop": "done"}  # hit max_iter
+
+
+def test_loop_condition_stops_early():
+    node = loop_factory({"max_iter": 10, "condition": "keep_going == True"}, CompileContext(tenant_id="t", project_id="p"))
+    assert node({"_loop_count": 0, "keep_going": True})["_loop"] == "continue"
+    assert node({"_loop_count": 0, "keep_going": False})["_loop"] == "done"

apps/api/tests/test_agent_tool_dedup.py

@@ -0,0 +1,30 @@
+"""An agent binds each tool NAME to the model at most once.
+
+Two layers protect the model call: `resolve_tool_ids` de-dups by id (a tool that lives in several
+tool sets is one record → sent once), and `_dedup_tools_by_name` is the final guard against a
+name collision from distinct records/sources (tool names aren't unique per project, and the list
+mixes tools + knowledge + MCP + components). Providers reject a duplicate function name.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from forge.nodes.agent_node import _dedup_tools_by_name
+
+
+def test_dedup_keeps_first_occurrence_by_name():
+    first = SimpleNamespace(name="get_orders")
+    dup = SimpleNamespace(name="get_orders")  # different object, same name (two Tool records)
+    other = SimpleNamespace(name="create_order")
+    out = _dedup_tools_by_name([first, other, dup])
+    assert [t.name for t in out] == ["get_orders", "create_order"]
+    assert out[0] is first  # the first occurrence wins
+
+
+def test_dedup_never_drops_unnamed_tools():
+    x = SimpleNamespace()  # no .name to key on
+    y = SimpleNamespace()
+    named = SimpleNamespace(name="t")
+    out = _dedup_tools_by_name([x, named, y])
+    assert len(out) == 3

apps/api/tests/test_app_event.py

@@ -0,0 +1,65 @@
+"""app_event polling trigger: dispatch a run per NEW item, dedup the rest."""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+import httpx
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.db.base import SessionLocal
+from forge.models import Trigger, Workflow
+from forge.services.dispatch import _poll_app_event
+from forge.services.runs import RunService
+
+_WF = {
+    "id": "wf_ae", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "agent",
+    "nodes": [
+        {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": "fake:ok"}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "agent", "target": "end"}],
+}
+
+_PAYLOAD = {"items": [{"id": "1", "msg": "first"}, {"id": "2", "msg": "second"}]}
+
+
+async def test_app_event_dispatches_new_then_dedupes(monkeypatch):
+    client = httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(200, json=_PAYLOAD)))
+    monkeypatch.setattr("forge.util.http.shared_async_client", lambda: client)
+
+    async with SessionLocal() as s:
+        wf = Workflow(tenant_id="t_ae", project_id="p_ae", name="AE", executable=_WF, status="active")
+        s.add(wf)
+        await s.flush()
+        trig = Trigger(
+            tenant_id="t_ae", project_id="p_ae", workflow_id=wf.id, node_id="evt", kind="app_event",
+            # last_fired_at in the past => not the baseline poll, so items dispatch
+            last_fired_at=datetime.utcnow() - timedelta(minutes=10),
+            config={"poll_url": "https://api.example.com/events", "items_path": "items",
+                    "dedupe_key": "id", "message_path": "msg", "interval_minutes": 1},
+            meta={},
+        )
+        s.add(trig)
+        await s.commit()
+        await s.refresh(trig)
+        tid = trig.id
+
+    rs = RunService(checkpointer=InMemorySaver())
+
+    # first poll: both items are new -> 2 dispatched
+    async with SessionLocal() as s:
+        t = await s.get(Trigger, tid)
+        n1 = await _poll_app_event(rs, t)
+    assert n1 == 2
+
+    # second poll: same items -> deduped -> 0 dispatched
+    async with SessionLocal() as s:
+        t = await s.get(Trigger, tid)
+        assert set(t.meta["seen"]) == {"1", "2"}  # cursor persisted
+        n2 = await _poll_app_event(rs, t)
+    assert n2 == 0
+
+    await client.aclose()

apps/api/tests/test_auth.py

@@ -0,0 +1,136 @@
+"""Auth, RBAC, and team-management tests (in-process ASGI)."""
+
+from __future__ import annotations
+
+import uuid
+
+import httpx
+
+from forge.config import settings
+from forge.main import create_app
+
+
+def _client() -> httpx.AsyncClient:
+    app = create_app()
+    return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test")
+
+
+def _email() -> str:
+    return f"u{uuid.uuid4().hex[:10]}@example.com"
+
+
+async def test_register_login_me_flow():
+    async with _client() as c:
+        email = _email()
+        r = await c.post("/v1/auth/register", json={"email": email, "password": "supersecret1"})
+        assert r.status_code == 201, r.text
+        body = r.json()
+        assert body["user"]["role"] == "owner"
+        token = body["access_token"]
+
+        # The same email may own another workspace. If both accounts use the same
+        # credentials, login requires the workspace id to disambiguate them.
+        second = await c.post(
+            "/v1/auth/register",
+            json={"email": email, "password": "supersecret1", "workspace_name": "Second"},
+        )
+        assert second.status_code == 201
+        assert second.json()["user"]["tenant_id"] != body["user"]["tenant_id"]
+
+        # wrong password
+        assert (await c.post("/v1/auth/login", json={"email": email, "password": "nope"})).status_code == 401
+        # correct password
+        assert (
+            await c.post("/v1/auth/login", json={"email": email, "password": "supersecret1"})
+        ).status_code == 401
+        r = await c.post(
+            "/v1/auth/login",
+            json={
+                "email": email,
+                "password": "supersecret1",
+                "workspace_id": body["user"]["tenant_id"],
+            },
+        )
+        assert r.status_code == 200
+
+        # me with token
+        r = await c.get("/v1/auth/me", headers={"Authorization": f"Bearer {token}"})
+        assert r.status_code == 200 and r.json()["email"] == email
+
+
+async def test_auth_required_blocks_anonymous(monkeypatch):
+    monkeypatch.setattr(settings, "auth_required", True)
+    async with _client() as c:
+        assert (await c.get("/v1/auth/me")).status_code == 401
+
+
+async def test_rbac_viewer_cannot_manage_team():
+    async with _client() as c:
+        owner_email, viewer_email = _email(), _email()
+        owner = (await c.post("/v1/auth/register", json={"email": owner_email, "password": "ownerpass1"})).json()
+        oh = {"Authorization": f"Bearer {owner['access_token']}"}
+
+        # owner invites a viewer with a password so they can log in
+        r = await c.post("/v1/team/members", json={"email": viewer_email, "role": "viewer", "password": "viewerpass1"}, headers=oh)
+        assert r.status_code == 201, r.text
+
+        viewer = (await c.post("/v1/auth/login", json={"email": viewer_email, "password": "viewerpass1"})).json()
+        vh = {"Authorization": f"Bearer {viewer['access_token']}"}
+
+        # viewer is forbidden from listing/managing the team
+        assert (await c.get("/v1/team/members", headers=vh)).status_code == 403
+        # owner can
+        r = await c.get("/v1/team/members", headers=oh)
+        assert r.status_code == 200 and len(r.json()) == 2
+
+
+async def test_invite_without_password_emails_link_and_can_be_accepted():
+    async with _client() as c:
+        owner_email, invitee_email = _email(), _email()
+        owner = (await c.post("/v1/auth/register", json={"email": owner_email, "password": "ownerpass1"})).json()
+        oh = {"Authorization": f"Bearer {owner['access_token']}"}
+
+        # invite with no password => a pending 'invited' user. SMTP is unconfigured in tests,
+        # so the API hands back a redeemable link instead of emailing it.
+        r = await c.post("/v1/team/members", json={"email": invitee_email, "role": "editor"}, headers=oh)
+        assert r.status_code == 201, r.text
+        body = r.json()
+        assert body["status"] == "invited" and body["email_sent"] is False
+        assert "invite=" in body.get("invite_url", "")
+        token = body["invite_url"].split("invite=", 1)[1]
+
+        # can't log in yet - no password set
+        assert (await c.post("/v1/auth/login", json={"email": invitee_email, "password": "whatever1"})).status_code == 401
+
+        # invite-info reflects the pending invite
+        info = await c.get(f"/v1/auth/invite-info?token={token}")
+        assert info.status_code == 200 and info.json()["email"] == invitee_email
+
+        # redeeming sets the invitee's own password and logs them in
+        acc = await c.post("/v1/auth/accept-invite", json={"token": token, "password": "myownpass1"})
+        assert acc.status_code == 200, acc.text
+        assert acc.json()["user"]["email"] == invitee_email
+        assert (await c.post("/v1/auth/login", json={"email": invitee_email, "password": "myownpass1"})).status_code == 200
+
+        # a used invite can't be redeemed again
+        assert (await c.post("/v1/auth/accept-invite", json={"token": token, "password": "another123"})).status_code == 400
+
+
+async def test_refresh_issues_new_access_token():
+    async with _client() as c:
+        email = _email()
+        body = (await c.post("/v1/auth/register", json={"email": email, "password": "supersecret1"})).json()
+        r = await c.post("/v1/auth/refresh", json={"refresh_token": body["refresh_token"]})
+        assert r.status_code == 200 and "access_token" in r.json()
+        # an access token is not accepted as a refresh token
+        assert (await c.post("/v1/auth/refresh", json={"refresh_token": body["access_token"]})).status_code == 401
+
+
+async def test_cannot_demote_only_owner():
+    async with _client() as c:
+        email = _email()
+        body = (await c.post("/v1/auth/register", json={"email": email, "password": "ownerpass1"})).json()
+        oh = {"Authorization": f"Bearer {body['access_token']}"}
+        uid = body["user"]["id"]
+        r = await c.patch(f"/v1/team/members/{uid}", json={"role": "viewer"}, headers=oh)
+        assert r.status_code == 400 and "owner" in r.json()["detail"].lower()

apps/api/tests/test_body_template_loop.py

@@ -0,0 +1,159 @@
+"""`$each` loop support in JSON body templates (batch many rows in one call).
+
+Before this, one REST tool call could only build a fixed-shape body, so an agent editing N rows
+made N calls (slow, context-heavy, and prone to blowing the graph recursion limit). A body
+template can now carry a `{"$each": "{{input.rows}}", "$as": "row", "$do": {...}}` directive that
+expands one list-valued arg into a variable-length JSON array - so N edits go out in ONE request.
+
+Rendering is structural (parse JSON, then walk with render_value): the output is always valid
+JSON with native types preserved, unlike string-concatenating an array. The path is opt-in on the
+"$each" marker, so existing string-substitution templates are untouched.
+"""
+from __future__ import annotations
+
+import json
+
+from forge.auth_providers.templates import has_each_directive, render_template, render_value
+from forge.tools.rest import _build_body
+
+
+def test_render_value_each_expands_per_item():
+    tmpl = {
+        "$each": "{{input.rows}}",
+        "$as": "row",
+        "$do": {"col": "{{row.editedCol}}", "val": "{{row.editedValue}}"},
+    }
+    rows = [
+        {"editedCol": "unitCost", "editedValue": "96"},
+        {"editedCol": "unitCost", "editedValue": "33"},
+    ]
+    out = render_value(tmpl, {"input": {"rows": rows}}, allow_each=True)
+    assert out == [
+        {"col": "unitCost", "val": "96"},
+        {"col": "unitCost", "val": "33"},
+    ]
+
+
+def test_render_value_each_preserves_native_types_and_outer_vars():
+    tmpl = {
+        "orderId": "{{input.orderId}}",
+        "rows": {"$each": "{{input.rows}}", "$as": "r", "$do": {"n": "{{r.n}}"}},
+    }
+    out = render_value(tmpl, {"input": {"orderId": "ORD-001", "rows": [{"n": 1}, {"n": 2}]}}, allow_each=True)
+    # line numbers stay ints (whole-string token preserves native type); orderId interpolated.
+    assert out == {"orderId": "ORD-001", "rows": [{"n": 1}, {"n": 2}]}
+
+
+def test_render_value_each_missing_yields_empty_list():
+    tmpl = {"$each": "{{input.rows}}", "$as": "row", "$do": {"x": "{{row.x}}"}}
+    assert render_value(tmpl, {"input": {}}, allow_each=True) == []
+
+
+def test_render_value_each_single_value_treated_as_one_item():
+    tmpl = {"$each": "{{input.row}}", "$as": "row", "$do": {"x": "{{row.x}}"}}
+    assert render_value(tmpl, {"input": {"row": {"x": "only"}}}, allow_each=True) == [{"x": "only"}]
+
+
+def test_render_value_each_is_opt_in_only():
+    # WITHOUT allow_each (the default for auth token_fetch / data-node payload callers), a dict that
+    # happens to have a "$each" KEY must stay an ordinary object, NOT be reinterpreted as a loop.
+    tmpl = {"$each": "{{input.rows}}", "$as": "row", "$do": {"x": "{{row.x}}"}}
+    out = render_value(tmpl, {"input": {"rows": [{"x": "a"}]}})
+    assert out == {"$each": [{"x": "a"}], "$as": "row", "$do": {"x": None}}
+
+
+def test_has_each_directive_ignores_literal_string_value():
+    # A directive is a "$each" KEY; the literal text "$each" inside a string value is not one.
+    assert has_each_directive({"note": "use $each to loop", "qty": "{{input.qty}}"}) is False
+    assert has_each_directive({"rows": {"$each": "{{x}}", "$do": {}}}) is True
+
+
+def test_build_body_literal_dollar_each_in_string_keeps_string_substitution():
+    # A valid-JSON template that merely MENTIONS "$each" in a value must not switch to structural
+    # rendering (which would change token type coercion). The quoted token stays a string "5".
+    body_template = json.dumps({"qty": "{{input.qty}}", "note": "$each is a keyword"})
+    body = _build_body({"body_template": body_template}, [], {"qty": 5}, {})
+    assert body == {"qty": "5", "note": "$each is a keyword"}
+
+
+def test_render_template_embedded_falsy_values_are_not_dropped():
+    # A falsy-but-real value embedded in a larger string must render literally (0 -> "0"), not be
+    # swallowed to "" - which is what a `_lookup(...) or ""` would wrongly do.
+    assert render_template("qty={{input.qty}}", {"input": {"qty": 0}}) == "qty=0"
+    assert render_template("on={{input.flag}}", {"input": {"flag": False}}) == "on=False"
+    assert render_template("x={{input.missing}}", {"input": {}}) == "x="
+
+
+def test_build_body_batches_multiple_rows_into_one_body():
+    body_template = json.dumps({
+        "orderId": "{{input.orderId}}",
+        "items": {
+            "$each": "{{input.rows}}",
+            "$as": "row",
+            "$do": {
+                "editedCol": "{{row.editedCol}}",
+                "editedValue": "{{row.editedValue}}",
+                "applyConversion": True,
+                "lineNums": ["{{row.lineNum}}"],
+                "enforcePolicy": False,
+            },
+        },
+    })
+    values = {
+        "orderId": "ORD-001",
+        "rows": [
+            {"editedCol": "unitCost", "editedValue": "96", "lineNum": 1},
+            {"editedCol": "unitCost", "editedValue": "33", "lineNum": 2},
+        ],
+    }
+    body = _build_body({"body_template": body_template}, [], values, {})
+    assert body["orderId"] == "ORD-001"
+    assert len(body["items"]) == 2
+    assert body["items"][0] == {
+        "editedCol": "unitCost", "editedValue": "96",
+        "applyConversion": True, "lineNums": [1], "enforcePolicy": False,
+    }
+    assert body["items"][1]["lineNums"] == [2]
+
+
+def test_build_body_passthrough_rows_and_injects_constants():
+    # Mirrors the live agent input: the model sends items items carrying an lineNums
+    # ARRAY and no constants; the template passes each row's fields through (preserving the array)
+    # and injects the fixed applyConversion/enforcePolicy server-side.
+    body_template = json.dumps({
+        "orderId": "{{input.orderId}}",
+        "items": {
+            "$each": "{{input.items}}",
+            "$as": "row",
+            "$do": {
+                "editedCol": "{{row.editedCol}}",
+                "editedValue": "{{row.editedValue}}",
+                "applyConversion": True,
+                "lineNums": "{{row.lineNums}}",
+                "enforcePolicy": False,
+            },
+        },
+    })
+    values = {
+        "orderId": "ORD-001",
+        "items": [
+            {"editedCol": "unitCost", "editedValue": "777", "lineNums": [1]},
+            {"editedCol": "unitCost", "editedValue": "777", "lineNums": [2]},
+        ],
+    }
+    body = _build_body({"body_template": body_template}, [], values, {})
+    assert body["orderId"] == "ORD-001"
+    assert body["items"] == [
+        {"editedCol": "unitCost", "editedValue": "777", "applyConversion": True,
+         "lineNums": [1], "enforcePolicy": False},
+        {"editedCol": "unitCost", "editedValue": "777", "applyConversion": True,
+         "lineNums": [2], "enforcePolicy": False},
+    ]
+
+
+def test_build_body_without_each_is_unchanged():
+    # Legacy unquoted-token template (not valid JSON as text) still uses string substitution and
+    # keeps producing a number for the bare {{token}}.
+    body_template = '{ "orderId": "{{input.orderId}}", "lineNums": [ {{input.lineNum}} ] }'
+    body = _build_body({"body_template": body_template}, [], {"orderId": "Q", "lineNum": 7}, {})
+    assert body == {"orderId": "Q", "lineNums": [7]}

apps/api/tests/test_channels.py

@@ -0,0 +1,23 @@
+"""Email channel parsing and reply construction."""
+
+from __future__ import annotations
+
+from forge.channels import email as email_ch
+
+# --- email parsing ---
+
+
+def test_email_parse_provider_dict():
+    p = email_ch.parse_inbound({"from": "Jane <jane@acme.com>", "subject": "Help", "text": "  my order is late  "})
+    assert p["from_addr"] == "jane@acme.com" and p["from_name"] == "Jane" and p["text"] == "my order is late"
+
+
+def test_email_parse_raw_mime():
+    raw = b"From: Bob <bob@x.com>\r\nSubject: Hi\r\nMessage-ID: <m1>\r\nContent-Type: text/plain\r\n\r\nHello body\r\n"
+    p = email_ch.parse_inbound(raw)
+    assert p["from_addr"] == "bob@x.com" and "Hello body" in p["text"] and p["message_id"] == "<m1>"
+
+
+def test_email_reply_threads_subject():
+    msg = email_ch.build_reply(to_addr="a@b.com", subject="Order", body="done", from_addr="bot@x.com", in_reply_to="<m1>")
+    assert msg["Subject"] == "Re: Order" and msg["In-Reply-To"] == "<m1>" and msg["To"] == "a@b.com"

apps/api/tests/test_channels_hitl.py

@@ -0,0 +1,464 @@
+"""Channels + HITL + run-cancel + semantic-cache wiring (audit a-i).
+
+Covers: the semantic_cache middleware (short-circuit on hit), handoff TOCTOU claim +
+delivery-status gating + chained-interrupt re-open + decision coercion, HITL timeout expiry,
+run cancel, email HTML fallback + threading, and pluggable webhook
+signatures. Uses the shared checkpointer pattern (InMemorySaver) so runs can be resumed by id.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import hmac
+import time
+from datetime import datetime, timedelta
+from types import SimpleNamespace
+
+from langchain_core.messages import AIMessage, HumanMessage
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.channels import email as email_ch
+from forge.db.base import SessionLocal
+from forge.models import HandoffRequest, Run, Workflow
+from forge.routers import hooks
+from forge.services.channels import ChannelService
+from forge.services.dispatch import dispatch_message
+from forge.services.handoff import HandoffService, coerce_to_allowed_decision
+from forge.services.runs import RunService, run_control
+
+# --- workflow fixtures --------------------------------------------------------------------
+
+_HITL_ONE = {
+    "id": "wf_one", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "hi1",
+    "on_error": {"message": "We'll follow up soon."},
+    "nodes": [
+        {"id": "hi1", "type": "human_input", "config": {"prompt": "Approve?", "allowed_decisions": ["approve", "reject"]}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "hi1", "target": "end"}],
+}
+
+_HITL_DEFAULT = {
+    "id": "wf_default", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "hi1",
+    "nodes": [
+        {"id": "hi1", "type": "human_input", "config": {
+            "prompt": "Approve?", "allowed_decisions": ["approve", "reject"],
+            "timeout_default": "reject",
+        }},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "hi1", "target": "end"}],
+}
+
+_HITL_CHAIN = {
+    "id": "wf_chain", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "hi1",
+    "nodes": [
+        {"id": "hi1", "type": "human_input", "config": {"prompt": "Approve step 1?", "allowed_decisions": ["approve", "reject"]}},
+        {"id": "hi2", "type": "human_input", "config": {"prompt": "Confirm step 2?", "ack_message": "One more step - confirming.", "allowed_decisions": ["approve", "reject"]}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "hi1", "target": "hi2"}, {"source": "hi2", "target": "end"}],
+}
+
+
+async def _mk_wf(executable, tenant, project) -> Workflow:
+    async with SessionLocal() as s:
+        wf = Workflow(tenant_id=tenant, project_id=project, name="W", executable=executable, status="active")
+        s.add(wf)
+        await s.commit()
+        await s.refresh(wf)
+        return wf
+
+
+# --- (a) semantic-cache middleware --------------------------------------------------------
+
+
+def test_semantic_cache_middleware_registered_and_builds():
+    from forge.engine.context import CompileContext
+    from forge.engine.middleware_compiler import (
+        MW_BUILDERS,
+        _SemanticCacheMiddleware,
+        build_middleware,
+    )
+
+    assert "semantic_cache" in MW_BUILDERS
+    mw = build_middleware([{"type": "semantic_cache", "config": {"threshold": 0.9, "ttl": 60}}],
+                          CompileContext(tenant_id="t", project_id="p"))
+    assert len(mw) == 1 and isinstance(mw[0], _SemanticCacheMiddleware)
+
+
+async def test_semantic_cache_middleware_short_circuits_on_hit(monkeypatch):
+    from langchain.agents.middleware.types import ModelResponse
+
+    from forge.engine.middleware_compiler import _SemanticCacheMiddleware
+
+    calls = {"handler": 0, "store": 0}
+    stored: dict[str, str] = {}
+
+    async def fake_lookup(session, t, p, q, *, scope, threshold, ttl):
+        return stored.get(q.strip().lower())
+
+    async def fake_store(session, t, p, q, a, *, scope):
+        calls["store"] += 1
+        stored[q.strip().lower()] = a
+
+    monkeypatch.setattr("forge.services.semantic_cache.SemanticCacheService.lookup", fake_lookup)
+    monkeypatch.setattr("forge.services.semantic_cache.SemanticCacheService.store", fake_store)
+
+    async def handler(_req):
+        calls["handler"] += 1
+        return ModelResponse(result=[AIMessage(content="We are open 9-5.")])
+
+    mw = _SemanticCacheMiddleware("t", "p", threshold=0.9, ttl=3600, scope="default", min_chars=3)
+    q = "what are your business hours?"
+
+    r1 = await mw.awrap_model_call(SimpleNamespace(messages=[HumanMessage(content=q)]), handler)
+    assert calls == {"handler": 1, "store": 1}
+    assert r1.result[0].content == "We are open 9-5."
+
+    # Same question again -> cache hit -> handler is NOT called, a cached AIMessage is returned.
+    r2 = await mw.awrap_model_call(SimpleNamespace(messages=[HumanMessage(content=q)]), handler)
+    assert calls["handler"] == 1
+    assert isinstance(r2, AIMessage) and "9-5" in r2.content
+
+
+async def test_semantic_cache_middleware_skips_mid_tool_loop(monkeypatch):
+    from langchain.agents.middleware.types import ModelResponse
+
+    from forge.engine.middleware_compiler import _SemanticCacheMiddleware
+
+    looked_up = []
+
+    async def fake_lookup(session, t, p, q, *, scope, threshold, ttl):
+        looked_up.append(q)
+        return None
+
+    monkeypatch.setattr("forge.services.semantic_cache.SemanticCacheService.lookup", fake_lookup)
+    monkeypatch.setattr("forge.services.semantic_cache.SemanticCacheService.store", lambda *a, **k: None)
+
+    async def handler(_req):
+        return ModelResponse(result=[AIMessage(content="x")])
+
+    mw = _SemanticCacheMiddleware("t", "p", threshold=0.9, ttl=3600, scope="default", min_chars=3)
+    # Last message is a tool result (not a fresh human question) -> no lookup / store.
+    from langchain_core.messages import ToolMessage
+    req = SimpleNamespace(messages=[HumanMessage(content="hi"), AIMessage(content=""), ToolMessage(content="42", tool_call_id="c1")])
+    await mw.awrap_model_call(req, handler)
+    assert looked_up == []
+
+
+async def test_semantic_cache_purge(monkeypatch):
+    from forge.services.semantic_cache import SemanticCacheService
+
+    t, p = "t_purge", "p_purge"
+    async with SessionLocal() as s:
+        await SemanticCacheService.store(s, t, p, "will this expire?", "yes")
+    async with SessionLocal() as s:
+        # ttl<=0 purges everything for the scope; returns the count purged.
+        purged = await SemanticCacheService.purge(s, t, p, ttl=0)
+    assert purged >= 1
+
+
+# --- (c) decision coercion ----------------------------------------------------------------
+
+
+def test_coerce_to_allowed_decision():
+    allowed = ["approve", "reject"]
+    assert coerce_to_allowed_decision("approve", allowed) == "approve"
+    assert coerce_to_allowed_decision("Yes, go ahead", allowed) == "approve"
+    assert coerce_to_allowed_decision("please approve this refund", allowed) == "approve"
+    assert coerce_to_allowed_decision("no, deny it", allowed) == "reject"
+    # Ambiguous free text fails safe to a negative decision when one is offered.
+    assert coerce_to_allowed_decision("hmm not sure yet", allowed) == "reject"
+    # No allowed list -> passthrough (unchanged behavior).
+    assert coerce_to_allowed_decision("whatever", []) == "whatever"
+
+
+# --- (b) handoff: TOCTOU claim, coercion end-to-end, chained re-open ----------------------
+
+
+async def test_handoff_reply_claims_row_toctou():
+    wf = await _mk_wf(_HITL_ONE, "t_toctou", "p_toctou")
+    rs = RunService(checkpointer=InMemorySaver())
+    result = await dispatch_message(rs, tenant_id="t_toctou", project_id="p_toctou", workflow_id=wf.id, text="please")
+    assert result["interrupted"] is True
+    async with SessionLocal() as s:
+        h = await HandoffService.create(
+            s, channel=None, tenant_id="t_toctou", project_id="p_toctou", workflow_id=wf.id,
+            run_id=result["run_id"], thread_id=result["thread_id"], customer="u",
+            customer_message="please", reason="approve?", reply_context={},
+        )
+        out1 = await HandoffService.reply(s, rs, handoff=h, agent_id="a1", message="approve")
+        assert out1["ok"] is True
+        # Second reply on the now-answered handoff is rejected by the atomic claim (no re-resume).
+        out2 = await HandoffService.reply(s, rs, handoff=h, agent_id="a2", message="approve")
+        assert out2["ok"] is False and out2["status"] == "answered"
+
+
+async def test_handoff_closed_filter_includes_all_terminal_outcomes():
+    tenant_id, project_id = "t_closed_queue", "p_closed_queue"
+    async with SessionLocal() as s:
+        open_item = await HandoffService.create(
+            s, channel=None, tenant_id=tenant_id, project_id=project_id, workflow_id=None,
+            run_id="run-open", thread_id=None, customer="Open", customer_message=None,
+            reason="waiting", reply_context={},
+        )
+        answered = await HandoffService.create(
+            s, channel=None, tenant_id=tenant_id, project_id=project_id, workflow_id=None,
+            run_id="run-answered", thread_id=None, customer="Answered", customer_message=None,
+            reason="done", reply_context={},
+        )
+        failed = await HandoffService.create(
+            s, channel=None, tenant_id=tenant_id, project_id=project_id, workflow_id=None,
+            run_id="run-failed", thread_id=None, customer="Failed", customer_message=None,
+            reason="delivery", reply_context={},
+        )
+        answered.status = "answered"
+        failed.status = "delivery_failed"
+        await s.commit()
+
+        open_rows = await HandoffService.list(s, tenant_id, project_id, status="open")
+        closed_rows = await HandoffService.list(s, tenant_id, project_id, status="closed")
+
+    assert [row.id for row in open_rows] == [open_item.id]
+    assert {row.id for row in closed_rows} == {answered.id, failed.id}
+
+
+async def test_handoff_reply_coerces_free_text_decision():
+    wf = await _mk_wf(_HITL_ONE, "t_coerce", "p_coerce")
+    rs = RunService(checkpointer=InMemorySaver())
+    result = await dispatch_message(rs, tenant_id="t_coerce", project_id="p_coerce", workflow_id=wf.id, text="please")
+    async with SessionLocal() as s:
+        h = await HandoffService.create(
+            s, channel=None, tenant_id="t_coerce", project_id="p_coerce", workflow_id=wf.id,
+            run_id=result["run_id"], thread_id=result["thread_id"], customer="u",
+            customer_message="please", reason="approve?",
+            reply_context={"_forge_hitl": {"allowed_decisions": ["approve", "reject"]}},
+        )
+        out = await HandoffService.reply(s, rs, handoff=h, agent_id="a1", message="yes, go ahead")
+    assert out["ok"] is True
+    # The free-text "yes, go ahead" was coerced to "approve" before resuming.
+    msgs = out["resume"]["messages"]
+    assert any("[human decision] approve" in str(m.get("content", "")) for m in msgs)
+
+
+async def test_handoff_reply_reopens_on_chained_interrupt():
+    wf = await _mk_wf(_HITL_CHAIN, "t_chain", "p_chain")
+    rs = RunService(checkpointer=InMemorySaver())
+    result = await dispatch_message(rs, tenant_id="t_chain", project_id="p_chain", workflow_id=wf.id, text="start")
+    assert result["interrupted"] is True
+    async with SessionLocal() as s:
+        ch = await ChannelService.create(s, "t_chain", "p_chain", type_="email", name="E", workflow_id=wf.id)
+        h = await HandoffService.create(
+            s, channel=ch, tenant_id="t_chain", project_id="p_chain", workflow_id=wf.id,
+            run_id=result["run_id"], thread_id=result["thread_id"], customer="u",
+            customer_message="start", reason="step1", reply_context={},
+        )
+        out = await HandoffService.reply(s, rs, handoff=h, agent_id="a1", message="approve")
+    assert out["ok"] is True and out["reinterrupted"] is True
+    assert out["new_handoff_id"]
+    # A fresh open handoff exists for the same run so the next step is actionable.
+    async with SessionLocal() as s:
+        fresh = await s.get(HandoffRequest, out["new_handoff_id"])
+        assert fresh is not None and fresh.status == "open" and fresh.run_id == result["run_id"]
+
+
+# --- (e) delivery status gates 'answered' -------------------------------------------------
+
+
+async def test_handoff_failed_send_not_marked_answered(monkeypatch):
+    async def _boom(*_a, **_k):
+        raise RuntimeError("smtp down")
+
+    monkeypatch.setattr(email_ch, "send_reply", _boom)
+    wf = await _mk_wf(_HITL_ONE, "t_fail", "p_fail")
+    rs = RunService(checkpointer=InMemorySaver())
+    result = await dispatch_message(rs, tenant_id="t_fail", project_id="p_fail", workflow_id=wf.id, text="please")
+    async with SessionLocal() as s:
+        ch = await ChannelService.create(s, "t_fail", "p_fail", type_="email", name="E", workflow_id=wf.id)
+        h = await HandoffService.create(
+            s, channel=ch, tenant_id="t_fail", project_id="p_fail", workflow_id=wf.id,
+            run_id=result["run_id"], thread_id=result["thread_id"], customer="c@x.com",
+            customer_message="please", reason="approve?", reply_context={"from_addr": "c@x.com"},
+        )
+        out = await HandoffService.reply(s, rs, handoff=h, agent_id="a1", message="Here you go")
+        assert out["ok"] is False and out["status"] == "delivery_failed"
+        refreshed = await s.get(HandoffRequest, h.id)
+        assert refreshed.status == "delivery_failed"  # NOT 'answered' on a failed send
+
+
+# --- (c) HITL timeout expiry --------------------------------------------------------------
+
+
+async def test_hitl_timeout_expires_interrupted_run(monkeypatch):
+    delivered: list[str] = []
+
+    async def _rec_deliver(channel, reply_ctx, text):
+        delivered.append(text)
+        return True
+
+    monkeypatch.setattr("forge.services.handoff._deliver", _rec_deliver)
+    wf = await _mk_wf(_HITL_ONE, "t_exp", "p_exp")
+    rs = RunService(checkpointer=InMemorySaver())
+    result = await dispatch_message(rs, tenant_id="t_exp", project_id="p_exp", workflow_id=wf.id, text="please")
+    run_id = result["run_id"]
+    async with SessionLocal() as s:
+        ch = await ChannelService.create(s, "t_exp", "p_exp", type_="email", name="E", workflow_id=wf.id)
+        h = await HandoffService.create(
+            s, channel=ch, tenant_id="t_exp", project_id="p_exp", workflow_id=wf.id,
+            run_id=run_id, thread_id=result["thread_id"], customer="u", customer_message="please",
+            reason="approve?", reply_context={"conversation_id": "c1"},
+        )
+        hid = h.id
+        # Backdate the pause so it's past the (tiny) timeout used below.
+        run = await s.get(Run, run_id)
+        run.ended_at = datetime.utcnow() - timedelta(hours=1)
+        await s.commit()
+
+    reaped = await rs.reap_stale_runs(hitl_timeout_s=1)
+    assert reaped >= 1
+    async with SessionLocal() as s:
+        run = await s.get(Run, run_id)
+        assert run.status == "error" and "HITL" in (run.error or "")
+        h = await s.get(HandoffRequest, hid)
+        assert h.status == "closed"
+    # The workflow's on_error fallback was pushed over the channel.
+    assert delivered and delivered[0] == "We'll follow up soon."
+
+
+async def test_hitl_timeout_resumes_with_configured_default():
+    wf = await _mk_wf(_HITL_DEFAULT, "t_exp_default", "p_exp_default")
+    rs = RunService(checkpointer=InMemorySaver())
+    result = await dispatch_message(
+        rs, tenant_id="t_exp_default", project_id="p_exp_default",
+        workflow_id=wf.id, text="please",
+    )
+    assert result["interrupted"] is True
+    async with SessionLocal() as s:
+        run = await s.get(Run, result["run_id"])
+        run.ended_at = datetime.utcnow() - timedelta(hours=1)
+        await s.commit()
+
+    assert await rs.reap_stale_runs(hitl_timeout_s=1) >= 1
+    async with SessionLocal() as s:
+        run = await s.get(Run, result["run_id"])
+        assert run.status == "done"
+        assert run.error is None
+
+
+# --- (h) run cancel -----------------------------------------------------------------------
+
+
+async def test_cancel_run_marks_canceled_and_is_idempotent():
+    wf = await _mk_wf(_HITL_ONE, "t_cancel", "p_cancel")
+    rs = RunService(checkpointer=InMemorySaver())
+    async with SessionLocal() as s:
+        run = await rs.create_run(s, tenant_id="t_cancel", project_id="p_cancel", workflow_id=wf.id, input={})
+        run_id = run.id
+    out = await rs.cancel_run(run_id=run_id, tenant_id="t_cancel", project_id="p_cancel")
+    assert out["ok"] is True and out["status"] == "canceled"
+    async with SessionLocal() as s:
+        assert (await s.get(Run, run_id)).status == "canceled"
+    # Cancelling an already-terminal run is a no-op.
+    out2 = await rs.cancel_run(run_id=run_id, tenant_id="t_cancel", project_id="p_cancel")
+    assert out2["ok"] is False and out2["status"] == "canceled"
+
+
+async def test_run_control_observes_cross_worker_database_cancel():
+    """A DB cancellation made outside the local registry hard-cancels the active task."""
+    wf = await _mk_wf(_HITL_ONE, "t_cancel_remote", "p_cancel_remote")
+    rs = RunService(checkpointer=InMemorySaver())
+    async with SessionLocal() as s:
+        run = await rs.create_run(
+            s, tenant_id="t_cancel_remote", project_id="p_cancel_remote",
+            workflow_id=wf.id, input={},
+        )
+        run_id = run.id
+
+    async def _active_run():
+        run_control.begin(run_id, "t_cancel_remote")
+        try:
+            await asyncio.Event().wait()
+        finally:
+            await run_control.end(run_id)
+
+    task = asyncio.create_task(_active_run())
+    await asyncio.sleep(0.05)
+    async with SessionLocal() as s:
+        run = await s.get(Run, run_id)
+        run.status = "canceled"
+        await s.commit()
+
+    canceled = False
+    try:
+        await asyncio.wait_for(task, timeout=2)
+    except asyncio.CancelledError:
+        canceled = True
+    assert canceled is True
+
+
+# --- (f) email HTML fallback + (g) threading ----------------------------------------------
+
+
+def test_email_html_only_fallback():
+    p = email_ch.parse_inbound({"from": "a@b.com", "subject": "Hi",
+                                "html": "<p>Hello <b>world</b></p><div>Line two</div>"})
+    assert "Hello world" in p["text"] and "Line two" in p["text"]
+
+
+def test_email_raw_html_only_fallback():
+    raw = (b"From: a@b.com\r\nSubject: Hi\r\nContent-Type: text/html\r\n\r\n"
+           b"<html><body><p>Body text here</p></body></html>\r\n")
+    p = email_ch.parse_inbound(raw)
+    assert "Body text here" in p["text"]
+
+
+def test_email_reply_preserves_references_and_sets_message_id():
+    msg = email_ch.build_reply(to_addr="a@b.com", subject="Order", body="ok", from_addr="bot@x.com",
+                               in_reply_to="<m2>", references="<m0> <m1>")
+    assert msg["In-Reply-To"] == "<m2>"
+    assert msg["References"].split() == ["<m0>", "<m1>", "<m2>"]
+    assert msg["Message-ID"]  # explicit id set on the outbound reply
+# --- (i) pluggable webhook signatures -----------------------------------------------------
+
+
+def test_webhook_stripe_signature():
+    secret = "whsec_test"
+    body = b'{"id":"evt_1"}'
+    ts = str(int(time.time()))
+    mac = hmac.new(secret.encode(), f"{ts}.".encode() + body, hashlib.sha256).hexdigest()
+    req = SimpleNamespace(headers={"Stripe-Signature": f"t={ts},v1={mac}"})
+    assert hooks._verify_stripe(secret, req, body, 300) is True
+    # Tampered body fails.
+    assert hooks._verify_stripe(secret, req, b'{"id":"evt_2"}', 300) is False
+    # Stale timestamp fails the replay window.
+    old = SimpleNamespace(headers={"Stripe-Signature": f"t=1,v1={mac}"})
+    assert hooks._verify_stripe(secret, old, body, 300) is False
+
+
+def test_webhook_slack_signature():
+    secret = "slack_secret"
+    body = b"token=abc&team_id=T1"
+    ts = str(int(time.time()))
+    mac = hmac.new(secret.encode(), b"v0:" + ts.encode() + b":" + body, hashlib.sha256).hexdigest()
+    req = SimpleNamespace(headers={"X-Slack-Request-Timestamp": ts, "X-Slack-Signature": f"v0={mac}"})
+    assert hooks._verify_slack(secret, req, body, 300) is True
+    bad = SimpleNamespace(headers={"X-Slack-Request-Timestamp": ts, "X-Slack-Signature": "v0=deadbeef"})
+    assert hooks._verify_slack(secret, bad, body, 300) is False
+
+
+def test_webhook_default_hmac_signature():
+    secret = "s3cr3t"
+    body = b"payload"
+    mac = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
+    req = SimpleNamespace(headers={"X-Hub-Signature-256": f"sha256={mac}"})
+    assert hooks._verify_hmac_sha256(secret, req, body) is True
+    req2 = SimpleNamespace(headers={"x-forge-signature": mac})
+    assert hooks._verify_hmac_sha256(secret, req2, body) is True

apps/api/tests/test_chunking_strategies.py

@@ -0,0 +1,120 @@
+"""Chunking strategies: pure splitter behavior + ingest wiring (strategy resolution
+and chunk_size/overlap sourced from the project's rag_defaults)."""
+
+from __future__ import annotations
+
+from forge.db.base import SessionLocal
+from forge.knowledge.splitter import chunk_text
+from forge.models import Project
+from forge.services.knowledge import KnowledgeService
+
+# --- pure splitter behavior ---
+
+
+def test_recursive_strategy_caps_chunk_size():
+    text = ("Sentence one. " * 200).strip()
+    chunks = chunk_text(text, strategy="recursive", chunk_size=300, overlap=50)
+    assert len(chunks) > 1
+    assert all(len(c) <= 360 for c in chunks)  # ~chunk_size + slack
+
+
+def test_section_strategy_splits_on_markdown_headers():
+    doc = (
+        "# Refunds\nRefunds go to the original method within 5-7 days.\n\n"
+        "# Shipping\nOrders ship in 2 days.\n\n"
+        "# Returns\nReturns accepted within 30 days."
+    )
+    chunks = chunk_text(doc, strategy="section", chunk_size=1000, overlap=100)
+    assert len(chunks) == 3
+    assert chunks[0].startswith("# Refunds") and "5-7 days" in chunks[0]
+    assert any(c.startswith("# Shipping") for c in chunks)
+
+
+def test_sentence_strategy_does_not_split_on_abbreviations():
+    prose = "Dr. Smith met Mr. Brown at 3 p.m. in the U.S. capital. They signed the deal."
+    chunks = chunk_text(prose, strategy="sentence", chunk_size=1000, overlap=0)
+    # Two real sentences fit in one chunk; abbreviations must not create extra splits.
+    assert len(chunks) == 1
+    assert "Dr. Smith" in chunks[0] and "U.S. capital" in chunks[0]
+
+
+def test_sentence_packs_to_chunk_size_with_overlap():
+    prose = (
+        "Alpha sentence here. Beta sentence here. Gamma sentence here. "
+        "Delta sentence here. Epsilon sentence here. "
+    ) * 4
+    chunks = chunk_text(prose, strategy="sentence", chunk_size=120, overlap=30)
+    assert len(chunks) > 1
+    assert all(len(c) <= 130 for c in chunks)
+
+
+def test_section_falls_back_to_recursive_without_headers():
+    text = ("No headers here at all. " * 80).strip()
+    chunks = chunk_text(text, strategy="section", chunk_size=200, overlap=40)
+    assert len(chunks) > 1  # no headers -> recursive fallback still splits
+
+
+def test_unknown_strategy_defaults_to_recursive():
+    text = ("word " * 300).strip()
+    assert chunk_text(text, strategy="nonsense", chunk_size=200) == chunk_text(
+        text, strategy="recursive", chunk_size=200
+    )
+
+
+def test_empty_text_yields_no_chunks():
+    assert chunk_text("", strategy="sentence") == []
+    assert chunk_text("   ", strategy="section") == []
+
+
+# --- ingest wiring ---
+
+
+async def _make_project(slug: str, rag_defaults: dict) -> str:
+    async with SessionLocal() as s:
+        proj = Project(tenant_id="t_chunk", name="Chunk", slug=slug, config={"rag_defaults": rag_defaults})
+        s.add(proj)
+        await s.commit()
+        await s.refresh(proj)
+        return proj.id
+
+
+async def test_ingest_uses_project_default_strategy(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma")
+    pid = await _make_project("chunk-default-section", {"chunking_strategy": "section"})
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(
+            s, "t_chunk", pid, kind="text", name="d", text="# A\nAlpha body.\n\n# B\nBeta body."
+        )
+        src = await KnowledgeService.ingest(s, src)
+    assert src.status == "ready"
+    assert src.chunking_strategy == "section"  # inherited project default, persisted on meta
+    assert src.chunks == 2  # one chunk per markdown section
+
+
+async def test_source_strategy_overrides_project_default(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma2")
+    pid = await _make_project("chunk-override", {"chunking_strategy": "section"})
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(
+            s, "t_chunk", pid, kind="text", name="d2",
+            text="One sentence. Two sentence. Three sentence.", chunking_strategy="sentence",
+        )
+        src = await KnowledgeService.ingest(s, src)
+    assert src.chunking_strategy == "sentence"  # per-source choice wins over project default
+
+
+async def test_ingest_reads_chunk_size_from_rag_defaults(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma3")
+    pid = await _make_project("chunk-size", {"chunk_size": 120, "chunk_overlap": 20, "chunking_strategy": "recursive"})
+    long_text = ("This is a sentence about refunds and shipping policies. " * 40).strip()
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_chunk", pid, kind="text", name="d3", text=long_text)
+        src = await KnowledgeService.ingest(s, src)
+    # A small project chunk_size yields many chunks (far fewer at the 1000 default).
+    assert src.chunks >= 10

apps/api/tests/test_clientip.py

@@ -0,0 +1,33 @@
+"""Client-IP trust rule (anti-spoofing): X-Forwarded-For is believed ONLY from a configured
+reverse proxy, so a directly-connected client cannot forge its IP for per-IP rate limits or
+audit logs. Covers both call sites (deps.client_ip and the audit middleware) via the shared
+helper they now both use."""
+
+from forge.util.clientip import resolve_client_ip
+
+
+def test_no_trusted_proxies_ignores_xff():
+    # Default (direct exposure): a client-supplied X-Forwarded-For must be ignored.
+    assert resolve_client_ip("203.0.113.9", "1.2.3.4", []) == "203.0.113.9"
+
+
+def test_trusted_proxy_uses_leftmost_xff():
+    # Behind a configured proxy, the original client is the left-most XFF entry.
+    assert resolve_client_ip("10.0.0.5", "1.2.3.4, 10.0.0.5", ["10.0.0.5"]) == "1.2.3.4"
+
+
+def test_untrusted_peer_ignores_xff():
+    # A peer that is NOT a configured proxy cannot get its XFF believed.
+    assert resolve_client_ip("203.0.113.9", "1.2.3.4", ["10.0.0.5"]) == "203.0.113.9"
+
+
+def test_wildcard_trusts_any_peer():
+    assert resolve_client_ip("203.0.113.9", "1.2.3.4", ["*"]) == "1.2.3.4"
+
+
+def test_no_xff_returns_peer():
+    assert resolve_client_ip("203.0.113.9", None, ["*"]) == "203.0.113.9"
+
+
+def test_no_client_and_no_xff_is_none():
+    assert resolve_client_ip(None, None, []) is None

apps/api/tests/test_config_env_lists.py

@@ -0,0 +1,43 @@
+"""List-of-strings settings parse leniently from env — JSON array, comma-separated, or blank —
+so a stray bracket/space (e.g. from a `${VAR:-[]}` compose interpolation) can't crash startup.
+
+Regression: the api container failed to boot with `FORGE_EGRESS_ALLOW_PRIVATE_HOSTS='['`
+(pydantic-settings tried json.loads('[') -> JSONDecodeError -> SettingsError).
+"""
+
+from forge.config import Settings, _as_str_list
+
+
+def test_blank_and_mangled_are_empty():
+    for v in ("", "   ", "[", "[]", None):
+        assert _as_str_list(v) == []
+
+
+def test_json_array():
+    assert _as_str_list('["localhost","127.0.0.1"]') == ["localhost", "127.0.0.1"]
+
+
+def test_comma_separated():
+    assert _as_str_list("localhost, 127.0.0.1 , host.docker.internal") == [
+        "localhost", "127.0.0.1", "host.docker.internal",
+    ]
+
+
+def test_unquoted_or_mangled_bracketed():
+    assert _as_str_list("[localhost,127.0.0.1]") == ["localhost", "127.0.0.1"]
+
+
+def test_passthrough_existing_list():
+    assert _as_str_list(["a", "b"]) == ["a", "b"]
+
+
+def test_settings_boots_with_the_crashing_value(monkeypatch):
+    monkeypatch.setenv("FORGE_EGRESS_ALLOW_PRIVATE_HOSTS", "[")  # exact value that crashed the container
+    s = Settings()
+    assert s.egress_allow_private_hosts == []
+
+
+def test_settings_parses_real_egress_list(monkeypatch):
+    monkeypatch.setenv("FORGE_EGRESS_ALLOW_PRIVATE_HOSTS", "localhost,127.0.0.1")
+    s = Settings()
+    assert s.egress_allow_private_hosts == ["localhost", "127.0.0.1"]

apps/api/tests/test_connected_credentials.py

@@ -0,0 +1,63 @@
+"""Per-user connected credentials: the AuthResolver picks each end user's own stored OAuth bundle,
+so a tool acts as the authenticated user downstream without the MCP token being passed through."""
+
+from __future__ import annotations
+
+import time
+
+import pytest
+
+from forge.auth_providers.resolver import AuthResolver
+from forge.db.base import SessionLocal
+from forge.services.auth_providers import AuthProviderService
+
+
+async def _provider(tenant: str, project: str) -> str:
+    async with SessionLocal() as s:
+        ap = await AuthProviderService.create(
+            s, tenant, project, name="portal", kind="oauth2_authorization_code",
+            config={
+                "per_user_context_keys": ["end_user_id"],
+                "token_url": "https://example.com/token",
+                "header_name": "Authorization",
+                "prefix": "Bearer ",
+            },
+        )
+        return ap.id
+
+
+async def test_per_user_connected_credentials_resolve_per_end_user():
+    tenant, project = "t_conn", "p_conn"
+    ap_id = await _provider(tenant, project)
+    async with SessionLocal() as s:
+        ap = await AuthProviderService.get(s, tenant, ap_id)
+        await AuthProviderService.set_user_connection(
+            s, tenant, project, ap, "user-A", bundle={"access_token": "tok-A", "expires_at": time.time() + 3600})
+        await AuthProviderService.set_user_connection(
+            s, tenant, project, ap, "user-B", bundle={"access_token": "tok-B", "expires_at": time.time() + 3600})
+
+    resolver = AuthResolver()
+    ra = await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id, context={"end_user_id": "user-A"})
+    assert ra.headers["Authorization"] == "Bearer tok-A"
+    rb = await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id, context={"end_user_id": "user-B"})
+    assert rb.headers["Authorization"] == "Bearer tok-B"
+
+    # a user who never connected their account cannot authenticate (no bundle -> "not connected")
+    with pytest.raises(KeyError):
+        await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id, context={"end_user_id": "user-C"})
+
+
+async def test_connection_status_and_clear():
+    tenant, project = "t_conn2", "p_conn2"
+    ap_id = await _provider(tenant, project)
+    async with SessionLocal() as s:
+        ap = await AuthProviderService.get(s, tenant, ap_id)
+        assert (await AuthProviderService.get_user_connection(tenant, project, ap, "u1"))["connected"] is False
+        await AuthProviderService.set_user_connection(s, tenant, project, ap, "u1", bundle={"access_token": "t", "expires_at": time.time() + 3600})
+    async with SessionLocal() as s:
+        ap = await AuthProviderService.get(s, tenant, ap_id)
+        assert (await AuthProviderService.get_user_connection(tenant, project, ap, "u1"))["connected"] is True
+        await AuthProviderService.clear_user_connection(s, tenant, project, ap, "u1")
+    async with SessionLocal() as s:
+        ap = await AuthProviderService.get(s, tenant, ap_id)
+        assert (await AuthProviderService.get_user_connection(tenant, project, ap, "u1"))["connected"] is False

apps/api/tests/test_conversations.py

@@ -0,0 +1,237 @@
+"""Conversation-centric Traces view: turns grouped by session/user, filters, capture, purge."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timedelta
+
+import httpx
+from langgraph.checkpoint.memory import InMemorySaver
+from sqlalchemy import select
+
+from forge.db.base import SessionLocal
+from forge.main import create_app
+from forge.models import Span, Trace
+from forge.services.conversations import ConversationService
+
+TENANT = "t_conv"
+
+
+def _pid() -> str:
+    """A fresh project id per test — the suite shares one DB file across tests, so isolate
+    each test's rows under its own project to keep assertions on the full list stable."""
+    return f"p_{uuid.uuid4().hex[:10]}"
+
+
+async def _add_trace(*, project, thread_id, actor, source, status="done", user="hi", ai="hello",
+                     tokens=10, cost=0.001, started=None, error=None, run_id=None):
+    async with SessionLocal() as s:
+        t = Trace(
+            tenant_id=TENANT, project_id=project, workflow_id="wf1",
+            run_id=run_id or str(uuid.uuid4()), thread_id=thread_id, name="run", status=status,
+            started_at=started or datetime.utcnow(), ended_at=started or datetime.utcnow(),
+            latency_ms=5, total_tokens=tokens, total_cost_usd=cost,
+            source=source, actor=actor, end_user_id=None,
+            user_message=user, ai_response=ai, error=error,
+        )
+        s.add(t)
+        await s.commit()
+        return t.id
+
+
+# --- grouping + summary --------------------------------------------------------
+
+async def test_turns_group_into_one_conversation_per_thread():
+    pid = _pid()
+    await _add_trace(project=pid, thread_id="th1", actor="Alice", source="api", user="q1", ai="a1",
+                     started=datetime.utcnow() - timedelta(minutes=5))
+    await _add_trace(project=pid, thread_id="th1", actor="Alice", source="api", user="q2", ai="a2",
+                     started=datetime.utcnow() - timedelta(minutes=4))
+    await _add_trace(project=pid, thread_id="th2", actor="System", source="playground", user="test", ai="ok")
+
+    async with SessionLocal() as s:
+        convos = await ConversationService.list(s, TENANT, pid)
+    by_thread = {c.thread_id: c for c in convos}
+    assert set(by_thread) == {"th1", "th2"}
+    assert by_thread["th1"].turns == 2 and by_thread["th1"].actor == "Alice"
+    assert by_thread["th1"].total_tokens == 20
+    assert by_thread["th1"].preview == "q1"          # earliest turn's user message
+    assert by_thread["th2"].actor == "System" and by_thread["th2"].source == "playground"
+
+
+async def test_pause_and_resume_of_one_run_count_as_one_turn():
+    # A HITL pause writes an `interrupted` Trace and the resume writes a `done` Trace, both under
+    # the SAME run_id (and same user_message, since run.input is unchanged). They must fold into
+    # ONE turn, not two - matching the run-grouped transcript in the Traces UI.
+    pid = _pid()
+    rid = str(uuid.uuid4())
+    await _add_trace(project=pid, thread_id="thHITL", actor="System", source="playground",
+                     status="interrupted", user="approve this", ai=None, run_id=rid,
+                     started=datetime.utcnow() - timedelta(minutes=2))
+    await _add_trace(project=pid, thread_id="thHITL", actor="System", source="playground",
+                     status="done", user="approve this", ai="done!", run_id=rid,
+                     started=datetime.utcnow() - timedelta(minutes=1))
+    async with SessionLocal() as s:
+        convos = await ConversationService.list(s, TENANT, pid)
+        turns = await ConversationService.turns(s, TENANT, pid, "thHITL")
+    conv = next(c for c in convos if c.thread_id == "thHITL")
+    assert conv.turns == 1, "pause + resume of one run is a single turn"
+    assert conv.status != "error"  # an interrupt is not a failure
+    assert len(turns) == 2  # both raw Trace segments are still returned; the UI groups by run_id
+
+
+async def test_conversation_status_is_error_if_any_turn_errored():
+    pid = _pid()
+    await _add_trace(project=pid, thread_id="thE", actor="Bob", source="embed", status="done")
+    await _add_trace(project=pid, thread_id="thE", actor="Bob", source="embed", status="error", error="boom")
+    async with SessionLocal() as s:
+        convos = await ConversationService.list(s, TENANT, pid)
+        errs = await ConversationService.list(s, TENANT, pid, status="error")
+        oks = await ConversationService.list(s, TENANT, pid, status="success")
+    assert next(c for c in convos if c.thread_id == "thE").status == "error"
+    assert [c.thread_id for c in errs] == ["thE"]
+    assert "thE" not in [c.thread_id for c in oks]
+
+
+async def test_filter_by_actor_and_source():
+    pid = _pid()
+    await _add_trace(project=pid, thread_id="thA", actor="Alice", source="api")
+    await _add_trace(project=pid, thread_id="thS", actor="System", source="playground")
+    async with SessionLocal() as s:
+        alice = await ConversationService.list(s, TENANT, pid, actor="Alice")
+        system = await ConversationService.list(s, TENANT, pid, source="playground")
+    assert [c.thread_id for c in alice] == ["thA"]
+    assert [c.thread_id for c in system] == ["thS"]
+
+
+async def test_search_matches_any_turn_and_keeps_the_full_conversation():
+    pid = _pid()
+    await _add_trace(project=pid, thread_id="match-user", actor="Alice", source="api",
+                     user="Find the quarterly invoice", ai="Here it is",
+                     started=datetime.utcnow() - timedelta(minutes=3))
+    await _add_trace(project=pid, thread_id="match-user", actor="Alice", source="api",
+                     user="Thanks", ai="You're welcome",
+                     started=datetime.utcnow() - timedelta(minutes=2))
+    await _add_trace(project=pid, thread_id="match-ai", actor="Bob", source="playground",
+                     user="What was the result?", ai="The needle is in this answer")
+    await _add_trace(project=pid, thread_id="miss", actor="Carol", source="api",
+                     user="Unrelated", ai="Nothing to see")
+
+    async with SessionLocal() as s:
+        user_match = await ConversationService.list(s, TENANT, pid, search="QUARTERLY")
+        ai_match = await ConversationService.list(s, TENANT, pid, search="needle")
+
+    assert [c.thread_id for c in user_match] == ["match-user"]
+    assert user_match[0].turns == 2
+    assert user_match[0].total_tokens == 20
+    assert [c.thread_id for c in ai_match] == ["match-ai"]
+
+
+async def test_turns_endpoint_returns_transcript_in_order():
+    pid = _pid()
+    await _add_trace(project=pid, thread_id="thT", actor="Al", source="api", user="first", ai="r1",
+                     started=datetime.utcnow() - timedelta(minutes=2))
+    await _add_trace(project=pid, thread_id="thT", actor="Al", source="api", user="second", ai="r2",
+                     started=datetime.utcnow() - timedelta(minutes=1))
+    async with SessionLocal() as s:
+        turns = await ConversationService.turns(s, TENANT, pid, "thT")
+    assert [t.user_message for t in turns] == ["first", "second"]
+    assert [t.ai_response for t in turns] == ["r1", "r2"]
+
+
+async def test_facets_lists_distinct_actors_and_sources():
+    pid = _pid()
+    await _add_trace(project=pid, thread_id="f1", actor="Alice", source="api")
+    await _add_trace(project=pid, thread_id="f2", actor="System", source="playground")
+    async with SessionLocal() as s:
+        facets = await ConversationService.facets(s, TENANT, pid)
+    assert "Alice" in facets["actors"] and "System" in facets["actors"]
+    assert "api" in facets["sources"] and "playground" in facets["sources"]
+
+
+async def test_purge_deletes_old_traces_and_spans():
+    pid = _pid()
+    old_id = await _add_trace(project=pid, thread_id="old", actor="X", source="api",
+                              started=datetime.utcnow() - timedelta(days=40))
+    await _add_trace(project=pid, thread_id="new", actor="X", source="api", started=datetime.utcnow())
+    async with SessionLocal() as s:
+        s.add(Span(tenant_id=TENANT, trace_id=old_id, name="tool", kind="tool"))
+        await s.commit()
+
+    async with SessionLocal() as s:
+        removed = await ConversationService.purge_older_than(s, TENANT, pid, days=30)
+    assert removed == 1
+    async with SessionLocal() as s:
+        convos = await ConversationService.list(s, TENANT, pid)
+        remaining_spans = (await s.execute(select(Span).where(Span.trace_id == old_id))).scalars().all()
+    assert [c.thread_id for c in convos] == ["new"]
+    assert remaining_spans == []  # the old trace's spans were purged too
+
+
+# --- end-to-end capture through a real run ------------------------------------
+
+_WF = {
+    "id": "wf_conv", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "agent",
+    "nodes": [
+        {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": "fake:Hi there!", "tools": []}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "agent", "target": "end"}],
+}
+
+
+def _client() -> httpx.AsyncClient:
+    app = create_app()
+    app.state.checkpointer = InMemorySaver()
+    return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test")
+
+
+async def test_run_captures_source_and_transcript_end_to_end():
+    """A run through the project /run endpoint (source='api') must land as a conversation with
+    the user message + AI response captured, so the Traces view can show it."""
+    async with _client() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": f"u{uuid.uuid4().hex[:8]}@x.com", "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        pid = (await c.post("/v1/projects", json={"name": "Conv"}, headers=h)).json()["id"]
+        wid = (await c.post(f"/v1/projects/{pid}/workflows", json={"name": "Chat", "executable": _WF}, headers=h)).json()["id"]
+        await c.patch(f"/v1/projects/{pid}", json={"config": {"api_workflow_id": wid}}, headers=h)
+
+        r = await c.post(f"/v1/projects/{pid}/run",
+                         json={"input": {"messages": [{"role": "user", "content": "what is forge"}]}, "stream": False},
+                         headers=h)
+        assert r.status_code == 200, r.text
+
+        convos = (await c.get(f"/v1/projects/{pid}/conversations", headers=h)).json()
+        assert len(convos) == 1, convos
+        conv = convos[0]
+        assert conv["source"] == "api"
+        assert conv["actor"] == "Unknown user"          # /run with no end_user identity
+        assert conv["preview"] == "what is forge"
+
+        detail = (await c.get(f"/v1/projects/{pid}/conversations/{conv['thread_id']}", headers=h)).json()
+        turn = detail["turns"][0]
+        assert turn["user_message"] == "what is forge"
+        assert "Hi there!" in (turn["ai_response"] or "")
+        # the AI-response click drills into the existing span waterfall by trace id
+        assert turn["trace_id"]
+        spans = (await c.get(f"/v1/projects/{pid}/traces/{turn['trace_id']}", headers=h)).json()
+        assert "spans" in spans
+
+        # The Traces "Run again" action must replay only through the original workflow.
+        other_wid = (await c.post(
+            f"/v1/projects/{pid}/workflows", json={"name": "Other", "executable": _WF}, headers=h,
+        )).json()["id"]
+        cross_workflow = await c.post(
+            f"/v1/projects/{pid}/workflows/{other_wid}/runs/{turn['run_id']}/rerun", headers=h,
+        )
+        assert cross_workflow.status_code == 404
+
+        replay = await c.post(
+            f"/v1/projects/{pid}/workflows/{wid}/runs/{turn['run_id']}/rerun", headers=h,
+        )
+        assert replay.status_code == 201, replay.text
+        replayed = replay.json()
+        assert replayed["id"] != turn["run_id"]
+        assert replayed["thread_id"] != conv["thread_id"]

apps/api/tests/test_cost.py

@@ -0,0 +1,25 @@
+"""Cost levers: accurate token counting + cheap-model defaults."""
+
+from __future__ import annotations
+
+from forge.engine.models import cheap_model_for_credentials, default_model_for_credentials
+from forge.tools.projection import count_tokens, estimate_tokens
+
+
+def test_count_tokens_nonzero_and_aliased():
+    n = count_tokens("hello world, this is a token counting test")
+    assert n > 0
+    assert estimate_tokens("hello world, this is a token counting test") == n  # alias
+
+
+def test_count_tokens_handles_objects_and_none():
+    assert count_tokens(None) == 0
+    assert count_tokens({"a": [1, 2, 3], "b": "text"}) > 0
+
+
+def test_cheap_model_prefers_provider_nano_tier():
+    assert cheap_model_for_credentials({"openai": "k"}) == "openai:gpt-4.1-nano"
+    assert cheap_model_for_credentials({"anthropic": "k"}) == "anthropic:claude-haiku-4-5"
+    assert cheap_model_for_credentials({}) is None
+    # the cheap model is distinct from the default (frontier-ish) model
+    assert cheap_model_for_credentials({"openai": "k"}) != default_model_for_credentials({"openai": "k"})

apps/api/tests/test_crawl.py

@@ -0,0 +1,33 @@
+"""Website crawl link extraction + source re-ingest."""
+
+from __future__ import annotations
+
+from forge.db.base import SessionLocal
+from forge.knowledge.crawl import extract_links
+from forge.services.knowledge import KnowledgeService
+
+
+def test_extract_links_same_domain_only():
+    html = """
+      <a href="/about">About</a>
+      <a href="https://acme.test/pricing">Pricing</a>
+      <a href="https://other.test/x">External</a>
+      <a href="/about#team">Fragment dup</a>
+      <a href="mailto:hi@acme.test">Mail</a>
+    """
+    links = extract_links(html, "https://acme.test/")
+    assert "https://acme.test/about" in links
+    assert "https://acme.test/pricing" in links
+    assert "https://other.test/x" not in links  # external domain excluded
+    assert "mailto:hi@acme.test" not in links
+    assert links.count("https://acme.test/about") == 1  # fragment deduped
+
+
+async def test_reingest_text_source_reembeds():
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_re", "p_re", kind="text", name="doc",
+                                                   text="Forge supports website crawling and re-ingest now.")
+        src = await KnowledgeService.ingest(s, src)
+        assert src.status == "ready" and src.chunks >= 1
+        src2 = await KnowledgeService.reingest(s, src)
+    assert src2.status == "ready" and src2.chunks >= 1

apps/api/tests/test_default_middleware.py

@@ -0,0 +1,57 @@
+"""Project-level default middleware (the Guardrails & Egress screen's `config.default_middleware`)
+is prepended to EVERY agent's stack, ahead of the agent's own middleware. This locks the
+"one enforcement point on every agent" guarantee the settings UI relies on."""
+
+from __future__ import annotations
+
+from langchain.agents.middleware import PIIMiddleware
+
+from forge.engine.context import CompileContext
+from forge.nodes.agent_node import _common_kwargs
+
+# A PII guardrail entry exactly as the Guardrails settings screen compiles it.
+_PII_DEFAULT = {
+    "type": "pii",
+    "config": {"_managed": True, "pii_type": "email", "strategy": "redact",
+               "apply_to_input": True, "apply_to_output": True},
+}
+
+
+def _ctx(**kw) -> CompileContext:
+    # `fake` model keeps resolve_model offline (no provider key / network).
+    return CompileContext(tenant_id="t", project_id="p", default_model="fake", **kw)
+
+
+def test_project_default_middleware_injected_when_agent_has_none():
+    ctx = _ctx(project_default_mw=[_PII_DEFAULT])
+    common = _common_kwargs({"model": "fake"}, ctx)
+    assert any(isinstance(m, PIIMiddleware) for m in common["middleware"]), "project default PII guardrail must reach an agent with no middleware of its own"
+
+
+def test_project_default_middleware_prepended_before_agent_middleware():
+    ctx = _ctx(project_default_mw=[_PII_DEFAULT])
+    config = {"model": "fake", "middleware": [{"type": "guardrail_regex", "config": {"patterns": ["secret"]}}]}
+    built = _common_kwargs(config, ctx)["middleware"]
+    names = [type(m).__name__ for m in built]
+    # Default runs first (outermost); the agent's own middleware follows.
+    assert names[0] == "PIIMiddleware"
+    assert "_GuardrailRegexMiddleware" in names
+
+
+def test_no_default_middleware_is_a_no_op():
+    # Empty policy adds nothing — an agent with no middleware compiles to an empty stack
+    # (so leaving the guardrails screen untouched has zero runtime effect / cost).
+    built = _common_kwargs({"model": "fake"}, _ctx(project_default_mw=[]))["middleware"]
+    assert built == []
+
+
+def test_custom_pattern_pii_guardrail_compiles():
+    # A "Custom pattern" row compiles to a `pii` entry with a regex `detector` and a custom
+    # `pii_type` label — the shape the Guardrails screen emits for e.g. phone / national-ID.
+    ctx = _ctx(project_default_mw=[{
+        "type": "pii",
+        "config": {"_managed": True, "pii_type": "phone", "detector": r"\d{3}-\d{3}-\d{4}", "strategy": "redact"},
+    }])
+    built = _common_kwargs({"model": "fake"}, ctx)["middleware"]
+    m = next((x for x in built if isinstance(x, PIIMiddleware)), None)
+    assert m is not None and m.pii_type == "phone"

apps/api/tests/test_e2e_regressions.py

@@ -0,0 +1,68 @@
+"""Regressions found by the full end-to-end application test (don't let them come back)."""
+
+from __future__ import annotations
+
+from forge.db.base import SessionLocal
+from forge.models import Trigger
+from forge.services.tools import ToolService
+from forge.services.validation import validate_workflow
+from forge.services.workflows import WorkflowService
+
+
+def test_trigger_and_flow_nodes_validate():
+    """node_schema_ref must resolve trigger/flow schemas whose file name != node type
+    (e.g. webhook_in -> forge/nodes/trigger_webhook). Previously: Unresolvable schema."""
+    wf = {
+        "id": "w", "version": 1,
+        "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+        "entry_node": "hook",
+        "nodes": [
+            {"id": "hook", "type": "webhook_in", "config": {"message_path": "text"}},
+            {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": "fake:hi"}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [{"source": "hook", "target": "agent"}, {"source": "agent", "target": "end"}],
+    }
+    res = validate_workflow(wf)
+    assert res.valid, res.errors
+
+
+async def test_tool_test_supports_code_and_sql(tmp_path, monkeypatch):
+    import sqlite3
+
+    from forge.config import settings
+    # Code tools are OFF by default (unsandboxed RestrictedPython is not an isolation
+    # boundary - audit S5); this test exercises the feature, so opt in explicitly.
+    monkeypatch.setattr(settings, "enable_code_tools", True)
+    code = {"name": "u", "kind": "code", "description": "upper", "language": "python",
+            "source": "def main(s):\n    return s.upper()", "args_schema": {"properties": {"s": {"type": "string"}}, "required": ["s"]}}
+    r = await ToolService.test("t", "p", code, {"s": "hi"})
+    assert r["ok"] and r["projected"] == "HI"
+
+    db = tmp_path / "t.db"
+    con = sqlite3.connect(db)
+    con.executescript("CREATE TABLE t(id int, name text); INSERT INTO t VALUES (1,'x');")
+    con.commit()
+    con.close()
+    sql = {"name": "q", "kind": "sql", "description": "q", "connection_url": f"sqlite+aiosqlite:///{db.as_posix()}",
+           "query": "SELECT name FROM t WHERE id = :id", "args_schema": {"properties": {"id": {"type": "integer"}}}}
+    r = await ToolService.test("t", "p", sql, {"id": 1})
+    assert r["ok"] and r["projected"] == [{"name": "x"}]
+
+
+async def test_workflow_create_syncs_triggers():
+    """Creating a workflow whose executable has a webhook_in must register a Trigger row."""
+    ex = {
+        "id": "w2", "version": 1,
+        "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+        "entry_node": "hook",
+        "nodes": [
+            {"id": "hook", "type": "webhook_in", "config": {}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [{"source": "hook", "target": "end"}],
+    }
+    async with SessionLocal() as s:
+        wf = await WorkflowService.create(s, "t_sync", "p_sync", name="Hooked", executable=ex)
+        rows = (await s.execute(Trigger.__table__.select().where(Trigger.workflow_id == wf.id))).fetchall()
+    assert len(rows) == 1 and rows[0].kind == "webhook_in" and rows[0].key

apps/api/tests/test_embedding_spans.py

@@ -0,0 +1,33 @@
+"""Embedding spans (item 9): embed calls run through the Embedder are timed + priced as
+`kind="embedding"` spans on the active run tracer, so RAG/memory embedding latency and cost
+show up in traces. Wrapping at the Embedder level covers every call site (knowledge/*,
+nodes/rag.py, services/memory.py) at once.
+"""
+
+from __future__ import annotations
+
+from forge.knowledge.embeddings import _est_tokens, resolve_embedder
+from forge.tracing.tracer import ForgeTracer, embedding_span
+
+
+def test_est_tokens():
+    assert _est_tokens(["aaaa"]) == 1  # 4 chars / 4
+    assert _est_tokens(["", None]) == 0  # handles empties
+    assert _est_tokens(["a" * 40, "b" * 40]) == 20
+
+
+def test_embedding_span_noop_off_run():
+    # No active tracer bound -> the context manager is a harmless no-op.
+    with embedding_span("some:model", n_texts=3):
+        pass
+
+
+async def test_embed_call_records_embedding_span():
+    tr = ForgeTracer()  # __init__ binds this as the active tracer for this async context
+    embedder = resolve_embedder(None)  # default local fastembed
+    await embedder.aembed_query("hello world")
+    spans = [s for s in tr.ordered() if s.kind == "embedding"]
+    assert spans, "an embed call should record an embedding span on the active tracer"
+    assert spans[0].model == embedder.name
+    assert spans[0].attributes.get("n_texts") == 1
+    assert spans[0].end is not None and spans[0].end >= spans[0].start  # latency captured

apps/api/tests/test_engine.py

@@ -0,0 +1,197 @@
+"""End-to-end validation of the Forge engine, fully offline (fake model).
+
+Proves: state TypedDict + reducers, the node registry, the workflow compiler,
+router expression routing, middleware attachment, and an actual graph run.
+"""
+
+from __future__ import annotations
+
+import pytest
+from langchain_core.messages import AIMessage, HumanMessage
+from langgraph.checkpoint.memory import InMemorySaver
+from langgraph.graph import START, StateGraph
+from langgraph.types import Command
+
+from forge.engine.compiler import compile_workflow
+from forge.engine.context import CompileContext
+from forge.engine.expressions import ExpressionError, eval_expression
+from forge.engine.state import build_state_typeddict
+from forge.tools.projection import estimate_tokens, project_response
+
+
+def _ctx() -> CompileContext:
+    return CompileContext(tenant_id="t1", project_id="p1", checkpointer=InMemorySaver())
+
+
+def _wf() -> dict:
+    return {
+        "id": "wf_test",
+        "version": 1,
+        "state": {
+            "messages": {"type": "list[message]", "reducer": "add_messages"},
+            "intent": {"type": "str", "reducer": "last"},
+        },
+        "entry_node": "start",
+        "nodes": [
+            {"id": "start", "type": "start", "config": {}},
+            {
+                "id": "route",
+                "type": "router",
+                "config": {
+                    "expression": "intent",
+                    "cases": {"billing": "billing_agent", "tech": "tech_agent"},
+                    "default": "billing_agent",
+                },
+            },
+            {
+                "id": "billing_agent",
+                "type": "agent",
+                "config": {
+                    "flavor": "agent",
+                    "model": "fake:Billing handled.",
+                    "system_prompt": "You are the billing agent.",
+                    "middleware": [
+                        {"type": "model_call_limit", "config": {"run_limit": 3}},
+                        {"type": "summarization", "config": {"trigger": ["tokens", 4000]}},
+                    ],
+                },
+            },
+            {
+                "id": "tech_agent",
+                "type": "agent",
+                "config": {"flavor": "agent", "model": "fake:Tech handled."},
+            },
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [
+            {"source": "start", "target": "route"},
+            {"source": "billing_agent", "target": "end"},
+            {"source": "tech_agent", "target": "end"},
+        ],
+    }
+
+
+# --- state builder --------------------------------------------------------
+
+
+def test_build_state_typeddict_injects_messages_and_reducers():
+    State = build_state_typeddict({"findings": {"type": "list[str]", "reducer": "add"}})
+    ann = State.__annotations__
+    assert "messages" in ann  # auto-injected
+    assert "findings" in ann
+
+
+async def test_add_reducer_accumulates_across_nodes():
+    State = build_state_typeddict(
+        {"items": {"type": "list[str]", "reducer": "add"}, "intent": {"type": "str", "reducer": "last"}}
+    )
+    g = StateGraph(State)
+    g.add_node("a", lambda s: {"items": ["a"], "intent": "x"})
+    g.add_node("b", lambda s: {"items": ["b"], "intent": "y"})
+    g.add_edge(START, "a")
+    g.add_edge("a", "b")
+    g.set_finish_point("b")
+    out = await g.compile().ainvoke({})
+    assert out["items"] == ["a", "b"]  # accumulated via operator.add
+    assert out["intent"] == "y"  # overwritten via "last"
+
+
+# --- expressions ----------------------------------------------------------
+
+
+def test_expression_sandbox_evaluates_and_blocks_imports():
+    assert eval_expression("intent == 'billing'", {"intent": "billing"}) is True
+    assert eval_expression("len(messages) > 1", {"messages": [1, 2, 3]}) is True
+    with pytest.raises(ExpressionError):
+        eval_expression("__import__('os').system('echo hi')", {})
+
+
+# --- projection (token lever) --------------------------------------------
+
+
+def test_projection_jmespath_then_fields_then_full():
+    raw = {"data": {"totals": {"subtotal": 90, "grand_total": 99}, "line_items": [1, 2, 3, 4]}}
+    jm = project_response(raw, {"projection_jmespath": "data.totals.{sub: subtotal, total: grand_total}"})
+    assert jm == {"sub": 90, "total": 99}
+
+    fld = project_response(
+        raw,
+        {"fields": [
+            {"path": "data.totals.subtotal", "include_in_llm": True},
+            {"path": "data.line_items", "include_in_llm": False},
+        ]},
+    )
+    assert fld == {"data.totals.subtotal": 90}
+
+    assert project_response(raw, {}) == raw
+    assert estimate_tokens(raw) > estimate_tokens(jm)  # the meter shrinks
+
+
+# --- compile + run --------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "intent,expected",
+    [("billing", "Billing handled."), ("tech", "Tech handled."), ("other", "Billing handled.")],
+)
+async def test_compile_and_run_routes_correctly(intent, expected):
+    graph = compile_workflow(_wf(), _ctx())
+    config = {"configurable": {"thread_id": f"thread-{intent}"}}
+    out = await graph.ainvoke(
+        {"messages": [HumanMessage(content="hi")], "intent": intent}, config
+    )
+    last = out["messages"][-1]
+    assert isinstance(last, AIMessage)
+    assert last.content == expected
+
+
+async def test_human_input_resume_can_drive_router_branch():
+    wf = {
+        "id": "wf_hitl_router",
+        "version": 1,
+        "state": {
+            "messages": {"type": "list[message]", "reducer": "add_messages"},
+            "decision": {"type": "str", "reducer": "last"},
+            "result": {"type": "str", "reducer": "last"},
+        },
+        "entry_node": "start",
+        "nodes": [
+            {"id": "start", "type": "start", "config": {}},
+            {
+                "id": "review",
+                "type": "human_input",
+                "config": {
+                    "prompt": "Approve?",
+                    "allowed_decisions": ["approve", "reject"],
+                    "output_key": "decision",
+                },
+            },
+            {
+                "id": "route",
+                "type": "router",
+                "config": {
+                    "expression": "decision",
+                    "cases": {"approve": "approved", "reject": "rejected"},
+                    "default": "rejected",
+                },
+            },
+            {"id": "approved", "type": "transform", "config": {"expression": "'approved'", "output_key": "result"}},
+            {"id": "rejected", "type": "transform", "config": {"expression": "'rejected'", "output_key": "result"}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [
+            {"source": "start", "target": "review"},
+            {"source": "review", "target": "route"},
+            {"source": "approved", "target": "end"},
+            {"source": "rejected", "target": "end"},
+        ],
+    }
+    graph = compile_workflow(wf, _ctx())
+    config = {"configurable": {"thread_id": "hitl-router"}}
+
+    first = await graph.ainvoke({"messages": [HumanMessage(content="needs review")]}, config)
+    assert "__interrupt__" in first
+
+    out = await graph.ainvoke(Command(resume="approve"), config)
+    assert out["decision"] == "approve"
+    assert out["result"] == "approved"

apps/api/tests/test_engine_knowledge_regressions.py

@@ -0,0 +1,286 @@
+"""Engine + knowledge regression tests:
+multi-label classifier + parallel router, knowledge_search builtin, KB folders,
+run-thread reuse, retry_on mapping, guardrail replacement, validation warnings,
+and the embedder cache.
+"""
+
+from __future__ import annotations
+
+import httpx
+from langchain_core.messages import HumanMessage
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.db.base import SessionLocal
+from forge.engine.compiler import compile_workflow
+from forge.engine.context import CompileContext
+from forge.engine.middleware_compiler import _retry_exceptions
+from forge.services.validation import validate_workflow
+
+
+def _ctx() -> CompileContext:
+    return CompileContext(tenant_id="t1", project_id="p1", checkpointer=InMemorySaver())
+
+
+def _cfg(thread: str) -> dict:
+    return {"configurable": {"thread_id": thread}}
+
+
+# ---------- multi-label classifier + parallel (multi) router ----------
+
+def _multi_wf() -> dict:
+    return {
+        "id": "wf_multi",
+        "version": 1,
+        "state": {
+            "messages": {"type": "list[message]", "reducer": "add_messages"},
+            "intents": {"type": "list[str]", "reducer": "last"},
+        },
+        "entry_node": "start",
+        "nodes": [
+            {"id": "start", "type": "start", "config": {}},
+            {
+                "id": "classify",
+                "type": "classifier",
+                # fake model can't do structured output -> keyword fallback collects
+                # EVERY matching label (multi_label).
+                "config": {"labels": ["weather", "billing"], "output_key": "intents",
+                           "multi_label": True, "model": "fake:n/a"},
+            },
+            {
+                "id": "route",
+                "type": "router",
+                "config": {"expression": "intents", "multi": True,
+                           "cases": {"weather": "weather_agent", "billing": "billing_agent"},
+                           "default": "general_agent"},
+            },
+            {"id": "weather_agent", "type": "agent",
+             "config": {"flavor": "agent", "model": "fake:WEATHER-ANSWER"}},
+            {"id": "billing_agent", "type": "agent",
+             "config": {"flavor": "agent", "model": "fake:BILLING-ANSWER"}},
+            {"id": "general_agent", "type": "agent",
+             "config": {"flavor": "agent", "model": "fake:GENERAL-ANSWER"}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [
+            {"source": "start", "target": "classify"},
+            {"source": "classify", "target": "route"},
+            {"source": "weather_agent", "target": "end"},
+            {"source": "billing_agent", "target": "end"},
+            {"source": "general_agent", "target": "end"},
+        ],
+    }
+
+
+async def test_multi_label_classifier_fallback_writes_list():
+    graph = compile_workflow(_multi_wf(), _ctx())
+    out = await graph.ainvoke(
+        {"messages": [HumanMessage(content="What's the weather like, and a question about my billing?")]},
+        _cfg("multi-1"),
+    )
+    assert sorted(out["intents"]) == ["billing", "weather"]
+
+
+async def test_multi_router_fans_out_to_all_matching_cases():
+    graph = compile_workflow(_multi_wf(), _ctx())
+    out = await graph.ainvoke(
+        {"messages": [HumanMessage(content="weather and billing please")]},
+        _cfg("multi-2"),
+    )
+    texts = [getattr(m, "content", "") for m in out["messages"]]
+    assert any("WEATHER-ANSWER" in t for t in texts)
+    assert any("BILLING-ANSWER" in t for t in texts)
+    assert not any("GENERAL-ANSWER" in t for t in texts)
+
+
+async def test_multi_router_falls_back_to_default_when_no_match():
+    graph = compile_workflow(_multi_wf(), _ctx())
+    out = await graph.ainvoke(
+        {"messages": [HumanMessage(content="hello there, completely unrelated")]},
+        _cfg("multi-3"),
+    )
+    texts = [getattr(m, "content", "") for m in out["messages"]]
+    assert any("GENERAL-ANSWER" in t for t in texts)
+
+
+# ---------- validation warnings ----------
+
+def test_router_without_default_warns():
+    wf = _multi_wf()
+    for n in wf["nodes"]:
+        if n["id"] == "route":
+            n["config"].pop("default")
+    wf["nodes"] = [n for n in wf["nodes"] if n["id"] != "general_agent"]
+    wf["edges"] = [e for e in wf["edges"] if e["source"] != "general_agent"]
+    res = validate_workflow(wf)
+    assert res.valid
+    assert any("no Default path" in w["message"] for w in res.warnings)
+
+
+# ---------- tool_retry retry_on mapping ----------
+
+def test_retry_exceptions_maps_names_to_types():
+    excs = _retry_exceptions(["timeout", "http_error", "value_error", "bogus_name"])
+    assert TimeoutError in excs
+    assert httpx.HTTPError in excs
+    assert ValueError in excs
+    assert len(excs) == 3  # unknown names are skipped
+
+
+# ---------- guardrail_regex block actually replaces ----------
+
+async def test_guardrail_block_replaces_reply():
+    wf = {
+        "id": "wf_guard", "version": 1,
+        "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+        "entry_node": "start",
+        "nodes": [
+            {"id": "start", "type": "start", "config": {}},
+            {"id": "agent", "type": "agent",
+             "config": {"flavor": "agent", "model": "fake:the forbidden secret",
+                        "middleware": [{"type": "guardrail_regex",
+                                        "config": {"patterns": ["forbidden"], "on_match": "block"}}]}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [{"source": "start", "target": "agent"}, {"source": "agent", "target": "end"}],
+    }
+    graph = compile_workflow(wf, _ctx())
+    out = await graph.ainvoke({"messages": [HumanMessage(content="hi")]}, _cfg("guard-1"))
+    texts = [getattr(m, "content", "") for m in out["messages"]]
+    assert any("[blocked by content guardrail]" in t for t in texts)
+    assert not any("forbidden secret" in t for t in texts)
+
+
+# ---------- knowledge_search builtin ----------
+
+async def test_knowledge_search_builtin_reports_empty_kb():
+    from forge.tools.materialize import materialize_tool
+
+    tool = materialize_tool({"kind": "builtin", "builtin": "knowledge_search", "name": "kb_search"}, _ctx())
+    out = await tool.ainvoke({"query": "anything at all"})
+    assert "No relevant knowledge" in out
+
+
+# ---------- KB folders ----------
+
+async def test_source_folders_scope_search_and_listing():
+    from forge.services.knowledge import KnowledgeService
+
+    async with SessionLocal() as s:
+        a = await KnowledgeService.create_source(
+            s, "t_fold", "p_fold", kind="text", name="manual",
+            text="The frobnicator manual explains frobnication in detail.", folder="Manuals")
+        await KnowledgeService.ingest(s, a)
+        b = await KnowledgeService.create_source(
+            s, "t_fold", "p_fold", kind="text", name="policy",
+            text="The vacation policy covers holidays and leave days.", folder="Policies")
+        await KnowledgeService.ingest(s, b)
+
+        folders = await KnowledgeService.list_folders(s, "t_fold", "p_fold")
+        assert folders == ["Manuals", "Policies"]
+
+        hits = await KnowledgeService.search(s, "t_fold", "p_fold", "frobnication manual", top_k=4, folders=["Manuals"])
+        assert hits and all(h.metadata.get("source_id") == a.id for h in hits)
+
+        none = await KnowledgeService.search(s, "t_fold", "p_fold", "frobnication", top_k=4, folders=["DoesNotExist"])
+        assert none == []
+
+
+# ---------- run thread reuse ----------
+
+async def test_create_run_reuses_thread():
+    from forge.services.runs import RunService
+    from forge.services.workflows import WorkflowService
+
+    wf_def = {
+        "id": "wf", "version": 1,
+        "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+        "entry_node": "start",
+        "nodes": [
+            {"id": "start", "type": "start", "config": {}},
+            {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": "fake:ok"}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [{"source": "start", "target": "agent"}, {"source": "agent", "target": "end"}],
+    }
+    svc = RunService(checkpointer=InMemorySaver())
+    async with SessionLocal() as s:
+        wf = await WorkflowService.create(s, "t_thr", "p_thr", name="thread-reuse")
+        await WorkflowService.save_canvas(s, wf, {}, wf_def)
+        run1 = await svc.create_run(s, tenant_id="t_thr", project_id="p_thr", workflow_id=wf.id,
+                                    input={"messages": [{"role": "user", "content": "one"}]})
+        run2 = await svc.create_run(s, tenant_id="t_thr", project_id="p_thr", workflow_id=wf.id,
+                                    input={"messages": [{"role": "user", "content": "two"}]},
+                                    thread_id=run1.thread_id)
+        run3 = await svc.create_run(s, tenant_id="t_thr", project_id="p_thr", workflow_id=wf.id,
+                                    input={"messages": [{"role": "user", "content": "three"}]})
+    assert run2.thread_id == run1.thread_id
+    assert run3.thread_id != run1.thread_id
+
+    # A caller may echo back the composite LangGraph id (`{tenant}:{uuid}`) instead of the DB
+    # Thread.id - create_run must resolve either to the SAME thread, else memory is not shared.
+    from sqlalchemy import select
+
+    from forge.models import Thread
+
+    async with SessionLocal() as s:
+        lg_id = (await s.execute(select(Thread.lg_thread_id).where(Thread.id == run1.thread_id))).scalar_one()
+        run4 = await svc.create_run(s, tenant_id="t_thr", project_id="p_thr", workflow_id=wf.id,
+                                    input={"messages": [{"role": "user", "content": "four"}]},
+                                    thread_id=lg_id)
+    assert run4.thread_id == run1.thread_id
+
+
+# ---------- embedder cache ----------
+
+def test_embedder_cache_returns_same_instance_and_right_dims():
+    from forge.knowledge.embeddings import resolve_embedder
+
+    a = resolve_embedder("openai:text-embedding-3-small", "sk-test-cache")
+    b = resolve_embedder("openai:text-embedding-3-small", "sk-test-cache")
+    other_key = resolve_embedder("openai:text-embedding-3-small", "sk-different")
+    large = resolve_embedder("openai:text-embedding-3-large", "sk-test-cache")
+    assert a is b
+    assert other_key is not a
+    assert a.dim == 1536
+    assert large.dim == 3072
+    assert large.name == "text-embedding-3-large"
+
+
+# ---------- qa kinds multi-filter ----------
+
+async def test_lookup_kinds_list_filters_multiple_categories():
+    from forge.services.knowledge import KnowledgeService
+
+    async with SessionLocal() as s:
+        await KnowledgeService.create_qa(s, "t_mk", "p_mk", question="alpha question", answer="a", kind="billing")
+        await KnowledgeService.create_qa(s, "t_mk", "p_mk", question="beta question", answer="b", kind="shipping")
+        await KnowledgeService.create_qa(s, "t_mk", "p_mk", question="gamma question", answer="c", kind="faq")
+        hit = await KnowledgeService.lookup(s, "t_mk", "p_mk", "alpha question", threshold=0.8, kinds=["billing", "shipping"])
+        assert hit and hit["kind"] == "billing"
+        miss = await KnowledgeService.lookup(s, "t_mk", "p_mk", "alpha question", threshold=0.8, kinds=["faq"])
+        assert miss is None
+        # empty kinds list = all kinds (no filter)
+        any_hit = await KnowledgeService.lookup(s, "t_mk", "p_mk", "gamma question", threshold=0.8, kinds=[])
+        assert any_hit and any_hit["kind"] == "faq"
+
+
+# Report-row grouping (workflow / assistant / other / deleted-workflow) is now covered
+# end-to-end against the SQL aggregate path in tests/test_stats.py.
+
+
+# ---------- qa custom kinds ----------
+
+async def test_qa_custom_kind_roundtrip_and_lookup_filter():
+    from forge.services.knowledge import KnowledgeService
+
+    async with SessionLocal() as s:
+        await KnowledgeService.create_qa(s, "t_kind", "p_kind", question="How do I reset the frobnicator?",
+                                         answer="Hold the red button for 5 seconds.", kind="troubleshooting")
+        await KnowledgeService.create_qa(s, "t_kind", "p_kind", question="What are your business hours?",
+                                         answer="9 to 5 on weekdays.", kind="faq")
+        hit = await KnowledgeService.lookup(s, "t_kind", "p_kind", "How do I reset the frobnicator?",
+                                            threshold=0.8, kind="troubleshooting")
+        assert hit and hit["kind"] == "troubleshooting"
+        miss = await KnowledgeService.lookup(s, "t_kind", "p_kind", "How do I reset the frobnicator?",
+                                             threshold=0.8, kind="faq")
+        assert miss is None

apps/api/tests/test_env_vars.py

@@ -0,0 +1,107 @@
+"""Per-environment endpoint substitution ({{env.*}} from FORGE_TOOL_VARS / settings.tool_vars).
+
+A tool/auth endpoint template references {{env.<key>}}; the value is supplied per environment via
+the tool_vars setting, so the SAME tool DB row resolves to a different real host in dev/qa/prod.
+Unlike {{ctx.*}} (lenient - a missing value is dropped/empty), a missing {{env.*}} key FAILS the
+call loudly (MissingTemplateVar), so a misconfigured environment never sends a broken URL.
+"""
+
+import json
+
+import httpx
+import pytest
+
+import forge.tools.graphql as gql_mod
+import forge.tools.rest as rest_mod
+from forge.auth_providers.templates import MissingTemplateVar
+from forge.tools.graphql import execute_graphql
+from forge.tools.rest import execute_rest
+
+
+def _capturing_client(sink: dict) -> httpx.AsyncClient:
+    async def handler(request: httpx.Request) -> httpx.Response:
+        sink["request"] = request
+        return httpx.Response(200, json={"ok": True})
+
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+
+async def test_env_substitution_in_url(monkeypatch):
+    """{{env.*}} in a url_template resolves from settings.tool_vars to the env's real host."""
+    monkeypatch.setattr(rest_mod.settings, "tool_vars", {"api_base": "https://api.qa.example.com"})
+    sink: dict = {}
+    cfg = {
+        "name": "orders_get",
+        "request": {
+            "method": "GET",
+            "url_template": "{{env.api_base}}/v1/orders/{id}",
+            "fields": [{"path": "id", "type": "string", "in": "path", "required": True, "llm_visible": True}],
+            "headers": [],
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {"id": "O-1"}, tenant_id="t", project_id="p", client=client)
+    assert str(sink["request"].url) == "https://api.qa.example.com/v1/orders/O-1"
+
+
+async def test_env_substitution_in_body_template(monkeypatch):
+    """{{env.*}} works alongside {{input.*}} in a JSON body template."""
+    monkeypatch.setattr(rest_mod.settings, "tool_vars", {"tenant": "acme-qa"})
+    sink: dict = {}
+    cfg = {
+        "name": "order_add",
+        "request": {
+            "method": "POST",
+            "url_template": "https://portal.example.dev/orders",
+            "fields": [{"path": "amount", "type": "integer", "in": "body", "llm_visible": True}],
+            "headers": [],
+            "body_template": '{"amount": {{ input.amount }}, "tenant": "{{ env.tenant }}"}',
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {"amount": 5}, tenant_id="t", project_id="p", client=client)
+    assert json.loads(sink["request"].content) == {"amount": 5, "tenant": "acme-qa"}
+
+
+async def test_undefined_env_var_fails_loud(monkeypatch):
+    """A template referencing an env var absent from tool_vars raises (never a broken request)."""
+    monkeypatch.setattr(rest_mod.settings, "tool_vars", {})  # nothing defined for this env
+    sink: dict = {}
+    cfg = {
+        "name": "x",
+        "request": {"method": "GET", "url_template": "{{env.api_base}}/x", "fields": [], "headers": []},
+    }
+    async with _capturing_client(sink) as client:
+        with pytest.raises(MissingTemplateVar) as e:
+            await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    assert "api_base" in str(e.value)
+    assert "request" not in sink  # the call never went out
+
+
+async def test_ctx_stays_lenient_alongside_strict_env(monkeypatch):
+    """env is strict, but ctx keeps its lenient behavior (missing -> empty) in the same template."""
+    monkeypatch.setattr(rest_mod.settings, "tool_vars", {"base": "https://api.example.com"})
+    sink: dict = {}
+    cfg = {
+        "name": "x",
+        "request": {
+            "method": "GET",
+            "url_template": "{{env.base}}/x?tok={{ctx.absent}}",
+            "fields": [],
+            "headers": [],
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", context={}, client=client)
+    # env resolved; the missing ctx token rendered empty rather than raising.
+    assert str(sink["request"].url) == "https://api.example.com/x?tok="
+
+
+async def test_graphql_endpoint_env_substitution(monkeypatch):
+    """The GraphQL endpoint (previously used verbatim) now resolves {{env.*}} too."""
+    monkeypatch.setattr(gql_mod.settings, "tool_vars", {"gql_base": "https://gql.prod.example.com"})
+    sink: dict = {}
+    cfg = {"endpoint": "{{env.gql_base}}/graphql", "query": "{ ping }", "variables": []}
+    async with _capturing_client(sink) as client:
+        await execute_graphql(cfg, {}, tenant_id="t", project_id="p", client=client)
+    assert str(sink["request"].url) == "https://gql.prod.example.com/graphql"

apps/api/tests/test_error_workflow.py

@@ -0,0 +1,37 @@
+"""Error-workflow fallback: an erroring run returns the on_error message gracefully."""
+
+from __future__ import annotations
+
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.db.base import SessionLocal
+from forge.models import Workflow
+from forge.services.dispatch import dispatch_message
+from forge.services.runs import RunService
+
+# agent with an unknown middleware type -> compile_workflow raises (deterministic, offline).
+# run_to_completion compiles inside its try block, so the on_error fallback covers it.
+_ERR_WF = {
+    "id": "wf_err", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "agent",
+    "on_error": {"message": "Sorry - something went wrong. A teammate will follow up."},
+    "nodes": [
+        {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": "fake:hi", "middleware": [{"type": "___nonexistent___", "enabled": True}]}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "agent", "target": "end"}],
+}
+
+
+async def test_errored_run_returns_on_error_message():
+    async with SessionLocal() as s:
+        wf = Workflow(tenant_id="t_err", project_id="p_err", name="Err", executable=_ERR_WF, status="active")
+        s.add(wf)
+        await s.commit()
+        await s.refresh(wf)
+    rs = RunService(checkpointer=InMemorySaver())
+    result = await dispatch_message(rs, tenant_id="t_err", project_id="p_err", workflow_id=wf.id, text="hi")
+    assert result.get("error")  # the run did fail
+    assert result.get("error_handled") is True
+    assert result.get("answer") == "Sorry - something went wrong. A teammate will follow up."

apps/api/tests/test_evals_and_retrieval.py

@@ -0,0 +1,65 @@
+"""Eval harness (dataset run + scoring) and ephemeral retrieval context."""
+
+from __future__ import annotations
+
+from langchain_core.messages import RemoveMessage, SystemMessage
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.db.base import SessionLocal
+from forge.engine.context import CompileContext
+from forge.models import Workflow
+from forge.nodes.rag import _KB_TAG, retrieval_factory
+from forge.services.evals import EvalService, _score_deterministic
+from forge.services.runs import RunService
+
+# --- ephemeral retrieval ---
+
+
+async def test_retrieval_removes_prior_kb_message():
+    ctx = CompileContext(tenant_id="t_r", project_id="p_r")
+    node = retrieval_factory({"announce_empty": True, "top_k": 2}, ctx)
+    prior = SystemMessage(content="old KB context", additional_kwargs={_KB_TAG: True})
+    prior.id = "kb-old"
+    user = {"role": "user", "content": "anything"}
+    out = await node({"messages": [prior, user]})
+    msgs = out.get("messages", [])
+    # prior KB message is removed; a fresh tagged one is added
+    assert any(isinstance(m, RemoveMessage) and m.id == "kb-old" for m in msgs)
+    assert any(isinstance(m, SystemMessage) and m.additional_kwargs.get(_KB_TAG) for m in msgs)
+
+
+def test_score_modes():
+    assert _score_deterministic("contains", "The answer is 42 friend", "42") is True
+    assert _score_deterministic("exact", "42", "42") is True
+    assert _score_deterministic("exact", "the answer is 42", "42") is False
+    assert _score_deterministic("regex", "order #A-1007 shipped", r"#A-\d+") is True
+
+
+# --- eval run ---
+
+_WF = {
+    "id": "wf_e", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "agent",
+    "nodes": [
+        {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": "fake:Your order total is 42 dollars."}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "agent", "target": "end"}],
+}
+
+
+async def test_eval_run_scores_dataset():
+    async with SessionLocal() as s:
+        wf = Workflow(tenant_id="t_e", project_id="p_e", name="E", executable=_WF, status="active")
+        s.add(wf)
+        await s.commit()
+        await s.refresh(wf)
+        ds = await EvalService.create(s, "t_e", "p_e", name="smoke", workflow_id=wf.id, score_mode="contains",
+                                      items=[{"input": "total?", "expected": "42"}, {"input": "hi", "expected": "nonexistent-string"}])
+        rs = RunService(checkpointer=InMemorySaver())
+        report = await EvalService.run(s, rs, ds)
+    assert report["summary"]["total"] == 2
+    assert report["summary"]["passed"] == 1  # first contains "42", second does not
+    assert report["results"][0]["passed"] is True and report["results"][1]["passed"] is False
+    assert ds.last_pass_rate == 0.5

apps/api/tests/test_evals_hardening.py

@@ -0,0 +1,256 @@
+"""Hardening for the eval harness, tracer, quota, and build assistant (findings F1-F7).
+
+House style: bare `async def` tests, offline `fake:` models, InMemorySaver, direct
+service calls. No provider keys / network (embedding + judge paths are exercised via
+their "unavailable" branches so the suite stays offline and fast).
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+from langgraph.checkpoint.memory import InMemorySaver
+from sqlalchemy import select
+
+from forge.db.base import SessionLocal
+from forge.engine.models import make_fake_model
+from forge.models import Component, Dataset, McpClient, Run, Tenant, Tool, Workflow
+from forge.services.evals import (
+    EvalService,
+    _is_real_judge,
+    _score_json,
+    _score_numeric,
+)
+from forge.services.quota import QuotaExceeded, check_run_quota, usage_today
+from forge.services.runs import RunService
+
+# A workflow whose (offline) agent always answers with "42 dollars" - lets us assert
+# deterministic pass/fail without any provider key.
+_WF = {
+    "id": "wf_h", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "agent",
+    "nodes": [
+        {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": "fake:Your order total is 42 dollars."}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "agent", "target": "end"}],
+}
+
+
+# --- F1 + F2: bounded-concurrency run, persisted history, regression gate ---
+
+
+async def test_eval_run_persists_history_and_regression_gate():
+    async with SessionLocal() as s:
+        wf = Workflow(tenant_id="t_h", project_id="p_h", name="H", executable=_WF, status="active")
+        s.add(wf)
+        await s.commit()
+        await s.refresh(wf)
+        ds = await EvalService.create(
+            s, "t_h", "p_h", name="h", workflow_id=wf.id, score_mode="contains",
+            items=[{"input": "a", "expected": "42"}, {"input": "b", "expected": "999"}, {"input": "c", "expected": "42"}],
+        )
+        ds.last_pass_rate = 1.0  # seed a prior high rate so the gate detects the drop
+        await s.commit()
+        rs = RunService(checkpointer=InMemorySaver())
+        report = await EvalService.run(s, rs, ds, regression_gate=True)
+        dsid = ds.id
+
+    # Three items ran (concurrently); two expect "42" (present) and one expects "999".
+    assert report["summary"]["total"] == 3
+    assert report["summary"]["passed"] == 2
+    assert report["summary"]["eval_run_id"]
+    assert report["summary"]["regression"]["regressed"] is True  # 0.667 < 1.0
+
+    async with SessionLocal() as s:
+        runs = await EvalService.history(s, "t_h", dsid)
+        assert len(runs) == 1
+        assert runs[0].total == 3 and runs[0].passed == 2 and runs[0].prev_pass_rate == 1.0
+        results = await EvalService.results(s, "t_h", runs[0].id)
+        assert sorted(r.item_index for r in results) == [0, 1, 2]
+        assert any(r.answer and "42" in r.answer for r in results)
+
+
+# --- F3: richer scorers + per-item assertion lists (AND/OR) ---
+
+
+def test_numeric_and_json_scorers():
+    ok, err = _score_numeric("about 42.5 units", "42", tolerance=1.0)
+    assert ok is True and err is not None
+    assert _score_numeric("100", "42", tolerance=1.0)[0] is False
+    assert _score_numeric("100", "90", rel_tolerance=0.2)[0] is True  # 10 <= 90*0.2
+    assert _score_json('{"a": 1, "b": 2}', {"a": 1}) is True          # subset match
+    assert _score_json('{"a": 1}', {"a": 1, "b": 2}) is False
+    assert _score_json('{"a": 1}', {"a": 1}, mode="exact") is True
+    assert _score_json("not json", {"a": 1}) is False
+
+
+async def test_assertion_list_and_or_combine():
+    ds = Dataset(tenant_id="t", project_id="p", name="x", score_mode="contains", items=[])
+    item = {"input": "q", "expected": "42",
+            "assertions": [{"type": "contains", "expected": "42"}, {"type": "contains", "expected": "zzz"}]}
+    # AND (default): one check fails -> item fails.
+    res_all = await EvalService._score_item(ds, {**item, "assert": "all"}, "the answer is 42", judge_model=None, embedder=None)
+    assert res_all["passed"] is False and len(res_all["checks"]) == 2
+    # OR: one check passes -> item passes.
+    res_any = await EvalService._score_item(ds, {**item, "assert": "any"}, "the answer is 42", judge_model=None, embedder=None)
+    assert res_any["passed"] is True
+
+
+# --- F4: LLM judge robustness (unavailable != silent contains pass) ---
+
+
+async def test_judge_unavailable_is_not_a_pass():
+    assert _is_real_judge(None) is False
+    assert _is_real_judge(make_fake_model("anything")) is False  # offline fake model is NOT a real judge
+
+    passed, _reason, status = await EvalService._judge(None, "in", "expected", "answer")
+    assert passed is False and status == "unavailable"
+
+    # The answer literally CONTAINS the expected string; the old code fell back to `contains`
+    # here and reported a (misleading) pass. Now a judge item with no model is "unavailable".
+    ds = Dataset(tenant_id="t", project_id="p", name="x", score_mode="judge", items=[])
+    res = await EvalService._score_item(ds, {"input": "i", "expected": "exp"}, "exp appears here",
+                                        judge_model=None, embedder=None)
+    assert res["passed"] is False and res["status"] == "unavailable"
+
+
+async def test_embedding_assertion_unavailable_without_embedder():
+    ds = Dataset(tenant_id="t", project_id="p", name="x", score_mode="contains", items=[])
+    item = {"input": "i", "expected": "e", "assertions": [{"type": "embedding", "expected": "e", "threshold": 0.9}]}
+    res = await EvalService._score_item(ds, item, "some answer", judge_model=None, embedder=None)
+    assert res["passed"] is False and res["status"] == "unavailable"
+
+
+# --- F5: tracer completeness (chain filtering, retriever + embedding spans) ---
+
+
+def test_tracer_filters_internal_chains_and_reparents():
+    from forge.tracing.tracer import ForgeTracer, _is_internal_chain
+
+    assert _is_internal_chain("RunnableSequence") and _is_internal_chain("chain") and _is_internal_chain("__start__")
+    assert not _is_internal_chain("agent_1") and not _is_internal_chain("support_agent")
+
+    tr = ForgeTracer()
+    tr.on_chain_start({"name": "RunnableSequence"}, {}, run_id="r1", parent_run_id=None)  # skipped
+    tr.on_chain_start({"name": "agent_1"}, {}, run_id="r2", parent_run_id="r1")            # kept, re-parented
+    tr.on_chat_model_start({}, [], run_id="r3", parent_run_id="r1")                        # kept, re-parented
+
+    names = [s.name for s in tr.ordered()]
+    assert "RunnableSequence" not in names
+    assert tr.spans["r2"].kind == "chain" and tr.spans["r2"].parent_id is None  # skipped r1 had no real parent
+    assert tr.spans["r3"].kind == "llm" and tr.spans["r3"].parent_id is None
+
+
+def test_tracer_retriever_and_embedding_spans():
+    from forge.tracing.tracer import ForgeTracer
+    from forge.tracing.tracer import embedding_span as active_embedding_span
+
+    tr = ForgeTracer()
+    tr.on_retriever_start({"name": "kb"}, "my query", run_id="rr", parent_run_id=None)
+    tr.on_retriever_end([{"page_content": "doc one"}, {"page_content": "doc two"}], run_id="rr")
+    rspan = tr.spans["rr"]
+    assert rspan.kind == "retriever" and rspan.attributes.get("docs") == 2
+
+    # Embedding span is priced via the embedding rate in pricing.price and rolled into totals.
+    with active_embedding_span("openai:text-embedding-3-small", n_texts=3, input_tokens=1000):
+        pass
+    emb = [s for s in tr.ordered() if s.kind == "embedding"]
+    assert emb and emb[0].cost_usd > 0 and emb[0].attributes.get("n_texts") == 3
+
+
+# --- F6: quota - projected-cost ceiling, per-project scoping ---
+
+
+async def test_quota_projected_cost_reserves_for_inflight_runs():
+    async with SessionLocal() as s:
+        t = Tenant(name="PC", settings={"max_cost_per_day_usd": 1.0, "projected_run_cost_usd": 5.0})
+        s.add(t)
+        await s.flush()
+        # An in-flight run is still $0 booked; the reservation (1 * $5) alone must trip the cap.
+        s.add(Run(tenant_id=t.id, project_id="p", workflow_id="w", thread_id="th", status="running", total_cost_usd=0.0))
+        await s.commit()
+        tid = t.id
+    async with SessionLocal() as s:
+        with pytest.raises(QuotaExceeded):
+            await check_run_quota(s, tid)
+        u = await usage_today(s, tid)
+        assert u["inflight"] == 1 and u["reserved_cost_usd"] == 5.0
+
+
+async def test_quota_no_reservation_when_projected_unset():
+    async with SessionLocal() as s:
+        t = Tenant(name="PC2", settings={"max_cost_per_day_usd": 1.0})  # no projected reservation
+        s.add(t)
+        await s.flush()
+        s.add(Run(tenant_id=t.id, project_id="p", workflow_id="w", thread_id="th", status="running", total_cost_usd=0.0))
+        await s.commit()
+        tid = t.id
+    async with SessionLocal() as s:
+        await check_run_quota(s, tid)  # booked cost 0 < 1 and nothing reserved -> must not raise
+
+
+async def test_quota_per_project_scoping():
+    async with SessionLocal() as s:
+        t = Tenant(name="PP", settings={"project_limits": {"proj_a": {"max_runs_per_day": 1}}})
+        s.add(t)
+        await s.flush()
+        s.add(Run(tenant_id=t.id, project_id="proj_a", workflow_id="w", thread_id="th", status="done"))
+        await s.commit()
+        tid = t.id
+    async with SessionLocal() as s:
+        with pytest.raises(QuotaExceeded):
+            await check_run_quota(s, tid, project_id="proj_a")  # per-project cap hit
+        # A project with no per-project limits (and no tenant-wide caps) is unlimited.
+        await check_run_quota(s, tid, project_id="proj_b")
+
+
+# --- F7: assistant build coverage (new tool kinds + component; offline judge unverified) ---
+
+
+async def test_assistant_builds_all_tool_kinds_and_component():
+    from forge.services.assistant import build_assistant_tools
+
+    tools = {t.name: t for t in build_assistant_tools("t_ab", "p_ab", [])}
+    for name in ("create_graphql_tool", "create_sql_tool", "create_code_tool", "create_mcp_tool", "create_component"):
+        assert name in tools
+
+    await tools["create_graphql_tool"].ainvoke({"name": "gql", "endpoint": "https://x/graphql", "query": "{ me }", "variables": "id"})
+    await tools["create_sql_tool"].ainvoke({"name": "sqltool", "query": "select 1", "arg_names": "limit"})
+    await tools["create_code_tool"].ainvoke({"name": "codetool", "source": "result = 1"})
+    await tools["create_mcp_tool"].ainvoke({"name": "mcpsrv", "url": "https://x/mcp"})
+    await tools["create_component"].ainvoke({"name": "card", "html": "<div>{{title}}</div>", "props_schema_json": '{"type": "object"}'})
+
+    async with SessionLocal() as s:
+        kinds = {t.kind for t in (await s.execute(select(Tool).where(Tool.project_id == "p_ab"))).scalars()}
+        assert {"graphql", "sql", "code"} <= kinds
+        assert (await s.execute(select(McpClient).where(McpClient.project_id == "p_ab"))).scalars().first() is not None
+        assert (await s.execute(select(Component).where(Component.project_id == "p_ab"))).scalars().first() is not None
+
+
+async def test_rest_tool_supports_body_query_header_params():
+    from forge.services.assistant import build_assistant_tools
+
+    tools = {t.name: t for t in build_assistant_tools("t_rt", "p_rt", [])}
+    await tools["create_rest_tool"].ainvoke({
+        "name": "post_it", "url_template": "https://x/{id}", "method": "POST",
+        "query_params": "q1", "header_params": "X-H", "body_params": "b1,b2",
+    })
+    async with SessionLocal() as s:
+        tool = (await s.execute(select(Tool).where(Tool.project_id == "p_rt"))).scalars().first()
+    fields = tool.config["request"]["fields"]
+    assert sorted({f["in"] for f in fields}) == ["body", "header", "path", "query"]
+    assert tool.config["request"]["method"] == "POST"
+
+
+async def test_evaluate_build_offline_is_unverified():
+    from forge.services.assistant import build_assistant_tools
+
+    tools = {t.name: t for t in build_assistant_tools("t_eb", "p_eb", [])}
+    out = json.loads(await tools["evaluate_build"].ainvoke(
+        {"user_request": "x", "what_was_built": "y", "test_results": "z"}
+    ))
+    # No provider key -> offline model -> NOT a bogus 'pass'.
+    assert out["verdict"] == "unverified"

apps/api/tests/test_form_encoding.py

@@ -0,0 +1,153 @@
+"""application/x-www-form-urlencoded request bodies (generic form-encoded POST support).
+
+`request.body_encoding = "form"` routes a structured body through httpx's `data=`, which
+URL-encodes every value (spaces, =, &, newlines, unicode), emits list values as repeated
+keys, and sets the Content-Type. This is generic to any form-encoded endpoint; the tests
+use a classic authenticated form post (orderNum + a multi-line productCodePost + a
+ctx-injected CSRFToken) as a representative shape, not a hardcoded API.
+"""
+
+import json
+from urllib.parse import parse_qs, parse_qsl
+
+import httpx
+
+from forge.tools.rest import build_args_schema, execute_rest
+from forge.util.ssrf import EgressPolicy
+
+# Permissive policy: the capturing client short-circuits the network, and block_private=False
+# keeps the SSRF guard from doing a real DNS lookup on the example host.
+_POLICY = EgressPolicy(block_private=False)
+
+
+def _capturing_client(sink: dict) -> httpx.AsyncClient:
+    async def handler(request: httpx.Request) -> httpx.Response:
+        sink["request"] = request
+        return httpx.Response(200, json={"ok": True})
+
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+
+def _form_cfg(fields, **req_extra) -> dict:
+    return {
+        "name": "form_post",
+        "request": {
+            "method": "POST",
+            "url_template": "https://portal.example.dev/shop/products/add",
+            "body_encoding": "form",
+            "fields": fields,
+            "headers": [],
+            **req_extra,
+        },
+    }
+
+
+async def test_form_encoding_urlencodes_multiline_and_special_chars():
+    """Headline case: a multi-line productCodePost value with `=`, spaces and a newline is
+    URL-encoded correctly and round-trips through a form parser - and the Content-Type is set
+    for the caller (no manual header)."""
+    sink: dict = {}
+    cfg = _form_cfg([
+        {"path": "orderNum", "type": "string", "in": "body", "llm_visible": True},
+        {"path": "pasteOption", "type": "string", "in": "body", "llm_visible": True},
+        {"path": "productCodePost", "type": "string", "in": "body", "llm_visible": True},
+    ])
+    multiline = "GLC-TE= 2\nFPR-9= 5"
+    async with _capturing_client(sink) as client:
+        await execute_rest(
+            cfg,
+            {"orderNum": "ORD-001", "pasteOption": "P-Q", "productCodePost": multiline},
+            tenant_id="t", project_id="p", client=client, egress_policy=_POLICY,
+        )
+    req = sink["request"]
+    assert req.headers["content-type"] == "application/x-www-form-urlencoded"
+    parsed = parse_qs(req.content.decode(), keep_blank_values=True)
+    assert parsed["orderNum"] == ["ORD-001"]
+    assert parsed["pasteOption"] == ["P-Q"]
+    assert parsed["productCodePost"] == [multiline]  # newline, `=`, space survived encode->decode
+    # The wire body must actually be percent-encoded, not literal, or a strict server misparses.
+    assert b"\n" not in req.content and b" " not in req.content
+
+
+async def test_form_encoding_includes_empty_field_and_ctx_injected_secret():
+    """An empty field is sent as `key=` (present, blank), and a hidden {{ctx.*}} in:body field
+    is injected into the encoded body without ever being an LLM arg."""
+    sink: dict = {}
+    cfg = _form_cfg([
+        {"path": "decimalPoints", "type": "string", "in": "body", "llm_visible": True, "default": ""},
+        {"path": "CSRFToken", "type": "string", "in": "body", "llm_visible": False, "default": "{{ctx.csrf}}"},
+    ])
+    assert "CSRFToken" not in build_args_schema(cfg).model_fields  # server-injected, not an LLM arg
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p",
+                           context={"csrf": "ffbe7ee3-29ff"}, client=client, egress_policy=_POLICY)
+    parsed = parse_qs(sink["request"].content.decode(), keep_blank_values=True)
+    assert parsed["decimalPoints"] == [""]  # empty value present on the wire
+    assert parsed["CSRFToken"] == ["ffbe7ee3-29ff"]
+
+
+async def test_form_encoding_list_value_becomes_repeated_keys():
+    """Generic repeated-key support: a list value serializes as `k=a&k=b` (not JSON)."""
+    sink: dict = {}
+    cfg = _form_cfg([{"path": "productCodePost", "type": "array", "in": "body", "llm_visible": True}])
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {"productCodePost": ["GLC-TE= 2", "FPR-9= 5"]},
+                           tenant_id="t", project_id="p", client=client, egress_policy=_POLICY)
+    pairs = parse_qsl(sink["request"].content.decode(), keep_blank_values=True)
+    assert pairs == [("productCodePost", "GLC-TE= 2"), ("productCodePost", "FPR-9= 5")]
+
+
+async def test_form_encoding_inferred_from_content_type_header():
+    """With no explicit body_encoding, a declared urlencoded Content-Type + structured body
+    infers form encoding."""
+    sink: dict = {}
+    cfg = {
+        "name": "x",
+        "request": {
+            "method": "POST",
+            "url_template": "https://portal.example.dev/x",
+            "fields": [{"path": "a", "type": "string", "in": "body", "llm_visible": True}],
+            "headers": [{"name": "Content-Type", "value": "application/x-www-form-urlencoded"}],
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {"a": "b c"}, tenant_id="t", project_id="p", client=client, egress_policy=_POLICY)
+    assert parse_qs(sink["request"].content.decode())["a"] == ["b c"]
+
+
+# --- regressions: an unset body_encoding keeps the legacy behavior -------------------------
+
+
+async def test_default_structured_body_still_json():
+    sink: dict = {}
+    cfg = {
+        "name": "x",
+        "request": {
+            "method": "POST",
+            "url_template": "https://portal.example.dev/x",
+            "fields": [{"path": "amount", "type": "integer", "in": "body", "llm_visible": True}],
+            "headers": [],
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {"amount": 5}, tenant_id="t", project_id="p", client=client, egress_policy=_POLICY)
+    assert json.loads(sink["request"].content) == {"amount": 5}
+    assert sink["request"].headers["content-type"].startswith("application/json")
+
+
+async def test_raw_body_template_unchanged():
+    sink: dict = {}
+    cfg = {
+        "name": "x",
+        "request": {
+            "method": "POST",
+            "url_template": "https://portal.example.dev/x",
+            "fields": [],
+            "headers": [],
+            "body_template": "CSRFToken={{ctx.csrf}}&scope=all",
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", context={"csrf": "C1"},
+                           client=client, egress_policy=_POLICY)
+    assert sink["request"].content == b"CSRFToken=C1&scope=all"

apps/api/tests/test_ghost_config_fixes.py

@@ -0,0 +1,459 @@
+"""Regression tests for the "ghost config" audit fixes - node/middleware schema options that
+were exposed in the UI but silently ignored by the compiler, plus the new validation rules.
+
+Everything here is engine-only (compile_workflow / validate_workflow with fake models and an
+InMemorySaver), so no database or network is needed.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+import pytest
+from langchain_core.messages import AIMessage, HumanMessage
+from langgraph.checkpoint.memory import InMemorySaver
+from langgraph.errors import GraphBubbleUp
+
+from forge.engine.compiler import compile_workflow
+from forge.engine.context import CompileContext
+from forge.nodes.flow import FANOUT_INDEX_KEY, join_factory, resilient_fanout_child
+from forge.services.validation import validate_workflow
+
+
+def _ctx() -> CompileContext:
+    return CompileContext(tenant_id="t1", project_id="p1", checkpointer=InMemorySaver())
+
+
+def _cfg(thread: str) -> dict:
+    return {"configurable": {"thread_id": thread}}
+
+
+# --------------------------------------------------------------------------------------------
+# Finding 1: join `reducer` is honored (was a pure passthrough).
+# --------------------------------------------------------------------------------------------
+
+def test_join_reducer_merge_first_last_concat():
+    ctx = _ctx()
+    assert join_factory({"reducer": "merge", "input_key": "p", "output_key": "o"}, ctx)(
+        {"p": [{"a": 1}, {"b": 2}]}
+    ) == {"o": {"a": 1, "b": 2}}
+    assert join_factory({"reducer": "last", "input_key": "p", "output_key": "o"}, ctx)(
+        {"p": [1, 2, 3]}
+    ) == {"o": 3}
+    assert join_factory({"reducer": "first", "input_key": "p", "output_key": "o"}, ctx)(
+        {"p": [1, 2, 3]}
+    ) == {"o": 1}
+    assert join_factory({"reducer": "concat", "input_key": "p", "output_key": "o"}, ctx)(
+        {"p": [[1], [2, 3]]}
+    ) == {"o": [1, 2, 3]}
+
+
+def test_join_without_input_key_is_passthrough_marker():
+    # No input_key -> convergence marker (aggregation stays with the state-key reducer).
+    assert join_factory({"reducer": "concat"}, _ctx())({"anything": 1}) == {}
+
+
+# --------------------------------------------------------------------------------------------
+# Finding 2: parallel_fanout index tagging + partial-failure isolation + per-item timeout.
+# --------------------------------------------------------------------------------------------
+
+_FANOUT_WF = {
+    "id": "fan", "version": 1,
+    "state": {
+        "messages": {"type": "list[message]", "reducer": "add_messages"},
+        "items": {"type": "list[json]", "reducer": "last"},
+        "results": {"type": "list[str]", "reducer": "add"},
+    },
+    "entry_node": "start",
+    "nodes": [
+        {"id": "start", "type": "start", "config": {}},
+        {"id": "fan", "type": "parallel_fanout", "config": {"over": "items", "child_node": "worker", "item_key": "item"}},
+        {"id": "worker", "type": "transform", "config": {"expression": "[item]", "output_key": "results"}},
+        {"id": "join", "type": "join", "config": {"reducer": "concat"}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [
+        {"source": "start", "target": "fan"},
+        {"source": "worker", "target": "join"},
+        {"source": "join", "target": "end"},
+    ],
+}
+
+
+async def test_fanout_still_maps_and_index_tag_does_not_leak():
+    graph = compile_workflow(_FANOUT_WF, _ctx())
+    out = await graph.ainvoke({"items": ["a", "b", "c"]}, _cfg("fan-1"))
+    assert sorted(out["results"]) == ["a", "b", "c"]
+    # The index/total ride only in the child's Send payload; they must not leak to run state.
+    assert FANOUT_INDEX_KEY not in out and "_fanout_total" not in out
+
+
+async def test_fanout_continue_on_error_still_runs():
+    graph = compile_workflow({**_FANOUT_WF, "error_policy": "continue"}, _ctx())
+    out = await graph.ainvoke({"items": ["x", "y"]}, _cfg("fan-2"))
+    assert sorted(out["results"]) == ["x", "y"]
+
+
+async def test_resilient_child_isolates_one_failure_but_keeps_the_rest():
+    def child(state):
+        if state.get(FANOUT_INDEX_KEY) == 1:
+            raise ValueError("boom")
+        return {"results": [state["item"]]}
+
+    skip = resilient_fanout_child(child, isolate=True)
+    assert await skip({"item": "a", FANOUT_INDEX_KEY: 0}) == {"results": ["a"]}
+    assert await skip({"item": "b", FANOUT_INDEX_KEY: 1}) == {}  # failure isolated
+
+    fail = resilient_fanout_child(child, isolate=False)
+    with pytest.raises(ValueError):
+        await fail({"item": "b", FANOUT_INDEX_KEY: 1})
+
+
+async def test_resilient_child_propagates_control_flow_and_honors_timeout():
+    async def bubbles(state):
+        raise GraphBubbleUp()  # interrupts / Command bubbling must NOT be swallowed
+
+    with pytest.raises(GraphBubbleUp):
+        await resilient_fanout_child(bubbles, isolate=True)({FANOUT_INDEX_KEY: 0})
+
+    async def slow(state):
+        await asyncio.sleep(1)
+        return {"results": ["late"]}
+
+    assert await resilient_fanout_child(slow, timeout=0.05, isolate=True)({FANOUT_INDEX_KEY: 0}) == {}
+
+
+# --------------------------------------------------------------------------------------------
+# Finding 3: tenant_budget honors max_usd_per_thread and scopes tokens to the run.
+# --------------------------------------------------------------------------------------------
+
+def test_tenant_budget_run_scoped_tokens():
+    from forge.engine.middleware_compiler import _tenant_budget
+
+    mw = _tenant_budget({"max_tokens_per_run": 10, "on_exceed": "end"}, None)
+    msg = AIMessage(content="x", usage_metadata={"input_tokens": 6, "output_tokens": 6, "total_tokens": 12})
+    assert mw.after_model({"messages": [msg]})["_forge_run_tokens"] == 12
+    stop = mw.before_model({"_forge_run_tokens": 12})
+    assert stop and stop.get("jump_to") == "end"
+    assert mw.before_model({"_forge_run_tokens": 0}) is None
+
+
+def test_tenant_budget_usd_accounting():
+    from forge.engine.middleware_compiler import _tenant_budget
+
+    mw = _tenant_budget({"max_usd_per_thread": 1.0, "on_exceed": "error"}, None)
+    # gpt-4.1-mini input is $0.40/1M tokens -> 1M input tokens == $0.40.
+    msg = AIMessage(
+        content="x",
+        usage_metadata={"input_tokens": 1_000_000, "output_tokens": 0, "total_tokens": 1_000_000},
+        response_metadata={"model_name": "gpt-4.1-mini"},
+    )
+    upd = mw.after_model({"messages": [msg]})
+    assert abs(upd["_forge_thread_cost_usd"] - 0.4) < 1e-6
+    with pytest.raises(RuntimeError):
+        mw.before_model({"_forge_thread_cost_usd": 2.0})
+
+
+# --------------------------------------------------------------------------------------------
+# Finding 4: guardrail_regex honors apply_to and implements redact/flag (block still replaces).
+# --------------------------------------------------------------------------------------------
+
+def test_guardrail_redact_masks_input_and_output():
+    from forge.engine.middleware_compiler import _guardrail_regex
+
+    mw = _guardrail_regex({"patterns": ["forbidden"], "on_match": "redact", "apply_to": "both"}, None)
+    out = mw.after_model({"messages": [AIMessage(content="the forbidden secret", id="a1")]})
+    assert "[redacted]" in out["messages"][-1].content and "forbidden" not in out["messages"][-1].content
+    inp = mw.before_model({"messages": [HumanMessage(content="my forbidden input", id="h1")]})
+    assert "[redacted]" in inp["messages"][-1].content
+
+
+def test_guardrail_flag_marks_without_changing_content():
+    from forge.engine.middleware_compiler import _guardrail_regex
+
+    mw = _guardrail_regex({"patterns": ["bad"], "on_match": "flag", "apply_to": "output"}, None)
+    out = mw.after_model({"messages": [AIMessage(content="this is bad", id="a2")]})
+    assert out["messages"][-1].additional_kwargs.get("guardrail_flagged")
+    assert out["messages"][-1].content == "this is bad"
+
+
+def test_guardrail_output_only_ignores_input():
+    from forge.engine.middleware_compiler import _guardrail_regex
+
+    mw = _guardrail_regex({"patterns": ["forbidden"], "on_match": "block", "apply_to": "output"}, None)
+    assert mw.before_model({"messages": [HumanMessage(content="forbidden", id="h9")]}) is None
+
+
+async def test_guardrail_block_still_replaces_reply():
+    wf = {
+        "id": "g", "version": 1,
+        "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+        "entry_node": "start",
+        "nodes": [
+            {"id": "start", "type": "start", "config": {}},
+            {"id": "agent", "type": "agent",
+             "config": {"flavor": "agent", "model": "fake:the forbidden secret",
+                        "middleware": [{"type": "guardrail_regex",
+                                        "config": {"patterns": ["forbidden"], "on_match": "block"}}]}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [{"source": "start", "target": "agent"}, {"source": "agent", "target": "end"}],
+    }
+    out = await compile_workflow(wf, _ctx()).ainvoke({"messages": [HumanMessage(content="hi")]}, _cfg("g1"))
+    texts = [getattr(m, "content", "") for m in out["messages"]]
+    assert any("[blocked by content guardrail]" in t for t in texts)
+    assert not any("forbidden secret" in t for t in texts)
+
+
+# --------------------------------------------------------------------------------------------
+# Finding 5: model_retry passes retry_on through.
+# --------------------------------------------------------------------------------------------
+
+def test_model_retry_passes_retry_on():
+    from forge.engine.middleware_compiler import _model_retry
+
+    mw = _model_retry({"max_retries": 1, "retry_on": ["timeout", "http_error"]}, None)
+    assert TimeoutError in mw.retry_on and httpx.HTTPError in mw.retry_on
+    assert _model_retry({"max_retries": 1}, None).retry_on == (Exception,)
+
+
+# --------------------------------------------------------------------------------------------
+# Advanced middleware are async-safe now (were sync-only -> crashed under ainvoke).
+# --------------------------------------------------------------------------------------------
+
+async def test_dynamic_model_by_state_middleware_runs_async():
+    # The advanced middleware were sync-only and raised NotImplementedError under ainvoke/astream
+    # (the real runtime path). They must now run async AND actually apply the model override.
+    # Rules evaluate against the AGENT's visible state; switching on an arbitrary parent-workflow
+    # state key is a separate, documented limitation (the agent subgraph boundary doesn't forward
+    # it), so this asserts the mechanism over a rule the agent can evaluate.
+    def wf(rule_when: str) -> dict:
+        return {
+            "id": "dm", "version": 1,
+            "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+            "entry_node": "start",
+            "nodes": [
+                {"id": "start", "type": "start", "config": {}},
+                {"id": "agent", "type": "agent", "config": {
+                    "flavor": "agent", "model": "fake:base",
+                    "middleware": [{"type": "dynamic_model_by_state",
+                                    "config": {"rules": [{"when": rule_when, "use": "fake:switched"}], "default": "fake:base"}}],
+                }},
+                {"id": "end", "type": "end", "config": {}},
+            ],
+            "edges": [{"source": "start", "target": "agent"}, {"source": "agent", "target": "end"}],
+        }
+    matched = await compile_workflow(wf("True"), _ctx()).ainvoke({"messages": [HumanMessage(content="hi")]}, _cfg("dm1"))
+    assert any("switched" in getattr(m, "content", "") for m in matched["messages"])
+    default = await compile_workflow(wf("False"), _ctx()).ainvoke({"messages": [HumanMessage(content="hi")]}, _cfg("dm2"))
+    assert any("base" in getattr(m, "content", "") for m in default["messages"])
+
+
+# --------------------------------------------------------------------------------------------
+# Finding 6: subworkflow input_mapping/output_mapping remap parent<->child keys.
+# --------------------------------------------------------------------------------------------
+
+_CHILD = {
+    "id": "child", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"},
+              "child_in": {"type": "str", "reducer": "last"},
+              "child_out": {"type": "str", "reducer": "last"}},
+    "entry_node": "t",
+    "nodes": [
+        {"id": "t", "type": "transform", "config": {"expression": "child_in", "output_key": "child_out"}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "t", "target": "end"}],
+}
+
+
+async def test_subworkflow_input_output_mapping():
+    parent = {
+        "id": "parent", "version": 1,
+        "state": {"messages": {"type": "list[message]", "reducer": "add_messages"},
+                  "p_val": {"type": "str", "reducer": "last"},
+                  "p_result": {"type": "str", "reducer": "last"}},
+        "entry_node": "start",
+        "nodes": [
+            {"id": "start", "type": "start", "config": {}},
+            {"id": "sub", "type": "subworkflow", "config": {
+                "workflow_id": "child_1",
+                "input_mapping": {"p_val": "child_in"},
+                "output_mapping": {"child_out": "p_result"}}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [{"source": "start", "target": "sub"}, {"source": "sub", "target": "end"}],
+    }
+    ctx = _ctx()
+    ctx.workflows = {"child_1": _CHILD}
+    out = await compile_workflow(parent, ctx).ainvoke({"p_val": "hello"}, _cfg("sub-map"))
+    assert out.get("p_result") == "hello"
+
+
+# --------------------------------------------------------------------------------------------
+# Finding 7: transform engine=jq raises clearly when jq missing; jmespath errors -> None.
+# --------------------------------------------------------------------------------------------
+
+def test_transform_jq_raises_when_unavailable():
+    from forge.nodes.data import transform_factory
+
+    node = transform_factory({"engine": "jq", "expression": ".x", "output_key": "data"}, _ctx())
+    with pytest.raises(ValueError, match="jq"):
+        node({"x": 1})
+
+
+def test_transform_bad_jmespath_returns_none():
+    from forge.nodes.data import transform_factory
+
+    node = transform_factory({"expression": "foo[", "output_key": "data"}, _ctx())
+    assert node({"foo": 1}) == {"data": None}
+
+
+# --------------------------------------------------------------------------------------------
+# Finding 10: new validation rules (+ the pre-existing fanout-adjacency false positive).
+# --------------------------------------------------------------------------------------------
+
+def _wf_with(nodes, edges, state=None):
+    return {
+        "id": "v", "version": 1,
+        "state": state or {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+        "entry_node": "start", "nodes": nodes, "edges": edges,
+    }
+
+
+def test_fanout_workflow_validates_cleanly():
+    # Pre-existing bug: the validator didn't model parallel_fanout -> child, so it wrongly
+    # reported worker/join/end unreachable + "no path to END". It should validate now.
+    res = validate_workflow(_FANOUT_WF)
+    assert res.valid, res.errors
+
+
+def test_undeclared_write_is_an_error():
+    wf = _wf_with(
+        [
+            {"id": "start", "type": "start", "config": {}},
+            {"id": "x", "type": "transform", "config": {"expression": "`1`", "output_key": "ghost_key"}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        [{"source": "start", "target": "x"}, {"source": "x", "target": "end"}],
+    )
+    res = validate_workflow(wf)
+    assert not res.valid
+    assert any("ghost_key" in e["message"] for e in res.errors)
+    # Declaring it clears the error.
+    wf2 = _wf_with(
+        wf["nodes"], wf["edges"],
+        state={"messages": {"type": "list[message]", "reducer": "add_messages"},
+               "ghost_key": {"type": "json", "reducer": "last"}},
+    )
+    assert validate_workflow(wf2).valid
+
+
+def test_reachable_dead_end_warns():
+    wf = _wf_with(
+        [
+            {"id": "start", "type": "start", "config": {}},
+            {"id": "a", "type": "agent", "config": {"flavor": "agent", "model": "fake:x"}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        # 'a' is reachable but has no outgoing edge; start also reaches end so the graph is valid.
+        [{"source": "start", "target": "a"}, {"source": "start", "target": "end"}],
+    )
+    res = validate_workflow(wf)
+    assert any("no outgoing edge" in w["message"] and w.get("node_id") == "a" for w in res.warnings)
+
+
+def test_branches_edge_requires_condition():
+    wf = _wf_with(
+        [
+            {"id": "start", "type": "start", "config": {}},
+            {"id": "a", "type": "agent", "config": {"flavor": "agent", "model": "fake:x"}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        [
+            {"source": "start", "target": "a"},
+            {"source": "a", "target": "end", "branches": {"yes": "end"}},  # missing condition
+        ],
+    )
+    res = validate_workflow(wf)
+    assert not res.valid
+    assert any("no condition" in e["message"] for e in res.errors)
+
+
+# --------------------------------------------------------------------------------------------
+# Finding 8: a branches edge routes on its condition, and an unmatched value ends the run
+# gracefully (END is a valid target) instead of raising KeyError('__end__') at runtime.
+# --------------------------------------------------------------------------------------------
+
+def _branch_wf():
+    return {
+        "id": "br", "version": 1,
+        "state": {"messages": {"type": "list[message]", "reducer": "add_messages"},
+                  "intent": {"type": "str", "reducer": "last"}},
+        "entry_node": "start",
+        "nodes": [
+            {"id": "start", "type": "start", "config": {}},
+            {"id": "gate", "type": "transform", "config": {"expression": "intent", "output_key": "intent"}},
+            {"id": "a", "type": "agent", "config": {"flavor": "agent", "model": "fake:A-ANSWER"}},
+            {"id": "b", "type": "agent", "config": {"flavor": "agent", "model": "fake:B-ANSWER"}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [
+            {"source": "start", "target": "gate"},
+            {"source": "gate", "target": "end", "condition": "intent", "branches": {"x": "a", "y": "b"}},
+            {"source": "a", "target": "end"},
+            {"source": "b", "target": "end"},
+        ],
+    }
+
+
+async def test_branch_edge_routes_and_ends_gracefully_on_no_match():
+    graph = compile_workflow(_branch_wf(), _ctx())
+    out = await graph.ainvoke({"messages": [HumanMessage(content="hi")], "intent": "y"}, _cfg("br-y"))
+    texts = [getattr(m, "content", "") for m in out["messages"]]
+    assert any("B-ANSWER" in t for t in texts) and not any("A-ANSWER" in t for t in texts)
+    # Unmatched value must route to END without crashing (was KeyError('__end__')).
+    out2 = await graph.ainvoke({"messages": [HumanMessage(content="hi")], "intent": "z"}, _cfg("br-z"))
+    assert not any("ANSWER" in getattr(m, "content", "") for m in out2["messages"])
+
+
+# --------------------------------------------------------------------------------------------
+# Finding 9: unwired agent fields (memory/filesystem/permissions) surface as warnings.
+# --------------------------------------------------------------------------------------------
+
+def test_unwired_agent_fields_warn():
+    wf = _wf_with(
+        [
+            {"id": "start", "type": "start", "config": {}},
+            {"id": "a", "type": "agent", "config": {
+                "flavor": "agent", "model": "fake:x",
+                "permissions": [{"path": "/x", "access": "read"}],
+                "memory": {"long_term": True}}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        [{"source": "start", "target": "a"}, {"source": "a", "target": "end"}],
+    )
+    res = validate_workflow(wf)
+    assert res.valid  # warnings only, never block save
+    assert any("permissions" in w["message"] for w in res.warnings)
+    assert any("memory" in w["message"].lower() for w in res.warnings)
+
+
+# --------------------------------------------------------------------------------------------
+# Library drift: langchain-openai (>=1.3) renamed OpenAIModerationMiddleware's apply_to_* flags
+# to check_*. Enabling `openai_moderation` used to crash at compile with
+# "__init__() got an unexpected keyword argument 'apply_to_input'"; the compiler now translates.
+# --------------------------------------------------------------------------------------------
+
+def test_openai_moderation_translates_apply_to_flags():
+    pytest.importorskip("langchain_openai")
+    from forge.engine.middleware_compiler import _openai_moderation
+
+    mw = _openai_moderation({"apply_to_input": True, "apply_to_output": False}, None)
+    assert mw.check_input is True and mw.check_output is False
+    # Empty config compiles to the library defaults (both checks on) without raising.
+    default = _openai_moderation({}, None)
+    assert default.check_input is True and default.check_output is True
+    # Advanced-JSON pass-through kwargs reach the library unchanged.
+    assert _openai_moderation({"exit_behavior": "replace"}, None).exit_behavior == "replace"

apps/api/tests/test_handoff.py

@@ -0,0 +1,53 @@
+"""Live-agent handoff: a channel run pauses at a handoff node, opens a queue item,
+and an agent reply resumes the run."""
+
+from __future__ import annotations
+
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.db.base import SessionLocal
+from forge.models import HandoffRequest, Workflow
+from forge.services.channels import ChannelService
+from forge.services.dispatch import dispatch_message
+from forge.services.handoff import HandoffService
+from forge.services.runs import RunService
+
+_WF = {
+    "id": "wf_h", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "handoff",
+    "nodes": [
+        {"id": "handoff", "type": "handoff", "config": {"reason": "needs a human", "ack_message": "Hold on, connecting you."}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "handoff", "target": "end"}],
+}
+
+
+async def test_handoff_node_interrupts_then_resumes():
+    async with SessionLocal() as s:
+        wf = Workflow(tenant_id="t_h", project_id="p_h", name="H", executable=_WF, status="active")
+        s.add(wf)
+        await s.commit()
+        await s.refresh(wf)
+        ch = await ChannelService.create(s, "t_h", "p_h", type_="email", name="W", workflow_id=wf.id)
+
+    # shared checkpointer so the run can be resumed by id
+    rs = RunService(checkpointer=InMemorySaver())
+    result = await dispatch_message(rs, tenant_id="t_h", project_id="p_h", workflow_id=wf.id, text="I need help")
+    assert result["interrupted"] is True
+    # the interrupt carries our handoff marker + reason
+    flat = [it for grp in result["interrupts"] for it in grp]
+    assert any(isinstance(i.get("value"), dict) and i["value"].get("handoff") for i in flat)
+
+    # open a handoff queue item and have an agent reply
+    async with SessionLocal() as s:
+        h = await HandoffService.create(
+            s, channel=ch, tenant_id="t_h", project_id="p_h", workflow_id=wf.id,
+            run_id=result["run_id"], thread_id=result["thread_id"], customer="widget-user",
+            customer_message="I need help", reason="needs a human", reply_context={},
+        )
+        out = await HandoffService.reply(s, rs, handoff=h, agent_id="agent1", message="Hi, this is Sam - happy to help!")
+        assert out["ok"] is True
+        refreshed = await s.get(HandoffRequest, h.id)
+        assert refreshed.status == "answered" and refreshed.agent_id == "agent1"

apps/api/tests/test_hybrid_search.py

@@ -0,0 +1,84 @@
+"""Hybrid retrieval: RRF fusion + BM25 primitives and the end-to-end search(hybrid=True)
+path (scoping, score normalization, source filtering, graceful vector fallback)."""
+
+from __future__ import annotations
+
+from forge.db.base import SessionLocal
+from forge.knowledge.hybrid import bm25_rank, rrf_fuse
+from forge.services.knowledge import KnowledgeService
+
+# --- pure primitives ---
+
+
+def test_rrf_fuse_rewards_agreement():
+    # "b" is near the top of BOTH lists; "a" and "z" each top only one.
+    fused = rrf_fuse(["a", "b", "c"], ["z", "b", "y"])
+    assert fused["b"] > fused["a"]
+    assert fused["b"] > fused["z"]
+
+
+def test_bm25_rank_surfaces_exact_term():
+    docs = [
+        ("d1", "general refund and shipping policy details"),
+        ("d2", "error code XJ9000 means a payment gateway timeout"),
+        ("d3", "how to contact our support team"),
+    ]
+    assert bm25_rank("XJ9000 gateway timeout", docs)[0] == "d2"
+
+
+def test_bm25_rank_empty_when_no_overlap():
+    docs = [("d1", "alpha beta gamma"), ("d2", "delta epsilon zeta")]
+    assert bm25_rank("zzz qqq wwww", docs) == []
+
+
+# --- end-to-end search(hybrid=True) ---
+
+
+async def test_hybrid_search_scoped_and_normalized(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma")
+    async with SessionLocal() as s:
+        for i, t in enumerate([
+            "Refunds go to the original payment method within 5-7 business days.",
+            "Error code XJ9000 indicates a payment gateway timeout; retry after 30 seconds.",
+            "Cancel an order from the Orders page before the item ships.",
+        ]):
+            src = await KnowledgeService.create_source(s, "t_hy", "p_hy", kind="text", name=f"d{i}", text=t)
+            await KnowledgeService.ingest(s, src)
+        # A different project must never leak into p_hy's results.
+        other = await KnowledgeService.create_source(s, "t_hy", "p_other", kind="text", name="x", text="XJ9000 belongs to another project")
+        await KnowledgeService.ingest(s, other)
+        hits = await KnowledgeService.search(s, "t_hy", "p_hy", "XJ9000 timeout", top_k=3, hybrid=True)
+    assert hits
+    assert all(0 < h.score <= 1.0 for h in hits)  # normalized fusion score
+    assert hits[0].score == 1.0  # best fused result anchors at 1.0
+    assert all(h.metadata.get("project_id") == "p_hy" for h in hits)  # tenant/project scoped
+    assert any("XJ9000" in h.text for h in hits)  # lexical match surfaced
+
+
+async def test_hybrid_respects_source_filter(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma2")
+    async with SessionLocal() as s:
+        a = await KnowledgeService.create_source(s, "t_sf", "p_sf", kind="text", name="a", text="XJ9000 appears in source A only")
+        await KnowledgeService.ingest(s, a)
+        b = await KnowledgeService.create_source(s, "t_sf", "p_sf", kind="text", name="b", text="source B is about refunds and shipping")
+        await KnowledgeService.ingest(s, b)
+        hits = await KnowledgeService.search(s, "t_sf", "p_sf", "XJ9000", top_k=5, hybrid=True, source_ids=[a.id])
+    assert hits
+    assert all(h.metadata.get("source_id") == a.id for h in hits)  # filter preserved under hybrid
+
+
+async def test_hybrid_degrades_to_vector_without_lexical_overlap(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma3")
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_dg", "p_dg", kind="text", name="d", text="Refunds are issued within five business days.")
+        await KnowledgeService.ingest(s, src)
+        # Query shares no tokens with the corpus -> BM25 contributes nothing -> vector fallback.
+        hits = await KnowledgeService.search(s, "t_dg", "p_dg", "zzz qqq wwww", top_k=3, hybrid=True)
+    assert len(hits) == 1  # no crash; vector path still returns the doc
+    assert "Refunds" in hits[0].text

apps/api/tests/test_knowledge.py

@@ -0,0 +1,52 @@
+"""Phase 5 validation: splitter, offline embedder, Q&A lookup, Chroma ingest→search."""
+
+from __future__ import annotations
+
+import pytest
+
+from forge.db.base import SessionLocal
+from forge.knowledge.embeddings import cosine, resolve_embedder
+from forge.knowledge.splitter import split_text
+from forge.services.knowledge import KnowledgeService
+
+
+def test_splitter_chunks_long_text():
+    text = ("Sentence one. " * 200).strip()
+    chunks = split_text(text, chunk_size=300, overlap=50)
+    assert len(chunks) > 1
+    assert all(len(c) <= 360 for c in chunks)  # ~chunk_size + overlap slack
+
+
+def test_embedder_similarity_reflects_overlap():
+    pytest.importorskip("fastembed")
+    e = resolve_embedder("fastembed:BAAI/bge-small-en-v1.5")
+    if getattr(e, "name", "") != "BAAI/bge-small-en-v1.5":
+        pytest.skip("fastembed model could not be loaded (offline)")
+    a = e.embed_query("refunds are issued to the original payment method")
+    b = e.embed_query("how long do refunds take to be issued")
+    c = e.embed_query("the weather in tokyo is sunny today")
+    assert cosine(a, b) > cosine(a, c)  # topical overlap > unrelated
+
+
+async def test_qa_create_and_lookup():
+    async with SessionLocal() as s:
+        await KnowledgeService.create_qa(s, "t_qa", "p_qa", question="How do I reset my password?", answer="Settings > Security > Reset password.", kind="faq")
+        match = await KnowledgeService.lookup(s, "t_qa", "p_qa", "how to reset password", threshold=0.2)
+    assert match and "Security" in match["answer"]
+
+
+async def test_ingest_text_and_search(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma")  # isolate Chroma for the test
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(
+            s, "t_kb", "p_kb", kind="text", name="help",
+            text="Refunds are issued to the original payment method within 5-7 business days. "
+                 "To cancel an order, open the Orders page before it ships.",
+        )
+        src = await KnowledgeService.ingest(s, src)
+        assert src.status == "ready" and src.chunks >= 1
+        hits = await KnowledgeService.search(s, "t_kb", "p_kb", "how long do refunds take", top_k=3)
+    assert hits, "expected at least one hit"
+    assert "refund" in hits[0].text.lower()

apps/api/tests/test_mcp.py

@@ -0,0 +1,63 @@
+"""MCP tool-kind wiring tests (monkeypatched server - no live MCP connection)."""
+
+from __future__ import annotations
+
+import pytest
+from langchain_core.tools import StructuredTool
+
+from forge.db.base import SessionLocal
+from forge.models import McpClient
+from forge.services.runtime import make_runtime_ctx
+from forge.tools import mcp as mcp_mod
+from forge.tools.mcp import McpUnavailable, load_mcp_tool
+
+
+def _fake_remote_tool(name="search"):
+    async def _run(q: str) -> str:
+        return f"results for {q}"
+    return StructuredTool.from_function(coroutine=_run, name=name, description="remote search")
+
+
+async def _make_client(tenant="t_mcp", project="p_mcp") -> str:
+    async with SessionLocal() as s:
+        row = McpClient(tenant_id=tenant, project_id=project, name="demo", transport="streamable_http", url="https://mcp.example/sse")
+        s.add(row)
+        await s.commit()
+        await s.refresh(row)
+        return row.id
+
+
+async def test_adapters_import_available():
+    # The optional extra is installed in this env; the loader resolves the client class.
+    assert mcp_mod._require_adapters() is not None
+
+
+async def test_load_mcp_tool_finds_remote_tool(monkeypatch):
+    cid = await _make_client()
+
+    async def fake_client_and_tools(row, tenant_id, project_id):
+        return object(), [_fake_remote_tool("search")]
+
+    monkeypatch.setattr(mcp_mod, "_client_and_tools", fake_client_and_tools)
+    ctx = make_runtime_ctx("t_mcp", "p_mcp")
+    tool = await load_mcp_tool({"mcp_client_id": cid, "remote_tool_name": "search"}, ctx)
+    assert tool.name == "search"
+    assert await tool.ainvoke({"q": "hello"}) == "results for hello"
+
+
+async def test_load_mcp_tool_unknown_remote(monkeypatch):
+    cid = await _make_client()
+    monkeypatch.setattr(mcp_mod, "_client_and_tools", lambda *a, **k: _noop([]))
+    ctx = make_runtime_ctx("t_mcp", "p_mcp")
+    with pytest.raises(McpUnavailable):
+        await load_mcp_tool({"mcp_client_id": cid, "remote_tool_name": "nope"}, ctx)
+
+
+async def test_load_mcp_tool_missing_client():
+    ctx = make_runtime_ctx("t_mcp", "p_mcp")
+    with pytest.raises(McpUnavailable):
+        await load_mcp_tool({"mcp_client_id": "does-not-exist", "remote_tool_name": "x"}, ctx)
+
+
+async def _noop(tools):
+    return object(), tools

apps/api/tests/test_mcp_oauth.py

@@ -0,0 +1,117 @@
+"""OAuth 2.1 for MCP: discovery, dynamic client registration, authorization-code + PKCE, single-use
+codes, audience binding, and token validation on the MCP endpoint. Gated by mcp_oauth_enabled."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import os
+import urllib.parse
+import uuid
+
+import httpx
+
+from forge.config import settings
+from forge.main import create_app
+
+
+def _pkce() -> tuple[str, str]:
+    verifier = base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode()
+    challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).rstrip(b"=").decode()
+    return verifier, challenge
+
+
+async def _authorize_code(c, *, email, password, client_id, redirect_uri, challenge, resource) -> str:
+    form = {
+        "email": email, "password": password, "workspace_id": "", "client_id": client_id,
+        "redirect_uri": redirect_uri, "code_challenge": challenge, "state": "s", "resource": resource, "scope": "",
+    }
+    sub = await c.post("/v1/oauth/authorize", data=form, follow_redirects=False)
+    assert sub.status_code == 302, sub.text
+    q = urllib.parse.parse_qs(urllib.parse.urlparse(sub.headers["location"]).query)
+    assert q.get("state") == ["s"]
+    return q["code"][0]
+
+
+async def test_mcp_oauth_disabled_returns_404():
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        assert (await c.get("/.well-known/oauth-authorization-server")).status_code == 404
+
+
+async def test_mcp_oauth_end_to_end(monkeypatch):
+    monkeypatch.setattr(settings, "mcp_oauth_enabled", True)
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        # a real user + a project (in the user's tenant) with one tool
+        email = f"o{uuid.uuid4().hex[:10]}@example.com"
+        reg = await c.post("/v1/auth/register", json={"email": email, "password": "supersecret1"})
+        assert reg.status_code == 201, reg.text
+        c.headers["Authorization"] = f"Bearer {reg.json()['access_token']}"
+        pid = (await c.post("/v1/projects", json={"name": "OAuth", "slug": "oauth-proj"})).json()["id"]
+        tid = (await c.post(f"/v1/projects/{pid}/tools", json={"name": "calc", "kind": "builtin", "config": {"builtin": "calculator", "description": "c"}})).json()["id"]
+        # publish the tool via an exposed tool set (the MCP surface = exposed sets' enabled tools)
+        await c.post(f"/v1/projects/{pid}/tool-sets", json={"name": "General", "tool_ids": [tid]})
+
+        # discovery is live when enabled
+        asm = (await c.get("/.well-known/oauth-authorization-server")).json()
+        assert asm["code_challenge_methods_supported"] == ["S256"]
+        prm = (await c.get(f"/.well-known/oauth-protected-resource/v1/mcp/{pid}")).json()
+        assert prm["resource"].endswith(f"/v1/mcp/{pid}") and prm["authorization_servers"]
+
+        # dynamic client registration (RFC 7591)
+        redirect_uri = "http://localhost/callback"
+        rc = await c.post("/v1/oauth/register", json={"redirect_uris": [redirect_uri], "client_name": "Test client"})
+        assert rc.status_code == 201, rc.text
+        client_id = rc.json()["client_id"]
+
+        # the consent form renders
+        verifier, challenge = _pkce()
+        resource = f"{settings.public_base_url.rstrip('/')}/v1/mcp/{pid}"
+        gf = await c.get("/v1/oauth/authorize", params={
+            "response_type": "code", "client_id": client_id, "redirect_uri": redirect_uri,
+            "code_challenge": challenge, "code_challenge_method": "S256", "resource": resource, "state": "s",
+        })
+        assert gf.status_code == 200 and "Authorize" in gf.text
+
+        # authorization-code exchange (PKCE verifier) -> access token
+        code = await _authorize_code(c, email=email, password="supersecret1", client_id=client_id,
+                                     redirect_uri=redirect_uri, challenge=challenge, resource=resource)
+        tok = await c.post("/v1/oauth/token", data={
+            "grant_type": "authorization_code", "code": code, "redirect_uri": redirect_uri,
+            "client_id": client_id, "code_verifier": verifier,
+        })
+        assert tok.status_code == 200, tok.text
+        access = tok.json()["access_token"]
+        assert tok.json()["token_type"] == "Bearer"
+
+        # the audience-bound token authorizes the MCP endpoint
+        body = {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}
+        oauth_headers = {"Authorization": f"Bearer {access}"}
+        r = await c.post(f"/v1/mcp/{pid}", headers=oauth_headers, json=body)
+        assert r.status_code == 200 and any(t["name"] == "calc" for t in r.json()["result"]["tools"])
+
+        # no credential -> 401 with an RFC 9728 discovery pointer
+        no = await c.post(f"/v1/mcp/{pid}", json=body)
+        assert no.status_code == 401 and "resource_metadata" in no.headers.get("www-authenticate", "")
+
+        # audience binding: the token must not work on a different project
+        pid2 = (await c.post("/v1/projects", json={"name": "Other", "slug": "oauth-other"})).json()["id"]
+        assert (await c.post(f"/v1/mcp/{pid2}", headers=oauth_headers, json=body)).status_code == 401
+
+        # single-use: replaying the same authorization code is rejected
+        replay = await c.post("/v1/oauth/token", data={
+            "grant_type": "authorization_code", "code": code, "redirect_uri": redirect_uri,
+            "client_id": client_id, "code_verifier": verifier,
+        })
+        assert replay.status_code == 400
+
+        # PKCE is enforced: a fresh code with the wrong verifier fails
+        v2, ch2 = _pkce()
+        code2 = await _authorize_code(c, email=email, password="supersecret1", client_id=client_id,
+                                      redirect_uri=redirect_uri, challenge=ch2, resource=resource)
+        bad = await c.post("/v1/oauth/token", data={
+            "grant_type": "authorization_code", "code": code2, "redirect_uri": redirect_uri,
+            "client_id": client_id, "code_verifier": verifier,  # wrong verifier
+        })
+        assert bad.status_code == 400

apps/api/tests/test_mcp_server.py

@@ -0,0 +1,109 @@
+"""Forge-as-an-MCP-server: initialize / tools/list / tools/call over JSON-RPC."""
+
+from __future__ import annotations
+
+import httpx
+
+from forge.db.base import SessionLocal
+from forge.main import create_app
+from forge.models import Project, Tool
+
+
+async def _seed_project_with_tool(slug="mcp-proj") -> str:
+    from forge.services.tool_sets import ToolSetService
+
+    async with SessionLocal() as s:
+        proj = Project(tenant_id="t_mcps", name="MCP Proj", slug=slug, config={})
+        s.add(proj)
+        await s.flush()
+        tool = Tool(tenant_id="t_mcps", project_id=proj.id, name="calculator", kind="builtin",
+                    config={"builtin": "calculator", "description": "Evaluate arithmetic."})
+        s.add(tool)
+        await s.commit()
+        await s.refresh(proj)
+        await s.refresh(tool)
+        # The MCP surface is the enabled tools of EXPOSED tool sets, so publish via a set.
+        await ToolSetService.create(s, "t_mcps", proj.id, name="General", tool_ids=[tool.id])
+        return proj.id
+
+
+async def test_mcp_initialize_and_list_and_call():
+    pid = await _seed_project_with_tool()
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        # initialize
+        r = await c.post(f"/v1/mcp/{pid}", json={"jsonrpc": "2.0", "id": 1, "method": "initialize"})
+        assert r.status_code == 200 and r.json()["result"]["serverInfo"]["name"].startswith("forge-")
+
+        # tools/list exposes the project's tools
+        r = await c.post(f"/v1/mcp/{pid}", json={"jsonrpc": "2.0", "id": 2, "method": "tools/list"})
+        names = [t["name"] for t in r.json()["result"]["tools"]]
+        assert "calculator" in names
+
+        # tools/call runs it
+        r = await c.post(f"/v1/mcp/{pid}", json={
+            "jsonrpc": "2.0", "id": 3, "method": "tools/call",
+            "params": {"name": "calculator", "arguments": {"expression": "6*7"}},
+        })
+        body = r.json()["result"]
+        assert body["isError"] is False and "42" in body["content"][0]["text"]
+
+
+async def test_mcp_exposes_project_tools():
+    """Project-level tools (workflow / knowledge / Q&A) are published on the base endpoint when
+    their project.config flags are set, and never on a per-set (toolset) endpoint."""
+    async with SessionLocal() as s:
+        proj = Project(tenant_id="t_mcps3", name="P3", slug="p3", config={
+            "mcp_expose_workflow": True, "mcp_workflow_tool_name": "run_it",
+            "mcp_expose_knowledge": True, "mcp_expose_faq": True,
+        })
+        s.add(proj)
+        await s.commit()
+        await s.refresh(proj)
+        pid = proj.id
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        # base endpoint: all three project tools are listed (custom workflow tool name honored)
+        r = await c.post(f"/v1/mcp/{pid}", json={"jsonrpc": "2.0", "id": 1, "method": "tools/list"})
+        names = [t["name"] for t in r.json()["result"]["tools"]]
+        assert "run_it" in names
+        assert "search_knowledge_base" in names
+        assert "lookup_faq" in names
+
+        # per-toolset endpoint: project tools are a whole-project surface, so none of them appear
+        r = await c.post(f"/v1/mcp/{pid}/toolset/general", json={"jsonrpc": "2.0", "id": 2, "method": "tools/list"})
+        names = [t["name"] for t in r.json()["result"]["tools"]]
+        assert not ({"run_it", "search_knowledge_base", "lookup_faq"} & set(names))
+
+
+async def test_mcp_project_tools_off_by_default():
+    """No flags => no project tools (unchanged surface for existing projects)."""
+    async with SessionLocal() as s:
+        proj = Project(tenant_id="t_mcps4", name="P4", slug="p4", config={})
+        s.add(proj)
+        await s.commit()
+        await s.refresh(proj)
+        pid = proj.id
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        r = await c.post(f"/v1/mcp/{pid}", json={"jsonrpc": "2.0", "id": 1, "method": "tools/list"})
+        names = [t["name"] for t in r.json()["result"]["tools"]]
+        assert not ({"run_workflow", "search_knowledge_base", "lookup_faq"} & set(names))
+
+
+async def test_mcp_requires_key_when_configured():
+    async with SessionLocal() as s:
+        proj = Project(tenant_id="t_mcps2", name="P2", slug="p2", config={"mcp_api_key": "secret-key"})
+        s.add(proj)
+        await s.commit()
+        await s.refresh(proj)
+        pid = proj.id
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        # no key -> 401
+        r = await c.post(f"/v1/mcp/{pid}", json={"jsonrpc": "2.0", "id": 1, "method": "initialize"})
+        assert r.status_code == 401
+        # correct key -> ok
+        r = await c.post(f"/v1/mcp/{pid}", headers={"Authorization": "Bearer secret-key"},
+                         json={"jsonrpc": "2.0", "id": 1, "method": "initialize"})
+        assert r.status_code == 200

apps/api/tests/test_mcp_streamable.py

@@ -0,0 +1,124 @@
+"""Forge-as-an-MCP-server over the Streamable-HTTP transport.
+
+The headline test drives the endpoint with the REAL `mcp` SDK client (the same protocol Claude
+Desktop / Cursor / VS Code speak), routed at the in-process ASGI app — proving a native client can
+initialize, list, and call tools with no `mcp-remote` proxy bridge. The rest pin the HTTP-level
+contract: a POST that accepts SSE gets a `text/event-stream` reply, a plain-JSON POST still gets the
+legacy JSON response, and the per-project auth applies to the streaming transport too.
+"""
+
+from __future__ import annotations
+
+import json
+
+import httpx
+
+from forge.db.base import SessionLocal
+from forge.main import create_app
+from forge.models import Project, Tool
+
+
+async def _seed_project_with_tool(tenant="t_stream", slug="mcp-stream", config=None) -> str:
+    from forge.services.tool_sets import ToolSetService
+
+    async with SessionLocal() as s:
+        proj = Project(tenant_id=tenant, name="Stream Proj", slug=slug, config=config or {})
+        s.add(proj)
+        await s.flush()
+        tool = Tool(tenant_id=tenant, project_id=proj.id, name="calculator", kind="builtin",
+                    config={"builtin": "calculator", "description": "Evaluate arithmetic."})
+        s.add(tool)
+        await s.commit()
+        await s.refresh(proj)
+        await s.refresh(tool)
+        await ToolSetService.create(s, tenant, proj.id, name="General", tool_ids=[tool.id])
+        return proj.id
+
+
+def _asgi_httpx_factory(app):
+    """An httpx client factory (the shape `streamablehttp_client` expects) that routes the MCP
+    client's real HTTP traffic through the in-process ASGI app instead of the network."""
+
+    def make(*, headers=None, timeout=None, auth=None, **_):
+        return httpx.AsyncClient(
+            transport=httpx.ASGITransport(app=app), base_url="http://test",
+            headers=headers, timeout=timeout, auth=auth,
+        )
+
+    return make
+
+
+async def test_streamable_real_mcp_client_end_to_end():
+    """A real MCP SDK client initializes, lists, and calls a tool over Streamable HTTP."""
+    from mcp.client.session import ClientSession
+    from mcp.client.streamable_http import streamablehttp_client
+
+    pid = await _seed_project_with_tool()
+    app = create_app()
+    url = f"http://test/v1/mcp/{pid}"
+
+    async with streamablehttp_client(url, httpx_client_factory=_asgi_httpx_factory(app)) as (read, write, _sid):
+        async with ClientSession(read, write) as session:
+            init = await session.initialize()
+            assert init.serverInfo.name.startswith("forge-")
+
+            tools = await session.list_tools()
+            assert "calculator" in [t.name for t in tools.tools]
+
+            res = await session.call_tool("calculator", {"expression": "6*7"})
+            assert res.isError is False
+            assert "42" in res.content[0].text
+
+
+def _parse_sse_json(body: str) -> dict:
+    """Pull the JSON-RPC payload out of a single-message `text/event-stream` response."""
+    for line in body.splitlines():
+        if line.startswith("data:"):
+            return json.loads(line[len("data:"):].strip())
+    raise AssertionError(f"no SSE data frame in response:\n{body}")
+
+
+async def test_streamable_post_negotiates_sse():
+    """A POST that accepts text/event-stream is answered with an SSE-framed JSON-RPC reply."""
+    pid = await _seed_project_with_tool(tenant="t_stream2", slug="mcp-stream2")
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        r = await c.post(
+            f"/v1/mcp/{pid}",
+            headers={"Accept": "application/json, text/event-stream"},
+            json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {
+                "protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "t", "version": "1"},
+            }},
+        )
+    assert r.status_code == 200
+    assert "text/event-stream" in r.headers["content-type"]
+    payload = _parse_sse_json(r.text)
+    assert payload["result"]["serverInfo"]["name"].startswith("forge-")
+
+
+async def test_plain_json_post_still_uses_legacy_json():
+    """A POST WITHOUT an SSE Accept stays on the legacy request/response path (application/json)."""
+    pid = await _seed_project_with_tool(tenant="t_stream3", slug="mcp-stream3")
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        r = await c.post(f"/v1/mcp/{pid}", json={"jsonrpc": "2.0", "id": 1, "method": "tools/list"})
+    assert r.status_code == 200
+    assert "application/json" in r.headers["content-type"]
+    assert "calculator" in [t["name"] for t in r.json()["result"]["tools"]]
+
+
+async def test_streamable_transport_enforces_project_key():
+    """The per-project mcp_api_key gates the streaming transport, not just the legacy path."""
+    pid = await _seed_project_with_tool(tenant="t_stream4", slug="mcp-stream4", config={"mcp_api_key": "sk-stream"})
+    app = create_app()
+    sse = {"Accept": "application/json, text/event-stream"}
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        init = {"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {
+            "protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "t", "version": "1"},
+        }}
+        # no key -> 401 even on the streamable transport
+        r = await c.post(f"/v1/mcp/{pid}", headers=sse, json=init)
+        assert r.status_code == 401
+        # correct key -> streamed 200
+        r = await c.post(f"/v1/mcp/{pid}", headers={**sse, "Authorization": "Bearer sk-stream"}, json=init)
+        assert r.status_code == 200 and "text/event-stream" in r.headers["content-type"]

apps/api/tests/test_memory.py

@@ -0,0 +1,37 @@
+"""Long-term memory: remember + semantic recall, and the builtin tools."""
+
+from __future__ import annotations
+
+from forge.db.base import SessionLocal
+from forge.services.memory import MemoryService
+from forge.services.runtime import make_runtime_ctx
+from forge.tools.materialize import materialize_tool
+
+T, P = "t_mem", "p_mem"
+
+
+async def test_remember_then_recall():
+    async with SessionLocal() as s:
+        await MemoryService.remember(s, T, P, "The customer prefers email over phone.")
+        await MemoryService.remember(s, T, P, "Our refund window is 30 days.")
+    async with SessionLocal() as s:
+        hits = await MemoryService.recall(s, T, P, "refund window days", top_k=5)
+    assert any("30 days" in h for h in hits)  # the refund memory is recalled
+
+
+async def test_scope_isolates_memories():
+    async with SessionLocal() as s:
+        await MemoryService.remember(s, "t_sc", "p_sc", "Alice's plan is enterprise.", scope="user:alice")
+        await MemoryService.remember(s, "t_sc", "p_sc", "Bob's plan is free.", scope="user:bob")
+    async with SessionLocal() as s:
+        alice = await MemoryService.recall(s, "t_sc", "p_sc", "what plan", scope="user:alice", top_k=5)
+    assert any("enterprise" in m for m in alice) and not any("free" in m for m in alice)
+
+
+async def test_memory_builtin_tools():
+    ctx = make_runtime_ctx(T, P)
+    remember = materialize_tool({"name": "remember", "kind": "builtin", "builtin": "remember"}, ctx)
+    recall = materialize_tool({"name": "recall", "kind": "builtin", "builtin": "recall"}, ctx)
+    await remember.ainvoke({"text": "The launch date is March 2027."})
+    out = await recall.ainvoke({"query": "when is the launch?"})
+    assert "March 2027" in out

apps/api/tests/test_model_catalog.py

@@ -0,0 +1,70 @@
+"""Model-catalog integrity.
+
+CHAT_MODELS is the single source of truth for the console's model picker, and the built-in
+pricing rates derive from it. These tests enforce the invariant that prevents cost tracking
+from silently reporting $0: every model a user can pick must be priced by the backend, and the
+model a blank ("Project default") node actually runs must be selectable in the UI.
+"""
+
+from __future__ import annotations
+
+from forge.engine.models import _PROVIDER_CHEAP_MODEL
+from forge.model_catalog import CHAT_MODELS, EMBEDDING_MODELS, RERANKER_MODELS, catalog_prices
+from forge.tracing.pricing import _resolve_rate, merged_prices
+
+
+def test_every_offered_model_is_priced():
+    # A model in the dropdown with no pricing entry would cost $0 at runtime (silent under-report).
+    for m in CHAT_MODELS:
+        if m.id.startswith("fake"):
+            continue
+        assert _resolve_rate(m.id) is not None, f"{m.id} is offered but has no pricing entry"
+
+
+def test_cheap_defaults_are_selectable():
+    # cheap_model_for_credentials picks these when a node leaves the model blank (e.g. the
+    # classifier's "Project default"); each must be in the catalog so the UI can show what runs.
+    ids = {m.id for m in CHAT_MODELS}
+    for default in _PROVIDER_CHEAP_MODEL.values():
+        assert default in ids, f"cheap default {default} is not in the model catalog"
+
+
+def test_no_duplicate_model_ids():
+    ids = [m.id for m in CHAT_MODELS]
+    assert len(ids) == len(set(ids)), "duplicate model id in CHAT_MODELS"
+
+
+def test_catalog_rates_are_the_ones_the_cost_engine_uses():
+    # The picker's rates must be the SAME table the tracer prices with - not a divergent copy.
+    prices = merged_prices()
+    for bare, rate in catalog_prices().items():
+        assert prices.get(bare) == rate, f"pricing for {bare} diverged from the catalog"
+
+
+def test_embedding_default_matches_backend():
+    # The picker's default embedder must be the one the backend actually falls back to.
+    from forge.knowledge.embeddings import _DEFAULT_FASTEMBED
+
+    defaults = [m for m in EMBEDDING_MODELS if m.default]
+    assert len(defaults) == 1, "exactly one default embedding model"
+    assert defaults[0].id.split(":", 1)[1] == _DEFAULT_FASTEMBED
+
+
+def test_reranker_default_matches_backend():
+    from forge.knowledge.rerank import DEFAULT_RERANKER
+
+    defaults = [m for m in RERANKER_MODELS if m.default]
+    assert len(defaults) == 1, "exactly one default reranker"
+    assert defaults[0].id == DEFAULT_RERANKER
+
+
+def test_billed_embeddings_are_priced():
+    # A billed embedder with no pricing entry would embed at $0 (silent cost under-report).
+    for m in EMBEDDING_MODELS:
+        if m.billed:
+            assert _resolve_rate(m.id) is not None, f"billed embedder {m.id} has no pricing entry"
+
+
+def test_no_duplicate_ids_across_catalogs():
+    all_ids = [m.id for m in CHAT_MODELS] + [m.id for m in EMBEDDING_MODELS] + [m.id for m in RERANKER_MODELS]
+    assert len(all_ids) == len(set(all_ids)), "duplicate id across model catalogs"

apps/api/tests/test_oauth.py

@@ -0,0 +1,127 @@
+"""3-legged OAuth (authorization_code) resolver + refresh + state-token tests."""
+
+from __future__ import annotations
+
+import time
+
+import httpx
+
+from forge.auth_providers.resolver import AuthResolver
+from forge.db.base import SessionLocal
+from forge.models import AuthProvider
+from forge.secrets.store import SecretStore
+from forge.security import create_state_token, decode_token
+
+_CFG = {
+    "kind": "oauth2_authorization_code",
+    "authorize_url": "https://idp.example/authorize",
+    "token_url": "https://idp.example/token",
+    "client_id_ref": "secret://proj/cid",
+    "client_secret_ref": "secret://proj/csec",
+}
+
+
+async def _store_bundle(tenant, project, ap_id, bundle):
+    async with SessionLocal() as s:
+        await SecretStore().write(s, tenant_id=tenant, project_id=project, name=f"oauth_token__{ap_id}", value=bundle, kind="oauth")
+
+
+def _ap(ap_id="ap_oauth", tenant="t_o", project="p_o"):
+    return AuthProvider(id=ap_id, tenant_id=tenant, project_id=project, name="idp", kind="oauth2_authorization_code", config=_CFG)
+
+
+async def test_oauth_resolves_valid_bundle():
+    await _store_bundle("t_o", "p_o", "ap_oauth", {"access_token": "TKN", "expires_at": time.time() + 3600})
+    resolved = await AuthResolver().resolve(tenant_id="t_o", project_id="p_o", provider_id="ap_oauth", provider=_ap(), force=True)
+    assert resolved.headers["Authorization"] == "Bearer TKN"
+
+
+async def test_oauth_refreshes_expired_token():
+    async with SessionLocal() as s:
+        await SecretStore().write(s, tenant_id="t_o2", project_id="p_o2", name="cid", value="client-id")
+        await SecretStore().write(s, tenant_id="t_o2", project_id="p_o2", name="csec", value="client-secret")
+    await _store_bundle("t_o2", "p_o2", "ap2", {"access_token": "OLD", "refresh_token": "R1", "expires_at": time.time() - 10})
+
+    seen = {}
+
+    def handler(req: httpx.Request) -> httpx.Response:
+        seen["body"] = req.content.decode()
+        return httpx.Response(200, json={"access_token": "NEW", "expires_in": 3600})
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    resolved = await AuthResolver().resolve(tenant_id="t_o2", project_id="p_o2", provider_id="ap2",
+                                            provider=_ap("ap2", "t_o2", "p_o2"), client=client, force=True)
+    await client.aclose()
+    assert resolved.headers["Authorization"] == "Bearer NEW"
+    assert "grant_type=refresh_token" in seen["body"]
+    # the refreshed bundle was persisted
+    new_bundle = await SecretStore().read_ref(tenant_id="t_o2", project_id="p_o2", ref="secret://proj/oauth_token__ap2")
+    assert new_bundle["access_token"] == "NEW"
+
+
+async def test_bearer_tolerates_stale_credentials_ref():
+    """Regression: a bearer provider whose own token_ref resolves must succeed even when the
+    legacy `credentials_ref` fallback points at a secret that no longer exists (the create flow
+    used to copy the template default there, then it rotted when the token_ref was edited)."""
+    async with SessionLocal() as s:
+        await SecretStore().write(s, tenant_id="t_b", project_id="p_b", name="git_token", value="ghp_real")
+    ap = AuthProvider(
+        id="ap_bearer", tenant_id="t_b", project_id="p_b", name="git", kind="bearer",
+        config={"kind": "bearer", "token_ref": "secret://proj/git_token", "header_name": "Authorization", "prefix": "Bearer "},
+        credentials_ref="secret://proj/token",  # stale - no secret named "token" exists
+    )
+    resolved = await AuthResolver().resolve(tenant_id="t_b", project_id="p_b", provider_id="ap_bearer", provider=ap, force=True)
+    assert resolved.headers["Authorization"] == "Bearer ghp_real"
+
+
+async def test_secret_usage_finds_references():
+    """The pre-delete guard must surface entities that reference a secret (here, an auth provider)."""
+    from forge.services.secrets import SecretService
+    async with SessionLocal() as s:
+        s.add(AuthProvider(
+            id="ap_use", tenant_id="t_u", project_id="p_u", name="shared", kind="bearer",
+            config={"kind": "bearer", "token_ref": "secret://proj/shared_key"},
+        ))
+        await s.commit()
+        refs = await SecretService.usage(s, "t_u", "p_u", name="shared_key")
+        none = await SecretService.usage(s, "t_u", "p_u", name="unreferenced")
+    assert any(r["type"] == "auth_provider" and r["label"] == "shared" for r in refs)
+    assert none == []
+
+
+async def test_oauth_state_token_roundtrip():
+    tok = create_state_token({"tid": "t", "pid": "p", "ap": "x"})
+    claims = decode_token(tok, expected_type="oauth_state")
+    assert claims["tid"] == "t" and claims["ap"] == "x"
+
+
+async def test_oauth_not_connected_raises():
+    import pytest
+    with pytest.raises((KeyError, Exception)):
+        await AuthResolver().resolve(tenant_id="t_none", project_id="p_none", provider_id="apx", provider=_ap("apx", "t_none", "p_none"), force=True)
+
+
+async def test_per_user_connect_bundle_is_resolvable():
+    """Item 5: the connect callback now stores the bundle under the SAME per-user secret name
+    that resolve/refresh read. Before the fix it wrote the default name, so a per-user provider's
+    token was invisible to resolve. This exercises the connect-time name -> resolve round trip."""
+    import pytest
+
+    cfg = {**_CFG, "per_user_context_keys": ["end_user.id"]}
+    ap = AuthProvider(id="ap_pu", tenant_id="t_pu", project_id="p_pu", name="idp",
+                      kind="oauth2_authorization_code", config=cfg)
+    ctx = {"end_user.id": "alice"}
+    # The per-user name the callback computes must differ from the default single-account name.
+    name = AuthResolver.bundle_secret_name("ap_pu", ctx, cfg["per_user_context_keys"])
+    assert name != AuthResolver.bundle_secret_name("ap_pu")
+    async with SessionLocal() as s:
+        await SecretStore().write(s, tenant_id="t_pu", project_id="p_pu", name=name,
+                                  value={"access_token": "ALICE", "expires_at": time.time() + 3600}, kind="oauth")
+    # Alice's context resolves to her token...
+    r = await AuthResolver().resolve(tenant_id="t_pu", project_id="p_pu", provider_id="ap_pu",
+                                     provider=ap, context=ctx, force=True)
+    assert r.headers["Authorization"] == "Bearer ALICE"
+    # ...a different end-user's context does not (proving the bundle is genuinely per-user).
+    with pytest.raises(KeyError):
+        await AuthResolver().resolve(tenant_id="t_pu", project_id="p_pu", provider_id="ap_pu",
+                                     provider=ap, context={"end_user.id": "bob"}, force=True)

apps/api/tests/test_otel.py

@@ -0,0 +1,46 @@
+"""OpenTelemetry export emits GenAI-semconv spans from run SpanRecords."""
+
+from __future__ import annotations
+
+from forge.tracing import otel
+from forge.tracing.tracer import SpanRecord
+
+
+def test_export_emits_spans_with_genai_attributes():
+    from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+    exporter = InMemorySpanExporter()
+    assert otel.configure(exporter=exporter) is True
+    assert otel.enabled() is True
+
+    records = [
+        SpanRecord(id="s1", parent_id=None, name="agent_1", kind="llm", start=1000.0, end=1000.5,
+                   start_wall=1_700_000_000.0, end_wall=1_700_000_000.5,
+                   model="openai:gpt-4o-mini", input_tokens=120, output_tokens=30, cost_usd=0.0001),
+        SpanRecord(id="s2", parent_id="s1", name="get_order", kind="tool", start=1000.5, end=1000.6,
+                   start_wall=1_700_000_000.5, end_wall=1_700_000_000.6, error="boom"),
+    ]
+    otel.export(records, trace_name="run")
+
+    spans = exporter.get_finished_spans()
+    # One root span ("run") now groups the child spans into a single trace with a real hierarchy.
+    assert len(spans) == 3
+    by_name = {s.name: s for s in spans}
+    root, llm, tool = by_name["run"], by_name["agent_1"], by_name["get_order"]
+    assert llm.attributes["gen_ai.request.model"] == "openai:gpt-4o-mini"
+    assert llm.attributes["gen_ai.system"] == "openai"
+    assert llm.attributes["gen_ai.usage.input_tokens"] == 120
+    assert tool.attributes["error"] is True
+    # Wall-clock start time (post-2020 in ns), NOT the ~1970 that exporting monotonic seconds gave.
+    assert llm.start_time > 1_600_000_000_000_000_000
+    # Single trace, correct parent/child nesting: run -> agent_1 -> get_order.
+    assert root.context.trace_id == llm.context.trace_id == tool.context.trace_id
+    assert root.parent is None
+    assert llm.parent.span_id == root.context.span_id
+    assert tool.parent.span_id == llm.context.span_id
+
+
+def test_export_noop_when_unconfigured():
+    # reset to unconfigured state
+    otel._tracer = None
+    otel.export([SpanRecord(id="x", parent_id=None, name="n", kind="node", start=0.0, end=1.0)])  # must not raise

apps/api/tests/test_per_user_token_auth.py

@@ -0,0 +1,164 @@
+"""Per-user bearer/api_key auth: a provider marked per-user (per_user_context_keys) injects EACH
+end user's OWN self-served token, with no shared secret and no passthrough of the inbound token.
+Companion to test_connected_credentials.py (which covers the oauth2_authorization_code path)."""
+
+from __future__ import annotations
+
+import uuid
+
+import httpx
+import pytest
+
+from forge.auth_providers.resolver import AuthResolver
+from forge.db.base import SessionLocal
+from forge.main import create_app
+from forge.services.auth_providers import AuthProviderService
+
+
+async def _provider(tenant: str, project: str, kind: str, extra: dict) -> str:
+    async with SessionLocal() as s:
+        ap = await AuthProviderService.create(
+            s, tenant, project, name="api", kind=kind,
+            config={"per_user_context_keys": ["end_user_id"], **extra},
+        )
+        return ap.id
+
+
+async def test_per_user_bearer_resolves_each_users_own_token():
+    tenant, project = "t_pu_b", "p_pu_b"
+    ap_id = await _provider(tenant, project, "bearer", {"header_name": "Authorization", "prefix": "Bearer "})
+    async with SessionLocal() as s:
+        ap = await AuthProviderService.get(s, tenant, ap_id)
+        await AuthProviderService.set_user_connection(s, tenant, project, ap, "user-A", bundle={"access_token": "PAT-A"})
+        await AuthProviderService.set_user_connection(s, tenant, project, ap, "user-B", bundle={"access_token": "PAT-B"})
+
+    resolver = AuthResolver()
+    ra = await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id, context={"end_user_id": "user-A"})
+    assert ra.headers["Authorization"] == "Bearer PAT-A"
+    rb = await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id, context={"end_user_id": "user-B"})
+    assert rb.headers["Authorization"] == "Bearer PAT-B"
+
+    # A user who hasn't connected their own token yet gets a clear "not connected" error, never
+    # a silent miss or another user's token.
+    with pytest.raises(KeyError):
+        await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id, context={"end_user_id": "user-C"})
+
+
+async def test_per_user_api_key_resolves_each_users_own_value():
+    tenant, project = "t_pu_k", "p_pu_k"
+    ap_id = await _provider(tenant, project, "api_key", {"in": "header", "name": "X-API-Key"})
+    async with SessionLocal() as s:
+        ap = await AuthProviderService.get(s, tenant, ap_id)
+        await AuthProviderService.set_user_connection(s, tenant, project, ap, "u1", bundle={"access_token": "KEY-1"})
+
+    resolver = AuthResolver()
+    ra = await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id, context={"end_user_id": "u1"})
+    assert ra.headers["X-API-Key"] == "KEY-1"
+    with pytest.raises(KeyError):
+        await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id, context={"end_user_id": "u2"})
+
+
+async def test_per_user_bearer_inline_token_wins_and_no_cache_collision():
+    """A per-user provider with token_ctx_key forwards an INLINE token from the run context (the
+    web-chat /run path) ahead of the stored connection, and different inline tokens sharing the same
+    end_user dims must not collide in the resolver cache."""
+    tenant, project = "t_pu_i", "p_pu_i"
+    ap_id = await _provider(tenant, project, "bearer",
+                            {"header_name": "Authorization", "prefix": "Bearer ", "token_ctx_key": "user_pat"})
+    resolver = AuthResolver()
+    ra = await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id,
+                                context={"end_user_id": "anon", "user_pat": "INLINE-A"})
+    assert ra.headers["Authorization"] == "Bearer INLINE-A"
+    # Different inline token, same end_user dims -> must NOT be served the cached "INLINE-A".
+    rb = await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id,
+                                context={"end_user_id": "anon", "user_pat": "INLINE-B"})
+    assert rb.headers["Authorization"] == "Bearer INLINE-B"
+    # No inline token -> falls back to the stored per-user connection (MCP/console path).
+    async with SessionLocal() as s:
+        ap = await AuthProviderService.get(s, tenant, ap_id)
+        await AuthProviderService.set_user_connection(s, tenant, project, ap, "stored-user", bundle={"access_token": "STORED"})
+    rc = await resolver.resolve(tenant_id=tenant, project_id=project, provider_id=ap_id,
+                                context={"end_user_id": "stored-user"})
+    assert rc.headers["Authorization"] == "Bearer STORED"
+
+
+async def test_extra_headers_literal_and_secret_ref():
+    """A provider's extra_headers are stamped on every call: literals verbatim, secret:// refs
+    resolved from the secret store — so a shared service token lives in the store, never hardcoded."""
+    from forge.secrets.store import SecretStore
+    tenant, project = "t_eh", "p_eh"
+    async with SessionLocal() as s:
+        await SecretStore().write(s, tenant_id=tenant, project_id=project, name="primary", value="PRIMARY")
+        await SecretStore().write(s, tenant_id=tenant, project_id=project, name="svc_tok", value="SVC-123")
+        ap = await AuthProviderService.create(
+            s, tenant, project, name="q", kind="bearer",
+            config={"kind": "bearer", "token_ref": "secret://proj/primary", "header_name": "Authorization",
+                    "prefix": "Bearer ", "extra_headers": {"X-Forge-Client-Id": "forge",
+                                                           "X-Forge-Service-Token": "secret://proj/svc_tok"}},
+        )
+        ap_id = ap.id
+
+    r = await AuthResolver().resolve(tenant_id=tenant, project_id=project, provider_id=ap_id)
+    assert r.headers["Authorization"] == "Bearer PRIMARY"
+    assert r.headers["X-Forge-Client-Id"] == "forge"           # literal, verbatim
+    assert r.headers["X-Forge-Service-Token"] == "SVC-123"     # resolved from the secret store
+
+
+def _http() -> httpx.AsyncClient:
+    return httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app()), base_url="http://test")
+
+
+async def test_connections_router_self_service_end_to_end():
+    """The connector-safe /connections router is served (not the /auth-providers admin surface) and
+    a logged-in user can discover per-user providers + set/read/clear THEIR OWN token. Role is not
+    gated here, so a connector uses the identical path; resolver injection keyed by end_user_id is
+    covered by the unit tests above."""
+    async with _http() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": f"pu{uuid.uuid4().hex[:8]}@ex.com", "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        pid = (await c.post("/v1/projects", json={"name": "PU"}, headers=h)).json()["id"]
+
+        # Owner creates a PER-USER bearer provider (no shared secret; each user brings their own).
+        ap = (await c.post(f"/v1/projects/{pid}/auth-providers", json={
+            "name": "api", "kind": "bearer",
+            "config": {"kind": "bearer", "header_name": "Authorization", "prefix": "Bearer ",
+                       "per_user_context_keys": ["end_user_id"]},
+        }, headers=h)).json()
+        ap_id = ap["id"]
+
+        # Discovery lists it as not-yet-connected for the caller.
+        lst = (await c.get(f"/v1/projects/{pid}/connections", headers=h)).json()
+        assert [x for x in lst if x["id"] == ap_id and x["connected"] is False], lst
+
+        # Set my own token -> 204, reads back connected.
+        r = await c.put(f"/v1/projects/{pid}/connections/{ap_id}", json={"access_token": "MY-TOKEN"}, headers=h)
+        assert r.status_code == 204, r.text
+        assert (await c.get(f"/v1/projects/{pid}/connections/{ap_id}", headers=h)).json()["connected"] is True
+        lst2 = (await c.get(f"/v1/projects/{pid}/connections", headers=h)).json()
+        assert [x for x in lst2 if x["id"] == ap_id and x["connected"] is True]
+
+        # The provider's own /test now resolves the CALLER's connected token: console tests run AS
+        # the current user, so a per-user provider no longer reports "not connected" for the tester.
+        t = (await c.post(f"/v1/projects/{pid}/auth-providers/{ap_id}/test", json={}, headers=h)).json()
+        assert t.get("ok") is True, t
+        assert "Authorization" in (t.get("headers") or {}), t
+
+        # Clear -> not connected again.
+        assert (await c.delete(f"/v1/projects/{pid}/connections/{ap_id}", headers=h)).status_code == 204
+        assert (await c.get(f"/v1/projects/{pid}/connections/{ap_id}", headers=h)).json()["connected"] is False
+
+
+async def test_connections_rejects_non_per_user_provider():
+    """A shared (non-per-user) provider can't take a per-user token via /connections."""
+    async with _http() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": f"pu{uuid.uuid4().hex[:8]}@ex.com", "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        pid = (await c.post("/v1/projects", json={"name": "PU2"}, headers=h)).json()["id"]
+        ap = (await c.post(f"/v1/projects/{pid}/auth-providers", json={
+            "name": "shared", "kind": "bearer",
+            "config": {"kind": "bearer", "token_ref": "secret://proj/token"},
+        }, headers=h)).json()
+        r = await c.put(f"/v1/projects/{pid}/connections/{ap['id']}", json={"access_token": "X"}, headers=h)
+        assert r.status_code == 400, r.text
+        # And it never appears in the connector's per-user discovery list.
+        assert all(x["id"] != ap["id"] for x in (await c.get(f"/v1/projects/{pid}/connections", headers=h)).json())

apps/api/tests/test_pgvector_store.py

@@ -0,0 +1,84 @@
+"""Unit tests for the pgvector backend's selection + Chroma-where translation.
+
+No live Postgres here: these cover the pure, bug-prone logic (Chroma where-dict -> SQL
+predicate, vector literal encoding, sync-DSN derivation) and the make_store() backend switch.
+The dense/hybrid search SQL shares its fusion + where semantics with the Chroma path that the
+other knowledge tests already exercise end to end.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from forge.config import settings
+from forge.knowledge import store as store_mod
+from forge.knowledge.pgvector_store import (
+    PgVectorStore,
+    _sync_dsn,
+    _to_vector_literal,
+    _translate_where,
+)
+from forge.knowledge.store import make_store
+
+
+def test_translate_where_and_eq():
+    params: list = []
+    sql = _translate_where({"$and": [{"tenant_id": {"$eq": "t1"}}, {"project_id": {"$eq": "p1"}}]}, params)
+    assert "metadata->>%s" in sql and " AND " in sql
+    # field names AND values are bound positionally, never string-formatted
+    assert params == ["tenant_id", "t1", "project_id", "p1"]
+
+
+def test_translate_where_in_and_shorthand():
+    params: list = []
+    sql = _translate_where({"kind": {"$in": ["faq", "hours"]}}, params)
+    assert "= ANY(%s)" in sql
+    assert params == ["kind", ["faq", "hours"]]
+    # {field: value} shorthand behaves as $eq
+    params2: list = []
+    _translate_where({"source_id": "s1"}, params2)
+    assert params2 == ["source_id", "s1"]
+
+
+def test_translate_where_empty_is_true():
+    assert _translate_where(None, []) == "TRUE"
+    assert _translate_where({}, []) == "TRUE"
+
+
+def test_translate_where_rejects_unknown_operator():
+    # fail closed: an operator we don't model must never silently drop the filter
+    with pytest.raises(ValueError):
+        _translate_where({"score": {"$gt": 0.5}}, [])
+
+
+def test_vector_literal_encoding():
+    assert _to_vector_literal([0.0, 1.5, -2.0]) == "[0.0,1.5,-2.0]"
+
+
+def test_sync_dsn_strips_async_driver(monkeypatch):
+    monkeypatch.setattr(settings, "database_url", "postgresql+asyncpg://u:p@h:5432/db", raising=False)
+    assert _sync_dsn() == "postgresql://u:p@h:5432/db"
+
+
+def test_sync_dsn_rejects_sqlite(monkeypatch):
+    monkeypatch.setattr(settings, "database_url", "sqlite+aiosqlite:///x.db", raising=False)
+    with pytest.raises(RuntimeError):
+        _sync_dsn()
+
+
+def test_make_store_defaults_to_chroma(monkeypatch):
+    # default backend selects Chroma; stub the ctor so no real on-disk client is built
+    monkeypatch.setattr(settings, "vector_backend", "chroma", raising=False)
+    marker = object()
+    monkeypatch.setattr(store_mod, "ChromaStore", lambda collection="forge_kb": marker)
+    assert make_store(collection="forge_kb_8") is marker
+
+
+def test_make_store_selects_pgvector(monkeypatch):
+    monkeypatch.setattr(settings, "vector_backend", "pgvector", raising=False)
+    monkeypatch.setattr(settings, "database_url", "postgresql+asyncpg://u:p@h/db", raising=False)
+    monkeypatch.setattr(PgVectorStore, "_ensure_schema", lambda self: None)  # skip the DB bootstrap
+    st = make_store(collection="forge_kb_16")
+    assert isinstance(st, PgVectorStore)
+    assert st._collection == "forge_kb_16"
+    assert st._dsn == "postgresql://u:p@h/db"

apps/api/tests/test_platform_hardening.py

@@ -0,0 +1,420 @@
+"""Platform-hardening tests (findings a-k): auth lifecycle, RBAC/API-keys, rate limiting,
+audit pagination/export, project budgets, retention, OAuth PKCE, and ops guards.
+
+In-process ASGI. Each test resets the shared in-process rate limiter + revocation state so the
+process-wide singletons can't leak between tests.
+"""
+
+from __future__ import annotations
+
+import json as _json
+import time
+import uuid
+
+import httpx
+import pytest
+
+from forge.config import settings
+from forge.main import create_app
+
+
+def _client() -> httpx.AsyncClient:
+    return httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app()), base_url="http://test")
+
+
+def _email() -> str:
+    return f"u{uuid.uuid4().hex[:10]}@example.com"
+
+
+@pytest.fixture(autouse=True)
+def _reset_platform_state():
+    from forge.security import _revocations
+    from forge.util.ratelimit import rate_limiter
+
+    try:
+        rate_limiter._local._buckets.clear()
+    except Exception:
+        pass
+    _revocations._jti.clear()
+    _revocations._user_cut.clear()
+    yield
+
+
+# --- c: production hardening guard ------------------------------------------------------
+
+def test_production_guard_flags_new_gaps():
+    from forge.config import Settings
+
+    s = Settings()
+    s.environment = "production"
+    s.jwt_secret = "x" * 40
+    s.auth_required = True
+    s.bootstrap_admin_password = "a-strong-password"
+    s.egress_block_private = True
+    s.database_url = "postgresql+asyncpg://u:p@db/forge"
+    s.checkpoint_backend = "postgres"
+    s.trusted_hosts = []                       # -> flagged
+    s.public_base_url = "http://forge.example.com"   # http -> flagged
+    s.public_console_url = "https://app.example.com"
+    s.service_api_token = "tooshort"           # < min length -> flagged
+    problems = s.validate_production()
+    assert any("TRUSTED_HOSTS" in p for p in problems)
+    assert any("PUBLIC_BASE_URL" in p for p in problems)
+    assert any("SERVICE_API_TOKEN" in p for p in problems)
+
+    # Fixing them clears exactly those problems.
+    s.trusted_hosts = ["forge.example.com"]
+    s.public_base_url = "https://forge.example.com"
+    s.service_api_token = ""  # empty = disabled, allowed
+    cleared = s.validate_production()
+    assert not any(("TRUSTED_HOSTS" in p or "PUBLIC_BASE_URL" in p or "SERVICE_API_TOKEN" in p) for p in cleared)
+
+
+def test_multi_worker_without_redis_warns():
+    from forge.config import Settings
+
+    s = Settings()
+    s.web_concurrency = 4
+    s.redis_url = None
+    assert any("Multiple workers" in w for w in s.startup_warnings())
+
+
+# --- b: rate limiter pruning + fail-closed public surface -------------------------------
+
+def test_inprocess_bucket_prunes_idle_keys():
+    import forge.util.ratelimit as rl
+
+    limiter = rl.RateLimiter()
+    limiter.allow("old", rate=100)
+    limiter._buckets["old"].updated -= (rl._BUCKET_IDLE_TTL + 10)   # look idle
+    limiter._last_prune -= (rl._BUCKET_PRUNE_EVERY + 10)            # allow a sweep
+    limiter.allow("new", rate=100)                                  # triggers prune
+    assert "old" not in limiter._buckets and "new" in limiter._buckets
+
+
+def test_public_surface_fails_closed_when_redis_unavailable():
+    from forge.util.ratelimit import ResilientRateLimiter, _RedisConn
+
+    class _Down(_RedisConn):
+        def __init__(self):
+            super().__init__("redis://unreachable")
+
+        def get(self):
+            return None  # configured but never connects
+
+    limiter = ResilientRateLimiter(_Down())
+    assert limiter.allow("embed:key", rate=5) is False       # public -> DENY (fail closed)
+    assert limiter.allow("runs:tenant", rate=5) is True      # non-public -> in-process fallback
+
+
+def test_public_surface_fails_closed_on_redis_error():
+    from forge.util.ratelimit import ResilientRateLimiter, _RedisConn
+
+    class _Broken:
+        def pipeline(self):
+            raise RuntimeError("redis down")
+
+    class _Conn(_RedisConn):
+        def __init__(self):
+            super().__init__("redis://x")
+            self._client = _Broken()
+
+        def get(self):
+            return self._client
+
+    limiter = ResilientRateLimiter(_Conn())
+    assert limiter.allow("embed:abc", rate=5) is False
+
+
+# --- f: project budgets + allowed-models ------------------------------------------------
+
+async def test_project_budget_and_allowed_models():
+    from forge.db.base import SessionLocal
+    from forge.models import Project, Run
+    from forge.services.budget import BudgetExceeded, ModelNotAllowed, enforce_project_budget
+
+    async with SessionLocal() as s:
+        p = Project(tenant_id="tb", name="B", slug="b", config={
+            "allowed_models": ["openai:gpt-4o"],
+            "budgets": {"monthly_usd_cap": 1.0, "max_usd_per_run": 0.0},
+        })
+        s.add(p)
+        await s.commit()
+        pid = p.id
+
+    async with SessionLocal() as s:
+        with pytest.raises(ModelNotAllowed):
+            await enforce_project_budget(s, "tb", pid, model="anthropic:claude")
+        await enforce_project_budget(s, "tb", pid, model="openai:gpt-4o")  # allowed, no spend yet
+
+    async with SessionLocal() as s:
+        s.add(Run(tenant_id="tb", project_id=pid, workflow_id="w", thread_id="t",
+                  status="done", total_cost_usd=1.5))
+        await s.commit()
+    async with SessionLocal() as s:
+        with pytest.raises(BudgetExceeded):
+            await enforce_project_budget(s, "tb", pid, model="openai:gpt-4o")
+
+
+def test_disallowed_workflow_models_at_publish():
+    """Per-node allowed_models validation (item 6): every chat model in a workflow's nodes is
+    checked at publish, mirroring the admission-time single-model check across all nodes."""
+    from forge.services.budget import collect_workflow_models, disallowed_workflow_models
+
+    executable = {
+        "nodes": [
+            {"id": "a", "type": "agent", "config": {"model": "openai:gpt-4o", "middleware": [
+                {"kind": "model_fallback", "config": {"models": ["anthropic:claude", "openai:gpt-4o"]}},
+            ]}},
+            {"id": "l", "type": "llm", "config": {"model": "openai:gpt-4o"}},
+            {"id": "r", "type": "retrieval", "config": {"embedding_model": "fastembed:bge"}},
+            {"id": "e", "type": "end", "config": {}},
+        ]
+    }
+    # agent/llm/classifier + nested middleware models are collected; the embedder is NOT.
+    assert collect_workflow_models(executable) == {"openai:gpt-4o", "anthropic:claude"}
+    # no allow-list => no-op (publish always allowed)
+    assert disallowed_workflow_models({}, executable) == []
+    # allow-list forbids the fallback's anthropic model
+    assert disallowed_workflow_models({"allowed_models": ["openai:gpt-4o"]}, executable) == ["anthropic:claude"]
+    # a fully-covered allow-list passes
+    assert disallowed_workflow_models({"allowed_models": ["openai:gpt-4o", "anthropic:claude"]}, executable) == []
+
+
+# --- a: auth endpoint throttling --------------------------------------------------------
+
+async def test_login_is_throttled_per_email(monkeypatch):
+    monkeypatch.setattr(settings, "auth_rate_limit_per_minute", 3)
+    async with _client() as c:
+        email = _email()
+        await c.post("/v1/auth/register", json={"email": email, "password": "supersecret1"})
+        codes = [
+            (await c.post("/v1/auth/login", json={"email": email, "password": "wrong"})).status_code
+            for _ in range(6)
+        ]
+        assert 429 in codes
+
+
+# --- d: refresh rotation, reuse detection, logout-all -----------------------------------
+
+async def test_refresh_rotates_and_detects_reuse():
+    async with _client() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        old_rt = reg["refresh_token"]
+        r1 = await c.post("/v1/auth/refresh", json={"refresh_token": old_rt})
+        assert r1.status_code == 200
+        new_rt = r1.json()["refresh_token"]
+        assert new_rt != old_rt
+        # Reusing the rotated (old) token is detected -> 401 and the whole family is revoked.
+        assert (await c.post("/v1/auth/refresh", json={"refresh_token": old_rt})).status_code == 401
+        assert (await c.post("/v1/auth/refresh", json={"refresh_token": new_rt})).status_code == 401
+
+
+async def test_logout_all_invalidates_existing_access_token():
+    async with _client() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        assert (await c.get("/v1/auth/me", headers=h)).status_code == 200
+        assert (await c.post("/v1/auth/logout-all", headers=h)).status_code == 200
+        assert (await c.get("/v1/auth/me", headers=h)).status_code == 401
+
+
+async def test_logout_revokes_refresh_token():
+    async with _client() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        rt = reg["refresh_token"]
+        assert (await c.post("/v1/auth/logout", json={"refresh_token": rt})).status_code == 200
+        assert (await c.post("/v1/auth/refresh", json={"refresh_token": rt})).status_code == 401
+
+
+# --- h: API keys + per-project RBAC -----------------------------------------------------
+
+async def test_api_key_lifecycle():
+    async with _client() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        created = await c.post("/v1/api-keys", json={"name": "ci", "role": "editor"}, headers=h)
+        assert created.status_code == 201, created.text
+        key, key_id = created.json()["key"], created.json()["id"]
+        assert key.startswith("forge_sk_")
+
+        kh = {"Authorization": f"Bearer {key}"}
+        assert (await c.get("/v1/projects", headers=kh)).status_code == 200
+        me = await c.get("/v1/auth/me", headers=kh)
+        assert me.status_code == 200 and me.json()["role"] == "editor"
+
+        assert (await c.delete(f"/v1/api-keys/{key_id}", headers=h)).status_code == 204
+        assert (await c.get("/v1/projects", headers=kh)).status_code == 401
+
+
+async def test_api_key_cannot_exceed_creator_role():
+    async with _client() as c:
+        # invite an editor, then that editor tries to mint an owner key
+        owner = (await c.post("/v1/auth/register", json={"email": _email(), "password": "ownerpass1"})).json()
+        oh = {"Authorization": f"Bearer {owner['access_token']}"}
+        ed_email = _email()
+        await c.post("/v1/team/members", json={"email": ed_email, "role": "admin", "password": "adminpass1"}, headers=oh)
+        ed = (await c.post("/v1/auth/login", json={"email": ed_email, "password": "adminpass1"})).json()
+        eh = {"Authorization": f"Bearer {ed['access_token']}"}
+        assert (await c.post("/v1/api-keys", json={"name": "x", "role": "owner"}, headers=eh)).status_code == 403
+
+
+async def test_per_project_membership_elevates_role():
+    async with _client() as c:
+        owner = (await c.post("/v1/auth/register", json={"email": _email(), "password": "ownerpass1"})).json()
+        oh = {"Authorization": f"Bearer {owner['access_token']}"}
+        member_email = _email()
+        inv = await c.post("/v1/team/members",
+                           json={"email": member_email, "role": "viewer", "password": "viewerpass1"}, headers=oh)
+        member_id = inv.json()["id"]
+        member = (await c.post("/v1/auth/login", json={"email": member_email, "password": "viewerpass1"})).json()
+        mh = {"Authorization": f"Bearer {member['access_token']}"}
+
+        pid = (await c.post("/v1/projects", json={"name": "P"}, headers=oh)).json()["id"]
+        # Global viewer can't PATCH (admin-gated) the project...
+        assert (await c.patch(f"/v1/projects/{pid}", json={"name": "X"}, headers=mh)).status_code == 403
+        # ...until granted admin ON THIS PROJECT.
+        assert (await c.put(f"/v1/projects/{pid}/members/{member_id}", json={"role": "admin"}, headers=oh)).status_code == 200
+        assert (await c.patch(f"/v1/projects/{pid}", json={"name": "Y"}, headers=mh)).status_code == 200
+
+
+# --- g: audit pagination, filters, export ----------------------------------------------
+
+async def test_audit_pagination_filter_and_export():
+    async with _client() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        for i in range(3):
+            await c.post("/v1/projects", json={"name": f"P{i}"}, headers=h)
+
+        p1 = await c.get("/v1/audit?limit=2", headers=h)
+        assert p1.status_code == 200 and len(p1.json()) == 2
+        cursor = p1.headers.get("X-Next-Cursor")
+        assert cursor
+        p2 = await c.get(f"/v1/audit?limit=2&cursor={cursor}", headers=h)
+        assert p2.status_code == 200 and len(p2.json()) >= 1
+        assert {a["id"] for a in p1.json()}.isdisjoint({a["id"] for a in p2.json()})
+
+        filtered = await c.get("/v1/audit", params={"action": "POST /v1/projects"}, headers=h)
+        rows = filtered.json()
+        assert len(rows) >= 3 and all(a["action"] == "POST /v1/projects" for a in rows)
+
+        export = await c.get("/v1/audit/export", headers=h)
+        assert export.status_code == 200
+        lines = [ln for ln in export.text.splitlines() if ln.strip()]
+        assert len(lines) >= 3 and all("action" in _json.loads(ln) for ln in lines)
+
+
+# --- j: password reset, email verification, TOTP MFA -----------------------------------
+
+async def test_password_reset_flow():
+    async with _client() as c:
+        email = _email()
+        await c.post("/v1/auth/register", json={"email": email, "password": "origpass1"})
+        rr = await c.post("/v1/auth/request-password-reset", json={"email": email})
+        assert rr.status_code == 200
+        url = rr.json().get("reset_url")   # no SMTP in tests -> link returned
+        assert url and "reset=" in url
+        token = url.split("reset=", 1)[1]
+        assert (await c.post("/v1/auth/reset-password", json={"token": token, "password": "newpass123"})).status_code == 200
+        assert (await c.post("/v1/auth/login", json={"email": email, "password": "origpass1"})).status_code == 401
+        assert (await c.post("/v1/auth/login", json={"email": email, "password": "newpass123"})).status_code == 200
+
+
+async def test_email_verification_flow():
+    async with _client() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        rv = await c.post("/v1/auth/request-email-verification", headers=h)
+        assert rv.status_code == 200
+        url = rv.json().get("verify_url")
+        assert url and "verify_email=" in url
+        token = url.split("verify_email=", 1)[1]
+        assert (await c.post("/v1/auth/verify-email", json={"token": token})).status_code == 200
+
+
+async def test_totp_enroll_confirm_and_enforced_at_login():
+    from forge.security import _totp_at
+
+    async with _client() as c:
+        email = _email()
+        reg = (await c.post("/v1/auth/register", json={"email": email, "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        secret = (await c.post("/v1/auth/mfa/totp/enroll", headers=h)).json()["secret"]
+        code = _totp_at(secret, int(time.time() // 30))
+        cf = await c.post("/v1/auth/mfa/totp/confirm", json={"code": code}, headers=h)
+        assert cf.status_code == 200 and cf.json()["mfa_enabled"] is True
+        # login now requires a valid code
+        assert (await c.post("/v1/auth/login", json={"email": email, "password": "supersecret1"})).status_code == 401
+        ok = await c.post("/v1/auth/login", json={
+            "email": email, "password": "supersecret1", "totp_code": _totp_at(secret, int(time.time() // 30))})
+        assert ok.status_code == 200
+
+
+# --- k: workspace admin + readiness -----------------------------------------------------
+
+async def test_workspace_get_and_update():
+    async with _client() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        assert (await c.get("/v1/workspace", headers=h)).status_code == 200
+        u = await c.patch("/v1/workspace", json={"name": "Renamed WS", "settings": {"max_runs_per_day": 5}}, headers=h)
+        assert u.status_code == 200
+        body = u.json()
+        assert body["name"] == "Renamed WS" and body["settings"]["max_runs_per_day"] == 5
+
+
+async def test_readyz_reports_dependency_checks():
+    async with _client() as c:
+        body = (await c.get("/readyz")).json()
+        assert "checks" in body
+        assert "db" in body["checks"] and "checkpointer" in body["checks"] and "vector_store" in body["checks"]
+
+
+async def test_global_rate_limit_middleware(monkeypatch):
+    monkeypatch.setattr(settings, "api_rate_limit_per_minute", 3)
+    async with _client() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        from forge.util.ratelimit import rate_limiter
+        rate_limiter._local._buckets.clear()  # isolate the GET burst from the register POST
+        codes = [(await c.get("/v1/auth/me", headers=h)).status_code for _ in range(6)]
+        assert codes.count(200) == 3 and 429 in codes  # burst of 3 (rate), then throttled
+
+
+# --- e: scheduled retention purge -------------------------------------------------------
+
+async def test_retention_purges_past_horizon():
+    from datetime import datetime, timedelta
+
+    from forge.db.base import SessionLocal
+    from forge.models import Project, Run, Span, Trace
+    from forge.services.retention import RetentionService
+
+    async with SessionLocal() as s:
+        p = Project(tenant_id="tret", name="R", slug="r", config={"tracing": {"retention_days": 7}})
+        s.add(p)
+        await s.flush()
+        pid = p.id
+        old = datetime.utcnow() - timedelta(days=30)
+        tr = Trace(tenant_id="tret", project_id=pid, run_id="r1", name="t", status="done")
+        old_run = Run(tenant_id="tret", project_id=pid, workflow_id="w", thread_id="th", status="done")
+        recent_run = Run(tenant_id="tret", project_id=pid, workflow_id="w", thread_id="th2", status="done")
+        s.add_all([tr, old_run, recent_run])
+        await s.flush()
+        sp = Span(tenant_id="tret", trace_id=tr.id, name="s", kind="node")
+        s.add(sp)
+        await s.flush()
+        tr.created_at = old
+        old_run.created_at = old
+        await s.commit()
+        trace_id, span_id, old_run_id, recent_run_id = tr.id, sp.id, old_run.id, recent_run.id
+
+    counts = await RetentionService.purge_expired()
+    assert counts["traces"] >= 1 and counts["runs"] >= 1 and counts["spans"] >= 1
+
+    async with SessionLocal() as s:
+        assert await s.get(Trace, trace_id) is None       # aged out
+        assert await s.get(Span, span_id) is None          # its span too
+        assert await s.get(Run, old_run_id) is None
+        assert await s.get(Run, recent_run_id) is not None  # within horizon -> kept

apps/api/tests/test_portability.py

@@ -0,0 +1,285 @@
+"""Import / export of tools, workflows, components, and agents (PortabilityService).
+
+Covers the guarantees the feature promises: a faithful round-trip (every authored field
+survives), fresh ids on import, auto-rename that never overwrites, intra-bundle id remap
+(a workflow's subworkflow ref follows the new ids), tool auth-provider resolution against
+the target project, runtime-junk stripping, and bundle-type validation.
+"""
+
+from __future__ import annotations
+
+import types
+import uuid
+
+import httpx
+import pytest
+
+from forge.db.base import SessionLocal
+from forge.main import create_app
+from forge.models import AuthProvider, Component
+from forge.services.agents import AgentService
+from forge.services.components import ComponentService
+from forge.services.portability import PortabilityService
+from forge.services.projects import ProjectService
+from forge.services.tools import ToolService
+from forge.services.versions import VersionService
+from forge.services.workflows import WorkflowService
+
+AUTHOR = types.SimpleNamespace(id="u_importer", email="importer@forge.local")
+
+
+async def _project(session, tenant_id, slug):
+    return await ProjectService.create(session, tenant_id, name=slug.title(), slug=slug)
+
+
+async def test_tool_export_import_roundtrip_strips_runtime_and_clears_missing_auth():
+    tenant = "t_tool_rt"
+    async with SessionLocal() as session:
+        src = await _project(session, tenant, "src")
+        dst = await _project(session, tenant, "dst")
+        ap = AuthProvider(tenant_id=tenant, project_id=src.id, name="bearer", kind="bearer", config={})
+        session.add(ap)
+        await session.commit()
+        cfg = {
+            "description": "call the widget API",
+            "request": {"method": "GET", "url_template": "https://api.example.com/widgets"},
+            "response": {"projection_jmespath": "data"},
+            "_last_test": {"status": 200, "raw": "junk"},  # runtime state; must not export
+        }
+        tool = await ToolService.create(
+            session, tenant, src.id, name="widget_api", kind="rest_api", config=cfg, auth_provider_id=ap.id
+        )
+
+        bundle = await PortabilityService.export(session, tenant, src.id, "tool", [tool.id])
+        assert bundle["type"] == "tool"
+        assert bundle["source"]["project_name"] == "Src"
+        assert len(bundle["items"]) == 1
+        item = bundle["items"][0]
+        assert item["name"] == "widget_api"
+        assert "_last_test" not in item["config"]  # runtime junk stripped
+        assert item["config"]["request"]["url_template"] == "https://api.example.com/widgets"
+        assert item["auth_provider_id"] == ap.id
+
+        # Import into a DIFFERENT project that has no such auth provider.
+        report = await PortabilityService.import_bundle(session, tenant, dst.id, bundle, author=AUTHOR)
+        assert report["imported"] == 1
+        assert report["items"][0]["renamed"] is False
+        assert any("auth provider" in w for w in report["warnings"])
+
+        # Filter out the auto-provisioned platform built-ins (never part of a bundle).
+        imported = [t for t in await ToolService.list(session, tenant, dst.id) if t.kind != "builtin"]
+        assert len(imported) == 1
+        it = imported[0]
+        assert it.id != tool.id  # fresh id
+        assert it.name == "widget_api"
+        assert it.config["request"]["url_template"] == "https://api.example.com/widgets"
+        assert "_last_test" not in it.config
+        assert it.auth_provider_id is None  # cleared: provider not in target project
+
+        # A version-history snapshot was recorded (imported == created + saved).
+        versions = await VersionService.list(session, tenant, "tool", it.id)
+        assert len(versions) >= 1
+
+
+async def test_tool_auth_provider_kept_when_present_in_target():
+    tenant = "t_tool_ap"
+    async with SessionLocal() as session:
+        src = await _project(session, tenant, "src")
+        dst = await _project(session, tenant, "dst")
+        # Same provider id must exist in the target for it to survive; simulate a same-project
+        # re-import by importing back into src (its provider exists).
+        ap = AuthProvider(tenant_id=tenant, project_id=src.id, name="bearer", kind="bearer", config={})
+        session.add(ap)
+        await session.commit()
+        tool = await ToolService.create(
+            session, tenant, src.id, name="api", kind="rest_api",
+            config={"description": "x", "request": {"method": "GET", "url_template": "https://x.test"}},
+            auth_provider_id=ap.id,
+        )
+        bundle = await PortabilityService.export(session, tenant, src.id, "tool", [tool.id])
+        report = await PortabilityService.import_bundle(session, tenant, src.id, bundle, author=AUTHOR)
+        assert report["imported"] == 1
+        # Re-imported into src: renamed (name clash) but auth kept.
+        new_id = report["items"][0]["id"]
+        assert report["items"][0]["renamed"] is True
+        tools = {t.id: t for t in await ToolService.list(session, tenant, src.id)}
+        assert tools[new_id].auth_provider_id == ap.id
+        assert not any("auth provider" in w for w in report["warnings"])
+        assert dst  # (unused target kept for symmetry)
+
+
+async def test_auto_rename_never_overwrites_on_repeated_import():
+    tenant = "t_rename"
+    async with SessionLocal() as session:
+        src = await _project(session, tenant, "src")
+        tool = await ToolService.create(
+            session, tenant, src.id, name="lookup", kind="rest_api",
+            config={"description": "look things up", "request": {"method": "GET", "url_template": "https://x.test"}},
+        )
+        bundle = await PortabilityService.export(session, tenant, src.id, "tool", [tool.id])
+        r1 = await PortabilityService.import_bundle(session, tenant, src.id, bundle, author=AUTHOR)
+        r2 = await PortabilityService.import_bundle(session, tenant, src.id, bundle, author=AUTHOR)
+        # Filter out the auto-provisioned platform built-ins (never part of a bundle).
+        names = sorted(t.name for t in await ToolService.list(session, tenant, src.id) if t.kind != "builtin")
+        assert names == ["lookup", "lookup_imported", "lookup_imported_2"]
+        assert r1["items"][0]["name"] == "lookup_imported"
+        assert r2["items"][0]["name"] == "lookup_imported_2"
+
+
+async def test_component_unique_name_autorenames_without_conflict():
+    tenant = "t_comp"
+    async with SessionLocal() as session:
+        src = await _project(session, tenant, "src")
+        comp = await ComponentService.create(
+            session, tenant, src.id, name="product_card", title="Card",
+            html="<div>{{title}}</div>", css=".x{}", props_schema={"type": "object"},
+            sample_props={"title": "Hi"}, actions=[{"id": "buy", "label": "Buy"}],
+        )
+        comp.enabled = False
+        await session.commit()
+
+        bundle = await PortabilityService.export(session, tenant, src.id, "component", [comp.id])
+        item = bundle["items"][0]
+        assert item["html"] == "<div>{{title}}</div>"
+        assert item["actions"] == [{"id": "buy", "label": "Buy"}]
+        assert item["enabled"] is False
+
+        # Import back into the same project twice: the unique-name constraint must never trip.
+        await PortabilityService.import_bundle(session, tenant, src.id, bundle, author=AUTHOR)
+        await PortabilityService.import_bundle(session, tenant, src.id, bundle, author=AUTHOR)
+        comps = sorted(c.name for c in await ComponentService.list(session, tenant, src.id))
+        assert comps == ["product_card", "product_card_imported", "product_card_imported_2"]
+        # Round-tripped fields survive (incl. the disabled state).
+        imported = [c for c in await ComponentService.list(session, tenant, src.id) if c.name == "product_card_imported"][0]
+        assert imported.enabled is False
+        assert imported.actions == [{"id": "buy", "label": "Buy"}]
+
+
+async def test_agent_config_preserved_and_attribution_is_importer():
+    tenant = "t_agent"
+    async with SessionLocal() as session:
+        src = await _project(session, tenant, "src")
+        dst = await _project(session, tenant, "dst")
+        config = {
+            "flavor": "agent",
+            "model": "openai:gpt-4o-mini",
+            "system_prompt": "Be helpful.",
+            "tools": ["some_tool_id"],
+            "components": ["some_component_id"],
+            "middleware": [{"type": "summarization", "enabled": True}],
+        }
+        agent = await AgentService.create(
+            session, tenant, src.id, name="support", config=config,
+            created_by="orig_user", created_by_email="orig@forge.local",
+        )
+        bundle = await PortabilityService.export(session, tenant, src.id, "agent", [agent.id])
+        assert bundle["items"][0]["created_by_email"] == "orig@forge.local"
+
+        await PortabilityService.import_bundle(session, tenant, dst.id, bundle, author=AUTHOR)
+        imported = (await AgentService.list(session, tenant, dst.id))[0]
+        assert imported.config == config  # full config verbatim (nothing dropped)
+        assert imported.created_by == "u_importer"
+        assert imported.created_by_email == "importer@forge.local"
+
+
+async def test_workflow_subworkflow_reference_is_remapped_to_new_ids():
+    tenant = "t_wf"
+    async with SessionLocal() as session:
+        src = await _project(session, tenant, "src")
+        dst = await _project(session, tenant, "dst")
+        child = await WorkflowService.create(
+            session, tenant, src.id, name="child",
+            executable={"id": "child", "version": 1, "nodes": [], "edges": []},
+            canvas={"nodes": [], "edges": []},
+        )
+        parent = await WorkflowService.create(
+            session, tenant, src.id, name="parent",
+            executable={
+                "id": "parent", "version": 1,
+                "nodes": [{"id": "sub_1", "type": "subworkflow", "config": {"workflow_id": child.id}}],
+                "edges": [],
+            },
+            canvas={"nodes": [], "edges": []},
+        )
+
+        bundle = await PortabilityService.export(session, tenant, src.id, "workflow", [child.id, parent.id])
+        report = await PortabilityService.import_bundle(session, tenant, dst.id, bundle, author=AUTHOR)
+        assert report["imported"] == 2
+
+        wfs = {w.name: w for w in await WorkflowService.list(session, tenant, dst.id)}
+        new_child, new_parent = wfs["child"], wfs["parent"]
+        assert new_child.id != child.id and new_parent.id != parent.id
+        ref = new_parent.executable["nodes"][0]["config"]["workflow_id"]
+        assert ref == new_child.id  # remapped to the freshly-created child, not the old id
+        # Imported workflows land as drafts (publish is a separate, governance-checked action).
+        assert new_parent.status == "draft"
+
+
+async def test_import_rejects_wrong_bundle_type():
+    tenant = "t_type"
+    async with SessionLocal() as session:
+        dst = await _project(session, tenant, "dst")
+        with pytest.raises(ValueError):
+            await PortabilityService.import_bundle(
+                session, tenant, dst.id, {"type": "nonsense", "items": []}, author=AUTHOR
+            )
+
+
+def _client() -> httpx.AsyncClient:
+    return httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app()), base_url="http://test")
+
+
+async def _owner(c: httpx.AsyncClient) -> dict:
+    reg = (await c.post("/v1/auth/register", json={"email": f"u{uuid.uuid4().hex[:8]}@x.com", "password": "supersecret1"})).json()
+    return {"Authorization": f"Bearer {reg['access_token']}"}
+
+
+async def test_http_component_export_import_roundtrip_across_projects():
+    """Full HTTP path: create → export → import into another project, over the real routes."""
+    async with _client() as c:
+        h = await _owner(c)
+        src = (await c.post("/v1/projects", json={"name": "Source"}, headers=h)).json()
+        dst = (await c.post("/v1/projects", json={"name": "Dest"}, headers=h)).json()
+        comp = (await c.post(
+            f"/v1/projects/{src['id']}/components",
+            json={"name": "price_card", "title": "Price", "html": "<b>{{p}}</b>", "css": ".b{}",
+                  "props_schema": {"type": "object"}, "sample_props": {"p": 9}, "actions": []},
+            headers=h,
+        )).json()
+
+        exp = await c.post(f"/v1/projects/{src['id']}/components/export", json={"ids": [comp["id"]]}, headers=h)
+        assert exp.status_code == 200, exp.text
+        bundle = exp.json()
+        assert bundle["type"] == "component" and len(bundle["items"]) == 1
+
+        imp = await c.post(f"/v1/projects/{dst['id']}/components/import", json=bundle, headers=h)
+        assert imp.status_code == 200, imp.text
+        report = imp.json()
+        assert report["imported"] == 1 and report["type"] == "component"
+
+        listed = (await c.get(f"/v1/projects/{dst['id']}/components", headers=h)).json()
+        assert [x["name"] for x in listed] == ["price_card"]
+        assert listed[0]["html"] == "<b>{{p}}</b>"
+
+
+async def test_http_import_wrong_type_returns_422():
+    async with _client() as c:
+        h = await _owner(c)
+        pid = (await c.post("/v1/projects", json={"name": "P"}, headers=h)).json()["id"]
+        # An agent bundle posted to the tools import endpoint must be rejected clearly.
+        r = await c.post(f"/v1/projects/{pid}/tools/import", json={"type": "agent", "items": []}, headers=h)
+        assert r.status_code == 422
+        assert "agent" in r.json()["detail"]
+
+
+async def test_export_ignores_ids_from_other_projects():
+    tenant = "t_scope"
+    async with SessionLocal() as session:
+        src = await _project(session, tenant, "src")
+        other = await _project(session, tenant, "other")
+        mine = await ToolService.create(session, tenant, src.id, name="mine", kind="rest_api", config={"request": {"method": "GET", "url_template": "https://x.test"}})
+        theirs = await ToolService.create(session, tenant, other.id, name="theirs", kind="rest_api", config={"request": {"method": "GET", "url_template": "https://y.test"}})
+        bundle = await PortabilityService.export(session, tenant, src.id, "tool", [mine.id, theirs.id])
+        names = [i["name"] for i in bundle["items"]]
+        assert names == ["mine"]  # cross-project id silently dropped
+        assert Component  # import kept tidy

apps/api/tests/test_pricing.py

@@ -0,0 +1,25 @@
+"""Admin pricing overrides overlay the built-in defaults."""
+
+from __future__ import annotations
+
+from forge.tracing.pricing import load_overrides, merged_prices, price, set_override
+
+
+def test_default_pricing_applies():
+    # gpt-4o-mini default is (0.15, 0.6) per 1M
+    cost = price("gpt-4o-mini", 1_000_000, 0)
+    assert abs(cost - 0.15) < 1e-9
+
+
+def test_override_takes_precedence_and_matches_bare_name():
+    set_override("gpt-4o-mini", 1.0, 2.0)
+    try:
+        assert abs(price("gpt-4o-mini", 1_000_000, 0) - 1.0) < 1e-9
+        assert abs(price("openai:gpt-4o-mini", 0, 1_000_000) - 2.0) < 1e-9  # provider-prefixed resolves bare
+        assert merged_prices()["gpt-4o-mini"] == (1.0, 2.0)
+    finally:
+        load_overrides({})  # reset so other tests see defaults
+
+
+def test_unknown_model_prices_zero():
+    assert price("totally-unknown-model", 1000, 1000) == 0.0

apps/api/tests/test_project_run_endpoint.py

@@ -0,0 +1,162 @@
+"""The single project-level run endpoint (POST /v1/projects/{id}/run).
+
+One generic surface over the existing run machinery: it runs the project's *configured*
+workflow (config.api_workflow_id), takes `stream` as the only per-request knob, and routes a
+`resume` body to the HITL machinery. These tests drive it in-process over ASGI with a fake
+model, so no real LLM is called.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+import httpx
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.main import create_app
+
+# A trivial one-agent workflow whose fake model always answers with this exact text - lets us
+# assert the endpoint actually ran the configured workflow end to end.
+_ANSWER = "Hello from Forge."
+_WF = {
+    "id": "wf_run_ep", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "agent",
+    "nodes": [
+        {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": f"fake:{_ANSWER}", "tools": []}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "agent", "target": "end"}],
+}
+
+
+def _client() -> httpx.AsyncClient:
+    app = create_app()
+    # No app lifespan runs in-process, so hand the run service a checkpointer directly
+    # (aget_state needs one); prod gets this from the lifespan.
+    app.state.checkpointer = InMemorySaver()
+    return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test")
+
+
+def _email() -> str:
+    return f"u{uuid.uuid4().hex[:10]}@example.com"
+
+
+async def _project_with_configured_workflow(c: httpx.AsyncClient) -> tuple[dict, str]:
+    """Register an owner, create a project + workflow, and pin the workflow as the project's
+    API workflow. Returns (auth header, project_id)."""
+    reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+    h = {"Authorization": f"Bearer {reg['access_token']}"}
+    pid = (await c.post("/v1/projects", json={"name": "Run API Project"}, headers=h)).json()["id"]
+    wid = (await c.post(f"/v1/projects/{pid}/workflows", json={"name": "Chat", "executable": _WF}, headers=h)).json()["id"]
+    # The saved project setting that designates which workflow the /run endpoint executes.
+    r = await c.patch(f"/v1/projects/{pid}", json={"config": {"api_workflow_id": wid}}, headers=h)
+    assert r.status_code == 200, r.text
+    return h, pid
+
+
+async def test_run_non_stream_returns_answer_and_thread():
+    async with _client() as c:
+        h, pid = await _project_with_configured_workflow(c)
+        r = await c.post(
+            f"/v1/projects/{pid}/run",
+            json={"input": {"messages": [{"role": "user", "content": "hi"}]}, "stream": False},
+            headers=h,
+        )
+        assert r.status_code == 200, r.text
+        body = r.json()
+        assert _ANSWER in (body.get("answer") or ""), body
+        assert body.get("thread_id"), body
+        assert body.get("status") == "done", body
+
+
+async def test_run_stream_emits_ready_and_done_frames():
+    async with _client() as c:
+        h, pid = await _project_with_configured_workflow(c)
+        r = await c.post(
+            f"/v1/projects/{pid}/run",
+            json={"input": {"messages": [{"role": "user", "content": "hi"}]}, "stream": True},
+            headers=h,
+        )
+        assert r.status_code == 200, r.text
+        text = r.text
+        # A leading `ready` frame hands the caller the canonical thread_id, and the run
+        # finishes with a `done` frame carrying the answer.
+        assert "event: ready" in text, text
+        assert '"thread_id"' in text
+        assert "event: done" in text, text
+        assert _ANSWER in text
+
+
+async def test_thread_id_from_run_continues_conversation():
+    async with _client() as c:
+        h, pid = await _project_with_configured_workflow(c)
+        first = (await c.post(
+            f"/v1/projects/{pid}/run",
+            json={"input": {"messages": [{"role": "user", "content": "hi"}]}, "stream": False},
+            headers=h,
+        )).json()
+        tid = first["thread_id"]
+        # Reusing the thread_id must be accepted (the checkpointer holds the history).
+        r = await c.post(
+            f"/v1/projects/{pid}/run",
+            json={"thread_id": tid, "input": {"messages": [{"role": "user", "content": "again"}]}, "stream": False},
+            headers=h,
+        )
+        assert r.status_code == 200, r.text
+        assert r.json().get("thread_id") == tid
+
+
+async def test_stream_exposes_one_consistent_thread_id_that_continues():
+    """Regression (shared chat memory): the streaming path once handed the caller TWO different
+    thread handles - the DB Thread.id in the `ready` frame and the composite LangGraph id
+    (`{tenant}:{uuid}`) in the `run` frame. A caller that stored the `run` frame's id echoed a
+    handle that never matched Thread.id, so every turn spun up a fresh thread and the agent
+    "forgot" the conversation. Every thread_id in the stream must now be identical and reusable."""
+    import re
+
+    async with _client() as c:
+        h, pid = await _project_with_configured_workflow(c)
+        r = await c.post(
+            f"/v1/projects/{pid}/run",
+            json={"input": {"messages": [{"role": "user", "content": "hi"}]}, "stream": True},
+            headers=h,
+        )
+        assert r.status_code == 200, r.text
+        tids = re.findall(r'"thread_id":\s*"([^"]+)"', r.text)
+        assert tids, r.text
+        assert len(set(tids)) == 1, f"stream exposed conflicting thread handles: {set(tids)}"
+        # The handle from the stream must reattach to the same thread (checkpointer holds history).
+        again = (await c.post(
+            f"/v1/projects/{pid}/run",
+            json={"thread_id": tids[0], "input": {"messages": [{"role": "user", "content": "again"}]}, "stream": False},
+            headers=h,
+        )).json()
+        assert again.get("thread_id") == tids[0]
+
+
+async def test_project_without_a_workflow_is_404():
+    async with _client() as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+        pid = (await c.post("/v1/projects", json={"name": "Empty"}, headers=h)).json()["id"]
+        r = await c.post(f"/v1/projects/{pid}/run", json={"input": {}, "stream": False}, headers=h)
+        assert r.status_code == 404, r.text
+
+
+async def test_resume_requires_thread_id():
+    async with _client() as c:
+        h, pid = await _project_with_configured_workflow(c)
+        r = await c.post(f"/v1/projects/{pid}/run", json={"resume": {"value": "approve"}}, headers=h)
+        assert r.status_code == 400, r.text
+
+
+async def test_resume_with_no_interrupted_run_is_409():
+    async with _client() as c:
+        h, pid = await _project_with_configured_workflow(c)
+        r = await c.post(
+            f"/v1/projects/{pid}/run",
+            json={"thread_id": "does-not-exist", "resume": {"value": "approve"}},
+            headers=h,
+        )
+        assert r.status_code == 409, r.text

apps/api/tests/test_projects.py

@@ -0,0 +1,108 @@
+"""Project lifecycle tests."""
+
+from __future__ import annotations
+
+from sqlalchemy import func, select
+
+from forge.db.base import SessionLocal
+from forge.models import (
+    Agent,
+    AuthProvider,
+    Component,
+    HandoffRequest,
+    KbSource,
+    McpClient,
+    QaPair,
+    Run,
+    Secret,
+    Span,
+    Thread,
+    Tool,
+    Trace,
+    Workflow,
+)
+from forge.services.projects import ProjectService
+
+
+async def _count(session, model, **where) -> int:
+    stmt = select(func.count()).select_from(model)
+    for key, value in where.items():
+        stmt = stmt.where(getattr(model, key) == value)
+    return int((await session.execute(stmt)).scalar_one())
+
+
+async def test_project_counts_are_scoped_to_project():
+    tenant_id = "tenant_counts"
+    async with SessionLocal() as session:
+        proj = await ProjectService.create(session, tenant_id, name="Counts", slug="counts")
+        other = await ProjectService.create(session, tenant_id, name="Other", slug="other")
+        session.add_all([
+            Workflow(tenant_id=tenant_id, project_id=proj.id, name="wf1"),
+            Workflow(tenant_id=tenant_id, project_id=proj.id, name="wf2"),
+            Agent(tenant_id=tenant_id, project_id=proj.id, name="a", config={}),
+            Tool(tenant_id=tenant_id, project_id=proj.id, name="t1", kind="builtin", config={}),
+            Tool(tenant_id=tenant_id, project_id=proj.id, name="t2", kind="builtin", config={}),
+            Tool(tenant_id=tenant_id, project_id=proj.id, name="t3", kind="builtin", config={}),
+            Component(tenant_id=tenant_id, project_id=proj.id, name="card"),
+            KbSource(tenant_id=tenant_id, project_id=proj.id, kind="text", name="src"),
+            AuthProvider(tenant_id=tenant_id, project_id=proj.id, name="auth", kind="bearer", config={}),
+            HandoffRequest(tenant_id=tenant_id, project_id=proj.id, run_id="run-open", status="open"),
+            HandoffRequest(tenant_id=tenant_id, project_id=proj.id, run_id="run-done", status="answered"),
+            # Belongs to a different project in the same tenant - must NOT be counted for `proj`.
+            Tool(tenant_id=tenant_id, project_id=other.id, name="other_tool", kind="builtin", config={}),
+            Workflow(tenant_id=tenant_id, project_id=other.id, name="other_wf"),
+        ])
+        await session.commit()
+
+        # ProjectService.create auto-provisions the platform built-ins, so each project starts with
+        # len(BUILTIN_DEFAULTS) tools before the 3 added here. Scoping still holds: `other`'s tool
+        # (and its own provisioned built-ins) are not counted for `proj`.
+        from forge.tools.builtin import BUILTIN_DEFAULTS
+        counts = await ProjectService.counts(session, tenant_id, proj.id)
+        assert counts == {
+            "workflows": 2, "agents": 1, "tools": 3 + len(BUILTIN_DEFAULTS),
+            "components": 1, "knowledge": 1, "auth": 1, "handoffs": 1,
+        }
+        # A fresh project is all zeros except the auto-provisioned platform built-ins (never None).
+        empty = await ProjectService.create(session, tenant_id, name="Empty", slug="empty")
+        assert await ProjectService.counts(session, tenant_id, empty.id) == {
+            "workflows": 0, "agents": 0, "tools": len(BUILTIN_DEFAULTS), "components": 0, "knowledge": 0, "auth": 0,
+            "handoffs": 0,
+        }
+
+
+async def test_delete_project_removes_project_scoped_data_and_trace_spans():
+    tenant_id = "tenant_delete_project"
+    deleted_threads: list[str] = []
+
+    class FakeCheckpointer:
+        async def adelete_thread(self, thread_id: str) -> None:
+            deleted_threads.append(thread_id)
+
+    async with SessionLocal() as session:
+        project = await ProjectService.create(session, tenant_id, name="Delete Me", slug="delete-me")
+        workflow = Workflow(tenant_id=tenant_id, project_id=project.id, name="wf")
+        thread = Thread(tenant_id=tenant_id, project_id=project.id, workflow_id="wf1", lg_thread_id="lg1")
+        run = Run(tenant_id=tenant_id, project_id=project.id, workflow_id="wf1", thread_id="thread1")
+        trace = Trace(tenant_id=tenant_id, project_id=project.id, workflow_id="wf1", run_id="run1", name="trace")
+        session.add_all([workflow, thread, run, trace])
+        await session.flush()
+        session.add_all([
+            Span(tenant_id=tenant_id, trace_id=trace.id, name="span", kind="node"),
+            Agent(tenant_id=tenant_id, project_id=project.id, name="agent", config={}),
+            Tool(tenant_id=tenant_id, project_id=project.id, name="tool", kind="builtin", config={}),
+            AuthProvider(tenant_id=tenant_id, project_id=project.id, name="auth", kind="bearer", config={}),
+            Secret(tenant_id=tenant_id, project_id=project.id, name="secret", kind="api_key", encrypted_value=b"x"),
+            KbSource(tenant_id=tenant_id, project_id=project.id, kind="text", name="source"),
+            QaPair(tenant_id=tenant_id, project_id=project.id, question="q", answer="a"),
+            McpClient(tenant_id=tenant_id, project_id=project.id, name="mcp"),
+        ])
+        await session.commit()
+
+        await ProjectService.delete(session, project, checkpointer=FakeCheckpointer())
+
+        assert await ProjectService.get(session, tenant_id, project.id) is None
+        for model in (Workflow, Thread, Run, Trace, Agent, Tool, AuthProvider, Secret, KbSource, QaPair, McpClient):
+            assert await _count(session, model, project_id=project.id) == 0
+        assert await _count(session, Span, trace_id=trace.id) == 0
+        assert deleted_threads == ["lg1"]

apps/api/tests/test_qa_vectorstore.py

@@ -0,0 +1,65 @@
+"""Q&A semantic match now runs through the vector store (forge_qa_<dim>), not a
+Python O(n) cosine over every row."""
+
+from __future__ import annotations
+
+from forge.db.base import SessionLocal
+from forge.models import QaPair
+from forge.services.knowledge import KnowledgeService
+
+T, P = "t_qa_vs", "p_qa_vs"
+
+
+async def _seed():
+    async with SessionLocal() as s:
+        await KnowledgeService.create_qa(s, T, P, question="How do I reset my password?", answer="Use the reset link on the login page.", kind="faq")
+        await KnowledgeService.create_qa(s, T, P, question="What are your support hours?", answer="9am-5pm ET, Mon-Fri.", kind="hours")
+
+
+async def test_top_qa_ranks_semantically():
+    await _seed()
+    async with SessionLocal() as s:
+        hits = await KnowledgeService.top_qa(s, T, P, "I forgot my password and need to reset it", top_k=1, threshold=0.0)
+    assert hits and "reset link" in hits[0]["answer"]
+
+
+async def test_lookup_kind_filter():
+    await _seed()
+    async with SessionLocal() as s:
+        # restricting to the 'hours' kind must not return the password FAQ
+        hit = await KnowledgeService.lookup(s, T, P, "reset password", threshold=0.0, kinds=["hours"])
+    assert hit is not None and "9am-5pm" in hit["answer"]
+
+
+async def test_lazy_reindex_backfills_existing_rows():
+    # Insert a QaPair directly (bypassing create_qa's upsert) to simulate pre-existing data,
+    # then top_qa must still find it via the lazy reindex.
+    async with SessionLocal() as s:
+        emb = await (await KnowledgeService.embedder_for_project(s, "t_bf", "p_bf")).aembed_query("billing question")
+        s.add(QaPair(tenant_id="t_bf", project_id="p_bf", question="How do I update my billing card?",
+                     answer="Settings → Billing → Update card.", kind="faq", q_embedding=emb))
+        await s.commit()
+    async with SessionLocal() as s:
+        hits = await KnowledgeService.top_qa(s, "t_bf", "p_bf", "change my billing card", top_k=1, threshold=0.0)
+    assert hits and "Billing" in hits[0]["answer"]
+
+
+async def test_update_qa_replaces_question_and_retrieval_metadata():
+    tenant_id, project_id = "t_qa_edit", "p_qa_edit"
+    async with SessionLocal() as s:
+        qa = await KnowledgeService.create_qa(
+            s, tenant_id, project_id, question="Where is the old handbook?",
+            answer="On the old portal.", kind="legacy", tags=["old"],
+        )
+        updated = await KnowledgeService.update_qa(
+            s, qa, question="Where is the employee handbook?",
+            answer="Open People, then Documents.", kind="hr", tags=["people"],
+        )
+        hits = await KnowledgeService.top_qa(
+            s, tenant_id, project_id, "employee handbook", top_k=1, threshold=0.0,
+        )
+
+    assert updated.question == "Where is the employee handbook?"
+    assert updated.kind == "hr" and updated.tags == ["people"]
+    assert hits and hits[0]["answer"] == "Open People, then Documents."
+    assert hits[0]["kind"] == "hr"

apps/api/tests/test_quota_audit.py

@@ -0,0 +1,99 @@
+"""Per-tenant daily quota, centralized mutation auditing, and the scoping helper."""
+
+from __future__ import annotations
+
+import uuid
+
+import httpx
+import pytest
+
+from forge.db.base import SessionLocal
+from forge.db.scoping import tenant_scoped
+from forge.main import create_app
+from forge.models import Run, Tenant
+from forge.services.quota import QuotaExceeded, check_run_quota, usage_today
+
+
+def _email() -> str:
+    return f"u{uuid.uuid4().hex[:10]}@example.com"
+
+
+# --- 1.7 quota ---
+
+
+async def test_quota_blocks_when_daily_run_cap_reached():
+    async with SessionLocal() as s:
+        t = Tenant(name="Q", settings={"max_runs_per_day": 1})
+        s.add(t)
+        await s.flush()
+        s.add(Run(tenant_id=t.id, project_id="p", workflow_id="w", thread_id="th", status="done"))
+        await s.commit()
+        tid = t.id
+    async with SessionLocal() as s:
+        with pytest.raises(QuotaExceeded):
+            await check_run_quota(s, tid)
+        usage = await usage_today(s, tid)
+        assert usage["runs"] == 1 and usage["limits"]["max_runs_per_day"] == 1
+
+
+async def test_no_quota_when_unset():
+    async with SessionLocal() as s:
+        t = Tenant(name="NoQ", settings={})
+        s.add(t)
+        await s.commit()
+        await check_run_quota(s, t.id)  # must not raise
+
+
+# --- 1.10 scoping helper ---
+
+
+def test_tenant_scoped_adds_filters():
+    from sqlalchemy import select
+
+    from forge.models import Workflow
+    sql = str(tenant_scoped(select(Workflow), Workflow, "t1", project_id="p1"))
+    assert "tenant_id" in sql and "project_id" in sql
+
+
+# --- 1.8 audit middleware ---
+
+
+async def test_mutations_are_audited():
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+
+        r = await c.post("/v1/projects", json={"name": "Audited Project"}, headers=h)
+        assert r.status_code in (200, 201), r.text
+
+        audit = (await c.get("/v1/audit", headers=h)).json()
+        actions = [a["action"] for a in audit]
+        assert any(a == "POST /v1/projects" for a in actions), actions
+        # auth endpoints are NOT double-audited by the middleware
+        assert "POST /v1/auth/register" not in actions
+
+
+async def test_audit_action_uses_route_template_and_keeps_concrete_path():
+    """A mutating request to a UUID path is audited as the route TEMPLATE, not the concrete
+    path. The template fits the action column (String(80)) and stays aggregatable; the concrete
+    path (with real ids) is preserved in meta for forensics. Regression for the varchar(80)
+    overflow that silently dropped every long-path audit row on Postgres."""
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        reg = (await c.post("/v1/auth/register", json={"email": _email(), "password": "supersecret1"})).json()
+        h = {"Authorization": f"Bearer {reg['access_token']}"}
+
+        pid = (await c.post("/v1/projects", json={"name": "P"}, headers=h)).json()["id"]
+        r = await c.patch(f"/v1/projects/{pid}", json={"name": "Renamed"}, headers=h)
+        assert r.status_code == 200, r.text
+
+        audit = (await c.get("/v1/audit", headers=h)).json()
+        patch_rows = [a for a in audit if a["action"].startswith("PATCH ")]
+        assert patch_rows, [a["action"] for a in audit]
+        row = patch_rows[0]
+        # templated (no UUID) -> within String(80) and aggregatable across projects
+        assert row["action"] == "PATCH /v1/projects/{project_id}"
+        assert len(row["action"]) <= 80
+        # concrete path with the real id is retained in meta
+        assert row["meta"]["path"] == f"/v1/projects/{pid}"

apps/api/tests/test_rag_floor_fixes.py

@@ -0,0 +1,313 @@
+"""Audit fixes for the RAG relevance floor + ingestion/retrieval gaps.
+
+Covers, per the feature audit:
+- effective cosine grounding floor (calibrated for the default BGE embedder) end to end,
+- min_score thresholding the TRUE cosine in hybrid mode (not the fused rank),
+- source-provenance citations persisted on chunks,
+- crawl honoring robots.txt + max_depth,
+- CSV/JSON per-record parsing + binary-upload rejection + HTML stripping,
+- chunk_size clamped to the embedder's input limit,
+- long-term-memory recall similarity floor.
+
+Model-backed tests use the local fastembed embedder and skip cleanly when it can't load
+(offline), mirroring the other knowledge tests.
+"""
+
+from __future__ import annotations
+
+import pytest
+from fastapi import HTTPException
+
+from forge.knowledge.crawl import MAX_DEPTH_CAP, MAX_PAGES_CAP
+from forge.knowledge.embeddings import DEFAULT_MIN_SCORE, DEFAULT_RERANK_MIN_SCORE, _max_input_chars
+from forge.knowledge.store import Hit, citation_for
+from forge.nodes.rag import _passes_floor
+from forge.routers.knowledge import _csv_to_text, _decode_upload, _json_to_text
+
+
+def _require_embedder():
+    """Return the local BGE embedder or skip (offline / model not cached)."""
+    pytest.importorskip("fastembed")
+    from forge.knowledge.embeddings import resolve_embedder
+
+    try:
+        e = resolve_embedder(None)
+    except Exception:  # noqa: BLE001
+        pytest.skip("fastembed model could not be loaded (offline)")
+    if getattr(e, "name", "") != "BAAI/bge-small-en-v1.5":
+        pytest.skip("fastembed model unavailable")
+    return e
+
+
+# --- calibrated floor defaults (pure) ---
+
+def test_default_floors_are_calibrated():
+    # BGE unrelated pairs measure ~0.4-0.52, related ~0.75+, so the floor must sit between.
+    assert 0.55 <= DEFAULT_MIN_SCORE <= 0.7
+    assert 0.0 < DEFAULT_RERANK_MIN_SCORE < DEFAULT_MIN_SCORE
+
+
+# --- min_score on the right scale (pure) ---
+
+def test_passes_floor_uses_cosine_not_fused_rank():
+    # vector-only: Hit.score IS the cosine.
+    assert _passes_floor(Hit("a", "t", 0.70, {}), 0.6, hybrid=False) is True
+    assert _passes_floor(Hit("a", "t", 0.50, {}), 0.6, hybrid=False) is False
+    # hybrid: Hit.score is the fused rank (top≈1.0); the floor must use vector_score.
+    assert _passes_floor(Hit("a", "t", 1.0, {}, vector_score=0.50), 0.6, hybrid=True) is False
+    assert _passes_floor(Hit("a", "t", 1.0, {}, vector_score=0.70), 0.6, hybrid=True) is True
+    # a BM25-only hit has no cosine -> kept (a strong exact-term match isn't floored out).
+    assert _passes_floor(Hit("a", "t", 0.9, {}, vector_score=None), 0.6, hybrid=True) is True
+
+
+# --- citations from chunk provenance (pure) ---
+
+def test_citation_for_prefers_page_then_source():
+    assert citation_for({"page_url": "https://x.test/pricing", "page_title": "pricing"}) == "pricing — https://x.test/pricing"
+    assert citation_for({"page_url": "https://x.test/p"}) == "https://x.test/p"
+    assert citation_for({"source_name": "help", "source_uri": "https://x.test/help"}) == "help — https://x.test/help"
+    assert citation_for({"source_name": "manual"}) == "manual"
+    assert citation_for({}) == ""
+    assert citation_for(None) == ""
+
+
+# --- BM25 cache split: index build + score parity (pure) ---
+
+def test_bm25_build_and_score_match_one_shot():
+    pytest.importorskip("rank_bm25")
+    from forge.knowledge.hybrid import bm25_rank, bm25_scores, build_bm25
+
+    docs = [
+        ("d1", "general refund and shipping policy details"),
+        ("d2", "error code XJ9000 means a payment gateway timeout"),
+        ("d3", "how to contact our support team"),
+    ]
+    idx = build_bm25(docs)
+    assert bm25_scores(idx, "XJ9000 gateway timeout")[0] == "d2"
+    # The cached-index path must match the one-shot bm25_rank exactly.
+    assert bm25_scores(idx, "XJ9000 gateway timeout") == bm25_rank("XJ9000 gateway timeout", docs)
+    assert bm25_scores(idx, "zzz qqq") == []
+    assert bm25_scores(None, "anything") == []
+    assert build_bm25([]) is None
+
+
+# --- chunk_size vs embedder token limit (pure) ---
+
+def test_max_input_chars_by_model():
+    assert _max_input_chars("BAAI/bge-small-en-v1.5") == 512 * 4
+    assert _max_input_chars("text-embedding-3-small") == 8191 * 4
+    assert _max_input_chars("some-unknown-model") == 512 * 4  # conservative default
+
+
+# --- CSV/JSON per-record parsing + upload guards (pure) ---
+
+def test_csv_parsed_into_header_qualified_records():
+    out = _csv_to_text("name,plan,seats\nAcme,enterprise,50\nBeta,free,3")
+    assert "name: Acme" in out and "plan: enterprise" in out and "seats: 50" in out
+    assert "name: Beta" in out
+    assert "\n\n" in out  # records separated so the chunker can split on record boundaries
+
+
+def test_csv_without_data_rows_falls_back():
+    assert _csv_to_text("only one line") == "only one line"
+
+
+def test_json_list_of_objects_becomes_records():
+    out = _json_to_text('[{"q": "hi", "a": "there"}, {"q": "bye", "a": "now"}]')
+    assert "q: hi | a: there" in out
+    assert "q: bye | a: now" in out
+    assert "\n\n" in out
+
+
+def test_json_single_list_value_is_expanded():
+    out = _json_to_text('{"items": [{"k": 1}, {"k": 2}]}')
+    assert "k: 1" in out and "k: 2" in out
+
+
+def test_json_invalid_falls_back_to_raw():
+    assert _json_to_text("not json {{{") == "not json {{{"
+
+
+def test_decode_upload_rejects_binary_extension():
+    with pytest.raises(HTTPException) as ei:
+        _decode_upload("report.docx", b"PK\x03\x04 not really text")
+    assert ei.value.status_code == 422
+
+
+def test_decode_upload_rejects_null_byte_binary():
+    with pytest.raises(HTTPException):
+        _decode_upload("mystery.dat", b"text\x00\x00\x01\x02 binary")
+
+
+def test_decode_upload_strips_html():
+    out = _decode_upload("page.html", b"<html><body><h1>Hi</h1><p>There</p></body></html>")
+    assert "Hi" in out and "There" in out
+    assert "<h1>" not in out and "<body>" not in out
+
+
+def test_decode_upload_plain_text_and_csv_dispatch():
+    assert _decode_upload("notes.txt", b"just some notes") == "just some notes"
+    out = _decode_upload("data.csv", b"a,b\n1,2")
+    assert "a: 1" in out and "b: 2" in out
+
+
+# --- crawl: robots.txt + depth (monkeypatched network, offline) ---
+
+def test_crawl_caps_are_bounded():
+    assert MAX_PAGES_CAP <= 500 and MAX_DEPTH_CAP <= 10
+
+
+async def test_crawl_honors_robots_and_max_depth(monkeypatch):
+    import forge.util.ssrf as ssrf
+    from forge.knowledge import crawl as crawl_mod
+
+    class _Resp:
+        def __init__(self, text: str, status: int = 200) -> None:
+            self.text = text
+            self.status_code = status
+
+    site = {
+        "https://acme.test/robots.txt": _Resp("User-agent: *\nDisallow: /private\n"),
+        "https://acme.test/": _Resp('<a href="/a">A</a> <a href="/private">P</a> <a href="/b">B</a>'),
+        "https://acme.test/a": _Resp('<a href="/c">C</a> alpha body'),
+        "https://acme.test/b": _Resp("bee body"),
+        "https://acme.test/c": _Resp("cee body"),
+        "https://acme.test/private": _Resp("secret body"),
+    }
+
+    async def fake_get(client, url, **kw):
+        if url in site:
+            return site[url]
+        raise RuntimeError("404")
+
+    monkeypatch.setattr(ssrf, "guarded_get", fake_get)
+    pages = await crawl_mod.crawl_site("https://acme.test/", max_pages=50, max_depth=1, delay=0.0)
+
+    crawled = set(pages)  # exact-URL membership (not substring) so this stays a set lookup
+    assert "https://acme.test/" in crawled
+    assert "https://acme.test/a" in crawled and "https://acme.test/b" in crawled
+    assert "https://acme.test/private" not in crawled  # robots.txt Disallow honored
+    assert "https://acme.test/c" not in crawled  # one hop beyond max_depth=1
+
+
+# --- model-backed end-to-end ---
+
+async def test_offtopic_query_is_floored_and_on_topic_cites(tmp_path):
+    _require_embedder()
+    from langchain_core.messages import SystemMessage
+
+    from forge.config import settings
+    from forge.db.base import SessionLocal
+    from forge.engine.context import CompileContext
+    from forge.nodes.rag import retrieval_factory
+    from forge.services.knowledge import KnowledgeService
+
+    settings.chroma_path = str(tmp_path / "chroma_floor")
+    t, p = "t_floor", "p_floor"
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(
+            s, t, p, kind="text", name="refunds",
+            text="Refunds are issued to the original payment method within 5-7 business days.",
+        )
+        await KnowledgeService.ingest(s, src)
+
+    node = retrieval_factory({"announce_empty": True, "top_k": 3}, CompileContext(tenant_id=t, project_id=p))
+    off = await node({"messages": [{"role": "user", "content": "what is the capital of France?"}]})
+    assert isinstance(off["messages"][-1], SystemMessage)
+    assert "no relevant" in off["messages"][-1].content.lower()  # off-topic floored -> empty note
+
+    on = await node({"messages": [{"role": "user", "content": "how long do refunds take?"}]})
+    body = on["messages"][-1].content
+    assert "KNOWLEDGE BASE context" in body
+    assert "Refunds" in body
+    assert "refunds" in body.lower().split("] ")[0]  # citation label carries the source name
+
+
+async def test_hybrid_hit_carries_true_cosine_vector_score(tmp_path):
+    _require_embedder()
+    from forge.config import settings
+    from forge.db.base import SessionLocal
+    from forge.services.knowledge import KnowledgeService
+
+    settings.chroma_path = str(tmp_path / "chroma_hy_vs")
+    t, p = "t_hyvs", "p_hyvs"
+    async with SessionLocal() as s:
+        for i, txt in enumerate([
+            "Refunds go to the original card within five business days.",
+            "Error code XJ9000 indicates a payment gateway timeout; retry after 30 seconds.",
+            "Cancel an order from the Orders page before it ships.",
+        ]):
+            src = await KnowledgeService.create_source(s, t, p, kind="text", name=f"d{i}", text=txt)
+            await KnowledgeService.ingest(s, src)
+        hits = await KnowledgeService.search(s, t, p, "XJ9000 timeout", top_k=3, hybrid=True)
+    assert hits
+    assert hits[0].score == 1.0  # fused rank still normalized to 1.0 at the top
+    # vector_score is the underlying cosine (0..1), a different scale from the fused score.
+    assert any(h.vector_score is not None for h in hits)
+    assert all(h.vector_score is None or 0.0 <= h.vector_score <= 1.0 for h in hits)
+
+
+async def test_ingest_persists_source_citation_metadata(tmp_path):
+    _require_embedder()
+    from forge.config import settings
+    from forge.db.base import SessionLocal
+    from forge.services.knowledge import KnowledgeService
+
+    settings.chroma_path = str(tmp_path / "chroma_cite")
+    t, p = "t_cite", "p_cite"
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(
+            s, t, p, kind="text", name="Refund Policy",
+            text="Refunds are issued to the original payment method within five business days.",
+        )
+        await KnowledgeService.ingest(s, src)
+        hits = await KnowledgeService.search(s, t, p, "refund timing", top_k=2)
+    assert hits
+    assert hits[0].metadata.get("source_name") == "Refund Policy"
+    assert hits[0].metadata.get("embedding_model") == "BAAI/bge-small-en-v1.5"
+    assert citation_for(hits[0].metadata) == "Refund Policy"
+
+
+async def test_ingest_clamps_chunk_size_to_embedder_limit(tmp_path):
+    _require_embedder()
+    from forge.config import settings
+    from forge.db.base import SessionLocal
+    from forge.models import Project
+    from forge.services.knowledge import KnowledgeService
+
+    settings.chroma_path = str(tmp_path / "chroma_clamp")
+    async with SessionLocal() as s:
+        proj = Project(tenant_id="t_cl", name="Cl", slug="clamp", config={"rag_defaults": {"chunk_size": 100000}})
+        s.add(proj)
+        await s.commit()
+        await s.refresh(proj)
+        src = await KnowledgeService.create_source(s, "t_cl", proj.id, kind="text", name="big", text="word " * 800)
+        src = await KnowledgeService.ingest(s, src)
+    assert src.status == "ready"
+    assert src.chunk_size <= _max_input_chars("BAAI/bge-small-en-v1.5")  # clamped down
+    assert (src.meta or {}).get("chunk_size_requested") == 100000  # original recorded
+
+
+async def test_memory_recall_similarity_floor(tmp_path):
+    _require_embedder()
+    from forge.config import settings
+    from forge.db.base import SessionLocal
+    from forge.services.memory import MemoryService
+
+    settings.chroma_path = str(tmp_path / "chroma_memfloor")
+    t, p = "t_memf", "p_memf"
+    async with SessionLocal() as s:
+        await MemoryService.remember(s, t, p, "Our refund window is 30 days.")
+
+    original = settings.memory_recall_min_score
+    try:
+        settings.memory_recall_min_score = 0.6
+        async with SessionLocal() as s:
+            off = await MemoryService.recall(s, t, p, "the capital of France", top_k=5)
+        assert off == []  # unrelated memory filtered by the floor
+        settings.memory_recall_min_score = 0.0
+        async with SessionLocal() as s:
+            on = await MemoryService.recall(s, t, p, "the capital of France", top_k=5)
+        assert on  # floor off (default) -> nearest returned regardless of distance
+    finally:
+        settings.memory_recall_min_score = original

apps/api/tests/test_rag_upgrades.py

@@ -0,0 +1,188 @@
+"""Production RAG upgrades: cross-encoder rerank (graceful), native semantic chunking,
+and parent-child retrieval. Q&A logic is deliberately NOT exercised here - it is unchanged."""
+
+from __future__ import annotations
+
+from forge.db.base import SessionLocal
+from forge.knowledge.rerank import _sigmoid, rerank_hits
+from forge.knowledge.splitter import _percentile, chunk_text
+from forge.knowledge.store import Hit
+from forge.models import Project
+from forge.services.knowledge import KnowledgeService
+
+# --- reranker: pure + graceful degradation (no model download needed) ---
+
+
+def test_sigmoid_bounds():
+    assert _sigmoid(0.0) == 0.5
+    assert _sigmoid(20) > 0.99 and _sigmoid(-20) < 0.01
+
+
+def test_rerank_missing_model_is_identity_truncated_to_top_k():
+    hits = [Hit(id=str(i), text=f"doc {i}", score=0.9 - i * 0.1, metadata={}) for i in range(5)]
+    out = rerank_hits("anything", hits, top_k=3, model="does/not-exist")
+    assert [h.id for h in out] == ["0", "1", "2"]  # unchanged order, capped
+
+
+def test_rerank_empty_hits_and_empty_query():
+    assert rerank_hits("q", [], top_k=3) == []
+    hits = [Hit(id="a", text="x", score=0.5, metadata={})]
+    assert rerank_hits("  ", hits, top_k=3) == hits  # blank query short-circuits
+
+
+def test_rerank_reorders_by_relevance():
+    """Real cross-encoder (small, cached after first run) must lift the relevant doc."""
+    hits = [
+        Hit(id="weather", text="The weather in Paris is mild in spring.", score=0.9, metadata={}),
+        Hit(id="fruit", text="Bananas are a good source of potassium.", score=0.8, metadata={}),
+        Hit(id="reset", text="To reset your password, use the Forgot Password link on the login screen.", score=0.1, metadata={}),
+    ]
+    out = rerank_hits("how do I reset my password?", hits, top_k=2)
+    assert out[0].id == "reset"
+    assert 0.0 <= out[0].score <= 1.0
+
+
+# --- semantic chunking: pure, deterministic via a fake embedder ---
+
+
+def test_percentile_interpolates():
+    assert _percentile([0.0, 1.0], 50) == 0.5
+    assert _percentile([1.0], 95) == 1.0
+    assert _percentile([], 95) == 0.0
+
+
+def test_semantic_splits_on_topic_shift():
+    # A fake embedder: refund sentences -> [1,0]; rocket sentences -> [0,1]. The single
+    # boundary between the two topics is the sharpest similarity drop -> exactly one cut.
+    refunds = "Refunds are issued to your original card. We process them within five days. Contact support for status."
+    rockets = "Rockets burn liquid oxygen. The first stage separates after ascent. Reentry heats the shield."
+    text = refunds + " " + rockets
+
+    def fake_embed(sentences):
+        return [[1.0, 0.0] if "efund" in s or "process" in s or "support" in s else [0.0, 1.0] for s in sentences]
+
+    chunks = chunk_text(text, strategy="semantic", chunk_size=1000, overlap=0, embed_fn=fake_embed)
+    assert len(chunks) == 2
+    assert "Refunds" in chunks[0] and "Rockets" in chunks[1]
+
+
+def test_semantic_without_embed_fn_falls_back_to_recursive():
+    text = ("Sentence one here. Sentence two here. Sentence three here. " * 20).strip()
+    got = chunk_text(text, strategy="semantic", chunk_size=200, overlap=40)
+    expected = chunk_text(text, strategy="recursive", chunk_size=200, overlap=40)
+    assert got == expected
+
+
+def test_semantic_too_few_sentences_falls_back():
+    def boom(_s):
+        raise AssertionError("embed_fn must not be called for <3 sentences")
+
+    assert chunk_text("Only one sentence here.", strategy="semantic", embed_fn=boom) == ["Only one sentence here."]
+
+
+# --- semantic + parent-child: ingest wiring (uses the real local embedder) ---
+
+
+async def _make_project(slug: str, rag_defaults: dict) -> str:
+    async with SessionLocal() as s:
+        proj = Project(tenant_id="t_rag", name="Rag", slug=slug, config={"rag_defaults": rag_defaults})
+        s.add(proj)
+        await s.commit()
+        await s.refresh(proj)
+        return proj.id
+
+
+async def test_semantic_ingest_does_not_crash(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_sem")
+    pid = await _make_project("rag-semantic", {"chunking_strategy": "semantic"})
+    text = ("Refunds go to the original card within five business days. Shipping takes two days. "
+            "Error XJ9000 is a gateway timeout; retry after 30 seconds. Our office is in Berlin. ") * 3
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_rag", pid, kind="text", name="d", text=text)
+        src = await KnowledgeService.ingest(s, src)
+    assert src.status == "ready"
+    assert src.chunks >= 1
+    assert src.chunking_strategy == "semantic"
+
+
+async def test_parent_child_ingest_and_retrieval(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_pc")
+    pid = await _make_project("rag-parentchild", {"retrieval_mode": "parent_child", "chunk_size": 400, "child_chunk_size": 120})
+    parent_a = ("Refund policy. Refunds are issued to the original payment method within five to "
+                "seven business days once the returned item is received and inspected at our warehouse.")
+    parent_b = ("Shipping policy. Standard orders ship within two business days and arrive in about "
+                "a week; expedited shipping is available at checkout for an additional fee.")
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_rag", pid, kind="text", name="policies",
+                                                   text=parent_a + "\n\n" + parent_b)
+        src = await KnowledgeService.ingest(s, src)
+        assert src.status == "ready"
+        assert (src.meta or {}).get("retrieval_mode") == "parent_child"
+        assert (src.meta or {}).get("parents", 0) >= 1
+        assert src.chunks >= (src.meta or {}).get("parents")  # at least one child per parent
+
+        hits = await KnowledgeService.search(s, "t_rag", pid, "how long does a refund take?", top_k=2)
+    assert hits
+    top = hits[0]
+    # Retrieval returns the PARENT window (the full paragraph), not just the matched child slice.
+    assert "five to seven business days" in top.text
+    assert top.metadata.get("parent_id")
+    # De-dup by parent: no two returned hits share a parent_id.
+    parent_ids = [h.metadata.get("parent_id") for h in hits]
+    assert len(parent_ids) == len(set(parent_ids))
+
+
+async def test_chunk_map_projects_and_marks_retrieved(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_map")
+    pid = await _make_project("rag-map", {"chunk_size": 200})
+    async with SessionLocal() as s:
+        for i, t in enumerate([
+            "Refunds go to the original payment method within five business days.",
+            "Error code XJ9000 indicates a payment gateway timeout; retry after 30 seconds.",
+            "Standard orders ship within two business days from our warehouse.",
+        ]):
+            src = await KnowledgeService.create_source(s, "t_rag", pid, kind="text", name=f"d{i}", text=t)
+            await KnowledgeService.ingest(s, src)
+        res = await KnowledgeService.chunk_map(s, "t_rag", pid, query="how long for a refund?", top_k=2)
+    assert res["total"] >= 3
+    assert len(res["points"]) >= 3
+    # every point has 2-D coords + a source
+    for p in res["points"]:
+        assert isinstance(p["x"], float) and isinstance(p["y"], float)
+        assert p["source_id"]
+    assert res["query_point"] and len(res["query_point"]) == 2
+    assert res["sources"]  # legend populated
+    # the query overlay tagged at least one chunk as retrieved (rank 1..top_k)
+    ranks = [p.get("retrieved") for p in res["points"] if p.get("retrieved")]
+    assert ranks and min(ranks) == 1
+
+
+async def test_chunk_map_empty_project_is_safe(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_map_empty")
+    pid = await _make_project("rag-map-empty", {})
+    async with SessionLocal() as s:
+        res = await KnowledgeService.chunk_map(s, "t_rag", pid, query="anything")
+    assert res == {"points": [], "sources": [], "query_point": None, "query": "anything", "total": 0, "truncated": False}
+
+
+async def test_flat_mode_unchanged(tmp_path):
+    """Default (no retrieval_mode) still returns plain chunks with no parent metadata."""
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_flat")
+    pid = await _make_project("rag-flat", {})
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_rag", pid, kind="text", name="d",
+                                                   text="Refunds are issued within five business days.")
+        await KnowledgeService.ingest(s, src)
+        hits = await KnowledgeService.search(s, "t_rag", pid, "refund time", top_k=3)
+    assert hits
+    assert not hits[0].metadata.get("parent_id")  # flat: no parent-child metadata

apps/api/tests/test_ratelimit.py

@@ -0,0 +1,37 @@
+"""Rate limiter + idempotency cache unit tests."""
+
+from __future__ import annotations
+
+from forge.util.ratelimit import IdempotencyCache, RateLimiter
+
+
+def test_token_bucket_blocks_after_burst():
+    rl = RateLimiter()
+    # rate=5/min, burst=5 → first 5 allowed, 6th blocked
+    allowed = [rl.allow("k", rate=5, per=60, burst=5) for _ in range(6)]
+    assert allowed == [True, True, True, True, True, False]
+
+
+def test_zero_rate_is_unlimited():
+    rl = RateLimiter()
+    assert all(rl.allow("k", rate=0) for _ in range(100))
+
+
+def test_keys_are_independent():
+    rl = RateLimiter()
+    assert rl.allow("a", rate=1, burst=1) is True
+    assert rl.allow("a", rate=1, burst=1) is False
+    assert rl.allow("b", rate=1, burst=1) is True  # different key unaffected
+
+
+def test_idempotency_returns_stored_value():
+    cache = IdempotencyCache(ttl_seconds=60)
+    assert cache.get("x") is None
+    cache.put("x", {"run_id": "r1"})
+    assert cache.get("x") == {"run_id": "r1"}
+
+
+def test_idempotency_expires():
+    cache = IdempotencyCache(ttl_seconds=-1)  # already expired
+    cache.put("x", 1)
+    assert cache.get("x") is None

apps/api/tests/test_rechunk_and_fastembed.py

@@ -0,0 +1,97 @@
+"""Per-source re-chunking overrides (strategy / size / overlap) and the local
+open-source fastembed embedder resolution."""
+
+from __future__ import annotations
+
+import pytest
+
+from forge.db.base import SessionLocal
+from forge.knowledge.embeddings import resolve_embedder
+from forge.services.knowledge import KnowledgeService
+
+# --- re-chunk overrides ---
+
+_BIG = " ".join(f"Refund policy note {i}: widgets ship within five business days." for i in range(120))
+
+
+async def test_ingest_records_chunk_settings(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_rec")
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_rec", "p_rec", kind="text", name="big", text=_BIG)
+        src = await KnowledgeService.ingest(s, src)
+    # Defaults (no project rag_defaults) are recorded in meta so the UI can display them.
+    assert src.chunking_strategy == "recursive"
+    assert src.chunk_size == 1000
+    assert src.chunk_overlap == 200
+    assert src.chunks >= 1
+
+
+async def test_rechunk_smaller_size_yields_more_chunks(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_rc")
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_rc", "p_rc", kind="text", name="big", text=_BIG)
+        src = await KnowledgeService.ingest(s, src)
+        base = src.chunks
+        # Shrinking the chunk size must split the same text into more pieces...
+        src = await KnowledgeService.rechunk(s, src, chunk_size=200, chunk_overlap=20)
+    assert src.chunk_size == 200
+    assert src.chunk_overlap == 20
+    assert src.chunks > base
+
+
+async def test_rechunk_honors_zero_overlap(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_ov0")
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_ov0", "p_ov0", kind="text", name="big", text=_BIG)
+        src = await KnowledgeService.ingest(s, src)
+        # An explicit 0 overlap must stick (not silently reset to the 200 default).
+        src = await KnowledgeService.rechunk(s, src, chunk_size=300, chunk_overlap=0)
+    assert src.chunk_overlap == 0
+    assert src.chunk_size == 300
+
+
+async def test_rechunk_strategy_only_preserves_size(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_rcs")
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_rcs", "p_rcs", kind="text", name="big", text=_BIG)
+        src = await KnowledgeService.ingest(s, src)
+        # ...while a strategy-only re-chunk leaves size/overlap untouched (None = keep).
+        src = await KnowledgeService.rechunk(s, src, chunking_strategy="sentence")
+    assert src.chunking_strategy == "sentence"
+    assert src.chunk_size == 1000
+    assert src.chunk_overlap == 200
+
+
+# --- fastembed resolution (local, open-source) ---
+
+
+def test_fastembed_resolves_local_model():
+    pytest.importorskip("fastembed")
+    try:
+        e = resolve_embedder("fastembed:BAAI/bge-small-en-v1.5")
+    except RuntimeError:
+        # No toy fallback anymore - resolve_embedder raises if the model can't load
+        # (e.g. offline with no cached model). Skip rather than fail the suite.
+        pytest.skip("fastembed model could not be loaded (offline)")
+    assert e.name == "BAAI/bge-small-en-v1.5"
+    assert e.dim == 384
+    assert len(e.embed_query("hello world")) == 384
+
+
+def test_default_embedder_is_local_fastembed():
+    """With no model ref, resolve_embedder returns the local open-source default
+    (no FakeEmbedder). Guards the 'remove fake embedder' decision."""
+    pytest.importorskip("fastembed")
+    try:
+        e = resolve_embedder(None)
+    except RuntimeError:
+        pytest.skip("fastembed model could not be loaded (offline)")
+    assert e.name == "BAAI/bge-small-en-v1.5" and e.dim == 384

apps/api/tests/test_rechunk_no_stale.py

@@ -0,0 +1,93 @@
+"""Re-chunk / re-embed must REPLACE a source's vectors, never accumulate stale ones.
+
+Guards the "chunk count keeps increasing every time I re-embed" class of bug: reingest deletes
+the source's old vectors (in the current dim collection AND every known dim collection) before
+re-ingesting, so the store row count always equals the source's reported chunk count.
+"""
+from __future__ import annotations
+
+from forge.db.base import SessionLocal
+from forge.knowledge.store import _where
+from forge.models import Project
+from forge.services.knowledge import KnowledgeService
+
+_TEXT = ("Refunds go to the original card within five business days. Shipping takes two days. "
+         "Error XJ9000 is a gateway timeout; retry after 30 seconds. Our office is in Berlin. ") * 5
+
+
+async def _make_project(slug: str, rag_defaults: dict) -> str:
+    async with SessionLocal() as s:
+        proj = Project(tenant_id="t_rc", name="Rc", slug=slug, config={"rag_defaults": rag_defaults})
+        s.add(proj)
+        await s.commit()
+        await s.refresh(proj)
+        return proj.id
+
+
+def _rows(embedder, pid, sid) -> int:
+    return KnowledgeService._store(embedder).count_where(_where("t_rc", pid, [sid]))
+
+
+async def test_rechunk_replaces_not_accumulates(tmp_path):
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_rc")
+    pid = await _make_project("rc", {"chunk_size": 200, "chunk_overlap": 0})
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_rc", pid, kind="text", name="d", text=_TEXT)
+        src = await KnowledgeService.ingest(s, src)
+        embedder = await KnowledgeService.embedder_for_project(s, "t_rc", pid)
+        assert _rows(embedder, pid, src.id) == src.chunks
+
+        # Bigger chunks -> fewer rows; smaller -> more. Row count must track src.chunks each time,
+        # proving the previous run's vectors were deleted, not left behind.
+        src = await KnowledgeService.rechunk(s, src, chunk_size=4000, chunk_overlap=0)
+        assert _rows(embedder, pid, src.id) == src.chunks
+        src = await KnowledgeService.rechunk(s, src, chunk_size=150, chunk_overlap=0)
+        assert _rows(embedder, pid, src.id) == src.chunks
+
+
+async def test_reembed_same_settings_stays_flat(tmp_path):
+    """The exact 'Apply & re-embed' path (run_ingest_bg with reingest=True) repeated with
+    UNCHANGED settings must not grow the row count."""
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_bg")
+    pid = await _make_project("bg", {"chunk_size": 180, "chunk_overlap": 0})
+    async with SessionLocal() as s:
+        src = await KnowledgeService.create_source(s, "t_rc", pid, kind="text", name="d", text=_TEXT)
+        sid = src.id
+        embedder = await KnowledgeService.embedder_for_project(s, "t_rc", pid)
+
+    counts = []
+    for _ in range(3):
+        await KnowledgeService.run_ingest_bg("t_rc", sid, reingest=True)
+        counts.append(_rows(embedder, pid, sid))
+    assert counts[0] == counts[1] == counts[2], f"re-embed accumulated stale rows: {counts}"
+
+
+async def test_dedupe_removes_exact_duplicate_chunks(tmp_path):
+    """The same document ingested twice -> identical chunks; dedupe keeps one copy of each and
+    fixes the affected sources' counts."""
+    from forge.config import settings
+
+    settings.chroma_path = str(tmp_path / "chroma_dedupe")
+    pid = await _make_project("dedupe", {"chunk_size": 200, "chunk_overlap": 0})
+    async with SessionLocal() as s:
+        a = await KnowledgeService.create_source(s, "t_rc", pid, kind="text", name="dup.md", text=_TEXT)
+        a = await KnowledgeService.ingest(s, a)
+        b = await KnowledgeService.create_source(s, "t_rc", pid, kind="text", name="dup-copy.md", text=_TEXT)
+        b = await KnowledgeService.ingest(s, b)
+        embedder = await KnowledgeService.embedder_for_project(s, "t_rc", pid)
+
+        before = _rows(embedder, pid, a.id) + _rows(embedder, pid, b.id)
+        res = await KnowledgeService.dedupe_chunks(s, "t_rc", pid)
+        after = _rows(embedder, pid, a.id) + _rows(embedder, pid, b.id)
+
+    # Two identical docs -> every chunk had exactly one duplicate; half are removed.
+    assert res["removed"] == before // 2
+    assert res["remaining"] == after
+    # A second run is a no-op (idempotent).
+    async with SessionLocal() as s:
+        res2 = await KnowledgeService.dedupe_chunks(s, "t_rc", pid)
+    assert res2["removed"] == 0

apps/api/tests/test_redirect_capture.py

@@ -0,0 +1,229 @@
+"""Service-tool redirect handling.
+
+Two behaviours, both new:
+1. A 3xx that is NOT followed (the default) is no longer an empty response - the
+   target `location` is captured and surfaced to the model.
+2. With `request.follow_redirects` on, redirects are chased SSRF-safely via
+   `guarded_request`: the final URL + hop chain are captured, and a hop pointing at
+   a blocked (private/metadata) address is refused.
+"""
+
+from __future__ import annotations
+
+import types
+import uuid
+
+import httpx
+import pytest
+
+from forge.tools import rest as rest_mod
+from forge.tools.rest import build_rest_tool, execute_rest
+from forge.util.ssrf import EgressBlocked, EgressPolicy
+
+
+def _cfg(**extra) -> dict:
+    return {
+        "name": f"t_{uuid.uuid4().hex[:8]}",
+        "kind": "rest_api",
+        "request": {"method": "GET", "url_template": "https://api.acme.dev/go", "fields": []},
+        **extra,
+    }
+
+
+def _redirecting_client(location: str, *, status: int = 302) -> httpx.AsyncClient:
+    """Mock client: /go -> 3xx(location); the target -> 200 JSON."""
+    def handler(req: httpx.Request) -> httpx.Response:
+        if req.url.path == "/go":
+            return httpx.Response(status, headers={"location": location})
+        return httpx.Response(200, json={"arrived": True, "path": req.url.path})
+
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+
+# --- 1. capture without following (default) --------------------------------------
+
+async def test_unfollowed_3xx_captures_location():
+    client = _redirecting_client("https://api.acme.dev/v2/final")
+    res = await execute_rest(_cfg(), {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+
+    assert res["status"] == 302
+    assert res["redirect"] is not None
+    assert res["redirect"]["followed"] is False
+    assert res["redirect"]["location"] == "https://api.acme.dev/v2/final"
+
+
+async def test_unfollowed_redirect_is_wrapped_for_the_agent():
+    """The StructuredTool the agent calls must surface the redirect, not an empty body."""
+    client = _redirecting_client("https://api.acme.dev/v2/final")
+    rest_mod_select = rest_mod.select_client
+    rest_mod.select_client = lambda *a, **k: client  # type: ignore[assignment]
+    try:
+        ctx = types.SimpleNamespace(tenant_id="t", project_id="p", auth_resolver=None, egress_policy=None)
+        tool = build_rest_tool(_cfg(), ctx)
+        out = await tool.ainvoke({})
+    finally:
+        rest_mod.select_client = rest_mod_select  # type: ignore[assignment]
+        await client.aclose()
+
+    assert isinstance(out, dict) and "redirect" in out
+    assert out["redirect"]["location"] == "https://api.acme.dev/v2/final"
+
+
+async def test_tool_test_preview_counts_the_redirect_not_an_empty_body():
+    """The /test preview (ToolService.test) must reflect what the agent actually gets:
+    the {body, redirect} envelope, not the empty 3xx body. Otherwise 'PROJECTED -> MODEL'
+    reads "" / 0 tok and looks like nothing reaches the model."""
+    from forge.services.tools import ToolService
+
+    client = _redirecting_client("/shop/addproduct/PROD-001")
+    rest_mod_select = rest_mod.select_client
+    rest_mod.select_client = lambda *a, **k: client  # type: ignore[assignment]
+    try:
+        r = await ToolService.test("t", "p", _cfg(), {})
+    finally:
+        rest_mod.select_client = rest_mod_select  # type: ignore[assignment]
+        await client.aclose()
+
+    assert r["ok"] and r["redirect"]["location"] == "/shop/addproduct/PROD-001"
+    # raw + projected now carry the observation shape, and the location survives into it.
+    assert isinstance(r["projected"], dict)
+    assert r["projected"]["redirect"]["location"] == "/shop/addproduct/PROD-001"
+    assert isinstance(r["raw"], dict) and "redirect" in r["raw"]
+    # ...so the meter shows a real cost instead of the old 0 tok.
+    assert r["projected_tokens"] > 0 and r["raw_tokens"] > 0
+
+
+async def test_tool_test_preview_leaves_non_redirect_body_bare():
+    """No redirect -> the preview stays the bare body (unchanged), not an envelope."""
+    from forge.services.tools import ToolService
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(200, json={"v": 1})))
+    rest_mod_select = rest_mod.select_client
+    rest_mod.select_client = lambda *a, **k: client  # type: ignore[assignment]
+    try:
+        r = await ToolService.test("t", "p", _cfg(), {})
+    finally:
+        rest_mod.select_client = rest_mod_select  # type: ignore[assignment]
+        await client.aclose()
+
+    assert r["ok"] and r["redirect"] is None
+    assert r["projected"] == {"v": 1} and r["raw"] == {"v": 1}  # bare body, not wrapped
+
+
+# --- 3. projecting the redirect location (strip a captured 3xx to just the URL) ----
+
+async def test_projection_can_select_the_redirect_location():
+    """`redirect.location` must reach the redirect envelope and collapse the observation to just
+    the target URL - so a 3xx can be stripped to the one thing the model needs, cheaply."""
+    from forge.services.tools import ToolService
+
+    client = _redirecting_client("/shop/addproduct/PROD-002")
+    cfg = _cfg(response={"projection_jmespath": "redirect.location"})
+    rest_mod_select = rest_mod.select_client
+    rest_mod.select_client = lambda *a, **k: client  # type: ignore[assignment]
+    try:
+        r = await ToolService.test("t", "p", cfg, {})
+    finally:
+        rest_mod.select_client = rest_mod_select  # type: ignore[assignment]
+        await client.aclose()
+
+    assert r["ok"]
+    assert r["projected"] == "/shop/addproduct/PROD-002"  # just the URL, not the envelope
+    assert 0 < r["projected_tokens"] < r["raw_tokens"]  # stripped -> cheaper than the full envelope
+
+
+async def test_tool_returns_bare_redirect_location_to_the_agent_when_projected():
+    """The agent-facing observation (not just the preview) is the bare URL string."""
+    client = _redirecting_client("/shop/addproduct/PROD-002")
+    cfg = _cfg(response={"projection_jmespath": "redirect.location"})
+    rest_mod_select = rest_mod.select_client
+    rest_mod.select_client = lambda *a, **k: client  # type: ignore[assignment]
+    try:
+        ctx = types.SimpleNamespace(tenant_id="t", project_id="p", auth_resolver=None, egress_policy=None)
+        tool = build_rest_tool(cfg, ctx)
+        out = await tool.ainvoke({})
+    finally:
+        rest_mod.select_client = rest_mod_select  # type: ignore[assignment]
+        await client.aclose()
+
+    assert out == "/shop/addproduct/PROD-002"
+
+
+async def test_no_redirect_returns_bare_body_unchanged():
+    """A normal 200 must behave exactly as before - no envelope, no redirect key."""
+    client = httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(200, json={"v": 1})))
+    res = await execute_rest(_cfg(), {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+
+    assert res["redirect"] is None
+    from forge.tools.rest import _tool_return
+
+    assert _tool_return(res, _cfg()) == {"v": 1}  # bare body, not wrapped
+
+
+# --- 2. SSRF-safe following ------------------------------------------------------
+
+async def test_follow_resolves_final_url_and_chain():
+    client = _redirecting_client("https://api.acme.dev/v2/final")
+    cfg = _cfg()
+    cfg["request"]["follow_redirects"] = True
+    res = await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+
+    assert res["status"] == 200 and res["raw"] == {"arrived": True, "path": "/v2/final"}
+    assert res["redirect"]["followed"] is True
+    assert res["redirect"]["final_url"] == "https://api.acme.dev/v2/final"
+    assert res["redirect"]["chain"] == ["https://api.acme.dev/go"]
+
+
+async def test_follow_revalidates_each_hop_and_blocks_private_target():
+    """A redirect to a blocked address (cloud metadata) must be refused on the hop -
+    the SSRF guard is not bypassed by following."""
+    client = _redirecting_client("http://169.254.169.254/latest/meta-data/")
+    cfg = _cfg()
+    cfg["request"]["url_template"] = "https://8.8.8.8/go"  # public literal so the first hop passes
+    cfg["request"]["follow_redirects"] = True
+    with pytest.raises(EgressBlocked):
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client,
+                           egress_policy=EgressPolicy(block_private=True))
+    await client.aclose()
+
+
+def _auth_capturing_client(location: str) -> tuple[httpx.AsyncClient, dict]:
+    """Mock client that records the Authorization header seen at each hop."""
+    seen: dict[str, str | None] = {}
+
+    def handler(req: httpx.Request) -> httpx.Response:
+        seen[str(req.url)] = req.headers.get("authorization")
+        if req.url.path == "/go":
+            return httpx.Response(302, headers={"location": location})
+        return httpx.Response(200, json={"ok": True})
+
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler)), seen
+
+
+async def test_follow_strips_authorization_on_cross_origin_redirect():
+    """SECURITY: a redirect to a DIFFERENT origin must NOT forward the tenant's
+    Authorization header - otherwise following redirects exfiltrates credentials."""
+    client, seen = _auth_capturing_client("https://evil.example/collect")
+    cfg = _cfg()
+    cfg["request"]["headers"] = [{"name": "Authorization", "value": "Bearer SECRET-TOKEN"}]
+    cfg["request"]["follow_redirects"] = True
+    await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+
+    assert seen["https://api.acme.dev/go"] == "Bearer SECRET-TOKEN"  # original origin keeps it
+    assert seen["https://evil.example/collect"] is None  # foreign origin must NOT receive it
+
+
+async def test_follow_preserves_authorization_on_same_origin_redirect():
+    """A same-origin redirect should keep the Authorization header (e.g. /v1 -> /v2)."""
+    client, seen = _auth_capturing_client("https://api.acme.dev/v2/final")
+    cfg = _cfg()
+    cfg["request"]["headers"] = [{"name": "Authorization", "value": "Bearer SECRET-TOKEN"}]
+    cfg["request"]["follow_redirects"] = True
+    await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+
+    assert seen["https://api.acme.dev/v2/final"] == "Bearer SECRET-TOKEN"

apps/api/tests/test_rls_scoping.py

@@ -0,0 +1,30 @@
+"""Postgres RLS tenant-GUC wiring.
+
+Verifies the request-scoped tenant contextvar round-trips (used by the GUC listener in
+forge.db.base to set app.current_tenant per transaction) and that the listener is a harmless
+no-op on the SQLite test DB, so ordinary queries keep working. The GUC's effect on real RLS
+policies is exercised only against Postgres (see infra/postgres_rls.sql)."""
+
+from __future__ import annotations
+
+from sqlalchemy import text
+
+from forge.db.base import SessionLocal
+from forge.db.scoping import current_tenant, set_current_tenant, tenant_guard
+
+
+def test_tenant_contextvar_roundtrip():
+    set_current_tenant(None)
+    assert current_tenant() is None
+    set_current_tenant("t-1")
+    assert current_tenant() == "t-1"
+    with tenant_guard("t-2"):
+        assert current_tenant() == "t-2"
+    assert current_tenant() == "t-1"
+    set_current_tenant(None)
+
+
+async def test_sqlite_session_unaffected_by_guc():
+    with tenant_guard("any-tenant"):
+        async with SessionLocal() as s:
+            assert (await s.execute(text("SELECT 1"))).scalar() == 1

apps/api/tests/test_run_context.py

@@ -0,0 +1,296 @@
+"""Per-run context injection (Feature: ephemeral per-run `run_context`).
+
+A server-side caller passes an `X-Forge-Context` header on a run's EXECUTION request (stream /
+resume); its values reach tools as {{ctx.<key>}} for on-behalf-of injection (e.g. a per-user
+API key or tenant identifier). They are NEVER persisted, NEVER placed in the LLM prompt, NEVER an
+LLM-visible arg, and cannot be overridden by an LLM-supplied value.
+
+Three lanes are kept distinct (the invariant these tests protect):
+  - input parameters : `fields` with llm_visible=True -> the MODEL decides them (in args_schema)
+  - injected context : {{ctx.*}} from run_context -> the SERVER injects them (not in schema)
+  - auth providers   : project-scoped stored secrets (covered elsewhere)
+"""
+
+import json
+
+import httpx
+import pytest
+from fastapi import HTTPException
+
+from forge.deps import FORGE_CONTEXT_HEADER, run_context
+from forge.engine.context import CompileContext
+from forge.tools.rest import build_args_schema, build_rest_tool, execute_rest
+
+
+def _capturing_client(sink: dict) -> httpx.AsyncClient:
+    async def handler(request: httpx.Request) -> httpx.Response:
+        sink["request"] = request
+        return httpx.Response(200, json={"ok": True})
+
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+
+# --- consumption: values reach the outbound request ---------------------------------------
+
+
+async def test_header_injection_from_context():
+    sink: dict = {}
+    cfg = {
+        "name": "items_list",
+        "request": {
+            "method": "GET",
+            "url_template": "https://api.example.dev/items",
+            "fields": [],
+            "headers": [
+                {"name": "Authorization", "value": "Bearer {{ctx.token}}"},
+                {"name": "X-Tenant-Id", "value": "{{ctx.tenant}}"},
+            ],
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(
+            cfg, {}, tenant_id="t", project_id="p",
+            context={"token": "tok-abc", "tenant": "acme"}, client=client,
+        )
+    req = sink["request"]
+    assert req.headers["authorization"] == "Bearer tok-abc"
+    assert req.headers["x-tenant-id"] == "acme"
+
+
+async def test_body_field_injection_and_hidden_from_schema():
+    """A non-llm-visible field with a {{ctx.*}} default injects into the body AND is not exposed
+    to the model (not in the tool args schema) - the input-vs-injected separation."""
+    sink: dict = {}
+    cfg = {
+        "name": "create_item",
+        "request": {
+            "method": "POST",
+            "url_template": "https://api.example.dev/items",
+            "fields": [
+                {"path": "name", "type": "string", "in": "body", "required": True, "llm_visible": True},
+                {"path": "api_key", "in": "body", "llm_visible": False, "default": "{{ctx.api_key}}"},
+            ],
+            "headers": [],
+        },
+    }
+    schema = build_args_schema(cfg)
+    assert "name" in schema.model_fields      # the model DOES decide this
+    assert "api_key" not in schema.model_fields  # the server injects this; model never sees it
+
+    async with _capturing_client(sink) as client:
+        await execute_rest(
+            cfg, {"name": "widget"}, tenant_id="t", project_id="p",
+            context={"api_key": "sk-123"}, client=client,
+        )
+    assert json.loads(sink["request"].content) == {"name": "widget", "api_key": "sk-123"}
+
+
+async def test_query_field_injection():
+    sink: dict = {}
+    cfg = {
+        "name": "search_items",
+        "request": {
+            "method": "GET",
+            "url_template": "https://api.example.dev/items/search",
+            "fields": [{"path": "api_key", "in": "query", "llm_visible": False, "default": "{{ctx.api_key}}"}],
+            "headers": [],
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", context={"api_key": "sk-123"}, client=client)
+    assert sink["request"].url.params["api_key"] == "sk-123"
+
+
+async def test_url_template_ctx_injection():
+    """{{ctx.*}} is honored directly in the URL/query string too."""
+    sink: dict = {}
+    cfg = {
+        "name": "x",
+        "request": {
+            "method": "GET",
+            "url_template": "https://api.example.dev/items?token={{ctx.token}}",
+            "fields": [],
+            "headers": [],
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", context={"token": "tok-abc"}, client=client)
+    assert sink["request"].url.params["token"] == "tok-abc"
+
+
+async def test_cookie_field_injection():
+    sink: dict = {}
+    cfg = {
+        "name": "x",
+        "request": {
+            "method": "GET",
+            "url_template": "https://api.example.dev/x",
+            "fields": [{"path": "sid", "in": "cookie", "llm_visible": False, "default": "{{ctx.session}}"}],
+            "headers": [],
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", context={"session": "s-xyz"}, client=client)
+    assert "sid=s-xyz" in sink["request"].headers.get("cookie", "")
+
+
+async def test_body_template_json_with_input_and_ctx():
+    """A free-form JSON body template mixes {{input.*}} (model args) and {{ctx.*}} (injected)."""
+    sink: dict = {}
+    cfg = {
+        "name": "create_item",
+        "request": {
+            "method": "POST",
+            "url_template": "https://api.example.dev/items",
+            "fields": [{"path": "amount", "type": "integer", "in": "body", "llm_visible": True}],
+            "headers": [],
+            "body_template": '{"amount": {{ input.amount }}, "api_key": "{{ ctx.api_key }}"}',
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {"amount": 5}, tenant_id="t", project_id="p", context={"api_key": "sk-123"}, client=client)
+    assert json.loads(sink["request"].content) == {"amount": 5, "api_key": "sk-123"}
+
+
+async def test_body_template_non_json_sent_raw():
+    """A body template that isn't JSON is sent as raw bytes unchanged."""
+    sink: dict = {}
+    cfg = {
+        "name": "x",
+        "request": {
+            "method": "POST",
+            "url_template": "https://api.example.dev/x",
+            "fields": [],
+            "headers": [],
+            "body_template": "token={{ctx.token}}&scope=all",
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", context={"token": "tok-abc"}, client=client)
+    assert sink["request"].content == b"token=tok-abc&scope=all"
+
+
+# --- security invariants -------------------------------------------------------------------
+
+
+async def test_ctx_header_is_authoritative_over_llm_field():
+    """An LLM-supplied header field must NOT override a server-injected ctx-templated header."""
+    sink: dict = {}
+    cfg = {
+        "name": "x",
+        "request": {
+            "method": "GET",
+            "url_template": "https://api.example.dev/x",
+            "fields": [{"path": "X-Api-Key", "type": "string", "in": "header", "llm_visible": True}],
+            "headers": [{"name": "X-Api-Key", "value": "{{ctx.api_key}}"}],
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(
+            cfg, {"X-Api-Key": "attacker-supplied"}, tenant_id="t", project_id="p",
+            context={"api_key": "sk-123"}, client=client,
+        )
+    assert sink["request"].headers["x-api-key"] == "sk-123"
+
+
+async def test_missing_context_value_is_dropped_not_literal():
+    """A {{ctx.*}} default with no matching context value is omitted - never sent as the raw
+    template string (regression for the _collect raw-default fallback)."""
+    sink: dict = {}
+    cfg = {
+        "name": "x",
+        "request": {
+            "method": "POST",
+            "url_template": "https://api.example.dev/x",
+            "fields": [{"path": "api_key", "in": "body", "llm_visible": False, "default": "{{ctx.api_key}}"}],
+            "headers": [],
+        },
+    }
+    async with _capturing_client(sink) as client:
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", context={}, client=client)
+    # No body was sent (the only field resolved to None and was dropped, not sent literally).
+    assert sink["request"].content in (b"", b"null")
+
+
+async def test_run_context_and_end_user_are_distinct_lanes():
+    """Via build_rest_tool: ctx.run_context supplies {{ctx.api_key}} while ctx.end_user supplies
+    {{ctx.end_user.id}} - both resolve, and the secret is not part of end_user (identity)."""
+    sink: dict = {}
+    cfg = {
+        "name": "on_behalf",
+        "request": {
+            "method": "GET",
+            "url_template": "https://api.example.dev/me",
+            "fields": [],
+            "headers": [
+                {"name": "X-Api-Key", "value": "{{ctx.api_key}}"},
+                {"name": "X-User", "value": "{{ctx.end_user.id}}"},
+            ],
+        },
+    }
+    ctx = CompileContext(tenant_id="t", project_id="p")
+    ctx.run_context = {"api_key": "sk-123"}
+    ctx.end_user = {"id": "u1"}
+
+    class _RT:
+        context: dict = {}
+        stream_writer = None
+
+    tool = build_rest_tool(cfg, ctx)
+    async with _capturing_client(sink) as client:
+        # execute_rest picks its client via select_client; point that at ours.
+        import forge.tools.rest as rest_mod
+
+        orig = rest_mod.select_client
+        rest_mod.select_client = lambda *a, **k: client  # type: ignore[assignment]
+        try:
+            await tool.coroutine(runtime=_RT())
+        finally:
+            rest_mod.select_client = orig  # type: ignore[assignment]
+
+    req = sink["request"]
+    assert req.headers["x-api-key"] == "sk-123"
+    assert req.headers["x-user"] == "u1"
+    # The secret is NOT part of identity (so it never reaches the end_user prompt block).
+    assert "api_key" not in ctx.end_user
+
+
+# --- transport: header parsing -------------------------------------------------------------
+
+
+class _Req:
+    def __init__(self, headers: dict):
+        self.headers = headers
+
+
+def test_run_context_parses_json_object():
+    r = _Req({FORGE_CONTEXT_HEADER: '{"token": "tok-abc", "tenant": "acme"}'})
+    assert run_context(r) == {"token": "tok-abc", "tenant": "acme"}
+
+
+def test_run_context_absent_is_none():
+    assert run_context(_Req({})) is None
+
+
+def test_run_context_strips_end_user():
+    r = _Req({FORGE_CONTEXT_HEADER: '{"end_user": {"id": "x"}, "token": "tok-abc"}'})
+    assert run_context(r) == {"token": "tok-abc"}  # identity is not settable via this channel
+
+
+def test_run_context_rejects_invalid_json():
+    with pytest.raises(HTTPException) as e:
+        run_context(_Req({FORGE_CONTEXT_HEADER: "not-json"}))
+    assert e.value.status_code == 400
+
+
+def test_run_context_rejects_non_object():
+    with pytest.raises(HTTPException) as e:
+        run_context(_Req({FORGE_CONTEXT_HEADER: '"just-a-string"'}))
+    assert e.value.status_code == 400
+
+
+def test_run_context_rejects_oversized():
+    big = json.dumps({"x": "a" * 9000})
+    with pytest.raises(HTTPException) as e:
+        run_context(_Req({FORGE_CONTEXT_HEADER: big}))
+    assert e.value.status_code == 413

apps/api/tests/test_secrets_auth_rest.py

@@ -0,0 +1,202 @@
+"""Secrets, auth resolver (csrf/bearer), REST tool + projection, and builtin /test validation."""
+
+from __future__ import annotations
+
+import httpx
+from sqlalchemy import select
+
+from forge.auth_providers.resolver import AuthResolver
+from forge.db.base import SessionLocal
+from forge.models import AuditLog, AuthProvider
+from forge.secrets.store import SecretStore
+from forge.services.tools import ToolService
+from forge.tools.rest import build_args_schema, execute_rest
+
+GET_ORDER = {
+    "name": "get_order",
+    "description": "Fetch an order.",
+    "kind": "rest_api",
+    "request": {
+        "method": "GET",
+        "url_template": "https://api.acme.dev/v2/orders/{order_id}",
+        "fields": [
+            {"path": "order_id", "type": "string", "in": "path", "required": True, "llm_visible": True},
+            {"path": "include", "type": "string", "in": "query", "required": False, "llm_visible": False, "default": "totals"},
+        ],
+        "headers": [{"name": "Accept", "value": "application/json"}],
+    },
+    "response": {"projection_jmespath": "data.{subtotal: totals.subtotal, total: totals.grand_total, status: status}"},
+}
+
+
+# --- secrets ---
+async def test_secret_roundtrip_encrypts_and_decrypts():
+    store = SecretStore()
+    async with SessionLocal() as s:
+        await store.write(s, tenant_id="t_sec", project_id="p_sec", name="creds", value={"u": "a", "p": "b"}, kind="generic")
+    got = await store.read_ref(tenant_id="t_sec", project_id="p_sec", ref="secret://proj/creds")
+    assert got == {"u": "a", "p": "b"}
+    async with SessionLocal() as s:
+        audit = (
+            await s.execute(
+                select(AuditLog).where(
+                    AuditLog.tenant_id == "t_sec",
+                    AuditLog.project_id == "p_sec",
+                    AuditLog.action == "secret.read",
+                    AuditLog.resource_type == "secret",
+                    AuditLog.resource_id == "creds",
+                )
+            )
+        ).scalar_one()
+    assert audit.meta == {"scheme": "secret"}
+
+
+# --- REST tool ---
+def test_args_schema_excludes_non_llm_visible_fields():
+    Args = build_args_schema(GET_ORDER)
+    assert set(Args.model_fields) == {"order_id"}  # `include` is hidden from the model
+
+
+async def test_rest_execute_projects_payload():
+    seen = {}
+
+    def handler(req: httpx.Request) -> httpx.Response:
+        seen["url"] = str(req.url)
+        return httpx.Response(200, json={
+            "data": {"totals": {"subtotal": 90, "grand_total": 99}, "line_items": [1, 2, 3, 4], "status": "open"},
+        })
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    res = await execute_rest(GET_ORDER, {"order_id": "A-1"}, tenant_id="t", project_id="p", context={}, auth_resolver=None, client=client)
+    await client.aclose()
+
+    assert "/orders/A-1" in seen["url"] and "include=totals" in seen["url"]
+    assert res["projected"] == {"subtotal": 90, "total": 99, "status": "open"}
+    assert res["raw"]["data"]["line_items"] == [1, 2, 3, 4]  # raw keeps everything
+
+
+# --- auth resolver ---
+async def test_csrf_session_extract_and_inject():
+    ap = AuthProvider(
+        id="ap1", tenant_id="t", project_id="p", name="orders", kind="csrf_session",
+        config={
+            "kind": "csrf_session",
+            "token_fetch": {"method": "POST", "url": "https://api.acme.dev/auth/login", "body": {}},
+            "extract": [
+                {"name": "csrf", "from": "header", "header": "X-CSRF-Token"},
+                {"name": "session", "from": "cookie", "cookie": "SESSIONID"},
+            ],
+            "inject": [
+                {"to": "header", "name": "X-CSRF-Token", "value": "{{extracted.csrf}}"},
+                {"to": "cookie", "name": "SESSIONID", "value": "{{extracted.session}}"},
+            ],
+            "cache_ttl_seconds": 1800,
+        },
+    )
+
+    def handler(req: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, headers=[("X-CSRF-Token", "abc123"), ("set-cookie", "SESSIONID=zzz; Path=/")])
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    resolved = await AuthResolver().resolve(tenant_id="t", project_id="p", provider_id="ap1", provider=ap, client=client, force=True)
+    await client.aclose()
+    assert resolved.headers["X-CSRF-Token"] == "abc123"
+    assert resolved.cookies["SESSIONID"] == "zzz"
+
+
+async def test_bearer_static_auth():
+    store = SecretStore()
+    async with SessionLocal() as s:
+        await store.write(s, tenant_id="t", project_id="p", name="tok", value="T0KEN", kind="bearer")
+    ap = AuthProvider(id="b1", tenant_id="t", project_id="p", name="b", kind="bearer", config={"kind": "bearer", "token_ref": "secret://proj/tok"})
+    resolved = await AuthResolver().resolve(tenant_id="t", project_id="p", provider_id="b1", provider=ap, force=True)
+    assert resolved.headers["Authorization"] == "Bearer T0KEN"
+
+
+# --- builtin tool /test ---
+def test_resolve_model_injects_project_provider_key(monkeypatch):
+    """A per-project key in ctx.provider_credentials is passed to init_chat_model."""
+    import langchain.chat_models as cm
+
+    captured: dict = {}
+
+    def fake_init(model, **kwargs):
+        captured["model"] = model
+        captured.update(kwargs)
+        return object()
+
+    monkeypatch.setattr(cm, "init_chat_model", fake_init)
+
+    from forge.engine.context import CompileContext
+    from forge.engine.models import resolve_model
+
+    ctx = CompileContext(tenant_id="t", project_id="p", provider_credentials={"openai": "sk-proj-test"})
+    resolve_model("openai:gpt-5.4-mini", ctx)
+    assert captured["model"] == "openai:gpt-5.4-mini"
+    assert captured.get("api_key") == "sk-proj-test"
+
+    # Google uses google_api_key, not api_key.
+    captured.clear()
+    ctx2 = CompileContext(tenant_id="t", project_id="p", provider_credentials={"google_genai": "g-key"})
+    resolve_model("google_genai:gemini-3.5-flash", ctx2)
+    assert captured.get("google_api_key") == "g-key"
+
+
+async def test_calculator_builtin_test_endpoint_logic():
+    res = await ToolService.test("t", "p", {"name": "calc", "kind": "builtin", "builtin": "calculator", "description": "d"}, {"expression": "2*(3+4)"})
+    assert res["ok"] is True
+    assert res["projected"] == "14"
+
+
+# --- tools wired into an agent ---
+_CALC = {"name": "calculator", "kind": "builtin", "builtin": "calculator", "description": "Evaluate arithmetic."}
+
+
+async def test_agent_compiles_and_runs_with_bound_tool():
+    from langchain_core.messages import HumanMessage
+    from langgraph.checkpoint.memory import InMemorySaver
+
+    from forge.engine.compiler import compile_workflow
+    from forge.services.runtime import make_runtime_ctx
+    from forge.tools.materialize import materialize_tool
+
+    ctx = make_runtime_ctx("t", "p")
+    ctx.checkpointer = InMemorySaver()
+    ctx.tool_registry = {"tool_calc": materialize_tool(_CALC, ctx)}
+    wf = {
+        "id": "wf_tool", "version": 1,
+        "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+        "entry_node": "agent",
+        "nodes": [
+            {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": "fake:Done.", "tools": ["tool_calc"]}},
+            {"id": "end", "type": "end", "config": {}},
+        ],
+        "edges": [{"source": "agent", "target": "end"}],
+    }
+    graph = compile_workflow(wf, ctx)
+    out = await graph.ainvoke({"messages": [HumanMessage(content="hi")]}, {"configurable": {"thread_id": "t1"}})
+    assert out["messages"][-1].content == "Done."
+
+
+async def test_agent_actually_invokes_tool_full_loop():
+    """Scripted fake model emits a tool call → calculator runs → 6*7 = 42 appears."""
+    from langchain.agents import create_agent
+    from langchain_core.language_models.fake_chat_models import GenericFakeChatModel
+    from langchain_core.messages import AIMessage, HumanMessage
+
+    from forge.services.runtime import make_runtime_ctx
+    from forge.tools.materialize import materialize_tool
+
+    class Fake(GenericFakeChatModel):
+        def bind_tools(self, tools=None, **kwargs):
+            return self
+
+    scripted = iter([
+        AIMessage(content="", tool_calls=[{"name": "calculator", "args": {"expression": "6*7"}, "id": "c1", "type": "tool_call"}]),
+        AIMessage(content="It is 42."),
+    ])
+    tool = materialize_tool(_CALC, make_runtime_ctx("t", "p"))
+    agent = create_agent(model=Fake(messages=scripted), tools=[tool], system_prompt="Use the calculator.")
+    out = await agent.ainvoke({"messages": [HumanMessage(content="what is 6*7")]})
+    tool_msgs = [m for m in out["messages"] if getattr(m, "type", None) == "tool"]
+    assert any("42" in str(m.content) for m in tool_msgs), [m.content for m in out["messages"]]

apps/api/tests/test_security_hardening.py

@@ -0,0 +1,179 @@
+"""Security + correctness hardening regression tests.
+
+Covers high-risk behaviors: embed/run project scoping, anonymous thread-identity
+isolation, atomic quota admission, the stale-run reaper, the resume-state guard,
+JWT revocation, and component tenant/project scoping.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+import pytest
+
+from forge.db.base import SessionLocal
+from forge.models import Run, Tenant, Workflow
+from forge.services.components import ComponentService
+from forge.services.runs import RunService
+
+# --- S1: runs are scoped by project, not just tenant -------------------------------------
+
+async def test_stream_is_scoped_by_project():
+    async with SessionLocal() as s:
+        run = Run(tenant_id="t1", project_id="pA", workflow_id="w", thread_id="th", status="queued", input={})
+        s.add(run)
+        await s.commit()
+        rid = run.id
+    rs = RunService(checkpointer=None)
+    # Streaming the run under a DIFFERENT project (same tenant) must not find it.
+    first = None
+    async for frame in rs.stream(run_id=rid, tenant_id="t1", project_id="pB", public=True):
+        first = frame
+        break
+    assert first["event"] == "error" and "not found" in first["data"]["message"]
+
+
+async def test_resume_is_scoped_by_project():
+    async with SessionLocal() as s:
+        run = Run(tenant_id="t1", project_id="pA", workflow_id="w", thread_id="th", status="interrupted", input={})
+        s.add(run)
+        await s.commit()
+        rid = run.id
+    rs = RunService(checkpointer=None)
+    res = await rs.resume(run_id=rid, tenant_id="t1", value=True, project_id="pB")
+    assert res.get("error") == "run not found"
+
+
+# --- S3: an anonymous caller can't attach to a thread bound to another identity ----------
+
+async def test_create_run_isolates_foreign_identity_threads():
+    async with SessionLocal() as s:
+        wf = Workflow(tenant_id="t2", project_id="p", name="w", executable={}, status="active")
+        s.add(wf)
+        await s.commit()
+        wid = wf.id
+    rs = RunService()
+    async with SessionLocal() as s:
+        r1 = await rs.create_run(s, tenant_id="t2", project_id="p", workflow_id=wid, input={}, end_user={"id": "alice"})
+        thread_alice = r1.thread_id
+    # Anonymous caller supplies alice's thread_id -> must start a FRESH thread, not attach.
+    async with SessionLocal() as s:
+        r2 = await rs.create_run(s, tenant_id="t2", project_id="p", workflow_id=wid, input={}, thread_id=thread_alice, end_user=None)
+        assert r2.thread_id != thread_alice
+    # The same identity CAN continue its own thread.
+    async with SessionLocal() as s:
+        r3 = await rs.create_run(s, tenant_id="t2", project_id="p", workflow_id=wid, input={}, thread_id=thread_alice, end_user={"id": "alice"})
+        assert r3.thread_id == thread_alice
+
+
+# --- F2 / S2: quota admission is enforced (and atomic) -----------------------------------
+
+async def test_run_admission_enforces_quota():
+    from forge.services.quota import QuotaExceeded, run_admission
+
+    async with SessionLocal() as s:
+        t = Tenant(name="QA", settings={"max_runs_per_day": 1})
+        s.add(t)
+        await s.flush()
+        s.add(Run(tenant_id=t.id, project_id="p", workflow_id="w", thread_id="th", status="done"))
+        await s.commit()
+        tid = t.id
+    async with SessionLocal() as s:
+        with pytest.raises(QuotaExceeded):
+            async with run_admission(s, tid):
+                pass
+
+
+async def test_quota_ignores_errored_runs():
+    from forge.services.quota import check_run_quota
+
+    async with SessionLocal() as s:
+        t = Tenant(name="QE", settings={"max_runs_per_day": 1})
+        s.add(t)
+        await s.flush()
+        # An errored run must not consume the daily allowance.
+        s.add(Run(tenant_id=t.id, project_id="p", workflow_id="w", thread_id="th", status="error"))
+        await s.commit()
+        await check_run_quota(s, t.id)  # must NOT raise
+
+
+# --- F3: the reaper resolves stale queued/running runs -----------------------------------
+
+async def test_reaper_marks_stale_runs():
+    old = datetime.utcnow() - timedelta(hours=3)
+    async with SessionLocal() as s:
+        q = Run(tenant_id="t3", project_id="p", workflow_id="w", thread_id="th", status="queued", input={})
+        r = Run(tenant_id="t3", project_id="p", workflow_id="w", thread_id="th", status="running", input={})
+        s.add_all([q, r])
+        await s.flush()
+        q.created_at = old
+        r.started_at = old
+        await s.commit()
+        qid, rid = q.id, r.id
+    reaped = await RunService().reap_stale_runs(queued_max_age_s=60, running_max_age_s=60)
+    assert reaped >= 2
+    async with SessionLocal() as s:
+        assert (await s.get(Run, qid)).status == "error"
+        assert (await s.get(Run, rid)).status == "error"
+
+
+# --- F-low: resume only an interrupted run -----------------------------------------------
+
+async def test_resume_rejects_non_interrupted_run():
+    async with SessionLocal() as s:
+        run = Run(tenant_id="t4", project_id="p", workflow_id="w", thread_id="th", status="done", input={})
+        s.add(run)
+        await s.commit()
+        rid = run.id
+    res = await RunService(checkpointer=None).resume(run_id=rid, tenant_id="t4", value=True)
+    assert res.get("error") and "not awaiting input" in res["error"]
+
+
+# --- S11: identity (session) tokens can be revoked ---------------------------------------
+
+def test_session_token_roundtrip_and_revocation():
+    from forge.security import TokenError, create_session_token, decode_token, revoke
+
+    tok = create_session_token(tenant_id="t", project_id="p", end_user={"id": "u"})
+    claims = decode_token(tok, expected_type="session")
+    assert claims["jti"] and claims["end_user"]["id"] == "u"
+    revoke(claims["jti"])
+    with pytest.raises(TokenError):
+        decode_token(tok, expected_type="session")
+
+
+# --- L2 / component isolation: get is scoped by project ----------------------------------
+
+async def test_component_get_is_project_scoped():
+    async with SessionLocal() as s:
+        comp = await ComponentService.create(s, "tc", "projA", name="card", html="<div>{{x}}</div>")
+        cid = comp.id
+    async with SessionLocal() as s:
+        assert await ComponentService.get(s, "tc", "projA", cid) is not None
+        # Same tenant, wrong project -> not found (no cross-project read).
+        assert await ComponentService.get(s, "tc", "projB", cid) is None
+
+
+# --- S7: the SQL-tool DSN guard honors the per-project EgressPolicy instance --------------
+
+async def test_sql_tool_honors_project_egress_policy():
+    from forge.tools.sql import execute_sql
+    from forge.util.ssrf import EgressBlocked, EgressPolicy
+
+    cfg = {"name": "q", "query": "SELECT 1",
+           "connection_url": "postgresql+asyncpg://u:p@blocked.example.com:5432/db"}
+    # A resolved EgressPolicy INSTANCE (what ctx.egress_policy is) must be applied directly -
+    # not silently rebuilt from global settings (which, in tests, block nothing).
+    policy = EgressPolicy(block_private=False, deny_hosts=("blocked.example.com",))
+    with pytest.raises(EgressBlocked):
+        await execute_sql(cfg, {}, tenant_id="t", project_id="p", egress=policy)
+
+
+# --- S4: a non-editor cannot self-assert privileged identity via the run body ------------
+
+def test_role_gate_blocks_viewer_entitlements():
+    from forge.services.auth import role_at_least
+
+    # The create_run route strips roles/entitlements unless the caller is editor+.
+    assert role_at_least("editor", "editor") is True
+    assert role_at_least("viewer", "editor") is False

apps/api/tests/test_semantic_cache.py

@@ -0,0 +1,32 @@
+"""Semantic response cache: paraphrased questions hit the cache."""
+
+from __future__ import annotations
+
+from forge.db.base import SessionLocal
+from forge.services.semantic_cache import SemanticCacheService
+
+T, P = "t_sc", "p_sc"
+
+
+async def test_store_then_lookup_hits_on_paraphrase():
+    async with SessionLocal() as s:
+        await SemanticCacheService.store(s, T, P, "What are your business hours?", "We're open 9am-5pm ET.")
+    async with SessionLocal() as s:
+        # near-duplicate question; fake embedder gives high overlap on shared words
+        hit = await SemanticCacheService.lookup(s, T, P, "what are your business hours", threshold=0.6)
+    assert hit == "We're open 9am-5pm ET."
+
+
+async def test_lookup_miss_below_threshold():
+    async with SessionLocal() as s:
+        await SemanticCacheService.store(s, "t_m", "p_m", "How do I reset my password?", "Use the reset link.")
+    async with SessionLocal() as s:
+        hit = await SemanticCacheService.lookup(s, "t_m", "p_m", "completely unrelated rocket science", threshold=0.9)
+    assert hit is None
+
+
+async def test_ttl_expiry():
+    async with SessionLocal() as s:
+        await SemanticCacheService.store(s, "t_t", "p_t", "ping?", "pong")
+    async with SessionLocal() as s:
+        assert await SemanticCacheService.lookup(s, "t_t", "p_t", "ping?", threshold=0.5, ttl=-1) is None

apps/api/tests/test_service_token_egress.py

@@ -0,0 +1,116 @@
+"""Server-to-server integration primitives:
+
+1. A static service API token (`FORGE_SERVICE_API_TOKEN`) that authenticates a trusted backend
+   as a least-privilege (editor) service identity - the outer "is this call from our backend"
+   barrier. Non-expiring, revoked by rotation.
+2. An egress allow-private-hosts toggle (`FORGE_EGRESS_ALLOW_PRIVATE_HOSTS`) that lets specific
+   trusted internal hosts (localhost, on-prem services) be reached even while the SSRF guard's
+   private-address block stays on globally (default-deny, explicit-allow).
+"""
+
+import pytest
+from fastapi import HTTPException
+
+from forge.config import settings
+from forge.deps import get_current_user
+from forge.util.ssrf import EgressBlocked, EgressPolicy, validate_url
+
+# --- service token ---------------------------------------------------------------------------
+
+
+class _State:
+    tenant_id = "t-seed"
+
+
+class _App:
+    state = _State()
+
+
+class _Req:
+    def __init__(self, authorization: str | None):
+        self.headers = {"authorization": authorization} if authorization else {}
+        self.app = _App()
+
+
+async def test_service_token_authenticates_as_editor(monkeypatch):
+    monkeypatch.setattr(settings, "service_api_token", "svc-secret-abc123")
+    user = await get_current_user(_Req("Bearer svc-secret-abc123"))
+    assert user.role == "editor"          # least privilege, but enough to assert end_user identity
+    assert user.tenant_id == "t-seed"     # bound to the seeded workspace
+    assert user.is_fallback is False
+
+
+async def test_wrong_service_token_is_rejected(monkeypatch):
+    # A present-but-wrong token must NOT fall back to an anonymous/owner identity: it fails the
+    # constant-time compare, then fails JWT decode -> 401.
+    monkeypatch.setattr(settings, "service_api_token", "svc-secret-abc123")
+    with pytest.raises(HTTPException) as e:
+        await get_current_user(_Req("Bearer not-the-token"))
+    assert e.value.status_code == 401
+
+
+async def test_empty_service_token_setting_is_disabled(monkeypatch):
+    # With no service token configured, the branch is inert (a random bearer is treated as a
+    # would-be JWT and rejected).
+    monkeypatch.setattr(settings, "service_api_token", "")
+    with pytest.raises(HTTPException) as e:
+        await get_current_user(_Req("Bearer anything"))
+    assert e.value.status_code == 401
+
+
+# --- egress allow-private-hosts --------------------------------------------------------------
+
+
+async def test_private_host_blocked_by_default():
+    with pytest.raises(EgressBlocked):
+        await validate_url("http://127.0.0.1:9002/x", EgressPolicy(block_private=True))
+
+
+async def test_allow_private_hosts_bypasses_block_by_ip():
+    url = await validate_url(
+        "http://127.0.0.1:9002/x",
+        EgressPolicy(block_private=True, allow_private_hosts=("127.0.0.1",)),
+    )
+    assert url == "http://127.0.0.1:9002/x"
+
+
+async def test_allow_private_hosts_bypasses_block_by_name():
+    # `localhost` resolves to a loopback IP, but the host is explicitly allow-listed.
+    url = await validate_url(
+        "http://localhost:9002/orders/detail/1",
+        EgressPolicy(block_private=True, allow_private_hosts=("localhost",)),
+    )
+    assert url.startswith("http://localhost")
+
+
+async def test_non_allowlisted_private_still_blocked():
+    with pytest.raises(EgressBlocked):
+        await validate_url(
+            "http://10.0.0.5/x",
+            EgressPolicy(block_private=True, allow_private_hosts=("localhost",)),
+        )
+
+
+def test_from_settings_reads_global_allow_private(monkeypatch):
+    monkeypatch.setattr(settings, "egress_allow_private_hosts", ["localhost", "127.0.0.1"])
+    p = EgressPolicy.from_settings()
+    assert "localhost" in p.allow_private_hosts and "127.0.0.1" in p.allow_private_hosts
+
+
+def test_from_settings_ignores_project_allow_private(monkeypatch):
+    # SECURITY (audit H1): project config is editable by any tenant member, so it must NOT be able
+    # to add a private-host bypass. allow_private_hosts is a deployment-level control only.
+    monkeypatch.setattr(settings, "egress_allow_private_hosts", [])
+    p = EgressPolicy.from_settings({"allow_private_hosts": ["10.0.0.5"]})
+    assert "10.0.0.5" not in p.allow_private_hosts
+    assert p.allow_private_hosts == ()
+
+
+def test_from_settings_project_cannot_loosen_block_private(monkeypatch):
+    # SECURITY (audit H1): a project may only TIGHTEN block_private (False->True), never turn the
+    # SSRF guard off. With the guard on globally, project block_private:false is ignored.
+    monkeypatch.setattr(settings, "egress_block_private", True)
+    assert EgressPolicy.from_settings({"block_private": False}).block_private is True
+    # And a project can still tighten when the deployment default is off.
+    monkeypatch.setattr(settings, "egress_block_private", False)
+    assert EgressPolicy.from_settings({"block_private": True}).block_private is True

apps/api/tests/test_sse_reconnect.py

@@ -0,0 +1,86 @@
+"""Durable SSE (finding #12): run execution is decoupled from the client connection.
+
+A mid-run client disconnect must NOT end the run - the graph keeps executing in a detached
+background task, and a client can reattach (same run_id) to get the final answer. Frames carry
+a monotonic id so a `Last-Event-ID` reconnect replays only what was missed.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.db.base import SessionLocal
+from forge.models import Run, Thread, Workflow
+from forge.services.runs import RunService
+
+_ANSWER = "Hello from the durable run."
+_WF = {
+    "id": "wf_sse", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "agent",
+    "nodes": [
+        {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": f"fake:{_ANSWER}", "tools": []}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "agent", "target": "end"}],
+}
+
+
+async def _seed_queued_run() -> tuple[str, str, str]:
+    """Insert a workflow + thread + a queued run directly; returns (tenant, project, run_id)."""
+    t, p = f"t_{uuid.uuid4().hex[:8]}", f"p_{uuid.uuid4().hex[:8]}"
+    async with SessionLocal() as s:
+        wf = Workflow(tenant_id=t, project_id=p, name="w", executable=_WF, status="active")
+        s.add(wf)
+        await s.flush()
+        thread = Thread(tenant_id=t, project_id=p, workflow_id=wf.id, lg_thread_id=f"lg_{uuid.uuid4().hex}", meta={})
+        s.add(thread)
+        await s.flush()
+        run = Run(tenant_id=t, project_id=p, workflow_id=wf.id, thread_id=thread.id, status="queued",
+                  input={"messages": [{"role": "user", "content": "hi"}]})
+        s.add(run)
+        await s.commit()
+        return t, p, run.id
+
+
+async def test_disconnect_leaves_run_running_and_reattach_gets_answer():
+    t, p, rid = await _seed_queued_run()
+    rs = RunService(checkpointer=InMemorySaver())
+
+    # First connection: take a single frame, then "disconnect" by closing the subscriber.
+    agen = rs.stream(run_id=rid, tenant_id=t, project_id=p)
+    first = await agen.__anext__()
+    assert first["event"] == "run" and first["id"] == "1"
+    await agen.aclose()  # client gone - the detached executor must keep running
+
+    # Reattach from where we left off; the run completes and we get the final answer.
+    frames = [f async for f in rs.stream(run_id=rid, tenant_id=t, project_id=p, last_event_id=int(first["id"]))]
+    done = [f for f in frames if f["event"] == "done"]
+    assert done, [f["event"] for f in frames]
+    assert _ANSWER in (done[-1]["data"].get("answer") or "")
+
+    # The disconnect did NOT cancel the run - it ran to completion.
+    async with SessionLocal() as s:
+        run = await s.get(Run, rid)
+        assert run.status == "done", run.status
+
+
+async def test_last_event_id_replays_only_later_frames():
+    t, p, rid = await _seed_queued_run()
+    rs = RunService(checkpointer=InMemorySaver())
+
+    # Drive the run to completion on the first connection, recording every frame id.
+    frames = [f async for f in rs.stream(run_id=rid, tenant_id=t, project_id=p)]
+    ids = [int(f["id"]) for f in frames if f.get("id")]
+    assert ids == sorted(ids) and ids[0] == 1  # monotonic, starting at 1
+    assert any(f["event"] == "done" for f in frames)
+
+    # Reattach with a mid-stream Last-Event-ID: only strictly-later frames are replayed.
+    cutoff = ids[len(ids) // 2]
+    replayed = [f async for f in rs.stream(run_id=rid, tenant_id=t, project_id=p, last_event_id=cutoff)]
+    assert replayed, "reattach after completion should replay the retained tail"
+    assert all(int(f["id"]) > cutoff for f in replayed)
+    # The retained tail still includes the terminal done frame.
+    assert any(f["event"] == "done" for f in replayed)

apps/api/tests/test_ssrf.py

@@ -0,0 +1,55 @@
+"""SSRF egress-guard tests (offline: uses IP literals + pre-resolution checks)."""
+
+from __future__ import annotations
+
+import pytest
+
+from forge.util.ssrf import EgressBlocked, EgressPolicy, validate_url
+
+_BLOCK = EgressPolicy(block_private=True)
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "http://127.0.0.1/x",
+        "http://localhost/x",  # resolves to loopback
+        "http://169.254.169.254/latest/meta-data/",  # cloud metadata
+        "http://10.0.0.5/x",
+        "http://192.168.1.1/x",
+        "http://[::1]/x",
+        "http://0.0.0.0/x",
+        "http://[::ffff:127.0.0.1]/x",  # IPv4-mapped loopback
+    ],
+)
+async def test_blocks_internal_targets(url):
+    with pytest.raises(EgressBlocked):
+        await validate_url(url, _BLOCK)
+
+
+@pytest.mark.parametrize("url", ["ftp://example.com", "file:///etc/passwd", "gopher://x"])
+async def test_blocks_non_http_schemes(url):
+    with pytest.raises(EgressBlocked):
+        await validate_url(url, _BLOCK)
+
+
+async def test_allows_public_ip_literal():
+    assert await validate_url("https://8.8.8.8/x", _BLOCK) == "https://8.8.8.8/x"
+
+
+async def test_deny_list_blocks_before_resolution():
+    pol = EgressPolicy(block_private=True, deny_hosts=("evil.example",))
+    with pytest.raises(EgressBlocked):
+        await validate_url("https://api.evil.example/x", pol)  # parent-domain match
+
+
+async def test_allow_list_blocks_other_hosts():
+    pol = EgressPolicy(block_private=True, allow_hosts=("good.example",))
+    with pytest.raises(EgressBlocked):
+        await validate_url("https://other.example/x", pol)
+    # host inside the allowed parent domain + public literal passes
+    assert await validate_url("https://8.8.8.8/x", EgressPolicy(allow_hosts=("8.8.8.8",)))
+
+
+async def test_block_private_disabled_allows_loopback():
+    assert await validate_url("http://127.0.0.1/x", EgressPolicy(block_private=False))

apps/api/tests/test_stats.py

@@ -0,0 +1,207 @@
+"""Characterization tests for the stats rollups (dashboard + project_stats).
+
+These pin the EXACT output of the two endpoints for a controlled dataset so the
+in-memory -> SQL-aggregate refactor can be proven behaviour-preserving. Every expected
+number here is hand-computed from the seeded traces below.
+
+Dataset (tenant t_stats), 6 traces across 2 projects:
+  A  P1 W1  run        done   tok 10  cost 0.10  lat 100   -1h   (in window)
+  B  P1 W1  run        error  tok 20  cost 0.20  lat 300   -2h   (in window)
+  C  P1 --  assistant  done   tok  5  cost 0.05  lat  50   -3h   (in window)
+  D  P2 --  adhoc      done   tok  8  cost 0.08  lat  80   -4h   (in window)
+  E  P2 wf_gone run     done  tok  2  cost 0.02  lat  20   -5h   (in window)
+  F  P1 W1  run        done   tok100  cost 1.00  lat1000  -10d   (OUT of window)
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from forge.db.base import SessionLocal
+from forge.models import Project, Span, Tool, Trace, Workflow
+from forge.routers.stats import dashboard, project_analytics, project_stats
+
+TENANT = "t_stats"
+
+
+async def _seed() -> tuple[str, str]:
+    now = datetime.utcnow()
+
+    def h(n):  # n hours ago
+        return now - timedelta(hours=n)
+
+    async with SessionLocal() as s:
+        p1 = Project(tenant_id=TENANT, name="Alpha", slug="alpha")
+        p2 = Project(tenant_id=TENANT, name="Beta", slug="beta")
+        s.add_all([p1, p2])
+        await s.flush()
+        w1 = Workflow(tenant_id=TENANT, project_id=p1.id, name="Support")
+        s.add(w1)
+        s.add(Tool(tenant_id=TENANT, project_id=p1.id, name="t", kind="builtin", config={}))
+        await s.flush()
+        rows = [
+            Trace(tenant_id=TENANT, project_id=p1.id, workflow_id=w1.id, run_id="rA", name="run", status="done", total_tokens=10, total_cost_usd=0.10, latency_ms=100, started_at=h(1)),
+            Trace(tenant_id=TENANT, project_id=p1.id, workflow_id=w1.id, run_id="rB", name="run", status="error", total_tokens=20, total_cost_usd=0.20, latency_ms=300, started_at=h(2)),
+            Trace(tenant_id=TENANT, project_id=p1.id, workflow_id=None, run_id="rC", name="assistant", status="done", total_tokens=5, total_cost_usd=0.05, latency_ms=50, started_at=h(3)),
+            Trace(tenant_id=TENANT, project_id=p2.id, workflow_id=None, run_id="rD", name="adhoc", status="done", total_tokens=8, total_cost_usd=0.08, latency_ms=80, started_at=h(4)),
+            Trace(tenant_id=TENANT, project_id=p2.id, workflow_id="wf_gone", run_id="rE", name="run", status="done", total_tokens=2, total_cost_usd=0.02, latency_ms=20, started_at=h(5)),
+            Trace(tenant_id=TENANT, project_id=p1.id, workflow_id=w1.id, run_id="rF", name="run", status="done", total_tokens=100, total_cost_usd=1.00, latency_ms=1000, started_at=now - timedelta(days=10)),
+        ]
+        s.add_all(rows)
+        await s.commit()
+        return p1.id, p2.id
+
+
+async def test_dashboard_rollups():
+    p1, p2 = await _seed()
+    async with SessionLocal() as s:
+        d = await dashboard(session=s, tenant_id=TENANT)
+
+    assert d["total_runs"] == 6
+    assert d["runs_7d"] == 5
+    assert d["success_rate"] == 80.0          # 4 done of 5 in-window
+    assert d["avg_latency_ms"] == 110         # int((100+300+50+80+20)/5)
+    assert d["spend_7d"] == 0.45
+
+    # per-project counts for the dashboard cards
+    assert d["projects"][p1] == {"workflows": 1, "tools": 1, "runs_7d": 3}
+    assert d["projects"][p2] == {"workflows": 0, "tools": 0, "runs_7d": 2}
+
+    # recent = 8 most recent all-time, newest first (A,B,C,D,E,F)
+    recent = [(r["workflow"], r["project"], r["status"], r["tokens"]) for r in d["recent"]]
+    assert recent == [
+        ("Support", "Alpha", "done", 10),
+        ("Support", "Alpha", "error", 20),
+        ("run", "Alpha", "done", 5),
+        ("run", "Beta", "done", 8),
+        ("run", "Beta", "done", 2),
+        ("Support", "Alpha", "done", 100),
+    ]
+
+    # totals (all-time)
+    assert d["totals"] == {
+        "runs": 6, "tokens": 145, "cost_usd": 1.45, "avg_latency_ms": 258,
+        "errors": 1, "error_rate": 16.7,
+    }
+
+    # reports (per project, all-time), cost desc
+    reps = {r["project_id"]: r for r in d["reports"]}
+    assert [r["project_id"] for r in d["reports"]] == [p1, p2]
+    assert reps[p1]["project"] == "Alpha" and reps[p1]["runs"] == 4 and reps[p1]["tokens"] == 135
+    assert reps[p1]["cost_usd"] == 1.35 and reps[p1]["avg_latency_ms"] == 362
+    assert reps[p1]["errors"] == 1 and reps[p1]["error_rate"] == 25.0
+    assert reps[p1]["assistant_cost_usd"] == 0.05 and reps[p1]["assistant_turns"] == 1
+    assert reps[p2]["runs"] == 2 and reps[p2]["cost_usd"] == 0.1 and reps[p2]["assistant_turns"] == 0
+
+
+async def test_project_stats_rollups():
+    p1, p2 = await _seed()
+    async with SessionLocal() as s:
+        d1 = await project_stats(project_id=p1, session=s, tenant_id=TENANT)
+        d2 = await project_stats(project_id=p2, session=s, tenant_id=TENANT)
+
+    assert d1["totals"] == {"runs": 4, "tokens": 135, "cost_usd": 1.35, "avg_latency_ms": 362, "errors": 1, "error_rate": 25.0}
+    assert d1["last_7d"] == {"runs": 3, "tokens": 35, "cost_usd": 0.35, "avg_latency_ms": 150, "errors": 1, "error_rate": 33.3}
+    assert d1["assistant"] == {"runs": 1, "tokens": 5, "cost_usd": 0.05, "avg_latency_ms": 50, "errors": 0, "error_rate": 0.0, "turns": 1}
+
+    # report rows for P1: workflow "Support" (A,B,F) then assistant (C), cost desc
+    r1 = d1["reports"]
+    assert [(r["kind"], r["label"], r["runs"], r["cost_usd"]) for r in r1] == [
+        ("workflow", "Support", 3, 1.3),
+        ("assistant", "Forge Assistant", 1, 0.05),
+    ]
+
+    # P2 exercises the 'other' (name) group and the deleted-workflow label
+    r2 = d2["reports"]
+    assert [(r["kind"], r["label"], r["runs"], r["cost_usd"]) for r in r2] == [
+        ("other", "adhoc", 1, 0.08),
+        ("workflow", "(deleted workflow)", 1, 0.02),
+    ]
+
+
+# --- analytics endpoint (time-series + breakdowns) ----------------------------------------
+# Isolated in its own tenant so it can't collide with the rollup fixtures above regardless of
+# test order (the suite shares one SQLite file; init_db only create_all's, it never truncates).
+ATENANT = "t_analytics"
+
+
+async def _seed_analytics() -> str:
+    now = datetime.utcnow()
+
+    def h(n):
+        return now - timedelta(hours=n)
+
+    async with SessionLocal() as s:
+        p = Project(tenant_id=ATENANT, name="Alpha", slug="alpha")
+        s.add(p)
+        await s.flush()
+        w = Workflow(tenant_id=ATENANT, project_id=p.id, name="Support")
+        s.add(w)
+        await s.flush()
+        t1 = Trace(tenant_id=ATENANT, project_id=p.id, workflow_id=w.id, run_id="rA1", name="run", status="done", source="playground", total_tokens=10, total_cost_usd=0.10, latency_ms=100, started_at=h(1))
+        t2 = Trace(tenant_id=ATENANT, project_id=p.id, workflow_id=w.id, run_id="rA2", name="run", status="error", source="playground", total_tokens=20, total_cost_usd=0.20, latency_ms=3000, started_at=h(2))
+        t3 = Trace(tenant_id=ATENANT, project_id=p.id, workflow_id=None, run_id="rA3", name="run", status="done", source="api", total_tokens=5, total_cost_usd=0.05, latency_ms=50, started_at=h(25))
+        t4 = Trace(tenant_id=ATENANT, project_id=p.id, workflow_id=None, run_id="rA4", name="assistant", status="done", source="assistant", total_tokens=8, total_cost_usd=0.08, latency_ms=5500, started_at=h(3))
+        # Out of the 30-day window but inside the previous (30-60d) window -> feeds prev_totals.
+        t5 = Trace(tenant_id=ATENANT, project_id=p.id, workflow_id=w.id, run_id="rA5", name="run", status="done", source="playground", total_tokens=100, total_cost_usd=1.00, latency_ms=200, started_at=now - timedelta(days=40))
+        s.add_all([t1, t2, t3, t4, t5])
+        await s.flush()
+        s.add_all([
+            Span(tenant_id=ATENANT, trace_id=t1.id, name="get_order", kind="tool", latency_ms=300, input_tokens=0, output_tokens=0, cost_usd=0.0),
+            Span(tenant_id=ATENANT, trace_id=t1.id, name="model", kind="llm", model="claude", latency_ms=800, input_tokens=4, output_tokens=6, cost_usd=0.09),
+            Span(tenant_id=ATENANT, trace_id=t2.id, name="get_order", kind="tool", latency_ms=200, input_tokens=0, output_tokens=0, cost_usd=0.0, error="boom"),
+        ])
+        await s.commit()
+        return p.id
+
+
+async def test_project_analytics():
+    pid = await _seed_analytics()
+    async with SessionLocal() as s:
+        a = await project_analytics(project_id=pid, days=30, session=s, tenant_id=ATENANT)
+
+    assert a["range"]["days"] == 30 and a["range"]["bucket"] == "day"
+
+    # Windowed totals (T1..T4); T5 is out of window.
+    assert a["totals"]["runs"] == 4
+    assert a["totals"]["tokens"] == 43
+    assert a["totals"]["cost_usd"] == 0.43
+    assert a["totals"]["errors"] == 1
+    assert a["totals"]["avg_latency_ms"] == 2162   # int((100+3000+50+5500)/4)
+
+    # Previous 30-60d window holds only T5.
+    assert a["prev_totals"]["runs"] == 1 and a["prev_totals"]["tokens"] == 100
+
+    # 30 days back -> 31 daily points (inclusive), continuous, summing to the 4 in-window runs.
+    assert len(a["timeseries"]) == 31
+    assert sum(p["runs"] for p in a["timeseries"]) == 4
+    assert sum(p["errors"] for p in a["timeseries"]) == 1
+
+    by_src = {r["source"]: r for r in a["by_source"]}
+    assert by_src["playground"]["runs"] == 2
+    assert by_src["api"]["runs"] == 1
+    assert by_src["assistant"]["runs"] == 1
+
+    by_wf = {(r["kind"], r["label"]): r for r in a["by_workflow"]}
+    assert by_wf[("workflow", "Support")]["runs"] == 2
+    assert by_wf[("assistant", "Forge Assistant")]["runs"] == 1
+    assert by_wf[("other", "run")]["runs"] == 1
+
+    tools = {t["name"]: t for t in a["tools"]}
+    assert tools["get_order"]["calls"] == 2
+    assert tools["get_order"]["errors"] == 1
+    assert tools["get_order"]["avg_latency_ms"] == 250   # (300+200)/2
+
+    models = {m["model"]: m for m in a["models"]}
+    assert models["claude"]["calls"] == 1
+    assert models["claude"]["tokens"] == 10
+    assert models["claude"]["cost_usd"] == 0.09
+
+    hist = {b["label"]: b["count"] for b in a["latency_histogram"]}
+    assert hist["<250ms"] == 2      # T1 (100), T3 (50)
+    assert hist["2-5s"] == 1        # T2 (3000)
+    assert hist["5-10s"] == 1       # T4 (5500)
+
+    # Recent activity feed: the 4 in-window runs, newest first.
+    assert len(a["recent"]) == 4
+    assert a["recent"][0]["status"] == "done"      # T1, most recent

apps/api/tests/test_stream_message_filtering.py

@@ -0,0 +1,41 @@
+"""Which nodes' streamed tokens reach the chat bubble.
+
+Classifier and structured-`llm` nodes still stream tokens over the messages channel even
+though their result (a routing label / structured_response) never enters `messages`. The run
+stream suppresses those by node id so a client no longer has to guess from node-name patterns
+(the old `classif|router|^start$` regex). Answer producers stream normally.
+"""
+
+from __future__ import annotations
+
+from forge.services.runs import _internal_message_nodes
+
+
+def test_suppresses_classifier_router_start_and_structured_llm():
+    nodes = [
+        {"id": "start", "type": "start"},
+        {"id": "intent", "type": "classifier"},
+        {"id": "route", "type": "router"},
+        {"id": "answer", "type": "agent"},
+        {"id": "reply", "type": "llm"},  # unstructured llm = an answer producer
+        {"id": "extract", "type": "llm", "config": {"response_format": {"mode": "structured", "schema": {}}}},
+        {"id": "finish", "type": "end"},
+    ]
+    suppressed = _internal_message_nodes(nodes)
+    assert suppressed == {"start", "intent", "route", "extract", "finish"}
+    # Answer-producing nodes stream normally.
+    assert "answer" not in suppressed
+    assert "reply" not in suppressed
+
+
+def test_unstructured_llm_is_not_suppressed():
+    # response_format present but not "structured" -> still an answer producer.
+    nodes = [{"id": "reply", "type": "llm", "config": {"response_format": {"mode": "text"}}}]
+    assert _internal_message_nodes(nodes) == set()
+
+
+def test_tolerates_malformed_nodes():
+    nodes = [None, "oops", {"type": "classifier"}, {"id": "c2", "type": "classifier"}]
+    # Non-dicts and a dict without an id are skipped (a null id would match tokens that have no
+    # langgraph_node and wrongly drop answers); only the well-formed classifier is suppressed.
+    assert _internal_message_nodes(nodes) == {"c2"}

apps/api/tests/test_stream_tool_display_name.py

@@ -0,0 +1,140 @@
+"""Human-readable tool names in the run stream.
+
+A tool's underscore identifier (e.g. `get_product_data`) is what the model
+calls, but end-user chat surfaces (the agent chat UI, embeds) should show a friendly
+label. The run stream binds a name->label map for the turn, and `jsonable` relabels every
+serialized tool_call with a `display_name` (falling back to the identifier when unmapped),
+while leaving the model-facing `name` untouched.
+"""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+from langchain_core.messages import AIMessage
+
+from forge.tools.rest import execute_rest
+from forge.util.serialize import (
+    jsonable,
+    reset_tool_display_names,
+    set_tool_display_names,
+)
+
+
+def _ai_with_calls(*names: str) -> AIMessage:
+    return AIMessage(
+        content="",
+        tool_calls=[{"name": n, "args": {}, "id": f"call_{i}"} for i, n in enumerate(names)],
+    )
+
+
+def test_display_name_maps_when_bound():
+    token = set_tool_display_names({"get_product_catalog": "Product catalog"})
+    try:
+        out = jsonable(_ai_with_calls("get_product_catalog"))
+    finally:
+        reset_tool_display_names(token)
+    call = out["tool_calls"][0]
+    # Model-facing name is unchanged; display_name carries the human label.
+    assert call["name"] == "get_product_catalog"
+    assert call["display_name"] == "Product catalog"
+
+
+def test_display_name_falls_back_to_identifier_when_unmapped():
+    # Bound map that doesn't cover this tool -> display_name == the identifier.
+    token = set_tool_display_names({"other_tool": "Other"})
+    try:
+        out = jsonable(_ai_with_calls("get_order_totals"))
+    finally:
+        reset_tool_display_names(token)
+    call = out["tool_calls"][0]
+    assert call["name"] == "get_order_totals"
+    assert call["display_name"] == "get_order_totals"
+
+
+def test_display_name_falls_back_with_no_map_bound():
+    # No map set for this context (e.g. resume/non-stream paths) -> identifier passthrough.
+    out = jsonable(_ai_with_calls("get_project_overview"))
+    call = out["tool_calls"][0]
+    assert call["display_name"] == "get_project_overview"
+
+
+def test_no_tool_calls_stays_none():
+    out = jsonable(AIMessage(content="hello"))
+    assert out["tool_calls"] is None
+
+
+# --- the LIVE "calling" + terminal "done"/"error" custom frames (REST tool) ---------------
+# The tool emits a "calling" frame before the request (label the spinner) and a paired terminal
+# frame when it ENDS (clear the spinner): "done" on success, "error" on failure. Both carry the
+# same tool id + display_name (config.display_name, else the identifier) so a client can pair them.
+
+def _rest_cfg(**extra) -> dict:
+    return {
+        "name": "get_order_totals",
+        "kind": "rest_api",
+        "request": {"method": "GET", "url_template": "https://api.acme.dev/totals", "fields": []},
+        **extra,
+    }
+
+
+def _client(handler) -> httpx.AsyncClient:
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+
+def _ok_client() -> httpx.AsyncClient:
+    return _client(lambda r: httpx.Response(200, json={"ok": True}))
+
+
+async def _run(cfg: dict, client: httpx.AsyncClient) -> list[dict]:
+    frames: list[dict] = []
+    try:
+        await execute_rest(
+            cfg, {}, tenant_id="t", project_id="p", context={}, client=client, stream_writer=frames.append,
+        )
+    finally:
+        await client.aclose()
+    return frames
+
+
+async def test_calling_frame_includes_display_name():
+    frames = await _run(_rest_cfg(display_name="Order totals"), _ok_client())
+    calling = [f for f in frames if f.get("status") == "calling"]
+    assert calling, "expected a 'calling' custom frame"
+    # model-facing id under `tool`; human label under `display_name`
+    assert calling[0]["tool"] == "get_order_totals"
+    assert calling[0]["display_name"] == "Order totals"
+
+
+async def test_calling_frame_display_name_falls_back_to_identifier():
+    frames = await _run(_rest_cfg(), _ok_client())
+    calling = [f for f in frames if f.get("status") == "calling"]
+    assert calling and calling[0]["display_name"] == "get_order_totals"
+
+
+async def test_terminal_done_frame_on_success():
+    frames = await _run(_rest_cfg(display_name="Order totals"), _ok_client())
+    statuses = [f.get("status") for f in frames]
+    # calling first, then a terminal done - so a client can clear the spinner.
+    assert statuses == ["calling", "done"]
+    done = frames[-1]
+    assert done["tool"] == "get_order_totals"
+    assert done["display_name"] == "Order totals"
+    assert done["status_code"] == 200
+    assert "latency_ms" in done
+
+
+async def test_terminal_error_frame_on_failure():
+    # A 500 raises out of execute_rest, but the spinner must still be cleared via an "error" frame.
+    client = _client(lambda r: httpx.Response(500, json={"detail": "boom"}))
+    frames: list[dict] = []
+    with pytest.raises(httpx.HTTPStatusError):
+        await execute_rest(
+            _rest_cfg(display_name="Order totals"), {},
+            tenant_id="t", project_id="p", context={}, client=client, stream_writer=frames.append,
+        )
+    await client.aclose()
+    statuses = [f.get("status") for f in frames]
+    assert statuses == ["calling", "error"]
+    assert frames[-1]["display_name"] == "Order totals"
+    assert frames[-1]["status_code"] == 500

apps/api/tests/test_subworkflow.py

@@ -0,0 +1,51 @@
+"""Subworkflow node: a parent workflow runs a referenced child as a reusable component."""
+
+from __future__ import annotations
+
+from langchain_core.messages import HumanMessage
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.engine.compiler import compile_workflow
+from forge.services.runtime import make_runtime_ctx
+
+_CHILD = {
+    "id": "child", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "agent",
+    "nodes": [
+        {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": "fake:Child answered."}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "agent", "target": "end"}],
+}
+
+_PARENT = {
+    "id": "parent", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "start",
+    "nodes": [
+        {"id": "start", "type": "start", "config": {}},
+        {"id": "sub", "type": "subworkflow", "config": {"workflow_id": "child_1"}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "start", "target": "sub"}, {"source": "sub", "target": "end"}],
+}
+
+
+async def test_subworkflow_runs_child():
+    ctx = make_runtime_ctx("t_sub", "p_sub")
+    ctx.checkpointer = InMemorySaver()
+    ctx.workflows = {"child_1": _CHILD}
+    graph = compile_workflow(_PARENT, ctx)
+    out = await graph.ainvoke({"messages": [HumanMessage(content="hi")]}, {"configurable": {"thread_id": "s1"}})
+    assert out["messages"][-1].content == "Child answered."
+
+
+async def test_missing_subworkflow_is_passthrough():
+    ctx = make_runtime_ctx("t_sub2", "p_sub2")
+    ctx.checkpointer = InMemorySaver()
+    ctx.workflows = {}  # referenced child not present
+    graph = compile_workflow(_PARENT, ctx)
+    out = await graph.ainvoke({"messages": [HumanMessage(content="hi")]}, {"configurable": {"thread_id": "s2"}})
+    # no child -> passthrough; the original message survives, no crash
+    assert out["messages"][-1].content == "hi"

apps/api/tests/test_tls_skip_verify.py

@@ -0,0 +1,90 @@
+"""`tls_skip_verify` is gated: TLS verification may be disabled only for a host explicitly on
+FORGE_EGRESS_ALLOW_PRIVATE_HOSTS. For any other host the flag is ignored and verification stays on,
+so certificate checks can never be turned off for an arbitrary/public endpoint."""
+
+from __future__ import annotations
+
+import httpx
+
+from forge.util.http import (
+    aclose_shared_client,
+    insecure_async_client,
+    select_client,
+    shared_async_client,
+)
+from forge.util.ssrf import EgressPolicy
+
+
+async def test_tls_skip_verify_is_gated_to_allow_private_hosts():
+    policy = EgressPolicy(block_private=True, allow_private_hosts=("host.docker.internal",))
+    try:
+        # opted-in internal host + skip_verify -> verification-disabled client
+        assert select_client(
+            "https://host.docker.internal:9002/x", skip_verify=True, policy=policy
+        ) is insecure_async_client()
+
+        # public host + skip_verify -> IGNORED, stays on the verified shared client
+        assert select_client(
+            "https://api.example.com/x", skip_verify=True, policy=policy
+        ) is shared_async_client()
+
+        # allow-private host but skip_verify off -> verified shared client
+        assert select_client(
+            "https://host.docker.internal/x", skip_verify=False, policy=policy
+        ) is shared_async_client()
+
+        # an explicit client (e.g. a test/mock) always wins over the gate
+        mock = httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(200)))
+        try:
+            assert select_client(
+                "https://host.docker.internal/x", skip_verify=True, policy=policy, override=mock
+            ) is mock
+        finally:
+            await mock.aclose()
+
+        # suffix match honored: a subdomain of an allow-private parent is still gated-in
+        policy2 = EgressPolicy(block_private=True, allow_private_hosts=("internal.corp",))
+        assert select_client(
+            "https://svc.internal.corp/x", skip_verify=True, policy=policy2
+        ) is insecure_async_client()
+    finally:
+        await aclose_shared_client()
+
+
+async def test_guarded_request_reselects_client_per_redirect_hop(monkeypatch):
+    """A redirect off an allow-private host to a public host must re-select the client from the
+    hop's OWN host, so verify-off (tls_skip_verify) never carries onto a non-allow-private hop.
+    Regression for the MEDIUM finding that guarded_request reused one client across all hops."""
+    import forge.util.ssrf as ssrf
+
+    seen_hosts: list[str] = []
+
+    async def _no_validate(url, policy=None):  # skip real DNS/SSRF resolution in the unit test
+        return url
+
+    def _handler(req: httpx.Request) -> httpx.Response:
+        if req.url.host == "internal.corp":
+            return httpx.Response(302, headers={"location": "https://public.example/final"})
+        return httpx.Response(200, json={"ok": True})
+
+    mock = httpx.AsyncClient(transport=httpx.MockTransport(_handler))
+
+    def _spy_select(url, *, skip_verify, policy, override=None):
+        seen_hosts.append(httpx.URL(url).host)
+        return mock
+
+    monkeypatch.setattr(ssrf, "validate_url", _no_validate)
+    monkeypatch.setattr("forge.util.http.select_client", _spy_select)
+
+    policy = EgressPolicy(block_private=True, allow_private_hosts=("internal.corp",))
+    try:
+        r = await ssrf.guarded_request(
+            None, "GET", "https://internal.corp/x", policy=policy, skip_verify=True, follow_redirects=True,
+        )
+        assert r.status_code == 200
+    finally:
+        await mock.aclose()
+
+    # select_client is consulted per hop with that hop's own host, so the real select_client would
+    # hand the public leg the VERIFIED client (proven by the gate test above), not the insecure one.
+    assert seen_hosts == ["internal.corp", "public.example"]

apps/api/tests/test_tool_io_capture.py

@@ -0,0 +1,151 @@
+"""Tool-I/O capture for traces.
+
+A REST tool records its FRAMED request (resolved URL, query, headers/cookies templated
+from {{ctx.*}}) plus the response into a context var; the ForgeTracer reads it back onto
+the tool span. This is what makes a run-time "works in test, 401s in a run" visible: the
+capture runs on failure too, so a dropped ctx cookie / a 401 shows up in the trace.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+import httpx
+import pytest
+
+from forge.config import settings
+from forge.tools.rest import execute_rest
+from forge.tracing import tool_io
+from forge.tracing.tracer import ForgeTracer
+
+
+def _cfg(**extra) -> dict:
+    return {
+        "name": f"tool_{uuid.uuid4().hex[:6]}",
+        "kind": "rest_api",
+        "request": {
+            "method": "GET",
+            "url_template": "https://api.acme.dev/items/{id}",
+            "fields": [
+                {"path": "id", "in": "path", "llm_visible": True},
+                {"path": "q", "in": "query", "llm_visible": True},
+                # server-injected session cookie, templated from run context (not an LLM arg)
+                {"path": "sid", "in": "cookie", "default": "{{ctx.jsessionid}}", "llm_visible": False},
+            ],
+            "headers": [{"name": "X-CSRF-Token", "value": "{{ctx.csrf}}"}],
+        },
+        **extra,
+    }
+
+
+def _client(handler) -> httpx.AsyncClient:
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+
+async def test_capture_frames_request_and_response():
+    tool_io.clear_tool_io()
+    client = _client(lambda r: httpx.Response(200, json={"ok": True}))
+    await execute_rest(
+        _cfg(), {"id": "123", "q": "widget"},
+        tenant_id="t", project_id="p", context={"jsessionid": "SESS", "csrf": "CSRF-XYZ"}, client=client,
+    )
+    await client.aclose()
+
+    rec = tool_io.take_tool_io()
+    assert rec is not None
+    inp, out = rec["input"], rec["output"]
+    # the agent's own args, and the fully framed request
+    assert inp["args"] == {"id": "123", "q": "widget"}
+    assert inp["method"] == "GET"
+    assert inp["url"] == "https://api.acme.dev/items/123"   # path param substituted
+    assert inp["query"] == {"q": "widget"}
+    assert inp["headers"]["X-CSRF-Token"] == "CSRF-XYZ"     # {{ctx.csrf}} resolved (full, redact off)
+    assert inp["cookies"]["sid"] == "SESS"                  # {{ctx.jsessionid}} resolved
+    assert out["status"] == 200 and out["response"] == {"ok": True} and out["error"] is None
+
+
+async def test_capture_records_a_failed_call():
+    """A 401 raises out of execute_rest, but the framed request + status must still be captured
+    so the failure is visible in the trace (not silently swallowed)."""
+    tool_io.clear_tool_io()
+    client = _client(lambda r: httpx.Response(401, json={"detail": "no session"}))
+    with pytest.raises(httpx.HTTPStatusError):
+        await execute_rest(
+            _cfg(), {"id": "9"},
+            tenant_id="t", project_id="p", context={}, client=client,  # no ctx -> cookie/csrf dropped
+        )
+    await client.aclose()
+
+    rec = tool_io.take_tool_io()
+    assert rec is not None
+    assert rec["output"]["status"] == 401 and rec["output"]["error"]
+    # the dropped session is visible by its ABSENCE: no sid cookie / empty csrf were sent
+    assert "sid" not in rec["input"]["cookies"]
+
+
+async def test_redaction_masks_secrets_when_enabled(monkeypatch):
+    tool_io.clear_tool_io()
+    monkeypatch.setattr(settings, "trace_tool_io_redact", True)
+    client = _client(lambda r: httpx.Response(200, json={"ok": True}))
+    await execute_rest(
+        _cfg(), {"id": "1"},
+        tenant_id="t", project_id="p", context={"jsessionid": "SESS", "csrf": "CSRF-XYZ"}, client=client,
+    )
+    await client.aclose()
+
+    rec = tool_io.take_tool_io()
+    assert rec["input"]["headers"]["X-CSRF-Token"].startswith("•••")   # masked
+    assert rec["input"]["cookies"]["sid"].startswith("•••")
+    assert "CSRF-XYZ" not in str(rec["input"])                          # secret not persisted
+
+
+async def test_disabled_capture_is_a_noop(monkeypatch):
+    tool_io.clear_tool_io()
+    monkeypatch.setattr(settings, "trace_tool_io", False)
+    client = _client(lambda r: httpx.Response(200, json={"ok": True}))
+    await execute_rest(_cfg(), {"id": "1"}, tenant_id="t", project_id="p", context={}, client=client)
+    await client.aclose()
+    assert tool_io.take_tool_io() is None
+
+
+def test_tracer_attaches_matching_record_to_tool_span():
+    """on_tool_end merges a record whose name matches the span's tool."""
+    tool_io.clear_tool_io()
+    tr = ForgeTracer()
+    rid = uuid.uuid4()
+    tr.on_tool_start({"name": "fetch_item"}, '{"id": "1"}', run_id=rid)
+    tool_io.set_tool_io("fetch_item", request={"method": "GET", "url": "u"}, response={"status": 200})
+    tr.on_tool_end({"ok": True}, run_id=rid)
+
+    sp = tr.spans[str(rid)]
+    assert sp.input == {"method": "GET", "url": "u"} and sp.output == {"status": 200}
+
+
+def test_span_dto_exposes_io():
+    """The traces API must serialize the captured input/output to the web client."""
+    from forge.models import Span
+    from forge.schemas.dto import SpanOut
+
+    sp = Span(
+        id="s1", tenant_id="t", trace_id="tr", name="tool · fetch_item", kind="tool", latency_ms=5,
+        input={"method": "GET", "url": "https://api.acme.dev/items/1"}, output={"status": 200},
+        input_tokens=0, output_tokens=0, cost_usd=0.0,
+    )
+    dto = SpanOut.model_validate(sp)
+    assert dto.input == {"method": "GET", "url": "https://api.acme.dev/items/1"}
+    assert dto.output == {"status": 200}
+
+
+def test_tracer_ignores_stale_record_from_another_tool():
+    """A record left by an EARLIER tool must not attach to a different tool's span; that span
+    falls back to its own raw return value instead."""
+    tool_io.clear_tool_io()
+    tr = ForgeTracer()
+    rid = uuid.uuid4()
+    tr.on_tool_start({"name": "lookup"}, "{}", run_id=rid)
+    tool_io.set_tool_io("some_other_tool", request={"method": "GET"}, response={"status": 200})
+    tr.on_tool_end("plain string result", run_id=rid)
+
+    sp = tr.spans[str(rid)]
+    assert sp.output == "plain string result"          # fell back to the raw return
+    assert sp.input == "{}"                             # kept the provisional LLM args

apps/api/tests/test_tool_reliability.py

@@ -0,0 +1,83 @@
+"""REST tool reliability config: cache, rate_limit, retry."""
+
+from __future__ import annotations
+
+import uuid
+
+import httpx
+import pytest
+
+from forge.tools.rest import execute_rest
+
+
+def _cfg(**extra) -> dict:
+    return {
+        "name": f"t_{uuid.uuid4().hex[:8]}",
+        "kind": "rest_api",
+        "request": {"method": "GET", "url_template": "https://api.acme.dev/v2/ping", "fields": []},
+        **extra,
+    }
+
+
+async def test_cache_serves_repeat_get_without_second_call():
+    calls = {"n": 0}
+
+    def handler(req):
+        calls["n"] += 1
+        return httpx.Response(200, json={"v": calls["n"]})
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    cfg = _cfg(cache={"ttl_seconds": 60})
+    a = await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    b = await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+    assert calls["n"] == 1 and a["raw"] == b["raw"]  # second served from cache
+
+
+async def test_rate_limit_blocks_second_call():
+    client = httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(200, json={})))
+    cfg = _cfg(rate_limit={"per_minute": 1})
+    await execute_rest(cfg, {}, tenant_id="t_rl", project_id="p", client=client)
+    with pytest.raises(RuntimeError):
+        await execute_rest(cfg, {}, tenant_id="t_rl", project_id="p", client=client)
+    await client.aclose()
+
+
+async def test_retry_recovers_from_transient_500():
+    state = {"n": 0}
+
+    def handler(req):
+        state["n"] += 1
+        return httpx.Response(500 if state["n"] == 1 else 200, json={"ok": state["n"]})
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    cfg = _cfg(retry={"max_retries": 2, "initial_delay": 0.001, "jitter": False, "retry_on": ["http_error"]})
+    res = await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+    assert state["n"] == 2 and res["status"] == 200
+
+
+async def test_no_retry_when_not_configured():
+    client = httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(500, json={})))
+    cfg = _cfg()  # no retry policy
+    with pytest.raises(httpx.HTTPStatusError):
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+
+
+async def test_missing_path_param_raises_clear_error():
+    client = httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(200, json={})))
+    cfg = _cfg()
+    cfg["request"]["url_template"] = "https://api.acme.dev/v2/orders/{order_id}"
+    with pytest.raises(ValueError, match="order_id"):
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+
+
+def test_cap_payload_truncates_large_only():
+    from forge.tools.projection import cap_payload
+
+    assert cap_payload({"a": 1}, 20000) == {"a": 1}  # small passes through unchanged
+    big = "x" * 50000
+    out = cap_payload(big, 100)
+    assert isinstance(out, str) and len(out) < 50000 and "truncated" in out

apps/api/tests/test_tool_runtime_injection.py

@@ -0,0 +1,122 @@
+"""Regression: ToolRuntime must be injected into materialized REST/GraphQL tools.
+
+Two historical failure modes (both caused by `from __future__ import annotations`
+in forge/tools/rest.py):
+1. compile time - NameError("ToolRuntime") when create_agent resolved the string
+   annotation against module globals where ToolRuntime wasn't imported.
+2. call time - "_call() missing 1 required positional argument: 'runtime'":
+   langchain_core's StructuredTool detects injectable params via
+   inspect.signature(fn) (raw, unevaluated annotations), so a string annotation
+   made the runtime arg invisible and it was stripped during validation.
+
+These tests exercise the real create_agent → ToolNode → StructuredTool path with a
+scripted model that actually calls the tool.
+"""
+
+import itertools
+
+import pytest
+from langchain_core.language_models.fake_chat_models import GenericFakeChatModel
+from langchain_core.messages import AIMessage
+
+from forge.engine.context import CompileContext
+from forge.tools import rest
+
+REST_CFG = {
+    "name": "get_thing",
+    "description": "Fetch a thing.",
+    "kind": "rest_api",
+    "request": {
+        "method": "GET",
+        "url_template": "https://api.example.dev/things/{thing_id}",
+        "fields": [
+            {"path": "thing_id", "type": "string", "in": "path", "required": True, "llm_visible": True},
+        ],
+    },
+    "response": {},
+}
+
+
+@pytest.fixture()
+def captured_exec(monkeypatch):
+    """Stub execute_rest and capture what the tool coroutine passes through."""
+    captured: dict = {}
+
+    async def fake_exec(cfg, kwargs, *, tenant_id, project_id, context=None, auth_resolver=None, stream_writer=None, client=None, egress_policy=None):
+        captured["kwargs"] = kwargs
+        captured["context"] = context
+        captured["tenant_id"] = tenant_id
+        return {"raw": {"ok": True}, "projected": {"ok": True}, "status": 200, "latency_ms": 1}
+
+    monkeypatch.setattr(rest, "execute_rest", fake_exec)
+    return captured
+
+
+def test_runtime_param_is_visible_to_langchain():
+    """The injected-arg detection must see `runtime` as a real ToolRuntime class."""
+    tool = rest.build_rest_tool(REST_CFG, CompileContext(tenant_id="t", project_id="p"))
+    assert tool._injected_args_keys == frozenset({"runtime"}), (
+        "StructuredTool can't see the runtime param - string annotations strike again "
+        "(check for `from __future__ import annotations` in forge/tools/rest.py)"
+    )
+    # The model-facing schema must NOT advertise runtime as an input.
+    assert "runtime" not in (tool.tool_call_schema.model_json_schema().get("properties") or {})
+
+
+async def test_zero_field_tool_executes_without_injection(captured_exec):
+    """Tools with NO llm-visible fields (empty args schema) hit langchain_core's empty-schema
+    short-circuit, which drops even injected args - the coroutine must tolerate runtime=None."""
+    from langchain.agents import create_agent
+
+    cfg = {
+        "name": "get_weather",
+        "description": "Fetch current weather.",
+        "kind": "rest_api",
+        "request": {"method": "GET", "url_template": "https://api.example.dev/weather", "fields": []},
+        "response": {},
+    }
+
+    class ScriptedModel(GenericFakeChatModel):
+        def bind_tools(self, tools=None, **kwargs):  # noqa: ANN001
+            return self
+
+    script = itertools.cycle(
+        [
+            AIMessage(content="", tool_calls=[{"name": "get_weather", "args": {}, "id": "c1", "type": "tool_call"}]),
+            AIMessage(content="done"),
+        ]
+    )
+    tool = rest.build_rest_tool(cfg, CompileContext(tenant_id="t", project_id="p"))
+    agent = create_agent(model=ScriptedModel(messages=script), tools=[tool])
+
+    out = await agent.ainvoke({"messages": [{"role": "user", "content": "weather?"}]})
+
+    assert captured_exec["kwargs"] == {}
+    tool_msgs = [m for m in out["messages"] if getattr(m, "type", "") == "tool"]
+    assert tool_msgs and "ok" in str(tool_msgs[-1].content)
+
+
+async def test_agent_tool_call_injects_runtime(captured_exec):
+    """Full loop: agent's model emits a tool call; ToolNode must inject runtime."""
+    from langchain.agents import create_agent
+
+    class ScriptedModel(GenericFakeChatModel):
+        def bind_tools(self, tools=None, **kwargs):  # noqa: ANN001
+            return self
+
+    script = itertools.cycle(
+        [
+            AIMessage(content="", tool_calls=[{"name": "get_thing", "args": {"thing_id": "42"}, "id": "c1", "type": "tool_call"}]),
+            AIMessage(content="done"),
+        ]
+    )
+    tool = rest.build_rest_tool(REST_CFG, CompileContext(tenant_id="t", project_id="p"))
+    agent = create_agent(model=ScriptedModel(messages=script), tools=[tool])
+
+    out = await agent.ainvoke({"messages": [{"role": "user", "content": "get thing 42"}]})
+
+    # The tool actually executed (no TypeError about 'runtime') with the model's args.
+    assert captured_exec["kwargs"] == {"thing_id": "42"}
+    tool_msgs = [m for m in out["messages"] if getattr(m, "type", "") == "tool"]
+    assert tool_msgs, "tool result message missing from agent transcript"
+    assert "ok" in str(tool_msgs[-1].content)

apps/api/tests/test_tool_sets.py

@@ -0,0 +1,333 @@
+"""Tool Sets: service CRUD + membership, agent toolset->tools resolution, and the REST API."""
+
+from __future__ import annotations
+
+import uuid
+
+import httpx
+
+from forge.db.base import SessionLocal
+from forge.main import create_app
+from forge.models import Project, Tool, User
+from forge.services.runtime import build_compile_context
+from forge.services.tool_sets import ToolSetService
+from forge.services.tools import ToolService
+
+
+async def _seed(tenant: str, slug: str) -> tuple[str, str, str]:
+    async with SessionLocal() as s:
+        proj = Project(tenant_id=tenant, name="TS Proj", slug=slug, config={})
+        s.add(proj)
+        await s.flush()
+        t1 = Tool(tenant_id=tenant, project_id=proj.id, name="alpha", kind="builtin",
+                  config={"builtin": "calculator", "description": "a"})
+        t2 = Tool(tenant_id=tenant, project_id=proj.id, name="beta", kind="builtin",
+                  config={"builtin": "current_time", "description": "b"})
+        s.add_all([t1, t2])
+        await s.commit()
+        for obj in (proj, t1, t2):
+            await s.refresh(obj)
+        return proj.id, t1.id, t2.id
+
+
+async def test_tool_set_service_crud_and_membership():
+    tenant = "t_ts_svc"
+    pid, t1, t2 = await _seed(tenant, "ts-svc")
+    async with SessionLocal() as s:
+        ts = await ToolSetService.create(s, tenant, pid, name="Billing Tools", description="billing", tool_ids=[t1, t2])
+        assert ts.slug == "billing-tools"
+        assert set(await ToolSetService.member_ids(s, tenant, ts.id)) == {t1, t2}
+        assert set((await ToolSetService.members_map(s, tenant, pid))[ts.id]) == {t1, t2}
+        assert set(await ToolSetService.tool_ids_for_sets(s, tenant, pid, [ts.id])) == {t1, t2}
+
+        # unknown / cross-project ids are filtered out of membership
+        ts2 = await ToolSetService.create(s, tenant, pid, name="X", tool_ids=[t1, "does-not-exist"])
+        assert await ToolSetService.member_ids(s, tenant, ts2.id) == [t1]
+
+        # add / remove membership
+        await ToolSetService.remove_member(s, ts, t1)
+        assert await ToolSetService.member_ids(s, tenant, ts.id) == [t2]
+        await ToolSetService.add_member(s, ts, t1)
+        assert set(await ToolSetService.member_ids(s, tenant, ts.id)) == {t1, t2}
+        await ToolSetService.add_member(s, ts, t1)  # idempotent (no duplicate row)
+        assert len(await ToolSetService.member_ids(s, tenant, ts.id)) == 2
+
+        # rename regenerates a unique slug (collides with ts2's "x")
+        ts = await ToolSetService.update(s, ts, name="X")
+        assert ts.slug == "x-2"
+
+        # update can replace membership wholesale
+        ts = await ToolSetService.update(s, ts, tool_ids=[t2])
+        assert await ToolSetService.member_ids(s, tenant, ts.id) == [t2]
+
+        # delete removes the set and its membership rows
+        set_id = ts.id
+        await ToolSetService.delete(s, ts)
+        assert await ToolSetService.get(s, tenant, set_id) is None
+        assert set_id not in await ToolSetService.members_map(s, tenant, pid)
+
+
+async def test_tool_deletion_removes_membership():
+    tenant = "t_ts_del"
+    pid, t1, t2 = await _seed(tenant, "ts-del")
+    async with SessionLocal() as s:
+        ts = await ToolSetService.create(s, tenant, pid, name="S", tool_ids=[t1, t2])
+        tool = await ToolService.get(s, tenant, t1)
+        await ToolService.delete(s, tool)  # deleting a tool must drop its membership rows
+        assert await ToolSetService.member_ids(s, tenant, ts.id) == [t2]
+
+
+async def test_build_compile_context_resolves_toolset_to_member_tools():
+    tenant = "t_ts_ctx"
+    pid, t1, t2 = await _seed(tenant, "ts-ctx")
+    async with SessionLocal() as s:
+        set_id = (await ToolSetService.create(s, tenant, pid, name="Set A", tool_ids=[t1, t2])).id
+    async with SessionLocal() as s:
+        ctx = await build_compile_context(s, tenant_id=tenant, project_id=pid)
+    # membership is loaded onto the compile context
+    assert set(ctx.toolset_members.get(set_id, [])) == {t1, t2}
+    # an agent granted only the set resolves to the set's member tool ids...
+    assert set(ctx.resolve_tool_ids([], [set_id])) == {t1, t2}
+    # ...and to the materialized tools (both builtins compiled into the registry)
+    assert len(ctx.tools_for(ctx.resolve_tool_ids([], [set_id]))) == 2
+
+
+async def test_tool_sets_api_end_to_end():
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        # register a real user (mutations require a real principal, not the dev fallback)
+        reg = await c.post("/v1/auth/register", json={"email": f"u{uuid.uuid4().hex[:10]}@example.com", "password": "supersecret1"})
+        assert reg.status_code == 201, reg.text
+        c.headers["Authorization"] = f"Bearer {reg.json()['access_token']}"
+        # project + two tools, all via the API (one consistent tenant = the registered workspace)
+        pid = (await c.post("/v1/projects", json={"name": "API TS", "slug": "api-ts"})).json()["id"]
+
+        def _mk(name: str, builtin: str) -> dict:
+            return {"name": name, "kind": "builtin", "config": {"builtin": builtin, "description": name}}
+
+        t1 = (await c.post(f"/v1/projects/{pid}/tools", json=_mk("aa", "calculator"))).json()["id"]
+        t2 = (await c.post(f"/v1/projects/{pid}/tools", json=_mk("bb", "current_time"))).json()["id"]
+
+        # create a set with one member
+        r = await c.post(f"/v1/projects/{pid}/tool-sets", json={"name": "Group One", "description": "g1", "tool_ids": [t1]})
+        assert r.status_code == 201, r.text
+        st = r.json()
+        assert st["slug"] == "group-one" and st["tool_ids"] == [t1] and st["description"] == "g1"
+        sid = st["id"]
+
+        # list
+        r = await c.get(f"/v1/projects/{pid}/tool-sets")
+        assert r.status_code == 200 and any(x["id"] == sid for x in r.json())
+
+        # add + remove via the membership endpoints
+        assert (await c.post(f"/v1/projects/{pid}/tool-sets/{sid}/tools/{t2}")).status_code == 204
+        assert set((await c.get(f"/v1/projects/{pid}/tool-sets/{sid}")).json()["tool_ids"]) == {t1, t2}
+        assert (await c.delete(f"/v1/projects/{pid}/tool-sets/{sid}/tools/{t1}")).status_code == 204
+
+        # patch: rename + replace membership
+        r = await c.patch(f"/v1/projects/{pid}/tool-sets/{sid}", json={"name": "Renamed", "tool_ids": [t1, t2]})
+        assert r.json()["slug"] == "renamed" and set(r.json()["tool_ids"]) == {t1, t2}
+
+        # delete
+        assert (await c.delete(f"/v1/projects/{pid}/tool-sets/{sid}")).status_code == 204
+        assert (await c.get(f"/v1/projects/{pid}/tool-sets/{sid}")).status_code == 404
+
+
+async def _seed_mcp_project(tenant: str, slug: str) -> tuple[str, str, str]:
+    """Project + two builtin tools; returns (project_id, calc_tool_id, clock_tool_id)."""
+    async with SessionLocal() as s:
+        proj = Project(tenant_id=tenant, name="MCP TS", slug=slug, config={})
+        s.add(proj)
+        await s.flush()
+        ta = Tool(tenant_id=tenant, project_id=proj.id, name="calc", kind="builtin",
+                  config={"builtin": "calculator", "description": "c"})
+        tb = Tool(tenant_id=tenant, project_id=proj.id, name="clock", kind="builtin",
+                  config={"builtin": "current_time", "description": "t"})
+        s.add_all([ta, tb])
+        await s.commit()
+        for obj in (proj, ta, tb):
+            await s.refresh(obj)
+        return proj.id, ta.id, tb.id
+
+
+async def _list_names(c: httpx.AsyncClient, path: str) -> set[str]:
+    r = await c.post(path, json={"jsonrpc": "2.0", "id": 1, "method": "tools/list"})
+    return {t["name"] for t in r.json()["result"]["tools"]}
+
+
+async def test_mcp_toolset_scoped_exposure():
+    tenant = "t_mcp_ts"
+    pid, a_id, b_id = await _seed_mcp_project(tenant, "mcp-ts")
+    async with SessionLocal() as s:
+        await ToolSetService.create(s, tenant, pid, name="Set A", tool_ids=[a_id])
+        set_b = await ToolSetService.create(s, tenant, pid, name="Set B", tool_ids=[b_id])
+        b_slug, b_set_id = set_b.slug, set_b.id
+
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        # base endpoint = flat union of every EXPOSED set's enabled tools
+        assert await _list_names(c, f"/v1/mcp/{pid}") == {"calc", "clock"}
+        # per-set endpoint = just that set's flat list
+        assert await _list_names(c, f"/v1/mcp/{pid}/toolset/{b_slug}") == {"clock"}
+        assert await _list_names(c, f"/v1/mcp/{pid}/toolset/nope") == set()  # unknown slug => empty
+        r = await c.post(f"/v1/mcp/{pid}/toolset/{b_slug}", json={"jsonrpc": "2.0", "id": 3, "method": "tools/call",
+                                                                  "params": {"name": "clock", "arguments": {}}})
+        assert r.json()["result"]["isError"] is False
+
+        # un-expose Set B -> it drops off both the base surface and its own endpoint
+        async with SessionLocal() as s:
+            await ToolSetService.update(s, await ToolSetService.get(s, tenant, b_set_id), exposed=False)
+        assert await _list_names(c, f"/v1/mcp/{pid}") == {"calc"}
+        assert await _list_names(c, f"/v1/mcp/{pid}/toolset/{b_slug}") == set()
+        r = await c.post(f"/v1/mcp/{pid}", json={"jsonrpc": "2.0", "id": 2, "method": "tools/call",
+                                                 "params": {"name": "clock", "arguments": {}}})
+        assert "not exposed" in r.json()["error"]["message"]
+
+
+async def test_mcp_tool_level_exclusion():
+    """Everything in an exposed set is published by default; an operator can untick individual
+    tools via project.config.mcp_excluded_tools."""
+    tenant = "t_mcp_excl"
+    pid, a_id, b_id = await _seed_mcp_project(tenant, "mcp-excl")
+    async with SessionLocal() as s:
+        await ToolSetService.create(s, tenant, pid, name="Set", tool_ids=[a_id, b_id])
+        proj = await s.get(Project, pid)
+        proj.config = {"mcp_excluded_tools": [b_id]}  # untick 'clock'
+        await s.commit()
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        assert await _list_names(c, f"/v1/mcp/{pid}") == {"calc"}
+
+
+async def test_mcp_no_toolsets_exposes_nothing():
+    tenant = "t_mcp_none"
+    pid, a_id, _b = await _seed_mcp_project(tenant, "mcp-none")
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        # No tool sets => nothing published (there are no loose / "direct" tools).
+        assert await _list_names(c, f"/v1/mcp/{pid}") == set()
+        # once a tool is placed in an exposed set, it appears
+        async with SessionLocal() as s:
+            await ToolSetService.create(s, tenant, pid, name="General", tool_ids=[a_id])
+        assert await _list_names(c, f"/v1/mcp/{pid}") == {"calc"}
+
+
+async def test_mcp_session_token_authorizes_as_end_user():
+    """A project-scoped Forge session token authenticates an MCP caller AS its end_user
+    (the portable per-user identity channel), alongside the shared project key."""
+    from forge.security import create_session_token
+
+    tenant = "t_mcp_sess"
+    async with SessionLocal() as s:
+        proj = Project(tenant_id=tenant, name="Sess", slug="mcp-sess", config={"mcp_api_key": "shared-key"})
+        s.add(proj)
+        await s.flush()
+        s.add(Tool(tenant_id=tenant, project_id=proj.id, name="calc", kind="builtin",
+                   config={"builtin": "calculator", "description": "c"}))
+        await s.commit()
+        await s.refresh(proj)
+        pid = proj.id
+
+    good = create_session_token(tenant_id=tenant, project_id=pid, end_user={"id": "u1", "entitlements": ["billing"]})
+    wrong_project = create_session_token(tenant_id=tenant, project_id="another", end_user={"id": "u2"})
+    body = {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        # shared key -> authorized (no per-user identity)
+        assert (await c.post(f"/v1/mcp/{pid}", headers={"Authorization": "Bearer shared-key"}, json=body)).status_code == 200
+        # project-scoped session token -> authorized as that end user
+        assert (await c.post(f"/v1/mcp/{pid}", headers={"Authorization": f"Bearer {good}"}, json=body)).status_code == 200
+        # session token scoped to a different project -> rejected
+        assert (await c.post(f"/v1/mcp/{pid}", headers={"Authorization": f"Bearer {wrong_project}"}, json=body)).status_code == 401
+        # no credential -> rejected (a key is configured)
+        assert (await c.post(f"/v1/mcp/{pid}", json=body)).status_code == 401
+
+
+async def test_mcp_personal_access_token_authorizes_as_user():
+    """A per-user Personal Access Token (forge_pat_) authenticates an MCP client as that user."""
+    from forge.services.apikeys import ApiKeyService
+
+    tenant = "t_mcp_pat"
+    async with SessionLocal() as s:
+        proj = Project(tenant_id=tenant, name="PAT", slug="mcp-pat", config={"mcp_api_key": "shared-key"})
+        s.add(proj)
+        await s.flush()
+        s.add(Tool(tenant_id=tenant, project_id=proj.id, name="calc", kind="builtin",
+                   config={"builtin": "calculator", "description": "c"}))
+        user = User(tenant_id=tenant, email="pat-user@example.com", role="editor", status="active")
+        s.add(user)
+        await s.commit()
+        await s.refresh(proj)
+        await s.refresh(user)
+        pid = proj.id
+        _k1, pat = await ApiKeyService.create_personal(s, tenant_id=tenant, user_id=user.id, name="t", project_id=pid)
+        key_id = _k1.id
+        _k2, pat_other = await ApiKeyService.create_personal(s, tenant_id=tenant, user_id=user.id, name="t2", project_id="another-project")
+
+    body = {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        # a project-scoped PAT authorizes
+        assert (await c.post(f"/v1/mcp/{pid}", headers={"Authorization": f"Bearer {pat}"}, json=body)).status_code == 200
+        # a PAT scoped to a different project is rejected here
+        assert (await c.post(f"/v1/mcp/{pid}", headers={"Authorization": f"Bearer {pat_other}"}, json=body)).status_code == 401
+        # once revoked, the PAT no longer authorizes
+        async with SessionLocal() as s:
+            await ApiKeyService.revoke_personal(s, tenant_id=tenant, user_id=user.id, key_id=key_id)
+        assert (await c.post(f"/v1/mcp/{pid}", headers={"Authorization": f"Bearer {pat}"}, json=body)).status_code == 401
+
+
+async def test_mcp_token_api_crud():
+    """The user-facing PAT endpoints mint / list / revoke tokens, and a minted token works on MCP."""
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        reg = await c.post("/v1/auth/register", json={"email": f"u{uuid.uuid4().hex[:10]}@example.com", "password": "supersecret1"})
+        assert reg.status_code == 201, reg.text
+        c.headers["Authorization"] = f"Bearer {reg.json()['access_token']}"
+        pid = (await c.post("/v1/projects", json={"name": "PAT API", "slug": "pat-api"})).json()["id"]
+        # lock the MCP surface behind a key so credential checks are meaningful
+        await c.patch(f"/v1/projects/{pid}", json={"config": {"mcp_api_key": "k"}})
+
+        r = await c.post(f"/v1/projects/{pid}/mcp-tokens", json={"name": "my token"})
+        assert r.status_code == 201, r.text
+        tok = r.json()
+        assert tok["token"].startswith("forge_pat_") and tok["status"] == "active"
+
+        body = {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}
+        # the freshly minted PAT authenticates against the MCP endpoint (auth is enforced by the key)
+        assert (await c.post(f"/v1/mcp/{pid}", json=body)).status_code == 401
+        pat_headers = {"Authorization": f"Bearer {tok['token']}"}
+        assert (await c.post(f"/v1/mcp/{pid}", headers=pat_headers, json=body)).status_code == 200
+
+        # listed without the plaintext, then revoked
+        lst = (await c.get(f"/v1/projects/{pid}/mcp-tokens")).json()
+        assert any(t["id"] == tok["id"] and t.get("token") is None for t in lst)
+        assert (await c.delete(f"/v1/projects/{pid}/mcp-tokens/{tok['id']}")).status_code == 204
+        assert (await c.post(f"/v1/mcp/{pid}", headers=pat_headers, json=body)).status_code == 401
+
+
+async def test_connector_role_is_mcp_only():
+    """A 'connector' user can manage their own MCP tokens but cannot mutate project resources."""
+    from forge.security import create_access_token
+
+    tenant = "t_conn_role"
+    async with SessionLocal() as s:
+        u = User(tenant_id=tenant, email="connector@example.com", role="connector", status="active")
+        proj = Project(tenant_id=tenant, name="Conn", slug="conn-p", config={})
+        s.add_all([u, proj])
+        await s.commit()
+        await s.refresh(u)
+        await s.refresh(proj)
+        uid, pid = u.id, proj.id
+
+    token = create_access_token(user_id=uid, tenant_id=tenant, role="connector")
+    app = create_app()
+    async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c:
+        c.headers["Authorization"] = f"Bearer {token}"
+        # cannot create/mutate project resources (needs editor+)
+        assert (await c.post("/v1/projects", json={"name": "X", "slug": "x-conn"})).status_code == 403
+        assert (await c.post(f"/v1/projects/{pid}/tool-sets", json={"name": "S"})).status_code == 403
+        # but can mint their own MCP personal access token
+        r = await c.post(f"/v1/projects/{pid}/mcp-tokens", json={"name": "my token"})
+        assert r.status_code == 201, r.text
+        assert r.json()["token"].startswith("forge_pat_")

apps/api/tests/test_tools_code_sql.py

@@ -0,0 +1,82 @@
+"""Code-tool (RestrictedPython sandbox) and SQL-tool (read-only) tests."""
+
+from __future__ import annotations
+
+import pytest
+
+from forge.services.runtime import make_runtime_ctx
+from forge.tools.code import CodeToolError, execute_code, run_code
+from forge.tools.materialize import materialize_tool
+from forge.tools.sql import SqlToolError, execute_sql
+
+
+@pytest.fixture(autouse=True)
+def _enable_code_tools(monkeypatch):
+    # Code tools are OFF by default in prod-safe config (unsandboxed RestrictedPython is not
+    # an isolation boundary - audit S5); these tests exercise the feature, so opt in.
+    from forge.config import settings
+
+    monkeypatch.setattr(settings, "enable_code_tools", True)
+
+
+# --- code tool ---
+
+
+async def test_code_tool_main_returns_value():
+    cfg = {"name": "adder", "kind": "code", "language": "python",
+           "source": "def main(a, b):\n    return a + b\n",
+           "args_schema": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}, "required": ["a", "b"]}}
+    assert await execute_code(cfg, {"a": 2, "b": 5}) == 7
+
+
+async def test_code_tool_allows_safe_import():
+    assert await execute_code({"source": "import math\ndef main(x):\n    return math.sqrt(x)\n"}, {"x": 9}) == 3.0
+
+
+async def test_code_tool_blocks_unsafe_import():
+    with pytest.raises(CodeToolError):
+        run_code("import os\ndef main():\n    return os.getcwd()\n", {})
+
+
+async def test_code_tool_blocks_dunder_escape():
+    with pytest.raises(CodeToolError):
+        run_code("def main():\n    return ().__class__.__bases__\n", {})
+
+
+async def test_code_tool_materializes_as_structured_tool():
+    cfg = {"name": "upper", "kind": "code",
+           "source": "def main(s):\n    return s.upper()\n",
+           "args_schema": {"properties": {"s": {"type": "string"}}, "required": ["s"]}}
+    tool = materialize_tool(cfg, make_runtime_ctx("t", "p"))
+    assert await tool.ainvoke({"s": "hi"}) == "HI"
+
+
+# --- sql tool (read-only) ---
+
+
+async def test_sql_tool_rejects_writes():
+    cfg = {"name": "q", "kind": "sql", "query": "DELETE FROM users", "connection_url": "sqlite+aiosqlite:///:memory:"}
+    with pytest.raises(SqlToolError):
+        await execute_sql(cfg, {}, tenant_id="t", project_id="p")
+
+
+async def test_sql_tool_rejects_multi_statement():
+    cfg = {"name": "q", "kind": "sql", "query": "SELECT 1; DROP TABLE users", "connection_url": "sqlite+aiosqlite:///:memory:"}
+    with pytest.raises(SqlToolError):
+        await execute_sql(cfg, {}, tenant_id="t", project_id="p")
+
+
+async def test_sql_tool_reads_rows(tmp_path):
+    import sqlite3
+
+    db = tmp_path / "demo.db"
+    con = sqlite3.connect(db)
+    con.executescript("CREATE TABLE t(id INTEGER, name TEXT); INSERT INTO t VALUES (1,'a'),(2,'b');")
+    con.commit()
+    con.close()
+
+    cfg = {"name": "q", "kind": "sql", "connection_url": f"sqlite+aiosqlite:///{db.as_posix()}",
+           "query": "SELECT id, name FROM t WHERE id = :id",
+           "args_schema": {"properties": {"id": {"type": "integer"}}, "required": ["id"]}}
+    res = await execute_sql(cfg, {"id": 2}, tenant_id="t", project_id="p")
+    assert res["rows"] == [{"id": 2, "name": "b"}]

apps/api/tests/test_tools_hardening.py

@@ -0,0 +1,356 @@
+"""Hardening tests for the tool executors (robustness-audit fixes).
+
+Covers: retry semantics (idempotency + transient-only default), the shared entitlement gate for
+every non-REST kind, reliability parity (rate_limit/cache) for graphql/sql/code via the shared
+wrapper, JMESPath projection error markers, SQL read-only/limit/cell-cap hardening, GraphQL
+in-band error handling + operationName, mcp being creatable, REST multipart + download-size guard,
+and production-default trace redaction.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import uuid
+
+import httpx
+import pytest
+
+from forge.services.runtime import make_runtime_ctx
+from forge.tools import rest as rest_mod
+from forge.tools.graphql import GraphQLToolError, execute_graphql
+from forge.tools.materialize import materialize_tool
+from forge.tools.projection import project_response
+from forge.tools.rest import _resolve_retry, _retry_types, _should_retry, execute_rest
+from forge.tools.sql import SqlToolError, execute_sql
+
+
+@pytest.fixture(autouse=True)
+def _enable_code_tools(monkeypatch):
+    from forge.config import settings
+
+    monkeypatch.setattr(settings, "enable_code_tools", True)
+
+
+def _rest_cfg(**extra) -> dict:
+    return {
+        "name": f"t_{uuid.uuid4().hex[:8]}",
+        "kind": "rest_api",
+        "request": {"method": "GET", "url_template": "https://api.acme.dev/v2/ping", "fields": []},
+        **extra,
+    }
+
+
+# --- Finding 1: retry semantics (transient-only default, idempotency gating, schema default) ----
+
+
+def test_should_retry_gates_on_idempotency_and_status():
+    transient = _retry_types([])
+    # 4xx is never retried by default (regression: HTTPError superclass used to retry it).
+    resp4 = httpx.Response(404, request=httpx.Request("GET", "https://x"))
+    err4 = httpx.HTTPStatusError("nf", request=resp4.request, response=resp4)
+    assert _should_retry(err4, transient, True, "GET", {}) is False
+    # 5xx IS retried on an idempotent method by the default classification...
+    resp5 = httpx.Response(503, request=httpx.Request("GET", "https://x"))
+    err5 = httpx.HTTPStatusError("boom", request=resp5.request, response=resp5)
+    assert _should_retry(err5, transient, True, "GET", {}) is True
+    # ...but NOT on a non-idempotent POST unless explicitly opted in.
+    assert _should_retry(err5, transient, True, "POST", {}) is False
+    assert _should_retry(err5, transient, True, "POST", {"retry_non_idempotent": True}) is True
+
+
+def test_resolve_retry_defaults_align_with_schema():
+    # No retry block => opt-out (no retries), preserving historic behavior.
+    assert _resolve_retry({})[0] == 0
+    # A retry block present but max_retries omitted => schema default of 2.
+    assert _resolve_retry({"retry": {}})[0] == 2
+    assert _resolve_retry({"retry": {"max_retries": 5}})[0] == 5
+
+
+async def test_default_retry_does_not_retry_4xx():
+    calls = {"n": 0}
+
+    def handler(req):
+        calls["n"] += 1
+        return httpx.Response(404, json={"e": "nope"})
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    cfg = _rest_cfg(retry={"max_retries": 3, "initial_delay": 0.001, "jitter": False})
+    with pytest.raises(httpx.HTTPStatusError):
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+    assert calls["n"] == 1  # a permanent 4xx is not retried
+
+
+async def test_default_retry_retries_transient_5xx_on_get():
+    state = {"n": 0}
+
+    def handler(req):
+        state["n"] += 1
+        return httpx.Response(500 if state["n"] == 1 else 200, json={"ok": state["n"]})
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    cfg = _rest_cfg(retry={"max_retries": 2, "initial_delay": 0.001, "jitter": False})  # no retry_on
+    res = await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+    assert state["n"] == 2 and res["status"] == 200
+
+
+async def test_post_5xx_not_retried_by_default():
+    calls = {"n": 0}
+
+    def handler(req):
+        calls["n"] += 1
+        return httpx.Response(500, json={})
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    cfg = _rest_cfg(retry={"max_retries": 3, "initial_delay": 0.001, "jitter": False})
+    cfg["request"]["method"] = "POST"
+    with pytest.raises(httpx.HTTPStatusError):
+        await execute_rest(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+    assert calls["n"] == 1  # non-idempotent POST is not auto-retried
+
+
+# --- Finding 4: broken JMESPath projection -> structured error marker (not silent full payload) --
+
+
+def test_broken_jmespath_returns_error_marker():
+    data = {"secret": "x" * 100, "items": [1, 2, 3]}
+    out = project_response(data, {"projection_jmespath": "items[?"})  # malformed expression
+    assert isinstance(out, dict) and out.get("error") == "projection_error"
+    assert "expression" in out and out != data  # did NOT masquerade as the full payload
+
+
+def test_valid_jmespath_missing_key_is_not_error():
+    out = project_response({"a": 1}, {"projection_jmespath": "nope.missing"})
+    assert out is None  # a valid expression selecting nothing is not an error
+
+
+# --- Finding 5: SQL read-only / limit / cell-cap hardening ---------------------------------------
+
+
+def _sqlite_url(tmp_path, rows=5) -> str:
+    db = tmp_path / f"h_{uuid.uuid4().hex[:6]}.db"
+    con = sqlite3.connect(db)
+    con.executescript("CREATE TABLE t(id INTEGER, name TEXT);")
+    con.executemany("INSERT INTO t VALUES (?, ?)", [(i, f"n{i}") for i in range(1, rows + 1)])
+    con.commit()
+    con.close()
+    return f"sqlite+aiosqlite:///{db.as_posix()}"
+
+
+async def test_sql_forbids_into_outfile(tmp_path):
+    cfg = {"name": "q", "kind": "sql", "connection_url": _sqlite_url(tmp_path),
+           "query": "SELECT * FROM t INTO OUTFILE '/tmp/x'"}
+    with pytest.raises(SqlToolError):
+        await execute_sql(cfg, {}, tenant_id="t", project_id="p")
+
+
+async def test_sql_streaming_truncation_is_accurate(tmp_path):
+    url = _sqlite_url(tmp_path, rows=5)
+    over = await execute_sql({"name": "q", "kind": "sql", "connection_url": url,
+                              "query": "SELECT id FROM t ORDER BY id", "max_rows": 3}, {},
+                             tenant_id="t", project_id="p")
+    assert over["row_count"] == 3 and over["truncated"] is True
+    exact = await execute_sql({"name": "q", "kind": "sql", "connection_url": url,
+                               "query": "SELECT id FROM t ORDER BY id", "max_rows": 5}, {},
+                              tenant_id="t", project_id="p")
+    assert exact["row_count"] == 5 and exact["truncated"] is False  # exactly max_rows is not truncated
+
+
+async def test_sql_caps_large_cell(tmp_path):
+    cfg = {"name": "q", "kind": "sql", "connection_url": _sqlite_url(tmp_path, rows=1),
+           "query": "SELECT printf('%.*c', 30000, 'x') AS big"}
+    res = await execute_sql(cfg, {}, tenant_id="t", project_id="p")
+    big = res["rows"][0]["big"]
+    assert len(big) < 30000 and "truncated" in big
+
+
+# --- Finding 7: GraphQL in-band errors + operationName -------------------------------------------
+
+
+async def test_graphql_errors_with_null_data_raise():
+    client = httpx.AsyncClient(transport=httpx.MockTransport(
+        lambda r: httpx.Response(200, json={"data": None, "errors": [{"message": "boom"}]})
+    ))
+    cfg = {"name": "g", "kind": "graphql", "endpoint": "https://api.acme.dev/graphql", "query": "{ me { id } }"}
+    with pytest.raises(GraphQLToolError, match="boom"):
+        await execute_graphql(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+
+
+async def test_graphql_partial_data_passes_through():
+    client = httpx.AsyncClient(transport=httpx.MockTransport(
+        lambda r: httpx.Response(200, json={"data": {"me": {"id": "1"}}, "errors": [{"message": "field x failed"}]})
+    ))
+    cfg = {"name": "g", "kind": "graphql", "endpoint": "https://api.acme.dev/graphql", "query": "{ me { id } }"}
+    res = await execute_graphql(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+    assert res["raw"]["data"] == {"me": {"id": "1"}}  # partial success is valid GraphQL
+
+
+async def test_graphql_sends_operation_name():
+    seen = {}
+
+    def handler(req):
+        import json as _j
+
+        seen.update(_j.loads(req.content))
+        return httpx.Response(200, json={"data": {"ok": True}})
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    cfg = {"name": "g", "kind": "graphql", "endpoint": "https://api.acme.dev/graphql",
+           "query": "query A { a } query B { b }", "operation_name": "B"}
+    await execute_graphql(cfg, {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+    assert seen.get("operationName") == "B"
+
+
+# --- Finding 2: entitlement gate for every non-REST kind (deny independently of the LLM) ---------
+
+
+def _ctx_without(entitlement="billing:read"):
+    ctx = make_runtime_ctx("t", "p")
+    ctx.end_user = {"id": "u1", "entitlements": []}  # user lacks the required entitlement
+    return ctx
+
+
+async def test_graphql_entitlement_denied():
+    ctx = _ctx_without()
+    cfg = {"name": "g", "kind": "graphql", "endpoint": "https://api.acme.dev/graphql",
+           "query": "{ me { id } }", "required_entitlements": ["billing:read"]}
+    tool = materialize_tool(cfg, ctx)
+    out = await tool.ainvoke({})
+    assert "Not permitted" in out  # denied before any network call
+
+
+async def test_sql_entitlement_denied(tmp_path):
+    ctx = _ctx_without()
+    cfg = {"name": "q", "kind": "sql", "connection_url": _sqlite_url(tmp_path),
+           "query": "SELECT id FROM t", "required_entitlements": ["billing:read"]}
+    tool = materialize_tool(cfg, ctx)
+    out = await tool.ainvoke({})
+    assert "Not permitted" in out
+
+
+async def test_code_entitlement_denied():
+    ctx = _ctx_without()
+    cfg = {"name": "c", "kind": "code", "language": "python",
+           "source": "def main():\n    return 1\n", "required_entitlements": ["billing:read"]}
+    tool = materialize_tool(cfg, ctx)
+    out = await tool.ainvoke({})
+    assert "Not permitted" in out
+
+
+async def test_component_entitlement_denied():
+    from forge.tools.components import build_component_tool
+
+    ctx = _ctx_without()
+    cfg = {"id": "c1", "name": "chart", "props_schema": {}, "required_entitlements": ["billing:read"]}
+    tool = build_component_tool(cfg, ctx)
+    out = await tool.ainvoke({})
+    assert "Not permitted" in out
+
+
+async def test_entitled_user_is_allowed(tmp_path):
+    ctx = make_runtime_ctx("t", "p")
+    ctx.end_user = {"id": "u2", "entitlements": ["billing:read"]}
+    cfg = {"name": "q", "kind": "sql", "connection_url": _sqlite_url(tmp_path, rows=2),
+           "query": "SELECT id FROM t ORDER BY id", "required_entitlements": ["billing:read"]}
+    tool = materialize_tool(cfg, ctx)
+    out = await tool.ainvoke({})
+    assert [r["id"] for r in out] == [1, 2]  # entitled -> query actually runs
+
+
+# --- Finding 3: reliability (rate_limit + cache) reach graphql/sql/code via the shared wrapper ---
+
+
+async def test_sql_rate_limit_via_wrapper(tmp_path):
+    ctx = make_runtime_ctx(f"t_{uuid.uuid4().hex[:6]}", "p")
+    cfg = {"name": f"q_{uuid.uuid4().hex[:6]}", "kind": "sql", "connection_url": _sqlite_url(tmp_path),
+           "query": "SELECT id FROM t", "rate_limit": {"per_minute": 1}}
+    tool = materialize_tool(cfg, ctx)
+    await tool.ainvoke({})
+    with pytest.raises(RuntimeError, match="rate limit"):
+        await tool.ainvoke({})
+
+
+async def test_sql_cache_via_wrapper(tmp_path):
+    url = _sqlite_url(tmp_path, rows=1)
+    ctx = make_runtime_ctx("t", "p")
+    cfg = {"name": f"q_{uuid.uuid4().hex[:6]}", "kind": "sql", "connection_url": url,
+           "query": "SELECT id FROM t ORDER BY id", "cache": {"ttl_seconds": 60}}
+    tool = materialize_tool(cfg, ctx)
+    first = await tool.ainvoke({})
+    # Mutate the DB behind the cache; a cache hit must still return the ORIGINAL rows.
+    path = url.split(":///", 1)[1]
+    con = sqlite3.connect(path)
+    con.execute("INSERT INTO t VALUES (99, 'new')")
+    con.commit()
+    con.close()
+    second = await tool.ainvoke({})
+    assert first == second == [{"id": 1}]  # served from cache, not re-queried
+
+
+# --- Finding 9: kind:"mcp" is creatable (materialize returns a deferred None, does not raise) -----
+
+
+def test_materialize_mcp_returns_none_instead_of_raising():
+    ctx = make_runtime_ctx("t", "p")
+    cfg = {"name": "gh", "kind": "mcp", "mcp_client_id": "srv1", "remote_tool_name": "list_issues"}
+    assert materialize_tool(cfg, ctx) is None
+
+
+def test_materialize_unknown_kind_still_raises():
+    ctx = make_runtime_ctx("t", "p")
+    with pytest.raises(ValueError, match="Unknown tool kind"):
+        materialize_tool({"name": "x", "kind": "bogus"}, ctx)
+
+
+# --- Finding 6: REST multipart encoding + download-size guard ------------------------------------
+
+
+async def test_multipart_body_encoding_sends_form_data():
+    seen = {}
+
+    def handler(req):
+        seen["ct"] = req.headers.get("content-type", "")
+        seen["body"] = req.content.decode("utf-8", "replace")
+        return httpx.Response(200, json={"ok": True})
+
+    client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    cfg = _rest_cfg()
+    cfg["request"] = {
+        "method": "POST", "url_template": "https://api.acme.dev/upload", "body_encoding": "multipart",
+        "fields": [{"path": "title", "type": "string", "in": "body"}],
+    }
+    await execute_rest(cfg, {"title": "hello"}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+    assert seen["ct"].startswith("multipart/form-data") and "hello" in seen["body"]
+
+
+async def test_download_size_guard_marks_oversized_body(monkeypatch):
+    monkeypatch.setattr(rest_mod, "_MAX_DOWNLOAD_BYTES", 50)
+    client = httpx.AsyncClient(transport=httpx.MockTransport(
+        lambda r: httpx.Response(200, json={"blob": "x" * 500})
+    ))
+    res = await execute_rest(_rest_cfg(), {}, tenant_id="t", project_id="p", client=client)
+    await client.aclose()
+    assert res["raw"]["error"] == "response_too_large" and res["raw"]["bytes"] > 50
+
+
+# --- Finding 10: trace I/O redaction defaults ON for a production install --------------------------
+
+
+def test_trace_redaction_defaults_on_in_production(monkeypatch):
+    from forge.config import settings
+    from forge.tracing import tool_io
+
+    monkeypatch.setattr(settings, "trace_tool_io_redact", False)
+    # Dev + flag off => values pass through.
+    monkeypatch.setattr(settings, "environment", "development")
+    assert tool_io.redact_headers({"Authorization": "Bearer secret"})["Authorization"] == "Bearer secret"
+    # Production + flag off => sensitive values are masked anyway.
+    monkeypatch.setattr(settings, "environment", "production")
+    masked = tool_io.redact_headers({"Authorization": "Bearer secret"})["Authorization"]
+    assert masked != "Bearer secret" and "secret" not in masked

apps/api/tests/test_tracer_interrupt.py

@@ -0,0 +1,42 @@
+"""HITL interrupts are control flow, not failures.
+
+When HumanInTheLoopMiddleware calls interrupt() it RAISES a GraphInterrupt (a GraphBubbleUp) to
+suspend the graph for approval. That raised signal reaches the tracer's on_*_error callbacks, so
+the tracer must recognize it and NOT record it as a span error - otherwise a run paused for human
+approval renders as a crash (the agent + HITL spans showing red), which is exactly the misleading
+trace this guards against. The run's own status is already a first-class `interrupted`.
+"""
+
+from __future__ import annotations
+
+from langgraph.errors import GraphInterrupt
+
+from forge.tracing.tracer import ForgeTracer
+
+
+def test_chain_interrupt_is_not_a_span_error():
+    tr = ForgeTracer()
+    tr.on_chain_start({"name": "agent_1"}, {}, run_id="agent-1")
+    tr.on_chain_error(GraphInterrupt(), run_id="agent-1")  # HITL pause bubbling up
+    sp = tr.spans["agent-1"]
+    assert sp.error is None, "an interrupt must not mark the span errored"
+    assert sp.attributes.get("interrupted") is True
+    assert sp.end is not None  # span is still closed (latency captured)
+
+
+def test_tool_interrupt_is_not_a_span_error():
+    tr = ForgeTracer()
+    tr.on_tool_start({"name": "get_weather"}, "args", run_id="tool-1")
+    tr.on_tool_error(GraphInterrupt(), run_id="tool-1")
+    sp = tr.spans["tool-1"]
+    assert sp.error is None
+    assert sp.attributes.get("interrupted") is True
+
+
+def test_real_error_is_still_recorded():
+    tr = ForgeTracer()
+    tr.on_chain_start({"name": "some_node"}, {}, run_id="node-1")
+    tr.on_chain_error(ValueError("boom"), run_id="node-1")
+    sp = tr.spans["node-1"]
+    assert sp.error == "boom"
+    assert not sp.attributes.get("interrupted")

apps/api/tests/test_triggers.py

@@ -0,0 +1,74 @@
+"""Trigger sync + scheduling + webhook dispatch (end-to-end via services)."""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from langgraph.checkpoint.memory import InMemorySaver
+
+from forge.db.base import SessionLocal
+from forge.models import Trigger, Workflow
+from forge.services.dispatch import dispatch_trigger
+from forge.services.runs import RunService
+from forge.services.triggers import TriggerService
+
+_WEBHOOK_WF = {
+    "id": "wf_hook", "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}},
+    "entry_node": "hook",
+    "nodes": [
+        {"id": "hook", "type": "webhook_in", "config": {"message_path": "text"}},
+        {"id": "agent", "type": "agent", "config": {"flavor": "agent", "model": "fake:Done."}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [{"source": "hook", "target": "agent"}, {"source": "agent", "target": "end"}],
+}
+
+
+async def _make_wf(tenant="t_trig", project="p_trig") -> Workflow:
+    async with SessionLocal() as s:
+        wf = Workflow(tenant_id=tenant, project_id=project, name="Hooked", executable=_WEBHOOK_WF, status="active")
+        s.add(wf)
+        await s.commit()
+        await s.refresh(wf)
+        await TriggerService.sync_from_workflow(s, wf)
+        return wf
+
+
+async def test_sync_creates_webhook_trigger_with_key():
+    wf = await _make_wf()
+    async with SessionLocal() as s:
+        trigs = (await s.execute(Trigger.__table__.select().where(Trigger.workflow_id == wf.id))).fetchall()
+    assert len(trigs) == 1
+    async with SessionLocal() as s:
+        t = await TriggerService.by_key(s, trigs[0].key)
+    assert t is not None and t.kind == "webhook_in" and t.key
+
+
+def test_build_input_message_path_extracts_field():
+    t = Trigger(tenant_id="t", project_id="p", workflow_id="w", node_id="hook", kind="webhook_in", config={"message_path": "text"})
+    assert TriggerService.build_input(t, {"text": "hello there"}) == {"messages": [{"role": "user", "content": "hello there"}]}
+
+
+def test_build_input_schedule_uses_config_message():
+    t = Trigger(tenant_id="t", project_id="p", workflow_id="w", node_id="s", kind="schedule", config={"message": "tick"})
+    assert TriggerService.build_input(t, None)["messages"][0]["content"] == "tick"
+
+
+def test_is_due_interval():
+    t = Trigger(tenant_id="t", project_id="p", workflow_id="w", node_id="s", kind="schedule", config={"every_minutes": 10}, enabled=True)
+    assert TriggerService.is_due(t, datetime.utcnow()) is True  # never fired -> due
+    t.last_fired_at = datetime.utcnow()
+    assert TriggerService.is_due(t, datetime.utcnow()) is False
+    t.last_fired_at = datetime.utcnow() - timedelta(minutes=11)
+    assert TriggerService.is_due(t, datetime.utcnow()) is True
+
+
+async def test_dispatch_webhook_runs_workflow():
+    wf = await _make_wf("t_d", "p_d")
+    async with SessionLocal() as s:
+        trig = (await s.execute(Trigger.__table__.select().where(Trigger.workflow_id == wf.id))).fetchone()
+        trigger = await TriggerService.by_key(s, trig.key)
+    rs = RunService(checkpointer=InMemorySaver())
+    result = await dispatch_trigger(rs, trigger, {"text": "ping"})
+    assert result.get("answer") == "Done." and result.get("status") == "done"

apps/api/tests/test_validation.py

@@ -0,0 +1,82 @@
+"""Validate the workflow validator: schema refs, per-node config, structural rules."""
+
+from __future__ import annotations
+
+import copy
+
+from forge.services.validation import validate_workflow
+
+GOOD = {
+    "id": "wf_ok",
+    "version": 1,
+    "state": {"messages": {"type": "list[message]", "reducer": "add_messages"},
+              "intent": {"type": "str", "reducer": "last"}},
+    "entry_node": "start",
+    "nodes": [
+        {"id": "start", "type": "start", "config": {}},
+        {"id": "route", "type": "router", "config": {
+            "expression": "intent", "cases": {"billing": "billing_agent"}, "default": "billing_agent"}},
+        {"id": "billing_agent", "type": "agent", "config": {
+            "flavor": "agent", "model": "anthropic:claude-sonnet-4-6",
+            "middleware": [{"type": "summarization", "config": {"trigger": ["tokens", 4000]}}]}},
+        {"id": "end", "type": "end", "config": {}},
+    ],
+    "edges": [
+        {"source": "start", "target": "route"},
+        {"source": "billing_agent", "target": "end"},
+    ],
+}
+
+
+def test_good_workflow_passes():
+    res = validate_workflow(GOOD)
+    assert res.valid, res.errors
+
+
+def test_unknown_node_type_flagged():
+    wf = copy.deepcopy(GOOD)
+    wf["nodes"][0]["type"] = "frobnicate"
+    res = validate_workflow(wf)
+    assert not res.valid
+    assert any("frobnicate" in e["message"] for e in res.errors)
+
+
+def test_bad_node_config_flagged_with_pointer():
+    wf = copy.deepcopy(GOOD)
+    # agent requires `model`; remove it
+    del wf["nodes"][2]["config"]["model"]
+    res = validate_workflow(wf)
+    assert not res.valid
+    assert any(e["pointer"].startswith("/nodes/2/config") for e in res.errors), res.errors
+
+
+def test_unknown_middleware_type_flagged():
+    wf = copy.deepcopy(GOOD)
+    wf["nodes"][2]["config"]["middleware"] = [{"type": "does_not_exist", "config": {}}]
+    res = validate_workflow(wf)
+    assert not res.valid
+    assert any("does_not_exist" in e["message"] for e in res.errors)
+
+
+def test_edge_to_unknown_node_flagged():
+    wf = copy.deepcopy(GOOD)
+    wf["edges"].append({"source": "billing_agent", "target": "ghost"})
+    res = validate_workflow(wf)
+    assert not res.valid
+    assert any("ghost" in e["message"] for e in res.errors)
+
+
+def test_orphan_node_flagged():
+    wf = copy.deepcopy(GOOD)
+    wf["nodes"].append({"id": "lonely", "type": "llm", "config": {"model": "fake:x", "prompt": "hi"}})
+    res = validate_workflow(wf)
+    assert not res.valid
+    assert any("lonely" in e["message"] and "unreachable" in e["message"] for e in res.errors)
+
+
+def test_no_path_to_end_flagged():
+    wf = copy.deepcopy(GOOD)
+    wf["edges"] = [e for e in wf["edges"] if e["target"] != "end"]  # cut the only END path
+    res = validate_workflow(wf)
+    assert not res.valid
+    assert any("END" in e["message"] for e in res.errors)

apps/api/tests/test_version_retention.py

@@ -0,0 +1,52 @@
+"""Version-history retention resolution (item 11).
+
+The console Settings > Versioning panel writes `version_history_limit` into project.config;
+snapshot()/prune must honor it, with precedence project.config > tenant.settings > global.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+from forge.config import settings
+from forge.db.base import SessionLocal
+from forge.models import Project, Workflow
+from forge.services.versions import VersionService, _limit_for
+
+
+def test_limit_precedence():
+    g = int(settings.version_history_limit)
+    # project.config wins over tenant + global
+    assert _limit_for({"version_history_limit": 7}, {"version_history_limit": 3}) == 3
+    # tenant override used when the project sets none
+    assert _limit_for({"version_history_limit": 7}, {}) == 7
+    assert _limit_for({"version_history_limit": 7}, None) == 7
+    # global default when neither is set
+    assert _limit_for(None, None) == g
+    # a non-integer value falls through to the next source
+    assert _limit_for(None, {"version_history_limit": "nope"}) == g
+
+
+async def test_project_config_limit_prunes_snapshots():
+    t = f"t_{uuid.uuid4().hex[:8]}"
+    async with SessionLocal() as s:
+        proj = Project(tenant_id=t, name="P", slug=f"p{uuid.uuid4().hex[:6]}", config={"version_history_limit": 2})
+        s.add(proj)
+        await s.flush()
+        wf = Workflow(tenant_id=t, project_id=proj.id, name="w", executable={}, status="draft")
+        s.add(wf)
+        await s.commit()
+        pid, wid = proj.id, wf.id
+
+    async with SessionLocal() as s:
+        for i in range(5):
+            await VersionService.snapshot(
+                s, tenant_id=t, entity_type="workflow", entity_id=wid,
+                data={"name": f"v{i}"}, project_id=pid,
+            )
+            await s.commit()
+
+    async with SessionLocal() as s:
+        rows = await VersionService.list(s, t, "workflow", wid)
+        # project.config limit=2 -> only the two newest snapshots survive
+        assert [r.version_no for r in rows] == [5, 4], [r.version_no for r in rows]