commit ae67bff5a3446ae7f1bba597abb5aadf106275f4 Author: nihalashetty Date: Tue Jul 28 01:49:19 2026 +0530 feat: deep-agent canvas, live observability, and multi-environment tooling Self-hosted platform for building, testing, and shipping LangChain/LangGraph agents. Deep-agent sub-agents on the canvas, a live tracing/observability timeline, auto-provisioned built-in tools with import/export, per-environment tool variables, streamed evaluations, and per-user auth token forwarding. diff --git a/.claude/launch.json b/.claude/launch.json new file mode 100644 index 0000000..396e984 --- /dev/null +++ b/.claude/launch.json @@ -0,0 +1,18 @@ +{ + "version": "0.0.1", + "configurations": [ + { + "name": "web", + "runtimeExecutable": "pnpm", + "runtimeArgs": ["--filter", "web", "dev"], + "port": 3000 + }, + { + "name": "web-dev", + "runtimeExecutable": "pnpm", + "runtimeArgs": ["--filter", "web", "exec", "next", "dev"], + "autoPort": true, + "port": 3000 + } + ] +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..81a6a80 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +**/node_modules +**/.next +**/.venv +**/.data +**/__pycache__ +**/*.pyc +**/.pytest_cache +**/.ruff_cache +**/.mypy_cache +**/*.egg-info +**/.git +**/*.db +**/*.sqlite +**/.env +**/.env.local +**/.DS_Store +# The api/worker builds now use context '.', so the old apps/api/.dockerignore is dead (Docker +# reads only the .dockerignore at the context root). Keep tests out of the api image (nothing +# at runtime imports them); explicit path avoids over-matching a top-level 'tests'. +apps/api/tests diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..afd03d4 --- /dev/null +++ b/.env.example @@ -0,0 +1,47 @@ +# Forge — environment config. Copy to `.env` at the repo ROOT (gitignored) and set real values. +# All backend vars are prefixed FORGE_. This file holds PLACEHOLDERS ONLY — never commit real secrets. + +# --- Core --- +FORGE_ENVIRONMENT=development # "production" enforces the checklist at the bottom + +# --- Platform auth --- +FORGE_JWT_SECRET=dev-insecure-change-me # MUST be a strong random value in production +# FORGE_AUTH_REQUIRED=true # default true — keep true so the service token is a real gate +FORGE_BOOTSTRAP_ADMIN_EMAIL=you@forge.local +FORGE_BOOTSTRAP_ADMIN_PASSWORD=forge-admin # change for production + +# --- Server-to-server barrier (an app backend → Forge) --- +# Static bearer that authenticates a trusted backend as a least-privilege service identity. +# Generate: python -c "import secrets; print(secrets.token_urlsafe(32))" +FORGE_SERVICE_API_TOKEN= + +# --- SSRF egress: allow specific private/loopback hosts (dev/testing only) --- +FORGE_EGRESS_ALLOW_PRIVATE_HOSTS=[] # e.g. ["localhost","127.0.0.1"] (Docker: ["host.docker.internal"]) + +# --- Tools: per-environment values --- +# A JSON map exposed to REST/GraphQL tool + auth templates as {{env.*}}, so the SAME tool row +# resolves to a different host per deploy (dev/qa/prod). A template referencing a key NOT in this +# map fails the call loudly. Blank/unset = {}. +FORGE_TOOL_VARS= # e.g. {"api_base":"https://api.example.com"} +# Deployment-wide fallback for a per-user auth provider's `token_ctx_key`: the run-context key an +# integration forwards its per-user token under (via X-Forge-Context) when a provider doesn't set +# its own. Empty = off. e.g. user_token +FORGE_DEFAULT_TOKEN_CTX_KEY= + +# --- Model (needed for live agent runs) --- +FORGE_DEFAULT_MODEL=fake:echo # offline-safe; set a real model for live runs, e.g. gpt-4.1-mini +# OPENAI_API_KEY=sk-... +# ANTHROPIC_API_KEY=sk-ant-... + +# --- Frontend --- +NEXT_PUBLIC_FORGE_API_URL=http://localhost:8000 + +# ============================================================================ +# PRODUCTION (Docker stack) — uncomment/set when deploying: +# POSTGRES_PASSWORD= # compose substitutes this into the DB URL +# FORGE_DATABASE_URL=postgresql+psycopg://forge:@localhost:5432/forge +# FORGE_CHECKPOINT_BACKEND=postgres +# FORGE_REDIS_URL=redis://localhost:6379/0 +# The app REFUSES to boot in production unless: strong FORGE_JWT_SECRET, +# FORGE_AUTH_REQUIRED=true, FORGE_EGRESS_BLOCK_PRIVATE=true, Postgres FORGE_DATABASE_URL. +# ============================================================================ \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..560cb39 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Report something that isn't working as expected +title: "[bug] " +labels: bug +--- + +**Describe the bug** +A clear and concise description of what's wrong. + +**To reproduce** +Steps to reproduce the behavior: +1. Go to '…' +2. Configure '…' +3. Run '…' +4. See error + +**Expected behavior** +What you expected to happen. + +**Screenshots / logs** +If applicable, add screenshots or relevant log output. **Redact any secrets, tokens, or +customer data.** + +**Environment** +- Forge version / commit SHA: +- Deployment: local (SQLite) / Docker (Postgres) / other +- OS & browser (for console issues): +- Python / Node version (for local dev): + +**Additional context** +Anything else that might help — workflow/tool config (redacted), provider, etc. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..5cc36c0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: 🔒 Report a security vulnerability + url: https://github.com/nihalashetty/Forge/security/advisories/new + about: Please report vulnerabilities privately via a GitHub Security Advisory, not a public issue. + - name: 💬 Questions & discussion + url: https://github.com/nihalashetty/Forge/discussions + about: Ask questions and share ideas with the community. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..5379255 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,19 @@ +--- +name: Feature request +about: Suggest an idea or improvement +title: "[feat] " +labels: enhancement +--- + +**What problem are you trying to solve?** +A clear description of the use case or pain point. "I'm always frustrated when …" + +**Proposed solution** +What you'd like Forge to do. + +**Alternatives considered** +Other approaches or workarounds you've thought about. + +**Additional context** +Mockups, links, related nodes/tools, or examples. Check the +[roadmap](../../docs/ROADMAP.md) first — it may already be planned. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..7464f6b --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,27 @@ + + +## What & why + + + +## Type of change + +- [ ] Bug fix +- [ ] New feature +- [ ] Performance +- [ ] Refactor (no behavior change) +- [ ] Docs / chore + +## Checklist + +- [ ] Backend: `ruff check forge migrations` is clean and `pytest -q` passes (from `apps/api`) +- [ ] New/changed backend code is `mypy`-clean +- [ ] Frontend: `pnpm --filter web build` passes (from repo root) +- [ ] Shared schemas (`packages/schemas`) updated if node/tool config changed +- [ ] Tests added/updated for the change (characterization test for behavior-preserving refactors) +- [ ] `CHANGELOG.md` updated under **Unreleased** (for user-facing changes) +- [ ] No secrets, tokens, or customer data in the diff + +## Notes for reviewers + + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8086efd --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,30 @@ +version: 2 +updates: + # Backend Python dependencies (PEP 621 pyproject.toml). + - package-ecosystem: "pip" + directory: "/apps/api" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + groups: + python: + patterns: ["*"] + + # Frontend / workspace JS dependencies (pnpm workspace at the repo root). + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + groups: + javascript: + patterns: ["*"] + + # GitHub Actions used by the workflows. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + actions: + patterns: ["*"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ff88677 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,51 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +# Least privilege: CI only reads the checked-out code; it never writes back to the repo. +permissions: + contents: read + +jobs: + api: + name: API - lint + tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: apps/api + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e ".[dev,all]" + - name: Ruff + run: ruff check forge migrations + - name: Mypy (advisory — gradual typing, see CONTRIBUTING.md) + continue-on-error: true + run: mypy forge + - name: Pytest + run: pytest -q + + web: + name: Web - typecheck + build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + with: + version: 9 + - uses: actions/setup-node@v7 + with: + node-version: "22" + cache: pnpm + - name: Install + run: pnpm install --frozen-lockfile || pnpm install + - name: Build + run: pnpm --filter web build diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..4b49672 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,34 @@ +name: CodeQL + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "0 6 * * 1" # weekly, Monday 06:00 UTC + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + permissions: + security-events: write + contents: read + actions: read + strategy: + fail-fast: false + matrix: + language: [python, javascript-typescript] + steps: + - uses: actions/checkout@v7 + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + language: ${{ matrix.language }} + - name: Autobuild + uses: github/codeql-action/autobuild@v4 + - name: Analyze + uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{ matrix.language }}" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d9b98f8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,44 @@ +# --- Python --- +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ + +# --- Node / Next --- +node_modules/ +.next/ +out/ +dist/ +build/ +*.tsbuildinfo +.turbo/ + +# --- Forge runtime data & secrets --- +apps/api/.data/ +*.db +*.sqlite +*.sqlite3 +.env +.env.local +master.key + +# --- Editors / OS / tooling --- +.vscode/ +.idea/ +.DS_Store +Thumbs.db + +# --- Claude Code / agent tooling (local dev only; not part of the product) --- +# Keep .claude/launch.json (the preview config) tracked; ignore everything else. +.agents/ +.claude/* +!.claude/launch.json + +# --- WebFetch / temp artifacts --- +*.bin + +# --- Local architecture notes (not part of the product) --- +/PENDING_FEATURES.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..996d6db --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,168 @@ +# Changelog + +All notable changes to Forge are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this +project adheres to [Semantic Versioning](https://semver.org/). + +## [Unreleased] + +### Deep agents on canvas, live observability & multi-environment tools +- **Deep-agent sub-agents on the canvas** (new): a Deep Agent node gains a third **subagents** handle — + drag it to any specialist agent node to fold that node in as a callable sub-agent (rendered as a + dashed org-chart branch). The compiler folds each wired agent into the supervisor's `subagents` + (name/description/system_prompt/tools/model), and the deep agent is now built as `create_agent` + + only the opt-in deepagents middleware you enable (planning / filesystem / sub-agents / skills) + instead of the full `create_deep_agent` harness — a concise task-tool prompt replaces deepagents' + ~536-token essay, cutting per-supervisor-turn cost. Skills stay wired via `SkillsMiddleware`. +- **Live agent-activity + named sub-agent traces** (new): the tracer records a deep-agent `task` + dispatch as a named `subagent · ` span (kind `subagent`) instead of a generic `tool · task`, + and streams a per-tool/per-sub-agent **activity** timeline over SSE. The Playground shows this live + ("Agent activity"), the canvas Test panel lights up folded sub-agent nodes, and the **Traces** view + is now a **collapsible span tree** (real `parent_span_id` hierarchy, friendly canvas node names, + per-kind colored dots, collapse/expand-all). +- **Per-environment tool values `{{env.*}}`** (new): a `FORGE_TOOL_VARS` JSON map is exposed to + REST/GraphQL tool + auth endpoint templates as `{{env.*}}`, so the SAME tool row resolves to a + different host per deploy (dev/qa/prod). Missing keys **fail loud** (never a broken URL); `{{ctx.*}}` + stays lenient. +- **Protected, auto-provisioned built-ins**: the platform built-ins (time, calculator, web + fetch/search, knowledge search, `remember`/`recall`) are provisioned into every project on create + and on tools-list read, pinned to the top of the Tools screen, and **cannot be deleted** (409). They + are excluded from import/export bundles, so importing a project neither duplicates nor loses them. +- **Import / export now carries tool sets**: a tool bundle includes the tool sets grouping its tools; + import re-creates them (remapping ids, auto-renaming on collision) and reports the count. +- **Live-streamed evaluations**: dataset runs stream `start` / `item` / `done` SSE frames — every case + renders immediately and resolves live with per-case status, latency, and tokens, plus a progress bar. +- **Console runs act as the logged-in operator**: Playground and canvas-test runs are attributed to + the signed-in user (removing the manual "Acting as" box), so per-user auth providers resolve the + operator's own connected credential — the same on-behalf-of path evals now use. +- **`FORGE_DEFAULT_TOKEN_CTX_KEY`** (new): a deployment-wide fallback for a per-user auth provider's + `token_ctx_key`, so an integration that always forwards its per-user token under one context key + works for every provider without per-provider configuration. +- **Fixed:** OpenAI streamed runs now report token usage / cost (`stream_usage`), instead of 0; a HITL + turn with several approval-gated tool calls resumes correctly (one decision replicated to each + hanging call) in both workflow runs and the Forge Assistant; the MCP endpoint shown on the Deploy + screen points at the API host directly so OAuth discovery works. + +### Tool sets, MCP server, governance & portability +- **Tool sets** (new): a describable, many-to-many group of tools that does two jobs at once — + it organizes the Tools screen (folders/filter chips) *and* is the unit of assignment and + exposure. Grant an agent a whole set (`agent.config.toolsets`, resolved to member tools at + compile time) and publish a set as a GitHub-style **MCP toolset**. +- **Per-project MCP server — full transport + auth.** The exposed server now speaks native + **Streamable-HTTP / SSE** (Claude Desktop, Cursor, VS Code connect **directly — no `mcp-remote` + bridge**), with the legacy request/response JSON-RPC POST preserved for simple clients; both + share one auth + tool-resolution core. Three ways to authenticate: a shared **project API key** + (server-to-server, no identity), a per-user **personal access token** (PAT, `forge_pat_…`), and + optional **OAuth 2.1** (Dynamic Client Registration + PKCE S256, audience-bound tokens; + default-off behind `FORGE_MCP_OAUTH_ENABLED`). The exposed surface is exactly the enabled tools + of **exposed tool sets**; knowledge, Q&A, and a whole workflow can also be published as MCP + tools. A least-privileged **connector** role can self-serve MCP tokens and call tools but sees + no projects/settings. +- **Per-user identity over MCP + connected credentials.** A project-scoped session token or PAT + resolves to an `end_user`, threaded into the run so entitlement gating and `{{ctx.*}}` injection + act per user. An OAuth auth-provider can key its token bundle **per end user** + (`per_user_context_keys`); the app owner stores each user's bundle via the new **connections API** + (`PUT/GET/DELETE /v1/projects/{id}/auth-providers/{apId}/connections/{endUserId}`). No MCP token + is ever passed downstream — Forge holds a separate per-user credential. +- **Guardrails & Egress** (new): a single project-level I/O policy in **Settings → Guardrails & + Egress** (admin-gated), enforced **by default on every agent** — no per-agent wiring. Content + guardrails (PII redaction, custom `Label = regex` patterns, blocked terms with + redact/mask/hash/block/flag) run locally in-process; the network egress policy (block-private + + allow/deny domain lists) is applied to every REST/GraphQL tool, webhook, `web_fetch`, and SQL + host. A project may only **tighten** inherited egress, never loosen it. +- **Import / Export (portability)** (new): export **tools, workflows, components, and agents** to a + portable `forge.bundle/1` JSON file and import them into another project. Secret **values never + leave** (only `secret://…` references travel; import warns you to recreate them); imports **never + overwrite** (new ids, auto-rename on name collision) and remap in-bundle references. Available + from each list screen's toolbar; import requires the `editor` role. +- **Model catalog from the backend**: the provider/model list is served by the API as one source + of truth (was duplicated in the web app). + +### Changed (console & runtime) +- **Console reskin**: a shadcn-style **neutral + indigo** design system and a minimal sidebar nav; + Traces now read like a chat history; unified screen headings and bare icons app-wide. +- **Performance:** cut interactive chat latency by eliminating a ~4s cold-connection DNS (AAAA) + stall on outbound LLM/REST calls and reusing pooled LLM connections + (`FORGE_PREFER_IPV4_EGRESS`, default on). + +### Fixed (this cycle) +- Map `openai_moderation` middleware flags to `langchain-openai >=1.3`. +- Group a HITL pause+resume into a single trace turn, and stop recording a HITL interrupt as a + span error. +- Classifier sends one bounded human turn for cross-provider compatibility; the agent binds at + most one tool per function name. + +### Feature-bounty fixes (correctness, governance, and DX) +- **Entity version history** (new): every save of a workflow/agent/tool/component/auth-provider/ + knowledge-source/project snapshots to `entity_versions`; view + restore in the console; retention + pruned to a configurable `version_history_limit`. +- **Engine correctness:** Loop nodes no longer crash past ~8 iterations (run `recursion_limit` is + set); many previously-ignored node/middleware options now work (Join reducer, parallel + isolation/ordering/timeout, tenant-budget USD cap + per-run token scope, guardrail + apply_to/redact/flag, model_retry retry_on, subworkflow input/output mapping, Transform jq, LLM + `{{state}}` templating, agent-node dynamic prompt/model); validation now errors on undeclared + state-key writes + branches-without-condition. +- **RAG grounding:** default relevance floor calibrated to the local embedder (0.18 → 0.6), + thresholds the true cosine in hybrid mode + a rerank floor (so off-topic → "I don't know"), + chunk citations, per-page crawl provenance (+robots/limits), MMR, resumable batched ingest. +- **Isolation/privacy:** per-user long-term memory scope; response-cache keyed by tenant/user/auth; + Postgres RLS actually wired (per-transaction tenant GUC); MCP `stdio` gated + external-MCP SSRF + screening; tool-I/O trace redaction on by default in production. +- **Reliability:** webhook + `/run` idempotency; scheduler on by default with an atomic + double-fire-safe claim; HITL TOCTOU + chained-interrupt + timeout fixes; outbound channel retry + with delivery status; run cancellation; transient-only tool retries. +- **Observability:** OTel export fixed (wall-clock times + real span hierarchy); cost accounting + handles prompt-cache tiers + dated/unlisted models; retriever/embedding spans; evals gain + concurrency, persisted history + regression gate, more scorers, robust judge. +- **Platform/governance:** per-project RBAC + scoped revocable API keys; auth rate-limiting, + refresh rotation, logout, password-reset/verify, optional TOTP MFA; project budgets + + allowed-models enforcement; scheduled retention purge; audit pagination/export + secret.read; + fail-closed public rate limiter; extended hardening guard; workspace management; worker DLQ. +- **MCP:** exposed-server rate-limited + per-project tool allow-list; expose a whole workflow as + an MCP tool (`mcp_expose_workflow`). +- **Semantic caching** wired as an agent middleware (was built but unreachable). +- **Console:** Settings redesigned with a section sidebar (incl. a model-pricing editor); a + restrained de-colored palette; version-history drawer; canvas unsaved-changes guard + undo/redo + + copy-paste; Playground Stop + real thread reset; Deep Agent config panel. + +### Added +- Project developer meta: `CONTRIBUTING.md`, `SECURITY.md`, `CODE_OF_CONDUCT.md`, this + changelog, and GitHub issue/PR templates. +- Lightweight `GET /v1/projects/{id}/counts` endpoint powering the sidebar badge counts. +- In-flight GET de-duplication in the web API client (collapses duplicate concurrent + requests into one). +- Characterization tests for the stats rollups (`apps/api/tests/test_stats.py`). +- Static type-checking with `mypy` (advisory in CI; gradual adoption) and a `CodeQL` + workflow + Dependabot for supply-chain updates. + +### Changed +- **Performance:** dashboard and project stats now compute rollups as SQL aggregates + (`COUNT`/`SUM` + `GROUP BY`) instead of loading a tenant's entire trace history into + memory. Output is unchanged. +- **Performance:** the project sidebar fetches one counts call instead of six full lists; + the dashboard fetches its stats once (was twice); Overview reuses counts. +- **Performance:** the Traces view loads conversations 20 at a time with infinite scroll + instead of fetching the entire history at once. +- Typed API responses for the counts and stats endpoints (`response_model`), improving the + generated OpenAPI schema. +- Pinned `ruff` to a reproducible range so CI lint doesn't drift with new rule sets. + +### Fixed +- Documentation drift: backend README layout, root README architecture description, + `TECH_STACK.md` embedder entry, and roadmap chunking strategies. +- Pre-existing lint findings (import order, statement style, mutable `ContextVar` default). + +### Removed +- Dead code: an orphaned frontend screen and a half-wired "code" workflow node (frontend + palette entry + orphan schema with no backend registration). +- Committed local agent tooling that was not part of the product. + +## [0.1.0] + +- Initial Forge platform: visual agent/workflow builder on LangChain + LangGraph, tools + (REST/GraphQL/Code/SQL/MCP/built-in), knowledge & RAG, generative-UI components, + embeddable widget, channels, triggers, evaluations, observability/traces, and a + production-shaped Docker stack. See the [README](README.md) and [ROADMAP](docs/ROADMAP.md). + +[Unreleased]: https://github.com/nihalashetty/Forge/compare/main...HEAD diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..ff35478 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,42 @@ +# Code of Conduct + +## Our commitment + +We want Forge to be a welcoming, harassment-free community for everyone, regardless of +experience level, background, or identity. We are committed to a respectful and +constructive environment for all contributors and users. + +## Expected behavior + +- Be respectful, considerate, and collaborative. +- Welcome newcomers and assume good intent. +- Give and gracefully accept constructive feedback; critique ideas, not people. +- Focus on what is best for the community and the project. + +## Unacceptable behavior + +- Harassment, insults, or derogatory comments; personal or political attacks. +- Discriminatory language or imagery, or unwelcome sexual attention. +- Publishing others' private information without explicit permission. +- Sustained disruption of discussions, issues, or reviews. + +## Scope + +This code applies in all project spaces — the repository, issues, pull requests, +discussions — and when an individual is representing the project in public spaces. + +## Enforcement + +Instances of abusive or unacceptable behavior may be reported to the maintainers privately +via a [GitHub Security Advisory](https://github.com/nihalashetty/Forge/security/advisories/new) +or by contacting a maintainer directly. All reports will be reviewed and investigated +promptly and fairly, and reporter confidentiality will be respected. + +Maintainers may take any action they deem appropriate, up to and including a temporary or +permanent ban from the community, and will communicate the reasons for moderation +decisions when appropriate. + +--- + +This Code of Conduct is adapted in spirit from the +[Contributor Covenant](https://www.contributor-covenant.org/). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ea8a12e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,74 @@ +# Contributing to Forge + +Thanks for your interest in improving Forge! This guide covers how to get set up, the +checks your change needs to pass, and our conventions. + +## Ways to contribute + +- **Report bugs** and **request features** via [issues](https://github.com/nihalashetty/Forge/issues) (templates provided). +- **Improve docs** — the [User Manual](docs/MANUAL.md), READMEs, and `TECH_STACK.md`. +- **Fix or build** — pick up an open issue or propose a change in a discussion first for anything large. + +## Development setup + +See the [README quick start](README.md#quick-start-local-zero-infra) for the full local +(zero-infra) setup. In short: + +```bash +# Backend (FastAPI engine) +cd apps/api +python -m venv .venv && source .venv/bin/activate # .venv\Scripts\activate on Windows +pip install -e ".[dev,all]" + +# Frontend (Next.js console) — from the repo root +pnpm install +pnpm --filter web dev +``` + +The local stack runs on SQLite + embedded Chroma + an in-process scheduler — no Docker, +Postgres, or Redis required. See [`apps/api/README.md`](apps/api/README.md) for the backend +layout and production swaps. + +## Before you open a pull request + +Run the same checks CI runs: + +**Backend** (from `apps/api`): +```bash +ruff check forge migrations # lint (pinned; must be clean) +mypy forge # type check (advisory today — keep new/changed code clean) +pytest -q # tests must pass +``` + +**Frontend** (from the repo root): +```bash +pnpm --filter web build # typecheck + build +``` + +Additional expectations: + +- **Keep the shared schemas authoritative.** `packages/schemas` is the single source of + truth for node/tool config; the backend validator, the compiler, and the frontend + `` all read from it. Update the schema, not one consumer. +- **Add tests** for new behavior. For refactors that must preserve behavior, add a + characterization test first (see `apps/api/tests/test_stats.py` for the pattern). +- **Type-checking is gradual.** `mypy` is advisory in CI while we clear a backlog on the + older engine modules, but any file you add or substantially change should be mypy-clean. + +## Commit & PR conventions + +- **Conventional Commits** for messages: `feat:`, `fix:`, `perf:`, `chore:`, `docs:`, + `refactor:`, `test:` — optionally scoped, e.g. `perf(stats): …`. +- **Atomic commits** — one logical change per commit; keep history bisectable. +- Open a PR describing the change and the reasoning. Link the issue it addresses. +- Update [`CHANGELOG.md`](CHANGELOG.md) under **Unreleased** for anything user-facing. + +## Versioning + +Forge follows [Semantic Versioning](https://semver.org/). User-facing changes are recorded +in the changelog and rolled into the next release. + +## License + +By contributing, you agree that your contributions are licensed under the +[MIT License](LICENSE), the same license as the project. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..937c1fd --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Nihal A Shetty + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..a025ba3 --- /dev/null +++ b/README.md @@ -0,0 +1,238 @@ +
+ +# Forge + +**The open-source, self-hosted platform for visually building, testing, and shipping AI agents & workflows.** + +Wire agents, tools, knowledge, and logic on a canvas - ground them in your data, connect them to your systems, and deploy to email, an API, an MCP server, or an embeddable web widget. No framework code required. + +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) +[![Python](https://img.shields.io/badge/Python-3.11–3.13-3776AB?logo=python&logoColor=white)](apps/api/pyproject.toml) +[![Node](https://img.shields.io/badge/Node-22%20LTS-339933?logo=nodedotjs&logoColor=white)](apps/web/package.json) +[![FastAPI](https://img.shields.io/badge/FastAPI-009688?logo=fastapi&logoColor=white)](apps/api) +[![Next.js](https://img.shields.io/badge/Next.js-14-000000?logo=nextdotjs&logoColor=white)](apps/web) +[![Built with LangChain v1 + LangGraph v1](https://img.shields.io/badge/Built%20with-LangChain%20v1%20%2B%20LangGraph%20v1-1C3C3C)](https://github.com/langchain-ai/langchain) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](#contributing) + +
+ +--- + +Forge is built directly on the **MIT-licensed LangChain v1 + LangGraph v1** framework - and **never** depends on `langgraph-api` (Elastic 2.0) or LangSmith (commercial). Everything you orchestrate runs on your own infrastructure; nothing is sent to a third-party orchestration service. + +- **Fully open source (MIT).** No proprietary core, no usage caps, no vendor lock-in. +- **Zero-infra local dev.** Boots on SQLite + embedded Chroma + an in-process scheduler - no Docker, Postgres, or Redis required to start. +- **Production-ready.** Swap to Postgres + pgvector, Redis, and a real secret store with config only - a hardening guard refuses to boot with insecure defaults. +- **Observable by default.** Every run is a span waterfall with tokens, latency, and cost down to fractions of a cent. + +## Table of contents + +- [Features](#features) +- [Architecture](#architecture) +- [Quick start](#quick-start-local-zero-infra) +- [Run with Docker](#run-with-docker-production-shaped) +- [Documentation](#documentation) +- [Tech stack](#tech-stack) +- [Contributing](#contributing) +- [License](#license) + +## Features + +
+ +
+ +> **[Watch the demo](docs/media/Forge_demo.mp4)** - the in-product **Forge Assistant** builds and runs a workflow end to end. *(If the player doesn't load inline, click the link to play.)* + +### Visual Workflow Builder + +Wire an entire app on a **drag-and-drop canvas** (React Flow): drop nodes from the palette - **agents** & deep agents, model calls, classifiers, tools, transforms, retrieval, human input/handoff, routers, loops, parallel fan-out/join, subworkflows, and triggers - and connect them with **typed, validated** edges. A per-node inspector and a live **state schema** keep runs type-safe, while a minimap, undo/redo, and copy/paste keep big graphs manageable. **Save**, **Test**, or open the **Playground** to watch nodes light up as the run streams - then **Publish**. + +

Forge visual workflow builder: a React Flow canvas wiring retrieval, router, agent, and end nodes, with a node palette, a state-schema inspector, and a minimap

+ +### Visual Agent Builder + +Compose an **Agent** or a **Deep Agent** (planning + subagents for long multi-step tasks) from a model, a system prompt, tools, knowledge, Q&A, and a reorderable **middleware stack** - all from friendly forms, no JSON. A live *"what the model sees"* panel shows the exact compiled prompt and middleware execution order before you ship. Build a **supervisor** visually: drag from a Deep Agent's **subagents** handle to any specialist agent node and it folds in as a callable sub-agent (an org-chart branch on the canvas) - each with its own model, tools, and prompt, dispatched via the `task` tool and shown as named sub-agent spans in the trace. + +

Forge visual agent builder with model, instructions, tools, and a 'what the model sees' panel

+ +### Tool Builder with Response Projection + +Register **REST, GraphQL, Code, SQL, MCP, or built-in** tools and test them live against real inputs. A **JMESPath response projection** trims bulky payloads *before* they reach the model - watch the raw → projected **token meter** shrink in real time to control cost. Every outbound call is screened by an **SSRF guard**, with optional retries, rate limits, and caching. Point a tool's endpoint at a **per-environment value** with `{{env.*}}` (from `FORGE_TOOL_VARS`) so the same tool row resolves to your dev / qa / prod host per deploy. The platform **built-ins** (time, calculator, web fetch/search, knowledge search, memory) are auto-provisioned into every project and protected from deletion. Organize tools into **tool sets** - reusable, many-to-many groups that double as folders on the screen, get granted to an agent in one click, and publish as MCP toolsets. + +

Forge tool builder with request config, live response, and a raw-to-projected token cost meter

+ +The Tools screen groups everything by set (switchable between grid and list), with one-click **export / import** to move tools between projects: + +

Forge Tools screen: the tool grid grouped into reusable tool sets, with grid/list views and one-click export and import

+ +### Knowledge & RAG + +Ground agents in **your own data**. Add pasted text, URLs, crawled sites, or uploaded files (`.txt/.md/.csv/.json/.html/.pdf`); Forge chunks, embeds (**offline-capable by default**), and stores them as vectors organized in folders. Curated **Q&A pairs** deflect common questions, and a **search debugger** lets you inspect exactly what retrieval returns. + +

Forge knowledge screen showing document sources, folders, chunking status, and Q&A

+ +The built-in **search debugger** plots every chunk by semantic similarity and overlays a query - so you can see exactly which chunks a search retrieves, and why: + +

Forge knowledge search debugger: a PCA chunk map colored by source, with a query overlay marking the retrieved chunks

+ +### Generative UI Components + +Let agents render **rich, interactive UI** - tables, cards, forms - instead of plain text. Author an HTML/CSS component once with a live preview; the model emits a tiny payload while the markup renders in a **sandboxed iframe** and never bloats the token stream. Buttons can send structured actions straight back to the agent. + +

Forge component builder with HTML/CSS editor and a live sandboxed preview of a weather card

+ +### Embeddable Web Widget + +Drop your assistant onto **any website** with a one-line script - a floating chat bubble locked to the origins you allow. End users see only the conversation; operational details (steps, tokens, cost, node names) stay private in the dashboard. + +

Forge embed screen with widget toggle, allowed origins, and a copy-paste launcher snippet

+ +### Deploy anywhere - one run API, MCP & channels + +Ship the same workflow through many surfaces without rewriting it: call it server-to-server over a **single run API** (`POST /run` handles new turns, streaming, and human-in-the-loop resumes), expose it as an **MCP server**, deploy it to **email**, or drop in the **web widget**. Per-request caller context (`X-Forge-Context`) lets tools act on behalf of your end users - with secrets never in the request body. + +

Forge Connect screen showing the run API: the Forge API base URL, the POST /run endpoint, and a copy-paste curl example

+ +> [!NOTE] +> **Connectors are not yet fully implemented.** The prebuilt, one-click **connector library / marketplace** (Google, Slack, Notion, GitHub, Salesforce, …) is on the [roadmap](docs/ROADMAP.md#planned--exploring) — it is **not shipped yet**. Today you integrate an external system by hand: create a **REST / GraphQL / MCP tool** and pair it with an **Auth Provider** (Bearer / API key / Basic / OAuth2 / CSRF-session). *(Note: the `connector` **role** — a least-privileged MCP-only user — is a separate, shipped feature and unrelated to the connector library.)* + +### Observability & Traces + +Every run is captured as a **collapsible span tree** - model calls, tools, chains, sub-agents, latency, tokens, and **cost**, nested by real parent/child so you can see exactly what happened and what it cost. Deep-agent dispatches appear as named `subagent · ` spans, and the Playground streams a **live agent-activity** timeline (which sub-agent/tool ran, as it happens). Pair it with **Evaluations** to catch regressions before you publish, and export traces to any **OpenTelemetry** collector (e.g. Langfuse). + +

Forge traces screen showing a run's span waterfall with per-step latency, tokens, and cost

+ +### Guardrails, budgets & governance + +Run it like production. A project-level **Guardrails & Egress** policy (PII redaction, blocked terms, and a network allow/deny list) applies to every agent by default; **budgets & quotas** cap spend and tokens; **versioning** snapshots every change; and it all sits behind per-project **roles / RBAC** and an audit log - from one Settings surface. + +

Forge project Settings with a section sidebar: General, Members and Roles, API Keys, Model Pricing, Budgets and Quotas, Guardrails and Egress, Versioning, and more

+ +### And many more + +- **Channels** - deploy a workflow to **Email**. +- **Triggers** - webhooks, schedules (interval/cron), inbound email, and polling "app events". +- **Human-in-the-loop** - approve/reject pauses and live **handoff** to an Agent inbox, with the reply delivered back over the same channel. +- **Auth Providers** - Bearer, API key, Basic, OAuth2 (client-credentials **and** 3-legged user login with auto-refresh), and CSRF/session - backed by encrypted, reference-only secrets (`secret://…`). +- **MCP, both ways** - expose your tools as an **MCP server** over native Streamable-HTTP/SSE (Claude Desktop, Cursor, and VS Code connect directly - no bridge), authenticated by a project key, per-user **personal access tokens**, or optional **OAuth 2.1**, publishing tool sets, knowledge, Q&A, or a whole workflow; and consume tools from external MCP servers. +- **Guardrails & egress policy** - one project-level I/O policy (PII redaction, blocked terms, and a network allow/deny list) enforced on every agent by default; a project can only *tighten* it, never loosen it. +- **Import & export** - move tools, workflows, agents, and components between projects as portable JSON bundles (secret *values* never leave). +- **Evaluations** - datasets scored by `contains` / `exact` / `regex` / LLM-`judge` for a pass rate per workflow, **streamed live** (each case resolves in the UI as it finishes, with per-case latency + tokens). +- **Long-term memory**, response caching, retries with backoff, and per-tenant **budgets**. +- **Multi-tenant projects & roles** (owner/admin/editor/viewer/connector) with **per-project RBAC**, scoped revocable API keys, entity **version history**, and an **audit log**. +- **Provider-agnostic models** - OpenAI, Anthropic, Google, or any LangChain provider, plus an offline `fake:` model so you can build the plumbing without spending a cent. + +> See the full **[User Manual](docs/MANUAL.md)** for an end-to-end tour and worked examples. + +## Architecture + +A pnpm + Python monorepo with a shared schema contract that keeps the backend and frontend in lockstep: + +``` +forge/ +├── apps/ +│ ├── api/ FastAPI backend - the engine (compiler, registry, middleware), tools, +│ │ auth, knowledge, tracing, MCP server, build assistant. Dockerfile + +│ │ Alembic migrations live here. [Python] +│ └── web/ Next.js console - canvas, config panels, playground, traces. Dockerfile +│ lives here. [TS/React] +├── packages/ +│ └── schemas/ Shared JSON Schemas - the single source of truth, imported by the +│ backend validator/compiler AND the frontend . +├── docs/ User manual, roadmap, and media. +├── infra/ Production database swaps (Postgres row-level-security policies). +└── docker-compose.yml Production-shaped stack (Postgres · Redis · api · worker · web). +``` + +The **shared schemas** are the contract behind three consumers: the backend **validator** (rejects bad configs on save), the **compiler** (`compile_workflow`, `build_middleware`), and the frontend **``** (forms generated from the same files). + +## Quick start (local, zero-infra) + +### Prerequisites + +- **Python** 3.11–3.13 - the backend engine +- **Node** 22 LTS and **pnpm** 9+ - the web console +- Nothing else - the local stack runs on SQLite + embedded Chroma + an in-process scheduler, so **no Docker, Postgres, or Redis** is required to start. + +### 1. Configure environment + +```bash +cp .env.example .env # macOS/Linux +copy .env.example .env # Windows +``` + +Open `.env` and fill in what you need (e.g. an LLM provider key). Everything is optional to boot; agents that call a model need a provider key. **Never commit your `.env`** - it is already git-ignored. + +### 2. Backend (FastAPI engine) + +```bash +cd apps/api +python -m venv .venv && source .venv/bin/activate # .venv\Scripts\activate on Windows +pip install -e ".[dev,all]" # engine + tests + vectors/providers/knowledge/MCP +pytest # optional: validate the engine (offline) +uvicorn forge.main:app --reload --port 8000 # http://localhost:8000/docs +``` + +### 3. Frontend (Next.js console) + +In a second terminal, from the repo root: + +```bash +pnpm install +pnpm --filter web dev # http://localhost:3000 +``` + +Open **http://localhost:3000** for the console and **http://localhost:8000/docs** for the API. On first run, sign in with `you@forge.local` / `forge-admin`, or create a fresh workspace. + +## Run with Docker (production-shaped) + +The included [`docker-compose.yml`](docker-compose.yml) brings up a production-shaped stack - **Postgres** (app DB + durable checkpointer), **Redis** (shared rate-limit/idempotency + worker queue), the **API**, a **worker**, and the **web** console: + +```bash +# Set real secrets first (FORGE_JWT_SECRET, FORGE_BOOTSTRAP_ADMIN_PASSWORD, provider keys) +docker compose up --build +``` + +With `FORGE_ENVIRONMENT=production`, Forge enables a hardening guard and **refuses to boot** with default secrets, SQLite, or a non-durable checkpointer. See [`apps/api/README.md`](apps/api/README.md) and **[Manual §13 - Going to production](docs/MANUAL.md)** for the full, annotated configuration. + +## Documentation + +| Doc | What's inside | +|---|---| +| **[User Manual](docs/MANUAL.md)** | Full feature tour, the node catalog, and end-to-end use cases (no developer knowledge needed). | +| **[Backend README](apps/api/README.md)** | API layout, local-vs-production swaps, and dependency notes. | +| **[Tech stack & architecture](TECH_STACK.md)** | Every dependency and why it's there, plus request/run sequence diagrams. | +| **[Roadmap & status](docs/ROADMAP.md)** | What's shipped, what's in progress, and what's planned next (connectors, more channels, and more). | +| **[Changelog](CHANGELOG.md)** | Notable changes, following Keep a Changelog + SemVer. | +| **[Contributing](CONTRIBUTING.md)** | Local setup, the checks CI runs, and commit/PR conventions. | + +## Tech stack + +| Layer | Technology | +|---|---| +| **Engine** | LangChain v1 · LangGraph v1 · Deep Agents - MIT framework only | +| **Backend** | Python · FastAPI · SQLAlchemy 2 (async) · Pydantic v2 | +| **Frontend** | Next.js 14 (App Router) · React 18 · TypeScript · React Flow | +| **Data (local)** | SQLite · embedded Chroma · in-process cache/scheduler | +| **Data (prod)** | Postgres 16 + pgvector · Redis 7 · Fernet/Vault secrets | +| **Observability** | Built-in tracer + cost accounting · OpenTelemetry / Langfuse export | + +## Contributing + +Contributions are welcome. Forge is MIT-licensed and built to be extended. + +1. Fork the repo and create a feature branch. +2. Backend changes: run `pytest` and `ruff check forge migrations` from `apps/api`. +3. Keep the **shared schemas** (`packages/schemas`) authoritative - the validator, compiler, and frontend forms all read from them. +4. Open a pull request describing the change and the reasoning. + +Found a bug or have an idea? Please [open an issue](https://github.com/nihalashetty/Forge/issues). + +## License + +Forge is released under the **[MIT License](LICENSE)** - free to use, modify, and distribute, including commercially. It builds only on the MIT-licensed LangChain/LangGraph ecosystem, with no Elastic-2.0 or commercial-license dependencies. + +
+Built on the open-source LangChain and LangGraph ecosystem. +
diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..a0717f9 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,57 @@ +# Security Policy + +Forge is self-hosted and handles sensitive material — provider API keys, encrypted +secrets, auth credentials, and outbound requests to your systems. We take security issues +seriously. + +## Supported versions + +Forge is pre-1.0 and under active development. Security fixes land on the `main` branch and +are included in the next release. Please test against the latest `main` before reporting. + +| Version | Supported | +|---|---| +| `main` (latest) | ✅ | +| older tags | ❌ (please upgrade) | + +## Reporting a vulnerability + +**Please do not open a public issue for security vulnerabilities.** + +Report privately via **[GitHub Security Advisories](https://github.com/nihalashetty/Forge/security/advisories/new)** +(Repository → *Security* → *Report a vulnerability*). This keeps the report confidential +until a fix is available. + +When reporting, include: + +- A description of the issue and its impact. +- Steps to reproduce (a minimal proof-of-concept helps). +- Affected component/version (commit SHA if possible). + +**Do not include real secrets, tokens, or customer data** in your report — redact or use +placeholders. + +We aim to acknowledge reports promptly, keep you updated on remediation, and credit +reporters (unless you prefer to remain anonymous) once a fix ships. + +## Built-in safeguards + +Forge ships several defense-in-depth controls; when reporting, note if your finding +bypasses one: + +- **SSRF guard** on every outbound call (tools, webhooks, fetch, crawl) — private, + loopback, and cloud-metadata addresses are blocked by default. +- **Encrypted, reference-only secrets** (`secret://…`) via a Fernet master key; secret + values are write-only and never returned. +- **Production hardening guard** that refuses to boot with default secrets, auth disabled, + a non-durable checkpointer, or the SSRF guard off. +- **Multi-tenant isolation** with query-level scoping and optional Postgres row-level + security. +- **Sandboxed code tools** (RestrictedPython) and origin-locked embed widgets. + +## Scope + +In scope: the Forge API, web console, engine, tools/auth subsystems, and the container +stack. Out of scope: vulnerabilities in third-party dependencies (report those upstream; +tell us if Forge's usage makes them exploitable), and issues requiring a +already-compromised host or admin credentials. diff --git a/TECH_STACK.md b/TECH_STACK.md new file mode 100644 index 0000000..d4e0705 --- /dev/null +++ b/TECH_STACK.md @@ -0,0 +1,211 @@ +# Forge — Technology Stack + +Every technology used in Forge, its purpose, and where it lives. Sourced from `apps/web/package.json`, `apps/api/pyproject.toml`, both `Dockerfile`s, `docker-compose.yml`, and `apps/api/forge/config.py`. Backend deps grouped `[in brackets]` are **optional extras** (installed on demand / prod); everything else is core. + +| Layer | Technology | Use case | Where it lies | +|---|---|---|---| +| **Frontend** | Next.js 14.2 | React framework; standalone server + build-time API proxy rewrites to the backend | `apps/web` (web console); `next.config.mjs` | +| **Frontend** | React 18.3 + React DOM | UI component rendering | `apps/web` | +| **Frontend** | TypeScript 5.6 | Typed frontend language | `apps/web` (`tsconfig.json`) | +| **Frontend** | @xyflow/react (React Flow) 12 | Visual drag-and-drop node-graph editor — the workflow builder canvas | `apps/web` (builder/canvas components) | +| **Frontend** | react-markdown 9 + remark-gfm 4 | Render agent/chat responses as GitHub-flavored Markdown | `apps/web` (chat UI) | +| **Frontend** | mustache 4 | Client-side `{{...}}` template rendering | `apps/web` | +| **Frontend** | jmespath 0.16 | JSON projection/query in the browser | `apps/web` | +| **Frontend / Build** | Node.js 22 | JS runtime for building and serving the console | `apps/web/Dockerfile` (`node:22-alpine`) | +| **Build / Monorepo** | pnpm (workspace) | Package manager + monorepo workspaces (`apps/web`, `packages/*`) | repo root (`pnpm-workspace.yaml`, `corepack`) | +| **Backend / API** | Python 3.11–3.13 | Backend language (runtime image: `python:3.12-slim`) | `apps/api` | +| **Backend / API** | FastAPI 0.115 | HTTP/REST API framework, routing, dependency injection, middleware | `apps/api/forge/main.py`, `forge/routers/*` | +| **Backend / API** | Uvicorn[standard] 0.32 | ASGI server that runs the app | serve command (`uvicorn forge.main:app`) | +| **Backend / API** | sse-starlette 2.1 | Server-Sent Events streaming of run event frames (`run`/`node_start`/`messages`/`done`) | `forge/routers/runs.py` | +| **Backend / API** | python-multipart | Multipart form / file upload parsing | `apps/api` | +| **Backend / API** | hatchling | Python package build backend | `apps/api/pyproject.toml` | +| **Config / Data** | Pydantic 2.9 | Request/response DTOs, data validation | `forge/schemas/dto.py`, models | +| **Config / Data** | pydantic-settings 2.6 | Environment-driven application settings | `forge/config.py` | +| **Config / Data** | email-validator 2.2 | Validate email fields (invites, auth) | `apps/api` | +| **Agent / Engine** | LangChain 1.3 + langchain-core | LLM orchestration primitives (messages, tools, model bindings) | `forge/engine/*` | +| **Agent / Engine** | LangGraph 1.2 | Stateful agent/workflow graph engine — compiles nodes into a runnable graph; the execution core | `forge/engine/compiler.py`, `forge/services/runs.py` | +| **Agent / Engine** | langgraph-checkpoint 4 | Checkpointer interface for durable / resumable run + HITL state | `forge/engine`, `forge/services/runs.py` | +| **Agent / Engine** | langgraph-checkpoint-sqlite 3 | SQLite-backed checkpointer (dev default) | `.data/checkpoints.sqlite` | +| **Agent / Engine** | deepagents 0.6 | Deep Agents harness (planning, subagents, virtual filesystem, sandbox) — always-registered `deep_agent` node | agent node palette | +| **Model Providers** | langchain-openai 1.x | OpenAI model access | `[providers]` extra | +| **Model Providers** | langchain-anthropic 1.x | Anthropic Claude models + prompt-caching middleware | `[providers]` extra; `default_anthropic_prompt_caching` | +| **Model Providers** | langchain-google-genai 4.2+ | Google Gemini models | `[providers]` extra | +| **Model Providers** | tiktoken 0.7 | Accurate token counting for the cost meter / budgets (falls back to len/4) | `forge/tracing/pricing.py` | +| **Tooling Primitives** | httpx 0.27 | Outbound HTTP for REST/GraphQL tools, webhooks, `web_fetch`, OAuth/token fetches | tool runtime + egress/SSRF guard | +| **Tooling Primitives** | jsonschema 4.23 | Validate node/tool config against the shared JSON Schemas | `packages/schemas` + engine | +| **Tooling Primitives** | jmespath 1.0 (py) | JSON projection of tool outputs | tool runtime | +| **Tooling Primitives** | RestrictedPython 7.4 | AST-sandboxed execution of code tools (opt-in; hardening layer, not OS isolation) | code tool runtime (`enable_code_tools`) | +| **MCP** | langchain-mcp-adapters 0.2 | Consume external MCP servers as tools | `[mcp]` extra | +| **MCP** | mcp 1.9 | Model Context Protocol SDK | `[mcp]` extra | +| **MCP** | fastmcp 3 | Expose Forge projects as MCP servers | `[mcp]` extra | +| **Knowledge / RAG** | chromadb 1.5 | Embedded persistent vector store (zero infra) | `[vectors]` extra; `.data/chroma` | +| **Knowledge / RAG** | fastembed 0.3+ | **Default embedder** — local open-source ONNX model (no API cost / no PyTorch); also powers the local cross-encoder **reranker** (`TextCrossEncoder`) | `[knowledge]` extra | +| **Knowledge / RAG** | langchain-text-splitters 1.x | Chunk/split documents for ingestion | `[knowledge]` extra | +| **Knowledge / RAG** | pypdf 5 | Extract text from PDF documents | `[knowledge]` extra | +| **Knowledge / RAG** | beautifulsoup4 4.12 | Parse HTML for URL ingestion | `[knowledge]` extra | +| **Knowledge / RAG** | rank-bm25 0.2 | Lexical (BM25) ranking for hybrid vector + keyword search | `[knowledge]` extra | +| **Persistence** | SQLAlchemy 2.0 [asyncio] | Async ORM / database access layer | `forge/models/entities.py` | +| **Persistence** | aiosqlite 0.20 | Async SQLite driver (dev default) | dev DB `.data/forge.db` | +| **Persistence** | Alembic 1.14 | Schema migrations (controlled prod path) | `apps/api/migrations`, `alembic.ini` | +| **Persistence** | greenlet 3.1 | Async/sync bridge required by SQLAlchemy asyncio | runtime dependency | +| **Persistence** | SQLite | Default dev database + checkpointer store | `.data/*.db` (dev only) | +| **Persistence** | PostgreSQL 16 | Production application database + shared durable checkpointer | `docker-compose.yml` (prod) | +| **Persistence** | asyncpg 0.30 / psycopg[binary,pool] 3.2 | Async Postgres drivers | `[postgres]` extra (prod) | +| **Persistence** | langgraph-checkpoint-postgres 2.x | Durable Postgres checkpointer shared across workers (prod/HITL) | `[postgres]` extra (prod) | +| **Secrets / Auth** | cryptography 43 (Fernet) | Encrypt stored secrets/credentials with a master key | `.data/master.key`; secrets service | +| **Secrets / Auth** | python-jose[cryptography] 3.3 | Mint/verify platform JWT access + refresh tokens (with `kid` rotation) | auth layer (`forge/config.py` JWT settings) | +| **Secrets / Auth** | bcrypt 4 | Password hashing for local accounts | auth layer | +| **Background / Workers** | Redis 7 | Shared rate-limit / idempotency store + worker queue backend | `docker-compose.yml` (prod); `[workers]` extra | +| **Background / Workers** | arq 0.26 | Async task queue + worker for offloaded run execution | `forge/worker.py`, `forge/queue.py`; `[workers]` | +| **Background / Workers** | croniter 2–6 | Evaluate cron `schedule` triggers | scheduler; `[workers]` | +| **Observability** | opentelemetry-sdk 1.20 | Emit run traces/spans (GenAI semantic conventions) | `forge/tracing/otel.py`; `[observability]` | +| **Observability** | opentelemetry-exporter-otlp-proto-http 1.20 | Export spans to an OTLP collector / Langfuse | `forge/tracing/otel.py`; `[observability]` (opt-in via `otel_enabled`) | +| **Infra / Deploy** | Docker + Docker Compose | Production-shaped container stack (postgres + redis + api + worker + web) | repo root (`docker-compose.yml`, `apps/*/Dockerfile`) | +| **Dev Tooling** | pytest 8.3 + pytest-asyncio 0.24 | Backend test suite (async mode auto) | `apps/api/tests`; `[dev]` extra | +| **Dev Tooling** | anyio 4.6 | Async test/runtime utilities | `[dev]` extra | +| **Dev Tooling** | ruff 0.15 (pinned `>=0.15,<0.16`) | Linting + import sorting/formatting; range-pinned so CI lint is reproducible | `pyproject.toml [tool.ruff]`; `[dev]` extra | +| **Dev Tooling** | mypy 1.13+ | Static type-checking (advisory in CI; gradual adoption) | `pyproject.toml [tool.mypy]`; `[dev]` extra | +| **Dev Tooling** | Vitest 2 + Testing Library | Frontend unit/component tests (`pnpm --filter web test`) | `apps/web` (`devDependencies`) | + +**Notes** +- **Local dev needs no external infra**: SQLite + embedded Chroma + in-process (fake) cache/queue. The prod swaps — Postgres, Redis, OTLP, Vault/KMS — are configuration-only (no code changes). +- **Framework is MIT-only**: LangChain/LangGraph OSS packages; deliberately **not** `langgraph-api` or LangSmith. +- **Optional extras** map to `pip install -e ".[...]"` groups in `pyproject.toml`: `providers`, `vectors`, `knowledge`, `mcp`, `workers`, `postgres`, `observability`, `dev` (and `all` = vectors+providers+knowledge+mcp). + +--- + +## Architecture + +How the pieces above fit together. The browser talks to a **same-origin** Next.js proxy (`/api/forge/*`) that rewrites to the FastAPI backend; every run — whatever triggers it — funnels through one `RunService`, and one `ForgeTracer` observes it. + +```mermaid +flowchart TB + subgraph Client["Client tier"] + Browser["Browser — Next.js console
(builder canvas · chat UI)"] + Embed["Embed widget
(anonymous · publishable key)"] + end + + subgraph Edge["Edge / web server"] + Next["Next.js 14 server
same-origin proxy: /api/forge/* to API"] + end + + subgraph Triggers["Trigger sources"] + UI["Interactive run (SSE)"] + WH["Webhook"] + SCH["Schedule (cron)"] + CH["Email channel"] + MCPin["MCP server surface"] + end + + subgraph API["FastAPI backend — apps/api/forge"] + MW["Middleware
Audit · TrustedHost · CORS · rate-limit
auth: JWT / service token · X-Forge-Context"] + Routers["Routers (/v1/...)"] + Dispatch["Dispatch service"] + RunSvc["RunService
create_run · stream · run_to_completion · resume"] + Compile["Engine: build_compile_context to compile_workflow"] + Graph["LangGraph StateGraph — graph.astream()"] + Tracer["ForgeTracer callback
spans: tokens · cost · latency"] + end + + subgraph Nodes["Graph nodes & tools"] + Models["Model providers
OpenAI · Anthropic · Google GenAI"] + Tools["Tools: httpx REST/GraphQL · MCP client
code (RestrictedPython) · knowledge/RAG"] + Deep["deep_agent (Deep Agents)"] + end + + subgraph DataTier["Persistence & state"] + DB["SQLAlchemy to SQLite (dev) / Postgres (prod)
Run · Trace · Span · Thread · AuditLog"] + CP["LangGraph checkpointer
SQLite / Postgres"] + Vec["Chroma vector store (.data/chroma)"] + Secrets["Fernet master.key (.data)"] + end + + subgraph Async["Async / prod tier"] + Redis["Redis — rate-limit · idempotency · queue"] + Worker["arq worker (offloaded runs)"] + OTel["OpenTelemetry to OTLP / Langfuse"] + end + + Browser --> Next + Embed --> Next + Next --> MW + UI --> MW + WH --> Dispatch + SCH --> Dispatch + CH --> Dispatch + MCPin --> Dispatch + MW --> Routers --> RunSvc + Dispatch --> RunSvc + RunSvc --> Compile --> Graph + Graph -. callbacks .-> Tracer + Graph --> Models + Graph --> Tools + Graph --> Deep + Tools --> Vec + Tools -. encrypt/decrypt .-> Secrets + RunSvc --> DB + Graph --> CP + Tracer --> DB + Tracer --> OTel + RunSvc <--> Redis + Dispatch --> Worker + Worker --> RunSvc + Routers -. SSE frames .-> Next + Next -. SSE .-> Browser +``` + +--- + +## Example flow — a user sends a chat message + +The interactive path (console or embed widget). Sending a message is **two HTTP calls**: a `POST` that creates the run row, then a `GET` that opens the SSE stream carrying tokens and lifecycle events back to the browser. + +```mermaid +sequenceDiagram + autonumber + actor U as User + participant B as Browser (chat UI) + participant N as Next.js proxy + participant M as FastAPI middleware + participant R as Runs router + participant S as RunService + participant G as LangGraph astream + participant T as ForgeTracer + participant P as Model provider + participant D as DB / checkpointer + + U->>B: type message, hit send + B->>N: POST /v1/projects/{p}/workflows/{w}/runs + N->>M: same-origin proxy to api:8000 + M->>R: authenticated request (audit · auth · rate-limit) + R->>R: idempotency · run rate limit · identity · daily quota + R->>S: create_run(input, thread_id) + S->>D: INSERT Run (queued) + Thread + S-->>R: run + R-->>B: 201 { run_id, thread_id } + + B->>N: GET /v1/.../runs/{run_id}/stream (EventSource / SSE) + N->>R: proxy stream request + R->>S: stream(run_id) + S->>D: Run to running + S->>G: build context to compile_workflow, astream(callbacks=[Tracer]) + activate G + loop each node / token + G->>T: on_llm_start / on_tool_start (open span) + G->>P: LLM call + P-->>G: tokens + G-->>B: SSE: node_start · messages (tokens) · custom + G->>T: on_llm_end (tokens · cost · latency) + end + deactivate G + alt HITL interrupt + S-->>B: SSE: interrupt (awaiting human input) + Note over B,S: user approves to resume the same run + else completed + S->>D: _write_trace to Trace + Span rows (+ OTel export) + S-->>B: SSE: done { answer, total_tokens, total_cost_usd } + end + B->>U: render streamed answer +``` + +> **Non-interactive triggers** (webhook / schedule / email) skip the browser and the SSE stream: they enter through the **Dispatch service** and call `RunService.run_to_completion()` instead of `stream()` — but the compile → LangGraph → ForgeTracer → Trace/Span path is identical, which is what keeps observability consistent across every entry point. diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile new file mode 100644 index 0000000..9bfa953 --- /dev/null +++ b/apps/api/Dockerfile @@ -0,0 +1,71 @@ +# Forge API - FastAPI on the MIT LangChain/LangGraph stack. +FROM python:3.12-slim AS base + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + # The default local (fastembed) embedder's model is baked into the image at this path + # (see the pre-download step below); the app reads the same dir at runtime, so ingestion + # works fully offline with no first-run download. Outside the /app/.data volume, so it's + # part of the image, not the persisted volume. + FORGE_FASTEMBED_CACHE_DIR=/app/.fastembed-cache + +# Build deps for psycopg/cryptography wheels (kept minimal; most ship manylinux wheels). +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Build context is the REPO ROOT (see docker-compose.yml: context: . / dockerfile: +# apps/api/Dockerfile) so we can COPY packages/schemas, which lives above apps/api. +# +# Third-party deps install in a layer keyed only on pyproject.toml + a package stub, so a +# pure forge/ source edit does NOT invalidate this (expensive) layer. The editable install +# just links /app/forge onto sys.path, so overwriting the stub with the real tree afterwards +# needs no reinstall. The BuildKit pip cache mount reuses already-downloaded wheels instead +# of re-hitting PyPI even when the layer does re-run (e.g. a pyproject.toml change), so a +# source edit can never turn into a PyPI round-trip / offline build failure. +COPY apps/api/pyproject.toml apps/api/README.md ./ +RUN mkdir -p forge && touch forge/__init__.py +# Production extras: model providers, vectors, knowledge, MCP, workers (Redis/arq), Postgres. +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --upgrade pip && \ + pip install -e ".[providers,vectors,knowledge,mcp,workers,postgres]" + +# Real source over the stub (this layer changes on every edit, but it is AFTER the install). +COPY apps/api/forge ./forge +COPY apps/api/alembic.ini ./alembic.ini +COPY apps/api/migrations ./migrations +# Shared JSON Schemas live at the repo root, outside the old ./apps/api context. Bake them at +# /app/packages/schemas so config.py's schemas_dir default (/app/packages/schemas) resolves. +COPY packages/schemas ./packages/schemas + +# Pre-download the default local embedder (fastembed) model into the image so the container +# ships self-contained: no ~130MB HuggingFace fetch on the first ingest, and no runtime egress +# to huggingface.co (which a locked-down deploy may block). Keep this model id in sync with +# forge.knowledge.embeddings._DEFAULT_FASTEMBED. Runs before the chown so the forge user owns it. +RUN python -c "import os; from fastembed import TextEmbedding; TextEmbedding(model_name='BAAI/bge-small-en-v1.5', cache_dir=os.environ['FORGE_FASTEMBED_CACHE_DIR'])" + +# Run as a non-root user. Pre-create /app/.data (holds the Fernet master.key + Chroma store) +# owned by forge: an empty named volume mounted there copies the image mountpoint's ownership on +# first use, so this keeps the volume forge-writable. Without it the volume would mount root-owned +# and the non-root process could not write the master key (PermissionError at startup). +RUN useradd --create-home --uid 10001 forge \ + && mkdir -p /app/.data \ + && chown -R forge:forge /app +USER forge + +EXPOSE 8000 + +# Container healthcheck hits the readiness probe (DB + checkpointer). +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD curl -fsS http://localhost:8000/readyz || exit 1 + +# Apply migrations, then serve. (Schema also self-bootstraps via create_all, but on managed +# Postgres `alembic upgrade head` is the controlled path.) +# No uvicorn --proxy-headers/--forwarded-allow-ips: the app derives the client IP itself and +# trusts X-Forwarded-For only from FORGE_TRUSTED_PROXIES (default none -> the real socket peer). +# Letting uvicorn rewrite the peer from XFF for ANY client (as --forwarded-allow-ips='*' did) +# lets clients spoof their IP to evade per-IP rate limits / poison audit logs. Behind a real +# proxy, set FORGE_TRUSTED_PROXIES to the proxy IP(s). +CMD ["sh", "-c", "alembic upgrade head && uvicorn forge.main:app --host 0.0.0.0 --port 8000"] diff --git a/apps/api/README.md b/apps/api/README.md new file mode 100644 index 0000000..e9a199c --- /dev/null +++ b/apps/api/README.md @@ -0,0 +1,61 @@ +# Forge API + +FastAPI backend for **Forge** - the self-hosted agent platform. Built directly on the +MIT-licensed LangChain v1 + LangGraph v1 framework. **Never** depends on `langgraph-api` +(Elastic 2.0) or LangSmith (commercial). + +## Local dev (zero external infra) + +The default local stack needs **no Docker, Postgres, or Redis**: + +| Concern | Local default | Production swap (config-only) | +|---|---|---| +| Relational DB | SQLite (`aiosqlite`) | Postgres 16 (`asyncpg` / `psycopg`) | +| Run durability | `langgraph-checkpoint-sqlite` | `langgraph-checkpoint-postgres` | +| Vectors | Chroma (embedded, persistent) | `pgvector` in the same Postgres - set `vector_backend=pgvector` | +| Cache / queue | in-process | Redis 7 + arq | +| Secrets | Fernet (local key file) | Vault / cloud KMS | + +```bash +cd apps/api +python -m venv .venv +.venv\Scripts\activate # Windows (source .venv/bin/activate on *nix) +pip install -e ".[dev]" # core + test deps only +pip install -e ".[dev,all]" # full local stack (vectors + providers + knowledge + MCP) + +cp ../../.env.example .env +uvicorn forge.main:app --reload --port 8000 +pytest # validate the engine +``` + +## Layout + +``` +apps/api/ + forge/ + main.py FastAPI app factory + lifespan (DB init, checkpointer, scheduler/reaper) + config.py Settings (pydantic-settings, env-driven) + production hardening guard + deps.py FastAPI dependencies: session, auth/tenant resolution, RBAC + security.py Auth primitives: bcrypt password hashing + JWT mint/verify + audit_middleware.py ASGI middleware that audits successful mutations + queue.py, worker.py Optional arq/Redis queue + worker for offloaded runs (prod) + db/ async engine, session, tenant scoping, dev seed/bootstrap + models/ SQLAlchemy ORM (tenants, projects, workflows, runs, traces, ...) + schemas/ Pydantic request/response DTOs + shared JSON-Schema loader/validator + services/ business logic (ProjectSvc, WorkflowSvc, RunSvc, assistant, + portability import/export, tool_sets, ...) + routers/ HTTP + SSE endpoints (incl. assistant, runs, mcp_server, mcp_oauth, + mcp_tokens, tool_sets, connections, models, versions, embed) + engine/ the heart: registry, compiler, state, middleware_compiler, context + nodes/ node-type factories (start, end, agent, llm, tool_call, flow, rag, triggers) + tools/ tool materialization (rest, graphql, code, sql, mcp, builtin) + projection + auth_providers/ Auth Provider resolver (csrf_session, oauth2, bearer, ...) + secrets/ Fernet-encrypted, reference-only secret store + channels/ email deployment surface + knowledge/ EmbeddingStore (Chroma), ingestion/crawl, splitter, hybrid + rerank + tracing/ ForgeTracer callback + span sink + pricing + tool-I/O capture + util/ cross-cutting helpers (SSRF guard, http client, rate limit, mailer, ...) + assistant_skills/ skill(s) the in-product build assistant loads + migrations/ Alembic migrations (prod schema path; SQLite auto-creates in dev) + tests/ pytest suite (engine, tools, knowledge, auth, security, ...) +``` diff --git a/apps/api/alembic.ini b/apps/api/alembic.ini new file mode 100644 index 0000000..0474c53 --- /dev/null +++ b/apps/api/alembic.ini @@ -0,0 +1,40 @@ +# Alembic config for Forge. The DB URL comes from FORGE_DATABASE_URL via env.py +# (this file's sqlalchemy.url is a placeholder and is overridden there). +[alembic] +script_location = migrations +prepend_sys_path = . +sqlalchemy.url = sqlite:///./.data/forge.db + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/apps/api/forge/__init__.py b/apps/api/forge/__init__.py new file mode 100644 index 0000000..5cada58 --- /dev/null +++ b/apps/api/forge/__init__.py @@ -0,0 +1,3 @@ +"""Forge API - self-hosted LangChain/LangGraph agent platform.""" + +__version__ = "0.1.0" diff --git a/apps/api/forge/assistant_skills/forge-platform/SKILL.md b/apps/api/forge/assistant_skills/forge-platform/SKILL.md new file mode 100644 index 0000000..bcf710a --- /dev/null +++ b/apps/api/forge/assistant_skills/forge-platform/SKILL.md @@ -0,0 +1,139 @@ +--- +name: forge-platform +description: Use when designing, building, debugging, or explaining Forge workflows, nodes, state, routing, middleware, tools, or knowledge - especially custom/complex shapes beyond the canned builders, AND for anything about deep agents, sub-agents, or supervisor patterns (how sub-agents are wired on the canvas), or per-environment tool endpoints ({{env.*}}). +--- + +# Forge platform deep guide + +You are embedded in Forge, a visual builder for LangChain/LangGraph agents. Workflows are +JSON definitions compiled to LangGraph StateGraphs. This guide covers the rules the canned +builder tools don't teach. For the live catalog always call `list_node_types`, +`get_node_schema(type)`, and `list_middleware_types` - they read the real registry. + +## Workflow definition shape + +```json +{ + "state": {"messages": {"type": "list[message]", "reducer": "add_messages"}, + "intent": {"type": "str", "reducer": "last"}}, + "entry_node": "start", + "nodes": [{"id": "start", "type": "start", "config": {}, "position": {"x": 40, "y": 200}}], + "edges": [{"source": "start", "target": "..."}] +} +``` + +Rules: +- Exactly one `start` node; at least one `end` node; every node reachable; a path must reach end. +- Every state key a node WRITES must be declared in `state` (LangGraph rejects undeclared + writes). `create_custom_workflow` auto-declares keys for known node configs, but declare + custom `output_key`s yourself. Types: str, int, float, bool, json, list[str], list[json], + list[message]. Reducers: last (overwrite), add (append lists), merge (dict merge), + add_messages (chat history). +- Messages flow on the `messages` channel; nodes append, never replace. + +## Routing patterns + +- Triage first (almost every support/chatbot graph): right after `start`, classify the + message into at least `general` vs `support`, then `router` it. Send `general` (greetings, + smalltalk, "what can you do?", capability/meta questions) to a small friendly agent that + answers directly and goes to `end`; route only `support` into the retrieval/ticket + pipeline. Without this, greetings and meta questions fall through retrieval, miss, + and dead-end at a "no relevant data → create a ticket" path - a bad first impression. + Shape: start → classify(general|support) → router → {general: greeter_agent → end, + support: retrieval → … → end}. (Simpler alt: one front agent with a knowledge_search tool + that both chats and answers.) +- Single intent: `classifier` (labels, output_key=intent) → `router` + (expression=intent, cases {label: node_id}, default=fallback_node). Case KEYS are the + exact VALUES the expression takes, not display labels. +- ALWAYS give routers a `default` - with no default, an unmatched value silently ends the + run with no answer. +- MULTI-INTENT (a question with several asks): classifier `multi_label: true` writes a + LIST to state (declare it `list[str]`); router `multi: true` routes to EVERY matching + case in parallel. All branches then converge on ONE synthesizer agent node before end - + its prompt: "compose the partial answers above into one coherent reply". Without a + synthesizer the user sees only the last branch's answer. +- Simpler multi-intent alternative (preferred for support bots): ONE agent with + `config.knowledge` enabled (rag and/or qa) plus any REST tools. The agent searches the KB + once per sub-question and composes one answer itself. Fewer nodes, no fan-out needed. +- Conditional on retrieval success: retrieval `route_key` writes "yes"/"no"; + human decisions: human_input `output_key` writes the decision string. + +## Knowledge + +- Sources (documents) live in folders (free-form names; "" = unfiled). retrieval node + `folders: ["Manuals"]` and the `knowledge_search` tool's `folder` arg filter by folder. +- Q&A pairs have a free-form `kind` (faq, error_workaround, or custom kinds the user + creates) + tags. The retrieval node (include_qa) and agent Q&A filter by `kinds`; empty = all. +- Three ways to ground an agent, pick by how much control you need: + 1. `retrieval` NODE = fixed pre-step grounding (one search over BOTH docs + Q&A per run, + before the agent; structurally guaranteed). Use when grounding MUST happen. + 2. Agent `config.knowledge` (PREFERRED for conversational/multi-part agents) = built-in, + agent-driven KB access, no separate Tool needed: + ```json + "knowledge": { + "rag": {"enabled": true, "folders": ["Manuals"], "top_k": 4}, + "qa": {"enabled": true, "kinds": ["faq"]} + } + ``` + Compiles to `search_knowledge_base` (documents) and/or `lookup_faq` (curated Q&A), + each toggled and scoped (folders / kinds) independently. The agent searches per + sub-question in its own phrasing - so ONE agent answers multi-part questions. + 3. `knowledge_search` builtin TOOL = same idea but as a standalone Tool row (use when you + want to share one tool across agents, or filter folder per-call). For a single agent, + `config.knowledge` is simpler. + +## Human-in-the-loop (real interrupts only) + +- `human_input` node pauses the run (LangGraph interrupt) until a human decides in the + Playground. `output_key` exposes the decision to a router. +- HumanInTheLoopMiddleware (`approve_tools` on builders / `human_in_the_loop` middleware) + pauses before specific TOOL calls. +- NEVER simulate approval via prompt text. Verify with test_workflow that nodes_visited + ends in `__interrupt__`. + +## Middleware (agent nodes) + +Per-agent `middleware: [{type, config, enabled}]`. Useful types: summarization, +model_fallback, model_retry, tool_retry (retry_on: timeout/connection/http_error/...), +pii, guardrail_regex (block actually replaces the reply), model_call_limit, +tool_call_limit, tenant_budget, llm_tool_selector, context_editing, tool_emulator, +dynamic_model_by_state, tool_filter_by_context, human_in_the_loop. Call +`list_middleware_types` for configs. + +## Deep agents & sub-agents (supervisor pattern) + +- A `deep_agent` node is a supervisor: `create_agent` + only the deepagents middleware its config + toggles on. `planning: true` adds a write_todos planner; `filesystem.enabled` adds a virtual FS; + `skills` adds skill files. All are OFF by default (each is pure token overhead until needed) - a + plain `agent` is cheaper for a simple lookup; reach for `deep_agent` for open-ended, multi-step + work you want to DELEGATE. +- **Sub-agents are wired on the canvas**, not inline JSON: an edge with `source_handle: "subagents"` + from the deep_agent to a specialist `agent` (or `deep_agent`) node folds that node in as a callable + sub-agent. The compiler lifts each wired node's config into the supervisor's `subagents` + (name/description/system_prompt/tools/model) and drops it as a standalone graph node. The + supervisor calls them via the `task` tool; results come back as named `subagent · ` spans. +- Give every specialist a clear **`config.description`** - the supervisor reads it (like a tool + description) to decide when to dispatch it. Each sub-agent keeps its OWN model + tools; if unset it + inherits the supervisor's model. +- Folded sub-agent nodes legitimately have NO outgoing flow edge (they run inside the supervisor, not + as graph steps) - validation exempts them from the dead-end warning. The supervisor still needs its + own `→ end` edge. Sub-agents can't see the chat history, so the supervisor must pass any needed + identifiers in the `task` description. +- Prefer a canvas supervisor over one mega-prompt when the work splits into distinct specialties + (e.g. lookup / create / update / validate); it keeps each sub-agent's tools + prompt focused. + +## Per-environment tool endpoints + +- A REST `url_template` / GraphQL `endpoint` (or an auth template) may reference `{{env.}}`, + resolved from the deployment's `FORGE_TOOL_VARS` map - so one tool row targets dev/qa/prod hosts + per deploy. A referenced key that isn't configured FAILS the call loudly (never a broken URL). + `{{ctx.*}}` (per-run injected values) stays lenient. Use `{{env.*}}` for the base host, `{{ctx.*}}` + for per-user/per-run values (tokens, ids). + +## Build discipline + +1. write_todos the plan. 2. list_resources (reuse, never duplicate names). +3. Build (canned builder if it fits, else create_custom_workflow). 4. test_workflow with +a realistic question, a greeting, an off-topic question - and every branch/intent. +5. evaluate_build to judge the results against the user's actual request. 6. Fix and +re-test until the judge passes. Only then report success. diff --git a/apps/api/forge/audit_middleware.py b/apps/api/forge/audit_middleware.py new file mode 100644 index 0000000..4f71141 --- /dev/null +++ b/apps/api/forge/audit_middleware.py @@ -0,0 +1,82 @@ +"""Centralized audit middleware. + +Records every successful mutating request (POST/PUT/PATCH/DELETE) as an AuditLog row - +so create/update/delete of any resource is audited without each router opting in. Pure +ASGI (peeks at the response-start status only) so it never buffers a body and can't break +the SSE run/assistant streams. The actor is taken from the JWT when present, else the +seeded dev tenant; auth endpoints are skipped (already audited in their router). +""" + +from __future__ import annotations + +from forge.config import settings +from forge.security import TokenError, decode_token +from forge.services.audit import AuditService +from forge.util.clientip import resolve_client_ip + +_SKIP_PREFIXES = ("/v1/auth", "/v1/audit") +_MUTATING = {"POST", "PUT", "PATCH", "DELETE"} + + +def _actor_from_headers(headers: dict[bytes, bytes]) -> tuple[str | None, str | None]: + auth = headers.get(b"authorization", b"").decode("latin-1") + if auth[:7].lower() == "bearer ": + try: + claims = decode_token(auth[7:].strip(), expected_type="access") + return claims.get("sub"), claims.get("tid") + except TokenError: + return None, None + return None, None + + +def _client_ip(scope, headers: dict[bytes, bytes]) -> str | None: + # Believe X-Forwarded-For only from a configured reverse proxy (settings.trusted_proxies) - + # the SAME rule as deps.client_ip. Previously this trusted XFF unconditionally, so any + # client could poison the audit IP. + client = scope.get("client") + peer = client[0] if client else None + fwd = headers.get(b"x-forwarded-for") + return resolve_client_ip(peer, fwd.decode("latin-1") if fwd else None, settings.trusted_proxies) + + +class AuditMiddleware: + def __init__(self, app) -> None: + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] != "http" or scope.get("method") not in _MUTATING: + return await self.app(scope, receive, send) + path = scope.get("path", "") + if any(path.startswith(p) for p in _SKIP_PREFIXES): + return await self.app(scope, receive, send) + + status_code = {"v": 0} + + async def send_wrapper(message): + if message["type"] == "http.response.start": + status_code["v"] = message["status"] + await send(message) + + await self.app(scope, receive, send_wrapper) + + if not (200 <= status_code["v"] < 400): + return + headers = dict(scope.get("headers") or []) + actor_id, tenant_id = _actor_from_headers(headers) + if tenant_id is None: + app = scope.get("app") + tenant_id = getattr(getattr(app, "state", None), "tenant_id", None) + if not tenant_id: + return + # Record the matched route TEMPLATE (e.g. "/v1/projects/{project_id}/tools/{tool_id}/test") + # rather than the concrete path. The concrete path carries UUIDs that overflow the action + # column (String(80)) and make actions un-aggregatable. FastAPI sets scope["route"] during + # routing, which ran inside self.app (already returned); fall back to the concrete path when + # nothing matched (404) or a non-APIRoute (bare Mount) handled it. The concrete path is kept + # in meta so the specific resource is still recoverable for forensics. + route_template = getattr(scope.get("route"), "path", None) or path + await AuditService.log( + tenant_id=tenant_id, action=f"{scope['method']} {route_template}", actor_id=actor_id, + ip=_client_ip(scope, headers), status="ok", + meta={"status_code": status_code["v"], "path": path}, + ) diff --git a/apps/api/forge/auth_providers/__init__.py b/apps/api/forge/auth_providers/__init__.py new file mode 100644 index 0000000..9b4649e --- /dev/null +++ b/apps/api/forge/auth_providers/__init__.py @@ -0,0 +1,5 @@ +"""Auth Providers: fetch/extract/inject credentials for downstream tool calls.""" + +from forge.auth_providers.resolver import AuthResolver, ResolvedAuth + +__all__ = ["AuthResolver", "ResolvedAuth"] diff --git a/apps/api/forge/auth_providers/extract.py b/apps/api/forge/auth_providers/extract.py new file mode 100644 index 0000000..bf7f3ef --- /dev/null +++ b/apps/api/forge/auth_providers/extract.py @@ -0,0 +1,34 @@ +"""Extract values from a token-fetch response (header / cookie / json path).""" + +from __future__ import annotations + +from typing import Any + +import httpx + + +def _json_path(data: Any, path: str) -> Any: + cur = data + for part in path.split("."): + if isinstance(cur, dict): + cur = cur.get(part) + elif isinstance(cur, list) and part.isdigit(): + i = int(part) + cur = cur[i] if 0 <= i < len(cur) else None + else: + return None + return cur + + +def extract_value(resp: httpx.Response, rule: dict) -> Any: + src = rule.get("from") + if src == "header": + return resp.headers.get(rule["header"]) + if src == "cookie": + return resp.cookies.get(rule["cookie"]) + if src == "json": + try: + return _json_path(resp.json(), rule["json_path"]) + except Exception: # noqa: BLE001 - non-JSON body + return None + return None diff --git a/apps/api/forge/auth_providers/resolver.py b/apps/api/forge/auth_providers/resolver.py new file mode 100644 index 0000000..6f94e34 --- /dev/null +++ b/apps/api/forge/auth_providers/resolver.py @@ -0,0 +1,316 @@ +"""AuthResolver - resolve an Auth Provider to headers/cookies/params for a tool call. + +Caches per (provider, per-user-context-hash) with TTL (in-process here; Redis in +prod). Invalidates on 401/403 (handled by the calling tool). Per-user secrets the +widget injects arrive in `context` and are never stored (Doc 2 §11). +""" + +from __future__ import annotations + +import base64 +import hashlib +import time +from dataclasses import dataclass, field +from typing import Any + +import httpx +from sqlalchemy import select + +from forge.auth_providers.extract import extract_value +from forge.auth_providers.templates import render_value +from forge.config import settings +from forge.db.base import SessionLocal +from forge.models import AuthProvider +from forge.secrets.store import SecretStore +from forge.util.http import shared_async_client +from forge.util.locks import KeyedLocks +from forge.util.ssrf import guarded_request + +# Serialize a provider's (per-user) OAuth refresh so concurrent resolves don't each POST the +# one-time refresh_token and clobber the rotated bundle - the loser would then hold a token the +# IdP has already invalidated (finding i). In-process (single worker); a distributed lock is +# needed for multi-worker, same as the rest of util.locks. +_oauth_refresh_locks = KeyedLocks() + + +@dataclass +class ResolvedAuth: + headers: dict[str, str] = field(default_factory=dict) + cookies: dict[str, str] = field(default_factory=dict) + params: dict[str, str] = field(default_factory=dict) + expires_at: float | None = None # monotonic seconds + + @property + def expired(self) -> bool: + return self.expires_at is not None and time.monotonic() >= self.expires_at + + +class AuthResolver: + def __init__(self, secrets: SecretStore | None = None, session_factory=SessionLocal) -> None: + self.secrets = secrets or SecretStore(session_factory) + self._sf = session_factory + self._cache: dict[str, ResolvedAuth] = {} + + async def _load(self, tenant_id: str, provider_id: str) -> AuthProvider | None: + async with self._sf() as session: + return ( + await session.execute( + select(AuthProvider).where( + AuthProvider.tenant_id == tenant_id, AuthProvider.id == provider_id + ) + ) + ).scalar_one_or_none() + + @staticmethod + def _key(provider_id: str, context: dict, per_user_keys: list[str]) -> str: + dims = "|".join(f"{k}={context.get(k)}" for k in sorted(per_user_keys or [])) + return provider_id + "::" + hashlib.sha256(dims.encode()).hexdigest()[:16] + + async def invalidate(self, key: str) -> None: + self._cache.pop(key, None) + + async def resolve( + self, + *, + tenant_id: str, + project_id: str, + provider_id: str, + context: dict | None = None, + force: bool = False, + client: httpx.AsyncClient | None = None, + provider: AuthProvider | None = None, + ) -> ResolvedAuth: + context = context or {} + provider = provider or await self._load(tenant_id, provider_id) + if provider is None: + raise KeyError(f"Auth provider {provider_id!r} not found") + cfg = provider.config or {} + per_user = cfg.get("per_user_context_keys", []) + # A per-user provider may also accept an INLINE token from the run context (token_ctx_key, + # for server-to-server /run forwarding). That value must vary the cache key too, or a cached + # ResolvedAuth for one caller's inline token could be served to another sharing the same + # end_user dims. The stored-connection path is unaffected (the key is absent from context). + effective_ctx_key = cfg.get("token_ctx_key") or settings.default_token_ctx_key + cache_dims = [*per_user, effective_ctx_key] if effective_ctx_key else per_user + key = self._key(provider_id, context, cache_dims) + if not force and (cached := self._cache.get(key)) and not cached.expired: + return cached + + async def read(ref: str | None) -> Any: + if not ref: + return None + return await self.secrets.read_ref(tenant_id=tenant_id, project_id=project_id, ref=ref) + + # `credentials_ref` is the primary secret for csrf_session/custom_script, but only a + # *fallback* for bearer/api_key (and unused for basic/oauth2). A stale or missing + # fallback must not abort a provider whose own ref (token_ref/value_ref/…) resolves - + # the per-kind branches below still raise clearly if their primary ref is absent. + try: + creds = await read(provider.credentials_ref or cfg.get("credentials_ref")) + except Exception: # noqa: BLE001 - absent fallback secret is tolerated + creds = None + kind = provider.kind + resolved = ResolvedAuth() + default_ttl = cfg.get("cache_ttl_seconds", 1800) + + if kind == "bearer": + token = await self._value_for(provider, cfg, read, context, shared_ref=cfg.get("token_ref"), fallback=creds) + resolved.headers[cfg.get("header_name", "Authorization")] = ( + cfg.get("prefix", "Bearer ") + str(token) + ) + resolved.expires_at = None if default_ttl == 0 else time.monotonic() + default_ttl + elif kind == "api_key": + value = await self._value_for(provider, cfg, read, context, shared_ref=cfg.get("value_ref"), fallback=creds) + where, name = cfg.get("in", "header"), cfg["name"] + (resolved.headers if where == "header" else resolved.params)[name] = str(value) + resolved.expires_at = None + elif kind == "basic": + user = await read(cfg.get("username_ref")) + pw = await read(cfg.get("password_ref")) + token = base64.b64encode(f"{user}:{pw}".encode()).decode() + resolved.headers["Authorization"] = "Basic " + token + resolved.expires_at = None + elif kind == "oauth2_client_credentials": + resolved = await self._oauth2(cfg, read, client) + elif kind == "oauth2_authorization_code": + resolved = await self._oauth2_auth_code(provider, cfg, read, tenant_id, project_id, client, context) + elif kind == "csrf_session": + resolved = await self._csrf_session(cfg, {"cred": creds, "ctx": context}, client, default_ttl) + elif kind == "custom_script": # pragma: no cover - advanced/audited + raise NotImplementedError("custom_script auth requires the advanced-scripts feature flag.") + else: + raise ValueError(f"Unknown auth kind {kind!r}") + + # Extra fixed headers stamped on every call (in addition to the primary auth header) - e.g. a + # constant client id + a service token defined ONCE on the provider instead of hardcoded per + # tool. Each value is a literal or a secret:// ref (resolved from the secret store), so a + # secret never has to live in plaintext in a tool's header config. + for hname, hval in (cfg.get("extra_headers") or {}).items(): + resolved_val = await read(hval) if isinstance(hval, str) and hval.startswith("secret://") else hval + if resolved_val is not None: + resolved.headers[hname] = str(resolved_val) + + self._cache[key] = resolved + return resolved + + async def _value_for(self, provider, cfg: dict, read, context: dict, *, shared_ref, fallback): + """The token/value for a bearer/api_key provider. + + When the provider is PER-USER (config.per_user_context_keys set, e.g. ["end_user_id"]) the + value is the acting user's OWN connected credential - read from the per-user bundle each user + deposits self-service (set_user_connection), keyed by the same per_user dims. So every end + user supplies their own token and a tool acts as them downstream, with NO shared secret and + NO passthrough of the inbound (MCP/session) token. A user who hasn't connected yet resolves + to a clear "not connected" error rather than a silent miss. + + Otherwise it's the shared secret ref (or the credentials_ref fallback) - the prior behavior, + preserved exactly for non-per-user providers.""" + per_user = cfg.get("per_user_context_keys") + if not per_user: + return await read(shared_ref) or fallback + # Inline per-request token (server-to-server /run forwarding via X-Forge-Context) takes + # precedence over the stored connection, so ONE per-user provider serves both delivery paths: + # a chat backend forwards the user's token inline, OR the user connected it once (MCP/console). + # The provider's own token_ctx_key wins; else fall back to the deployment-wide default + # (settings.default_token_ctx_key) so an integration that always forwards the same key works + # for every per-user provider WITHOUT per-provider config (survives project re-creation). + ctx_key = cfg.get("token_ctx_key") or settings.default_token_ctx_key + if ctx_key and (context or {}).get(ctx_key): + return context[ctx_key] + name = self.bundle_secret_name(provider.id, context, per_user) + try: + bundle = await read(f"secret://proj/{name}") + except Exception: # noqa: BLE001 - "not connected" surfaces as a missing secret + bundle = None + if not isinstance(bundle, dict) or not bundle.get("access_token"): + raise KeyError( + f"Auth provider {provider.id!r} is per-user and the acting user has not connected " + f"their credential yet (and no inline token was forwarded)" + ) + return bundle["access_token"] + + async def _oauth2(self, cfg: dict, read, client: httpx.AsyncClient | None) -> ResolvedAuth: + data = { + "grant_type": "client_credentials", + "client_id": await read(cfg.get("client_id_ref")), + "client_secret": await read(cfg.get("client_secret_ref")), + } + if cfg.get("scope"): + data["scope"] = cfg["scope"] + if cfg.get("audience"): + data["audience"] = cfg["audience"] + client = client or shared_async_client() + # SSRF-guarded (host validated pre-connect + every redirect hop) so a tenant-configured + # token_url can't be aimed at an internal/metadata endpoint while carrying secrets (S8). + r = await guarded_request(client, "POST", cfg["token_url"], data=data, timeout=30, follow_redirects=True) + r.raise_for_status() + body = r.json() + token = body.get("access_token", "") + ttl = body.get("expires_in", cfg.get("cache_ttl_seconds", 3600)) + return ResolvedAuth(headers={"Authorization": f"Bearer {token}"}, expires_at=time.monotonic() + ttl) + + @staticmethod + def _per_user_suffix(context: dict | None, per_user_keys: list[str] | None) -> str: + """A stable short hash of the per-user context dims, so each end-user's OAuth bundle is + stored under its own secret name when the provider is configured per-user (finding i).""" + if not per_user_keys: + return "" + dims = "|".join(f"{k}={(context or {}).get(k)}" for k in sorted(per_user_keys)) + return "__u" + hashlib.sha256(dims.encode()).hexdigest()[:12] + + @staticmethod + def bundle_secret_name(provider_id: str, context: dict | None = None, + per_user_keys: list[str] | None = None) -> str: + # Default (no per_user_keys) preserves the original single-account name. + return f"oauth_token__{provider_id}" + AuthResolver._per_user_suffix(context, per_user_keys) + + async def _store_bundle(self, tenant_id: str, project_id: str, provider_id: str, bundle: dict, + *, name: str | None = None) -> None: + async with self._sf() as session: + await self.secrets.write( + session, tenant_id=tenant_id, project_id=project_id, + name=name or self.bundle_secret_name(provider_id), value=bundle, kind="oauth", + ) + + async def _oauth2_auth_code( + self, provider, cfg: dict, read, tenant_id: str, project_id: str, + client: httpx.AsyncClient | None, context: dict | None = None + ) -> ResolvedAuth: + # Per-user bundle name when the provider keys tokens per end-user (finding i). + per_user = cfg.get("per_user_context_keys") + bundle_name = self.bundle_secret_name(provider.id, context, per_user) + bundle_ref = cfg.get("token_bundle_ref") or f"secret://proj/{bundle_name}" + bundle = await read(bundle_ref) + if not isinstance(bundle, dict) or not bundle.get("access_token"): + raise KeyError(f"OAuth provider {provider.id!r} is not connected - run the connect flow first") + now = time.time() + expires_at = bundle.get("expires_at") + if expires_at and now >= (float(expires_at) - 60) and bundle.get("refresh_token"): + # Serialize refresh per (tenant, provider, per-user bundle) so concurrent resolves + # don't race the one-time refresh_token; re-read inside the lock in case a peer + # already refreshed it. + lock = await _oauth_refresh_locks.acquire_cm(f"{tenant_id}:{provider.id}:{bundle_name}") + async with lock: + fresh = await read(bundle_ref) + if isinstance(fresh, dict) and fresh.get("access_token"): + bundle = fresh + expires_at = bundle.get("expires_at") + if expires_at and time.time() >= (float(expires_at) - 60) and bundle.get("refresh_token"): + bundle = await self._refresh_oauth(provider, cfg, read, bundle, tenant_id, + project_id, client, bundle_name=bundle_name) + expires_at = bundle.get("expires_at") + header = cfg.get("header_name", "Authorization") + prefix = cfg.get("prefix", "Bearer ") + ttl_left = (float(expires_at) - now) if expires_at else None + cache_exp = time.monotonic() + max(0.0, ttl_left - 60) if ttl_left and ttl_left > 0 else None + return ResolvedAuth(headers={header: prefix + str(bundle["access_token"])}, expires_at=cache_exp) + + async def _refresh_oauth(self, provider, cfg: dict, read, bundle: dict, tenant_id, project_id, + client, *, bundle_name: str | None = None) -> dict: + data = { + "grant_type": "refresh_token", + "refresh_token": bundle["refresh_token"], + "client_id": await read(cfg.get("client_id_ref")), + "client_secret": await read(cfg.get("client_secret_ref")), + } + client = client or shared_async_client() + r = await guarded_request( + client, "POST", cfg["token_url"], + data={k: v for k, v in data.items() if v is not None}, timeout=30, follow_redirects=True, + ) + r.raise_for_status() + body = r.json() + new = dict(bundle) + new["access_token"] = body.get("access_token", bundle["access_token"]) + if body.get("refresh_token"): + new["refresh_token"] = body["refresh_token"] + if body.get("expires_in"): + new["expires_at"] = time.time() + int(body["expires_in"]) + await self._store_bundle(tenant_id, project_id, provider.id, new, name=bundle_name) + return new + + async def _csrf_session(self, cfg: dict, vars: dict, client: httpx.AsyncClient | None, default_ttl: int) -> ResolvedAuth: + fetch = render_value(cfg["token_fetch"], vars) + client = client or shared_async_client() + r = await guarded_request( + client, fetch["method"], fetch["url"], + headers=fetch.get("headers"), json=fetch.get("body"), timeout=30, follow_redirects=True, + ) + r.raise_for_status() + + extracted: dict[str, Any] = {} + ttl = None + for rule in cfg.get("extract", []): + val = extract_value(r, rule) + if rule.get("kind") == "ttl": + ttl = int(val) if val else None + else: + extracted[rule["name"]] = val + + out = ResolvedAuth(expires_at=time.monotonic() + (ttl or default_ttl)) + for rule in cfg.get("inject", []): + value = render_value(rule["value"], {"extracted": extracted}) + where = rule["to"] + target = {"header": out.headers, "cookie": out.cookies, "query": out.params}[where] + target[rule["name"]] = str(value) + return out diff --git a/apps/api/forge/auth_providers/templates.py b/apps/api/forge/auth_providers/templates.py new file mode 100644 index 0000000..d468cf2 --- /dev/null +++ b/apps/api/forge/auth_providers/templates.py @@ -0,0 +1,116 @@ +"""Token-template rendering for auth recipes. + +Supports `{{a.b}}` references resolved against a vars dict, e.g. +`{{cred.username}}`, `{{ctx.csrf}}`, `{{extracted.session}}`. Non-string leaves +pass through; whole-string matches preserve the resolved value's native type. + +`render_value` walks a parsed JSON structure (dict/list) and additionally honors a +`{"$each": "{{input.rows}}", "$as": "row", "$do": {...}}` loop directive, so a JSON body +template can build a variable-length array from one list-valued arg (see `_render_each`). +""" + +from __future__ import annotations + +import re +from collections.abc import Collection +from typing import Any + +_TOKEN = re.compile(r"\{\{\s*([a-zA-Z0-9_.]+)\s*\}\}") + + +class MissingTemplateVar(ValueError): + """A template referenced a variable in a STRICT namespace that isn't defined. + + Raised for `{{env.*}}` when the key isn't in FORGE_TOOL_VARS for the current environment, + so a misconfigured deploy fails loudly at call/test time instead of sending a broken URL. + The lenient namespaces (ctx/input) keep their historic behavior: a missing key renders empty + / is dropped.""" + + +def _lookup(path: str, vars: dict, strict_ns: Collection[str] = ()) -> Any: + parts = path.split(".") + cur: Any = vars + for part in parts: + if isinstance(cur, dict): + cur = cur.get(part) + else: + cur = None + break + # A missing value under a STRICT namespace (e.g. env) is an error, not an empty string. + if cur is None and parts and parts[0] in strict_ns: + raise MissingTemplateVar(f"undefined template variable {{{{{path}}}}} (namespace '{parts[0]}')") + return cur + + +def _sub_one(mm: re.Match, vars: dict, strict_ns: Collection[str] = ()) -> str: + # Embedded token (not a whole-string match): stringify the resolved value. Only a missing + # value (None) becomes empty - a falsy-but-real value like 0 or False must render as "0"/ + # "False", not "" (an `x or ""` here would silently drop legitimate zeros/booleans). + v = _lookup(mm.group(1), vars, strict_ns) + return "" if v is None else str(v) + + +def render_template(s: str, vars: dict, *, strict_ns: Collection[str] = ()) -> Any: + # `strict_ns` names namespaces whose missing keys raise MissingTemplateVar instead of + # rendering empty (used for {{env.*}}). Defaults to lenient for all namespaces. + # Whole-string single token -> preserve native type (numbers, objects). + m = _TOKEN.fullmatch(s.strip()) + if m: + return _lookup(m.group(1), vars, strict_ns) + return _TOKEN.sub(lambda mm: _sub_one(mm, vars, strict_ns), s) + + +def has_each_directive(obj: Any) -> bool: + """True if `obj` (a parsed JSON structure) contains a `$each` loop directive anywhere - i.e. + a dict that has "$each" as a KEY. Used to decide whether a body template needs structural + rendering; a literal "$each" appearing inside a string value is NOT a directive and must not + trigger it (that would silently change type coercion for unrelated templates).""" + if isinstance(obj, dict): + if "$each" in obj: + return True + return any(has_each_directive(v) for v in obj.values()) + if isinstance(obj, list): + return any(has_each_directive(v) for v in obj) + return False + + +def render_value(obj: Any, vars: dict, *, allow_each: bool = False, strict_ns: Collection[str] = ()) -> Any: + """Walk a parsed JSON structure, rendering `{{token}}` leaves. `$each` loop directives are + honored ONLY when `allow_each=True` (the REST body-template path opts in); every other caller + - auth token_fetch/extract rules, data-node payloads - passes the default False, so a literal + object key named "$each" stays an ordinary key instead of being reinterpreted as a loop. + `strict_ns` propagates the fail-loud namespaces (e.g. env) to every leaf.""" + if isinstance(obj, str): + return render_template(obj, vars, strict_ns=strict_ns) + if isinstance(obj, dict): + if allow_each and "$each" in obj: + return _render_each(obj, vars, strict_ns=strict_ns) + return {k: render_value(v, vars, allow_each=allow_each, strict_ns=strict_ns) for k, v in obj.items()} + if isinstance(obj, list): + return [render_value(v, vars, allow_each=allow_each, strict_ns=strict_ns) for v in obj] + return obj + + +def _render_each(directive: dict, vars: dict, *, strict_ns: Collection[str] = ()) -> list: + """Expand a `{"$each": "{{input.rows}}", "$as": "row", "$do": {...}}` loop directive into a + list: render `$do` once per item of the array `$each` resolves to, with the item bound under + the `$as` name (default "item"). Outer vars (input/ctx/state) stay visible inside the loop, so + a nested template can still read e.g. `{{input.orderId}}`. A missing/None `$each` yields []; + a single non-list value is treated as one item. + + This lets a JSON body template build a variable-length array (e.g. one productRow per edited + cell) WITHOUT string-concatenating JSON - so the output is always valid JSON with native types + preserved, and one tool call can carry many rows instead of one call per row. + """ + each = directive.get("$each") + seq = render_value(each, vars, strict_ns=strict_ns) if isinstance(each, str) else each + if seq is None: + items: list = [] + elif isinstance(seq, list): + items = seq + else: + items = [seq] + as_name = directive.get("$as") or "item" + body = directive.get("$do") + # allow_each=True so a `$do` body can itself contain a nested `$each` (loops within loops). + return [render_value(body, {**vars, as_name: item}, allow_each=True, strict_ns=strict_ns) for item in items] diff --git a/apps/api/forge/channels/__init__.py b/apps/api/forge/channels/__init__.py new file mode 100644 index 0000000..3094681 --- /dev/null +++ b/apps/api/forge/channels/__init__.py @@ -0,0 +1 @@ +"""Channel adapters.""" diff --git a/apps/api/forge/channels/email.py b/apps/api/forge/channels/email.py new file mode 100644 index 0000000..46e05bb --- /dev/null +++ b/apps/api/forge/channels/email.py @@ -0,0 +1,238 @@ +"""Email channel - inbound parsing and outbound (SMTP) replies. + +Inbound supports two shapes: +- A raw RFC-822 message (bytes/str), e.g. from an IMAP poll. +- A provider inbound-parse payload (Mailgun/SendGrid/Postmark post form/JSON fields). + +Outbound sends a threaded reply via SMTP (creds resolved from the channel's secret +refs). SMTP/IMAP run in worker threads so they don't block the event loop. +""" + +from __future__ import annotations + +import asyncio +import email +import logging +import re +import smtplib +from email.message import EmailMessage +from email.utils import make_msgid, parseaddr +from html.parser import HTMLParser +from typing import Any + +from forge.channels.retry import retry_send +from forge.secrets.store import SecretStore + +log = logging.getLogger("forge.channels.email") + +_BLOCK_ENDERS = {"br", "p", "div", "li", "tr"} +_SKIP_TAGS = {"script", "style"} + + +class _HTMLTextExtractor(HTMLParser): + """Pull visible text out of HTML with the stdlib parser instead of regexes, so a hostile + body can't trigger catastrophic backtracking (ReDoS): skip ` + ); + }, [def.html, def.css, props]); + + useEffect(() => { + function onMsg(e: MessageEvent) { + if (!ref.current || e.source !== ref.current.contentWindow) return; + // Sandboxed (allow-scripts, no same-origin) iframes post from the opaque "null" origin; + // accept that or our own origin, reject anything else (audit F27). + if (e.origin !== "null" && e.origin !== window.location.origin) return; + const d: any = e.data || {}; + if (!d.__forge) return; + if (d.type === "size" && typeof d.height === "number") { + setHeight(Math.min(2000, Math.max(40, Math.ceil(d.height)))); + } else if (d.type === "action") { + actionRef.current?.(String(d.action || ""), d.fields || {}, defRef.current); + } + } + window.addEventListener("message", onMsg); + return () => window.removeEventListener("message", onMsg); + }, []); + + const actions = def.actions || []; + return ( +
+ ` : null; + + return ( +
+
+ +
+ + +
Which workflow the widget runs.
+
+
+ +