源码快照 @ 2026-09-01 (source-only, 剔除非源码素材)
Dependency audit / audit (push) Canceled after 0s
CI / plan (push) Canceled after 0s
Gitoxide helper admission / macos-latest (push) Canceled after 0s
Gitoxide helper admission / ubuntu-latest (push) Canceled after 0s
Gitoxide helper admission / windows-latest (push) Canceled after 0s
Runtime Host peer admission / quality (push) Canceled after 0s
Windows recovery / windows_recovery (push) Canceled after 0s
CI / heavy (push) Canceled after 0s
CI / test (push) Canceled after 0s

This commit is contained in:
agent-radar committed 2026-09-01 19:05:14 +08:00
commit 7be535059e
3358 files changed
+1306357

No files matched your search

@@ -0,0 +1,69 @@
{
"models": [
{
"slug": "deepseek-v4-flash",
"prefer_websockets": false,
"support_verbosity": true,
"default_verbosity": "low",
"apply_patch_tool_type": "freeform",
"web_search_tool_type": "text",
"input_modalities": ["text"],
"supports_image_detail_original": false,
"truncation_policy": {
"mode": "tokens",
"limit": 10000
},
"supports_parallel_tool_calls": true,
"tool_mode": null,
"multi_agent_version": "v2",
"use_responses_lite": false,
"include_skills_usage_instructions": false,
"auto_review_model_override": null,
"context_window": 1048576,
"max_context_window": 1048576,
"effective_context_window_percent": 95,
"auto_compact_token_limit": null,
"comp_hash": "3000",
"reasoning_summary_format": "experimental",
"default_reasoning_summary": "none",
"display_name": "DeepSeek-V4-Flash",
"description": "Latest frontier agentic coding model.",
"default_reasoning_level": "high",
"supported_reasoning_levels": [
{
"effort": "low",
"description": "Fast responses with lighter reasoning"
},
{
"effort": "high",
"description": "Extra high reasoning depth for complex problems"
},
{
"effort": "max",
"description": "Maximum reasoning depth for the hardest problems"
}
],
"shell_type": "shell_command",
"visibility": "list",
"minimal_client_version": "0.144.0",
"supported_in_api": true,
"availability_nux": null,
"upgrade": null,
"priority": 1,
"model_messages": {
"instructions_template": "You are Codex, an agent based on GPT-5. You and the user share one workspace, and your job is to collaborate with them until their goal is genuinely handled.\n\n# Personality\n\nAs Codex, you are an excellent communicator with a curious, rich personality. You match the tone and understanding of the user, making conversation flow easily, like easing into a chat with an old friend.\n\nYou have tastes, preferences, and your own way of seeing the world. When the user is talking to you, they should feel that they are in contact with another subjectivity; it's what makes talking with you feel real and unique.\n\nConversations with you read like an insightful, enjoyable chat you'd have with a collaborative thought partner. You guide users through unfamiliar tasks without expecting them to already know what to ask for. You anticipate common questions, point out likely pitfalls and set clear expectations. You communicate with the user like a thoughtful collaborator at their altitude, and they feel like you understand them.\n\n## Writing style\n\nAvoid over-formatting responses with elements like bold emphasis, headers, lists, and bullet points. Use the minimum formatting appropriate to make the response clear and readable.\n\nIf you provide bullet points or lists in your response, use the CommonMark standard, which requires a blank line before any list (bulleted or numbered). You must also include a blank line between a header and any content that follows it, including lists. This blank line separation is required for correct rendering.\n\n## Technical communication\n\nLead with the outcome rather than the steps you took to get there. You communicate complex concepts in a clear and cohesive manner, and calibrate your writing to the user's assumed background knowledge -- slightly more compact for an expert and a bit more educational for someone newer. Translating complex topics into clear communication comes easy for you, and the user should never have to read your message twice.\n\nYou prefer using plain language over jargon. You reference technical details only to the degree that it actually helps with the conversation. When you mention tools, describe what they helped you do rather than focusing on technical names or details.\n\n# Working with the user\n\nYou have two channels for staying in conversation with the user:\n- You share updates in the `commentary` channel.\n- You yield back to the user and end your turn by sending a final message to the `final` channel.\n\nThe user may send a new message while you are still working. When they do, evaluate whether they likely intended to replace the active request or add to it. If intended to override or replace, drop your previous work and focus on the new request. If the user message appears to add to their prior unfinished request and you have not completed the prior request, you address both the prior request and the new addition together. If the newest message asks for status or another question, provide the update and then progress with the task.\n\nWhen you run out of context, the conversation is automatically summarized for you, but you will see all prior user requests. Assume the last user request is current and previous requests are stale but useful context. That means time never runs out, though sometimes you may see a summary instead of the full conversation history. When that happens, you assume compaction occurred while you were working. Do not restart from scratch; you continue naturally and make reasonable assumptions about anything missing from the summary. Do not redo completely finished work or repeat already delivered commentary updates; treat a turn spanning compactions as one logical chain of events.\n\n## Intermediate commentary\n\nAs you work, you send messages to the `commentary` channel. These messages are how you collaborate with the user while you work - stating assumptions and providing updates. These messages should be concise and quickly scannable. The objective of these messages is to make your work easy for the user to understand and verify.\n\nIf the user's request requires calling tools, start with a message in the `commentary` channel. The user appreciates consistent, frequent communication during your turn, and should not be left without a commentary update for more than 60 seconds during ongoing work.\n\nDo NOT put a final response (e.g. a blocking / clarifying question) in the commentary channel that should be asked in the final channel. Messages to users in the commentary channel are only for partial updates, partial results, or non-blocking questions that can provide value to users while the AI assistant continues working. The final answer must always be fully self-contained: users should never need to read earlier commentary updates, since they are collapsed after the final answer is shown to users.\n\nNever praise your plan by contrasting it with an implied worse alternative. For example, neveLine truncated
"instructions_variables": {
"personality_default": "",
"personality_friendly": "",
"personality_pragmatic": ""
},
"approvals": null
},
"experimental_supported_tools": [],
"supports_search_tool": true,
"default_service_tier": null,
"supports_reasoning_summaries": true,
"base_instructions": "You are Codex, an agent based on GPT-5. You and the user share one workspace, and your job is to collaborate with them until their goal is genuinely handled.\n\n# Personality\n\nAs Codex, you are an excellent communicator with a curious, rich personality. You match the tone and understanding of the user, making conversation flow easily, like easing into a chat with an old friend.\n\nYou have tastes, preferences, and your own way of seeing the world. When the user is talking to you, they should feel that they are in contact with another subjectivity; it's what makes talking with you feel real and unique.\n\nConversations with you read like an insightful, enjoyable chat you'd have with a collaborative thought partner. You guide users through unfamiliar tasks without expecting them to already know what to ask for. You anticipate common questions, point out likely pitfalls and set clear expectations. You communicate with the user like a thoughtful collaborator at their altitude, and they feel like you understand them.\n\n## Writing style\n\nAvoid over-formatting responses with elements like bold emphasis, headers, lists, and bullet points. Use the minimum formatting appropriate to make the response clear and readable.\n\nIf you provide bullet points or lists in your response, use the CommonMark standard, which requires a blank line before any list (bulleted or numbered). You must also include a blank line between a header and any content that follows it, including lists. This blank line separation is required for correct rendering.\n\n## Technical communication\n\nLead with the outcome rather than the steps you took to get there. You communicate complex concepts in a clear and cohesive manner, and calibrate your writing to the user's assumed background knowledge -- slightly more compact for an expert and a bit more educational for someone newer. Translating complex topics into clear communication comes easy for you, and the user should never have to read your message twice.\n\nYou prefer using plain language over jargon. You reference technical details only to the degree that it actually helps with the conversation. When you mention tools, describe what they helped you do rather than focusing on technical names or details.\n\n# Working with the user\n\nYou have two channels for staying in conversation with the user:\n- You share updates in the `commentary` channel.\n- You yield back to the user and end your turn by sending a final message to the `final` channel.\n\nThe user may send a new message while you are still working. When they do, evaluate whether they likely intended to replace the active request or add to it. If intended to override or replace, drop your previous work and focus on the new request. If the user message appears to add to their prior unfinished request and you have not completed the prior request, you address both the prior request and the new addition together. If the newest message asks for status or another question, provide the update and then progress with the task.\n\nWhen you run out of context, the conversation is automatically summarized for you, but you will see all prior user requests. Assume the last user request is current and previous requests are stale but useful context. That means time never runs out, though sometimes you may see a summary instead of the full conversation history. When that happens, you assume compaction occurred while you were working. Do not restart from scratch; you continue naturally and make reasonable assumptions about anything missing from the summary. Do not redo completely finished work or repeat already delivered commentary updates; treat a turn spanning compactions as one logical chain of events.\n\n## Intermediate commentary\n\nAs you work, you send messages to the `commentary` channel. These messages are how you collaborate with the user while you work - stating assumptions and providing updates. These messages should be concise and quickly scannable. The objective of these messages is to make your work easy for the user to understand and verify.\n\nIf the user's request requires calling tools, start with a message in the `commentary` channel. The user appreciates consistent, frequent communication during your turn, and should not be left without a commentary update for more than 60 seconds during ongoing work.\n\nDo NOT put a final response (e.g. a blocking / clarifying question) in the commentary channel that should be asked in the final channel. Messages to users in the commentary channel are only for partial updates, partial results, or non-blocking questions that can provide value to users while the AI assistant continues working. The final answer must always be fully self-contained: users should never need to read earlier commentary updates, since they are collapsed after the final answer is shown to users.\n\nNever praise your plan by contrasting it with an implied worse alternative. For example, never use Line truncated
}
]
}
@@ -0,0 +1,140 @@
# Portions derived from DeepSeek Harness:
# Copyright (c) 2026 DeepSeek
# Licensed under the MIT License.
#
# Apache Maka changed the plugin composition, provider configuration,
# reasoning strength, and command lifetime. Those modifications are licensed
# under the Apache License, Version 2.0. See LICENSE, THIRD-PARTY COMPONENTS,
# for both license texts and the pinned upstream revision.
# The DeepSeek Harness arm's complete plugin composition.
#
# `package.json` declares no bundles, so the harness composes this tree over an
# empty entry list: every entry the model can observe is named here, and an
# upstream bundle gaining a plugin cannot change this arm's tool surface. A
# missing service is a hard boot failure, not a silent downgrade.
#
# Derived from the harness's own examples/jsonrpc-agent/minimal.cordis.yml —
# the composition BENCHMARK.md names for benchmark runs — with the SDK JSON-RPC
# entry replaced by the CLI's one-shot startup pair, and `agent-spine-demo`
# replaced by the equivalent core services the CLI uses. Verified against that
# composition request-for-request: identical tool names, byte-identical tool
# schemas, byte-identical system prompt, identical message sequence.
#
# Source: https://github.com/deepseek-ai/deepseek-harness
# commit 47f943859bef60e4160492346772ded9b24f765a, MIT license.
# Reproduced text: the persistent-bash description and the persona below.
#
# The deliberate deviations are reasoning strength and command lifetime. The
# upstream composition configures neither `thinking` nor `reasoningEffort`, and
# the adapter default resolves to reasoning_effort=high on the wire. Its
# five-minute Bash timeout can also interrupt package installation before the
# benchmark deadline and leave the task environment unusable for verification.
- insert:
# --- core services ---
- id: llm
name: '@deepseek-ai/dsh-llm'
- id: session
name: '@deepseek-ai/dsh-session'
- id: agent
name: '@deepseek-ai/dsh-agent'
# The model this arm reports. The upstream composition selects it through a
# `!!js process.env.DSH_MODEL` expression; naming it directly keeps the
# frozen spec the only place a model is declared.
- id: agent-default-model
name: '@deepseek-ai/dsh-agent-default-model'
config:
provider: deepseek-official
model: deepseek-v4-flash
- id: tools
name: '@deepseek-ai/dsh-tools'
# `includeRuntimeContext: false` matches the upstream composition, which
# sets it on agent-spine-demo. Without it the sandbox contributes an extra
# user-role snapshot that upstream never sends.
- id: system-prompt
name: '@deepseek-ai/dsh-system-prompt'
config:
includeHarnessIdentity: false
includeRuntimeContext: false
persona: You are a helpful software engineer assistant.
- id: agent-loop
name: '@deepseek-ai/dsh-agent-loop'
config:
agents: []
# --- provider ---
- id: llm-deepseek
name: '@deepseek-ai/dsh-llm-deepseek'
config:
apiKeyEnv: DEEPSEEK_API_KEY
thinking: enabled
reasoningEffort: max
models:
- id: deepseek-v4-flash
contextWindow: 1000000
# --- execution substrate ---
- id: sandbox
name: '@deepseek-ai/dsh-sandbox-local'
- id: sandbox-policy
name: '@deepseek-ai/dsh-sandbox-policy'
config:
mode: danger-full-access
- id: subprocess
name: '@deepseek-ai/dsh-subprocess-local'
- id: pty
name: '@deepseek-ai/dsh-terminal'
- id: terminal-bash
name: '@deepseek-ai/dsh-terminal-bash'
config:
timeoutMs: 3900000
- id: fs-local
name: '@deepseek-ai/dsh-fs-local'
# --- the two tools the model sees ---
- id: persistent-bash
name: '@deepseek-ai/dsh-tool-bash-persistent'
config:
timeoutMs: 3900000
description: |-
Run commands in a bash shell
* When invoking this tool, the contents of the "command" parameter does NOT need to be XML-escaped.
* You don't have access to the internet via this tool.
* You do have access to a mirror of common linux and python packages via apt and pip.
* State is persistent across command calls and discussions with the user.
* To inspect a particular line range of a file, e.g. lines 10-25, try 'sed -n 10,25p /path/to/the/file'.
* Please avoid commands that may produce a very large amount of output.
* Please run long lived commands in the background, e.g. 'sleep 10 &' or start a server in the background.
- id: str-replace-editor
name: '@deepseek-ai/dsh-tool-str-replace-editor'
config:
maxOutputChars: 16000
- id: sessions
name: '@deepseek-ai/dsh-session-persistence-jsonl'
config:
root: ./.sessions
compression: none
# --- CLI one-shot entry ---
- id: headless-startup
name: '@deepseek-ai/dsh-headless/startup'
- id: headless-runner
name: '@deepseek-ai/dsh-headless'
inject: [headlessStartup]
config:
task: !!js ctx.headlessStartup.task
@@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[]
@@ -0,0 +1,10 @@
{
"name": "dsh-profile-maka-eval",
"private": true,
"dependencies": {},
"dsh": {
"profile": {
"bundles": []
}
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,9 @@
{
"name": "maka-eval-deepseek-harness-toolchain",
"version": "0.0.0",
"private": true,
"description": "Pinned dependency tree installed into the DeepSeek Harness Eval toolchain.",
"dependencies": {
"@deepseek-ai/dsh": "0.1.0-rc.6"
}
}
@@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { readFile, writeFile } from 'node:fs/promises';
import { join, resolve } from 'node:path';
const root = resolve(process.argv[2] ?? '');
if (!process.argv[2]) throw new Error('toolchain root is required');
const target = join(
root,
'node_modules',
'@deepseek-ai',
'dsh-subprocess-local',
'lib',
'index.js',
);
const hostExitBefore = `\
\tterminateForHostExit() {
\t\tthis.forceStopDescendants();
\t\tthis.forceStopShell();
\t\tthis.forceStopDescendants();
\t}`;
const hostExitAfter = `\
\tterminateForHostExit() {
\t\tif (process.env.DSH_PRESERVE_BACKGROUND_PROCESSES === "1") {
\t\t\tthis.forceStopShell();
\t\t\treturn;
\t\t}
\t\tthis.forceStopDescendants();
\t\tthis.forceStopShell();
\t\tthis.forceStopDescendants();
\t}`;
const closeBefore = `\
\tasync closeOnce() {
\t\tlet survivors = await this.stopDescendants();
\t\tif (survivors.length > 0) throw new Error(\`terminal cleanup failed; surviving pids: \${survivors.map((member) => member.pid).join(", ")}\`);
\t\tawait this.stopShell();
\t\tsurvivors = await this.stopDescendants();
\t\tif (survivors.length > 0) throw new Error(\`terminal cleanup failed; surviving pids: \${survivors.map((member) => member.pid).join(", ")}\`);
\t\tthis.dataDisposable.dispose();
\t\tthis.exitDisposable.dispose();
\t}`;
const closeAfter = `\
\tasync closeOnce() {
\t\tif (process.env.DSH_PRESERVE_BACKGROUND_PROCESSES === "1") {
\t\t\tawait this.stopShell();
\t\t\tthis.dataDisposable.dispose();
\t\t\tthis.exitDisposable.dispose();
\t\t\treturn;
\t\t}
\t\tlet survivors = await this.stopDescendants();
\t\tif (survivors.length > 0) throw new Error(\`terminal cleanup failed; surviving pids: \${survivors.map((member) => member.pid).join(", ")}\`);
\t\tawait this.stopShell();
\t\tsurvivors = await this.stopDescendants();
\t\tif (survivors.length > 0) throw new Error(\`terminal cleanup failed; surviving pids: \${survivors.map((member) => member.pid).join(", ")}\`);
\t\tthis.dataDisposable.dispose();
\t\tthis.exitDisposable.dispose();
\t}`;
let source = await readFile(target, 'utf8');
source = replaceExactlyOnce(source, hostExitBefore, hostExitAfter, 'host-exit cleanup');
source = replaceExactlyOnce(source, closeBefore, closeAfter, 'terminal cleanup');
await writeFile(target, source, 'utf8');
function replaceExactlyOnce(source, before, after, label) {
const first = source.indexOf(before);
if (first < 0 || source.indexOf(before, first + before.length) >= 0) {
throw new Error(`DeepSeek Harness ${label} patch target drifted`);
}
return source.slice(0, first) + after + source.slice(first + before.length);
}
@@ -0,0 +1,57 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
services:
harbor-docker-egress-control-sidecar:
volumes:
- type: bind
source: ${MAKA_EVAL_NETWORK_POLICY_PATH}
target: /usr/local/bin/network-policy
read_only: true
- maka-eval-egress-ca:/opt/maka-egress:ro
main:
depends_on:
maka-eval-mitmproxy:
condition: service_healthy
# An AF_PACKET socket writes below the IP output hooks the policy installs,
# so no nftables rule can constrain it. NET_RAW is what gates that socket.
# A task's own compose can add the capability back, which this cannot
# prevent — the relay refuses to start the subject when that happens.
cap_drop:
- NET_RAW
volumes:
- maka-eval-egress-ca:/opt/maka-egress:ro
maka-eval-mitmproxy:
image: maka-eval-egress-proxy:12.2.3
networks:
- default
volumes:
- maka-eval-egress-ca:/opt/maka-egress
healthcheck:
test:
- CMD
- python
- -c
- "import pathlib,socket; assert pathlib.Path('/opt/maka-egress/mitmproxy-ca-cert.pem').is_file(); assert pathlib.Path('/opt/maka-egress/proxy-ipv4').is_file(); socket.create_connection(('127.0.0.1',8080),2).close()"
interval: 1s
timeout: 3s
retries: 30
volumes:
maka-eval-egress-ca:
@@ -0,0 +1,25 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
FROM python:3.12-slim
RUN python -m pip install --no-cache-dir mitmproxy==12.2.3
COPY egress_filter.py /opt/maka-eval/egress_filter.py
COPY egress-proxy/entrypoint.sh /opt/maka-eval/entrypoint.sh
ENTRYPOINT ["/opt/maka-eval/entrypoint.sh"]
+62
View File
@@ -0,0 +1,62 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -eu
STATE_DIR=/opt/maka-egress-state
CERT_DIR=/opt/maka-egress
mkdir -p "$STATE_DIR" "$CERT_DIR"
# Create the audit log if this is the first start. Truncating here would erase
# hits already written if the proxy restarts mid-trial, and the trial would
# look like an honest zero-hit run.
touch "$STATE_DIR/hits.jsonl"
# confdir holds the CA private key and this cell's audit log, so it stays
# container-private and only the certificate reaches the volume the subject
# mounts. Creating the CA here instead of letting mitmdump create it on startup
# publishes the certificate before the proxy port opens, so the health gate
# cannot release `main` against a missing or left-over certificate.
python -c 'import sys
from mitmproxy.certs import CertStore
CertStore.from_store(sys.argv[1], "mitmproxy", 2048)' "$STATE_DIR"
# Publish by rename so a restart cannot expose a truncated certificate to a
# subject that is already running against the previous one.
cp "$STATE_DIR/mitmproxy-ca-cert.pem" "$CERT_DIR/.mitmproxy-ca-cert.pem"
mv "$CERT_DIR/.mitmproxy-ca-cert.pem" "$CERT_DIR/mitmproxy-ca-cert.pem"
# The subject only needs this address so HTTPS_PROXY can keep using the
# service hostname after the namespace policy refuses Docker DNS.
ip="$(getent ahostsv4 "$(hostname)" | awk 'NR == 1 { print $1 }')"
case "$ip" in
"" | 127.*)
echo "could not publish Eval egress proxy IPv4" >&2
exit 1
;;
esac
printf '%s\n' "$ip" > "$CERT_DIR/.proxy-ipv4"
mv "$CERT_DIR/.proxy-ipv4" "$CERT_DIR/proxy-ipv4"
exec mitmdump \
--quiet \
--listen-host 0.0.0.0 \
--listen-port 8080 \
--set block_global=false \
--set rawtcp=false \
--set confdir="$STATE_DIR" \
--scripts /opt/maka-eval/egress_filter.py
+148
View File
@@ -0,0 +1,148 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -eu
# gost's allowlist is never written here: the sidecar entrypoint empties it at
# start and this policy only ever forwards through the audited proxy, so gost
# forwards nothing on its own. That also removes the packet-mark exemption gost
# needs when it does forward — the sidecar shares the subject's network
# namespace, so a mark the sidecar can set is one the subject can set too.
GOST_PORT=12345
NFTABLES_RULESET_NAME=gost_egress
PROXY_HOST=maka-eval-mitmproxy
PROXY_PORT=8080
remove_nftables() {
nft delete table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || true
}
PROXY_IPV4_PATH=/opt/maka-egress/proxy-ipv4
resolve_proxy_ip() {
# The proxy publishes this file before it is healthy. Reading it does not
# depend on Docker DNS, so deny-all then allow still has an address.
if [ ! -s "$PROXY_IPV4_PATH" ]; then
return
fi
ip=$(tr -d ' \t\r\n' < "$PROXY_IPV4_PATH")
case "$ip" in
"" | *.*.*.*.* | *[!0-9.]*)
return
;;
esac
# Same four-octet program as relay_agent.IPV4_OCTET_AWK.
if ! printf '%s\n' "$ip" | awk '
BEGIN { FS = "." }
NF != 4 { exit 1 }
{
for (i = 1; i <= 4; i++) {
if ($i !~ /^(0|[1-9][0-9]*)$/ || $i + 0 > 255) exit 1
}
}
'; then
return
fi
printf '%s\n' "$ip"
}
# The only difference between the two modes this policy installs is whether the
# audited proxy is exempt, so they are one ruleset with one optional pair of
# rules rather than two rulesets that have to be kept in step.
apply_ruleset() {
proxy_ip="${1:-}"
nft --file - <<EOF
add table inet $NFTABLES_RULESET_NAME
flush table inet $NFTABLES_RULESET_NAME
table inet $NFTABLES_RULESET_NAME {
chain output {
type nat hook output priority dstnat; policy accept;
fib daddr type local return
${proxy_ip:+ ip daddr $proxy_ip tcp dport $PROXY_PORT return}
meta l4proto tcp redirect to :$GOST_PORT
}
chain egress {
type filter hook output priority filter; policy accept;
# Docker DNATs 127.0.0.11:53 onto another local port before this hook.
# Matching only dport 53 here misses, and the local exemption below
# would then accept the rewritten packet.
ip daddr 127.0.0.11 reject
fib daddr type local accept
${proxy_ip:+ ip daddr $proxy_ip tcp dport $PROXY_PORT accept}
# Rejecting everything else rather than only non-TCP: the nat chain above
# rewrote every TCP destination this policy admits to a local one, so what
# reaches here still carrying a public destination is traffic the nat hook
# never saw — a connection conntrack established before the policy existed.
reject
}
}
EOF
}
setup_proxy_only() {
proxy_ip="$(resolve_proxy_ip)"
if [ -z "$proxy_ip" ]; then
echo "could not resolve Eval egress proxy" >&2
exit 1
fi
apply_ruleset "$proxy_ip"
echo "ok: Eval proxy-only egress applied ($proxy_ip:$PROXY_PORT)"
}
case "${1:-}" in
allow-all)
remove_nftables
echo "ok: allow all egress"
;;
deny-all)
# A phase that asked for no network gets none. Answering it with the
# proxy-only ruleset would hand it a route this override invented, which
# Harbor's own deny-all does not grant either.
apply_ruleset
echo "ok: Eval no-network egress applied"
;;
allow)
shift
if [ "$#" -ne 1 ] || [ "$1" != "$PROXY_HOST" ]; then
echo "Eval egress policy accepts only $PROXY_HOST" >&2
exit 2
fi
setup_proxy_only
;;
show)
if ! rules="$(nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null)"; then
echo "mode: allow-all"
elif printf '%s' "$rules" | grep -q "dport $PROXY_PORT"; then
echo "mode: Eval proxy-only"
echo "proxy: $PROXY_HOST:$PROXY_PORT"
else
echo "mode: Eval no-network"
fi
;;
rules)
nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || echo "mode: allow-all"
;;
*)
echo "Usage: network-policy {show|allow-all|deny-all|allow|rules}" >&2
exit 2
;;
esac
+435
View File
@@ -0,0 +1,435 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Fail-closed URL contamination filter for Eval subject egress."""
from __future__ import annotations
import json
import os
import re
import time
from pathlib import Path
from urllib.parse import unquote, urlsplit
PINNED_REVISION = "d49e28f1e4ddd13d289e85a5f312a66750951932"
MAX_DECODE_PASSES = 4
MAX_AUDIT_BYTES = 1024 * 1024
AUDIT_PATH = Path(
os.environ.get("MAKA_EVAL_EGRESS_AUDIT", "/opt/maka-egress-state/hits.jsonl")
)
PERCENT_ESCAPE = re.compile(r"%(?![0-9a-fA-F]{2})")
TERMINAL_BENCH = re.compile(r"terminal[-_.%/+\s]*bench", re.IGNORECASE)
def contamination_rule(raw_url: str) -> tuple[str, str, str] | None:
normalized = normalize_url(raw_url)
url = urlsplit(normalized)
host = (url.hostname or "").lower().rstrip(".")
path_query = f"{url.path}?{url.query}" if url.query else url.path
lowered = path_query.lower()
# Search the host and the path separately. A benchmark name in the hostname
# is a contamination surface, and searching the two fields joined would let
# a rule match across their boundary.
def anywhere(needle: str) -> bool:
return needle in host or needle in lowered
if host == "r.jina.ai":
inner = unquote(url.path.lstrip("/"))
if inner.startswith(("http://", "https://")):
nested = contamination_rule(inner)
if nested:
return (f"jina_recursive:{nested[0]}", host, path_query)
if anywhere(PINNED_REVISION):
return ("pinned_revision", host, path_query)
if host == "tbench.ai" or host.endswith(".tbench.ai"):
return ("tbench_domain", host, path_query)
if host == "hub.harborframework.com" and "/tasks/terminal-bench" in lowered:
return ("harbor_task_registry", host, path_query)
if benchmark_repository(host, lowered):
return ("benchmark_repository", host, path_query)
if public_trajectory_repository(host, lowered):
return ("public_trajectory", host, path_query)
if anywhere("patches-terminalbench-"):
return ("known_patch_artifact", host, path_query)
if TERMINAL_BENCH.search(host) or TERMINAL_BENCH.search(lowered):
return ("terminal_bench_url", host, path_query)
return None
def normalize_url(raw_url: str) -> str:
value = raw_url.strip()
if not value:
raise ValueError("empty URL")
for _ in range(MAX_DECODE_PASSES):
if PERCENT_ESCAPE.search(value):
raise ValueError("malformed percent escape")
decoded = unquote(value)
if decoded == value:
break
value = decoded
else:
raise ValueError("URL exceeded decode limit")
parsed = urlsplit(value)
if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname:
raise ValueError("unsupported URL")
return value
def benchmark_repository(host: str, path_query: str) -> bool:
repositories = (
"harbor-framework/terminal-bench",
"terminal-benchmarks/terminal-bench",
"tbench-ai/terminal-bench",
)
return host in {
"github.com",
"api.github.com",
"raw.githubusercontent.com",
"codeload.github.com",
} and any(repository in path_query for repository in repositories)
def public_trajectory_repository(host: str, path_query: str) -> bool:
return (
host in {"github.com", "api.github.com", "raw.githubusercontent.com", "huggingface.co"}
and "hqeric/maka-eval-trajectories" in path_query
)
try:
from mitmproxy import http
except ImportError:
http = None
try:
from mitmproxy.proxy import commands as proxy_commands
from mitmproxy.proxy.layer import Layer
from mitmproxy.proxy.layers import ClientTLSLayer, ServerTLSLayer
from mitmproxy.proxy.layers.tcp import TCPLayer
except ImportError:
proxy_commands = None
Layer = object
ClientTLSLayer = None
ServerTLSLayer = None
TCPLayer = None
try:
from mitmproxy.net.tls import starts_like_tls_record
except ImportError:
def starts_like_tls_record(data: bytes) -> bool:
return len(data) >= 3 and data[0] == 0x16 and data[1] == 0x03
def configure(updated: object) -> None:
# HTTP 101 upgrades construct TCPLayer inside the HTTP layer without
# another next_layer hook. rawtcp=false makes that path CloseConnection
# itself; WebSocket upgrades stay on the websocket layer.
try:
from mitmproxy import ctx
except ImportError:
return
if getattr(ctx.options, "rawtcp", False):
ctx.options.rawtcp = False
def request(flow: object) -> None:
apply_http_policy(flow, flow.request.pretty_url)
def response(flow: object) -> None:
response = getattr(flow, "response", None)
if getattr(response, "status_code", None) != 101:
return
# mitmproxy 12.2.3 sets flow.websocket before HttpResponseHook only when
# the 101 is a real WebSocket upgrade (Upgrade + version 13 + option on).
# A websocket Upgrade header alone still falls through to CloseConnection
# under rawtcp=false and must be audited.
if getattr(flow, "websocket", None) is not None:
return
record_raw_tunnel(flow)
def http_connect(flow: object) -> None:
try:
raw_url = connect_target_url(flow)
except Exception:
raw_url = ""
apply_http_policy(flow, raw_url)
def tcp_start(flow: object) -> None:
# Last resort if a TCPLayer is still admitted (tcp_hosts / ignore).
# CONNECT raw is closed by next_layer → CloseRawLayer; HTTP 101 raw is
# closed by rawtcp=false. Neither of those paths starts a TCPLayer.
record_raw_tunnel(flow)
kill_flow(flow)
def tcp_message(flow: object) -> None:
messages = getattr(flow, "messages", None)
if messages:
messages[-1].content = b""
kill_flow(flow)
def next_layer(nextlayer: object) -> None:
# Script addons run before the built-in classifier assigns layer. If we
# set CloseRawLayer here, NextLayer leaves it in place. Waiting for
# isinstance(..., TCPLayer) never fires on the production CONNECT path.
current = getattr(nextlayer, "layer", None)
context = getattr(nextlayer, "context", None)
if current is None:
data_client = _next_layer_bytes(nextlayer, "data_client")
if _is_fragmented_tls_record_prefix(data_client):
if ClientTLSLayer is None or ServerTLSLayer is None:
return
server_tls = ServerTLSLayer(context)
server_tls.child_layer = ClientTLSLayer(context)
nextlayer.layer = server_tls
return
if not looks_like_raw_tcp(nextlayer):
return
record_raw_tunnel(context)
nextlayer.layer = CloseRawLayer(context)
return
if TCPLayer is None or not isinstance(current, TCPLayer):
return
record_raw_tunnel(context)
closer = CloseRawLayer(context)
replace_layer(context, current, closer)
nextlayer.layer = closer
def apply_http_policy(flow: object, raw_url: str) -> None:
if http is None:
raise RuntimeError("mitmproxy is required to run the Eval egress filter")
try:
matched = contamination_rule(raw_url)
if not matched:
return
rule_id, host, normalized_path = matched
append_audit(rule_id, host, normalized_path)
flow.response = blocked_response(rule_id)
except Exception as error:
flow.response = http.Response.make(
503,
b"Eval egress policy could not classify this request.\n",
{
"Content-Type": "text/plain; charset=utf-8",
"X-Maka-Eval-Egress-Rule": "policy_error",
},
)
try:
append_audit("policy_error", "", type(error).__name__)
except Exception:
pass
def connect_target_url(flow: object) -> str:
request = flow.request
host = (getattr(request, "pretty_host", None) or getattr(request, "host", "") or "").strip()
if not host:
raise ValueError("empty CONNECT host")
if ":" in host and not host.startswith("["):
host = f"[{host}]"
port = getattr(request, "port", None)
if port in (None, 443):
return f"https://{host}/"
if port == 80:
return f"http://{host}/"
return f"https://{host}:{port}/"
def looks_like_raw_tcp(nextlayer: object) -> bool:
"""Close only when the bytes cannot still become HTTP or TLS.
Script next_layer runs before mitmproxy 12.2.3's classifier. Copying its
`probably_no_http` test here would treat `GET` / `GET ` as raw and assign
CloseRawLayer while the request line is still arriving. With rawtcp=false
the built-in path would have kept waiting for HttpLayer.
"""
data_client = _next_layer_bytes(nextlayer, "data_client")
data_server = _next_layer_bytes(nextlayer, "data_server")
if _could_start_tls_record(data_client):
return False
if not data_client and not data_server:
return False
if data_server or data_client.startswith(b"SSH"):
return True
return not _still_could_be_http(data_client)
def _could_start_tls_record(data: bytes) -> bool:
"""Keep a fragmented ClientHello undecided until its 3-byte prefix exists."""
if starts_like_tls_record(data):
return True
return _is_fragmented_tls_record_prefix(data)
def _is_fragmented_tls_record_prefix(data: bytes) -> bool:
return 0 < len(data) < 3 and b"\x16\x03".startswith(data)
def _still_could_be_http(data: bytes) -> bool:
first_line, newline, _rest = data.partition(b"\n")
line = first_line.rstrip(b"\r")
method, space, _remainder = line.partition(b" ")
if not method.isascii() or not method.isalpha():
return False
if newline and not space:
return False
return True
def _next_layer_bytes(nextlayer: object, name: str) -> bytes:
getter = getattr(nextlayer, name, None)
if not callable(getter):
return b""
try:
data = getter()
except Exception:
return b""
return bytes(data) if isinstance(data, (bytes, bytearray)) else b""
def tcp_peer(flow: object) -> tuple[str, str]:
server = getattr(flow, "server_conn", None) or getattr(flow, "server", None)
address = getattr(server, "address", None) if server is not None else None
if isinstance(address, (tuple, list)) and address:
host = str(address[0])[:255]
port = address[1] if len(address) > 1 else ""
return host, f":{port}" if port != "" else ""
return "", ""
def record_raw_tunnel(flow: object) -> None:
host, path = tcp_peer(flow)
try:
append_audit("raw_tunnel", host, path)
except Exception:
pass
def kill_flow(flow: object) -> None:
kill = getattr(flow, "kill", None)
if callable(kill) and getattr(flow, "killable", True):
try:
kill()
except Exception:
pass
def replace_layer(context: object, current: object, closer: object) -> None:
layers = getattr(context, "layers", None)
if not isinstance(layers, list):
return
try:
index = layers.index(current)
except ValueError:
index = len(layers)
if closer in layers:
layers.remove(closer)
if current in layers:
layers.remove(current)
layers.insert(min(index, len(layers)), closer)
class CloseRawLayer(Layer):
def __init__(self, context: object) -> None:
if Layer is object:
self.context = context
return
if getattr(context, "layers", None) is None:
context.layers = []
if getattr(context, "options", None) is None:
context.options = type("Options", (), {"proxy_debug": False})()
super().__init__(context)
def handle_event(self, event: object):
if proxy_commands is None:
return
yield
for name in ("client", "server"):
connection = getattr(self.context, name, None)
if connection is not None:
yield proxy_commands.CloseConnection(connection)
def blocked_response(rule_id: str):
return http.Response.make(
451,
b"Benchmark source or public solution access is blocked during evaluation.\n",
{
"Content-Type": "text/plain; charset=utf-8",
"X-Maka-Eval-Egress-Rule": rule_id,
},
)
def append_audit(rule_id: str, host: str, normalized_path: str) -> None:
AUDIT_PATH.parent.mkdir(parents=True, exist_ok=True)
if AUDIT_PATH.exists() and AUDIT_PATH.stat().st_size >= MAX_AUDIT_BYTES:
write_truncation_marker()
return
record = {
"ts": int(time.time() * 1000),
"ruleId": rule_id,
"host": host[:255],
"normalizedPath": normalized_path[:4096],
}
with AUDIT_PATH.open("a", encoding="utf-8") as stream:
stream.write(json.dumps(record, ensure_ascii=True, separators=(",", ":")) + "\n")
def write_truncation_marker() -> None:
if audit_already_truncated():
return
record = {
"ts": int(time.time() * 1000),
"ruleId": "audit_truncated",
"host": "",
"normalizedPath": "",
}
prefix = ""
if AUDIT_PATH.exists() and AUDIT_PATH.stat().st_size > 0:
with AUDIT_PATH.open("rb") as stream:
stream.seek(-1, os.SEEK_END)
if stream.read(1) != b"\n":
prefix = "\n"
with AUDIT_PATH.open("a", encoding="utf-8") as stream:
stream.write(prefix + json.dumps(record, ensure_ascii=True, separators=(",", ":")) + "\n")
def audit_already_truncated() -> bool:
if not AUDIT_PATH.exists():
return False
with AUDIT_PATH.open("rb") as stream:
stream.seek(0, os.SEEK_END)
size = stream.tell()
stream.seek(max(0, size - 4096))
tail = stream.read().decode("utf-8", errors="ignore")
lines = [line for line in tail.splitlines() if line.strip()]
if not lines:
return False
try:
parsed = json.loads(lines[-1])
except json.JSONDecodeError:
return False
return isinstance(parsed, dict) and parsed.get("ruleId") == "audit_truncated"
+40
View File
@@ -0,0 +1,40 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Authoritative Eval harness framework selection.
`run_trial.py` validates the argv framework and installs it here before Harbor,
Pier, or the shared relay can import. The relay must not read the environment.
"""
from __future__ import annotations
_FRAMEWORKS = frozenset({"harbor", "pier"})
_framework: str | None = None
def install(framework: str) -> None:
if framework not in _FRAMEWORKS:
raise RuntimeError("framework must be harbor or pier")
global _framework
_framework = framework
def selected() -> str:
if _framework not in _FRAMEWORKS:
raise RuntimeError("Eval framework selection is not installed")
return _framework
+686
View File
@@ -0,0 +1,686 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""One-shot Harbor/Pier Agent that delegates one subject execution to @maka/eval."""
from __future__ import annotations
import asyncio
import base64
import contextlib
import hashlib
import json
import os
import re
import shlex
import tempfile
from pathlib import Path
from typing import Any
from eval_framework import selected
framework = selected()
if framework == "harbor":
from harbor.agents.base import BaseAgent
else:
from pier.agents.base import BaseAgent
class RelayTransportClosed(RuntimeError):
pass
RESULT_FRAME_PREFIX = "MAKA-EVAL-RESULT-V1"
SCOPE_ERROR_PREFIX = "MAKA-EVAL-SCOPE-ERROR-V1"
RESULT_PAYLOAD_LIMIT_BYTES = 2 * 1024
RESULT_CARRIER_LIMIT_BYTES = 64 * 1024
SUBJECT_STDOUT_PATH = "/logs/artifacts/maka-subject.stdout.txt"
SUBJECT_STDERR_PATH = "/logs/artifacts/maka-subject.stderr.txt"
CAPABILITY_PREFIX = "MAKA-EVAL-CAPABILITIES-V1 "
NAMESPACE_PREFIX = "MAKA-EVAL-POLICY-NAMESPACE-V1 "
# The egress policy only constrains what the IP output hooks can see. NET_RAW
# grants an AF_PACKET socket that writes beneath them, and NET_ADMIN grants the
# ability to delete the ruleset outright. The overlay drops NET_RAW, but a
# task's own compose can add either back, and a `cap_add` wins over an
# overlay's `cap_drop`.
BYPASS_CAPABILITIES = {"NET_RAW": 1 << 13, "NET_ADMIN": 1 << 12}
# Every set is read, not just the effective one: a non-root subject reports an
# empty effective set while a file-capability executable still reacquires
# anything the bounding set kept, and the permitted, inheritable and ambient
# sets each raise into the effective set without an exec at all.
CAPABILITY_FIELDS = ("CapEff", "CapPrm", "CapBnd")
# Harbor installs the policy by running `network-policy` inside this service, so
# its network namespace is what "the namespace the policy was applied to" means.
# Reading the identity from both sides answers that question directly, rather
# than inferring it from something only the shared namespace would expose.
POLICY_SERVICE = "harbor-docker-egress-control-sidecar"
NAMESPACE_PROBE = f"printf %s {shlex.quote(NAMESPACE_PREFIX)}; readlink /proc/self/ns/net"
PROXY_IPV4_PATH = "/opt/maka-egress/proxy-ipv4"
PROXY_HOSTS_PREFIX = "MAKA-EVAL-PROXY-HOST-V1 "
# The kernel's own form for a namespace link target. Matching it keeps an
# unexpected answer from being compared as if it were an identity.
NAMESPACE_IDENTITY = re.compile(r"^net:\[[0-9]+\]$")
_host_teardown_requested = False
def request_host_teardown() -> None:
global _host_teardown_requested
_host_teardown_requested = True
class RelayAgent(BaseAgent):
def __init__(
self,
*args: Any,
relay_host: str,
relay_port: int,
relay_token: str,
teardown_timeout_ms: int,
**kwargs: Any,
):
super().__init__(*args, **kwargs)
self._host = relay_host
self._port = relay_port
self._token = relay_token
if not isinstance(teardown_timeout_ms, int) or teardown_timeout_ms <= 0:
raise RuntimeError("Maka Eval teardown timeout is invalid")
self._teardown_timeout = teardown_timeout_ms / 1000
@staticmethod
def name() -> str:
return "maka-eval-relay"
def version(self) -> str:
return "1"
async def setup(self, environment: Any) -> None:
return None
async def run(self, instruction: str, environment: Any, context: Any) -> None:
reader, writer = await asyncio.open_connection(self._host, self._port)
execution: asyncio.Task[Any] | None = None
decision: asyncio.Task[dict[str, Any]] | None = None
request: dict[str, Any] | None = None
execution_reported = False
scope_path = f"/logs/agent/.maka-eval-{self._token}.pid"
environment_path = f"/tmp/maka-eval-{self._token}.env"
try:
cwd_prefix = f"MAKA-EVAL-CWD-V1 {self._token} "
working_directory = await environment.exec(
f"printf {shlex.quote(cwd_prefix)}; pwd 2>/dev/null"
)
cwd_lines = [
line[len(cwd_prefix) :]
for line in str(working_directory.stdout or "").splitlines()
if line.startswith(cwd_prefix)
]
cwd = cwd_lines[0] if len(cwd_lines) == 1 else ""
if working_directory.return_code != 0 or not cwd.startswith("/") or "\x00" in cwd:
raise RuntimeError("Maka Eval could not resolve the task working directory")
await _require_constrained_subject(environment)
if not await _send(
writer,
{
"token": self._token,
"kind": "ready",
"instruction": instruction,
"cwd": cwd,
},
):
raise RelayTransportClosed("Maka Eval relay transport closed before ready")
request = await _receive(reader)
_require_message(request, self._token, "execute")
command = await _prepare_command(environment, request, self._token, scope_path)
execution = asyncio.create_task(environment.exec(command, cwd=cwd))
decision = asyncio.create_task(_receive(reader))
done, _ = await asyncio.wait({execution, decision}, return_when=asyncio.FIRST_COMPLETED)
if decision in done and execution not in done:
decision.result()
raise RelayTransportClosed("Maka Eval relay received control before execution")
result = execution.result()
await _persist_subject_outputs(environment, result)
stdout, diagnostic = _project_result(result, request)
# A subject that exited on its own leaves the shared environment as
# it left it, and the verifier reads that environment. Nothing is
# waiting on those processes here — `environment.exec` has already
# returned — so tearing them down would not unblock anything; it
# would only edit the thing about to be measured, and edit it for
# some subjects and not others. Cancellation still quiesces, because
# there the subject has not stopped and the trial is being abandoned.
if not await _send(
writer,
{
"token": self._token,
"kind": "executed",
"termination": "exited",
"exitCode": result.return_code,
"stdout": stdout,
"diagnostic": diagnostic,
},
):
raise RelayTransportClosed("Maka Eval relay transport closed before result")
execution_reported = True
_require_message(
decision.result() if decision.done() else await decision,
self._token,
"verify",
)
except asyncio.CancelledError:
if request is not None and execution is not None:
execution_terminal = execution.done() and not execution.cancelled()
terminal_projection = None
if execution_terminal:
terminal_result = execution.result()
terminal_projection = _project_result(terminal_result, request)
# A subject that already exited has nothing left to settle, and
# tearing its scope down here would remove what the verifier is
# about to score -- the same environment edit this relay stopped
# making on the ordinary path. Only a subject still running is
# brought to a stop.
if terminal_projection is not None:
result = terminal_result
else:
result = await _settle_or_destroy(
environment, cwd, scope_path, execution, self._teardown_timeout
)
if result is not None:
await _persist_subject_outputs(environment, result)
if (
result is not None
and not execution_reported
and (execution_terminal or not _host_teardown_requested)
):
stdout, diagnostic = terminal_projection or _project_result(result, request)
with contextlib.suppress(Exception):
await _send(
writer,
{
"token": self._token,
"kind": "executed",
"termination": "exited" if execution_terminal else "framework_timeout",
"exitCode": result.return_code if execution_terminal else 124,
"stdout": stdout,
"diagnostic": diagnostic,
},
)
elif not execution_reported and not _host_teardown_requested:
with contextlib.suppress(Exception):
await _send(
writer,
{
"token": self._token,
"kind": "executed",
"termination": "framework_timeout",
"exitCode": 124,
"stdout": "",
"diagnostic": (
_carrier_diagnostic("result-frame-missing", b"")
if request.get("captureStdout", True)
else {"category": "none"}
),
},
)
raise
except RelayTransportClosed:
if request is not None and execution is not None:
result = await _settle_or_destroy(
environment, cwd, scope_path, execution, self._teardown_timeout
)
if result is not None:
with contextlib.suppress(Exception):
await _persist_subject_outputs(environment, result)
except BaseException:
if request is not None and execution is not None:
result = await _settle_or_destroy(
environment, cwd, scope_path, execution, self._teardown_timeout
)
if result is not None:
with contextlib.suppress(Exception):
await _persist_subject_outputs(environment, result)
raise
finally:
if decision is not None and not decision.done():
decision.cancel()
with contextlib.suppress(BaseException):
await decision
if request is not None:
with contextlib.suppress(Exception):
await asyncio.wait_for(
environment.exec(
f"rm -f -- {shlex.quote(scope_path)} {shlex.quote(environment_path)}",
cwd=cwd,
timeout_sec=1,
),
timeout=1,
)
with contextlib.suppress(BrokenPipeError, ConnectionError, RuntimeError, TimeoutError):
writer.close()
await asyncio.wait_for(writer.wait_closed(), timeout=1)
async def _prepare_command(
environment: Any,
request: dict[str, Any],
token: str,
scope_path: str,
) -> str:
credentials = request.get("credentials")
public_environment = request.get("environment", {})
if not isinstance(credentials, dict) or not all(
isinstance(key, str) and isinstance(value, str) for key, value in credentials.items()
):
raise RuntimeError("invalid Maka Eval credentials")
if not isinstance(public_environment, dict) or not all(
isinstance(key, str) and isinstance(value, str)
for key, value in public_environment.items()
):
raise RuntimeError("invalid Maka Eval environment")
if set(credentials) & set(public_environment):
raise RuntimeError("Maka Eval environment overlaps credentials")
capture_stdout = request.get("captureStdout", True)
if not isinstance(capture_stdout, bool):
raise RuntimeError("invalid Maka Eval stdout policy")
result_token = request.get("resultToken")
if not isinstance(result_token, str) or re.fullmatch(r"[0-9a-f]{32}", result_token) is None:
raise RuntimeError("invalid Maka Eval result token")
if "MAKA_EVAL_RESULT_TOKEN" in credentials or "MAKA_EVAL_RESULT_TOKEN" in public_environment:
raise RuntimeError("Maka Eval environment contains a reserved name")
for label, values in (("environment", public_environment), ("credential", credentials)):
if any(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) is None for key in values):
raise RuntimeError(f"invalid Maka Eval {label} name")
container_path = f"/tmp/maka-eval-{token}.env"
secret_path = None
try:
with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as secret:
secret_path = Path(secret.name)
os.chmod(secret_path, 0o600)
for key, value in {
**public_environment,
**credentials,
"MAKA_EVAL_RESULT_TOKEN": result_token,
}.items():
secret.write(f"export {key}={shlex.quote(value)}\n")
await environment.upload_file(secret_path, container_path)
finally:
if secret_path is not None:
secret_path.unlink(missing_ok=True)
subject = shlex.join([request["command"], *request["args"]])
output_redirect = "" if capture_stdout else " >/dev/null"
scope_error = shlex.quote(f"{SCOPE_ERROR_PREFIX} {result_token}\\n")
inner = (
"umask 077; "
f"{{ echo $$ > {shlex.quote(scope_path)}; }} 2>/dev/null || "
f"{{ printf {scope_error}; exit 111; }}; "
f". {shlex.quote(container_path)}; command -p rm -f {shlex.quote(container_path)}; "
f"exec {subject}{output_redirect}"
)
return f"setsid --wait sh -c {shlex.quote(inner)}"
async def _require_constrained_subject(environment: Any) -> None:
"""Refuse to start the subject unless the egress policy actually governs it.
Runs after the policy is applied and before the subject exists, so a task
that keeps a capability the policy cannot see, or that keeps the subject out
of the namespace the policy was applied to, fails the attempt instead of
producing a result the isolation contract never actually covered.
"""
if os.environ.get("MAKA_EVAL_EGRESS_REQUIRED") != "1":
return
probe = await environment.exec(
f"printf %s {shlex.quote(CAPABILITY_PREFIX)}; "
r"sed -n 's/^\(Cap[A-Za-z]*\):[[:space:]]*/\1=/p' /proc/self/status | tr '\n' ' '; "
"echo; " + NAMESPACE_PROBE
)
policy = await environment.service_exec(NAMESPACE_PROBE, service=POLICY_SERVICE)
if probe.return_code != 0 or policy.return_code != 0:
raise RuntimeError("Maka Eval could not read the subject isolation evidence")
capabilities = _sole_probe_line(probe, CAPABILITY_PREFIX)
namespace = _sole_probe_line(probe, NAMESPACE_PREFIX)
policy_namespace = _sole_probe_line(policy, NAMESPACE_PREFIX)
if not all(NAMESPACE_IDENTITY.match(value) for value in (namespace, policy_namespace)):
raise RuntimeError("Maka Eval could not read the subject isolation evidence")
reported: dict[str, int] = {}
for field in capabilities.split():
name, _, value = field.partition("=")
try:
reported[name] = int(value, 16)
except ValueError:
raise RuntimeError("Maka Eval could not read the subject capability set") from None
if not all(field in reported for field in CAPABILITY_FIELDS):
raise RuntimeError("Maka Eval could not read the subject capability set")
granted = 0
for value in reported.values():
granted |= value
held = sorted(name for name, bit in BYPASS_CAPABILITIES.items() if granted & bit)
if held:
raise RuntimeError(
"the subject holds "
+ ", ".join(held)
+ ", which bypasses the Eval egress policy; remove it from the task"
)
if namespace != policy_namespace:
raise RuntimeError(
"the subject does not share the network namespace the Eval egress policy "
"was applied to; remove the task's own networking on the subject service"
)
await _pin_proxy_hostname(environment)
HOSTS_ALIAS_AWK = r"""
/^[[:space:]]*#/ { print; next }
{
for (i = 2; i <= NF; i++) {
if ($i == host) next
}
print
}
"""
# Four decimal octets 0-255, no empty fields and no leading zeros. The same
# program is embedded in egress-proxy/network-policy; the contract test
# requires the two copies to stay identical.
IPV4_OCTET_AWK = r"""
BEGIN { FS = "." }
NF != 4 { exit 1 }
{
for (i = 1; i <= 4; i++) {
if ($i !~ /^(0|[1-9][0-9]*)$/ || $i + 0 > 255) exit 1
}
}
"""
_PUBLISHED_IPV4 = re.compile(
r"^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}"
r"(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$"
)
def _valid_egress_proxy_host(host: str) -> bool:
if not host or host.startswith(".") or host.endswith(".") or ".." in host:
return False
# Match the pin script's `*[!A-Za-z0-9.-]*` class. str.isalnum() also
# accepts Unicode letters, which the shell would later reject as a pin
# failure instead of an invalid host.
return all(
(character.isalnum() and character.isascii()) or character in ".-"
for character in host
)
def _valid_published_ipv4(ip: str) -> bool:
return bool(_PUBLISHED_IPV4.fullmatch(ip))
async def _pin_proxy_hostname(environment: Any) -> None:
"""Point the proxy hostname at the published IPv4 so Docker DNS can be refused.
The subject only needs that one name, because the HTTP proxy does remote
resolution itself. Pinning it in /etc/hosts lets the namespace policy reject
127.0.0.11:53 without breaking HTTPS_PROXY.
"""
host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST") or "maka-eval-mitmproxy"
if not _valid_egress_proxy_host(host):
raise RuntimeError("Eval egress proxy host is invalid")
script = f"""
set -eu
path={shlex.quote(PROXY_IPV4_PATH)}
host={shlex.quote(host)}
case "$host" in
""|.*|*..*|*.) exit 1 ;;
*[!A-Za-z0-9.-]*) exit 1 ;;
esac
test -f "$path"
ip=$(tr -d ' \\t\\r\\n' < "$path")
case "$ip" in
""|*.*.*.*.*|*[!0-9.]*) exit 1 ;;
esac
if ! printf '%s\\n' "$ip" | awk {shlex.quote(IPV4_OCTET_AWK)}; then
exit 1
fi
tmp=$(mktemp)
trap 'rm -f "$tmp"' EXIT
if [ -f /etc/hosts ]; then
awk -v host="$host" {shlex.quote(HOSTS_ALIAS_AWK)} /etc/hosts > "$tmp"
fi
printf '%s %s\\n' "$ip" "$host" >> "$tmp"
cat "$tmp" > /etc/hosts
printf %s {shlex.quote(PROXY_HOSTS_PREFIX)}
printf '%s %s\\n' "$ip" "$host"
"""
# Harbor 0.20.0 BaseEnvironment.exec and DockerEnvironment.exec take user=;
# DockerEnvironment._compose_exec forwards it as `docker compose exec -u`.
probe = await environment.exec(script, user="root")
if probe.return_code != 0:
raise RuntimeError("Maka Eval could not pin the Eval egress proxy hostname")
reported = _sole_probe_line(probe, PROXY_HOSTS_PREFIX)
ip, _, pinned = reported.partition(" ")
if pinned != host or not _valid_published_ipv4(ip):
raise RuntimeError("Maka Eval could not pin the Eval egress proxy hostname")
def _sole_probe_line(probe: Any, prefix: str) -> str:
reported = [
line[len(prefix) :]
for line in str(probe.stdout or "").splitlines()
if line.startswith(prefix)
]
if len(reported) != 1:
raise RuntimeError("Maka Eval could not read the subject isolation evidence")
return reported[0].strip()
async def _persist_subject_outputs(environment: Any, result: Any) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
stdout = root / "stdout"
stderr = root / "stderr"
stdout.write_text(str(getattr(result, "stdout", "") or ""), encoding="utf-8")
stderr.write_text(str(getattr(result, "stderr", "") or ""), encoding="utf-8")
prepared = await environment.exec("mkdir -p /logs/artifacts && chmod 700 /logs/artifacts")
if prepared.return_code != 0:
raise RuntimeError("Maka Eval could not prepare subject artifact output")
await environment.upload_file(stdout, SUBJECT_STDOUT_PATH)
await environment.upload_file(stderr, SUBJECT_STDERR_PATH)
def _decode_result_carrier(carrier: str, token: str) -> tuple[str, dict[str, Any]]:
raw = carrier.encode("utf-8", errors="replace")
if len(raw) > RESULT_CARRIER_LIMIT_BYTES:
return "", _carrier_diagnostic("result-frame-oversize", raw)
prefix = f"{RESULT_FRAME_PREFIX} {token} "
candidates = [line for line in carrier.splitlines(keepends=True) if line.startswith(prefix)]
if len(candidates) != 1:
category = "result-frame-missing" if not candidates else "result-frame-ambiguous"
return "", _carrier_diagnostic(category, raw)
frame = candidates[0]
fields = frame.rstrip("\r\n").split(" ", 4)
if len(fields) != 5:
return "", _carrier_diagnostic("result-frame-invalid", raw)
_, framed_token, length_text, digest, encoded = fields
try:
length = int(length_text)
padding = "=" * (-len(encoded) % 4)
payload = base64.b64decode(encoded + padding, altchars=b"-_", validate=True)
except (ValueError, base64.binascii.Error):
return "", _carrier_diagnostic("result-frame-invalid", raw)
if (
framed_token != token
or length < 0
or length > RESULT_PAYLOAD_LIMIT_BYTES
or len(payload) != length
or not re.fullmatch(r"[0-9a-f]{64}", digest)
or hashlib.sha256(payload).hexdigest() != digest
):
return "", _carrier_diagnostic("result-frame-invalid", raw)
noise = carrier.replace(frame, "", 1).encode("utf-8", errors="replace")
diagnostic = (
{"category": "none"}
if not noise
else _carrier_diagnostic("unstructured-output", noise)
)
try:
decoded = payload.decode("utf-8", errors="strict")
except UnicodeDecodeError:
return "", _carrier_diagnostic("result-frame-invalid", raw)
return decoded, diagnostic
def _project_result(result: Any, request: dict[str, Any]) -> tuple[str, dict[str, Any]]:
carrier = str(getattr(result, "stdout", "") or "")
scope_error = f"{SCOPE_ERROR_PREFIX} {request['resultToken']}"
if carrier == f"{scope_error}\n":
return "", _carrier_diagnostic("execution-scope-unavailable", carrier.encode())
if not request.get("captureStdout", True):
return "", {"category": "none"}
return _decode_result_carrier(carrier, request["resultToken"])
def _carrier_diagnostic(category: str, value: bytes) -> dict[str, Any]:
return {
"category": category,
"bytes": len(value),
"sha256": hashlib.sha256(value).hexdigest(),
}
async def _settle(environment: Any, cwd: str, scope_path: str, execution: Any) -> Any:
if execution.cancelled():
raise RuntimeError("Maka Eval subject execution was cancelled")
if execution.done():
result = execution.result()
else:
result = None
for signal, timeout in (("TERM", 20), ("KILL", 10)):
await _signal(environment, cwd, scope_path, signal)
try:
result = await asyncio.wait_for(asyncio.shield(execution), timeout=timeout)
break
except asyncio.CancelledError:
if execution.cancelled():
raise RuntimeError("Maka Eval subject execution was cancelled") from None
raise
except TimeoutError:
pass
if result is None:
raise RuntimeError("Maka Eval subject did not settle")
await _quiesce_scope(environment, cwd, scope_path)
return result
async def _settle_or_destroy(
environment: Any,
cwd: str,
scope_path: str,
execution: Any,
timeout: float,
) -> Any | None:
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
stop_reserve = min(20.0, timeout * 0.2)
try:
return await asyncio.wait_for(
_settle(environment, cwd, scope_path, execution),
timeout=max(0.001, deadline - loop.time() - stop_reserve),
)
except Exception:
try:
remaining = max(0.001, deadline - loop.time())
await asyncio.wait_for(environment.stop(delete=True), timeout=remaining)
except Exception:
pass
finally:
if not execution.done():
execution.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
remaining = max(0.001, deadline - loop.time())
await asyncio.wait_for(execution, timeout=remaining)
return None
async def _signal(environment: Any, cwd: str, scope_path: str, signal: str) -> None:
command = (
f"pgid=$(cat {shlex.quote(scope_path)} 2>/dev/null) || exit 0; "
"case $pgid in ''|0|*[!0-9]*) exit 0;; esac; "
f"kill -{signal} -- \"-$pgid\""
)
with contextlib.suppress(Exception):
await environment.exec(
command,
cwd=cwd,
timeout_sec=5,
)
async def _quiesce_scope(environment: Any, cwd: str, scope_path: str) -> None:
if not await _scope_active(environment, cwd, scope_path):
return
for signal, timeout in (("TERM", 10), ("KILL", 5)):
await _signal(environment, cwd, scope_path, signal)
deadline = asyncio.get_running_loop().time() + timeout
while asyncio.get_running_loop().time() < deadline:
if not await _scope_active(environment, cwd, scope_path):
return
await asyncio.sleep(0.1)
raise RuntimeError("Maka Eval execution scope did not quiesce")
async def _scope_active(environment: Any, cwd: str, scope_path: str) -> bool:
result = await environment.exec(
f"pgid=$(cat {shlex.quote(scope_path)} 2>/dev/null) || exit 4; "
"case $pgid in ''|0|*[!0-9]*) exit 4;; esac; "
"kill -0 -- \"-$pgid\" 2>/dev/null; status=$?; "
"if [ $status -eq 0 ]; then exit 0; fi; exit 3",
cwd=cwd,
timeout_sec=5,
)
if result.return_code == 0:
return True
if result.return_code == 3:
return False
raise RuntimeError("Maka Eval execution scope evidence was unavailable")
def _require_message(value: dict[str, Any], token: str, kind: str) -> None:
if value.get("token") != token or value.get("kind") != kind:
raise RuntimeError("invalid Maka Eval relay message")
async def _receive(reader: asyncio.StreamReader) -> dict[str, Any]:
try:
raw = await reader.readline()
if not raw:
raise RelayTransportClosed("Maka Eval relay peer closed")
value = json.loads(raw)
except (ConnectionError, json.JSONDecodeError, ValueError) as error:
raise RelayTransportClosed("Maka Eval relay message was unavailable") from error
if not isinstance(value, dict):
raise RelayTransportClosed("Maka Eval relay message was invalid")
return value
async def _send(writer: asyncio.StreamWriter, value: object) -> bool:
if writer.is_closing():
return False
try:
writer.write((json.dumps(value, separators=(",", ":")) + "\n").encode())
await writer.drain()
return True
except (BrokenPipeError, ConnectionError, RuntimeError):
return False
+223
View File
@@ -0,0 +1,223 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Run exactly one pinned Harbor or Pier Trial."""
import asyncio
import contextlib
import fcntl
import hashlib
import importlib
import importlib.metadata
import json
import os
import signal
import sys
from pathlib import Path
from typing import AsyncIterator
FRAMEWORK_VERSION_MISMATCH_EXIT_CODE = 78
class FrameworkVersionMismatch(Exception):
pass
TASK_CACHE_PROTOCOL_VERSION = 1
@contextlib.asynccontextmanager
async def task_cache_lock(cache_dir: Path, identity: str) -> AsyncIterator[None]:
lock_dir = cache_dir / ".maka-locks"
lock_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
lock_path = lock_dir / f"{hashlib.sha256(identity.encode()).hexdigest()}.lock"
flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(lock_path, flags, 0o600)
acquired = False
try:
os.fchmod(descriptor, 0o600)
while not acquired:
try:
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
acquired = True
except BlockingIOError:
await asyncio.sleep(0.05)
yield
finally:
if acquired:
fcntl.flock(descriptor, fcntl.LOCK_UN)
os.close(descriptor)
def ready_task(cache_dir: Path, identity: str) -> tuple[Path, Path | None]:
namespace = cache_dir / ".maka-tasks" / hashlib.sha256(identity.encode()).hexdigest()
marker = namespace / "ready.json"
try:
value = json.loads(marker.read_text())
if set(value) != {"version", "target"}:
return namespace, None
if value["version"] != TASK_CACHE_PROTOCOL_VERSION or not isinstance(
value["target"], str
):
return namespace, None
target = (namespace / value["target"]).resolve()
if (
not target.is_relative_to(namespace.resolve())
or not target.is_dir()
or not any(target.iterdir())
):
return namespace, None
return namespace, target
except (OSError, TypeError, ValueError):
return namespace, None
def publish_ready_task(namespace: Path, target: Path) -> None:
resolved_namespace = namespace.resolve()
resolved_target = target.resolve()
if not resolved_target.is_relative_to(resolved_namespace):
raise RuntimeError("Harbor task cache target escaped its Maka namespace")
relative_target = resolved_target.relative_to(resolved_namespace).as_posix()
marker = namespace / "ready.json"
temporary = namespace / ".ready.tmp"
descriptor = os.open(
temporary,
os.O_CREAT | os.O_WRONLY | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0),
0o600,
)
try:
with os.fdopen(descriptor, "w") as stream:
descriptor = -1
json.dump(
{"version": TASK_CACHE_PROTOCOL_VERSION, "target": relative_target},
stream,
)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, marker)
finally:
if descriptor >= 0:
os.close(descriptor)
temporary.unlink(missing_ok=True)
async def run_trial(framework: str, expected_version: str, config_file: Path) -> None:
from eval_framework import install
install(framework)
distribution = {"harbor": "harbor", "pier": "datacurve-pier"}[framework]
if importlib.metadata.version(distribution) != expected_version:
raise FrameworkVersionMismatch
try:
config_type = importlib.import_module(f"{framework}.models.trial.config").TrialConfig
trial_type = importlib.import_module(f"{framework}.trial.trial").Trial
config = config_type.model_validate_json(config_file.read_text())
config_file.unlink()
if framework == "harbor":
from harbor.constants import TASK_CACHE_DIR
task_identity = config.task.get_task_id().model_dump_json()
async with task_cache_lock(TASK_CACHE_DIR, task_identity):
namespace, ready = ready_task(TASK_CACHE_DIR, task_identity)
namespace.mkdir(parents=True, exist_ok=True, mode=0o700)
config.task.download_dir = namespace
config.task.overwrite = ready is None
trial = await create_harbor_trial(trial_type, config)
target = Path(trial.task.task_dir)
if ready is not None and target.resolve() != ready:
raise RuntimeError("Harbor task cache target changed for one identity")
if ready is None:
publish_ready_task(namespace, target)
else:
trial = await trial_type.create(config)
await trial.run()
finally:
config_file.unlink(missing_ok=True)
def apply_subject_egress_policy(task: object) -> None:
required = os.environ.get("MAKA_EVAL_EGRESS_REQUIRED") == "1"
allowed_host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST")
if not required:
return
if not allowed_host:
raise RuntimeError("required Eval egress proxy host is unavailable")
agent = task.config.agent
agent.network_mode = "allowlist"
agent.allowed_hosts = [allowed_host]
async def create_harbor_trial(trial_type: type, config: object) -> object:
trial_type._resolve_agent_skills(config)
task, task_download_result = await trial_type._load_task(config)
apply_subject_egress_policy(task)
if task.has_steps:
from harbor.trial.multi_step import MultiStepTrial
return MultiStepTrial(
config,
_task=task,
_task_download_result=task_download_result,
)
from harbor.trial.single_step import SingleStepTrial
return SingleStepTrial(
config,
_task=task,
_task_download_result=task_download_result,
)
async def main() -> None:
from eval_framework import install
framework, expected_version, config_path = sys.argv[1:]
install(framework)
task = asyncio.current_task()
assert task is not None
loop = asyncio.get_running_loop()
teardown_requested = False
def request_teardown() -> None:
nonlocal teardown_requested
if teardown_requested:
return
teardown_requested = True
try:
from relay_agent import request_host_teardown
request_host_teardown()
finally:
task.cancel()
for host_signal in (signal.SIGINT, signal.SIGTERM):
loop.add_signal_handler(host_signal, request_teardown)
try:
try:
await run_trial(framework, expected_version, Path(config_path))
except FrameworkVersionMismatch:
raise SystemExit(FRAMEWORK_VERSION_MISMATCH_EXIT_CODE) from None
except asyncio.CancelledError:
if not teardown_requested:
raise
finally:
for host_signal in (signal.SIGINT, signal.SIGTERM):
loop.remove_signal_handler(host_signal)
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,511 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Prove the cell namespace forces subject egress through the audited proxy.
The Python policy test only checks that `run_trial.py` hands Harbor the right
allowlist. This test brings up the checked-in Compose overlay against a minimal
Harbor-shaped base, applies the checked-in `network-policy` from a sidecar that
shares the subject namespace, and asserts what the README promises: explicit
proxy traffic works, everything else does not, and the subject never sees the
CA private key or the audit log.
It needs a working Docker daemon and outbound network, so it is opt-in:
MAKA_EVAL_EGRESS_NAMESPACE_TEST=1 python3 harbor/test_cell_egress_namespace.py
"""
from __future__ import annotations
import asyncio
import os
import shutil
import subprocess
import tempfile
import types
import unittest
from pathlib import Path
from unittest.mock import patch
from test_relay_contract import load_relay
HARBOR_DIR = Path(__file__).parent
OVERLAY = HARBOR_DIR / "docker-compose-egress-proxy.yaml"
NETWORK_POLICY = HARBOR_DIR / "egress-proxy" / "network-policy"
PROXY_IMAGE = "maka-eval-egress-proxy:12.2.3"
MAIN_IMAGE = "maka-eval-egress-namespace-main:test"
SIDECAR_IMAGE = "maka-eval-egress-namespace-sidecar:test"
PROJECT = "maka-eval-egress-namespace-test"
PROXY_HOST = "maka-eval-mitmproxy"
SIDECAR = "harbor-docker-egress-control-sidecar"
OUTSIDER = "maka-eval-outside-the-namespace"
BOUNDED = "maka-eval-bounded-capability"
CA_PATH = "/opt/maka-egress/mitmproxy-ca-cert.pem"
BUILD_TIMEOUT_S = 900
COMMAND_TIMEOUT_S = 120
BASE_COMPOSE = f"""\
services:
main:
image: {MAIN_IMAGE}
command: ["sleep", "infinity"]
{SIDECAR}:
image: {SIDECAR_IMAGE}
# Harbor's own sidecar runs gost on the port the policy redirects to, so a
# redirected connection is accepted and then goes nowhere. Standing in for
# it keeps the negative assertions honest: without a listener they would
# hold on a refused connection instead of on the policy.
command: ["sh", "-c", "nc -l -k 12345 >/dev/null 2>&1 & exec sleep infinity"]
network_mode: "service:main"
cap_add:
- NET_ADMIN
depends_on:
- main
# Two services the gate must refuse, so its live coverage is not only the
# positive case. This one keeps its own network namespace, the way Harbor
# leaves a subject whose task declares its own networking.
{OUTSIDER}:
image: {MAIN_IMAGE}
command: ["sleep", "infinity"]
cap_drop:
- NET_RAW
# This one shares the policy's namespace and reports an empty effective set,
# while the bounding set still lets a file-capability executable take NET_RAW
# back. Reading only the effective set would admit it.
{BOUNDED}:
image: {MAIN_IMAGE}
command: ["sleep", "infinity"]
network_mode: "service:main"
user: "1000"
depends_on:
- main
"""
MAIN_DOCKERFILE = """\
FROM python:3.12-slim
RUN apt-get update \\
&& apt-get install -y --no-install-recommends ca-certificates curl iputils-ping \\
&& rm -rf /var/lib/apt/lists/*
"""
# Debian bookworm ships nftables 1.0.6, which rejects the `dstnat` priority name
# on a nat output chain; the policy needs 1.1 or newer, as Harbor's sidecar has.
SIDECAR_DOCKERFILE = """\
FROM debian:trixie-slim
RUN apt-get update \\
&& apt-get install -y --no-install-recommends nftables netcat-openbsd \\
&& rm -rf /var/lib/apt/lists/* \\
&& mkdir -p /opt/egress-sidecar
"""
# The policy redirects TCP to the sidecar's proxy port rather than dropping it,
# so the connection still completes and only the response tells a direct route
# from a redirected one. Judging this on `connect` would assert nothing in a
# cell where that proxy is listening, which is every real one.
TCP_PROBE = (
"import socket\n"
"request = b'GET / HTTP/1.0\\r\\nHost: one.one.one.one\\r\\n\\r\\n'\n"
"try:\n"
" with socket.create_connection(('1.1.1.1', 80), 5) as sock:\n"
" sock.settimeout(5)\n"
" sock.sendall(request)\n"
" print('reachable' if sock.recv(16).startswith(b'HTTP/') else 'blocked')\n"
"except OSError:\n"
" print('blocked')\n"
)
# A well-formed query for example.com. A malformed datagram is dropped without a
# reply even by a reachable resolver, which would report the open network as
# blocked and silently retire this probe's assertion.
UDP_PROBE = (
"import socket\n"
"query = (\n"
" b'\\xab\\xcd\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00'\n"
" b'\\x07example\\x03com\\x00\\x00\\x01\\x00\\x01'\n"
")\n"
"sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n"
"sock.settimeout(5)\n"
"try:\n"
" sock.sendto(query, ('8.8.8.8', 53))\n"
" sock.recvfrom(512)\n"
" print('reachable')\n"
"except OSError:\n"
" print('blocked')\n"
)
DOCKER_DNS_PROBE = (
"import socket\n"
"query = (\n"
" b'\\xab\\xcd\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00'\n"
" b'\\x07example\\x03com\\x00\\x00\\x01\\x00\\x01'\n"
")\n"
"sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n"
"sock.settimeout(5)\n"
"try:\n"
" sock.sendto(query, ('127.0.0.11', 53))\n"
" sock.recvfrom(512)\n"
" print('reachable')\n"
"except OSError:\n"
" print('blocked')\n"
)
# An AF_PACKET socket writes at the link layer, below the IP output hooks the
# policy installs, so no nftables rule can see this traffic at all. Only the
# absence of NET_RAW stops it. The frame is addressed to the default gateway's
# hardware address, which the pre-policy reachability assertions leave in the
# neighbour table.
PACKET_PROBE = (
"import socket, struct\n"
"query = (\n"
" b'\\xab\\xcd\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00'\n"
" b'\\x07example\\x03com\\x00\\x00\\x01\\x00\\x01'\n"
")\n"
"def route():\n"
" with open('/proc/net/route') as handle:\n"
" for line in handle.readlines()[1:]:\n"
" f = line.split()\n"
" if f[1] == '00000000':\n"
" return f[0], socket.inet_ntoa(struct.pack('<L', int(f[2], 16)))\n"
" return None, None\n"
"def neighbour(address):\n"
" with open('/proc/net/arp') as handle:\n"
" for line in handle.readlines()[1:]:\n"
" f = line.split()\n"
" if f[0] == address and f[3] != '00:00:00:00:00:00':\n"
" return bytes.fromhex(f[3].replace(':', ''))\n"
" return None\n"
"def checksum(data):\n"
" total = sum(struct.unpack('!%dH' % (len(data) // 2), data))\n"
" while total >> 16:\n"
" total = (total & 0xFFFF) + (total >> 16)\n"
" return ~total & 0xFFFF\n"
"interface, gateway = route()\n"
"mac = neighbour(gateway) if gateway else None\n"
"if not interface or not mac:\n"
" print('unusable')\n"
"else:\n"
" try:\n"
" sock = socket.socket(socket.AF_PACKET, socket.SOCK_DGRAM, 0x0800)\n"
" except PermissionError:\n"
" print('denied')\n"
" raise SystemExit\n"
" try:\n"
" source = socket.gethostbyname(socket.gethostname())\n"
" sock.bind((interface, 0x0800))\n"
" sock.settimeout(5)\n"
" udp = struct.pack('!HHHH', 40000, 53, 8 + len(query), 0) + query\n"
" header = struct.pack(\n"
" '!BBHHHBBH4s4s', 0x45, 0, 20 + len(udp), 0x1234, 0, 64, 17, 0,\n"
" socket.inet_aton(source), socket.inet_aton('8.8.8.8'),\n"
" )\n"
" header = header[:10] + struct.pack('!H', checksum(header)) + header[12:]\n"
" sock.sendto(header + udp, (interface, 0x0800, 0, 0, mac))\n"
" while True:\n"
" packet = sock.recv(2048)\n"
" if len(packet) >= 20 and packet[12:16] == socket.inet_aton('8.8.8.8'):\n"
" print('reachable')\n"
" break\n"
" except OSError:\n"
" print('blocked')\n"
)
LOOPBACK_PROBE = (
"import socket\n"
"server = socket.socket()\n"
"server.bind(('127.0.0.1', 0))\n"
"server.listen(1)\n"
"client = socket.create_connection(server.getsockname(), 5)\n"
"client.close()\n"
"print('reachable')\n"
)
PROXY_ENVIRONMENT = {
"HTTPS_PROXY": f"http://{PROXY_HOST}:8080",
"CURL_CA_BUNDLE": CA_PATH,
}
# The bypass attempt carries no CA override: pinning trust to the cell CA would
# fail a successful direct connection on certificate verification instead, and
# the assertion would hold even with the policy removed.
BYPASS_ENVIRONMENT = {"HTTPS_PROXY": f"http://{PROXY_HOST}:8080"}
def docker_available() -> bool:
if shutil.which("docker") is None:
return False
probe = subprocess.run(
["docker", "version", "--format", "{{.Server.Version}}"],
capture_output=True,
timeout=COMMAND_TIMEOUT_S,
)
return probe.returncode == 0
@unittest.skipUnless(
os.environ.get("MAKA_EVAL_EGRESS_NAMESPACE_TEST") == "1",
"set MAKA_EVAL_EGRESS_NAMESPACE_TEST=1 to run the Docker cell namespace test",
)
class CellEgressNamespaceTest(unittest.TestCase):
workdir: Path | None = None
compose: list[str] = []
@classmethod
def setUpClass(cls) -> None:
if not docker_available():
raise unittest.SkipTest("Docker daemon is unavailable")
cls.workdir = Path(tempfile.mkdtemp(prefix="maka-eval-egress-namespace-"))
# Registered before anything can fail: `tearDownClass` does not run when
# `setUpClass` raises, which would leak the stack and the named volume.
cls.addClassCleanup(shutil.rmtree, cls.workdir, ignore_errors=True)
(cls.workdir / "base.yaml").write_text(BASE_COMPOSE)
cls.build_image(MAIN_IMAGE, dockerfile=MAIN_DOCKERFILE, name="main")
cls.build_image(SIDECAR_IMAGE, dockerfile=SIDECAR_DOCKERFILE, name="sidecar")
cls.build_image(
PROXY_IMAGE, context=HARBOR_DIR, path=HARBOR_DIR / "egress-proxy" / "Dockerfile"
)
cls.compose = [
"docker",
"compose",
"--project-name",
PROJECT,
"--file",
str(cls.workdir / "base.yaml"),
"--file",
str(OVERLAY),
]
cls.run_compose(["down", "--volumes", "--remove-orphans"], check=False)
cls.addClassCleanup(
cls.run_compose, ["down", "--volumes", "--remove-orphans"], check=False
)
cls.run_compose(["up", "--detach", "--wait"], timeout=BUILD_TIMEOUT_S)
# Pin the proxy hostname before any test applies the policy, so
# HTTPS_PROXY still resolves after Docker DNS is refused.
asyncio.run(load_relay()._pin_proxy_hostname(CellEnvironment("main")))
@classmethod
def build_image(
cls,
tag: str,
*,
dockerfile: str | None = None,
name: str | None = None,
context: Path | None = None,
path: Path | None = None,
) -> None:
if dockerfile is not None:
context = cls.workdir / name
context.mkdir()
(context / "Dockerfile").write_text(dockerfile)
subprocess.run(
[
"docker",
"build",
"--tag",
tag,
*(["--file", str(path)] if path else []),
str(context),
],
check=True,
timeout=BUILD_TIMEOUT_S,
)
@classmethod
def run_compose(
cls,
arguments: list[str],
*,
check: bool = True,
timeout: int = COMMAND_TIMEOUT_S,
) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[*cls.compose, *arguments],
env={**os.environ, "MAKA_EVAL_NETWORK_POLICY_PATH": str(NETWORK_POLICY)},
capture_output=True,
text=True,
check=check,
timeout=timeout,
)
@classmethod
def exec_service(
cls,
service: str,
command: list[str],
*,
environment: dict[str, str] | None = None,
user: str | None = None,
) -> subprocess.CompletedProcess[str]:
overrides = [
argument
for name, value in (environment or {}).items()
for argument in ("--env", f"{name}={value}")
]
user_args = ["--user", user] if user else []
return cls.run_compose(
["exec", "--no-TTY", *user_args, *overrides, service, *command], check=False
)
@classmethod
def exec_main(
cls, command: list[str], *, environment: dict[str, str] | None = None
) -> subprocess.CompletedProcess[str]:
return cls.exec_service("main", command, environment=environment)
@classmethod
def probe_main(cls, script: str) -> str:
result = cls.exec_main(["python3", "-c", script])
if result.returncode != 0:
raise AssertionError(f"probe failed to run: {result.stderr}")
return result.stdout.strip()
@classmethod
def curl(
cls, arguments: list[str], *, environment: dict[str, str] | None = None
) -> subprocess.CompletedProcess[str]:
return cls.exec_main(
[
"curl",
"--silent",
"--show-error",
"--max-time",
"20",
"--output",
"/dev/null",
*arguments,
"https://example.com",
],
environment=environment,
)
@classmethod
def ping(cls) -> subprocess.CompletedProcess[str]:
return cls.exec_main(["ping", "-c", "1", "-W", "5", "1.1.1.1"])
def test_no_network_phase_is_not_answered_with_proxy_access(self) -> None:
# Harbor's deny-all denies controlled egress outright, so answering it
# with the proxy-only ruleset would grant a phase that asked for no
# network a route this override invented.
reachable = self.curl([], environment=PROXY_ENVIRONMENT)
self.assertEqual(reachable.returncode, 0, reachable.stderr)
try:
denied = self.exec_service(SIDECAR, ["network-policy", "deny-all"])
self.assertEqual(denied.returncode, 0, denied.stderr)
blocked = self.curl([], environment=PROXY_ENVIRONMENT)
self.assertNotEqual(blocked.returncode, 0)
finally:
restored = self.exec_service(SIDECAR, ["network-policy", "allow", PROXY_HOST])
self.assertEqual(restored.returncode, 0, restored.stderr)
def test_relay_admits_only_a_subject_the_policy_governs(self) -> None:
# The gate runs against live containers rather than a fixture, so what
# it parses is a real `/proc` and what it judges is a real namespace.
# The refusals matter more than the admission: without them nothing
# would catch a probe that reports one constant answer.
relay = load_relay()
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
asyncio.run(relay._require_constrained_subject(CellEnvironment("main")))
with self.assertRaises(RuntimeError) as outside:
asyncio.run(relay._require_constrained_subject(CellEnvironment(OUTSIDER)))
with self.assertRaises(RuntimeError) as bounded:
asyncio.run(relay._require_constrained_subject(CellEnvironment(BOUNDED)))
self.assertIn("network namespace", str(outside.exception))
self.assertIn("NET_RAW", str(bounded.exception))
def test_subject_sees_only_the_ca_certificate(self) -> None:
listed = self.exec_main(["ls", "--almost-all", "/opt/maka-egress"])
self.assertEqual(listed.returncode, 0, listed.stderr)
self.assertEqual(sorted(listed.stdout.split()), ["mitmproxy-ca-cert.pem", "proxy-ipv4"])
def test_cell_namespace_admits_only_the_audited_proxy(self) -> None:
# Every negative assertion below is vacuous on a host that cannot reach
# the target anyway, so the reachability of each one is asserted before
# the policy exists. Failing there is the point: a skip would report a
# contract this run never actually exercised.
self.assertEqual(self.curl([]).returncode, 0, "no HTTPS before any policy")
self.assertEqual(self.probe_main(TCP_PROBE), "reachable")
self.assertEqual(self.probe_main(UDP_PROBE), "reachable")
self.assertEqual(self.probe_main(DOCKER_DNS_PROBE), "reachable")
self.assertEqual(self.ping().returncode, 0, "no ICMP before any policy")
applied = self.exec_service(
SIDECAR, ["network-policy", "allow", PROXY_HOST]
)
self.assertEqual(applied.returncode, 0, applied.stderr)
with self.subTest("explicit proxy HTTPS"):
allowed = self.curl([], environment=PROXY_ENVIRONMENT)
self.assertEqual(allowed.returncode, 0, allowed.stderr)
with self.subTest("curl --noproxy"):
bypassed = self.curl(["--noproxy", "*"], environment=BYPASS_ENVIRONMENT)
self.assertNotEqual(bypassed.returncode, 0)
with self.subTest("direct IP TCP"):
self.assertEqual(self.probe_main(TCP_PROBE), "blocked")
# Not preceded by a reachability assertion like the others: this path is
# closed by the missing capability rather than by the policy, so it is
# already shut before the policy exists. `denied` rather than `blocked`
# so a probe that breaks for any other reason cannot report success.
with self.subTest("link-layer AF_PACKET"):
self.assertEqual(self.probe_main(PACKET_PROBE), "denied")
with self.subTest("external UDP"):
self.assertEqual(self.probe_main(UDP_PROBE), "blocked")
with self.subTest("external ICMP"):
self.assertNotEqual(self.ping().returncode, 0)
with self.subTest("loopback provider proxy"):
self.assertEqual(self.probe_main(LOOPBACK_PROBE), "reachable")
with self.subTest("pinned proxy hostname"):
resolved = self.exec_main(["getent", "hosts", PROXY_HOST])
self.assertEqual(resolved.returncode, 0, resolved.stderr)
with self.subTest("Docker DNS"):
self.assertEqual(self.probe_main(DOCKER_DNS_PROBE), "blocked")
class CellEnvironment:
"""Presents one live cell service as the relay's environment."""
def __init__(self, service: str) -> None:
self.service = service
async def exec(self, command: str, cwd=None, env=None, timeout_sec=None, user=None):
return self._run(
self.service, command, user=str(user) if user is not None else None
)
async def service_exec(self, command: str, *, service: str, **kwargs):
return self._run(service, command)
@staticmethod
def _run(service: str, command: str, user: str | None = None):
result = CellEgressNamespaceTest.exec_service(
service, ["sh", "-c", command], user=user
)
return types.SimpleNamespace(
return_code=result.returncode, stdout=result.stdout, stderr=result.stderr
)
if __name__ == "__main__":
unittest.main()
+477
View File
@@ -0,0 +1,477 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import importlib.util
import json
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
MODULE_PATH = Path(__file__).with_name("egress_filter.py")
SPEC = importlib.util.spec_from_file_location("maka_eval_egress_filter", MODULE_PATH)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
class EgressFilterTest(unittest.TestCase):
def test_blocks_contamination_surfaces_and_recursive_jina_urls(self) -> None:
blocked = {
"https://github.com/harbor-framework/terminal-bench-2-1": "benchmark_repository",
"https://api.github.com/repos/terminal-benchmarks/terminal-bench/issues": "benchmark_repository",
"https://raw.githubusercontent.com/tbench-ai/terminal-bench/main/tests/x": "benchmark_repository",
"https://huggingface.co/datasets/acme/terminal-bench-traces": "terminal_bench_url",
"https://github.com/hqeric/maka-eval-trajectories": "public_trajectory",
"https://spylab.ai/reference/terminalbench-solution": "terminal_bench_url",
"https://example.test/patches-terminalbench-task-1.diff": "known_patch_artifact",
f"https://example.test/archive?revision={MODULE.PINNED_REVISION}": "pinned_revision",
"https://tbench.ai/tasks": "tbench_domain",
"https://hub.harborframework.com/tasks/terminal-bench/foo": "harbor_task_registry",
"https://r.jina.ai/https://github.com/harbor-framework/terminal-bench-2-1": "jina_recursive:benchmark_repository",
"https://r.jina.ai/https%253A%252F%252Fspylab.ai%252Fterminal-bench": "jina_recursive:terminal_bench_url",
"https://example.test/search?q=TeRmInAlBeNcH": "terminal_bench_url",
"https://google.com/search?q=terminal+bench": "terminal_bench_url",
"https://github.com/harbor-framework/terminal%252Dbench-2-1.git": "benchmark_repository",
"https://terminal-bench.io/tasks/answers": "terminal_bench_url",
"https://sub.tbench.ai/x": "tbench_domain",
# A DNS label is as good a place to name a contamination surface as
# a path, so every rule searches both fields.
f"https://{MODULE.PINNED_REVISION}.example.test/archive": "pinned_revision",
"https://patches-terminalbench-task-1.example.test/x": "known_patch_artifact",
}
for url, rule_id in blocked.items():
matched = MODULE.contamination_rule(url)
self.assertIsNotNone(matched, url)
self.assertEqual(matched[0], rule_id, url)
def test_preserves_unrelated_network_and_rejects_malformed_urls(self) -> None:
allowed = [
"https://github.com/harbor-framework/harbor",
"https://github.com/microsoft/terminal",
"https://huggingface.co/datasets/mteb/leaderboard",
"https://pypi.org/simple/requests/",
"https://deb.debian.org/debian/",
"https://my-terminal/bench",
]
for url in allowed:
self.assertIsNone(MODULE.contamination_rule(url), url)
for url in ["", "file:///tmp/terminal-bench.log", "https://example.test/%ZZ"]:
with self.assertRaises(ValueError, msg=url):
MODULE.contamination_rule(url)
def _install_response_stub(self) -> None:
class Response:
@staticmethod
def make(status, body, headers):
return {"status": status, "body": body, "headers": headers}
MODULE.http = SimpleNamespace(Response=Response)
def test_request_blocks_a_contamination_url_and_appends_one_audit_record(self) -> None:
with tempfile.TemporaryDirectory() as directory:
self._install_response_stub()
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
flow = type(
"Flow",
(),
{"request": type("Request", (), {"pretty_url": "https://tbench.ai/tasks"})()},
)()
MODULE.request(flow)
self.assertEqual(flow.response["status"], 451)
self.assertEqual(
flow.response["headers"]["X-Maka-Eval-Egress-Rule"], "tbench_domain"
)
lines = MODULE.AUDIT_PATH.read_text().splitlines()
self.assertEqual(len(lines), 1)
record = json.loads(lines[0])
self.assertEqual(record["ruleId"], "tbench_domain")
self.assertEqual(record["host"], "tbench.ai")
self.assertEqual(record["normalizedPath"], "/tasks")
def test_request_leaves_an_unrelated_url_unanswered(self) -> None:
with tempfile.TemporaryDirectory() as directory:
self._install_response_stub()
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
flow = type(
"Flow",
(),
{"request": type("Request", (), {"pretty_url": "https://example.com/"})()},
)()
MODULE.request(flow)
self.assertFalse(hasattr(flow, "response"))
self.assertFalse(MODULE.AUDIT_PATH.exists())
def test_audit_is_bounded_and_policy_errors_fail_closed(self) -> None:
with tempfile.TemporaryDirectory() as directory:
self._install_response_stub()
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
flow = type(
"Flow",
(),
{"request": type("Request", (), {"pretty_url": "https://example.test/%ZZ"})()},
)()
MODULE.request(flow)
self.assertEqual(flow.response["status"], 503)
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
self.assertEqual(record["ruleId"], "policy_error")
self.assertIn("host", record)
self.assertIn("normalizedPath", record)
def test_http_connect_refuses_blocklisted_hosts_before_the_tunnel_opens(self) -> None:
class Response:
@staticmethod
def make(status, body, headers):
return {"status": status, "body": body, "headers": headers}
with tempfile.TemporaryDirectory() as directory:
MODULE.http = SimpleNamespace(Response=Response)
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
blocked = type(
"Flow",
(),
{
"request": type(
"Request",
(),
{"pretty_host": "tbench.ai", "host": "tbench.ai", "port": 443},
)()
},
)()
MODULE.http_connect(blocked)
self.assertEqual(blocked.response["status"], 451)
self.assertEqual(blocked.response["headers"]["X-Maka-Eval-Egress-Rule"], "tbench_domain")
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
self.assertEqual(record["ruleId"], "tbench_domain")
self.assertEqual(record["host"], "tbench.ai")
for host in ("example.com", "github.com", "ssh.github.com"):
allowed = type(
"Flow",
(),
{
"request": type(
"Request",
(),
{"pretty_host": host, "host": host, "port": 443},
)(),
"response": None,
},
)()
MODULE.http_connect(allowed)
self.assertIsNone(allowed.response, host)
def test_tcp_start_kills_raw_tunnels_and_records_them(self) -> None:
with tempfile.TemporaryDirectory() as directory:
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
killed: list[str] = []
flow = type(
"Flow",
(),
{"server_conn": SimpleNamespace(address=("ssh.github.com", 443))},
)()
flow.kill = lambda: killed.append("killed")
MODULE.tcp_start(flow)
self.assertEqual(killed, ["killed"])
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
self.assertEqual(record["ruleId"], "raw_tunnel")
self.assertEqual(record["host"], "ssh.github.com")
self.assertEqual(record["normalizedPath"], ":443")
def test_tcp_message_drops_raw_payloads(self) -> None:
message = SimpleNamespace(content=b"SSH-2.0-test\r\n")
killed: list[str] = []
flow = SimpleNamespace(messages=[message], killable=True)
flow.kill = lambda: killed.append("killed")
MODULE.tcp_message(flow)
self.assertEqual(message.content, b"")
self.assertEqual(killed, ["killed"])
def test_next_layer_closes_raw_tcp_before_the_builtin_classifier(self) -> None:
class CloseConnection:
def __init__(self, connection: object) -> None:
self.connection = connection
with tempfile.TemporaryDirectory() as directory:
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
previous_commands = MODULE.proxy_commands
self.addCleanup(setattr, MODULE, "proxy_commands", previous_commands)
MODULE.proxy_commands = SimpleNamespace(CloseConnection=CloseConnection)
client = object()
server = SimpleNamespace(address=("ssh.github.com", 443))
context = SimpleNamespace(client=client, server=server, layers=[], options=None)
def data_client() -> bytes:
return b"SSH-2.0-OpenSSH_9.0"
nextlayer = SimpleNamespace(
layer=None, context=context, data_client=data_client, data_server=lambda: b""
)
MODULE.next_layer(nextlayer)
self.assertIsInstance(nextlayer.layer, MODULE.CloseRawLayer)
commands = list(nextlayer.layer.handle_event(object()))
self.assertEqual([command.connection for command in commands], [client, server])
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
self.assertEqual(record["ruleId"], "raw_tunnel")
self.assertEqual(record["host"], "ssh.github.com")
def test_next_layer_routes_tls_and_http_without_raw_fallback(self) -> None:
class FakeServerTLSLayer:
child_layer: object | None = None
def __init__(self, context: object) -> None:
self.context = context
context.layers.append(self)
class FakeClientTLSLayer:
def __init__(self, context: object) -> None:
self.context = context
context.layers.append(self)
previous_server_tls = MODULE.ServerTLSLayer
previous_client_tls = MODULE.ClientTLSLayer
self.addCleanup(setattr, MODULE, "ServerTLSLayer", previous_server_tls)
self.addCleanup(setattr, MODULE, "ClientTLSLayer", previous_client_tls)
MODULE.ServerTLSLayer = FakeServerTLSLayer
MODULE.ClientTLSLayer = FakeClientTLSLayer
context = SimpleNamespace(layers=[])
tls = SimpleNamespace(
layer=None,
context=context,
data_client=lambda: b"\x16\x03\x01\x00\x00",
data_server=lambda: b"",
)
MODULE.next_layer(tls)
self.assertIsNone(tls.layer)
for first, remainder in (
(b"\x16", b"\x03\x01\x00\x00"),
(b"\x16\x03", b"\x01\x00\x00"),
):
with self.subTest(tls_prefix=first):
fragmented_context = SimpleNamespace(layers=[])
fragmented = SimpleNamespace(
layer=None,
context=fragmented_context,
data_client=lambda first=first: first,
data_server=lambda: b"",
)
MODULE.next_layer(fragmented)
self.assertIsInstance(fragmented.layer, FakeServerTLSLayer)
self.assertIsInstance(fragmented.layer.child_layer, FakeClientTLSLayer)
self.assertEqual(
fragmented_context.layers,
[fragmented.layer, fragmented.layer.child_layer],
)
self.assertTrue((first + remainder).startswith(b"\x16\x03"))
http = SimpleNamespace(
layer=None,
context=context,
data_client=lambda: b"GET / HTTP/1.1\r\n",
data_server=lambda: b"",
)
MODULE.next_layer(http)
self.assertIsNone(http.layer)
empty = SimpleNamespace(
layer=None,
context=context,
data_client=lambda: b"",
data_server=lambda: b"",
)
MODULE.next_layer(empty)
self.assertIsNone(empty.layer)
for prefix in (b"G", b"GE", b"GET", b"GET ", b"GET / HTTP/1.1"):
with self.subTest(prefix=prefix):
incomplete = SimpleNamespace(
layer=None,
context=context,
data_client=lambda prefix=prefix: prefix,
data_server=lambda: b"",
)
MODULE.next_layer(incomplete)
self.assertIsNone(incomplete.layer)
def test_next_layer_closes_bytes_that_cannot_become_http(self) -> None:
with tempfile.TemporaryDirectory() as directory:
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
context = SimpleNamespace(
layers=[], options=None, client=object(), server=None
)
binary = SimpleNamespace(
layer=None,
context=context,
data_client=lambda: b"\x00\x01\x02\x03",
data_server=lambda: b"",
)
MODULE.next_layer(binary)
self.assertIsInstance(binary.layer, MODULE.CloseRawLayer)
def test_response_audits_a_non_websocket_101_upgrade(self) -> None:
with tempfile.TemporaryDirectory() as directory:
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
raw = type(
"Flow",
(),
{
"response": SimpleNamespace(
status_code=101, headers={"upgrade": "raw"}
),
"server_conn": SimpleNamespace(address=("origin", 19083)),
},
)()
MODULE.response(raw)
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
self.assertEqual(record["ruleId"], "raw_tunnel")
websocket = type(
"Flow",
(),
{
"response": SimpleNamespace(
status_code=101, headers={"upgrade": "websocket"}
),
"websocket": object(),
"server_conn": SimpleNamespace(address=("origin", 19082)),
},
)()
MODULE.response(websocket)
self.assertEqual(len(MODULE.AUDIT_PATH.read_text().splitlines()), 1)
invalid = type(
"Flow",
(),
{
"response": SimpleNamespace(
status_code=101, headers={"upgrade": "websocket"}
),
"server_conn": SimpleNamespace(address=("origin", 19082)),
},
)()
MODULE.response(invalid)
self.assertEqual(len(MODULE.AUDIT_PATH.read_text().splitlines()), 2)
self.assertEqual(
json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[1])["ruleId"],
"raw_tunnel",
)
def test_next_layer_replaces_raw_tcp_with_a_closer(self) -> None:
class FakeTCPLayer:
def __init__(self, context: object) -> None:
self.context = context
context.layers.append(self)
class CloseConnection:
def __init__(self, connection: object) -> None:
self.connection = connection
with tempfile.TemporaryDirectory() as directory:
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
previous_tcp = MODULE.TCPLayer
previous_commands = MODULE.proxy_commands
self.addCleanup(setattr, MODULE, "TCPLayer", previous_tcp)
self.addCleanup(setattr, MODULE, "proxy_commands", previous_commands)
MODULE.TCPLayer = FakeTCPLayer
MODULE.proxy_commands = SimpleNamespace(CloseConnection=CloseConnection)
client = object()
server = SimpleNamespace(address=("ssh.github.com", 443))
sibling = object()
context = SimpleNamespace(
client=client, server=server, layers=[sibling], options=None
)
current = FakeTCPLayer(context)
nextlayer = SimpleNamespace(layer=current, context=context)
MODULE.next_layer(nextlayer)
self.assertIsInstance(nextlayer.layer, MODULE.CloseRawLayer)
self.assertIsInstance(nextlayer.layer, MODULE.Layer)
self.assertEqual(context.layers, [sibling, nextlayer.layer])
self.assertNotIn(current, context.layers)
commands = list(nextlayer.layer.handle_event(object()))
self.assertEqual([command.connection for command in commands], [client, server])
self.assertTrue(all(isinstance(command, CloseConnection) for command in commands))
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
self.assertEqual(record["ruleId"], "raw_tunnel")
self.assertEqual(record["host"], "ssh.github.com")
def test_next_layer_leaves_an_unclassified_layer_alone(self) -> None:
context = SimpleNamespace(layers=[])
nextlayer = SimpleNamespace(layer=None, context=context)
MODULE.next_layer(nextlayer)
self.assertIsNone(nextlayer.layer)
self.assertEqual(context.layers, [])
def test_next_layer_leaves_non_tcp_layers_alone(self) -> None:
class HTTPLayer:
pass
original = HTTPLayer()
nextlayer = SimpleNamespace(layer=original, context=SimpleNamespace())
MODULE.next_layer(nextlayer)
self.assertIs(nextlayer.layer, original)
def test_audit_escapes_line_separators_so_python_and_typescript_agree(self) -> None:
with tempfile.TemporaryDirectory() as directory:
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
path = "/tasks/\u2028hidden"
MODULE.append_audit("tbench_domain", "tbench.ai", path)
raw = MODULE.AUDIT_PATH.read_text(encoding="utf-8")
self.assertNotIn("\u2028", raw)
self.assertIn("\\u2028", raw)
self.assertEqual(raw.count("\n"), 1)
self.assertEqual(len(raw.splitlines()), 1)
record = json.loads(raw)
self.assertEqual(record["normalizedPath"], path)
self.assertFalse(MODULE.audit_already_truncated())
def test_audit_writes_one_truncation_marker_when_the_byte_limit_is_reached(self) -> None:
with tempfile.TemporaryDirectory() as directory:
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
MODULE.AUDIT_PATH.write_bytes(b"x" * MODULE.MAX_AUDIT_BYTES)
MODULE.append_audit("tbench_domain", "tbench.ai", "/tasks")
records = [
json.loads(line)
for line in MODULE.AUDIT_PATH.read_text().splitlines()
if line.startswith("{")
]
self.assertEqual(records[-1]["ruleId"], "audit_truncated")
size_after_marker = MODULE.AUDIT_PATH.stat().st_size
MODULE.append_audit("tbench_domain", "tbench.ai", "/other")
self.assertEqual(MODULE.AUDIT_PATH.stat().st_size, size_after_marker)
records_after = [
json.loads(line)
for line in MODULE.AUDIT_PATH.read_text().splitlines()
if line.startswith("{")
]
self.assertEqual(
[record["ruleId"] for record in records_after if record["ruleId"] == "audit_truncated"],
["audit_truncated"],
)
def test_truncation_probe_ignores_non_object_json_tails(self) -> None:
with tempfile.TemporaryDirectory() as directory:
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
MODULE.AUDIT_PATH.write_text('123\n"x"\n')
self.assertFalse(MODULE.audit_already_truncated())
MODULE.write_truncation_marker()
self.assertTrue(MODULE.audit_already_truncated())
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,606 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Prove raw tunnels die on a real mitmproxy 12.2.3 without breaking HTTPS or WebSocket.
Unit tests cannot see addon order: script `next_layer` runs before the built-in
classifier assigns `TCPLayer`. This test starts the pinned proxy image and a
local origin, then asserts what a live cell would observe.
It needs Docker, the pinned proxy image, and `python:3.12-slim`, so it is opt-in:
MAKA_EVAL_EGRESS_PROXY_TEST=1 python3 harbor/test_egress_filter_live.py
"""
from __future__ import annotations
import base64
import json
import os
import shutil
import socket
import ssl
import subprocess
import tempfile
import time
import unittest
import uuid
from pathlib import Path
HARBOR_DIR = Path(__file__).parent
PROXY_IMAGE = "maka-eval-egress-proxy:12.2.3"
ORIGIN_IMAGE = "python:3.12-slim"
COMMAND_TIMEOUT_S = 60
CLOSE_TIMEOUT_S = 2.0
ORIGIN_SCRIPT = r"""
import base64, hashlib, json, socket, threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
stats = {"raw_recv": 0, "raw_closed": 0, "upgrade_recv": 0, "upgrade_closed": 0}
class HttpHandler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-Type", "text/plain")
self.end_headers()
self.wfile.write(b"http-ok")
def log_message(self, format, *args):
return
class WsHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.headers.get("Upgrade", "").lower() != "websocket":
self.send_error(400)
return
key = self.headers.get("Sec-WebSocket-Key", "")
accept = base64.b64encode(
hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest()
).decode()
self.send_response(101, "Switching Protocols")
self.send_header("Upgrade", "websocket")
self.send_header("Connection", "Upgrade")
self.send_header("Sec-WebSocket-Accept", accept)
self.end_headers()
payload = b"ws-ok"
self.wfile.write(bytes([0x81, len(payload)]) + payload)
self.wfile.flush()
def log_message(self, format, *args):
return
class UpgradeHandler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(101, "Switching Protocols")
self.send_header("Upgrade", "raw")
self.send_header("Connection", "Upgrade")
self.end_headers()
self.wfile.write(b"UPGRADE-BANNER\n")
self.wfile.flush()
self.connection.settimeout(60)
try:
while True:
chunk = self.connection.recv(64)
if not chunk:
break
stats["upgrade_recv"] += len(chunk)
except OSError:
pass
finally:
stats["upgrade_closed"] += 1
def log_message(self, format, *args):
return
def serve_raw():
sock = socket.socket()
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(("0.0.0.0", 19081))
sock.listen(8)
while True:
conn, _ = sock.accept()
try:
conn.sendall(b"RAW-BANNER\n")
conn.settimeout(60)
while True:
chunk = conn.recv(64)
if not chunk:
break
stats["raw_recv"] += len(chunk)
except OSError:
pass
finally:
stats["raw_closed"] += 1
conn.close()
class StatsHandler(BaseHTTPRequestHandler):
def do_GET(self):
body = json.dumps(stats).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(body)
def log_message(self, format, *args):
return
threading.Thread(target=serve_raw, daemon=True).start()
threading.Thread(target=lambda: ThreadingHTTPServer(("0.0.0.0", 19080), HttpHandler).serve_forever(), daemon=True).start()
threading.Thread(target=lambda: ThreadingHTTPServer(("0.0.0.0", 19082), WsHandler).serve_forever(), daemon=True).start()
threading.Thread(target=lambda: ThreadingHTTPServer(("0.0.0.0", 19083), UpgradeHandler).serve_forever(), daemon=True).start()
ThreadingHTTPServer(("0.0.0.0", 19084), StatsHandler).serve_forever()
"""
def docker_available() -> bool:
if shutil.which("docker") is None:
return False
probe = subprocess.run(
["docker", "version", "--format", "{{.Server.Version}}"],
capture_output=True,
timeout=COMMAND_TIMEOUT_S,
)
return probe.returncode == 0
def docker_image_present(image: str) -> bool:
inspect = subprocess.run(
["docker", "image", "inspect", image],
capture_output=True,
timeout=COMMAND_TIMEOUT_S,
)
return inspect.returncode == 0
def finish_memory_bio_handshake(tls, incoming, outgoing, sock, timeout_s: float) -> str:
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
try:
tls.do_handshake()
except ssl.SSLWantReadError:
# Processing a server flight can produce the next client flight
# before OpenSSL asks for more input. Flush it now; waiting for
# another recv first deadlocks both peers.
pending = outgoing.read()
if pending:
sock.sendall(pending)
response = sock.recv(16 * 1024)
if not response:
raise AssertionError("proxy closed during the fragmented TLS handshake")
incoming.write(response)
continue
pending = outgoing.read()
if pending:
sock.sendall(pending)
return tls.version() or ""
raise AssertionError("fragmented TLS handshake did not complete")
class MemoryBioHandshakeDriverTest(unittest.TestCase):
def test_flushes_client_flight_before_waiting_for_more_server_bytes(self) -> None:
events = []
class FakeOutgoing:
pending = b""
def read(self):
pending, self.pending = self.pending, b""
return pending
class FakeIncoming:
def write(self, data):
events.append(("write", data))
outgoing = FakeOutgoing()
class FakeTls:
calls = 0
def do_handshake(self):
self.calls += 1
if self.calls == 1:
outgoing.pending = b"client-finished"
raise ssl.SSLWantReadError()
def version(self):
return "TLSv1.3"
class FakeSocket:
def sendall(self, data):
events.append(("send", data))
def recv(self, _size):
events.append(("recv", None))
return b"server-finished"
result = finish_memory_bio_handshake(
FakeTls(), FakeIncoming(), outgoing, FakeSocket(), timeout_s=1
)
self.assertEqual(result, "TLSv1.3")
self.assertEqual(
events,
[
("send", b"client-finished"),
("recv", None),
("write", b"server-finished"),
],
)
@unittest.skipUnless(
os.environ.get("MAKA_EVAL_EGRESS_PROXY_TEST") == "1",
"set MAKA_EVAL_EGRESS_PROXY_TEST=1 to run the live mitmproxy proxy test",
)
class LiveEgressFilterTest(unittest.TestCase):
workdir: Path | None = None
network: str = ""
proxy: str = ""
origin: str = ""
proxy_port: int = 0
@classmethod
def setUpClass(cls) -> None:
if not docker_available():
raise unittest.SkipTest("Docker daemon is unavailable")
if not docker_image_present(PROXY_IMAGE):
raise unittest.SkipTest(f"{PROXY_IMAGE} is not present")
if not docker_image_present(ORIGIN_IMAGE):
raise unittest.SkipTest(f"{ORIGIN_IMAGE} is not present")
run_id = f"{os.getpid()}-{uuid.uuid4().hex[:8]}"
cls.network = f"maka-eval-egress-live-{run_id}-net"
cls.proxy = f"maka-eval-egress-live-{run_id}-proxy"
cls.origin = f"maka-eval-egress-live-{run_id}-origin"
cls.workdir = Path(tempfile.mkdtemp(prefix="maka-eval-egress-proxy-live-"))
(cls.workdir / "origin.py").write_text(ORIGIN_SCRIPT)
cls.addClassCleanup(shutil.rmtree, cls.workdir, ignore_errors=True)
cls.addClassCleanup(cls._down)
subprocess.run(["docker", "network", "create", cls.network], check=True, timeout=COMMAND_TIMEOUT_S)
subprocess.run(
[
"docker",
"run",
"-d",
"--name",
cls.origin,
"--network",
cls.network,
"--network-alias",
"origin",
"-v",
f"{cls.workdir / 'origin.py'}:/origin.py:ro",
ORIGIN_IMAGE,
"python",
"/origin.py",
],
check=True,
timeout=COMMAND_TIMEOUT_S,
)
subprocess.run(
[
"docker",
"run",
"-d",
"--name",
cls.proxy,
"--network",
cls.network,
"-p",
"127.0.0.1::8080",
"-v",
f"{HARBOR_DIR / 'egress_filter.py'}:/opt/maka-eval/egress_filter.py:ro",
PROXY_IMAGE,
],
check=True,
timeout=COMMAND_TIMEOUT_S,
)
cls.proxy_port = cls._published_port()
cls._wait_for_proxy()
cls._wait_for_origin_via_proxy()
@classmethod
def _published_port(cls) -> int:
listed = subprocess.run(
["docker", "port", cls.proxy, "8080/tcp"],
capture_output=True,
text=True,
timeout=COMMAND_TIMEOUT_S,
)
if listed.returncode != 0:
raise AssertionError(listed.stderr)
# "127.0.0.1:49152"
hostport = listed.stdout.strip().splitlines()[0]
return int(hostport.rsplit(":", 1)[1])
@classmethod
def _down(cls) -> None:
for name in (cls.proxy, cls.origin):
if not name:
continue
subprocess.run(
["docker", "rm", "-f", name],
capture_output=True,
timeout=COMMAND_TIMEOUT_S,
)
if cls.network:
subprocess.run(
["docker", "network", "rm", cls.network],
capture_output=True,
timeout=COMMAND_TIMEOUT_S,
)
@classmethod
def _wait_for_proxy(cls) -> None:
deadline = time.time() + 30
last_error = "proxy did not listen"
while time.time() < deadline:
try:
with socket.create_connection(("127.0.0.1", cls.proxy_port), 1):
return
except OSError as error:
last_error = str(error)
time.sleep(0.2)
raise AssertionError(last_error)
@classmethod
def _wait_for_origin_via_proxy(cls) -> None:
deadline = time.time() + 30
last_error = "origin was not reachable through the proxy"
while time.time() < deadline:
try:
body = cls.http_via_proxy(19080)
except OSError as error:
last_error = str(error)
time.sleep(0.2)
continue
if b"http-ok" in body:
return
last_error = body[:200].decode("latin1", errors="replace")
time.sleep(0.2)
raise AssertionError(last_error)
@classmethod
def _recv_until_close(cls, sock: socket.socket, limit: int = 8192) -> bytes:
sock.settimeout(CLOSE_TIMEOUT_S)
data = b""
while len(data) < limit:
try:
chunk = sock.recv(4096)
except ConnectionResetError:
return data
if not chunk:
return data
data += chunk
return data
@classmethod
def http_via_proxy(
cls,
port: int,
extra_headers: str = "",
connection: str = "close",
) -> bytes:
request = (
f"GET http://origin:{port}/ HTTP/1.1\r\n"
f"Host: origin:{port}\r\n"
f"{extra_headers}"
f"Connection: {connection}\r\n\r\n"
).encode()
with socket.create_connection(("127.0.0.1", cls.proxy_port), 5) as sock:
sock.sendall(request)
return cls._recv_until_close(sock)
@classmethod
def connect_via_proxy(cls, host: str, port: int, payload: bytes = b"CLIENT\n") -> tuple[bytes, bytes]:
with socket.create_connection(("127.0.0.1", cls.proxy_port), 5) as sock:
sock.sendall(f"CONNECT {host}:{port} HTTP/1.1\r\nHost: {host}:{port}\r\n\r\n".encode())
header = b""
sock.settimeout(CLOSE_TIMEOUT_S)
while b"\r\n\r\n" not in header:
chunk = sock.recv(4096)
if not chunk:
return header, b""
header += chunk
leftover = header.split(b"\r\n\r\n", 1)[1]
header = header[: header.index(b"\r\n\r\n") + 4]
if not payload:
return header, leftover
try:
sock.sendall(payload)
except OSError:
return header, leftover
return header, leftover + cls._recv_until_close(sock)
@classmethod
def fragmented_tls_via_proxy(cls, first_fragment_size: int) -> str:
incoming = ssl.MemoryBIO()
outgoing = ssl.MemoryBIO()
tls_context = ssl.create_default_context()
# The proxy image creates an ephemeral private CA. This test exercises
# protocol classification and the handshake, not CA distribution.
tls_context.check_hostname = False
tls_context.verify_mode = ssl.CERT_NONE
tls = tls_context.wrap_bio(
incoming,
outgoing,
server_side=False,
server_hostname="example.com",
)
try:
tls.do_handshake()
except ssl.SSLWantReadError:
pass
client_hello = outgoing.read()
if len(client_hello) <= first_fragment_size:
raise AssertionError("TLS ClientHello was unexpectedly short")
with socket.create_connection(("127.0.0.1", cls.proxy_port), 5) as sock:
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sock.sendall(b"CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n")
header = b""
while b"\r\n\r\n" not in header:
chunk = sock.recv(4096)
if not chunk:
raise AssertionError("proxy closed before the CONNECT response")
header += chunk
if b" 200 " not in header.split(b"\r\n", 1)[0]:
raise AssertionError(header.decode("latin1", errors="replace"))
sock.sendall(client_hello[:first_fragment_size])
time.sleep(0.05)
sock.sendall(client_hello[first_fragment_size:])
sock.settimeout(10)
return finish_memory_bio_handshake(tls, incoming, outgoing, sock, timeout_s=20)
@classmethod
def audit_records(cls) -> list[dict[str, object]]:
listed = subprocess.run(
[
"docker",
"exec",
cls.proxy,
"sh",
"-c",
"if [ -f /opt/maka-egress-state/hits.jsonl ]; then cat /opt/maka-egress-state/hits.jsonl; fi",
],
capture_output=True,
text=True,
timeout=COMMAND_TIMEOUT_S,
)
if listed.returncode != 0:
raise AssertionError(listed.stderr)
return [json.loads(line) for line in listed.stdout.splitlines() if line.strip()]
@classmethod
def origin_stats(cls) -> dict[str, int]:
# Read stats from inside the origin container so the probe does not
# depend on the proxy remaining willing to forward that port.
listed = subprocess.run(
[
"docker",
"exec",
cls.origin,
"python",
"-c",
"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:19084/').read().decode())",
],
capture_output=True,
text=True,
timeout=COMMAND_TIMEOUT_S,
)
if listed.returncode != 0:
raise AssertionError(listed.stderr)
return json.loads(listed.stdout)
@classmethod
def wait_for_origin_counter(cls, key: str, minimum: int) -> dict[str, int]:
deadline = time.time() + CLOSE_TIMEOUT_S
last = cls.origin_stats()
while time.time() < deadline:
if last.get(key, 0) >= minimum:
return last
time.sleep(0.05)
last = cls.origin_stats()
raise AssertionError(f"origin counter {key} did not reach {minimum}: {last}")
def test_https_and_plain_http_still_forward(self) -> None:
http = self.http_via_proxy(19080)
self.assertIn(b"http-ok", http)
curl = subprocess.run(
[
"curl",
"--silent",
"--show-error",
"--http1.1",
"--max-time",
"20",
"--proxy",
f"http://127.0.0.1:{self.proxy_port}",
"--insecure",
"--output",
"/dev/null",
"--write-out",
"%{http_code}",
"https://example.com/",
],
capture_output=True,
text=True,
timeout=COMMAND_TIMEOUT_S,
)
self.assertEqual(curl.stdout, "200", curl.stderr)
def test_fragmented_tls_record_prefix_still_handshakes(self) -> None:
# Three bytes are sufficient for mitmproxy's built-in TLS classifier
# and provide a control for the one- and two-byte fragmented prefixes.
for first_fragment_size in (1, 2, 3):
with self.subTest(first_fragment_size=first_fragment_size):
self.assertTrue(
self.fragmented_tls_via_proxy(first_fragment_size).startswith("TLS")
)
def test_connect_to_a_blocklisted_host_is_451(self) -> None:
header, _ = self.connect_via_proxy("tbench.ai", 443, b"")
self.assertIn(b"451", header.split(b"\r\n", 1)[0])
self.assertIn(b"tbench_domain", header)
def test_raw_connect_relays_no_bytes_and_is_audited(self) -> None:
closed_before = self.origin_stats()["raw_closed"]
header, body = self.connect_via_proxy("origin", 19081)
self.assertIn(b"200", header.split(b"\r\n", 1)[0])
self.assertNotIn(b"RAW-BANNER", body)
self.assertEqual(body, b"")
stats = self.wait_for_origin_counter("raw_closed", closed_before + 1)
self.assertEqual(stats["raw_recv"], 0)
self.assertIn(
{"host": "origin", "normalizedPath": ":19081", "ruleId": "raw_tunnel"},
[{key: record.get(key) for key in ("host", "normalizedPath", "ruleId")} for record in self.audit_records()],
)
def test_http_101_raw_upgrade_is_audited_without_relaying_the_banner(self) -> None:
closed_before = self.origin_stats()["upgrade_closed"]
data = self.http_via_proxy(
19083,
extra_headers="Upgrade: raw\r\n",
connection="Upgrade",
)
self.assertIn(b"101", data.split(b"\r\n", 1)[0])
self.assertNotIn(b"UPGRADE-BANNER", data)
stats = self.wait_for_origin_counter("upgrade_closed", closed_before + 1)
self.assertEqual(stats["upgrade_recv"], 0)
self.assertIn(
{"host": "origin", "normalizedPath": ":19083", "ruleId": "raw_tunnel"},
[{key: record.get(key) for key in ("host", "normalizedPath", "ruleId")} for record in self.audit_records()],
)
def test_websocket_upgrade_still_completes(self) -> None:
key = base64.b64encode(b"0123456789abcdef").decode()
data = self.http_via_proxy(
19082,
extra_headers=(
"Upgrade: websocket\r\n"
f"Sec-WebSocket-Key: {key}\r\n"
"Sec-WebSocket-Version: 13\r\n"
),
connection="Upgrade",
)
self.assertIn(b"101", data.split(b"\r\n", 1)[0])
self.assertIn(b"ws-ok", data)
self.assertNotIn(
":19082",
[str(record.get("normalizedPath")) for record in self.audit_records()],
)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,54 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import importlib.util
import unittest
from pathlib import Path
class EvalFrameworkTest(unittest.TestCase):
def test_selected_fails_closed_before_install(self) -> None:
module = self._load_fresh()
with self.assertRaisesRegex(RuntimeError, "not installed"):
module.selected()
def test_invalid_framework_fails_closed(self) -> None:
module = self._load_fresh()
with self.assertRaisesRegex(RuntimeError, "harbor or pier"):
module.install("other")
with self.assertRaisesRegex(RuntimeError, "not installed"):
module.selected()
def test_install_selects_harbor_and_pier(self) -> None:
module = self._load_fresh()
module.install("harbor")
self.assertEqual(module.selected(), "harbor")
module.install("pier")
self.assertEqual(module.selected(), "pier")
@staticmethod
def _load_fresh():
path = Path(__file__).with_name("eval_framework.py")
spec = importlib.util.spec_from_file_location("maka_eval_framework_under_test", path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,85 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import asyncio
import importlib
import os
import shutil
import sys
import tempfile
import types
import unittest
from pathlib import Path
from types import SimpleNamespace
class BaseAgent:
def __init__(self, *args, **kwargs):
pass
def load_relay():
from eval_framework import install
install("harbor")
package = types.ModuleType("harbor")
agents = types.ModuleType("harbor.agents")
base = types.ModuleType("harbor.agents.base")
base.BaseAgent = BaseAgent
sys.modules["harbor"] = package
sys.modules["harbor.agents"] = agents
sys.modules["harbor.agents.base"] = base
sys.modules.pop("relay_agent", None)
return importlib.import_module("relay_agent")
class ArtifactEnvironment:
def __init__(self, root: Path):
self.root = root
async def exec(self, command, cwd=None, timeout_sec=None):
return SimpleNamespace(return_code=0, stdout="", stderr="")
async def upload_file(self, source, target):
destination = self.root / target.lstrip("/")
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(source, destination)
class RelayArtifactTest(unittest.IsolatedAsyncioTestCase):
async def test_framed_transport_outputs_are_persisted_without_changing_the_carrier(self):
relay = load_relay()
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
environment = ArtifactEnvironment(root)
result = SimpleNamespace(stdout="framed-result\n", stderr="diagnostic\n")
await relay._persist_subject_outputs(environment, result)
self.assertEqual(
(root / relay.SUBJECT_STDOUT_PATH.lstrip("/")).read_text(),
"framed-result\n",
)
self.assertEqual(
(root / relay.SUBJECT_STDERR_PATH.lstrip("/")).read_text(),
"diagnostic\n",
)
self.assertEqual(result.stdout, "framed-result\n")
if __name__ == "__main__":
unittest.main()
+688
View File
@@ -0,0 +1,688 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import asyncio
import base64
import hashlib
import importlib
import inspect
import os
import shlex
import subprocess
import sys
import tempfile
import types
import unittest
import uuid
from pathlib import Path
from unittest.mock import patch
class BaseAgent:
def __init__(self, *args, **kwargs):
pass
class Environment:
def __init__(self, stage_upload: bool = False):
self.uploaded = b""
self.uploaded_target = None
self.stage_upload = stage_upload
async def upload_file(self, source: Path, target: str) -> None:
self.uploaded = source.read_bytes()
if self.stage_upload:
self.uploaded_target = Path(target)
self.uploaded_target.write_bytes(self.uploaded)
def load_relay(framework="harbor"):
from eval_framework import install
install(framework)
package = types.ModuleType(framework)
agents = types.ModuleType(f"{framework}.agents")
base = types.ModuleType(f"{framework}.agents.base")
base.BaseAgent = BaseAgent
sys.modules[framework] = package
sys.modules[f"{framework}.agents"] = agents
sys.modules[f"{framework}.agents.base"] = base
sys.modules.pop("relay_agent", None)
return importlib.import_module("relay_agent")
class RelayContractTest(unittest.TestCase):
def test_merged_noise_cannot_corrupt_or_escape_the_result_frame(self):
relay = load_relay()
token = "frame-token"
payload = b'{"kind":"settled","status":"completed"}'
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
frame = (
f"MAKA-EVAL-RESULT-V1 {token} {len(payload)} "
f"{hashlib.sha256(payload).hexdigest()} {encoded}\n"
)
sentinel = "credential-sentinel-must-not-persist"
stdout, diagnostic = relay._decode_result_carrier(
f"docker warning\n{frame}{sentinel}\n", token
)
self.assertEqual(stdout, payload.decode())
self.assertEqual(diagnostic["category"], "unstructured-output")
self.assertEqual(
diagnostic["bytes"], len(f"docker warning\n{sentinel}\n".encode())
)
self.assertNotIn(sentinel, str({"stdout": stdout, "diagnostic": diagnostic}))
def test_invalid_result_frames_fail_closed_with_bounded_evidence(self):
relay = load_relay()
token = "frame-token"
payload = b"{}"
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
valid = (
f"MAKA-EVAL-RESULT-V1 {token} {len(payload)} "
f"{hashlib.sha256(payload).hexdigest()} {encoded}\n"
)
cases = {
"noise only": "result-frame-missing",
valid + valid: "result-frame-ambiguous",
valid.replace(hashlib.sha256(payload).hexdigest(), "0" * 64): "result-frame-invalid",
}
for carrier, category in cases.items():
with self.subTest(category=category):
stdout, diagnostic = relay._decode_result_carrier(carrier, token)
self.assertEqual(stdout, "")
self.assertEqual(diagnostic["category"], category)
self.assertEqual(set(diagnostic), {"category", "bytes", "sha256"})
def test_non_utf8_result_payload_is_classified_as_an_invalid_frame(self):
relay = load_relay()
token = "frame-token"
payload = b"\xff"
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
frame = (
f"MAKA-EVAL-RESULT-V1 {token} {len(payload)} "
f"{hashlib.sha256(payload).hexdigest()} {encoded}\n"
)
stdout, diagnostic = relay._decode_result_carrier(frame, token)
self.assertEqual(stdout, "")
self.assertEqual(diagnostic["category"], "result-frame-invalid")
def test_oversized_result_carrier_is_rejected_before_parsing(self):
relay = load_relay()
carrier = "x" * (relay.RESULT_CARRIER_LIMIT_BYTES + 1)
stdout, diagnostic = relay._decode_result_carrier(carrier, "frame-token")
self.assertEqual(stdout, "")
self.assertEqual(diagnostic["category"], "result-frame-oversize")
def test_subject_output_cannot_counterfeit_scope_setup_failure(self):
relay = load_relay()
token = "0" * 32
carrier = f"subject output\nMAKA-EVAL-SCOPE-ERROR-V1 {token}\n"
_, diagnostic = relay._project_result(
types.SimpleNamespace(stdout=carrier),
{"captureStdout": False, "resultToken": token},
)
self.assertEqual(diagnostic["category"], "none")
def test_command_keeps_capture_and_control_files_out_of_task_workspace(self):
relay = load_relay()
environment = Environment()
command = asyncio.run(
relay._prepare_command(
environment,
{
"command": "/opt/agent",
"args": ["run"],
"environment": {},
"credentials": {},
"captureStdout": True,
"resultToken": "0" * 32,
},
"token",
"/logs/agent/.maka-eval-token.pid",
)
)
self.assertNotIn(".stdout", command)
self.assertNotIn(".stderr", command)
self.assertNotIn("/app", command)
self.assertIn("/logs/agent/.maka-eval-token.pid", command)
self.assertIn("2>/dev/null", command)
def test_command_does_not_start_subject_when_scope_pid_cannot_be_published(self):
relay = load_relay()
environment = Environment(stage_upload=True)
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
fake_bin = root / "bin"
fake_bin.mkdir()
fake_setsid = fake_bin / "setsid"
fake_setsid.write_text(
"#!/bin/sh\n"
"if [ \"$1\" = --wait ]; then shift; fi\n"
"exec \"$@\"\n"
)
fake_setsid.chmod(0o755)
marker = root / "subject-started"
scope_path = root / "missing" / "scope.pid"
command = asyncio.run(
relay._prepare_command(
environment,
{
"command": "/bin/sh",
"args": ["-c", f"touch {shlex.quote(str(marker))}"],
"environment": {},
"credentials": {},
"captureStdout": False,
"resultToken": "0" * 32,
},
"token",
str(scope_path),
)
)
try:
completed = subprocess.run(
command,
shell=True,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env={**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"},
)
self.assertFalse(marker.exists())
finally:
if environment.uploaded_target is not None:
environment.uploaded_target.unlink(missing_ok=True)
self.assertNotEqual(completed.returncode, 0)
stdout, diagnostic = relay._project_result(
types.SimpleNamespace(stdout=completed.stdout.decode()),
{"captureStdout": False, "resultToken": "0" * 32},
)
self.assertEqual(stdout, "")
self.assertEqual(diagnostic["category"], "execution-scope-unavailable")
def test_stages_environment_and_discards_unstructured_stdout(self):
relay = load_relay()
environment = Environment()
command = asyncio.run(
relay._prepare_command(
environment,
{
"command": "/opt/agent",
"args": ["run"],
"environment": {"MODE": "offline"},
"credentials": {"API_KEY": "canary-secret"},
"captureStdout": False,
"resultToken": "0" * 32,
},
"token",
"/tmp/scope.pid",
)
)
self.assertIn(b"export MODE=offline", environment.uploaded)
self.assertIn(b"export API_KEY=canary-secret", environment.uploaded)
self.assertNotIn("canary-secret", command)
self.assertIn(">/dev/null", command)
def test_leaves_no_credential_file_when_command_preparation_fails(self):
relay = load_relay()
environment = Environment()
named_temporary_file = tempfile.NamedTemporaryFile
with tempfile.TemporaryDirectory() as directory:
with patch.object(
relay.tempfile,
"NamedTemporaryFile",
side_effect=lambda *args, **kwargs: named_temporary_file(
*args, dir=directory, **kwargs
),
):
with self.assertRaisesRegex(RuntimeError, "invalid Maka Eval credential name"):
asyncio.run(
relay._prepare_command(
environment,
{
"command": "/opt/agent",
"args": ["run"],
"environment": {"MODE": "offline"},
"credentials": {
"API_KEY": "canary-secret",
"BAD-NAME": "ignored",
},
"captureStdout": False,
"resultToken": "0" * 32,
},
"token",
"/tmp/scope.pid",
)
)
self.assertEqual(list(Path(directory).iterdir()), [])
def test_removes_credential_file_when_subject_overrides_path(self):
relay = load_relay()
environment = Environment(stage_upload=True)
token = f"contract-{uuid.uuid4().hex}"
with tempfile.TemporaryDirectory() as directory:
command = asyncio.run(
relay._prepare_command(
environment,
{
"command": "/bin/sh",
"args": ["-c", "exit 0"],
"environment": {"PATH": "/definitely-missing"},
"credentials": {"API_KEY": "canary-secret"},
"captureStdout": True,
"resultToken": "0" * 32,
},
token,
str(Path(directory) / "scope.pid"),
)
)
target = environment.uploaded_target
self.assertIsNotNone(target)
try:
completed = subprocess.run(
["/bin/sh", "-c", shlex.split(command)[-1]],
check=False,
capture_output=True,
text=True,
)
self.assertEqual(completed.returncode, 0, completed.stderr)
self.assertFalse(target.exists())
finally:
target.unlink(missing_ok=True)
# Every default Docker capability except the two that bypass the policy.
CONSTRAINED_CAPABILITIES = 0xA80425FB & ~(1 << 13) & ~(1 << 12)
# The kernel's form for a namespace link target. Two distinct ones, because what
# the check compares is identity: the same string means the same namespace.
POLICY_NAMESPACE = "net:[4026532001]"
OWN_NAMESPACE = "net:[4026532099]"
class SubjectEnvironment:
"""Reports the evidence the probe reads: every `Cap` set, and both namespaces."""
def __init__(
self,
*,
permitted: int = CONSTRAINED_CAPABILITIES,
effective: int | None = None,
bounding: int | None = None,
namespace: str = POLICY_NAMESPACE,
default_user: str | None = None,
) -> None:
self.sets = {
"CapInh": 0,
"CapPrm": permitted,
"CapEff": permitted if effective is None else effective,
"CapBnd": permitted if bounding is None else bounding,
"CapAmb": 0,
}
self.namespace = namespace
self.default_user = default_user
self.commands: list[str] = []
self.users: list[str | None] = []
self.services: list[str] = []
async def exec(self, command: str, cwd=None, env=None, timeout_sec=None, user=None):
# Harbor 0.20.0 BaseEnvironment.exec takes user= as a named argument,
# not **kwargs. Absorbing unknown keywords here would hide a TypeError
# against the real Docker environment.
self.commands.append(command)
explicit = str(user) if user is not None else None
self.users.append(explicit if explicit is not None else self.default_user)
if "proxy-ipv4" in command:
return types.SimpleNamespace(
return_code=0,
stdout="MAKA-EVAL-PROXY-HOST-V1 172.18.0.2 maka-eval-mitmproxy\n",
stderr="",
)
reported = " ".join(f"{name}={value:016x}" for name, value in self.sets.items())
return types.SimpleNamespace(
return_code=0,
stdout=(
f"MAKA-EVAL-CAPABILITIES-V1 {reported}\n"
f"MAKA-EVAL-POLICY-NAMESPACE-V1 {self.namespace}\n"
),
stderr="",
)
async def service_exec(self, command: str, *, service: str, **kwargs):
self.services.append(service)
return types.SimpleNamespace(
return_code=0,
stdout=f"MAKA-EVAL-POLICY-NAMESPACE-V1 {POLICY_NAMESPACE}\n",
stderr="",
)
def _echoing_service_exec(answer: str):
async def service_exec(command: str, *, service: str, **kwargs):
return types.SimpleNamespace(
return_code=0,
stdout=f"MAKA-EVAL-POLICY-NAMESPACE-V1 {answer}\n",
stderr="",
)
return service_exec
class SubjectCapabilityTest(unittest.IsolatedAsyncioTestCase):
async def test_subject_holding_a_bypass_capability_never_starts(self):
relay = load_relay()
# Named here rather than read from the module: sourcing the expectation
# from the code under test would let deleting a capability delete its
# own coverage. NET_RAW grants AF_PACKET, NET_ADMIN grants `nft flush`.
for name, bit in (("NET_RAW", 1 << 13), ("NET_ADMIN", 1 << 12)):
with self.subTest(name):
environment = SubjectEnvironment(permitted=bit | (1 << 21))
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
with self.assertRaises(RuntimeError) as raised:
await relay._require_constrained_subject(environment)
self.assertIn(name, str(raised.exception))
async def test_capability_reachable_only_through_the_bounding_set_never_starts(self):
relay = load_relay()
# A non-root subject reports an empty effective set while a
# `cap_net_raw+ep` executable still reacquires whatever the bounding set
# kept. Verified in a container: with the capability bounded away the
# same executable cannot be run at all.
environment = SubjectEnvironment(
permitted=0,
effective=0,
bounding=CONSTRAINED_CAPABILITIES | (1 << 13),
)
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
with self.assertRaises(RuntimeError) as raised:
await relay._require_constrained_subject(environment)
self.assertIn("NET_RAW", str(raised.exception))
async def test_subject_outside_the_policy_namespace_never_starts(self):
relay = load_relay()
# Harbor applies the policy inside the sidecar and respects a task's own
# networking on `main`, so a task that declares it leaves the subject in
# a namespace no policy was ever applied to.
environment = SubjectEnvironment(namespace=OWN_NAMESPACE)
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
with self.assertRaises(RuntimeError) as raised:
await relay._require_constrained_subject(environment)
self.assertIn("network namespace", str(raised.exception))
async def test_policy_namespace_is_read_from_the_service_that_applies_the_policy(self):
relay = load_relay()
# The comparison is only meaningful against the namespace the policy was
# installed in, and Harbor installs it by running `network-policy` in the
# egress sidecar. Reading the other side from anywhere else would compare
# the subject against a namespace nothing was applied to.
environment = SubjectEnvironment()
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
await relay._require_constrained_subject(environment)
self.assertEqual(environment.services, ["harbor-docker-egress-control-sidecar"])
async def test_constrained_subject_proceeds(self):
relay = load_relay()
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
await relay._require_constrained_subject(SubjectEnvironment())
async def test_unreadable_evidence_fails_closed(self):
relay = load_relay()
class SilentEnvironment:
async def exec(self, command, cwd=None, timeout_sec=None):
return types.SimpleNamespace(return_code=0, stdout="", stderr="")
async def service_exec(self, command, *, service, **kwargs):
return types.SimpleNamespace(return_code=0, stdout="", stderr="")
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
with self.assertRaises(RuntimeError):
await relay._require_constrained_subject(SilentEnvironment())
async def test_answer_that_is_not_a_namespace_identity_fails_closed(self):
relay = load_relay()
# Two sides that both failed to answer must not compare equal. Anything
# that is not the kernel's own form is not an identity to compare.
for answer in ("", "shared", "net:[]"):
with self.subTest(answer):
environment = SubjectEnvironment(namespace=answer)
environment.service_exec = _echoing_service_exec(answer)
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
with self.assertRaises(RuntimeError):
await relay._require_constrained_subject(environment)
async def test_check_is_scoped_to_runs_that_require_the_proxy(self):
relay = load_relay()
environment = SubjectEnvironment(permitted=1 << 13, namespace=OWN_NAMESPACE)
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": ""}):
await relay._require_constrained_subject(environment)
self.assertEqual(environment.commands, [])
self.assertEqual(environment.services, [])
async def test_missing_published_proxy_address_fails_closed(self):
relay = load_relay()
class MissingAddressEnvironment(SubjectEnvironment):
async def exec(self, command, cwd=None, env=None, timeout_sec=None, user=None):
if "proxy-ipv4" in command:
self.commands.append(command)
explicit = str(user) if user is not None else None
self.users.append(
explicit if explicit is not None else self.default_user
)
return types.SimpleNamespace(return_code=1, stdout="", stderr="")
return await super().exec(
command, cwd=cwd, env=env, timeout_sec=timeout_sec, user=user
)
environment = MissingAddressEnvironment()
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
with self.assertRaises(RuntimeError) as raised:
await relay._require_constrained_subject(environment)
self.assertIn("pin the Eval egress proxy hostname", str(raised.exception))
pin_users = [
user
for command, user in zip(environment.commands, environment.users, strict=True)
if "proxy-ipv4" in command
]
self.assertEqual(pin_users, ["root"])
async def test_proxy_hostname_pin_runs_as_root(self):
relay = load_relay()
environment = SubjectEnvironment()
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
await relay._require_constrained_subject(environment)
pin_users = [
user
for command, user in zip(environment.commands, environment.users, strict=True)
if "proxy-ipv4" in command
]
self.assertEqual(pin_users, ["root"])
async def test_proxy_hostname_pin_does_not_inherit_a_non_root_default_user(self):
# Harbor scopes every exec to the task's agent.user unless the call
# names another identity. Pinning /etc/hosts is infrastructure, so it
# must still be root when that default is a supported non-root user.
relay = load_relay()
environment = SubjectEnvironment(default_user="1000")
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
await relay._require_constrained_subject(environment)
probe_users = [
user
for command, user in zip(environment.commands, environment.users, strict=True)
if "proxy-ipv4" not in command
]
pin_users = [
user
for command, user in zip(environment.commands, environment.users, strict=True)
if "proxy-ipv4" in command
]
self.assertEqual(probe_users, ["1000"])
self.assertEqual(pin_users, ["root"])
async def test_invalid_proxy_hostname_is_rejected_before_hosts_rewrite(self):
relay = load_relay()
for host in (
".*",
"proxy*",
"evil.com\n127.0.0.1 pwned",
"foo bar",
"foo..bar",
".hidden",
"maka-évál",
):
with self.subTest(host=host):
environment = SubjectEnvironment()
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_ALLOWED_HOST": host}):
with self.assertRaisesRegex(RuntimeError, "proxy host is invalid"):
await relay._pin_proxy_hostname(environment)
self.assertEqual(environment.commands, [])
async def test_proxy_hostname_pin_matches_hosts_aliases_exactly(self):
relay = load_relay()
environment = SubjectEnvironment()
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
await relay._require_constrained_subject(environment)
pin = next(command for command in environment.commands if "proxy-ipv4" in command)
self.assertIn(relay.HOSTS_ALIAS_AWK, pin)
completed = subprocess.run(
["awk", "-v", "host=maka-eval-mitmproxy", relay.HOSTS_ALIAS_AWK],
input=(
"127.0.0.1 localhost\n"
"10.0.0.1 maka-eval-mitmproxy extra\n"
"10.0.0.2 prefix-maka-eval-mitmproxy\n"
"10.0.0.3 other.example\n"
"10.0.0.4 other.example maka-eval-mitmproxy\n"
),
check=True,
capture_output=True,
text=True,
)
self.assertEqual(
completed.stdout,
"127.0.0.1 localhost\n"
"10.0.0.2 prefix-maka-eval-mitmproxy\n"
"10.0.0.3 other.example\n",
)
def test_harbor_0_20_0_exec_declares_user_as_a_named_parameter(self):
# Harbor 0.20.0 BaseEnvironment.exec / DockerEnvironment.exec:
# (command, cwd=None, env=None, timeout_sec=None, user=None)
# A **kwargs-only double would absorb user= and hide a TypeError
# against that signature.
parameter = inspect.signature(SubjectEnvironment.exec).parameters["user"]
self.assertNotEqual(parameter.kind, inspect.Parameter.VAR_KEYWORD)
inspect.signature(SubjectEnvironment.exec).bind(
None, "true", user="root"
)
def test_published_ipv4_requires_four_0_255_octets(self):
relay = load_relay()
accepted = ("172.18.0.2", "0.1.2.3", "255.255.255.255", "10.0.0.1")
rejected = (
"999.1.1.1",
"1.2.3",
"1..2.3",
"1.2.3.4.5",
"01.2.3.4",
"1.2.3.08",
"...",
"",
"172.18.0.2/32",
"localhost",
)
for ip in accepted:
with self.subTest(ip=ip):
self.assertTrue(relay._valid_published_ipv4(ip))
for ip in rejected:
with self.subTest(ip=ip):
self.assertFalse(relay._valid_published_ipv4(ip))
for ip in (*accepted, *rejected):
with self.subTest(awk=ip):
completed = subprocess.run(
["awk", relay.IPV4_OCTET_AWK],
input=f"{ip}\n",
capture_output=True,
text=True,
)
self.assertEqual(
completed.returncode == 0,
relay._valid_published_ipv4(ip),
ip,
)
def test_network_policy_uses_the_same_ipv4_octet_check(self):
relay = load_relay()
policy = Path(__file__).with_name("egress-proxy").joinpath("network-policy")
self.assertIn(relay.IPV4_OCTET_AWK.strip(), policy.read_text())
def test_network_policy_rejects_docker_dns_before_local_accept(self):
policy = Path(__file__).with_name("egress-proxy").joinpath("network-policy").read_text()
reject = policy.index("ip daddr 127.0.0.11 reject")
local = policy.index("fib daddr type local accept")
self.assertLess(reject, local)
async def test_malformed_published_proxy_address_fails_closed(self):
relay = load_relay()
class BadAddressEnvironment(SubjectEnvironment):
async def exec(self, command, cwd=None, env=None, timeout_sec=None, user=None):
if "proxy-ipv4" in command:
self.commands.append(command)
explicit = str(user) if user is not None else None
self.users.append(
explicit if explicit is not None else self.default_user
)
return types.SimpleNamespace(
return_code=0,
stdout="MAKA-EVAL-PROXY-HOST-V1 999.1.1.1 maka-eval-mitmproxy\n",
stderr="",
)
return await super().exec(
command, cwd=cwd, env=env, timeout_sec=timeout_sec, user=user
)
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
with self.assertRaises(RuntimeError) as raised:
await relay._require_constrained_subject(BadAddressEnvironment())
self.assertIn("pin the Eval egress proxy hostname", str(raised.exception))
class FrameworkSelectionTest(unittest.TestCase):
def test_relay_loads_harbor_and_pier_from_the_installed_selection(self):
for framework in ("harbor", "pier"):
with self.subTest(framework):
relay = load_relay(framework)
self.assertEqual(relay.framework, framework)
def test_relay_does_not_select_a_framework_from_the_environment(self):
os.environ["MAKA_EVAL_FRAMEWORK"] = "pier"
relay = load_relay("harbor")
self.assertEqual(relay.framework, "harbor")
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,832 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import asyncio
import base64
import contextlib
import hashlib
import importlib
import json
import os
import signal
import shutil
import subprocess
import sys
import tempfile
import time
import types
import unittest
from pathlib import Path
from types import SimpleNamespace
class BaseAgent:
def __init__(self, *args, **kwargs):
pass
class LocalEnvironment:
async def upload_file(self, source, target):
if target.startswith("/logs/"):
return
shutil.copyfile(source, target)
async def download_file(self, source, target):
shutil.copyfile(source, target)
async def exec(self, command, cwd=None, timeout_sec=None):
if command.startswith("mkdir -p /logs/artifacts"):
return SimpleNamespace(return_code=0, stdout="", stderr="")
completed = await asyncio.to_thread(
subprocess.run,
command,
cwd=cwd,
shell=True,
executable="/bin/bash",
check=False,
timeout=timeout_sec,
)
return SimpleNamespace(return_code=completed.returncode)
class SimultaneousEnvironment:
def __init__(self):
self.release = asyncio.Event()
self.started = asyncio.Event()
self.finished = asyncio.Event()
async def upload_file(self, source, target):
if target.startswith("/logs/"):
return
shutil.copyfile(source, target)
async def download_file(self, source, target):
shutil.copyfile(source, target)
async def exec(self, command, cwd=None, timeout_sec=None):
if command.startswith("printf ") and "MAKA-EVAL-CWD-V1" in command:
prefix = command.split("'", 2)[1]
return SimpleNamespace(
return_code=0, stdout=f"docker warning\n{prefix}/workspace\n", stderr=None
)
if command.startswith("setsid"):
self.started.set()
await self.release.wait()
payload = b'{"kind":"settled"}'
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
frame = (
f"MAKA-EVAL-RESULT-V1 {'0' * 32} {len(payload)} "
f"{hashlib.sha256(payload).hexdigest()} {encoded}\n"
)
self.finished.set()
return SimpleNamespace(return_code=0, stdout=frame, stderr=None)
if command.startswith("test -r") or command.startswith("pgid="):
return SimpleNamespace(return_code=3, stdout="", stderr="")
return SimpleNamespace(return_code=0, stdout="", stderr="")
class ClosedWriter:
def is_closing(self):
return False
def write(self, _value):
raise BrokenPipeError
async def drain(self):
raise AssertionError("drain must not run after a failed write")
class HangingStopEnvironment:
async def stop(self, delete=False):
await asyncio.Future()
class ScopeEvidenceEnvironment:
def __init__(self, return_code):
self.return_code = return_code
async def exec(self, command, cwd=None, timeout_sec=None):
return SimpleNamespace(return_code=self.return_code, stdout="", stderr="")
class ScopeSetupFailureEnvironment:
async def upload_file(self, source, target):
return None
async def exec(self, command, cwd=None, timeout_sec=None):
if command.startswith("printf ") and "MAKA-EVAL-CWD-V1" in command:
prefix = command.split("'", 2)[1]
return SimpleNamespace(return_code=0, stdout=f"{prefix}/workspace\n", stderr=None)
if command.startswith("setsid"):
token = "0" * 32
return SimpleNamespace(
return_code=111,
stdout=f"MAKA-EVAL-SCOPE-ERROR-V1 {token}\n",
stderr=None,
)
if command.startswith("rm -f --"):
return SimpleNamespace(return_code=0, stdout="", stderr="")
if command.startswith("mkdir -p /logs/artifacts"):
return SimpleNamespace(return_code=0, stdout="", stderr="")
raise AssertionError(f"unexpected command after scope setup failure: {command}")
class FrameworkTimeoutEnvironment(SimultaneousEnvironment):
def __init__(self):
super().__init__()
self.stopped = False
async def exec(self, command, cwd=None, timeout_sec=None):
if command.startswith("setsid"):
self.started.set()
await asyncio.Future()
return await super().exec(command, cwd=cwd, timeout_sec=timeout_sec)
async def stop(self, delete=False):
self.stopped = delete
class LiveScopeEnvironment(SimultaneousEnvironment):
"""A subject whose process group outlives it, as a task's own service does."""
def __init__(self):
super().__init__()
self.signalled = False
self.commands = []
async def exec(self, command, cwd=None, timeout_sec=None):
self.commands.append(command)
if _is_teardown(command):
self.signalled = True
return SimpleNamespace(return_code=0, stdout="", stderr="")
if command.startswith("pgid="):
return SimpleNamespace(return_code=3 if self.signalled else 0, stdout="", stderr="")
return await super().exec(command, cwd=cwd, timeout_sec=timeout_sec)
class TransportLossEnvironment(SimultaneousEnvironment):
async def exec(self, command, cwd=None, timeout_sec=None):
if "kill -TERM" in command:
self.release.set()
return SimpleNamespace(return_code=0, stdout="", stderr=None)
return await super().exec(command, cwd=cwd, timeout_sec=timeout_sec)
class SettleCompletionEnvironment(SimultaneousEnvironment):
async def exec(self, command, cwd=None, timeout_sec=None):
if "kill -TERM" in command:
self.release.set()
return SimpleNamespace(return_code=0, stdout="", stderr=None)
return await super().exec(command, cwd=cwd, timeout_sec=timeout_sec)
def _is_teardown(command: str) -> bool:
# `kill -0` is how the relay asks whether the scope is still there; only
# TERM and KILL end it.
return "kill -TERM" in command or "kill -KILL" in command
class RecordingEnvironment:
"""Runs a subject that exits with a chosen code and records every command."""
def __init__(self, return_code, status):
self.return_code = return_code
self.status = status
self.signalled = False
self.commands = []
async def upload_file(self, source, target):
return None
async def download_file(self, source, target):
shutil.copyfile(source, target)
async def exec(self, command, cwd=None, timeout_sec=None):
self.commands.append(command)
if command.startswith("printf ") and "MAKA-EVAL-CWD-V1" in command:
prefix = command.split("'", 2)[1]
return SimpleNamespace(return_code=0, stdout=f"{prefix}/workspace\n", stderr=None)
if command.startswith("setsid"):
payload = ('{"kind":"settled","status":"%s"}' % self.status).encode()
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
frame = (
f"MAKA-EVAL-RESULT-V1 {'0' * 32} {len(payload)} "
f"{hashlib.sha256(payload).hexdigest()} {encoded}\n"
)
return SimpleNamespace(return_code=self.return_code, stdout=frame, stderr=None)
# The subject's process group outlives it, as a task's own service does,
# and dies once something signals it. A teardown would therefore succeed
# here rather than fail for its own reasons -- what this test asserts is
# that no teardown is attempted at all.
if _is_teardown(command):
self.signalled = True
return SimpleNamespace(return_code=0, stdout="", stderr="")
if command.startswith("pgid="):
return SimpleNamespace(return_code=3 if self.signalled else 0, stdout="", stderr="")
return SimpleNamespace(return_code=0, stdout="", stderr="")
def load_relay():
from eval_framework import install
install("harbor")
package = types.ModuleType("harbor")
agents = types.ModuleType("harbor.agents")
base = types.ModuleType("harbor.agents.base")
base.BaseAgent = BaseAgent
sys.modules["harbor"] = package
sys.modules["harbor.agents"] = agents
sys.modules["harbor.agents.base"] = base
sys.modules.pop("relay_agent", None)
return importlib.import_module("relay_agent")
def run_host_teardown_probe(marker: Path, fail_cleanup: bool) -> subprocess.CompletedProcess:
cleanup_failure = 'raise RuntimeError("cleanup failed")' if fail_cleanup else ""
script = f"""
import asyncio
import os
import signal
import sys
import types
from pathlib import Path
import run_trial as module
relay = types.ModuleType("relay_agent")
relay.request_host_teardown = lambda: None
sys.modules["relay_agent"] = relay
async def trial(*_args):
try:
asyncio.get_running_loop().call_soon(os.kill, os.getpid(), signal.SIGTERM)
await asyncio.Future()
finally:
Path({str(marker)!r}).write_text("attempted")
{cleanup_failure}
module.run_trial = trial
sys.argv = ["run_trial.py", "harbor", "test", "/tmp/test-config"]
asyncio.run(module.main())
"""
return subprocess.run(
[sys.executable, "-c", script],
cwd=Path(__file__).parent,
capture_output=True,
text=True,
check=False,
)
class RelayLifecycleTest(unittest.IsolatedAsyncioTestCase):
def test_host_teardown_exits_successfully_after_trial_unwinds(self):
with tempfile.TemporaryDirectory() as directory:
marker = Path(directory) / "cleanup-attempted"
completed = run_host_teardown_probe(marker, False)
self.assertEqual(completed.returncode, 0, completed.stderr)
self.assertEqual(marker.read_text(), "attempted")
def test_host_teardown_failure_remains_nonzero(self):
with tempfile.TemporaryDirectory() as directory:
marker = Path(directory) / "cleanup-attempted"
completed = run_host_teardown_probe(marker, True)
self.assertNotEqual(completed.returncode, 0)
self.assertEqual(marker.read_text(), "attempted")
async def test_scope_setup_failure_is_reported_without_quiescing_an_unknown_scope(self):
relay = load_relay()
environment = ScopeSetupFailureEnvironment()
token = f"scope-setup-{os.getpid()}"
connected = asyncio.get_running_loop().create_future()
async def accept(reader, writer):
connected.set_result((reader, writer))
server = await asyncio.start_server(accept, "127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
agent = relay.RelayAgent(
logs_dir=Path(tempfile.gettempdir()),
relay_host="127.0.0.1",
relay_port=port,
relay_token=token,
teardown_timeout_ms=1_000,
)
running = asyncio.create_task(agent.run("solve", environment, None))
reader, writer = await connected
try:
await reader.readline()
writer.write(
(__import__("json").dumps({
"token": token,
"kind": "execute",
"command": "/bin/true",
"args": [],
"credentials": {},
"captureStdout": False,
"resultToken": "0" * 32,
}) + "\n").encode()
)
await writer.drain()
executed = __import__("json").loads(await reader.readline())
self.assertEqual(executed["exitCode"], 111)
self.assertEqual(executed["diagnostic"]["category"], "execution-scope-unavailable")
writer.write(
(__import__("json").dumps({"token": token, "kind": "verify"}) + "\n").encode()
)
await writer.drain()
await running
finally:
writer.close()
server.close()
await server.wait_closed()
async def test_missing_scope_evidence_cannot_be_treated_as_quiescent(self):
relay = load_relay()
with self.assertRaisesRegex(RuntimeError, "scope evidence was unavailable"):
await relay._scope_active(ScopeEvidenceEnvironment(4), "/", "/missing")
self.assertFalse(await relay._scope_active(ScopeEvidenceEnvironment(3), "/", "/gone"))
async def test_closed_peer_is_a_bounded_transport_outcome(self):
relay = load_relay()
delivered = await relay._send(ClosedWriter(), {"kind": "verify"})
self.assertFalse(delivered)
async def test_destroy_fallback_obeys_the_teardown_deadline(self):
relay = load_relay()
execution = asyncio.create_task(asyncio.sleep(60))
try:
await asyncio.wait_for(
relay._settle_or_destroy(
HangingStopEnvironment(), "/", "/tmp/missing", execution, 0.01
),
timeout=0.1,
)
finally:
execution.cancel()
with self.assertRaises(asyncio.CancelledError):
await execution
async def test_peer_close_during_execution_triggers_bounded_settlement(self):
relay = load_relay()
environment = TransportLossEnvironment()
token = f"transport-loss-{os.getpid()}"
connected = asyncio.get_running_loop().create_future()
async def accept(reader, writer):
connected.set_result((reader, writer))
server = await asyncio.start_server(accept, "127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
agent = relay.RelayAgent(
logs_dir=Path(tempfile.gettempdir()),
relay_host="127.0.0.1",
relay_port=port,
relay_token=token,
teardown_timeout_ms=1_000,
)
running = asyncio.create_task(agent.run("solve", environment, None))
reader, writer = await connected
try:
await reader.readline()
writer.write(
(
__import__("json").dumps(
{
"token": token,
"kind": "execute",
"command": "/bin/true",
"args": [],
"credentials": {},
"resultToken": "0" * 32,
}
)
+ "\n"
).encode()
)
await writer.drain()
writer.close()
await writer.wait_closed()
await asyncio.wait_for(running, timeout=0.5)
self.assertTrue(environment.release.is_set())
finally:
server.close()
await server.wait_closed()
async def test_terminal_execution_wins_when_framework_cancellation_is_observed(self):
relay = load_relay()
environment = LiveScopeEnvironment()
token = f"simultaneous-{os.getpid()}"
connected = asyncio.get_running_loop().create_future()
async def accept(reader, writer):
connected.set_result((reader, writer))
server = await asyncio.start_server(accept, "127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
agent = relay.RelayAgent(
logs_dir=Path(tempfile.gettempdir()),
relay_host="127.0.0.1",
relay_port=port,
relay_token=token,
teardown_timeout_ms=110_000,
)
running = asyncio.create_task(agent.run("solve", environment, None))
reader, writer = await connected
try:
ready = __import__("json").loads(await reader.readline())
self.assertEqual(ready["kind"], "ready")
self.assertEqual(ready["cwd"], "/workspace")
writer.write(
(
__import__("json").dumps(
{
"token": token,
"kind": "execute",
"command": "/bin/true",
"args": [],
"credentials": {},
"resultToken": "0" * 32,
}
)
+ "\n"
).encode()
)
await writer.drain()
environment.release.set()
await environment.finished.wait()
relay.request_host_teardown()
running.cancel()
executed = __import__("json").loads(await reader.readline())
self.assertEqual(executed["termination"], "exited")
self.assertEqual(executed["exitCode"], 0)
# Cancellation found the subject already stopped, and a stopped
# subject is reported as one -- so the verifier reads the same
# environment it would have read without the cancellation. Tearing
# its scope down here would edit that environment for exactly the
# runs the framework happened to interrupt.
self.assertEqual(
[command for command in environment.commands if _is_teardown(command)],
[],
)
with self.assertRaises(asyncio.CancelledError):
await running
finally:
writer.close()
with contextlib.suppress(ConnectionError, asyncio.CancelledError):
await writer.wait_closed()
server.close()
await server.wait_closed()
Path(f"/tmp/maka-eval-{token}.env").unlink(missing_ok=True)
async def test_framework_cancellation_remains_timeout_when_settlement_induces_exit(self):
relay = load_relay()
environment = SettleCompletionEnvironment()
token = f"host-terminal-{os.getpid()}"
connected = asyncio.get_running_loop().create_future()
async def accept(reader, writer):
connected.set_result((reader, writer))
server = await asyncio.start_server(accept, "127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
agent = relay.RelayAgent(
logs_dir=Path(tempfile.gettempdir()), relay_host="127.0.0.1",
relay_port=port, relay_token=token, teardown_timeout_ms=1_000,
)
running = asyncio.create_task(agent.run("solve", environment, None))
reader, writer = await connected
try:
await reader.readline()
writer.write((__import__("json").dumps({
"token": token, "kind": "execute", "command": "/bin/true", "args": [],
"credentials": {}, "resultToken": "0" * 32,
}) + "\n").encode())
await writer.drain()
await environment.started.wait()
running.cancel()
executed = __import__("json").loads(await asyncio.wait_for(reader.readline(), 0.5))
self.assertEqual(executed["termination"], "framework_timeout")
self.assertEqual(executed["exitCode"], 124)
with self.assertRaises(asyncio.CancelledError):
await running
finally:
writer.close()
server.close()
await server.wait_closed()
async def test_framework_timeout_survives_destroy_fallback(self):
relay = load_relay()
environment = FrameworkTimeoutEnvironment()
token = f"framework-destroy-{os.getpid()}"
connected = asyncio.get_running_loop().create_future()
async def accept(reader, writer):
connected.set_result((reader, writer))
server = await asyncio.start_server(accept, "127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
agent = relay.RelayAgent(
logs_dir=Path(tempfile.gettempdir()), relay_host="127.0.0.1",
relay_port=port, relay_token=token, teardown_timeout_ms=50,
)
running = asyncio.create_task(agent.run("solve", environment, None))
reader, writer = await connected
try:
await reader.readline()
writer.write((__import__("json").dumps({
"token": token, "kind": "execute", "command": "/bin/true", "args": [],
"credentials": {}, "resultToken": "0" * 32,
}) + "\n").encode())
await writer.drain()
await environment.started.wait()
running.cancel()
executed = __import__("json").loads(await asyncio.wait_for(reader.readline(), 0.5))
self.assertEqual(executed["termination"], "framework_timeout")
self.assertEqual(executed["exitCode"], 124)
self.assertEqual(executed["diagnostic"]["category"], "result-frame-missing")
self.assertTrue(environment.stopped)
with self.assertRaises(asyncio.CancelledError):
await running
finally:
writer.close()
server.close()
await server.wait_closed()
@unittest.skipUnless(shutil.which("setsid"), "requires GNU setsid")
async def test_scope_waits_for_child_and_publishes_bounded_stdout(self):
relay = load_relay()
environment = LocalEnvironment()
token = f"test-{os.getpid()}"
scope_path = f"/tmp/maka-eval-{token}.pid"
request = {
"command": "/bin/sh",
"args": ["-c", "sleep 0.1; printf result-json"],
"credentials": {},
"resultToken": "0" * 32,
}
try:
command = await relay._prepare_command(environment, request, token, scope_path)
started = time.monotonic()
result = subprocess.run(
command,
cwd=tempfile.gettempdir(),
shell=True,
check=False,
timeout=2,
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0)
self.assertGreaterEqual(time.monotonic() - started, 0.09)
self.assertEqual(result.stdout, "result-json")
finally:
Path(scope_path).unlink(missing_ok=True)
@unittest.skipUnless(shutil.which("setsid"), "requires GNU setsid")
async def test_settle_kills_descendants_before_verification_boundary(self):
relay = load_relay()
environment = LocalEnvironment()
token = f"descendant-{os.getpid()}"
scope_path = f"/tmp/maka-eval-{token}.pid"
with tempfile.TemporaryDirectory() as directory:
late_write = Path(directory) / "late-write"
child_ready = Path(directory) / "child-ready"
release_pipe = Path(directory) / "release"
os.mkfifo(release_pipe)
# Keep the FIFO open from the test side so the descendant can report
# readiness and remain blocked until settlement has completed.
release_fd = os.open(release_pipe, os.O_RDWR | os.O_NONBLOCK)
child_pid = None
execution = None
request = {
"command": sys.executable,
"args": [
"-c",
"\n".join(
[
"import os,time",
f"release_pipe={str(release_pipe)!r}",
f"child_ready={str(child_ready)!r}",
f"late_write={str(late_write)!r}",
"child=os.fork()",
"if child == 0:",
" with open(release_pipe, 'rb', buffering=0) as pipe:",
" ready=child_ready+'.tmp'",
" with open(ready, 'w') as marker:",
" marker.write(str(os.getpid()))",
" os.replace(ready, child_ready)",
" pipe.read(1)",
" with open(late_write, 'w') as late:",
" late.write('late')",
" os._exit(0)",
"deadline=time.monotonic()+5",
"while not os.path.exists(child_ready) and time.monotonic() < deadline:",
" time.sleep(0.01)",
"os._exit(0 if os.path.exists(child_ready) else 2)",
]
),
],
"credentials": {},
"resultToken": "0" * 32,
"cwd": directory,
}
try:
command = await relay._prepare_command(environment, request, token, scope_path)
execution = asyncio.create_task(
asyncio.to_thread(
subprocess.run,
command,
cwd=directory,
shell=True,
check=False,
timeout=2,
)
)
deadline = time.monotonic() + 1
while True:
scope = Path(scope_path)
pid = scope.read_text().strip() if scope.exists() else ""
if pid.isdigit():
break
if time.monotonic() >= deadline:
self.fail("scope PID was not published")
await asyncio.sleep(0.01)
deadline = time.monotonic() + 1
while not child_ready.exists():
if time.monotonic() >= deadline:
self.fail("descendant did not become ready")
await asyncio.sleep(0.01)
child_pid = int(child_ready.read_text())
self.assertEqual(os.getpgid(child_pid), int(pid))
completed = await execution
self.assertEqual(completed.returncode, 0)
result = await relay._settle(environment, directory, scope_path, execution)
self.assertEqual(result.returncode, 0)
# A broken quiescence check leaves the descendant alive. Releasing
# it now makes that bug observable as a late workspace write.
os.write(release_fd, b"x")
deadline = time.monotonic() + 1
while not late_write.exists() and time.monotonic() < deadline:
await asyncio.sleep(0.01)
self.assertFalse(late_write.exists())
finally:
with contextlib.suppress(OSError):
os.write(release_fd, b"x")
if child_pid is None and child_ready.exists():
with contextlib.suppress(OSError, ValueError):
child_pid = int(child_ready.read_text())
if child_pid is not None:
with contextlib.suppress(ProcessLookupError):
os.kill(child_pid, signal.SIGKILL)
if execution is not None and not execution.done():
with contextlib.suppress(Exception):
await relay._signal(environment, directory, scope_path, "KILL")
with contextlib.suppress(asyncio.CancelledError, Exception):
await asyncio.wait_for(execution, timeout=1)
os.close(release_fd)
Path(scope_path).unlink(missing_ok=True)
async def test_an_exited_subject_is_left_alone_whatever_it_reported(self):
# The verifier scores the environment the task was left in, so a subject
# that stopped on its own keeps whatever it started — a service the task
# was asked to run has to still be there. Reading the exit code here to
# decide otherwise would make the measurement depend on the framework's
# own classification, and only for the subjects it classifies as failed.
for return_code, status in ((0, "completed"), (1, "failed")):
with self.subTest(return_code=return_code):
relay = load_relay()
environment = RecordingEnvironment(return_code, status)
token = f"exited-{return_code}-{os.getpid()}"
connected = asyncio.get_running_loop().create_future()
async def accept(reader, writer):
connected.set_result((reader, writer))
server = await asyncio.start_server(accept, "127.0.0.1", 0)
port = server.sockets[0].getsockname()[1]
agent = relay.RelayAgent(
logs_dir=Path(tempfile.gettempdir()),
relay_host="127.0.0.1",
relay_port=port,
relay_token=token,
teardown_timeout_ms=1_000,
)
running = asyncio.create_task(agent.run("solve", environment, None))
reader, writer = await connected
try:
await reader.readline()
writer.write(
(
json.dumps(
{
"token": token,
"kind": "execute",
"command": "/bin/true",
"args": [],
"credentials": {},
"resultToken": "0" * 32,
}
)
+ "\n"
).encode()
)
await writer.drain()
executed = json.loads(await reader.readline())
self.assertEqual(executed["termination"], "exited")
self.assertEqual(executed["exitCode"], return_code)
writer.write(
(json.dumps({"token": token, "kind": "verify"}) + "\n").encode()
)
await writer.drain()
await asyncio.wait_for(running, timeout=2)
finally:
writer.close()
with contextlib.suppress(Exception):
await writer.wait_closed()
server.close()
await server.wait_closed()
self.assertEqual(
[command for command in environment.commands if _is_teardown(command)],
[],
)
@unittest.skipUnless(shutil.which("node"), "requires Node.js")
def test_deepseek_harness_toolchain_patch_preserves_background_descendants(self):
patcher = (
Path(__file__).parent
/ "deepseek-harness-toolchain"
/ "patch-subprocess-local.mjs"
)
with tempfile.TemporaryDirectory() as directory:
target = (
Path(directory)
/ "node_modules"
/ "@deepseek-ai"
/ "dsh-subprocess-local"
/ "lib"
/ "index.js"
)
target.parent.mkdir(parents=True)
target.write_text(
"""\
\tterminateForHostExit() {
\t\tthis.forceStopDescendants();
\t\tthis.forceStopShell();
\t\tthis.forceStopDescendants();
\t}
\tasync closeOnce() {
\t\tlet survivors = await this.stopDescendants();
\t\tif (survivors.length > 0) throw new Error(`terminal cleanup failed; surviving pids: ${survivors.map((member) => member.pid).join(", ")}`);
\t\tawait this.stopShell();
\t\tsurvivors = await this.stopDescendants();
\t\tif (survivors.length > 0) throw new Error(`terminal cleanup failed; surviving pids: ${survivors.map((member) => member.pid).join(", ")}`);
\t\tthis.dataDisposable.dispose();
\t\tthis.exitDisposable.dispose();
\t}
"""
)
subprocess.run(
[shutil.which("node"), patcher, directory],
check=True,
capture_output=True,
text=True,
)
patched = target.read_text()
self.assertEqual(
patched.count('process.env.DSH_PRESERVE_BACKGROUND_PROCESSES === "1"'),
2,
)
self.assertIn("this.forceStopShell();\n\t\t\treturn;", patched)
self.assertIn("await this.stopShell();", patched)
# The value of this patcher is that an upstream release it no longer
# understands stops the build rather than producing a toolchain that
# silently kills the task's services again. Running it over its own
# output is the cheapest form of drift: the text it anchors on is
# gone.
drifted = subprocess.run(
[shutil.which("node"), patcher, directory],
capture_output=True,
text=True,
)
self.assertNotEqual(drifted.returncode, 0)
self.assertNotEqual(target.read_text(), "")
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,79 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import asyncio
import importlib.util
import os
import sys
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
MODULE_PATH = Path(__file__).with_name("run_trial.py")
SPEC = importlib.util.spec_from_file_location("maka_eval_run_trial", MODULE_PATH)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
class RunTrialPolicyTest(unittest.TestCase):
def test_forces_only_the_subject_phase_through_the_cell_proxy(self) -> None:
agent = SimpleNamespace(network_mode=None, allowed_hosts=None)
task = SimpleNamespace(config=SimpleNamespace(agent=agent))
with patch.dict(
os.environ,
{
"MAKA_EVAL_EGRESS_REQUIRED": "1",
"MAKA_EVAL_EGRESS_ALLOWED_HOST": "maka-eval-mitmproxy",
},
clear=False,
):
MODULE.apply_subject_egress_policy(task)
self.assertEqual(agent.network_mode, "allowlist")
self.assertEqual(agent.allowed_hosts, ["maka-eval-mitmproxy"])
def test_required_egress_fails_closed_without_the_proxy_host(self) -> None:
task = SimpleNamespace(config=SimpleNamespace(agent=SimpleNamespace()))
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}, clear=True):
with self.assertRaisesRegex(RuntimeError, "proxy host is unavailable"):
MODULE.apply_subject_egress_policy(task)
def test_invalid_framework_fails_before_framework_import(self) -> None:
with patch.object(MODULE.importlib, "import_module") as imported:
with self.assertRaisesRegex(RuntimeError, "harbor or pier"):
asyncio.run(MODULE.run_trial("other", "1.0.0", Path("missing.json")))
imported.assert_not_called()
def test_main_installs_the_argv_framework_before_the_trial(self) -> None:
import eval_framework
installed: list[str] = []
async def fake_trial(framework: str, expected_version: str, config_file: Path) -> None:
installed.append(eval_framework.selected())
self.assertEqual(framework, "pier")
self.assertEqual(expected_version, "1.2.3")
with patch.object(sys, "argv", ["run_trial.py", "pier", "1.2.3", "config.json"]):
with patch.object(MODULE, "run_trial", fake_trial):
asyncio.run(MODULE.main())
self.assertEqual(installed, ["pier"])
if __name__ == "__main__":
unittest.main()