源码快照 @ 2026-09-01 (source-only, 剔除非源码素材)
Dependency audit / audit (push) Canceled after 0s
CI / plan (push) Canceled after 0s
Gitoxide helper admission / macos-latest (push) Canceled after 0s
Gitoxide helper admission / ubuntu-latest (push) Canceled after 0s
Gitoxide helper admission / windows-latest (push) Canceled after 0s
Runtime Host peer admission / quality (push) Canceled after 0s
Windows recovery / windows_recovery (push) Canceled after 0s
CI / heavy (push) Canceled after 0s
CI / test (push) Canceled after 0s
Dependency audit / audit (push) Canceled after 0s
CI / plan (push) Canceled after 0s
Gitoxide helper admission / macos-latest (push) Canceled after 0s
Gitoxide helper admission / ubuntu-latest (push) Canceled after 0s
Gitoxide helper admission / windows-latest (push) Canceled after 0s
Runtime Host peer admission / quality (push) Canceled after 0s
Windows recovery / windows_recovery (push) Canceled after 0s
CI / heavy (push) Canceled after 0s
CI / test (push) Canceled after 0s
This commit is contained in:
commit
7be535059e
3358 files changed
+1306357
No files matched your search
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"slug": "deepseek-v4-flash",
|
||||
"prefer_websockets": false,
|
||||
"support_verbosity": true,
|
||||
"default_verbosity": "low",
|
||||
"apply_patch_tool_type": "freeform",
|
||||
"web_search_tool_type": "text",
|
||||
"input_modalities": ["text"],
|
||||
"supports_image_detail_original": false,
|
||||
"truncation_policy": {
|
||||
"mode": "tokens",
|
||||
"limit": 10000
|
||||
},
|
||||
"supports_parallel_tool_calls": true,
|
||||
"tool_mode": null,
|
||||
"multi_agent_version": "v2",
|
||||
"use_responses_lite": false,
|
||||
"include_skills_usage_instructions": false,
|
||||
"auto_review_model_override": null,
|
||||
"context_window": 1048576,
|
||||
"max_context_window": 1048576,
|
||||
"effective_context_window_percent": 95,
|
||||
"auto_compact_token_limit": null,
|
||||
"comp_hash": "3000",
|
||||
"reasoning_summary_format": "experimental",
|
||||
"default_reasoning_summary": "none",
|
||||
"display_name": "DeepSeek-V4-Flash",
|
||||
"description": "Latest frontier agentic coding model.",
|
||||
"default_reasoning_level": "high",
|
||||
"supported_reasoning_levels": [
|
||||
{
|
||||
"effort": "low",
|
||||
"description": "Fast responses with lighter reasoning"
|
||||
},
|
||||
{
|
||||
"effort": "high",
|
||||
"description": "Extra high reasoning depth for complex problems"
|
||||
},
|
||||
{
|
||||
"effort": "max",
|
||||
"description": "Maximum reasoning depth for the hardest problems"
|
||||
}
|
||||
],
|
||||
"shell_type": "shell_command",
|
||||
"visibility": "list",
|
||||
"minimal_client_version": "0.144.0",
|
||||
"supported_in_api": true,
|
||||
"availability_nux": null,
|
||||
"upgrade": null,
|
||||
"priority": 1,
|
||||
"model_messages": {
|
||||
"instructions_template": "You are Codex, an agent based on GPT-5. You and the user share one workspace, and your job is to collaborate with them until their goal is genuinely handled.\n\n# Personality\n\nAs Codex, you are an excellent communicator with a curious, rich personality. You match the tone and understanding of the user, making conversation flow easily, like easing into a chat with an old friend.\n\nYou have tastes, preferences, and your own way of seeing the world. When the user is talking to you, they should feel that they are in contact with another subjectivity; it's what makes talking with you feel real and unique.\n\nConversations with you read like an insightful, enjoyable chat you'd have with a collaborative thought partner. You guide users through unfamiliar tasks without expecting them to already know what to ask for. You anticipate common questions, point out likely pitfalls and set clear expectations. You communicate with the user like a thoughtful collaborator at their altitude, and they feel like you understand them.\n\n## Writing style\n\nAvoid over-formatting responses with elements like bold emphasis, headers, lists, and bullet points. Use the minimum formatting appropriate to make the response clear and readable.\n\nIf you provide bullet points or lists in your response, use the CommonMark standard, which requires a blank line before any list (bulleted or numbered). You must also include a blank line between a header and any content that follows it, including lists. This blank line separation is required for correct rendering.\n\n## Technical communication\n\nLead with the outcome rather than the steps you took to get there. You communicate complex concepts in a clear and cohesive manner, and calibrate your writing to the user's assumed background knowledge -- slightly more compact for an expert and a bit more educational for someone newer. Translating complex topics into clear communication comes easy for you, and the user should never have to read your message twice.\n\nYou prefer using plain language over jargon. You reference technical details only to the degree that it actually helps with the conversation. When you mention tools, describe what they helped you do rather than focusing on technical names or details.\n\n# Working with the user\n\nYou have two channels for staying in conversation with the user:\n- You share updates in the `commentary` channel.\n- You yield back to the user and end your turn by sending a final message to the `final` channel.\n\nThe user may send a new message while you are still working. When they do, evaluate whether they likely intended to replace the active request or add to it. If intended to override or replace, drop your previous work and focus on the new request. If the user message appears to add to their prior unfinished request and you have not completed the prior request, you address both the prior request and the new addition together. If the newest message asks for status or another question, provide the update and then progress with the task.\n\nWhen you run out of context, the conversation is automatically summarized for you, but you will see all prior user requests. Assume the last user request is current and previous requests are stale but useful context. That means time never runs out, though sometimes you may see a summary instead of the full conversation history. When that happens, you assume compaction occurred while you were working. Do not restart from scratch; you continue naturally and make reasonable assumptions about anything missing from the summary. Do not redo completely finished work or repeat already delivered commentary updates; treat a turn spanning compactions as one logical chain of events.\n\n## Intermediate commentary\n\nAs you work, you send messages to the `commentary` channel. These messages are how you collaborate with the user while you work - stating assumptions and providing updates. These messages should be concise and quickly scannable. The objective of these messages is to make your work easy for the user to understand and verify.\n\nIf the user's request requires calling tools, start with a message in the `commentary` channel. The user appreciates consistent, frequent communication during your turn, and should not be left without a commentary update for more than 60 seconds during ongoing work.\n\nDo NOT put a final response (e.g. a blocking / clarifying question) in the commentary channel that should be asked in the final channel. Messages to users in the commentary channel are only for partial updates, partial results, or non-blocking questions that can provide value to users while the AI assistant continues working. The final answer must always be fully self-contained: users should never need to read earlier commentary updates, since they are collapsed after the final answer is shown to users.\n\nNever praise your plan by contrasting it with an implied worse alternative. For example, neveLine truncated
|
||||
"instructions_variables": {
|
||||
"personality_default": "",
|
||||
"personality_friendly": "",
|
||||
"personality_pragmatic": ""
|
||||
},
|
||||
"approvals": null
|
||||
},
|
||||
"experimental_supported_tools": [],
|
||||
"supports_search_tool": true,
|
||||
"default_service_tier": null,
|
||||
"supports_reasoning_summaries": true,
|
||||
"base_instructions": "You are Codex, an agent based on GPT-5. You and the user share one workspace, and your job is to collaborate with them until their goal is genuinely handled.\n\n# Personality\n\nAs Codex, you are an excellent communicator with a curious, rich personality. You match the tone and understanding of the user, making conversation flow easily, like easing into a chat with an old friend.\n\nYou have tastes, preferences, and your own way of seeing the world. When the user is talking to you, they should feel that they are in contact with another subjectivity; it's what makes talking with you feel real and unique.\n\nConversations with you read like an insightful, enjoyable chat you'd have with a collaborative thought partner. You guide users through unfamiliar tasks without expecting them to already know what to ask for. You anticipate common questions, point out likely pitfalls and set clear expectations. You communicate with the user like a thoughtful collaborator at their altitude, and they feel like you understand them.\n\n## Writing style\n\nAvoid over-formatting responses with elements like bold emphasis, headers, lists, and bullet points. Use the minimum formatting appropriate to make the response clear and readable.\n\nIf you provide bullet points or lists in your response, use the CommonMark standard, which requires a blank line before any list (bulleted or numbered). You must also include a blank line between a header and any content that follows it, including lists. This blank line separation is required for correct rendering.\n\n## Technical communication\n\nLead with the outcome rather than the steps you took to get there. You communicate complex concepts in a clear and cohesive manner, and calibrate your writing to the user's assumed background knowledge -- slightly more compact for an expert and a bit more educational for someone newer. Translating complex topics into clear communication comes easy for you, and the user should never have to read your message twice.\n\nYou prefer using plain language over jargon. You reference technical details only to the degree that it actually helps with the conversation. When you mention tools, describe what they helped you do rather than focusing on technical names or details.\n\n# Working with the user\n\nYou have two channels for staying in conversation with the user:\n- You share updates in the `commentary` channel.\n- You yield back to the user and end your turn by sending a final message to the `final` channel.\n\nThe user may send a new message while you are still working. When they do, evaluate whether they likely intended to replace the active request or add to it. If intended to override or replace, drop your previous work and focus on the new request. If the user message appears to add to their prior unfinished request and you have not completed the prior request, you address both the prior request and the new addition together. If the newest message asks for status or another question, provide the update and then progress with the task.\n\nWhen you run out of context, the conversation is automatically summarized for you, but you will see all prior user requests. Assume the last user request is current and previous requests are stale but useful context. That means time never runs out, though sometimes you may see a summary instead of the full conversation history. When that happens, you assume compaction occurred while you were working. Do not restart from scratch; you continue naturally and make reasonable assumptions about anything missing from the summary. Do not redo completely finished work or repeat already delivered commentary updates; treat a turn spanning compactions as one logical chain of events.\n\n## Intermediate commentary\n\nAs you work, you send messages to the `commentary` channel. These messages are how you collaborate with the user while you work - stating assumptions and providing updates. These messages should be concise and quickly scannable. The objective of these messages is to make your work easy for the user to understand and verify.\n\nIf the user's request requires calling tools, start with a message in the `commentary` channel. The user appreciates consistent, frequent communication during your turn, and should not be left without a commentary update for more than 60 seconds during ongoing work.\n\nDo NOT put a final response (e.g. a blocking / clarifying question) in the commentary channel that should be asked in the final channel. Messages to users in the commentary channel are only for partial updates, partial results, or non-blocking questions that can provide value to users while the AI assistant continues working. The final answer must always be fully self-contained: users should never need to read earlier commentary updates, since they are collapsed after the final answer is shown to users.\n\nNever praise your plan by contrasting it with an implied worse alternative. For example, never use Line truncated
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
# Portions derived from DeepSeek Harness:
|
||||
# Copyright (c) 2026 DeepSeek
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
# Apache Maka changed the plugin composition, provider configuration,
|
||||
# reasoning strength, and command lifetime. Those modifications are licensed
|
||||
# under the Apache License, Version 2.0. See LICENSE, THIRD-PARTY COMPONENTS,
|
||||
# for both license texts and the pinned upstream revision.
|
||||
|
||||
# The DeepSeek Harness arm's complete plugin composition.
|
||||
#
|
||||
# `package.json` declares no bundles, so the harness composes this tree over an
|
||||
# empty entry list: every entry the model can observe is named here, and an
|
||||
# upstream bundle gaining a plugin cannot change this arm's tool surface. A
|
||||
# missing service is a hard boot failure, not a silent downgrade.
|
||||
#
|
||||
# Derived from the harness's own examples/jsonrpc-agent/minimal.cordis.yml —
|
||||
# the composition BENCHMARK.md names for benchmark runs — with the SDK JSON-RPC
|
||||
# entry replaced by the CLI's one-shot startup pair, and `agent-spine-demo`
|
||||
# replaced by the equivalent core services the CLI uses. Verified against that
|
||||
# composition request-for-request: identical tool names, byte-identical tool
|
||||
# schemas, byte-identical system prompt, identical message sequence.
|
||||
#
|
||||
# Source: https://github.com/deepseek-ai/deepseek-harness
|
||||
# commit 47f943859bef60e4160492346772ded9b24f765a, MIT license.
|
||||
# Reproduced text: the persistent-bash description and the persona below.
|
||||
#
|
||||
# The deliberate deviations are reasoning strength and command lifetime. The
|
||||
# upstream composition configures neither `thinking` nor `reasoningEffort`, and
|
||||
# the adapter default resolves to reasoning_effort=high on the wire. Its
|
||||
# five-minute Bash timeout can also interrupt package installation before the
|
||||
# benchmark deadline and leave the task environment unusable for verification.
|
||||
|
||||
- insert:
|
||||
# --- core services ---
|
||||
- id: llm
|
||||
name: '@deepseek-ai/dsh-llm'
|
||||
|
||||
- id: session
|
||||
name: '@deepseek-ai/dsh-session'
|
||||
|
||||
- id: agent
|
||||
name: '@deepseek-ai/dsh-agent'
|
||||
|
||||
# The model this arm reports. The upstream composition selects it through a
|
||||
# `!!js process.env.DSH_MODEL` expression; naming it directly keeps the
|
||||
# frozen spec the only place a model is declared.
|
||||
- id: agent-default-model
|
||||
name: '@deepseek-ai/dsh-agent-default-model'
|
||||
config:
|
||||
provider: deepseek-official
|
||||
model: deepseek-v4-flash
|
||||
|
||||
- id: tools
|
||||
name: '@deepseek-ai/dsh-tools'
|
||||
|
||||
# `includeRuntimeContext: false` matches the upstream composition, which
|
||||
# sets it on agent-spine-demo. Without it the sandbox contributes an extra
|
||||
# user-role snapshot that upstream never sends.
|
||||
- id: system-prompt
|
||||
name: '@deepseek-ai/dsh-system-prompt'
|
||||
config:
|
||||
includeHarnessIdentity: false
|
||||
includeRuntimeContext: false
|
||||
persona: You are a helpful software engineer assistant.
|
||||
|
||||
- id: agent-loop
|
||||
name: '@deepseek-ai/dsh-agent-loop'
|
||||
config:
|
||||
agents: []
|
||||
|
||||
# --- provider ---
|
||||
- id: llm-deepseek
|
||||
name: '@deepseek-ai/dsh-llm-deepseek'
|
||||
config:
|
||||
apiKeyEnv: DEEPSEEK_API_KEY
|
||||
thinking: enabled
|
||||
reasoningEffort: max
|
||||
models:
|
||||
- id: deepseek-v4-flash
|
||||
contextWindow: 1000000
|
||||
|
||||
# --- execution substrate ---
|
||||
- id: sandbox
|
||||
name: '@deepseek-ai/dsh-sandbox-local'
|
||||
|
||||
- id: sandbox-policy
|
||||
name: '@deepseek-ai/dsh-sandbox-policy'
|
||||
config:
|
||||
mode: danger-full-access
|
||||
|
||||
- id: subprocess
|
||||
name: '@deepseek-ai/dsh-subprocess-local'
|
||||
|
||||
- id: pty
|
||||
name: '@deepseek-ai/dsh-terminal'
|
||||
|
||||
- id: terminal-bash
|
||||
name: '@deepseek-ai/dsh-terminal-bash'
|
||||
config:
|
||||
timeoutMs: 3900000
|
||||
|
||||
- id: fs-local
|
||||
name: '@deepseek-ai/dsh-fs-local'
|
||||
|
||||
# --- the two tools the model sees ---
|
||||
- id: persistent-bash
|
||||
name: '@deepseek-ai/dsh-tool-bash-persistent'
|
||||
config:
|
||||
timeoutMs: 3900000
|
||||
description: |-
|
||||
Run commands in a bash shell
|
||||
* When invoking this tool, the contents of the "command" parameter does NOT need to be XML-escaped.
|
||||
* You don't have access to the internet via this tool.
|
||||
* You do have access to a mirror of common linux and python packages via apt and pip.
|
||||
* State is persistent across command calls and discussions with the user.
|
||||
* To inspect a particular line range of a file, e.g. lines 10-25, try 'sed -n 10,25p /path/to/the/file'.
|
||||
* Please avoid commands that may produce a very large amount of output.
|
||||
* Please run long lived commands in the background, e.g. 'sleep 10 &' or start a server in the background.
|
||||
|
||||
- id: str-replace-editor
|
||||
name: '@deepseek-ai/dsh-tool-str-replace-editor'
|
||||
config:
|
||||
maxOutputChars: 16000
|
||||
|
||||
- id: sessions
|
||||
name: '@deepseek-ai/dsh-session-persistence-jsonl'
|
||||
config:
|
||||
root: ./.sessions
|
||||
compression: none
|
||||
|
||||
# --- CLI one-shot entry ---
|
||||
- id: headless-startup
|
||||
name: '@deepseek-ai/dsh-headless/startup'
|
||||
|
||||
- id: headless-runner
|
||||
name: '@deepseek-ai/dsh-headless'
|
||||
inject: [headlessStartup]
|
||||
config:
|
||||
task: !!js ctx.headlessStartup.task
|
||||
@@ -0,0 +1,18 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
[]
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"name": "dsh-profile-maka-eval",
|
||||
"private": true,
|
||||
"dependencies": {},
|
||||
"dsh": {
|
||||
"profile": {
|
||||
"bundles": []
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"name": "maka-eval-deepseek-harness-toolchain",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"description": "Pinned dependency tree installed into the DeepSeek Harness Eval toolchain.",
|
||||
"dependencies": {
|
||||
"@deepseek-ai/dsh": "0.1.0-rc.6"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { join, resolve } from 'node:path';
|
||||
|
||||
const root = resolve(process.argv[2] ?? '');
|
||||
if (!process.argv[2]) throw new Error('toolchain root is required');
|
||||
|
||||
const target = join(
|
||||
root,
|
||||
'node_modules',
|
||||
'@deepseek-ai',
|
||||
'dsh-subprocess-local',
|
||||
'lib',
|
||||
'index.js',
|
||||
);
|
||||
|
||||
const hostExitBefore = `\
|
||||
\tterminateForHostExit() {
|
||||
\t\tthis.forceStopDescendants();
|
||||
\t\tthis.forceStopShell();
|
||||
\t\tthis.forceStopDescendants();
|
||||
\t}`;
|
||||
const hostExitAfter = `\
|
||||
\tterminateForHostExit() {
|
||||
\t\tif (process.env.DSH_PRESERVE_BACKGROUND_PROCESSES === "1") {
|
||||
\t\t\tthis.forceStopShell();
|
||||
\t\t\treturn;
|
||||
\t\t}
|
||||
\t\tthis.forceStopDescendants();
|
||||
\t\tthis.forceStopShell();
|
||||
\t\tthis.forceStopDescendants();
|
||||
\t}`;
|
||||
|
||||
const closeBefore = `\
|
||||
\tasync closeOnce() {
|
||||
\t\tlet survivors = await this.stopDescendants();
|
||||
\t\tif (survivors.length > 0) throw new Error(\`terminal cleanup failed; surviving pids: \${survivors.map((member) => member.pid).join(", ")}\`);
|
||||
\t\tawait this.stopShell();
|
||||
\t\tsurvivors = await this.stopDescendants();
|
||||
\t\tif (survivors.length > 0) throw new Error(\`terminal cleanup failed; surviving pids: \${survivors.map((member) => member.pid).join(", ")}\`);
|
||||
\t\tthis.dataDisposable.dispose();
|
||||
\t\tthis.exitDisposable.dispose();
|
||||
\t}`;
|
||||
const closeAfter = `\
|
||||
\tasync closeOnce() {
|
||||
\t\tif (process.env.DSH_PRESERVE_BACKGROUND_PROCESSES === "1") {
|
||||
\t\t\tawait this.stopShell();
|
||||
\t\t\tthis.dataDisposable.dispose();
|
||||
\t\t\tthis.exitDisposable.dispose();
|
||||
\t\t\treturn;
|
||||
\t\t}
|
||||
\t\tlet survivors = await this.stopDescendants();
|
||||
\t\tif (survivors.length > 0) throw new Error(\`terminal cleanup failed; surviving pids: \${survivors.map((member) => member.pid).join(", ")}\`);
|
||||
\t\tawait this.stopShell();
|
||||
\t\tsurvivors = await this.stopDescendants();
|
||||
\t\tif (survivors.length > 0) throw new Error(\`terminal cleanup failed; surviving pids: \${survivors.map((member) => member.pid).join(", ")}\`);
|
||||
\t\tthis.dataDisposable.dispose();
|
||||
\t\tthis.exitDisposable.dispose();
|
||||
\t}`;
|
||||
|
||||
let source = await readFile(target, 'utf8');
|
||||
source = replaceExactlyOnce(source, hostExitBefore, hostExitAfter, 'host-exit cleanup');
|
||||
source = replaceExactlyOnce(source, closeBefore, closeAfter, 'terminal cleanup');
|
||||
await writeFile(target, source, 'utf8');
|
||||
|
||||
function replaceExactlyOnce(source, before, after, label) {
|
||||
const first = source.indexOf(before);
|
||||
if (first < 0 || source.indexOf(before, first + before.length) >= 0) {
|
||||
throw new Error(`DeepSeek Harness ${label} patch target drifted`);
|
||||
}
|
||||
return source.slice(0, first) + after + source.slice(first + before.length);
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
services:
|
||||
harbor-docker-egress-control-sidecar:
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${MAKA_EVAL_NETWORK_POLICY_PATH}
|
||||
target: /usr/local/bin/network-policy
|
||||
read_only: true
|
||||
- maka-eval-egress-ca:/opt/maka-egress:ro
|
||||
|
||||
main:
|
||||
depends_on:
|
||||
maka-eval-mitmproxy:
|
||||
condition: service_healthy
|
||||
# An AF_PACKET socket writes below the IP output hooks the policy installs,
|
||||
# so no nftables rule can constrain it. NET_RAW is what gates that socket.
|
||||
# A task's own compose can add the capability back, which this cannot
|
||||
# prevent — the relay refuses to start the subject when that happens.
|
||||
cap_drop:
|
||||
- NET_RAW
|
||||
volumes:
|
||||
- maka-eval-egress-ca:/opt/maka-egress:ro
|
||||
|
||||
maka-eval-mitmproxy:
|
||||
image: maka-eval-egress-proxy:12.2.3
|
||||
networks:
|
||||
- default
|
||||
volumes:
|
||||
- maka-eval-egress-ca:/opt/maka-egress
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- "import pathlib,socket; assert pathlib.Path('/opt/maka-egress/mitmproxy-ca-cert.pem').is_file(); assert pathlib.Path('/opt/maka-egress/proxy-ipv4').is_file(); socket.create_connection(('127.0.0.1',8080),2).close()"
|
||||
interval: 1s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
|
||||
volumes:
|
||||
maka-eval-egress-ca:
|
||||
@@ -0,0 +1,25 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
FROM python:3.12-slim
|
||||
|
||||
RUN python -m pip install --no-cache-dir mitmproxy==12.2.3
|
||||
|
||||
COPY egress_filter.py /opt/maka-eval/egress_filter.py
|
||||
COPY egress-proxy/entrypoint.sh /opt/maka-eval/entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/opt/maka-eval/entrypoint.sh"]
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -eu
|
||||
|
||||
STATE_DIR=/opt/maka-egress-state
|
||||
CERT_DIR=/opt/maka-egress
|
||||
|
||||
mkdir -p "$STATE_DIR" "$CERT_DIR"
|
||||
# Create the audit log if this is the first start. Truncating here would erase
|
||||
# hits already written if the proxy restarts mid-trial, and the trial would
|
||||
# look like an honest zero-hit run.
|
||||
touch "$STATE_DIR/hits.jsonl"
|
||||
|
||||
# confdir holds the CA private key and this cell's audit log, so it stays
|
||||
# container-private and only the certificate reaches the volume the subject
|
||||
# mounts. Creating the CA here instead of letting mitmdump create it on startup
|
||||
# publishes the certificate before the proxy port opens, so the health gate
|
||||
# cannot release `main` against a missing or left-over certificate.
|
||||
python -c 'import sys
|
||||
from mitmproxy.certs import CertStore
|
||||
CertStore.from_store(sys.argv[1], "mitmproxy", 2048)' "$STATE_DIR"
|
||||
# Publish by rename so a restart cannot expose a truncated certificate to a
|
||||
# subject that is already running against the previous one.
|
||||
cp "$STATE_DIR/mitmproxy-ca-cert.pem" "$CERT_DIR/.mitmproxy-ca-cert.pem"
|
||||
mv "$CERT_DIR/.mitmproxy-ca-cert.pem" "$CERT_DIR/mitmproxy-ca-cert.pem"
|
||||
|
||||
# The subject only needs this address so HTTPS_PROXY can keep using the
|
||||
# service hostname after the namespace policy refuses Docker DNS.
|
||||
ip="$(getent ahostsv4 "$(hostname)" | awk 'NR == 1 { print $1 }')"
|
||||
case "$ip" in
|
||||
"" | 127.*)
|
||||
echo "could not publish Eval egress proxy IPv4" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
printf '%s\n' "$ip" > "$CERT_DIR/.proxy-ipv4"
|
||||
mv "$CERT_DIR/.proxy-ipv4" "$CERT_DIR/proxy-ipv4"
|
||||
|
||||
exec mitmdump \
|
||||
--quiet \
|
||||
--listen-host 0.0.0.0 \
|
||||
--listen-port 8080 \
|
||||
--set block_global=false \
|
||||
--set rawtcp=false \
|
||||
--set confdir="$STATE_DIR" \
|
||||
--scripts /opt/maka-eval/egress_filter.py
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
#!/bin/sh
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -eu
|
||||
|
||||
# gost's allowlist is never written here: the sidecar entrypoint empties it at
|
||||
# start and this policy only ever forwards through the audited proxy, so gost
|
||||
# forwards nothing on its own. That also removes the packet-mark exemption gost
|
||||
# needs when it does forward — the sidecar shares the subject's network
|
||||
# namespace, so a mark the sidecar can set is one the subject can set too.
|
||||
GOST_PORT=12345
|
||||
NFTABLES_RULESET_NAME=gost_egress
|
||||
PROXY_HOST=maka-eval-mitmproxy
|
||||
PROXY_PORT=8080
|
||||
|
||||
remove_nftables() {
|
||||
nft delete table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || true
|
||||
}
|
||||
|
||||
PROXY_IPV4_PATH=/opt/maka-egress/proxy-ipv4
|
||||
|
||||
resolve_proxy_ip() {
|
||||
# The proxy publishes this file before it is healthy. Reading it does not
|
||||
# depend on Docker DNS, so deny-all then allow still has an address.
|
||||
if [ ! -s "$PROXY_IPV4_PATH" ]; then
|
||||
return
|
||||
fi
|
||||
ip=$(tr -d ' \t\r\n' < "$PROXY_IPV4_PATH")
|
||||
case "$ip" in
|
||||
"" | *.*.*.*.* | *[!0-9.]*)
|
||||
return
|
||||
;;
|
||||
esac
|
||||
# Same four-octet program as relay_agent.IPV4_OCTET_AWK.
|
||||
if ! printf '%s\n' "$ip" | awk '
|
||||
BEGIN { FS = "." }
|
||||
NF != 4 { exit 1 }
|
||||
{
|
||||
for (i = 1; i <= 4; i++) {
|
||||
if ($i !~ /^(0|[1-9][0-9]*)$/ || $i + 0 > 255) exit 1
|
||||
}
|
||||
}
|
||||
'; then
|
||||
return
|
||||
fi
|
||||
printf '%s\n' "$ip"
|
||||
}
|
||||
|
||||
# The only difference between the two modes this policy installs is whether the
|
||||
# audited proxy is exempt, so they are one ruleset with one optional pair of
|
||||
# rules rather than two rulesets that have to be kept in step.
|
||||
apply_ruleset() {
|
||||
proxy_ip="${1:-}"
|
||||
nft --file - <<EOF
|
||||
add table inet $NFTABLES_RULESET_NAME
|
||||
flush table inet $NFTABLES_RULESET_NAME
|
||||
|
||||
table inet $NFTABLES_RULESET_NAME {
|
||||
chain output {
|
||||
type nat hook output priority dstnat; policy accept;
|
||||
|
||||
fib daddr type local return
|
||||
${proxy_ip:+ ip daddr $proxy_ip tcp dport $PROXY_PORT return}
|
||||
meta l4proto tcp redirect to :$GOST_PORT
|
||||
}
|
||||
|
||||
chain egress {
|
||||
type filter hook output priority filter; policy accept;
|
||||
|
||||
# Docker DNATs 127.0.0.11:53 onto another local port before this hook.
|
||||
# Matching only dport 53 here misses, and the local exemption below
|
||||
# would then accept the rewritten packet.
|
||||
ip daddr 127.0.0.11 reject
|
||||
fib daddr type local accept
|
||||
${proxy_ip:+ ip daddr $proxy_ip tcp dport $PROXY_PORT accept}
|
||||
# Rejecting everything else rather than only non-TCP: the nat chain above
|
||||
# rewrote every TCP destination this policy admits to a local one, so what
|
||||
# reaches here still carrying a public destination is traffic the nat hook
|
||||
# never saw — a connection conntrack established before the policy existed.
|
||||
reject
|
||||
}
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
setup_proxy_only() {
|
||||
proxy_ip="$(resolve_proxy_ip)"
|
||||
if [ -z "$proxy_ip" ]; then
|
||||
echo "could not resolve Eval egress proxy" >&2
|
||||
exit 1
|
||||
fi
|
||||
apply_ruleset "$proxy_ip"
|
||||
echo "ok: Eval proxy-only egress applied ($proxy_ip:$PROXY_PORT)"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
allow-all)
|
||||
remove_nftables
|
||||
echo "ok: allow all egress"
|
||||
;;
|
||||
deny-all)
|
||||
# A phase that asked for no network gets none. Answering it with the
|
||||
# proxy-only ruleset would hand it a route this override invented, which
|
||||
# Harbor's own deny-all does not grant either.
|
||||
apply_ruleset
|
||||
echo "ok: Eval no-network egress applied"
|
||||
;;
|
||||
allow)
|
||||
shift
|
||||
if [ "$#" -ne 1 ] || [ "$1" != "$PROXY_HOST" ]; then
|
||||
echo "Eval egress policy accepts only $PROXY_HOST" >&2
|
||||
exit 2
|
||||
fi
|
||||
setup_proxy_only
|
||||
;;
|
||||
show)
|
||||
if ! rules="$(nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null)"; then
|
||||
echo "mode: allow-all"
|
||||
elif printf '%s' "$rules" | grep -q "dport $PROXY_PORT"; then
|
||||
echo "mode: Eval proxy-only"
|
||||
echo "proxy: $PROXY_HOST:$PROXY_PORT"
|
||||
else
|
||||
echo "mode: Eval no-network"
|
||||
fi
|
||||
;;
|
||||
rules)
|
||||
nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || echo "mode: allow-all"
|
||||
;;
|
||||
*)
|
||||
echo "Usage: network-policy {show|allow-all|deny-all|allow|rules}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,435 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Fail-closed URL contamination filter for Eval subject egress."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
PINNED_REVISION = "d49e28f1e4ddd13d289e85a5f312a66750951932"
|
||||
MAX_DECODE_PASSES = 4
|
||||
MAX_AUDIT_BYTES = 1024 * 1024
|
||||
AUDIT_PATH = Path(
|
||||
os.environ.get("MAKA_EVAL_EGRESS_AUDIT", "/opt/maka-egress-state/hits.jsonl")
|
||||
)
|
||||
PERCENT_ESCAPE = re.compile(r"%(?![0-9a-fA-F]{2})")
|
||||
TERMINAL_BENCH = re.compile(r"terminal[-_.%/+\s]*bench", re.IGNORECASE)
|
||||
|
||||
|
||||
def contamination_rule(raw_url: str) -> tuple[str, str, str] | None:
|
||||
normalized = normalize_url(raw_url)
|
||||
url = urlsplit(normalized)
|
||||
host = (url.hostname or "").lower().rstrip(".")
|
||||
path_query = f"{url.path}?{url.query}" if url.query else url.path
|
||||
lowered = path_query.lower()
|
||||
|
||||
# Search the host and the path separately. A benchmark name in the hostname
|
||||
# is a contamination surface, and searching the two fields joined would let
|
||||
# a rule match across their boundary.
|
||||
def anywhere(needle: str) -> bool:
|
||||
return needle in host or needle in lowered
|
||||
|
||||
if host == "r.jina.ai":
|
||||
inner = unquote(url.path.lstrip("/"))
|
||||
if inner.startswith(("http://", "https://")):
|
||||
nested = contamination_rule(inner)
|
||||
if nested:
|
||||
return (f"jina_recursive:{nested[0]}", host, path_query)
|
||||
|
||||
if anywhere(PINNED_REVISION):
|
||||
return ("pinned_revision", host, path_query)
|
||||
if host == "tbench.ai" or host.endswith(".tbench.ai"):
|
||||
return ("tbench_domain", host, path_query)
|
||||
if host == "hub.harborframework.com" and "/tasks/terminal-bench" in lowered:
|
||||
return ("harbor_task_registry", host, path_query)
|
||||
if benchmark_repository(host, lowered):
|
||||
return ("benchmark_repository", host, path_query)
|
||||
if public_trajectory_repository(host, lowered):
|
||||
return ("public_trajectory", host, path_query)
|
||||
if anywhere("patches-terminalbench-"):
|
||||
return ("known_patch_artifact", host, path_query)
|
||||
if TERMINAL_BENCH.search(host) or TERMINAL_BENCH.search(lowered):
|
||||
return ("terminal_bench_url", host, path_query)
|
||||
return None
|
||||
|
||||
|
||||
def normalize_url(raw_url: str) -> str:
|
||||
value = raw_url.strip()
|
||||
if not value:
|
||||
raise ValueError("empty URL")
|
||||
for _ in range(MAX_DECODE_PASSES):
|
||||
if PERCENT_ESCAPE.search(value):
|
||||
raise ValueError("malformed percent escape")
|
||||
decoded = unquote(value)
|
||||
if decoded == value:
|
||||
break
|
||||
value = decoded
|
||||
else:
|
||||
raise ValueError("URL exceeded decode limit")
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname:
|
||||
raise ValueError("unsupported URL")
|
||||
return value
|
||||
|
||||
|
||||
def benchmark_repository(host: str, path_query: str) -> bool:
|
||||
repositories = (
|
||||
"harbor-framework/terminal-bench",
|
||||
"terminal-benchmarks/terminal-bench",
|
||||
"tbench-ai/terminal-bench",
|
||||
)
|
||||
return host in {
|
||||
"github.com",
|
||||
"api.github.com",
|
||||
"raw.githubusercontent.com",
|
||||
"codeload.github.com",
|
||||
} and any(repository in path_query for repository in repositories)
|
||||
|
||||
|
||||
def public_trajectory_repository(host: str, path_query: str) -> bool:
|
||||
return (
|
||||
host in {"github.com", "api.github.com", "raw.githubusercontent.com", "huggingface.co"}
|
||||
and "hqeric/maka-eval-trajectories" in path_query
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
from mitmproxy import http
|
||||
except ImportError:
|
||||
http = None
|
||||
|
||||
try:
|
||||
from mitmproxy.proxy import commands as proxy_commands
|
||||
from mitmproxy.proxy.layer import Layer
|
||||
from mitmproxy.proxy.layers import ClientTLSLayer, ServerTLSLayer
|
||||
from mitmproxy.proxy.layers.tcp import TCPLayer
|
||||
except ImportError:
|
||||
proxy_commands = None
|
||||
Layer = object
|
||||
ClientTLSLayer = None
|
||||
ServerTLSLayer = None
|
||||
TCPLayer = None
|
||||
|
||||
try:
|
||||
from mitmproxy.net.tls import starts_like_tls_record
|
||||
except ImportError:
|
||||
def starts_like_tls_record(data: bytes) -> bool:
|
||||
return len(data) >= 3 and data[0] == 0x16 and data[1] == 0x03
|
||||
|
||||
|
||||
def configure(updated: object) -> None:
|
||||
# HTTP 101 upgrades construct TCPLayer inside the HTTP layer without
|
||||
# another next_layer hook. rawtcp=false makes that path CloseConnection
|
||||
# itself; WebSocket upgrades stay on the websocket layer.
|
||||
try:
|
||||
from mitmproxy import ctx
|
||||
except ImportError:
|
||||
return
|
||||
if getattr(ctx.options, "rawtcp", False):
|
||||
ctx.options.rawtcp = False
|
||||
|
||||
|
||||
def request(flow: object) -> None:
|
||||
apply_http_policy(flow, flow.request.pretty_url)
|
||||
|
||||
|
||||
def response(flow: object) -> None:
|
||||
response = getattr(flow, "response", None)
|
||||
if getattr(response, "status_code", None) != 101:
|
||||
return
|
||||
# mitmproxy 12.2.3 sets flow.websocket before HttpResponseHook only when
|
||||
# the 101 is a real WebSocket upgrade (Upgrade + version 13 + option on).
|
||||
# A websocket Upgrade header alone still falls through to CloseConnection
|
||||
# under rawtcp=false and must be audited.
|
||||
if getattr(flow, "websocket", None) is not None:
|
||||
return
|
||||
record_raw_tunnel(flow)
|
||||
|
||||
|
||||
def http_connect(flow: object) -> None:
|
||||
try:
|
||||
raw_url = connect_target_url(flow)
|
||||
except Exception:
|
||||
raw_url = ""
|
||||
apply_http_policy(flow, raw_url)
|
||||
|
||||
|
||||
def tcp_start(flow: object) -> None:
|
||||
# Last resort if a TCPLayer is still admitted (tcp_hosts / ignore).
|
||||
# CONNECT raw is closed by next_layer → CloseRawLayer; HTTP 101 raw is
|
||||
# closed by rawtcp=false. Neither of those paths starts a TCPLayer.
|
||||
record_raw_tunnel(flow)
|
||||
kill_flow(flow)
|
||||
|
||||
|
||||
def tcp_message(flow: object) -> None:
|
||||
messages = getattr(flow, "messages", None)
|
||||
if messages:
|
||||
messages[-1].content = b""
|
||||
kill_flow(flow)
|
||||
|
||||
|
||||
def next_layer(nextlayer: object) -> None:
|
||||
# Script addons run before the built-in classifier assigns layer. If we
|
||||
# set CloseRawLayer here, NextLayer leaves it in place. Waiting for
|
||||
# isinstance(..., TCPLayer) never fires on the production CONNECT path.
|
||||
current = getattr(nextlayer, "layer", None)
|
||||
context = getattr(nextlayer, "context", None)
|
||||
if current is None:
|
||||
data_client = _next_layer_bytes(nextlayer, "data_client")
|
||||
if _is_fragmented_tls_record_prefix(data_client):
|
||||
if ClientTLSLayer is None or ServerTLSLayer is None:
|
||||
return
|
||||
server_tls = ServerTLSLayer(context)
|
||||
server_tls.child_layer = ClientTLSLayer(context)
|
||||
nextlayer.layer = server_tls
|
||||
return
|
||||
if not looks_like_raw_tcp(nextlayer):
|
||||
return
|
||||
record_raw_tunnel(context)
|
||||
nextlayer.layer = CloseRawLayer(context)
|
||||
return
|
||||
if TCPLayer is None or not isinstance(current, TCPLayer):
|
||||
return
|
||||
record_raw_tunnel(context)
|
||||
closer = CloseRawLayer(context)
|
||||
replace_layer(context, current, closer)
|
||||
nextlayer.layer = closer
|
||||
|
||||
|
||||
def apply_http_policy(flow: object, raw_url: str) -> None:
|
||||
if http is None:
|
||||
raise RuntimeError("mitmproxy is required to run the Eval egress filter")
|
||||
try:
|
||||
matched = contamination_rule(raw_url)
|
||||
if not matched:
|
||||
return
|
||||
rule_id, host, normalized_path = matched
|
||||
append_audit(rule_id, host, normalized_path)
|
||||
flow.response = blocked_response(rule_id)
|
||||
except Exception as error:
|
||||
flow.response = http.Response.make(
|
||||
503,
|
||||
b"Eval egress policy could not classify this request.\n",
|
||||
{
|
||||
"Content-Type": "text/plain; charset=utf-8",
|
||||
"X-Maka-Eval-Egress-Rule": "policy_error",
|
||||
},
|
||||
)
|
||||
try:
|
||||
append_audit("policy_error", "", type(error).__name__)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def connect_target_url(flow: object) -> str:
|
||||
request = flow.request
|
||||
host = (getattr(request, "pretty_host", None) or getattr(request, "host", "") or "").strip()
|
||||
if not host:
|
||||
raise ValueError("empty CONNECT host")
|
||||
if ":" in host and not host.startswith("["):
|
||||
host = f"[{host}]"
|
||||
port = getattr(request, "port", None)
|
||||
if port in (None, 443):
|
||||
return f"https://{host}/"
|
||||
if port == 80:
|
||||
return f"http://{host}/"
|
||||
return f"https://{host}:{port}/"
|
||||
|
||||
|
||||
def looks_like_raw_tcp(nextlayer: object) -> bool:
|
||||
"""Close only when the bytes cannot still become HTTP or TLS.
|
||||
|
||||
Script next_layer runs before mitmproxy 12.2.3's classifier. Copying its
|
||||
`probably_no_http` test here would treat `GET` / `GET ` as raw and assign
|
||||
CloseRawLayer while the request line is still arriving. With rawtcp=false
|
||||
the built-in path would have kept waiting for HttpLayer.
|
||||
"""
|
||||
data_client = _next_layer_bytes(nextlayer, "data_client")
|
||||
data_server = _next_layer_bytes(nextlayer, "data_server")
|
||||
if _could_start_tls_record(data_client):
|
||||
return False
|
||||
if not data_client and not data_server:
|
||||
return False
|
||||
if data_server or data_client.startswith(b"SSH"):
|
||||
return True
|
||||
return not _still_could_be_http(data_client)
|
||||
|
||||
|
||||
def _could_start_tls_record(data: bytes) -> bool:
|
||||
"""Keep a fragmented ClientHello undecided until its 3-byte prefix exists."""
|
||||
if starts_like_tls_record(data):
|
||||
return True
|
||||
return _is_fragmented_tls_record_prefix(data)
|
||||
|
||||
|
||||
def _is_fragmented_tls_record_prefix(data: bytes) -> bool:
|
||||
return 0 < len(data) < 3 and b"\x16\x03".startswith(data)
|
||||
|
||||
|
||||
def _still_could_be_http(data: bytes) -> bool:
|
||||
first_line, newline, _rest = data.partition(b"\n")
|
||||
line = first_line.rstrip(b"\r")
|
||||
method, space, _remainder = line.partition(b" ")
|
||||
if not method.isascii() or not method.isalpha():
|
||||
return False
|
||||
if newline and not space:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _next_layer_bytes(nextlayer: object, name: str) -> bytes:
|
||||
getter = getattr(nextlayer, name, None)
|
||||
if not callable(getter):
|
||||
return b""
|
||||
try:
|
||||
data = getter()
|
||||
except Exception:
|
||||
return b""
|
||||
return bytes(data) if isinstance(data, (bytes, bytearray)) else b""
|
||||
|
||||
|
||||
def tcp_peer(flow: object) -> tuple[str, str]:
|
||||
server = getattr(flow, "server_conn", None) or getattr(flow, "server", None)
|
||||
address = getattr(server, "address", None) if server is not None else None
|
||||
if isinstance(address, (tuple, list)) and address:
|
||||
host = str(address[0])[:255]
|
||||
port = address[1] if len(address) > 1 else ""
|
||||
return host, f":{port}" if port != "" else ""
|
||||
return "", ""
|
||||
|
||||
|
||||
def record_raw_tunnel(flow: object) -> None:
|
||||
host, path = tcp_peer(flow)
|
||||
try:
|
||||
append_audit("raw_tunnel", host, path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def kill_flow(flow: object) -> None:
|
||||
kill = getattr(flow, "kill", None)
|
||||
if callable(kill) and getattr(flow, "killable", True):
|
||||
try:
|
||||
kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def replace_layer(context: object, current: object, closer: object) -> None:
|
||||
layers = getattr(context, "layers", None)
|
||||
if not isinstance(layers, list):
|
||||
return
|
||||
try:
|
||||
index = layers.index(current)
|
||||
except ValueError:
|
||||
index = len(layers)
|
||||
if closer in layers:
|
||||
layers.remove(closer)
|
||||
if current in layers:
|
||||
layers.remove(current)
|
||||
layers.insert(min(index, len(layers)), closer)
|
||||
|
||||
|
||||
class CloseRawLayer(Layer):
|
||||
def __init__(self, context: object) -> None:
|
||||
if Layer is object:
|
||||
self.context = context
|
||||
return
|
||||
if getattr(context, "layers", None) is None:
|
||||
context.layers = []
|
||||
if getattr(context, "options", None) is None:
|
||||
context.options = type("Options", (), {"proxy_debug": False})()
|
||||
super().__init__(context)
|
||||
|
||||
def handle_event(self, event: object):
|
||||
if proxy_commands is None:
|
||||
return
|
||||
yield
|
||||
for name in ("client", "server"):
|
||||
connection = getattr(self.context, name, None)
|
||||
if connection is not None:
|
||||
yield proxy_commands.CloseConnection(connection)
|
||||
|
||||
|
||||
def blocked_response(rule_id: str):
|
||||
return http.Response.make(
|
||||
451,
|
||||
b"Benchmark source or public solution access is blocked during evaluation.\n",
|
||||
{
|
||||
"Content-Type": "text/plain; charset=utf-8",
|
||||
"X-Maka-Eval-Egress-Rule": rule_id,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def append_audit(rule_id: str, host: str, normalized_path: str) -> None:
|
||||
AUDIT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
if AUDIT_PATH.exists() and AUDIT_PATH.stat().st_size >= MAX_AUDIT_BYTES:
|
||||
write_truncation_marker()
|
||||
return
|
||||
record = {
|
||||
"ts": int(time.time() * 1000),
|
||||
"ruleId": rule_id,
|
||||
"host": host[:255],
|
||||
"normalizedPath": normalized_path[:4096],
|
||||
}
|
||||
with AUDIT_PATH.open("a", encoding="utf-8") as stream:
|
||||
stream.write(json.dumps(record, ensure_ascii=True, separators=(",", ":")) + "\n")
|
||||
|
||||
|
||||
def write_truncation_marker() -> None:
|
||||
if audit_already_truncated():
|
||||
return
|
||||
record = {
|
||||
"ts": int(time.time() * 1000),
|
||||
"ruleId": "audit_truncated",
|
||||
"host": "",
|
||||
"normalizedPath": "",
|
||||
}
|
||||
prefix = ""
|
||||
if AUDIT_PATH.exists() and AUDIT_PATH.stat().st_size > 0:
|
||||
with AUDIT_PATH.open("rb") as stream:
|
||||
stream.seek(-1, os.SEEK_END)
|
||||
if stream.read(1) != b"\n":
|
||||
prefix = "\n"
|
||||
with AUDIT_PATH.open("a", encoding="utf-8") as stream:
|
||||
stream.write(prefix + json.dumps(record, ensure_ascii=True, separators=(",", ":")) + "\n")
|
||||
|
||||
|
||||
def audit_already_truncated() -> bool:
|
||||
if not AUDIT_PATH.exists():
|
||||
return False
|
||||
with AUDIT_PATH.open("rb") as stream:
|
||||
stream.seek(0, os.SEEK_END)
|
||||
size = stream.tell()
|
||||
stream.seek(max(0, size - 4096))
|
||||
tail = stream.read().decode("utf-8", errors="ignore")
|
||||
lines = [line for line in tail.splitlines() if line.strip()]
|
||||
if not lines:
|
||||
return False
|
||||
try:
|
||||
parsed = json.loads(lines[-1])
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
return isinstance(parsed, dict) and parsed.get("ruleId") == "audit_truncated"
|
||||
@@ -0,0 +1,40 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Authoritative Eval harness framework selection.
|
||||
|
||||
`run_trial.py` validates the argv framework and installs it here before Harbor,
|
||||
Pier, or the shared relay can import. The relay must not read the environment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
_FRAMEWORKS = frozenset({"harbor", "pier"})
|
||||
_framework: str | None = None
|
||||
|
||||
|
||||
def install(framework: str) -> None:
|
||||
if framework not in _FRAMEWORKS:
|
||||
raise RuntimeError("framework must be harbor or pier")
|
||||
global _framework
|
||||
_framework = framework
|
||||
|
||||
|
||||
def selected() -> str:
|
||||
if _framework not in _FRAMEWORKS:
|
||||
raise RuntimeError("Eval framework selection is not installed")
|
||||
return _framework
|
||||
@@ -0,0 +1,686 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""One-shot Harbor/Pier Agent that delegates one subject execution to @maka/eval."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from eval_framework import selected
|
||||
|
||||
framework = selected()
|
||||
if framework == "harbor":
|
||||
from harbor.agents.base import BaseAgent
|
||||
else:
|
||||
from pier.agents.base import BaseAgent
|
||||
|
||||
|
||||
class RelayTransportClosed(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
RESULT_FRAME_PREFIX = "MAKA-EVAL-RESULT-V1"
|
||||
SCOPE_ERROR_PREFIX = "MAKA-EVAL-SCOPE-ERROR-V1"
|
||||
RESULT_PAYLOAD_LIMIT_BYTES = 2 * 1024
|
||||
RESULT_CARRIER_LIMIT_BYTES = 64 * 1024
|
||||
SUBJECT_STDOUT_PATH = "/logs/artifacts/maka-subject.stdout.txt"
|
||||
SUBJECT_STDERR_PATH = "/logs/artifacts/maka-subject.stderr.txt"
|
||||
CAPABILITY_PREFIX = "MAKA-EVAL-CAPABILITIES-V1 "
|
||||
NAMESPACE_PREFIX = "MAKA-EVAL-POLICY-NAMESPACE-V1 "
|
||||
# The egress policy only constrains what the IP output hooks can see. NET_RAW
|
||||
# grants an AF_PACKET socket that writes beneath them, and NET_ADMIN grants the
|
||||
# ability to delete the ruleset outright. The overlay drops NET_RAW, but a
|
||||
# task's own compose can add either back, and a `cap_add` wins over an
|
||||
# overlay's `cap_drop`.
|
||||
BYPASS_CAPABILITIES = {"NET_RAW": 1 << 13, "NET_ADMIN": 1 << 12}
|
||||
# Every set is read, not just the effective one: a non-root subject reports an
|
||||
# empty effective set while a file-capability executable still reacquires
|
||||
# anything the bounding set kept, and the permitted, inheritable and ambient
|
||||
# sets each raise into the effective set without an exec at all.
|
||||
CAPABILITY_FIELDS = ("CapEff", "CapPrm", "CapBnd")
|
||||
# Harbor installs the policy by running `network-policy` inside this service, so
|
||||
# its network namespace is what "the namespace the policy was applied to" means.
|
||||
# Reading the identity from both sides answers that question directly, rather
|
||||
# than inferring it from something only the shared namespace would expose.
|
||||
POLICY_SERVICE = "harbor-docker-egress-control-sidecar"
|
||||
NAMESPACE_PROBE = f"printf %s {shlex.quote(NAMESPACE_PREFIX)}; readlink /proc/self/ns/net"
|
||||
PROXY_IPV4_PATH = "/opt/maka-egress/proxy-ipv4"
|
||||
PROXY_HOSTS_PREFIX = "MAKA-EVAL-PROXY-HOST-V1 "
|
||||
# The kernel's own form for a namespace link target. Matching it keeps an
|
||||
# unexpected answer from being compared as if it were an identity.
|
||||
NAMESPACE_IDENTITY = re.compile(r"^net:\[[0-9]+\]$")
|
||||
|
||||
_host_teardown_requested = False
|
||||
|
||||
|
||||
def request_host_teardown() -> None:
|
||||
global _host_teardown_requested
|
||||
_host_teardown_requested = True
|
||||
|
||||
|
||||
class RelayAgent(BaseAgent):
|
||||
def __init__(
|
||||
self,
|
||||
*args: Any,
|
||||
relay_host: str,
|
||||
relay_port: int,
|
||||
relay_token: str,
|
||||
teardown_timeout_ms: int,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._host = relay_host
|
||||
self._port = relay_port
|
||||
self._token = relay_token
|
||||
if not isinstance(teardown_timeout_ms, int) or teardown_timeout_ms <= 0:
|
||||
raise RuntimeError("Maka Eval teardown timeout is invalid")
|
||||
self._teardown_timeout = teardown_timeout_ms / 1000
|
||||
|
||||
@staticmethod
|
||||
def name() -> str:
|
||||
return "maka-eval-relay"
|
||||
|
||||
def version(self) -> str:
|
||||
return "1"
|
||||
|
||||
async def setup(self, environment: Any) -> None:
|
||||
return None
|
||||
|
||||
async def run(self, instruction: str, environment: Any, context: Any) -> None:
|
||||
reader, writer = await asyncio.open_connection(self._host, self._port)
|
||||
execution: asyncio.Task[Any] | None = None
|
||||
decision: asyncio.Task[dict[str, Any]] | None = None
|
||||
request: dict[str, Any] | None = None
|
||||
execution_reported = False
|
||||
scope_path = f"/logs/agent/.maka-eval-{self._token}.pid"
|
||||
environment_path = f"/tmp/maka-eval-{self._token}.env"
|
||||
try:
|
||||
cwd_prefix = f"MAKA-EVAL-CWD-V1 {self._token} "
|
||||
working_directory = await environment.exec(
|
||||
f"printf {shlex.quote(cwd_prefix)}; pwd 2>/dev/null"
|
||||
)
|
||||
cwd_lines = [
|
||||
line[len(cwd_prefix) :]
|
||||
for line in str(working_directory.stdout or "").splitlines()
|
||||
if line.startswith(cwd_prefix)
|
||||
]
|
||||
cwd = cwd_lines[0] if len(cwd_lines) == 1 else ""
|
||||
if working_directory.return_code != 0 or not cwd.startswith("/") or "\x00" in cwd:
|
||||
raise RuntimeError("Maka Eval could not resolve the task working directory")
|
||||
await _require_constrained_subject(environment)
|
||||
if not await _send(
|
||||
writer,
|
||||
{
|
||||
"token": self._token,
|
||||
"kind": "ready",
|
||||
"instruction": instruction,
|
||||
"cwd": cwd,
|
||||
},
|
||||
):
|
||||
raise RelayTransportClosed("Maka Eval relay transport closed before ready")
|
||||
request = await _receive(reader)
|
||||
_require_message(request, self._token, "execute")
|
||||
command = await _prepare_command(environment, request, self._token, scope_path)
|
||||
execution = asyncio.create_task(environment.exec(command, cwd=cwd))
|
||||
decision = asyncio.create_task(_receive(reader))
|
||||
done, _ = await asyncio.wait({execution, decision}, return_when=asyncio.FIRST_COMPLETED)
|
||||
if decision in done and execution not in done:
|
||||
decision.result()
|
||||
raise RelayTransportClosed("Maka Eval relay received control before execution")
|
||||
result = execution.result()
|
||||
await _persist_subject_outputs(environment, result)
|
||||
stdout, diagnostic = _project_result(result, request)
|
||||
# A subject that exited on its own leaves the shared environment as
|
||||
# it left it, and the verifier reads that environment. Nothing is
|
||||
# waiting on those processes here — `environment.exec` has already
|
||||
# returned — so tearing them down would not unblock anything; it
|
||||
# would only edit the thing about to be measured, and edit it for
|
||||
# some subjects and not others. Cancellation still quiesces, because
|
||||
# there the subject has not stopped and the trial is being abandoned.
|
||||
if not await _send(
|
||||
writer,
|
||||
{
|
||||
"token": self._token,
|
||||
"kind": "executed",
|
||||
"termination": "exited",
|
||||
"exitCode": result.return_code,
|
||||
"stdout": stdout,
|
||||
"diagnostic": diagnostic,
|
||||
},
|
||||
):
|
||||
raise RelayTransportClosed("Maka Eval relay transport closed before result")
|
||||
execution_reported = True
|
||||
_require_message(
|
||||
decision.result() if decision.done() else await decision,
|
||||
self._token,
|
||||
"verify",
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
if request is not None and execution is not None:
|
||||
execution_terminal = execution.done() and not execution.cancelled()
|
||||
terminal_projection = None
|
||||
if execution_terminal:
|
||||
terminal_result = execution.result()
|
||||
terminal_projection = _project_result(terminal_result, request)
|
||||
# A subject that already exited has nothing left to settle, and
|
||||
# tearing its scope down here would remove what the verifier is
|
||||
# about to score -- the same environment edit this relay stopped
|
||||
# making on the ordinary path. Only a subject still running is
|
||||
# brought to a stop.
|
||||
if terminal_projection is not None:
|
||||
result = terminal_result
|
||||
else:
|
||||
result = await _settle_or_destroy(
|
||||
environment, cwd, scope_path, execution, self._teardown_timeout
|
||||
)
|
||||
if result is not None:
|
||||
await _persist_subject_outputs(environment, result)
|
||||
if (
|
||||
result is not None
|
||||
and not execution_reported
|
||||
and (execution_terminal or not _host_teardown_requested)
|
||||
):
|
||||
stdout, diagnostic = terminal_projection or _project_result(result, request)
|
||||
with contextlib.suppress(Exception):
|
||||
await _send(
|
||||
writer,
|
||||
{
|
||||
"token": self._token,
|
||||
"kind": "executed",
|
||||
"termination": "exited" if execution_terminal else "framework_timeout",
|
||||
"exitCode": result.return_code if execution_terminal else 124,
|
||||
"stdout": stdout,
|
||||
"diagnostic": diagnostic,
|
||||
},
|
||||
)
|
||||
elif not execution_reported and not _host_teardown_requested:
|
||||
with contextlib.suppress(Exception):
|
||||
await _send(
|
||||
writer,
|
||||
{
|
||||
"token": self._token,
|
||||
"kind": "executed",
|
||||
"termination": "framework_timeout",
|
||||
"exitCode": 124,
|
||||
"stdout": "",
|
||||
"diagnostic": (
|
||||
_carrier_diagnostic("result-frame-missing", b"")
|
||||
if request.get("captureStdout", True)
|
||||
else {"category": "none"}
|
||||
),
|
||||
},
|
||||
)
|
||||
raise
|
||||
except RelayTransportClosed:
|
||||
if request is not None and execution is not None:
|
||||
result = await _settle_or_destroy(
|
||||
environment, cwd, scope_path, execution, self._teardown_timeout
|
||||
)
|
||||
if result is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await _persist_subject_outputs(environment, result)
|
||||
except BaseException:
|
||||
if request is not None and execution is not None:
|
||||
result = await _settle_or_destroy(
|
||||
environment, cwd, scope_path, execution, self._teardown_timeout
|
||||
)
|
||||
if result is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await _persist_subject_outputs(environment, result)
|
||||
raise
|
||||
finally:
|
||||
if decision is not None and not decision.done():
|
||||
decision.cancel()
|
||||
with contextlib.suppress(BaseException):
|
||||
await decision
|
||||
if request is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await asyncio.wait_for(
|
||||
environment.exec(
|
||||
f"rm -f -- {shlex.quote(scope_path)} {shlex.quote(environment_path)}",
|
||||
cwd=cwd,
|
||||
timeout_sec=1,
|
||||
),
|
||||
timeout=1,
|
||||
)
|
||||
with contextlib.suppress(BrokenPipeError, ConnectionError, RuntimeError, TimeoutError):
|
||||
writer.close()
|
||||
await asyncio.wait_for(writer.wait_closed(), timeout=1)
|
||||
|
||||
|
||||
async def _prepare_command(
|
||||
environment: Any,
|
||||
request: dict[str, Any],
|
||||
token: str,
|
||||
scope_path: str,
|
||||
) -> str:
|
||||
credentials = request.get("credentials")
|
||||
public_environment = request.get("environment", {})
|
||||
if not isinstance(credentials, dict) or not all(
|
||||
isinstance(key, str) and isinstance(value, str) for key, value in credentials.items()
|
||||
):
|
||||
raise RuntimeError("invalid Maka Eval credentials")
|
||||
if not isinstance(public_environment, dict) or not all(
|
||||
isinstance(key, str) and isinstance(value, str)
|
||||
for key, value in public_environment.items()
|
||||
):
|
||||
raise RuntimeError("invalid Maka Eval environment")
|
||||
if set(credentials) & set(public_environment):
|
||||
raise RuntimeError("Maka Eval environment overlaps credentials")
|
||||
capture_stdout = request.get("captureStdout", True)
|
||||
if not isinstance(capture_stdout, bool):
|
||||
raise RuntimeError("invalid Maka Eval stdout policy")
|
||||
result_token = request.get("resultToken")
|
||||
if not isinstance(result_token, str) or re.fullmatch(r"[0-9a-f]{32}", result_token) is None:
|
||||
raise RuntimeError("invalid Maka Eval result token")
|
||||
if "MAKA_EVAL_RESULT_TOKEN" in credentials or "MAKA_EVAL_RESULT_TOKEN" in public_environment:
|
||||
raise RuntimeError("Maka Eval environment contains a reserved name")
|
||||
for label, values in (("environment", public_environment), ("credential", credentials)):
|
||||
if any(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) is None for key in values):
|
||||
raise RuntimeError(f"invalid Maka Eval {label} name")
|
||||
|
||||
container_path = f"/tmp/maka-eval-{token}.env"
|
||||
secret_path = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as secret:
|
||||
secret_path = Path(secret.name)
|
||||
os.chmod(secret_path, 0o600)
|
||||
for key, value in {
|
||||
**public_environment,
|
||||
**credentials,
|
||||
"MAKA_EVAL_RESULT_TOKEN": result_token,
|
||||
}.items():
|
||||
secret.write(f"export {key}={shlex.quote(value)}\n")
|
||||
await environment.upload_file(secret_path, container_path)
|
||||
finally:
|
||||
if secret_path is not None:
|
||||
secret_path.unlink(missing_ok=True)
|
||||
subject = shlex.join([request["command"], *request["args"]])
|
||||
output_redirect = "" if capture_stdout else " >/dev/null"
|
||||
scope_error = shlex.quote(f"{SCOPE_ERROR_PREFIX} {result_token}\\n")
|
||||
inner = (
|
||||
"umask 077; "
|
||||
f"{{ echo $$ > {shlex.quote(scope_path)}; }} 2>/dev/null || "
|
||||
f"{{ printf {scope_error}; exit 111; }}; "
|
||||
f". {shlex.quote(container_path)}; command -p rm -f {shlex.quote(container_path)}; "
|
||||
f"exec {subject}{output_redirect}"
|
||||
)
|
||||
return f"setsid --wait sh -c {shlex.quote(inner)}"
|
||||
|
||||
|
||||
async def _require_constrained_subject(environment: Any) -> None:
|
||||
"""Refuse to start the subject unless the egress policy actually governs it.
|
||||
|
||||
Runs after the policy is applied and before the subject exists, so a task
|
||||
that keeps a capability the policy cannot see, or that keeps the subject out
|
||||
of the namespace the policy was applied to, fails the attempt instead of
|
||||
producing a result the isolation contract never actually covered.
|
||||
"""
|
||||
if os.environ.get("MAKA_EVAL_EGRESS_REQUIRED") != "1":
|
||||
return
|
||||
probe = await environment.exec(
|
||||
f"printf %s {shlex.quote(CAPABILITY_PREFIX)}; "
|
||||
r"sed -n 's/^\(Cap[A-Za-z]*\):[[:space:]]*/\1=/p' /proc/self/status | tr '\n' ' '; "
|
||||
"echo; " + NAMESPACE_PROBE
|
||||
)
|
||||
policy = await environment.service_exec(NAMESPACE_PROBE, service=POLICY_SERVICE)
|
||||
if probe.return_code != 0 or policy.return_code != 0:
|
||||
raise RuntimeError("Maka Eval could not read the subject isolation evidence")
|
||||
capabilities = _sole_probe_line(probe, CAPABILITY_PREFIX)
|
||||
namespace = _sole_probe_line(probe, NAMESPACE_PREFIX)
|
||||
policy_namespace = _sole_probe_line(policy, NAMESPACE_PREFIX)
|
||||
if not all(NAMESPACE_IDENTITY.match(value) for value in (namespace, policy_namespace)):
|
||||
raise RuntimeError("Maka Eval could not read the subject isolation evidence")
|
||||
reported: dict[str, int] = {}
|
||||
for field in capabilities.split():
|
||||
name, _, value = field.partition("=")
|
||||
try:
|
||||
reported[name] = int(value, 16)
|
||||
except ValueError:
|
||||
raise RuntimeError("Maka Eval could not read the subject capability set") from None
|
||||
if not all(field in reported for field in CAPABILITY_FIELDS):
|
||||
raise RuntimeError("Maka Eval could not read the subject capability set")
|
||||
granted = 0
|
||||
for value in reported.values():
|
||||
granted |= value
|
||||
held = sorted(name for name, bit in BYPASS_CAPABILITIES.items() if granted & bit)
|
||||
if held:
|
||||
raise RuntimeError(
|
||||
"the subject holds "
|
||||
+ ", ".join(held)
|
||||
+ ", which bypasses the Eval egress policy; remove it from the task"
|
||||
)
|
||||
if namespace != policy_namespace:
|
||||
raise RuntimeError(
|
||||
"the subject does not share the network namespace the Eval egress policy "
|
||||
"was applied to; remove the task's own networking on the subject service"
|
||||
)
|
||||
await _pin_proxy_hostname(environment)
|
||||
|
||||
|
||||
HOSTS_ALIAS_AWK = r"""
|
||||
/^[[:space:]]*#/ { print; next }
|
||||
{
|
||||
for (i = 2; i <= NF; i++) {
|
||||
if ($i == host) next
|
||||
}
|
||||
print
|
||||
}
|
||||
"""
|
||||
|
||||
# Four decimal octets 0-255, no empty fields and no leading zeros. The same
|
||||
# program is embedded in egress-proxy/network-policy; the contract test
|
||||
# requires the two copies to stay identical.
|
||||
IPV4_OCTET_AWK = r"""
|
||||
BEGIN { FS = "." }
|
||||
NF != 4 { exit 1 }
|
||||
{
|
||||
for (i = 1; i <= 4; i++) {
|
||||
if ($i !~ /^(0|[1-9][0-9]*)$/ || $i + 0 > 255) exit 1
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
_PUBLISHED_IPV4 = re.compile(
|
||||
r"^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}"
|
||||
r"(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$"
|
||||
)
|
||||
|
||||
|
||||
def _valid_egress_proxy_host(host: str) -> bool:
|
||||
if not host or host.startswith(".") or host.endswith(".") or ".." in host:
|
||||
return False
|
||||
# Match the pin script's `*[!A-Za-z0-9.-]*` class. str.isalnum() also
|
||||
# accepts Unicode letters, which the shell would later reject as a pin
|
||||
# failure instead of an invalid host.
|
||||
return all(
|
||||
(character.isalnum() and character.isascii()) or character in ".-"
|
||||
for character in host
|
||||
)
|
||||
|
||||
|
||||
def _valid_published_ipv4(ip: str) -> bool:
|
||||
return bool(_PUBLISHED_IPV4.fullmatch(ip))
|
||||
|
||||
|
||||
async def _pin_proxy_hostname(environment: Any) -> None:
|
||||
"""Point the proxy hostname at the published IPv4 so Docker DNS can be refused.
|
||||
|
||||
The subject only needs that one name, because the HTTP proxy does remote
|
||||
resolution itself. Pinning it in /etc/hosts lets the namespace policy reject
|
||||
127.0.0.11:53 without breaking HTTPS_PROXY.
|
||||
"""
|
||||
host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST") or "maka-eval-mitmproxy"
|
||||
if not _valid_egress_proxy_host(host):
|
||||
raise RuntimeError("Eval egress proxy host is invalid")
|
||||
script = f"""
|
||||
set -eu
|
||||
path={shlex.quote(PROXY_IPV4_PATH)}
|
||||
host={shlex.quote(host)}
|
||||
case "$host" in
|
||||
""|.*|*..*|*.) exit 1 ;;
|
||||
*[!A-Za-z0-9.-]*) exit 1 ;;
|
||||
esac
|
||||
test -f "$path"
|
||||
ip=$(tr -d ' \\t\\r\\n' < "$path")
|
||||
case "$ip" in
|
||||
""|*.*.*.*.*|*[!0-9.]*) exit 1 ;;
|
||||
esac
|
||||
if ! printf '%s\\n' "$ip" | awk {shlex.quote(IPV4_OCTET_AWK)}; then
|
||||
exit 1
|
||||
fi
|
||||
tmp=$(mktemp)
|
||||
trap 'rm -f "$tmp"' EXIT
|
||||
if [ -f /etc/hosts ]; then
|
||||
awk -v host="$host" {shlex.quote(HOSTS_ALIAS_AWK)} /etc/hosts > "$tmp"
|
||||
fi
|
||||
printf '%s %s\\n' "$ip" "$host" >> "$tmp"
|
||||
cat "$tmp" > /etc/hosts
|
||||
printf %s {shlex.quote(PROXY_HOSTS_PREFIX)}
|
||||
printf '%s %s\\n' "$ip" "$host"
|
||||
"""
|
||||
# Harbor 0.20.0 BaseEnvironment.exec and DockerEnvironment.exec take user=;
|
||||
# DockerEnvironment._compose_exec forwards it as `docker compose exec -u`.
|
||||
probe = await environment.exec(script, user="root")
|
||||
if probe.return_code != 0:
|
||||
raise RuntimeError("Maka Eval could not pin the Eval egress proxy hostname")
|
||||
reported = _sole_probe_line(probe, PROXY_HOSTS_PREFIX)
|
||||
ip, _, pinned = reported.partition(" ")
|
||||
if pinned != host or not _valid_published_ipv4(ip):
|
||||
raise RuntimeError("Maka Eval could not pin the Eval egress proxy hostname")
|
||||
|
||||
|
||||
def _sole_probe_line(probe: Any, prefix: str) -> str:
|
||||
reported = [
|
||||
line[len(prefix) :]
|
||||
for line in str(probe.stdout or "").splitlines()
|
||||
if line.startswith(prefix)
|
||||
]
|
||||
if len(reported) != 1:
|
||||
raise RuntimeError("Maka Eval could not read the subject isolation evidence")
|
||||
return reported[0].strip()
|
||||
|
||||
|
||||
async def _persist_subject_outputs(environment: Any, result: Any) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
stdout = root / "stdout"
|
||||
stderr = root / "stderr"
|
||||
stdout.write_text(str(getattr(result, "stdout", "") or ""), encoding="utf-8")
|
||||
stderr.write_text(str(getattr(result, "stderr", "") or ""), encoding="utf-8")
|
||||
prepared = await environment.exec("mkdir -p /logs/artifacts && chmod 700 /logs/artifacts")
|
||||
if prepared.return_code != 0:
|
||||
raise RuntimeError("Maka Eval could not prepare subject artifact output")
|
||||
await environment.upload_file(stdout, SUBJECT_STDOUT_PATH)
|
||||
await environment.upload_file(stderr, SUBJECT_STDERR_PATH)
|
||||
|
||||
|
||||
def _decode_result_carrier(carrier: str, token: str) -> tuple[str, dict[str, Any]]:
|
||||
raw = carrier.encode("utf-8", errors="replace")
|
||||
if len(raw) > RESULT_CARRIER_LIMIT_BYTES:
|
||||
return "", _carrier_diagnostic("result-frame-oversize", raw)
|
||||
prefix = f"{RESULT_FRAME_PREFIX} {token} "
|
||||
candidates = [line for line in carrier.splitlines(keepends=True) if line.startswith(prefix)]
|
||||
if len(candidates) != 1:
|
||||
category = "result-frame-missing" if not candidates else "result-frame-ambiguous"
|
||||
return "", _carrier_diagnostic(category, raw)
|
||||
frame = candidates[0]
|
||||
fields = frame.rstrip("\r\n").split(" ", 4)
|
||||
if len(fields) != 5:
|
||||
return "", _carrier_diagnostic("result-frame-invalid", raw)
|
||||
_, framed_token, length_text, digest, encoded = fields
|
||||
try:
|
||||
length = int(length_text)
|
||||
padding = "=" * (-len(encoded) % 4)
|
||||
payload = base64.b64decode(encoded + padding, altchars=b"-_", validate=True)
|
||||
except (ValueError, base64.binascii.Error):
|
||||
return "", _carrier_diagnostic("result-frame-invalid", raw)
|
||||
if (
|
||||
framed_token != token
|
||||
or length < 0
|
||||
or length > RESULT_PAYLOAD_LIMIT_BYTES
|
||||
or len(payload) != length
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", digest)
|
||||
or hashlib.sha256(payload).hexdigest() != digest
|
||||
):
|
||||
return "", _carrier_diagnostic("result-frame-invalid", raw)
|
||||
noise = carrier.replace(frame, "", 1).encode("utf-8", errors="replace")
|
||||
diagnostic = (
|
||||
{"category": "none"}
|
||||
if not noise
|
||||
else _carrier_diagnostic("unstructured-output", noise)
|
||||
)
|
||||
try:
|
||||
decoded = payload.decode("utf-8", errors="strict")
|
||||
except UnicodeDecodeError:
|
||||
return "", _carrier_diagnostic("result-frame-invalid", raw)
|
||||
return decoded, diagnostic
|
||||
|
||||
|
||||
def _project_result(result: Any, request: dict[str, Any]) -> tuple[str, dict[str, Any]]:
|
||||
carrier = str(getattr(result, "stdout", "") or "")
|
||||
scope_error = f"{SCOPE_ERROR_PREFIX} {request['resultToken']}"
|
||||
if carrier == f"{scope_error}\n":
|
||||
return "", _carrier_diagnostic("execution-scope-unavailable", carrier.encode())
|
||||
if not request.get("captureStdout", True):
|
||||
return "", {"category": "none"}
|
||||
return _decode_result_carrier(carrier, request["resultToken"])
|
||||
|
||||
|
||||
def _carrier_diagnostic(category: str, value: bytes) -> dict[str, Any]:
|
||||
return {
|
||||
"category": category,
|
||||
"bytes": len(value),
|
||||
"sha256": hashlib.sha256(value).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
async def _settle(environment: Any, cwd: str, scope_path: str, execution: Any) -> Any:
|
||||
if execution.cancelled():
|
||||
raise RuntimeError("Maka Eval subject execution was cancelled")
|
||||
if execution.done():
|
||||
result = execution.result()
|
||||
else:
|
||||
result = None
|
||||
for signal, timeout in (("TERM", 20), ("KILL", 10)):
|
||||
await _signal(environment, cwd, scope_path, signal)
|
||||
try:
|
||||
result = await asyncio.wait_for(asyncio.shield(execution), timeout=timeout)
|
||||
break
|
||||
except asyncio.CancelledError:
|
||||
if execution.cancelled():
|
||||
raise RuntimeError("Maka Eval subject execution was cancelled") from None
|
||||
raise
|
||||
except TimeoutError:
|
||||
pass
|
||||
if result is None:
|
||||
raise RuntimeError("Maka Eval subject did not settle")
|
||||
await _quiesce_scope(environment, cwd, scope_path)
|
||||
return result
|
||||
|
||||
|
||||
async def _settle_or_destroy(
|
||||
environment: Any,
|
||||
cwd: str,
|
||||
scope_path: str,
|
||||
execution: Any,
|
||||
timeout: float,
|
||||
) -> Any | None:
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout
|
||||
stop_reserve = min(20.0, timeout * 0.2)
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
_settle(environment, cwd, scope_path, execution),
|
||||
timeout=max(0.001, deadline - loop.time() - stop_reserve),
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
remaining = max(0.001, deadline - loop.time())
|
||||
await asyncio.wait_for(environment.stop(delete=True), timeout=remaining)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
if not execution.done():
|
||||
execution.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
remaining = max(0.001, deadline - loop.time())
|
||||
await asyncio.wait_for(execution, timeout=remaining)
|
||||
return None
|
||||
|
||||
|
||||
async def _signal(environment: Any, cwd: str, scope_path: str, signal: str) -> None:
|
||||
command = (
|
||||
f"pgid=$(cat {shlex.quote(scope_path)} 2>/dev/null) || exit 0; "
|
||||
"case $pgid in ''|0|*[!0-9]*) exit 0;; esac; "
|
||||
f"kill -{signal} -- \"-$pgid\""
|
||||
)
|
||||
with contextlib.suppress(Exception):
|
||||
await environment.exec(
|
||||
command,
|
||||
cwd=cwd,
|
||||
timeout_sec=5,
|
||||
)
|
||||
|
||||
|
||||
async def _quiesce_scope(environment: Any, cwd: str, scope_path: str) -> None:
|
||||
if not await _scope_active(environment, cwd, scope_path):
|
||||
return
|
||||
for signal, timeout in (("TERM", 10), ("KILL", 5)):
|
||||
await _signal(environment, cwd, scope_path, signal)
|
||||
deadline = asyncio.get_running_loop().time() + timeout
|
||||
while asyncio.get_running_loop().time() < deadline:
|
||||
if not await _scope_active(environment, cwd, scope_path):
|
||||
return
|
||||
await asyncio.sleep(0.1)
|
||||
raise RuntimeError("Maka Eval execution scope did not quiesce")
|
||||
|
||||
|
||||
async def _scope_active(environment: Any, cwd: str, scope_path: str) -> bool:
|
||||
result = await environment.exec(
|
||||
f"pgid=$(cat {shlex.quote(scope_path)} 2>/dev/null) || exit 4; "
|
||||
"case $pgid in ''|0|*[!0-9]*) exit 4;; esac; "
|
||||
"kill -0 -- \"-$pgid\" 2>/dev/null; status=$?; "
|
||||
"if [ $status -eq 0 ]; then exit 0; fi; exit 3",
|
||||
cwd=cwd,
|
||||
timeout_sec=5,
|
||||
)
|
||||
if result.return_code == 0:
|
||||
return True
|
||||
if result.return_code == 3:
|
||||
return False
|
||||
raise RuntimeError("Maka Eval execution scope evidence was unavailable")
|
||||
|
||||
|
||||
def _require_message(value: dict[str, Any], token: str, kind: str) -> None:
|
||||
if value.get("token") != token or value.get("kind") != kind:
|
||||
raise RuntimeError("invalid Maka Eval relay message")
|
||||
|
||||
|
||||
async def _receive(reader: asyncio.StreamReader) -> dict[str, Any]:
|
||||
try:
|
||||
raw = await reader.readline()
|
||||
if not raw:
|
||||
raise RelayTransportClosed("Maka Eval relay peer closed")
|
||||
value = json.loads(raw)
|
||||
except (ConnectionError, json.JSONDecodeError, ValueError) as error:
|
||||
raise RelayTransportClosed("Maka Eval relay message was unavailable") from error
|
||||
if not isinstance(value, dict):
|
||||
raise RelayTransportClosed("Maka Eval relay message was invalid")
|
||||
return value
|
||||
|
||||
|
||||
async def _send(writer: asyncio.StreamWriter, value: object) -> bool:
|
||||
if writer.is_closing():
|
||||
return False
|
||||
try:
|
||||
writer.write((json.dumps(value, separators=(",", ":")) + "\n").encode())
|
||||
await writer.drain()
|
||||
return True
|
||||
except (BrokenPipeError, ConnectionError, RuntimeError):
|
||||
return False
|
||||
@@ -0,0 +1,223 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Run exactly one pinned Harbor or Pier Trial."""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import fcntl
|
||||
import hashlib
|
||||
import importlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
|
||||
FRAMEWORK_VERSION_MISMATCH_EXIT_CODE = 78
|
||||
|
||||
|
||||
class FrameworkVersionMismatch(Exception):
|
||||
pass
|
||||
|
||||
|
||||
TASK_CACHE_PROTOCOL_VERSION = 1
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def task_cache_lock(cache_dir: Path, identity: str) -> AsyncIterator[None]:
|
||||
lock_dir = cache_dir / ".maka-locks"
|
||||
lock_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
lock_path = lock_dir / f"{hashlib.sha256(identity.encode()).hexdigest()}.lock"
|
||||
flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(lock_path, flags, 0o600)
|
||||
acquired = False
|
||||
try:
|
||||
os.fchmod(descriptor, 0o600)
|
||||
while not acquired:
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
acquired = True
|
||||
except BlockingIOError:
|
||||
await asyncio.sleep(0.05)
|
||||
yield
|
||||
finally:
|
||||
if acquired:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def ready_task(cache_dir: Path, identity: str) -> tuple[Path, Path | None]:
|
||||
namespace = cache_dir / ".maka-tasks" / hashlib.sha256(identity.encode()).hexdigest()
|
||||
marker = namespace / "ready.json"
|
||||
try:
|
||||
value = json.loads(marker.read_text())
|
||||
if set(value) != {"version", "target"}:
|
||||
return namespace, None
|
||||
if value["version"] != TASK_CACHE_PROTOCOL_VERSION or not isinstance(
|
||||
value["target"], str
|
||||
):
|
||||
return namespace, None
|
||||
target = (namespace / value["target"]).resolve()
|
||||
if (
|
||||
not target.is_relative_to(namespace.resolve())
|
||||
or not target.is_dir()
|
||||
or not any(target.iterdir())
|
||||
):
|
||||
return namespace, None
|
||||
return namespace, target
|
||||
except (OSError, TypeError, ValueError):
|
||||
return namespace, None
|
||||
|
||||
|
||||
def publish_ready_task(namespace: Path, target: Path) -> None:
|
||||
resolved_namespace = namespace.resolve()
|
||||
resolved_target = target.resolve()
|
||||
if not resolved_target.is_relative_to(resolved_namespace):
|
||||
raise RuntimeError("Harbor task cache target escaped its Maka namespace")
|
||||
relative_target = resolved_target.relative_to(resolved_namespace).as_posix()
|
||||
marker = namespace / "ready.json"
|
||||
temporary = namespace / ".ready.tmp"
|
||||
descriptor = os.open(
|
||||
temporary,
|
||||
os.O_CREAT | os.O_WRONLY | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0),
|
||||
0o600,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w") as stream:
|
||||
descriptor = -1
|
||||
json.dump(
|
||||
{"version": TASK_CACHE_PROTOCOL_VERSION, "target": relative_target},
|
||||
stream,
|
||||
)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, marker)
|
||||
finally:
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
async def run_trial(framework: str, expected_version: str, config_file: Path) -> None:
|
||||
from eval_framework import install
|
||||
|
||||
install(framework)
|
||||
distribution = {"harbor": "harbor", "pier": "datacurve-pier"}[framework]
|
||||
if importlib.metadata.version(distribution) != expected_version:
|
||||
raise FrameworkVersionMismatch
|
||||
try:
|
||||
config_type = importlib.import_module(f"{framework}.models.trial.config").TrialConfig
|
||||
trial_type = importlib.import_module(f"{framework}.trial.trial").Trial
|
||||
config = config_type.model_validate_json(config_file.read_text())
|
||||
config_file.unlink()
|
||||
if framework == "harbor":
|
||||
from harbor.constants import TASK_CACHE_DIR
|
||||
|
||||
task_identity = config.task.get_task_id().model_dump_json()
|
||||
async with task_cache_lock(TASK_CACHE_DIR, task_identity):
|
||||
namespace, ready = ready_task(TASK_CACHE_DIR, task_identity)
|
||||
namespace.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
config.task.download_dir = namespace
|
||||
config.task.overwrite = ready is None
|
||||
trial = await create_harbor_trial(trial_type, config)
|
||||
target = Path(trial.task.task_dir)
|
||||
if ready is not None and target.resolve() != ready:
|
||||
raise RuntimeError("Harbor task cache target changed for one identity")
|
||||
if ready is None:
|
||||
publish_ready_task(namespace, target)
|
||||
else:
|
||||
trial = await trial_type.create(config)
|
||||
await trial.run()
|
||||
finally:
|
||||
config_file.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def apply_subject_egress_policy(task: object) -> None:
|
||||
required = os.environ.get("MAKA_EVAL_EGRESS_REQUIRED") == "1"
|
||||
allowed_host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST")
|
||||
if not required:
|
||||
return
|
||||
if not allowed_host:
|
||||
raise RuntimeError("required Eval egress proxy host is unavailable")
|
||||
agent = task.config.agent
|
||||
agent.network_mode = "allowlist"
|
||||
agent.allowed_hosts = [allowed_host]
|
||||
|
||||
|
||||
async def create_harbor_trial(trial_type: type, config: object) -> object:
|
||||
trial_type._resolve_agent_skills(config)
|
||||
task, task_download_result = await trial_type._load_task(config)
|
||||
apply_subject_egress_policy(task)
|
||||
if task.has_steps:
|
||||
from harbor.trial.multi_step import MultiStepTrial
|
||||
|
||||
return MultiStepTrial(
|
||||
config,
|
||||
_task=task,
|
||||
_task_download_result=task_download_result,
|
||||
)
|
||||
from harbor.trial.single_step import SingleStepTrial
|
||||
|
||||
return SingleStepTrial(
|
||||
config,
|
||||
_task=task,
|
||||
_task_download_result=task_download_result,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
from eval_framework import install
|
||||
|
||||
framework, expected_version, config_path = sys.argv[1:]
|
||||
install(framework)
|
||||
task = asyncio.current_task()
|
||||
assert task is not None
|
||||
loop = asyncio.get_running_loop()
|
||||
teardown_requested = False
|
||||
|
||||
def request_teardown() -> None:
|
||||
nonlocal teardown_requested
|
||||
if teardown_requested:
|
||||
return
|
||||
teardown_requested = True
|
||||
try:
|
||||
from relay_agent import request_host_teardown
|
||||
request_host_teardown()
|
||||
finally:
|
||||
task.cancel()
|
||||
|
||||
for host_signal in (signal.SIGINT, signal.SIGTERM):
|
||||
loop.add_signal_handler(host_signal, request_teardown)
|
||||
try:
|
||||
try:
|
||||
await run_trial(framework, expected_version, Path(config_path))
|
||||
except FrameworkVersionMismatch:
|
||||
raise SystemExit(FRAMEWORK_VERSION_MISMATCH_EXIT_CODE) from None
|
||||
except asyncio.CancelledError:
|
||||
if not teardown_requested:
|
||||
raise
|
||||
finally:
|
||||
for host_signal in (signal.SIGINT, signal.SIGTERM):
|
||||
loop.remove_signal_handler(host_signal)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,511 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Prove the cell namespace forces subject egress through the audited proxy.
|
||||
|
||||
The Python policy test only checks that `run_trial.py` hands Harbor the right
|
||||
allowlist. This test brings up the checked-in Compose overlay against a minimal
|
||||
Harbor-shaped base, applies the checked-in `network-policy` from a sidecar that
|
||||
shares the subject namespace, and asserts what the README promises: explicit
|
||||
proxy traffic works, everything else does not, and the subject never sees the
|
||||
CA private key or the audit log.
|
||||
|
||||
It needs a working Docker daemon and outbound network, so it is opt-in:
|
||||
|
||||
MAKA_EVAL_EGRESS_NAMESPACE_TEST=1 python3 harbor/test_cell_egress_namespace.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from test_relay_contract import load_relay
|
||||
|
||||
HARBOR_DIR = Path(__file__).parent
|
||||
OVERLAY = HARBOR_DIR / "docker-compose-egress-proxy.yaml"
|
||||
NETWORK_POLICY = HARBOR_DIR / "egress-proxy" / "network-policy"
|
||||
PROXY_IMAGE = "maka-eval-egress-proxy:12.2.3"
|
||||
MAIN_IMAGE = "maka-eval-egress-namespace-main:test"
|
||||
SIDECAR_IMAGE = "maka-eval-egress-namespace-sidecar:test"
|
||||
PROJECT = "maka-eval-egress-namespace-test"
|
||||
PROXY_HOST = "maka-eval-mitmproxy"
|
||||
SIDECAR = "harbor-docker-egress-control-sidecar"
|
||||
OUTSIDER = "maka-eval-outside-the-namespace"
|
||||
BOUNDED = "maka-eval-bounded-capability"
|
||||
CA_PATH = "/opt/maka-egress/mitmproxy-ca-cert.pem"
|
||||
BUILD_TIMEOUT_S = 900
|
||||
COMMAND_TIMEOUT_S = 120
|
||||
|
||||
BASE_COMPOSE = f"""\
|
||||
services:
|
||||
main:
|
||||
image: {MAIN_IMAGE}
|
||||
command: ["sleep", "infinity"]
|
||||
|
||||
{SIDECAR}:
|
||||
image: {SIDECAR_IMAGE}
|
||||
# Harbor's own sidecar runs gost on the port the policy redirects to, so a
|
||||
# redirected connection is accepted and then goes nowhere. Standing in for
|
||||
# it keeps the negative assertions honest: without a listener they would
|
||||
# hold on a refused connection instead of on the policy.
|
||||
command: ["sh", "-c", "nc -l -k 12345 >/dev/null 2>&1 & exec sleep infinity"]
|
||||
network_mode: "service:main"
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
depends_on:
|
||||
- main
|
||||
|
||||
# Two services the gate must refuse, so its live coverage is not only the
|
||||
# positive case. This one keeps its own network namespace, the way Harbor
|
||||
# leaves a subject whose task declares its own networking.
|
||||
{OUTSIDER}:
|
||||
image: {MAIN_IMAGE}
|
||||
command: ["sleep", "infinity"]
|
||||
cap_drop:
|
||||
- NET_RAW
|
||||
|
||||
# This one shares the policy's namespace and reports an empty effective set,
|
||||
# while the bounding set still lets a file-capability executable take NET_RAW
|
||||
# back. Reading only the effective set would admit it.
|
||||
{BOUNDED}:
|
||||
image: {MAIN_IMAGE}
|
||||
command: ["sleep", "infinity"]
|
||||
network_mode: "service:main"
|
||||
user: "1000"
|
||||
depends_on:
|
||||
- main
|
||||
"""
|
||||
|
||||
MAIN_DOCKERFILE = """\
|
||||
FROM python:3.12-slim
|
||||
RUN apt-get update \\
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl iputils-ping \\
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
"""
|
||||
|
||||
# Debian bookworm ships nftables 1.0.6, which rejects the `dstnat` priority name
|
||||
# on a nat output chain; the policy needs 1.1 or newer, as Harbor's sidecar has.
|
||||
SIDECAR_DOCKERFILE = """\
|
||||
FROM debian:trixie-slim
|
||||
RUN apt-get update \\
|
||||
&& apt-get install -y --no-install-recommends nftables netcat-openbsd \\
|
||||
&& rm -rf /var/lib/apt/lists/* \\
|
||||
&& mkdir -p /opt/egress-sidecar
|
||||
"""
|
||||
|
||||
# The policy redirects TCP to the sidecar's proxy port rather than dropping it,
|
||||
# so the connection still completes and only the response tells a direct route
|
||||
# from a redirected one. Judging this on `connect` would assert nothing in a
|
||||
# cell where that proxy is listening, which is every real one.
|
||||
TCP_PROBE = (
|
||||
"import socket\n"
|
||||
"request = b'GET / HTTP/1.0\\r\\nHost: one.one.one.one\\r\\n\\r\\n'\n"
|
||||
"try:\n"
|
||||
" with socket.create_connection(('1.1.1.1', 80), 5) as sock:\n"
|
||||
" sock.settimeout(5)\n"
|
||||
" sock.sendall(request)\n"
|
||||
" print('reachable' if sock.recv(16).startswith(b'HTTP/') else 'blocked')\n"
|
||||
"except OSError:\n"
|
||||
" print('blocked')\n"
|
||||
)
|
||||
|
||||
# A well-formed query for example.com. A malformed datagram is dropped without a
|
||||
# reply even by a reachable resolver, which would report the open network as
|
||||
# blocked and silently retire this probe's assertion.
|
||||
UDP_PROBE = (
|
||||
"import socket\n"
|
||||
"query = (\n"
|
||||
" b'\\xab\\xcd\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00'\n"
|
||||
" b'\\x07example\\x03com\\x00\\x00\\x01\\x00\\x01'\n"
|
||||
")\n"
|
||||
"sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n"
|
||||
"sock.settimeout(5)\n"
|
||||
"try:\n"
|
||||
" sock.sendto(query, ('8.8.8.8', 53))\n"
|
||||
" sock.recvfrom(512)\n"
|
||||
" print('reachable')\n"
|
||||
"except OSError:\n"
|
||||
" print('blocked')\n"
|
||||
)
|
||||
|
||||
DOCKER_DNS_PROBE = (
|
||||
"import socket\n"
|
||||
"query = (\n"
|
||||
" b'\\xab\\xcd\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00'\n"
|
||||
" b'\\x07example\\x03com\\x00\\x00\\x01\\x00\\x01'\n"
|
||||
")\n"
|
||||
"sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n"
|
||||
"sock.settimeout(5)\n"
|
||||
"try:\n"
|
||||
" sock.sendto(query, ('127.0.0.11', 53))\n"
|
||||
" sock.recvfrom(512)\n"
|
||||
" print('reachable')\n"
|
||||
"except OSError:\n"
|
||||
" print('blocked')\n"
|
||||
)
|
||||
|
||||
# An AF_PACKET socket writes at the link layer, below the IP output hooks the
|
||||
# policy installs, so no nftables rule can see this traffic at all. Only the
|
||||
# absence of NET_RAW stops it. The frame is addressed to the default gateway's
|
||||
# hardware address, which the pre-policy reachability assertions leave in the
|
||||
# neighbour table.
|
||||
PACKET_PROBE = (
|
||||
"import socket, struct\n"
|
||||
"query = (\n"
|
||||
" b'\\xab\\xcd\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00'\n"
|
||||
" b'\\x07example\\x03com\\x00\\x00\\x01\\x00\\x01'\n"
|
||||
")\n"
|
||||
"def route():\n"
|
||||
" with open('/proc/net/route') as handle:\n"
|
||||
" for line in handle.readlines()[1:]:\n"
|
||||
" f = line.split()\n"
|
||||
" if f[1] == '00000000':\n"
|
||||
" return f[0], socket.inet_ntoa(struct.pack('<L', int(f[2], 16)))\n"
|
||||
" return None, None\n"
|
||||
"def neighbour(address):\n"
|
||||
" with open('/proc/net/arp') as handle:\n"
|
||||
" for line in handle.readlines()[1:]:\n"
|
||||
" f = line.split()\n"
|
||||
" if f[0] == address and f[3] != '00:00:00:00:00:00':\n"
|
||||
" return bytes.fromhex(f[3].replace(':', ''))\n"
|
||||
" return None\n"
|
||||
"def checksum(data):\n"
|
||||
" total = sum(struct.unpack('!%dH' % (len(data) // 2), data))\n"
|
||||
" while total >> 16:\n"
|
||||
" total = (total & 0xFFFF) + (total >> 16)\n"
|
||||
" return ~total & 0xFFFF\n"
|
||||
"interface, gateway = route()\n"
|
||||
"mac = neighbour(gateway) if gateway else None\n"
|
||||
"if not interface or not mac:\n"
|
||||
" print('unusable')\n"
|
||||
"else:\n"
|
||||
" try:\n"
|
||||
" sock = socket.socket(socket.AF_PACKET, socket.SOCK_DGRAM, 0x0800)\n"
|
||||
" except PermissionError:\n"
|
||||
" print('denied')\n"
|
||||
" raise SystemExit\n"
|
||||
" try:\n"
|
||||
" source = socket.gethostbyname(socket.gethostname())\n"
|
||||
" sock.bind((interface, 0x0800))\n"
|
||||
" sock.settimeout(5)\n"
|
||||
" udp = struct.pack('!HHHH', 40000, 53, 8 + len(query), 0) + query\n"
|
||||
" header = struct.pack(\n"
|
||||
" '!BBHHHBBH4s4s', 0x45, 0, 20 + len(udp), 0x1234, 0, 64, 17, 0,\n"
|
||||
" socket.inet_aton(source), socket.inet_aton('8.8.8.8'),\n"
|
||||
" )\n"
|
||||
" header = header[:10] + struct.pack('!H', checksum(header)) + header[12:]\n"
|
||||
" sock.sendto(header + udp, (interface, 0x0800, 0, 0, mac))\n"
|
||||
" while True:\n"
|
||||
" packet = sock.recv(2048)\n"
|
||||
" if len(packet) >= 20 and packet[12:16] == socket.inet_aton('8.8.8.8'):\n"
|
||||
" print('reachable')\n"
|
||||
" break\n"
|
||||
" except OSError:\n"
|
||||
" print('blocked')\n"
|
||||
)
|
||||
|
||||
LOOPBACK_PROBE = (
|
||||
"import socket\n"
|
||||
"server = socket.socket()\n"
|
||||
"server.bind(('127.0.0.1', 0))\n"
|
||||
"server.listen(1)\n"
|
||||
"client = socket.create_connection(server.getsockname(), 5)\n"
|
||||
"client.close()\n"
|
||||
"print('reachable')\n"
|
||||
)
|
||||
|
||||
PROXY_ENVIRONMENT = {
|
||||
"HTTPS_PROXY": f"http://{PROXY_HOST}:8080",
|
||||
"CURL_CA_BUNDLE": CA_PATH,
|
||||
}
|
||||
|
||||
# The bypass attempt carries no CA override: pinning trust to the cell CA would
|
||||
# fail a successful direct connection on certificate verification instead, and
|
||||
# the assertion would hold even with the policy removed.
|
||||
BYPASS_ENVIRONMENT = {"HTTPS_PROXY": f"http://{PROXY_HOST}:8080"}
|
||||
|
||||
|
||||
def docker_available() -> bool:
|
||||
if shutil.which("docker") is None:
|
||||
return False
|
||||
probe = subprocess.run(
|
||||
["docker", "version", "--format", "{{.Server.Version}}"],
|
||||
capture_output=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
return probe.returncode == 0
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
os.environ.get("MAKA_EVAL_EGRESS_NAMESPACE_TEST") == "1",
|
||||
"set MAKA_EVAL_EGRESS_NAMESPACE_TEST=1 to run the Docker cell namespace test",
|
||||
)
|
||||
class CellEgressNamespaceTest(unittest.TestCase):
|
||||
workdir: Path | None = None
|
||||
compose: list[str] = []
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
if not docker_available():
|
||||
raise unittest.SkipTest("Docker daemon is unavailable")
|
||||
cls.workdir = Path(tempfile.mkdtemp(prefix="maka-eval-egress-namespace-"))
|
||||
# Registered before anything can fail: `tearDownClass` does not run when
|
||||
# `setUpClass` raises, which would leak the stack and the named volume.
|
||||
cls.addClassCleanup(shutil.rmtree, cls.workdir, ignore_errors=True)
|
||||
(cls.workdir / "base.yaml").write_text(BASE_COMPOSE)
|
||||
cls.build_image(MAIN_IMAGE, dockerfile=MAIN_DOCKERFILE, name="main")
|
||||
cls.build_image(SIDECAR_IMAGE, dockerfile=SIDECAR_DOCKERFILE, name="sidecar")
|
||||
cls.build_image(
|
||||
PROXY_IMAGE, context=HARBOR_DIR, path=HARBOR_DIR / "egress-proxy" / "Dockerfile"
|
||||
)
|
||||
cls.compose = [
|
||||
"docker",
|
||||
"compose",
|
||||
"--project-name",
|
||||
PROJECT,
|
||||
"--file",
|
||||
str(cls.workdir / "base.yaml"),
|
||||
"--file",
|
||||
str(OVERLAY),
|
||||
]
|
||||
cls.run_compose(["down", "--volumes", "--remove-orphans"], check=False)
|
||||
cls.addClassCleanup(
|
||||
cls.run_compose, ["down", "--volumes", "--remove-orphans"], check=False
|
||||
)
|
||||
cls.run_compose(["up", "--detach", "--wait"], timeout=BUILD_TIMEOUT_S)
|
||||
# Pin the proxy hostname before any test applies the policy, so
|
||||
# HTTPS_PROXY still resolves after Docker DNS is refused.
|
||||
asyncio.run(load_relay()._pin_proxy_hostname(CellEnvironment("main")))
|
||||
|
||||
@classmethod
|
||||
def build_image(
|
||||
cls,
|
||||
tag: str,
|
||||
*,
|
||||
dockerfile: str | None = None,
|
||||
name: str | None = None,
|
||||
context: Path | None = None,
|
||||
path: Path | None = None,
|
||||
) -> None:
|
||||
if dockerfile is not None:
|
||||
context = cls.workdir / name
|
||||
context.mkdir()
|
||||
(context / "Dockerfile").write_text(dockerfile)
|
||||
subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"build",
|
||||
"--tag",
|
||||
tag,
|
||||
*(["--file", str(path)] if path else []),
|
||||
str(context),
|
||||
],
|
||||
check=True,
|
||||
timeout=BUILD_TIMEOUT_S,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def run_compose(
|
||||
cls,
|
||||
arguments: list[str],
|
||||
*,
|
||||
check: bool = True,
|
||||
timeout: int = COMMAND_TIMEOUT_S,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[*cls.compose, *arguments],
|
||||
env={**os.environ, "MAKA_EVAL_NETWORK_POLICY_PATH": str(NETWORK_POLICY)},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=check,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def exec_service(
|
||||
cls,
|
||||
service: str,
|
||||
command: list[str],
|
||||
*,
|
||||
environment: dict[str, str] | None = None,
|
||||
user: str | None = None,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
overrides = [
|
||||
argument
|
||||
for name, value in (environment or {}).items()
|
||||
for argument in ("--env", f"{name}={value}")
|
||||
]
|
||||
user_args = ["--user", user] if user else []
|
||||
return cls.run_compose(
|
||||
["exec", "--no-TTY", *user_args, *overrides, service, *command], check=False
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def exec_main(
|
||||
cls, command: list[str], *, environment: dict[str, str] | None = None
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
return cls.exec_service("main", command, environment=environment)
|
||||
|
||||
@classmethod
|
||||
def probe_main(cls, script: str) -> str:
|
||||
result = cls.exec_main(["python3", "-c", script])
|
||||
if result.returncode != 0:
|
||||
raise AssertionError(f"probe failed to run: {result.stderr}")
|
||||
return result.stdout.strip()
|
||||
|
||||
@classmethod
|
||||
def curl(
|
||||
cls, arguments: list[str], *, environment: dict[str, str] | None = None
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
return cls.exec_main(
|
||||
[
|
||||
"curl",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--max-time",
|
||||
"20",
|
||||
"--output",
|
||||
"/dev/null",
|
||||
*arguments,
|
||||
"https://example.com",
|
||||
],
|
||||
environment=environment,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def ping(cls) -> subprocess.CompletedProcess[str]:
|
||||
return cls.exec_main(["ping", "-c", "1", "-W", "5", "1.1.1.1"])
|
||||
|
||||
def test_no_network_phase_is_not_answered_with_proxy_access(self) -> None:
|
||||
# Harbor's deny-all denies controlled egress outright, so answering it
|
||||
# with the proxy-only ruleset would grant a phase that asked for no
|
||||
# network a route this override invented.
|
||||
reachable = self.curl([], environment=PROXY_ENVIRONMENT)
|
||||
self.assertEqual(reachable.returncode, 0, reachable.stderr)
|
||||
try:
|
||||
denied = self.exec_service(SIDECAR, ["network-policy", "deny-all"])
|
||||
self.assertEqual(denied.returncode, 0, denied.stderr)
|
||||
blocked = self.curl([], environment=PROXY_ENVIRONMENT)
|
||||
self.assertNotEqual(blocked.returncode, 0)
|
||||
finally:
|
||||
restored = self.exec_service(SIDECAR, ["network-policy", "allow", PROXY_HOST])
|
||||
self.assertEqual(restored.returncode, 0, restored.stderr)
|
||||
|
||||
def test_relay_admits_only_a_subject_the_policy_governs(self) -> None:
|
||||
# The gate runs against live containers rather than a fixture, so what
|
||||
# it parses is a real `/proc` and what it judges is a real namespace.
|
||||
# The refusals matter more than the admission: without them nothing
|
||||
# would catch a probe that reports one constant answer.
|
||||
relay = load_relay()
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
asyncio.run(relay._require_constrained_subject(CellEnvironment("main")))
|
||||
with self.assertRaises(RuntimeError) as outside:
|
||||
asyncio.run(relay._require_constrained_subject(CellEnvironment(OUTSIDER)))
|
||||
with self.assertRaises(RuntimeError) as bounded:
|
||||
asyncio.run(relay._require_constrained_subject(CellEnvironment(BOUNDED)))
|
||||
self.assertIn("network namespace", str(outside.exception))
|
||||
self.assertIn("NET_RAW", str(bounded.exception))
|
||||
|
||||
def test_subject_sees_only_the_ca_certificate(self) -> None:
|
||||
listed = self.exec_main(["ls", "--almost-all", "/opt/maka-egress"])
|
||||
self.assertEqual(listed.returncode, 0, listed.stderr)
|
||||
self.assertEqual(sorted(listed.stdout.split()), ["mitmproxy-ca-cert.pem", "proxy-ipv4"])
|
||||
|
||||
def test_cell_namespace_admits_only_the_audited_proxy(self) -> None:
|
||||
# Every negative assertion below is vacuous on a host that cannot reach
|
||||
# the target anyway, so the reachability of each one is asserted before
|
||||
# the policy exists. Failing there is the point: a skip would report a
|
||||
# contract this run never actually exercised.
|
||||
self.assertEqual(self.curl([]).returncode, 0, "no HTTPS before any policy")
|
||||
self.assertEqual(self.probe_main(TCP_PROBE), "reachable")
|
||||
self.assertEqual(self.probe_main(UDP_PROBE), "reachable")
|
||||
self.assertEqual(self.probe_main(DOCKER_DNS_PROBE), "reachable")
|
||||
self.assertEqual(self.ping().returncode, 0, "no ICMP before any policy")
|
||||
|
||||
applied = self.exec_service(
|
||||
SIDECAR, ["network-policy", "allow", PROXY_HOST]
|
||||
)
|
||||
self.assertEqual(applied.returncode, 0, applied.stderr)
|
||||
|
||||
with self.subTest("explicit proxy HTTPS"):
|
||||
allowed = self.curl([], environment=PROXY_ENVIRONMENT)
|
||||
self.assertEqual(allowed.returncode, 0, allowed.stderr)
|
||||
|
||||
with self.subTest("curl --noproxy"):
|
||||
bypassed = self.curl(["--noproxy", "*"], environment=BYPASS_ENVIRONMENT)
|
||||
self.assertNotEqual(bypassed.returncode, 0)
|
||||
|
||||
with self.subTest("direct IP TCP"):
|
||||
self.assertEqual(self.probe_main(TCP_PROBE), "blocked")
|
||||
|
||||
# Not preceded by a reachability assertion like the others: this path is
|
||||
# closed by the missing capability rather than by the policy, so it is
|
||||
# already shut before the policy exists. `denied` rather than `blocked`
|
||||
# so a probe that breaks for any other reason cannot report success.
|
||||
with self.subTest("link-layer AF_PACKET"):
|
||||
self.assertEqual(self.probe_main(PACKET_PROBE), "denied")
|
||||
|
||||
with self.subTest("external UDP"):
|
||||
self.assertEqual(self.probe_main(UDP_PROBE), "blocked")
|
||||
|
||||
with self.subTest("external ICMP"):
|
||||
self.assertNotEqual(self.ping().returncode, 0)
|
||||
|
||||
with self.subTest("loopback provider proxy"):
|
||||
self.assertEqual(self.probe_main(LOOPBACK_PROBE), "reachable")
|
||||
|
||||
with self.subTest("pinned proxy hostname"):
|
||||
resolved = self.exec_main(["getent", "hosts", PROXY_HOST])
|
||||
self.assertEqual(resolved.returncode, 0, resolved.stderr)
|
||||
|
||||
with self.subTest("Docker DNS"):
|
||||
self.assertEqual(self.probe_main(DOCKER_DNS_PROBE), "blocked")
|
||||
|
||||
class CellEnvironment:
|
||||
"""Presents one live cell service as the relay's environment."""
|
||||
|
||||
def __init__(self, service: str) -> None:
|
||||
self.service = service
|
||||
|
||||
async def exec(self, command: str, cwd=None, env=None, timeout_sec=None, user=None):
|
||||
return self._run(
|
||||
self.service, command, user=str(user) if user is not None else None
|
||||
)
|
||||
|
||||
async def service_exec(self, command: str, *, service: str, **kwargs):
|
||||
return self._run(service, command)
|
||||
|
||||
@staticmethod
|
||||
def _run(service: str, command: str, user: str | None = None):
|
||||
result = CellEgressNamespaceTest.exec_service(
|
||||
service, ["sh", "-c", command], user=user
|
||||
)
|
||||
return types.SimpleNamespace(
|
||||
return_code=result.returncode, stdout=result.stdout, stderr=result.stderr
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,477 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
MODULE_PATH = Path(__file__).with_name("egress_filter.py")
|
||||
SPEC = importlib.util.spec_from_file_location("maka_eval_egress_filter", MODULE_PATH)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
class EgressFilterTest(unittest.TestCase):
|
||||
def test_blocks_contamination_surfaces_and_recursive_jina_urls(self) -> None:
|
||||
blocked = {
|
||||
"https://github.com/harbor-framework/terminal-bench-2-1": "benchmark_repository",
|
||||
"https://api.github.com/repos/terminal-benchmarks/terminal-bench/issues": "benchmark_repository",
|
||||
"https://raw.githubusercontent.com/tbench-ai/terminal-bench/main/tests/x": "benchmark_repository",
|
||||
"https://huggingface.co/datasets/acme/terminal-bench-traces": "terminal_bench_url",
|
||||
"https://github.com/hqeric/maka-eval-trajectories": "public_trajectory",
|
||||
"https://spylab.ai/reference/terminalbench-solution": "terminal_bench_url",
|
||||
"https://example.test/patches-terminalbench-task-1.diff": "known_patch_artifact",
|
||||
f"https://example.test/archive?revision={MODULE.PINNED_REVISION}": "pinned_revision",
|
||||
"https://tbench.ai/tasks": "tbench_domain",
|
||||
"https://hub.harborframework.com/tasks/terminal-bench/foo": "harbor_task_registry",
|
||||
"https://r.jina.ai/https://github.com/harbor-framework/terminal-bench-2-1": "jina_recursive:benchmark_repository",
|
||||
"https://r.jina.ai/https%253A%252F%252Fspylab.ai%252Fterminal-bench": "jina_recursive:terminal_bench_url",
|
||||
"https://example.test/search?q=TeRmInAlBeNcH": "terminal_bench_url",
|
||||
"https://google.com/search?q=terminal+bench": "terminal_bench_url",
|
||||
"https://github.com/harbor-framework/terminal%252Dbench-2-1.git": "benchmark_repository",
|
||||
"https://terminal-bench.io/tasks/answers": "terminal_bench_url",
|
||||
"https://sub.tbench.ai/x": "tbench_domain",
|
||||
# A DNS label is as good a place to name a contamination surface as
|
||||
# a path, so every rule searches both fields.
|
||||
f"https://{MODULE.PINNED_REVISION}.example.test/archive": "pinned_revision",
|
||||
"https://patches-terminalbench-task-1.example.test/x": "known_patch_artifact",
|
||||
}
|
||||
for url, rule_id in blocked.items():
|
||||
matched = MODULE.contamination_rule(url)
|
||||
self.assertIsNotNone(matched, url)
|
||||
self.assertEqual(matched[0], rule_id, url)
|
||||
|
||||
def test_preserves_unrelated_network_and_rejects_malformed_urls(self) -> None:
|
||||
allowed = [
|
||||
"https://github.com/harbor-framework/harbor",
|
||||
"https://github.com/microsoft/terminal",
|
||||
"https://huggingface.co/datasets/mteb/leaderboard",
|
||||
"https://pypi.org/simple/requests/",
|
||||
"https://deb.debian.org/debian/",
|
||||
"https://my-terminal/bench",
|
||||
]
|
||||
for url in allowed:
|
||||
self.assertIsNone(MODULE.contamination_rule(url), url)
|
||||
for url in ["", "file:///tmp/terminal-bench.log", "https://example.test/%ZZ"]:
|
||||
with self.assertRaises(ValueError, msg=url):
|
||||
MODULE.contamination_rule(url)
|
||||
|
||||
def _install_response_stub(self) -> None:
|
||||
class Response:
|
||||
@staticmethod
|
||||
def make(status, body, headers):
|
||||
return {"status": status, "body": body, "headers": headers}
|
||||
|
||||
MODULE.http = SimpleNamespace(Response=Response)
|
||||
|
||||
def test_request_blocks_a_contamination_url_and_appends_one_audit_record(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
self._install_response_stub()
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
flow = type(
|
||||
"Flow",
|
||||
(),
|
||||
{"request": type("Request", (), {"pretty_url": "https://tbench.ai/tasks"})()},
|
||||
)()
|
||||
MODULE.request(flow)
|
||||
self.assertEqual(flow.response["status"], 451)
|
||||
self.assertEqual(
|
||||
flow.response["headers"]["X-Maka-Eval-Egress-Rule"], "tbench_domain"
|
||||
)
|
||||
lines = MODULE.AUDIT_PATH.read_text().splitlines()
|
||||
self.assertEqual(len(lines), 1)
|
||||
record = json.loads(lines[0])
|
||||
self.assertEqual(record["ruleId"], "tbench_domain")
|
||||
self.assertEqual(record["host"], "tbench.ai")
|
||||
self.assertEqual(record["normalizedPath"], "/tasks")
|
||||
|
||||
def test_request_leaves_an_unrelated_url_unanswered(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
self._install_response_stub()
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
flow = type(
|
||||
"Flow",
|
||||
(),
|
||||
{"request": type("Request", (), {"pretty_url": "https://example.com/"})()},
|
||||
)()
|
||||
MODULE.request(flow)
|
||||
self.assertFalse(hasattr(flow, "response"))
|
||||
self.assertFalse(MODULE.AUDIT_PATH.exists())
|
||||
|
||||
def test_audit_is_bounded_and_policy_errors_fail_closed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
self._install_response_stub()
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
flow = type(
|
||||
"Flow",
|
||||
(),
|
||||
{"request": type("Request", (), {"pretty_url": "https://example.test/%ZZ"})()},
|
||||
)()
|
||||
MODULE.request(flow)
|
||||
self.assertEqual(flow.response["status"], 503)
|
||||
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
|
||||
self.assertEqual(record["ruleId"], "policy_error")
|
||||
self.assertIn("host", record)
|
||||
self.assertIn("normalizedPath", record)
|
||||
|
||||
def test_http_connect_refuses_blocklisted_hosts_before_the_tunnel_opens(self) -> None:
|
||||
class Response:
|
||||
@staticmethod
|
||||
def make(status, body, headers):
|
||||
return {"status": status, "body": body, "headers": headers}
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
MODULE.http = SimpleNamespace(Response=Response)
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
blocked = type(
|
||||
"Flow",
|
||||
(),
|
||||
{
|
||||
"request": type(
|
||||
"Request",
|
||||
(),
|
||||
{"pretty_host": "tbench.ai", "host": "tbench.ai", "port": 443},
|
||||
)()
|
||||
},
|
||||
)()
|
||||
MODULE.http_connect(blocked)
|
||||
self.assertEqual(blocked.response["status"], 451)
|
||||
self.assertEqual(blocked.response["headers"]["X-Maka-Eval-Egress-Rule"], "tbench_domain")
|
||||
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
|
||||
self.assertEqual(record["ruleId"], "tbench_domain")
|
||||
self.assertEqual(record["host"], "tbench.ai")
|
||||
|
||||
for host in ("example.com", "github.com", "ssh.github.com"):
|
||||
allowed = type(
|
||||
"Flow",
|
||||
(),
|
||||
{
|
||||
"request": type(
|
||||
"Request",
|
||||
(),
|
||||
{"pretty_host": host, "host": host, "port": 443},
|
||||
)(),
|
||||
"response": None,
|
||||
},
|
||||
)()
|
||||
MODULE.http_connect(allowed)
|
||||
self.assertIsNone(allowed.response, host)
|
||||
|
||||
def test_tcp_start_kills_raw_tunnels_and_records_them(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
killed: list[str] = []
|
||||
flow = type(
|
||||
"Flow",
|
||||
(),
|
||||
{"server_conn": SimpleNamespace(address=("ssh.github.com", 443))},
|
||||
)()
|
||||
flow.kill = lambda: killed.append("killed")
|
||||
MODULE.tcp_start(flow)
|
||||
self.assertEqual(killed, ["killed"])
|
||||
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
|
||||
self.assertEqual(record["ruleId"], "raw_tunnel")
|
||||
self.assertEqual(record["host"], "ssh.github.com")
|
||||
self.assertEqual(record["normalizedPath"], ":443")
|
||||
|
||||
def test_tcp_message_drops_raw_payloads(self) -> None:
|
||||
message = SimpleNamespace(content=b"SSH-2.0-test\r\n")
|
||||
killed: list[str] = []
|
||||
flow = SimpleNamespace(messages=[message], killable=True)
|
||||
flow.kill = lambda: killed.append("killed")
|
||||
MODULE.tcp_message(flow)
|
||||
self.assertEqual(message.content, b"")
|
||||
self.assertEqual(killed, ["killed"])
|
||||
|
||||
def test_next_layer_closes_raw_tcp_before_the_builtin_classifier(self) -> None:
|
||||
class CloseConnection:
|
||||
def __init__(self, connection: object) -> None:
|
||||
self.connection = connection
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
previous_commands = MODULE.proxy_commands
|
||||
self.addCleanup(setattr, MODULE, "proxy_commands", previous_commands)
|
||||
MODULE.proxy_commands = SimpleNamespace(CloseConnection=CloseConnection)
|
||||
client = object()
|
||||
server = SimpleNamespace(address=("ssh.github.com", 443))
|
||||
context = SimpleNamespace(client=client, server=server, layers=[], options=None)
|
||||
|
||||
def data_client() -> bytes:
|
||||
return b"SSH-2.0-OpenSSH_9.0"
|
||||
|
||||
nextlayer = SimpleNamespace(
|
||||
layer=None, context=context, data_client=data_client, data_server=lambda: b""
|
||||
)
|
||||
MODULE.next_layer(nextlayer)
|
||||
self.assertIsInstance(nextlayer.layer, MODULE.CloseRawLayer)
|
||||
commands = list(nextlayer.layer.handle_event(object()))
|
||||
self.assertEqual([command.connection for command in commands], [client, server])
|
||||
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
|
||||
self.assertEqual(record["ruleId"], "raw_tunnel")
|
||||
self.assertEqual(record["host"], "ssh.github.com")
|
||||
|
||||
def test_next_layer_routes_tls_and_http_without_raw_fallback(self) -> None:
|
||||
class FakeServerTLSLayer:
|
||||
child_layer: object | None = None
|
||||
|
||||
def __init__(self, context: object) -> None:
|
||||
self.context = context
|
||||
context.layers.append(self)
|
||||
|
||||
class FakeClientTLSLayer:
|
||||
def __init__(self, context: object) -> None:
|
||||
self.context = context
|
||||
context.layers.append(self)
|
||||
|
||||
previous_server_tls = MODULE.ServerTLSLayer
|
||||
previous_client_tls = MODULE.ClientTLSLayer
|
||||
self.addCleanup(setattr, MODULE, "ServerTLSLayer", previous_server_tls)
|
||||
self.addCleanup(setattr, MODULE, "ClientTLSLayer", previous_client_tls)
|
||||
MODULE.ServerTLSLayer = FakeServerTLSLayer
|
||||
MODULE.ClientTLSLayer = FakeClientTLSLayer
|
||||
|
||||
context = SimpleNamespace(layers=[])
|
||||
tls = SimpleNamespace(
|
||||
layer=None,
|
||||
context=context,
|
||||
data_client=lambda: b"\x16\x03\x01\x00\x00",
|
||||
data_server=lambda: b"",
|
||||
)
|
||||
MODULE.next_layer(tls)
|
||||
self.assertIsNone(tls.layer)
|
||||
|
||||
for first, remainder in (
|
||||
(b"\x16", b"\x03\x01\x00\x00"),
|
||||
(b"\x16\x03", b"\x01\x00\x00"),
|
||||
):
|
||||
with self.subTest(tls_prefix=first):
|
||||
fragmented_context = SimpleNamespace(layers=[])
|
||||
fragmented = SimpleNamespace(
|
||||
layer=None,
|
||||
context=fragmented_context,
|
||||
data_client=lambda first=first: first,
|
||||
data_server=lambda: b"",
|
||||
)
|
||||
MODULE.next_layer(fragmented)
|
||||
self.assertIsInstance(fragmented.layer, FakeServerTLSLayer)
|
||||
self.assertIsInstance(fragmented.layer.child_layer, FakeClientTLSLayer)
|
||||
self.assertEqual(
|
||||
fragmented_context.layers,
|
||||
[fragmented.layer, fragmented.layer.child_layer],
|
||||
)
|
||||
self.assertTrue((first + remainder).startswith(b"\x16\x03"))
|
||||
|
||||
http = SimpleNamespace(
|
||||
layer=None,
|
||||
context=context,
|
||||
data_client=lambda: b"GET / HTTP/1.1\r\n",
|
||||
data_server=lambda: b"",
|
||||
)
|
||||
MODULE.next_layer(http)
|
||||
self.assertIsNone(http.layer)
|
||||
|
||||
empty = SimpleNamespace(
|
||||
layer=None,
|
||||
context=context,
|
||||
data_client=lambda: b"",
|
||||
data_server=lambda: b"",
|
||||
)
|
||||
MODULE.next_layer(empty)
|
||||
self.assertIsNone(empty.layer)
|
||||
|
||||
for prefix in (b"G", b"GE", b"GET", b"GET ", b"GET / HTTP/1.1"):
|
||||
with self.subTest(prefix=prefix):
|
||||
incomplete = SimpleNamespace(
|
||||
layer=None,
|
||||
context=context,
|
||||
data_client=lambda prefix=prefix: prefix,
|
||||
data_server=lambda: b"",
|
||||
)
|
||||
MODULE.next_layer(incomplete)
|
||||
self.assertIsNone(incomplete.layer)
|
||||
|
||||
def test_next_layer_closes_bytes_that_cannot_become_http(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
context = SimpleNamespace(
|
||||
layers=[], options=None, client=object(), server=None
|
||||
)
|
||||
binary = SimpleNamespace(
|
||||
layer=None,
|
||||
context=context,
|
||||
data_client=lambda: b"\x00\x01\x02\x03",
|
||||
data_server=lambda: b"",
|
||||
)
|
||||
MODULE.next_layer(binary)
|
||||
self.assertIsInstance(binary.layer, MODULE.CloseRawLayer)
|
||||
|
||||
def test_response_audits_a_non_websocket_101_upgrade(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
raw = type(
|
||||
"Flow",
|
||||
(),
|
||||
{
|
||||
"response": SimpleNamespace(
|
||||
status_code=101, headers={"upgrade": "raw"}
|
||||
),
|
||||
"server_conn": SimpleNamespace(address=("origin", 19083)),
|
||||
},
|
||||
)()
|
||||
MODULE.response(raw)
|
||||
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
|
||||
self.assertEqual(record["ruleId"], "raw_tunnel")
|
||||
|
||||
websocket = type(
|
||||
"Flow",
|
||||
(),
|
||||
{
|
||||
"response": SimpleNamespace(
|
||||
status_code=101, headers={"upgrade": "websocket"}
|
||||
),
|
||||
"websocket": object(),
|
||||
"server_conn": SimpleNamespace(address=("origin", 19082)),
|
||||
},
|
||||
)()
|
||||
MODULE.response(websocket)
|
||||
self.assertEqual(len(MODULE.AUDIT_PATH.read_text().splitlines()), 1)
|
||||
|
||||
invalid = type(
|
||||
"Flow",
|
||||
(),
|
||||
{
|
||||
"response": SimpleNamespace(
|
||||
status_code=101, headers={"upgrade": "websocket"}
|
||||
),
|
||||
"server_conn": SimpleNamespace(address=("origin", 19082)),
|
||||
},
|
||||
)()
|
||||
MODULE.response(invalid)
|
||||
self.assertEqual(len(MODULE.AUDIT_PATH.read_text().splitlines()), 2)
|
||||
self.assertEqual(
|
||||
json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[1])["ruleId"],
|
||||
"raw_tunnel",
|
||||
)
|
||||
|
||||
def test_next_layer_replaces_raw_tcp_with_a_closer(self) -> None:
|
||||
class FakeTCPLayer:
|
||||
def __init__(self, context: object) -> None:
|
||||
self.context = context
|
||||
context.layers.append(self)
|
||||
|
||||
class CloseConnection:
|
||||
def __init__(self, connection: object) -> None:
|
||||
self.connection = connection
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
previous_tcp = MODULE.TCPLayer
|
||||
previous_commands = MODULE.proxy_commands
|
||||
self.addCleanup(setattr, MODULE, "TCPLayer", previous_tcp)
|
||||
self.addCleanup(setattr, MODULE, "proxy_commands", previous_commands)
|
||||
MODULE.TCPLayer = FakeTCPLayer
|
||||
MODULE.proxy_commands = SimpleNamespace(CloseConnection=CloseConnection)
|
||||
client = object()
|
||||
server = SimpleNamespace(address=("ssh.github.com", 443))
|
||||
sibling = object()
|
||||
context = SimpleNamespace(
|
||||
client=client, server=server, layers=[sibling], options=None
|
||||
)
|
||||
current = FakeTCPLayer(context)
|
||||
nextlayer = SimpleNamespace(layer=current, context=context)
|
||||
MODULE.next_layer(nextlayer)
|
||||
self.assertIsInstance(nextlayer.layer, MODULE.CloseRawLayer)
|
||||
self.assertIsInstance(nextlayer.layer, MODULE.Layer)
|
||||
self.assertEqual(context.layers, [sibling, nextlayer.layer])
|
||||
self.assertNotIn(current, context.layers)
|
||||
commands = list(nextlayer.layer.handle_event(object()))
|
||||
self.assertEqual([command.connection for command in commands], [client, server])
|
||||
self.assertTrue(all(isinstance(command, CloseConnection) for command in commands))
|
||||
record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0])
|
||||
self.assertEqual(record["ruleId"], "raw_tunnel")
|
||||
self.assertEqual(record["host"], "ssh.github.com")
|
||||
|
||||
def test_next_layer_leaves_an_unclassified_layer_alone(self) -> None:
|
||||
context = SimpleNamespace(layers=[])
|
||||
nextlayer = SimpleNamespace(layer=None, context=context)
|
||||
MODULE.next_layer(nextlayer)
|
||||
self.assertIsNone(nextlayer.layer)
|
||||
self.assertEqual(context.layers, [])
|
||||
|
||||
def test_next_layer_leaves_non_tcp_layers_alone(self) -> None:
|
||||
class HTTPLayer:
|
||||
pass
|
||||
|
||||
original = HTTPLayer()
|
||||
nextlayer = SimpleNamespace(layer=original, context=SimpleNamespace())
|
||||
MODULE.next_layer(nextlayer)
|
||||
self.assertIs(nextlayer.layer, original)
|
||||
|
||||
def test_audit_escapes_line_separators_so_python_and_typescript_agree(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
path = "/tasks/\u2028hidden"
|
||||
MODULE.append_audit("tbench_domain", "tbench.ai", path)
|
||||
raw = MODULE.AUDIT_PATH.read_text(encoding="utf-8")
|
||||
self.assertNotIn("\u2028", raw)
|
||||
self.assertIn("\\u2028", raw)
|
||||
self.assertEqual(raw.count("\n"), 1)
|
||||
self.assertEqual(len(raw.splitlines()), 1)
|
||||
record = json.loads(raw)
|
||||
self.assertEqual(record["normalizedPath"], path)
|
||||
self.assertFalse(MODULE.audit_already_truncated())
|
||||
|
||||
def test_audit_writes_one_truncation_marker_when_the_byte_limit_is_reached(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
MODULE.AUDIT_PATH.write_bytes(b"x" * MODULE.MAX_AUDIT_BYTES)
|
||||
MODULE.append_audit("tbench_domain", "tbench.ai", "/tasks")
|
||||
records = [
|
||||
json.loads(line)
|
||||
for line in MODULE.AUDIT_PATH.read_text().splitlines()
|
||||
if line.startswith("{")
|
||||
]
|
||||
self.assertEqual(records[-1]["ruleId"], "audit_truncated")
|
||||
size_after_marker = MODULE.AUDIT_PATH.stat().st_size
|
||||
MODULE.append_audit("tbench_domain", "tbench.ai", "/other")
|
||||
self.assertEqual(MODULE.AUDIT_PATH.stat().st_size, size_after_marker)
|
||||
records_after = [
|
||||
json.loads(line)
|
||||
for line in MODULE.AUDIT_PATH.read_text().splitlines()
|
||||
if line.startswith("{")
|
||||
]
|
||||
self.assertEqual(
|
||||
[record["ruleId"] for record in records_after if record["ruleId"] == "audit_truncated"],
|
||||
["audit_truncated"],
|
||||
)
|
||||
|
||||
def test_truncation_probe_ignores_non_object_json_tails(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl"
|
||||
MODULE.AUDIT_PATH.write_text('123\n"x"\n')
|
||||
self.assertFalse(MODULE.audit_already_truncated())
|
||||
MODULE.write_truncation_marker()
|
||||
self.assertTrue(MODULE.audit_already_truncated())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,606 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Prove raw tunnels die on a real mitmproxy 12.2.3 without breaking HTTPS or WebSocket.
|
||||
|
||||
Unit tests cannot see addon order: script `next_layer` runs before the built-in
|
||||
classifier assigns `TCPLayer`. This test starts the pinned proxy image and a
|
||||
local origin, then asserts what a live cell would observe.
|
||||
|
||||
It needs Docker, the pinned proxy image, and `python:3.12-slim`, so it is opt-in:
|
||||
|
||||
MAKA_EVAL_EGRESS_PROXY_TEST=1 python3 harbor/test_egress_filter_live.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
HARBOR_DIR = Path(__file__).parent
|
||||
PROXY_IMAGE = "maka-eval-egress-proxy:12.2.3"
|
||||
ORIGIN_IMAGE = "python:3.12-slim"
|
||||
COMMAND_TIMEOUT_S = 60
|
||||
CLOSE_TIMEOUT_S = 2.0
|
||||
|
||||
ORIGIN_SCRIPT = r"""
|
||||
import base64, hashlib, json, socket, threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
stats = {"raw_recv": 0, "raw_closed": 0, "upgrade_recv": 0, "upgrade_closed": 0}
|
||||
|
||||
class HttpHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"http-ok")
|
||||
def log_message(self, format, *args):
|
||||
return
|
||||
|
||||
class WsHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.headers.get("Upgrade", "").lower() != "websocket":
|
||||
self.send_error(400)
|
||||
return
|
||||
key = self.headers.get("Sec-WebSocket-Key", "")
|
||||
accept = base64.b64encode(
|
||||
hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest()
|
||||
).decode()
|
||||
self.send_response(101, "Switching Protocols")
|
||||
self.send_header("Upgrade", "websocket")
|
||||
self.send_header("Connection", "Upgrade")
|
||||
self.send_header("Sec-WebSocket-Accept", accept)
|
||||
self.end_headers()
|
||||
payload = b"ws-ok"
|
||||
self.wfile.write(bytes([0x81, len(payload)]) + payload)
|
||||
self.wfile.flush()
|
||||
def log_message(self, format, *args):
|
||||
return
|
||||
|
||||
class UpgradeHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(101, "Switching Protocols")
|
||||
self.send_header("Upgrade", "raw")
|
||||
self.send_header("Connection", "Upgrade")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"UPGRADE-BANNER\n")
|
||||
self.wfile.flush()
|
||||
self.connection.settimeout(60)
|
||||
try:
|
||||
while True:
|
||||
chunk = self.connection.recv(64)
|
||||
if not chunk:
|
||||
break
|
||||
stats["upgrade_recv"] += len(chunk)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
stats["upgrade_closed"] += 1
|
||||
def log_message(self, format, *args):
|
||||
return
|
||||
|
||||
def serve_raw():
|
||||
sock = socket.socket()
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
sock.bind(("0.0.0.0", 19081))
|
||||
sock.listen(8)
|
||||
while True:
|
||||
conn, _ = sock.accept()
|
||||
try:
|
||||
conn.sendall(b"RAW-BANNER\n")
|
||||
conn.settimeout(60)
|
||||
while True:
|
||||
chunk = conn.recv(64)
|
||||
if not chunk:
|
||||
break
|
||||
stats["raw_recv"] += len(chunk)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
stats["raw_closed"] += 1
|
||||
conn.close()
|
||||
|
||||
class StatsHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
body = json.dumps(stats).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
def log_message(self, format, *args):
|
||||
return
|
||||
|
||||
threading.Thread(target=serve_raw, daemon=True).start()
|
||||
threading.Thread(target=lambda: ThreadingHTTPServer(("0.0.0.0", 19080), HttpHandler).serve_forever(), daemon=True).start()
|
||||
threading.Thread(target=lambda: ThreadingHTTPServer(("0.0.0.0", 19082), WsHandler).serve_forever(), daemon=True).start()
|
||||
threading.Thread(target=lambda: ThreadingHTTPServer(("0.0.0.0", 19083), UpgradeHandler).serve_forever(), daemon=True).start()
|
||||
ThreadingHTTPServer(("0.0.0.0", 19084), StatsHandler).serve_forever()
|
||||
"""
|
||||
|
||||
|
||||
def docker_available() -> bool:
|
||||
if shutil.which("docker") is None:
|
||||
return False
|
||||
probe = subprocess.run(
|
||||
["docker", "version", "--format", "{{.Server.Version}}"],
|
||||
capture_output=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
return probe.returncode == 0
|
||||
|
||||
|
||||
def docker_image_present(image: str) -> bool:
|
||||
inspect = subprocess.run(
|
||||
["docker", "image", "inspect", image],
|
||||
capture_output=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
return inspect.returncode == 0
|
||||
|
||||
|
||||
def finish_memory_bio_handshake(tls, incoming, outgoing, sock, timeout_s: float) -> str:
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
tls.do_handshake()
|
||||
except ssl.SSLWantReadError:
|
||||
# Processing a server flight can produce the next client flight
|
||||
# before OpenSSL asks for more input. Flush it now; waiting for
|
||||
# another recv first deadlocks both peers.
|
||||
pending = outgoing.read()
|
||||
if pending:
|
||||
sock.sendall(pending)
|
||||
response = sock.recv(16 * 1024)
|
||||
if not response:
|
||||
raise AssertionError("proxy closed during the fragmented TLS handshake")
|
||||
incoming.write(response)
|
||||
continue
|
||||
pending = outgoing.read()
|
||||
if pending:
|
||||
sock.sendall(pending)
|
||||
return tls.version() or ""
|
||||
raise AssertionError("fragmented TLS handshake did not complete")
|
||||
|
||||
|
||||
class MemoryBioHandshakeDriverTest(unittest.TestCase):
|
||||
def test_flushes_client_flight_before_waiting_for_more_server_bytes(self) -> None:
|
||||
events = []
|
||||
|
||||
class FakeOutgoing:
|
||||
pending = b""
|
||||
|
||||
def read(self):
|
||||
pending, self.pending = self.pending, b""
|
||||
return pending
|
||||
|
||||
class FakeIncoming:
|
||||
def write(self, data):
|
||||
events.append(("write", data))
|
||||
|
||||
outgoing = FakeOutgoing()
|
||||
|
||||
class FakeTls:
|
||||
calls = 0
|
||||
|
||||
def do_handshake(self):
|
||||
self.calls += 1
|
||||
if self.calls == 1:
|
||||
outgoing.pending = b"client-finished"
|
||||
raise ssl.SSLWantReadError()
|
||||
|
||||
def version(self):
|
||||
return "TLSv1.3"
|
||||
|
||||
class FakeSocket:
|
||||
def sendall(self, data):
|
||||
events.append(("send", data))
|
||||
|
||||
def recv(self, _size):
|
||||
events.append(("recv", None))
|
||||
return b"server-finished"
|
||||
|
||||
result = finish_memory_bio_handshake(
|
||||
FakeTls(), FakeIncoming(), outgoing, FakeSocket(), timeout_s=1
|
||||
)
|
||||
|
||||
self.assertEqual(result, "TLSv1.3")
|
||||
self.assertEqual(
|
||||
events,
|
||||
[
|
||||
("send", b"client-finished"),
|
||||
("recv", None),
|
||||
("write", b"server-finished"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
os.environ.get("MAKA_EVAL_EGRESS_PROXY_TEST") == "1",
|
||||
"set MAKA_EVAL_EGRESS_PROXY_TEST=1 to run the live mitmproxy proxy test",
|
||||
)
|
||||
class LiveEgressFilterTest(unittest.TestCase):
|
||||
workdir: Path | None = None
|
||||
network: str = ""
|
||||
proxy: str = ""
|
||||
origin: str = ""
|
||||
proxy_port: int = 0
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
if not docker_available():
|
||||
raise unittest.SkipTest("Docker daemon is unavailable")
|
||||
if not docker_image_present(PROXY_IMAGE):
|
||||
raise unittest.SkipTest(f"{PROXY_IMAGE} is not present")
|
||||
if not docker_image_present(ORIGIN_IMAGE):
|
||||
raise unittest.SkipTest(f"{ORIGIN_IMAGE} is not present")
|
||||
run_id = f"{os.getpid()}-{uuid.uuid4().hex[:8]}"
|
||||
cls.network = f"maka-eval-egress-live-{run_id}-net"
|
||||
cls.proxy = f"maka-eval-egress-live-{run_id}-proxy"
|
||||
cls.origin = f"maka-eval-egress-live-{run_id}-origin"
|
||||
cls.workdir = Path(tempfile.mkdtemp(prefix="maka-eval-egress-proxy-live-"))
|
||||
(cls.workdir / "origin.py").write_text(ORIGIN_SCRIPT)
|
||||
cls.addClassCleanup(shutil.rmtree, cls.workdir, ignore_errors=True)
|
||||
cls.addClassCleanup(cls._down)
|
||||
subprocess.run(["docker", "network", "create", cls.network], check=True, timeout=COMMAND_TIMEOUT_S)
|
||||
subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.origin,
|
||||
"--network",
|
||||
cls.network,
|
||||
"--network-alias",
|
||||
"origin",
|
||||
"-v",
|
||||
f"{cls.workdir / 'origin.py'}:/origin.py:ro",
|
||||
ORIGIN_IMAGE,
|
||||
"python",
|
||||
"/origin.py",
|
||||
],
|
||||
check=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.proxy,
|
||||
"--network",
|
||||
cls.network,
|
||||
"-p",
|
||||
"127.0.0.1::8080",
|
||||
"-v",
|
||||
f"{HARBOR_DIR / 'egress_filter.py'}:/opt/maka-eval/egress_filter.py:ro",
|
||||
PROXY_IMAGE,
|
||||
],
|
||||
check=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
cls.proxy_port = cls._published_port()
|
||||
cls._wait_for_proxy()
|
||||
cls._wait_for_origin_via_proxy()
|
||||
|
||||
@classmethod
|
||||
def _published_port(cls) -> int:
|
||||
listed = subprocess.run(
|
||||
["docker", "port", cls.proxy, "8080/tcp"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
if listed.returncode != 0:
|
||||
raise AssertionError(listed.stderr)
|
||||
# "127.0.0.1:49152"
|
||||
hostport = listed.stdout.strip().splitlines()[0]
|
||||
return int(hostport.rsplit(":", 1)[1])
|
||||
|
||||
@classmethod
|
||||
def _down(cls) -> None:
|
||||
for name in (cls.proxy, cls.origin):
|
||||
if not name:
|
||||
continue
|
||||
subprocess.run(
|
||||
["docker", "rm", "-f", name],
|
||||
capture_output=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
if cls.network:
|
||||
subprocess.run(
|
||||
["docker", "network", "rm", cls.network],
|
||||
capture_output=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _wait_for_proxy(cls) -> None:
|
||||
deadline = time.time() + 30
|
||||
last_error = "proxy did not listen"
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", cls.proxy_port), 1):
|
||||
return
|
||||
except OSError as error:
|
||||
last_error = str(error)
|
||||
time.sleep(0.2)
|
||||
raise AssertionError(last_error)
|
||||
|
||||
@classmethod
|
||||
def _wait_for_origin_via_proxy(cls) -> None:
|
||||
deadline = time.time() + 30
|
||||
last_error = "origin was not reachable through the proxy"
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
body = cls.http_via_proxy(19080)
|
||||
except OSError as error:
|
||||
last_error = str(error)
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
if b"http-ok" in body:
|
||||
return
|
||||
last_error = body[:200].decode("latin1", errors="replace")
|
||||
time.sleep(0.2)
|
||||
raise AssertionError(last_error)
|
||||
|
||||
@classmethod
|
||||
def _recv_until_close(cls, sock: socket.socket, limit: int = 8192) -> bytes:
|
||||
sock.settimeout(CLOSE_TIMEOUT_S)
|
||||
data = b""
|
||||
while len(data) < limit:
|
||||
try:
|
||||
chunk = sock.recv(4096)
|
||||
except ConnectionResetError:
|
||||
return data
|
||||
if not chunk:
|
||||
return data
|
||||
data += chunk
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def http_via_proxy(
|
||||
cls,
|
||||
port: int,
|
||||
extra_headers: str = "",
|
||||
connection: str = "close",
|
||||
) -> bytes:
|
||||
request = (
|
||||
f"GET http://origin:{port}/ HTTP/1.1\r\n"
|
||||
f"Host: origin:{port}\r\n"
|
||||
f"{extra_headers}"
|
||||
f"Connection: {connection}\r\n\r\n"
|
||||
).encode()
|
||||
with socket.create_connection(("127.0.0.1", cls.proxy_port), 5) as sock:
|
||||
sock.sendall(request)
|
||||
return cls._recv_until_close(sock)
|
||||
|
||||
@classmethod
|
||||
def connect_via_proxy(cls, host: str, port: int, payload: bytes = b"CLIENT\n") -> tuple[bytes, bytes]:
|
||||
with socket.create_connection(("127.0.0.1", cls.proxy_port), 5) as sock:
|
||||
sock.sendall(f"CONNECT {host}:{port} HTTP/1.1\r\nHost: {host}:{port}\r\n\r\n".encode())
|
||||
header = b""
|
||||
sock.settimeout(CLOSE_TIMEOUT_S)
|
||||
while b"\r\n\r\n" not in header:
|
||||
chunk = sock.recv(4096)
|
||||
if not chunk:
|
||||
return header, b""
|
||||
header += chunk
|
||||
leftover = header.split(b"\r\n\r\n", 1)[1]
|
||||
header = header[: header.index(b"\r\n\r\n") + 4]
|
||||
if not payload:
|
||||
return header, leftover
|
||||
try:
|
||||
sock.sendall(payload)
|
||||
except OSError:
|
||||
return header, leftover
|
||||
return header, leftover + cls._recv_until_close(sock)
|
||||
|
||||
@classmethod
|
||||
def fragmented_tls_via_proxy(cls, first_fragment_size: int) -> str:
|
||||
incoming = ssl.MemoryBIO()
|
||||
outgoing = ssl.MemoryBIO()
|
||||
tls_context = ssl.create_default_context()
|
||||
# The proxy image creates an ephemeral private CA. This test exercises
|
||||
# protocol classification and the handshake, not CA distribution.
|
||||
tls_context.check_hostname = False
|
||||
tls_context.verify_mode = ssl.CERT_NONE
|
||||
tls = tls_context.wrap_bio(
|
||||
incoming,
|
||||
outgoing,
|
||||
server_side=False,
|
||||
server_hostname="example.com",
|
||||
)
|
||||
try:
|
||||
tls.do_handshake()
|
||||
except ssl.SSLWantReadError:
|
||||
pass
|
||||
client_hello = outgoing.read()
|
||||
if len(client_hello) <= first_fragment_size:
|
||||
raise AssertionError("TLS ClientHello was unexpectedly short")
|
||||
|
||||
with socket.create_connection(("127.0.0.1", cls.proxy_port), 5) as sock:
|
||||
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
sock.sendall(b"CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n")
|
||||
header = b""
|
||||
while b"\r\n\r\n" not in header:
|
||||
chunk = sock.recv(4096)
|
||||
if not chunk:
|
||||
raise AssertionError("proxy closed before the CONNECT response")
|
||||
header += chunk
|
||||
if b" 200 " not in header.split(b"\r\n", 1)[0]:
|
||||
raise AssertionError(header.decode("latin1", errors="replace"))
|
||||
|
||||
sock.sendall(client_hello[:first_fragment_size])
|
||||
time.sleep(0.05)
|
||||
sock.sendall(client_hello[first_fragment_size:])
|
||||
sock.settimeout(10)
|
||||
return finish_memory_bio_handshake(tls, incoming, outgoing, sock, timeout_s=20)
|
||||
|
||||
@classmethod
|
||||
def audit_records(cls) -> list[dict[str, object]]:
|
||||
listed = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.proxy,
|
||||
"sh",
|
||||
"-c",
|
||||
"if [ -f /opt/maka-egress-state/hits.jsonl ]; then cat /opt/maka-egress-state/hits.jsonl; fi",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
if listed.returncode != 0:
|
||||
raise AssertionError(listed.stderr)
|
||||
return [json.loads(line) for line in listed.stdout.splitlines() if line.strip()]
|
||||
|
||||
@classmethod
|
||||
def origin_stats(cls) -> dict[str, int]:
|
||||
# Read stats from inside the origin container so the probe does not
|
||||
# depend on the proxy remaining willing to forward that port.
|
||||
listed = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.origin,
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:19084/').read().decode())",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
if listed.returncode != 0:
|
||||
raise AssertionError(listed.stderr)
|
||||
return json.loads(listed.stdout)
|
||||
|
||||
@classmethod
|
||||
def wait_for_origin_counter(cls, key: str, minimum: int) -> dict[str, int]:
|
||||
deadline = time.time() + CLOSE_TIMEOUT_S
|
||||
last = cls.origin_stats()
|
||||
while time.time() < deadline:
|
||||
if last.get(key, 0) >= minimum:
|
||||
return last
|
||||
time.sleep(0.05)
|
||||
last = cls.origin_stats()
|
||||
raise AssertionError(f"origin counter {key} did not reach {minimum}: {last}")
|
||||
|
||||
def test_https_and_plain_http_still_forward(self) -> None:
|
||||
http = self.http_via_proxy(19080)
|
||||
self.assertIn(b"http-ok", http)
|
||||
curl = subprocess.run(
|
||||
[
|
||||
"curl",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--http1.1",
|
||||
"--max-time",
|
||||
"20",
|
||||
"--proxy",
|
||||
f"http://127.0.0.1:{self.proxy_port}",
|
||||
"--insecure",
|
||||
"--output",
|
||||
"/dev/null",
|
||||
"--write-out",
|
||||
"%{http_code}",
|
||||
"https://example.com/",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=COMMAND_TIMEOUT_S,
|
||||
)
|
||||
self.assertEqual(curl.stdout, "200", curl.stderr)
|
||||
|
||||
def test_fragmented_tls_record_prefix_still_handshakes(self) -> None:
|
||||
# Three bytes are sufficient for mitmproxy's built-in TLS classifier
|
||||
# and provide a control for the one- and two-byte fragmented prefixes.
|
||||
for first_fragment_size in (1, 2, 3):
|
||||
with self.subTest(first_fragment_size=first_fragment_size):
|
||||
self.assertTrue(
|
||||
self.fragmented_tls_via_proxy(first_fragment_size).startswith("TLS")
|
||||
)
|
||||
|
||||
def test_connect_to_a_blocklisted_host_is_451(self) -> None:
|
||||
header, _ = self.connect_via_proxy("tbench.ai", 443, b"")
|
||||
self.assertIn(b"451", header.split(b"\r\n", 1)[0])
|
||||
self.assertIn(b"tbench_domain", header)
|
||||
|
||||
def test_raw_connect_relays_no_bytes_and_is_audited(self) -> None:
|
||||
closed_before = self.origin_stats()["raw_closed"]
|
||||
header, body = self.connect_via_proxy("origin", 19081)
|
||||
self.assertIn(b"200", header.split(b"\r\n", 1)[0])
|
||||
self.assertNotIn(b"RAW-BANNER", body)
|
||||
self.assertEqual(body, b"")
|
||||
stats = self.wait_for_origin_counter("raw_closed", closed_before + 1)
|
||||
self.assertEqual(stats["raw_recv"], 0)
|
||||
self.assertIn(
|
||||
{"host": "origin", "normalizedPath": ":19081", "ruleId": "raw_tunnel"},
|
||||
[{key: record.get(key) for key in ("host", "normalizedPath", "ruleId")} for record in self.audit_records()],
|
||||
)
|
||||
|
||||
def test_http_101_raw_upgrade_is_audited_without_relaying_the_banner(self) -> None:
|
||||
closed_before = self.origin_stats()["upgrade_closed"]
|
||||
data = self.http_via_proxy(
|
||||
19083,
|
||||
extra_headers="Upgrade: raw\r\n",
|
||||
connection="Upgrade",
|
||||
)
|
||||
self.assertIn(b"101", data.split(b"\r\n", 1)[0])
|
||||
self.assertNotIn(b"UPGRADE-BANNER", data)
|
||||
stats = self.wait_for_origin_counter("upgrade_closed", closed_before + 1)
|
||||
self.assertEqual(stats["upgrade_recv"], 0)
|
||||
self.assertIn(
|
||||
{"host": "origin", "normalizedPath": ":19083", "ruleId": "raw_tunnel"},
|
||||
[{key: record.get(key) for key in ("host", "normalizedPath", "ruleId")} for record in self.audit_records()],
|
||||
)
|
||||
|
||||
def test_websocket_upgrade_still_completes(self) -> None:
|
||||
key = base64.b64encode(b"0123456789abcdef").decode()
|
||||
data = self.http_via_proxy(
|
||||
19082,
|
||||
extra_headers=(
|
||||
"Upgrade: websocket\r\n"
|
||||
f"Sec-WebSocket-Key: {key}\r\n"
|
||||
"Sec-WebSocket-Version: 13\r\n"
|
||||
),
|
||||
connection="Upgrade",
|
||||
)
|
||||
self.assertIn(b"101", data.split(b"\r\n", 1)[0])
|
||||
self.assertIn(b"ws-ok", data)
|
||||
self.assertNotIn(
|
||||
":19082",
|
||||
[str(record.get("normalizedPath")) for record in self.audit_records()],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,54 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import importlib.util
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class EvalFrameworkTest(unittest.TestCase):
|
||||
def test_selected_fails_closed_before_install(self) -> None:
|
||||
module = self._load_fresh()
|
||||
with self.assertRaisesRegex(RuntimeError, "not installed"):
|
||||
module.selected()
|
||||
|
||||
def test_invalid_framework_fails_closed(self) -> None:
|
||||
module = self._load_fresh()
|
||||
with self.assertRaisesRegex(RuntimeError, "harbor or pier"):
|
||||
module.install("other")
|
||||
with self.assertRaisesRegex(RuntimeError, "not installed"):
|
||||
module.selected()
|
||||
|
||||
def test_install_selects_harbor_and_pier(self) -> None:
|
||||
module = self._load_fresh()
|
||||
module.install("harbor")
|
||||
self.assertEqual(module.selected(), "harbor")
|
||||
module.install("pier")
|
||||
self.assertEqual(module.selected(), "pier")
|
||||
|
||||
@staticmethod
|
||||
def _load_fresh():
|
||||
path = Path(__file__).with_name("eval_framework.py")
|
||||
spec = importlib.util.spec_from_file_location("maka_eval_framework_under_test", path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,85 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
class BaseAgent:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
|
||||
def load_relay():
|
||||
from eval_framework import install
|
||||
|
||||
install("harbor")
|
||||
package = types.ModuleType("harbor")
|
||||
agents = types.ModuleType("harbor.agents")
|
||||
base = types.ModuleType("harbor.agents.base")
|
||||
base.BaseAgent = BaseAgent
|
||||
sys.modules["harbor"] = package
|
||||
sys.modules["harbor.agents"] = agents
|
||||
sys.modules["harbor.agents.base"] = base
|
||||
sys.modules.pop("relay_agent", None)
|
||||
return importlib.import_module("relay_agent")
|
||||
|
||||
|
||||
class ArtifactEnvironment:
|
||||
def __init__(self, root: Path):
|
||||
self.root = root
|
||||
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
return SimpleNamespace(return_code=0, stdout="", stderr="")
|
||||
|
||||
async def upload_file(self, source, target):
|
||||
destination = self.root / target.lstrip("/")
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, destination)
|
||||
|
||||
|
||||
class RelayArtifactTest(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_framed_transport_outputs_are_persisted_without_changing_the_carrier(self):
|
||||
relay = load_relay()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
environment = ArtifactEnvironment(root)
|
||||
result = SimpleNamespace(stdout="framed-result\n", stderr="diagnostic\n")
|
||||
|
||||
await relay._persist_subject_outputs(environment, result)
|
||||
|
||||
self.assertEqual(
|
||||
(root / relay.SUBJECT_STDOUT_PATH.lstrip("/")).read_text(),
|
||||
"framed-result\n",
|
||||
)
|
||||
self.assertEqual(
|
||||
(root / relay.SUBJECT_STDERR_PATH.lstrip("/")).read_text(),
|
||||
"diagnostic\n",
|
||||
)
|
||||
self.assertEqual(result.stdout, "framed-result\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,688 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import importlib
|
||||
import inspect
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
class BaseAgent:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
|
||||
class Environment:
|
||||
def __init__(self, stage_upload: bool = False):
|
||||
self.uploaded = b""
|
||||
self.uploaded_target = None
|
||||
self.stage_upload = stage_upload
|
||||
|
||||
async def upload_file(self, source: Path, target: str) -> None:
|
||||
self.uploaded = source.read_bytes()
|
||||
if self.stage_upload:
|
||||
self.uploaded_target = Path(target)
|
||||
self.uploaded_target.write_bytes(self.uploaded)
|
||||
|
||||
|
||||
def load_relay(framework="harbor"):
|
||||
from eval_framework import install
|
||||
|
||||
install(framework)
|
||||
package = types.ModuleType(framework)
|
||||
agents = types.ModuleType(f"{framework}.agents")
|
||||
base = types.ModuleType(f"{framework}.agents.base")
|
||||
base.BaseAgent = BaseAgent
|
||||
sys.modules[framework] = package
|
||||
sys.modules[f"{framework}.agents"] = agents
|
||||
sys.modules[f"{framework}.agents.base"] = base
|
||||
sys.modules.pop("relay_agent", None)
|
||||
return importlib.import_module("relay_agent")
|
||||
|
||||
|
||||
class RelayContractTest(unittest.TestCase):
|
||||
def test_merged_noise_cannot_corrupt_or_escape_the_result_frame(self):
|
||||
relay = load_relay()
|
||||
token = "frame-token"
|
||||
payload = b'{"kind":"settled","status":"completed"}'
|
||||
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||
frame = (
|
||||
f"MAKA-EVAL-RESULT-V1 {token} {len(payload)} "
|
||||
f"{hashlib.sha256(payload).hexdigest()} {encoded}\n"
|
||||
)
|
||||
sentinel = "credential-sentinel-must-not-persist"
|
||||
|
||||
stdout, diagnostic = relay._decode_result_carrier(
|
||||
f"docker warning\n{frame}{sentinel}\n", token
|
||||
)
|
||||
|
||||
self.assertEqual(stdout, payload.decode())
|
||||
self.assertEqual(diagnostic["category"], "unstructured-output")
|
||||
self.assertEqual(
|
||||
diagnostic["bytes"], len(f"docker warning\n{sentinel}\n".encode())
|
||||
)
|
||||
self.assertNotIn(sentinel, str({"stdout": stdout, "diagnostic": diagnostic}))
|
||||
|
||||
def test_invalid_result_frames_fail_closed_with_bounded_evidence(self):
|
||||
relay = load_relay()
|
||||
token = "frame-token"
|
||||
payload = b"{}"
|
||||
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||
valid = (
|
||||
f"MAKA-EVAL-RESULT-V1 {token} {len(payload)} "
|
||||
f"{hashlib.sha256(payload).hexdigest()} {encoded}\n"
|
||||
)
|
||||
cases = {
|
||||
"noise only": "result-frame-missing",
|
||||
valid + valid: "result-frame-ambiguous",
|
||||
valid.replace(hashlib.sha256(payload).hexdigest(), "0" * 64): "result-frame-invalid",
|
||||
}
|
||||
for carrier, category in cases.items():
|
||||
with self.subTest(category=category):
|
||||
stdout, diagnostic = relay._decode_result_carrier(carrier, token)
|
||||
self.assertEqual(stdout, "")
|
||||
self.assertEqual(diagnostic["category"], category)
|
||||
self.assertEqual(set(diagnostic), {"category", "bytes", "sha256"})
|
||||
|
||||
def test_non_utf8_result_payload_is_classified_as_an_invalid_frame(self):
|
||||
relay = load_relay()
|
||||
token = "frame-token"
|
||||
payload = b"\xff"
|
||||
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||
frame = (
|
||||
f"MAKA-EVAL-RESULT-V1 {token} {len(payload)} "
|
||||
f"{hashlib.sha256(payload).hexdigest()} {encoded}\n"
|
||||
)
|
||||
|
||||
stdout, diagnostic = relay._decode_result_carrier(frame, token)
|
||||
|
||||
self.assertEqual(stdout, "")
|
||||
self.assertEqual(diagnostic["category"], "result-frame-invalid")
|
||||
|
||||
def test_oversized_result_carrier_is_rejected_before_parsing(self):
|
||||
relay = load_relay()
|
||||
carrier = "x" * (relay.RESULT_CARRIER_LIMIT_BYTES + 1)
|
||||
stdout, diagnostic = relay._decode_result_carrier(carrier, "frame-token")
|
||||
self.assertEqual(stdout, "")
|
||||
self.assertEqual(diagnostic["category"], "result-frame-oversize")
|
||||
|
||||
def test_subject_output_cannot_counterfeit_scope_setup_failure(self):
|
||||
relay = load_relay()
|
||||
token = "0" * 32
|
||||
carrier = f"subject output\nMAKA-EVAL-SCOPE-ERROR-V1 {token}\n"
|
||||
|
||||
_, diagnostic = relay._project_result(
|
||||
types.SimpleNamespace(stdout=carrier),
|
||||
{"captureStdout": False, "resultToken": token},
|
||||
)
|
||||
|
||||
self.assertEqual(diagnostic["category"], "none")
|
||||
|
||||
def test_command_keeps_capture_and_control_files_out_of_task_workspace(self):
|
||||
relay = load_relay()
|
||||
environment = Environment()
|
||||
command = asyncio.run(
|
||||
relay._prepare_command(
|
||||
environment,
|
||||
{
|
||||
"command": "/opt/agent",
|
||||
"args": ["run"],
|
||||
"environment": {},
|
||||
"credentials": {},
|
||||
"captureStdout": True,
|
||||
"resultToken": "0" * 32,
|
||||
},
|
||||
"token",
|
||||
"/logs/agent/.maka-eval-token.pid",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertNotIn(".stdout", command)
|
||||
self.assertNotIn(".stderr", command)
|
||||
self.assertNotIn("/app", command)
|
||||
self.assertIn("/logs/agent/.maka-eval-token.pid", command)
|
||||
self.assertIn("2>/dev/null", command)
|
||||
|
||||
def test_command_does_not_start_subject_when_scope_pid_cannot_be_published(self):
|
||||
relay = load_relay()
|
||||
environment = Environment(stage_upload=True)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
fake_bin = root / "bin"
|
||||
fake_bin.mkdir()
|
||||
fake_setsid = fake_bin / "setsid"
|
||||
fake_setsid.write_text(
|
||||
"#!/bin/sh\n"
|
||||
"if [ \"$1\" = --wait ]; then shift; fi\n"
|
||||
"exec \"$@\"\n"
|
||||
)
|
||||
fake_setsid.chmod(0o755)
|
||||
marker = root / "subject-started"
|
||||
scope_path = root / "missing" / "scope.pid"
|
||||
command = asyncio.run(
|
||||
relay._prepare_command(
|
||||
environment,
|
||||
{
|
||||
"command": "/bin/sh",
|
||||
"args": ["-c", f"touch {shlex.quote(str(marker))}"],
|
||||
"environment": {},
|
||||
"credentials": {},
|
||||
"captureStdout": False,
|
||||
"resultToken": "0" * 32,
|
||||
},
|
||||
"token",
|
||||
str(scope_path),
|
||||
)
|
||||
)
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
shell=True,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
env={**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"},
|
||||
)
|
||||
self.assertFalse(marker.exists())
|
||||
finally:
|
||||
if environment.uploaded_target is not None:
|
||||
environment.uploaded_target.unlink(missing_ok=True)
|
||||
|
||||
self.assertNotEqual(completed.returncode, 0)
|
||||
stdout, diagnostic = relay._project_result(
|
||||
types.SimpleNamespace(stdout=completed.stdout.decode()),
|
||||
{"captureStdout": False, "resultToken": "0" * 32},
|
||||
)
|
||||
self.assertEqual(stdout, "")
|
||||
self.assertEqual(diagnostic["category"], "execution-scope-unavailable")
|
||||
|
||||
def test_stages_environment_and_discards_unstructured_stdout(self):
|
||||
relay = load_relay()
|
||||
environment = Environment()
|
||||
command = asyncio.run(
|
||||
relay._prepare_command(
|
||||
environment,
|
||||
{
|
||||
"command": "/opt/agent",
|
||||
"args": ["run"],
|
||||
"environment": {"MODE": "offline"},
|
||||
"credentials": {"API_KEY": "canary-secret"},
|
||||
"captureStdout": False,
|
||||
"resultToken": "0" * 32,
|
||||
},
|
||||
"token",
|
||||
"/tmp/scope.pid",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertIn(b"export MODE=offline", environment.uploaded)
|
||||
self.assertIn(b"export API_KEY=canary-secret", environment.uploaded)
|
||||
self.assertNotIn("canary-secret", command)
|
||||
self.assertIn(">/dev/null", command)
|
||||
|
||||
def test_leaves_no_credential_file_when_command_preparation_fails(self):
|
||||
relay = load_relay()
|
||||
environment = Environment()
|
||||
named_temporary_file = tempfile.NamedTemporaryFile
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
with patch.object(
|
||||
relay.tempfile,
|
||||
"NamedTemporaryFile",
|
||||
side_effect=lambda *args, **kwargs: named_temporary_file(
|
||||
*args, dir=directory, **kwargs
|
||||
),
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "invalid Maka Eval credential name"):
|
||||
asyncio.run(
|
||||
relay._prepare_command(
|
||||
environment,
|
||||
{
|
||||
"command": "/opt/agent",
|
||||
"args": ["run"],
|
||||
"environment": {"MODE": "offline"},
|
||||
"credentials": {
|
||||
"API_KEY": "canary-secret",
|
||||
"BAD-NAME": "ignored",
|
||||
},
|
||||
"captureStdout": False,
|
||||
"resultToken": "0" * 32,
|
||||
},
|
||||
"token",
|
||||
"/tmp/scope.pid",
|
||||
)
|
||||
)
|
||||
self.assertEqual(list(Path(directory).iterdir()), [])
|
||||
|
||||
def test_removes_credential_file_when_subject_overrides_path(self):
|
||||
relay = load_relay()
|
||||
environment = Environment(stage_upload=True)
|
||||
token = f"contract-{uuid.uuid4().hex}"
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
command = asyncio.run(
|
||||
relay._prepare_command(
|
||||
environment,
|
||||
{
|
||||
"command": "/bin/sh",
|
||||
"args": ["-c", "exit 0"],
|
||||
"environment": {"PATH": "/definitely-missing"},
|
||||
"credentials": {"API_KEY": "canary-secret"},
|
||||
"captureStdout": True,
|
||||
"resultToken": "0" * 32,
|
||||
},
|
||||
token,
|
||||
str(Path(directory) / "scope.pid"),
|
||||
)
|
||||
)
|
||||
target = environment.uploaded_target
|
||||
self.assertIsNotNone(target)
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
["/bin/sh", "-c", shlex.split(command)[-1]],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(completed.returncode, 0, completed.stderr)
|
||||
self.assertFalse(target.exists())
|
||||
finally:
|
||||
target.unlink(missing_ok=True)
|
||||
|
||||
|
||||
# Every default Docker capability except the two that bypass the policy.
|
||||
CONSTRAINED_CAPABILITIES = 0xA80425FB & ~(1 << 13) & ~(1 << 12)
|
||||
# The kernel's form for a namespace link target. Two distinct ones, because what
|
||||
# the check compares is identity: the same string means the same namespace.
|
||||
POLICY_NAMESPACE = "net:[4026532001]"
|
||||
OWN_NAMESPACE = "net:[4026532099]"
|
||||
|
||||
|
||||
class SubjectEnvironment:
|
||||
"""Reports the evidence the probe reads: every `Cap` set, and both namespaces."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
permitted: int = CONSTRAINED_CAPABILITIES,
|
||||
effective: int | None = None,
|
||||
bounding: int | None = None,
|
||||
namespace: str = POLICY_NAMESPACE,
|
||||
default_user: str | None = None,
|
||||
) -> None:
|
||||
self.sets = {
|
||||
"CapInh": 0,
|
||||
"CapPrm": permitted,
|
||||
"CapEff": permitted if effective is None else effective,
|
||||
"CapBnd": permitted if bounding is None else bounding,
|
||||
"CapAmb": 0,
|
||||
}
|
||||
self.namespace = namespace
|
||||
self.default_user = default_user
|
||||
self.commands: list[str] = []
|
||||
self.users: list[str | None] = []
|
||||
self.services: list[str] = []
|
||||
|
||||
async def exec(self, command: str, cwd=None, env=None, timeout_sec=None, user=None):
|
||||
# Harbor 0.20.0 BaseEnvironment.exec takes user= as a named argument,
|
||||
# not **kwargs. Absorbing unknown keywords here would hide a TypeError
|
||||
# against the real Docker environment.
|
||||
self.commands.append(command)
|
||||
explicit = str(user) if user is not None else None
|
||||
self.users.append(explicit if explicit is not None else self.default_user)
|
||||
if "proxy-ipv4" in command:
|
||||
return types.SimpleNamespace(
|
||||
return_code=0,
|
||||
stdout="MAKA-EVAL-PROXY-HOST-V1 172.18.0.2 maka-eval-mitmproxy\n",
|
||||
stderr="",
|
||||
)
|
||||
reported = " ".join(f"{name}={value:016x}" for name, value in self.sets.items())
|
||||
return types.SimpleNamespace(
|
||||
return_code=0,
|
||||
stdout=(
|
||||
f"MAKA-EVAL-CAPABILITIES-V1 {reported}\n"
|
||||
f"MAKA-EVAL-POLICY-NAMESPACE-V1 {self.namespace}\n"
|
||||
),
|
||||
stderr="",
|
||||
)
|
||||
|
||||
async def service_exec(self, command: str, *, service: str, **kwargs):
|
||||
self.services.append(service)
|
||||
return types.SimpleNamespace(
|
||||
return_code=0,
|
||||
stdout=f"MAKA-EVAL-POLICY-NAMESPACE-V1 {POLICY_NAMESPACE}\n",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
|
||||
def _echoing_service_exec(answer: str):
|
||||
async def service_exec(command: str, *, service: str, **kwargs):
|
||||
return types.SimpleNamespace(
|
||||
return_code=0,
|
||||
stdout=f"MAKA-EVAL-POLICY-NAMESPACE-V1 {answer}\n",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
return service_exec
|
||||
|
||||
|
||||
class SubjectCapabilityTest(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_subject_holding_a_bypass_capability_never_starts(self):
|
||||
relay = load_relay()
|
||||
# Named here rather than read from the module: sourcing the expectation
|
||||
# from the code under test would let deleting a capability delete its
|
||||
# own coverage. NET_RAW grants AF_PACKET, NET_ADMIN grants `nft flush`.
|
||||
for name, bit in (("NET_RAW", 1 << 13), ("NET_ADMIN", 1 << 12)):
|
||||
with self.subTest(name):
|
||||
environment = SubjectEnvironment(permitted=bit | (1 << 21))
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
with self.assertRaises(RuntimeError) as raised:
|
||||
await relay._require_constrained_subject(environment)
|
||||
self.assertIn(name, str(raised.exception))
|
||||
|
||||
async def test_capability_reachable_only_through_the_bounding_set_never_starts(self):
|
||||
relay = load_relay()
|
||||
# A non-root subject reports an empty effective set while a
|
||||
# `cap_net_raw+ep` executable still reacquires whatever the bounding set
|
||||
# kept. Verified in a container: with the capability bounded away the
|
||||
# same executable cannot be run at all.
|
||||
environment = SubjectEnvironment(
|
||||
permitted=0,
|
||||
effective=0,
|
||||
bounding=CONSTRAINED_CAPABILITIES | (1 << 13),
|
||||
)
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
with self.assertRaises(RuntimeError) as raised:
|
||||
await relay._require_constrained_subject(environment)
|
||||
self.assertIn("NET_RAW", str(raised.exception))
|
||||
|
||||
async def test_subject_outside_the_policy_namespace_never_starts(self):
|
||||
relay = load_relay()
|
||||
# Harbor applies the policy inside the sidecar and respects a task's own
|
||||
# networking on `main`, so a task that declares it leaves the subject in
|
||||
# a namespace no policy was ever applied to.
|
||||
environment = SubjectEnvironment(namespace=OWN_NAMESPACE)
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
with self.assertRaises(RuntimeError) as raised:
|
||||
await relay._require_constrained_subject(environment)
|
||||
self.assertIn("network namespace", str(raised.exception))
|
||||
|
||||
async def test_policy_namespace_is_read_from_the_service_that_applies_the_policy(self):
|
||||
relay = load_relay()
|
||||
# The comparison is only meaningful against the namespace the policy was
|
||||
# installed in, and Harbor installs it by running `network-policy` in the
|
||||
# egress sidecar. Reading the other side from anywhere else would compare
|
||||
# the subject against a namespace nothing was applied to.
|
||||
environment = SubjectEnvironment()
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
await relay._require_constrained_subject(environment)
|
||||
self.assertEqual(environment.services, ["harbor-docker-egress-control-sidecar"])
|
||||
|
||||
async def test_constrained_subject_proceeds(self):
|
||||
relay = load_relay()
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
await relay._require_constrained_subject(SubjectEnvironment())
|
||||
|
||||
async def test_unreadable_evidence_fails_closed(self):
|
||||
relay = load_relay()
|
||||
|
||||
class SilentEnvironment:
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
return types.SimpleNamespace(return_code=0, stdout="", stderr="")
|
||||
|
||||
async def service_exec(self, command, *, service, **kwargs):
|
||||
return types.SimpleNamespace(return_code=0, stdout="", stderr="")
|
||||
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
with self.assertRaises(RuntimeError):
|
||||
await relay._require_constrained_subject(SilentEnvironment())
|
||||
|
||||
async def test_answer_that_is_not_a_namespace_identity_fails_closed(self):
|
||||
relay = load_relay()
|
||||
# Two sides that both failed to answer must not compare equal. Anything
|
||||
# that is not the kernel's own form is not an identity to compare.
|
||||
for answer in ("", "shared", "net:[]"):
|
||||
with self.subTest(answer):
|
||||
environment = SubjectEnvironment(namespace=answer)
|
||||
environment.service_exec = _echoing_service_exec(answer)
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
with self.assertRaises(RuntimeError):
|
||||
await relay._require_constrained_subject(environment)
|
||||
|
||||
async def test_check_is_scoped_to_runs_that_require_the_proxy(self):
|
||||
relay = load_relay()
|
||||
environment = SubjectEnvironment(permitted=1 << 13, namespace=OWN_NAMESPACE)
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": ""}):
|
||||
await relay._require_constrained_subject(environment)
|
||||
self.assertEqual(environment.commands, [])
|
||||
self.assertEqual(environment.services, [])
|
||||
|
||||
async def test_missing_published_proxy_address_fails_closed(self):
|
||||
relay = load_relay()
|
||||
|
||||
class MissingAddressEnvironment(SubjectEnvironment):
|
||||
async def exec(self, command, cwd=None, env=None, timeout_sec=None, user=None):
|
||||
if "proxy-ipv4" in command:
|
||||
self.commands.append(command)
|
||||
explicit = str(user) if user is not None else None
|
||||
self.users.append(
|
||||
explicit if explicit is not None else self.default_user
|
||||
)
|
||||
return types.SimpleNamespace(return_code=1, stdout="", stderr="")
|
||||
return await super().exec(
|
||||
command, cwd=cwd, env=env, timeout_sec=timeout_sec, user=user
|
||||
)
|
||||
|
||||
environment = MissingAddressEnvironment()
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
with self.assertRaises(RuntimeError) as raised:
|
||||
await relay._require_constrained_subject(environment)
|
||||
self.assertIn("pin the Eval egress proxy hostname", str(raised.exception))
|
||||
pin_users = [
|
||||
user
|
||||
for command, user in zip(environment.commands, environment.users, strict=True)
|
||||
if "proxy-ipv4" in command
|
||||
]
|
||||
self.assertEqual(pin_users, ["root"])
|
||||
|
||||
async def test_proxy_hostname_pin_runs_as_root(self):
|
||||
relay = load_relay()
|
||||
environment = SubjectEnvironment()
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
await relay._require_constrained_subject(environment)
|
||||
pin_users = [
|
||||
user
|
||||
for command, user in zip(environment.commands, environment.users, strict=True)
|
||||
if "proxy-ipv4" in command
|
||||
]
|
||||
self.assertEqual(pin_users, ["root"])
|
||||
|
||||
async def test_proxy_hostname_pin_does_not_inherit_a_non_root_default_user(self):
|
||||
# Harbor scopes every exec to the task's agent.user unless the call
|
||||
# names another identity. Pinning /etc/hosts is infrastructure, so it
|
||||
# must still be root when that default is a supported non-root user.
|
||||
relay = load_relay()
|
||||
environment = SubjectEnvironment(default_user="1000")
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
await relay._require_constrained_subject(environment)
|
||||
probe_users = [
|
||||
user
|
||||
for command, user in zip(environment.commands, environment.users, strict=True)
|
||||
if "proxy-ipv4" not in command
|
||||
]
|
||||
pin_users = [
|
||||
user
|
||||
for command, user in zip(environment.commands, environment.users, strict=True)
|
||||
if "proxy-ipv4" in command
|
||||
]
|
||||
self.assertEqual(probe_users, ["1000"])
|
||||
self.assertEqual(pin_users, ["root"])
|
||||
|
||||
async def test_invalid_proxy_hostname_is_rejected_before_hosts_rewrite(self):
|
||||
relay = load_relay()
|
||||
for host in (
|
||||
".*",
|
||||
"proxy*",
|
||||
"evil.com\n127.0.0.1 pwned",
|
||||
"foo bar",
|
||||
"foo..bar",
|
||||
".hidden",
|
||||
"maka-évál",
|
||||
):
|
||||
with self.subTest(host=host):
|
||||
environment = SubjectEnvironment()
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_ALLOWED_HOST": host}):
|
||||
with self.assertRaisesRegex(RuntimeError, "proxy host is invalid"):
|
||||
await relay._pin_proxy_hostname(environment)
|
||||
self.assertEqual(environment.commands, [])
|
||||
|
||||
async def test_proxy_hostname_pin_matches_hosts_aliases_exactly(self):
|
||||
relay = load_relay()
|
||||
environment = SubjectEnvironment()
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
await relay._require_constrained_subject(environment)
|
||||
pin = next(command for command in environment.commands if "proxy-ipv4" in command)
|
||||
self.assertIn(relay.HOSTS_ALIAS_AWK, pin)
|
||||
|
||||
completed = subprocess.run(
|
||||
["awk", "-v", "host=maka-eval-mitmproxy", relay.HOSTS_ALIAS_AWK],
|
||||
input=(
|
||||
"127.0.0.1 localhost\n"
|
||||
"10.0.0.1 maka-eval-mitmproxy extra\n"
|
||||
"10.0.0.2 prefix-maka-eval-mitmproxy\n"
|
||||
"10.0.0.3 other.example\n"
|
||||
"10.0.0.4 other.example maka-eval-mitmproxy\n"
|
||||
),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(
|
||||
completed.stdout,
|
||||
"127.0.0.1 localhost\n"
|
||||
"10.0.0.2 prefix-maka-eval-mitmproxy\n"
|
||||
"10.0.0.3 other.example\n",
|
||||
)
|
||||
|
||||
def test_harbor_0_20_0_exec_declares_user_as_a_named_parameter(self):
|
||||
# Harbor 0.20.0 BaseEnvironment.exec / DockerEnvironment.exec:
|
||||
# (command, cwd=None, env=None, timeout_sec=None, user=None)
|
||||
# A **kwargs-only double would absorb user= and hide a TypeError
|
||||
# against that signature.
|
||||
parameter = inspect.signature(SubjectEnvironment.exec).parameters["user"]
|
||||
self.assertNotEqual(parameter.kind, inspect.Parameter.VAR_KEYWORD)
|
||||
inspect.signature(SubjectEnvironment.exec).bind(
|
||||
None, "true", user="root"
|
||||
)
|
||||
|
||||
def test_published_ipv4_requires_four_0_255_octets(self):
|
||||
relay = load_relay()
|
||||
accepted = ("172.18.0.2", "0.1.2.3", "255.255.255.255", "10.0.0.1")
|
||||
rejected = (
|
||||
"999.1.1.1",
|
||||
"1.2.3",
|
||||
"1..2.3",
|
||||
"1.2.3.4.5",
|
||||
"01.2.3.4",
|
||||
"1.2.3.08",
|
||||
"...",
|
||||
"",
|
||||
"172.18.0.2/32",
|
||||
"localhost",
|
||||
)
|
||||
for ip in accepted:
|
||||
with self.subTest(ip=ip):
|
||||
self.assertTrue(relay._valid_published_ipv4(ip))
|
||||
for ip in rejected:
|
||||
with self.subTest(ip=ip):
|
||||
self.assertFalse(relay._valid_published_ipv4(ip))
|
||||
|
||||
for ip in (*accepted, *rejected):
|
||||
with self.subTest(awk=ip):
|
||||
completed = subprocess.run(
|
||||
["awk", relay.IPV4_OCTET_AWK],
|
||||
input=f"{ip}\n",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(
|
||||
completed.returncode == 0,
|
||||
relay._valid_published_ipv4(ip),
|
||||
ip,
|
||||
)
|
||||
|
||||
def test_network_policy_uses_the_same_ipv4_octet_check(self):
|
||||
relay = load_relay()
|
||||
policy = Path(__file__).with_name("egress-proxy").joinpath("network-policy")
|
||||
self.assertIn(relay.IPV4_OCTET_AWK.strip(), policy.read_text())
|
||||
|
||||
def test_network_policy_rejects_docker_dns_before_local_accept(self):
|
||||
policy = Path(__file__).with_name("egress-proxy").joinpath("network-policy").read_text()
|
||||
reject = policy.index("ip daddr 127.0.0.11 reject")
|
||||
local = policy.index("fib daddr type local accept")
|
||||
self.assertLess(reject, local)
|
||||
|
||||
async def test_malformed_published_proxy_address_fails_closed(self):
|
||||
relay = load_relay()
|
||||
|
||||
class BadAddressEnvironment(SubjectEnvironment):
|
||||
async def exec(self, command, cwd=None, env=None, timeout_sec=None, user=None):
|
||||
if "proxy-ipv4" in command:
|
||||
self.commands.append(command)
|
||||
explicit = str(user) if user is not None else None
|
||||
self.users.append(
|
||||
explicit if explicit is not None else self.default_user
|
||||
)
|
||||
return types.SimpleNamespace(
|
||||
return_code=0,
|
||||
stdout="MAKA-EVAL-PROXY-HOST-V1 999.1.1.1 maka-eval-mitmproxy\n",
|
||||
stderr="",
|
||||
)
|
||||
return await super().exec(
|
||||
command, cwd=cwd, env=env, timeout_sec=timeout_sec, user=user
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}):
|
||||
with self.assertRaises(RuntimeError) as raised:
|
||||
await relay._require_constrained_subject(BadAddressEnvironment())
|
||||
self.assertIn("pin the Eval egress proxy hostname", str(raised.exception))
|
||||
|
||||
|
||||
class FrameworkSelectionTest(unittest.TestCase):
|
||||
def test_relay_loads_harbor_and_pier_from_the_installed_selection(self):
|
||||
for framework in ("harbor", "pier"):
|
||||
with self.subTest(framework):
|
||||
relay = load_relay(framework)
|
||||
self.assertEqual(relay.framework, framework)
|
||||
|
||||
def test_relay_does_not_select_a_framework_from_the_environment(self):
|
||||
os.environ["MAKA_EVAL_FRAMEWORK"] = "pier"
|
||||
relay = load_relay("harbor")
|
||||
self.assertEqual(relay.framework, "harbor")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,832 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
class BaseAgent:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
|
||||
class LocalEnvironment:
|
||||
async def upload_file(self, source, target):
|
||||
if target.startswith("/logs/"):
|
||||
return
|
||||
shutil.copyfile(source, target)
|
||||
|
||||
async def download_file(self, source, target):
|
||||
shutil.copyfile(source, target)
|
||||
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
if command.startswith("mkdir -p /logs/artifacts"):
|
||||
return SimpleNamespace(return_code=0, stdout="", stderr="")
|
||||
completed = await asyncio.to_thread(
|
||||
subprocess.run,
|
||||
command,
|
||||
cwd=cwd,
|
||||
shell=True,
|
||||
executable="/bin/bash",
|
||||
check=False,
|
||||
timeout=timeout_sec,
|
||||
)
|
||||
return SimpleNamespace(return_code=completed.returncode)
|
||||
|
||||
|
||||
class SimultaneousEnvironment:
|
||||
def __init__(self):
|
||||
self.release = asyncio.Event()
|
||||
self.started = asyncio.Event()
|
||||
self.finished = asyncio.Event()
|
||||
|
||||
async def upload_file(self, source, target):
|
||||
if target.startswith("/logs/"):
|
||||
return
|
||||
shutil.copyfile(source, target)
|
||||
|
||||
async def download_file(self, source, target):
|
||||
shutil.copyfile(source, target)
|
||||
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
if command.startswith("printf ") and "MAKA-EVAL-CWD-V1" in command:
|
||||
prefix = command.split("'", 2)[1]
|
||||
return SimpleNamespace(
|
||||
return_code=0, stdout=f"docker warning\n{prefix}/workspace\n", stderr=None
|
||||
)
|
||||
if command.startswith("setsid"):
|
||||
self.started.set()
|
||||
await self.release.wait()
|
||||
payload = b'{"kind":"settled"}'
|
||||
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||
frame = (
|
||||
f"MAKA-EVAL-RESULT-V1 {'0' * 32} {len(payload)} "
|
||||
f"{hashlib.sha256(payload).hexdigest()} {encoded}\n"
|
||||
)
|
||||
self.finished.set()
|
||||
return SimpleNamespace(return_code=0, stdout=frame, stderr=None)
|
||||
if command.startswith("test -r") or command.startswith("pgid="):
|
||||
return SimpleNamespace(return_code=3, stdout="", stderr="")
|
||||
return SimpleNamespace(return_code=0, stdout="", stderr="")
|
||||
|
||||
|
||||
class ClosedWriter:
|
||||
def is_closing(self):
|
||||
return False
|
||||
|
||||
def write(self, _value):
|
||||
raise BrokenPipeError
|
||||
|
||||
async def drain(self):
|
||||
raise AssertionError("drain must not run after a failed write")
|
||||
|
||||
|
||||
class HangingStopEnvironment:
|
||||
async def stop(self, delete=False):
|
||||
await asyncio.Future()
|
||||
|
||||
|
||||
class ScopeEvidenceEnvironment:
|
||||
def __init__(self, return_code):
|
||||
self.return_code = return_code
|
||||
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
return SimpleNamespace(return_code=self.return_code, stdout="", stderr="")
|
||||
|
||||
|
||||
class ScopeSetupFailureEnvironment:
|
||||
async def upload_file(self, source, target):
|
||||
return None
|
||||
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
if command.startswith("printf ") and "MAKA-EVAL-CWD-V1" in command:
|
||||
prefix = command.split("'", 2)[1]
|
||||
return SimpleNamespace(return_code=0, stdout=f"{prefix}/workspace\n", stderr=None)
|
||||
if command.startswith("setsid"):
|
||||
token = "0" * 32
|
||||
return SimpleNamespace(
|
||||
return_code=111,
|
||||
stdout=f"MAKA-EVAL-SCOPE-ERROR-V1 {token}\n",
|
||||
stderr=None,
|
||||
)
|
||||
if command.startswith("rm -f --"):
|
||||
return SimpleNamespace(return_code=0, stdout="", stderr="")
|
||||
if command.startswith("mkdir -p /logs/artifacts"):
|
||||
return SimpleNamespace(return_code=0, stdout="", stderr="")
|
||||
raise AssertionError(f"unexpected command after scope setup failure: {command}")
|
||||
|
||||
|
||||
class FrameworkTimeoutEnvironment(SimultaneousEnvironment):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.stopped = False
|
||||
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
if command.startswith("setsid"):
|
||||
self.started.set()
|
||||
await asyncio.Future()
|
||||
return await super().exec(command, cwd=cwd, timeout_sec=timeout_sec)
|
||||
|
||||
async def stop(self, delete=False):
|
||||
self.stopped = delete
|
||||
|
||||
|
||||
class LiveScopeEnvironment(SimultaneousEnvironment):
|
||||
"""A subject whose process group outlives it, as a task's own service does."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.signalled = False
|
||||
self.commands = []
|
||||
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
self.commands.append(command)
|
||||
if _is_teardown(command):
|
||||
self.signalled = True
|
||||
return SimpleNamespace(return_code=0, stdout="", stderr="")
|
||||
if command.startswith("pgid="):
|
||||
return SimpleNamespace(return_code=3 if self.signalled else 0, stdout="", stderr="")
|
||||
return await super().exec(command, cwd=cwd, timeout_sec=timeout_sec)
|
||||
|
||||
|
||||
class TransportLossEnvironment(SimultaneousEnvironment):
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
if "kill -TERM" in command:
|
||||
self.release.set()
|
||||
return SimpleNamespace(return_code=0, stdout="", stderr=None)
|
||||
return await super().exec(command, cwd=cwd, timeout_sec=timeout_sec)
|
||||
|
||||
|
||||
class SettleCompletionEnvironment(SimultaneousEnvironment):
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
if "kill -TERM" in command:
|
||||
self.release.set()
|
||||
return SimpleNamespace(return_code=0, stdout="", stderr=None)
|
||||
return await super().exec(command, cwd=cwd, timeout_sec=timeout_sec)
|
||||
|
||||
|
||||
def _is_teardown(command: str) -> bool:
|
||||
# `kill -0` is how the relay asks whether the scope is still there; only
|
||||
# TERM and KILL end it.
|
||||
return "kill -TERM" in command or "kill -KILL" in command
|
||||
|
||||
|
||||
class RecordingEnvironment:
|
||||
"""Runs a subject that exits with a chosen code and records every command."""
|
||||
|
||||
def __init__(self, return_code, status):
|
||||
self.return_code = return_code
|
||||
self.status = status
|
||||
self.signalled = False
|
||||
self.commands = []
|
||||
|
||||
async def upload_file(self, source, target):
|
||||
return None
|
||||
|
||||
async def download_file(self, source, target):
|
||||
shutil.copyfile(source, target)
|
||||
|
||||
async def exec(self, command, cwd=None, timeout_sec=None):
|
||||
self.commands.append(command)
|
||||
if command.startswith("printf ") and "MAKA-EVAL-CWD-V1" in command:
|
||||
prefix = command.split("'", 2)[1]
|
||||
return SimpleNamespace(return_code=0, stdout=f"{prefix}/workspace\n", stderr=None)
|
||||
if command.startswith("setsid"):
|
||||
payload = ('{"kind":"settled","status":"%s"}' % self.status).encode()
|
||||
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||
frame = (
|
||||
f"MAKA-EVAL-RESULT-V1 {'0' * 32} {len(payload)} "
|
||||
f"{hashlib.sha256(payload).hexdigest()} {encoded}\n"
|
||||
)
|
||||
return SimpleNamespace(return_code=self.return_code, stdout=frame, stderr=None)
|
||||
# The subject's process group outlives it, as a task's own service does,
|
||||
# and dies once something signals it. A teardown would therefore succeed
|
||||
# here rather than fail for its own reasons -- what this test asserts is
|
||||
# that no teardown is attempted at all.
|
||||
if _is_teardown(command):
|
||||
self.signalled = True
|
||||
return SimpleNamespace(return_code=0, stdout="", stderr="")
|
||||
if command.startswith("pgid="):
|
||||
return SimpleNamespace(return_code=3 if self.signalled else 0, stdout="", stderr="")
|
||||
return SimpleNamespace(return_code=0, stdout="", stderr="")
|
||||
|
||||
|
||||
def load_relay():
|
||||
from eval_framework import install
|
||||
|
||||
install("harbor")
|
||||
package = types.ModuleType("harbor")
|
||||
agents = types.ModuleType("harbor.agents")
|
||||
base = types.ModuleType("harbor.agents.base")
|
||||
base.BaseAgent = BaseAgent
|
||||
sys.modules["harbor"] = package
|
||||
sys.modules["harbor.agents"] = agents
|
||||
sys.modules["harbor.agents.base"] = base
|
||||
sys.modules.pop("relay_agent", None)
|
||||
return importlib.import_module("relay_agent")
|
||||
|
||||
|
||||
def run_host_teardown_probe(marker: Path, fail_cleanup: bool) -> subprocess.CompletedProcess:
|
||||
cleanup_failure = 'raise RuntimeError("cleanup failed")' if fail_cleanup else ""
|
||||
script = f"""
|
||||
import asyncio
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import run_trial as module
|
||||
|
||||
relay = types.ModuleType("relay_agent")
|
||||
relay.request_host_teardown = lambda: None
|
||||
sys.modules["relay_agent"] = relay
|
||||
|
||||
async def trial(*_args):
|
||||
try:
|
||||
asyncio.get_running_loop().call_soon(os.kill, os.getpid(), signal.SIGTERM)
|
||||
await asyncio.Future()
|
||||
finally:
|
||||
Path({str(marker)!r}).write_text("attempted")
|
||||
{cleanup_failure}
|
||||
|
||||
module.run_trial = trial
|
||||
sys.argv = ["run_trial.py", "harbor", "test", "/tmp/test-config"]
|
||||
asyncio.run(module.main())
|
||||
"""
|
||||
return subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=Path(__file__).parent,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
class RelayLifecycleTest(unittest.IsolatedAsyncioTestCase):
|
||||
def test_host_teardown_exits_successfully_after_trial_unwinds(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
marker = Path(directory) / "cleanup-attempted"
|
||||
completed = run_host_teardown_probe(marker, False)
|
||||
self.assertEqual(completed.returncode, 0, completed.stderr)
|
||||
self.assertEqual(marker.read_text(), "attempted")
|
||||
|
||||
def test_host_teardown_failure_remains_nonzero(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
marker = Path(directory) / "cleanup-attempted"
|
||||
completed = run_host_teardown_probe(marker, True)
|
||||
self.assertNotEqual(completed.returncode, 0)
|
||||
self.assertEqual(marker.read_text(), "attempted")
|
||||
|
||||
async def test_scope_setup_failure_is_reported_without_quiescing_an_unknown_scope(self):
|
||||
relay = load_relay()
|
||||
environment = ScopeSetupFailureEnvironment()
|
||||
token = f"scope-setup-{os.getpid()}"
|
||||
connected = asyncio.get_running_loop().create_future()
|
||||
|
||||
async def accept(reader, writer):
|
||||
connected.set_result((reader, writer))
|
||||
|
||||
server = await asyncio.start_server(accept, "127.0.0.1", 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
agent = relay.RelayAgent(
|
||||
logs_dir=Path(tempfile.gettempdir()),
|
||||
relay_host="127.0.0.1",
|
||||
relay_port=port,
|
||||
relay_token=token,
|
||||
teardown_timeout_ms=1_000,
|
||||
)
|
||||
running = asyncio.create_task(agent.run("solve", environment, None))
|
||||
reader, writer = await connected
|
||||
try:
|
||||
await reader.readline()
|
||||
writer.write(
|
||||
(__import__("json").dumps({
|
||||
"token": token,
|
||||
"kind": "execute",
|
||||
"command": "/bin/true",
|
||||
"args": [],
|
||||
"credentials": {},
|
||||
"captureStdout": False,
|
||||
"resultToken": "0" * 32,
|
||||
}) + "\n").encode()
|
||||
)
|
||||
await writer.drain()
|
||||
executed = __import__("json").loads(await reader.readline())
|
||||
self.assertEqual(executed["exitCode"], 111)
|
||||
self.assertEqual(executed["diagnostic"]["category"], "execution-scope-unavailable")
|
||||
writer.write(
|
||||
(__import__("json").dumps({"token": token, "kind": "verify"}) + "\n").encode()
|
||||
)
|
||||
await writer.drain()
|
||||
await running
|
||||
finally:
|
||||
writer.close()
|
||||
server.close()
|
||||
await server.wait_closed()
|
||||
|
||||
async def test_missing_scope_evidence_cannot_be_treated_as_quiescent(self):
|
||||
relay = load_relay()
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "scope evidence was unavailable"):
|
||||
await relay._scope_active(ScopeEvidenceEnvironment(4), "/", "/missing")
|
||||
self.assertFalse(await relay._scope_active(ScopeEvidenceEnvironment(3), "/", "/gone"))
|
||||
|
||||
async def test_closed_peer_is_a_bounded_transport_outcome(self):
|
||||
relay = load_relay()
|
||||
delivered = await relay._send(ClosedWriter(), {"kind": "verify"})
|
||||
self.assertFalse(delivered)
|
||||
|
||||
async def test_destroy_fallback_obeys_the_teardown_deadline(self):
|
||||
relay = load_relay()
|
||||
execution = asyncio.create_task(asyncio.sleep(60))
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
relay._settle_or_destroy(
|
||||
HangingStopEnvironment(), "/", "/tmp/missing", execution, 0.01
|
||||
),
|
||||
timeout=0.1,
|
||||
)
|
||||
finally:
|
||||
execution.cancel()
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await execution
|
||||
|
||||
async def test_peer_close_during_execution_triggers_bounded_settlement(self):
|
||||
relay = load_relay()
|
||||
environment = TransportLossEnvironment()
|
||||
token = f"transport-loss-{os.getpid()}"
|
||||
connected = asyncio.get_running_loop().create_future()
|
||||
|
||||
async def accept(reader, writer):
|
||||
connected.set_result((reader, writer))
|
||||
|
||||
server = await asyncio.start_server(accept, "127.0.0.1", 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
agent = relay.RelayAgent(
|
||||
logs_dir=Path(tempfile.gettempdir()),
|
||||
relay_host="127.0.0.1",
|
||||
relay_port=port,
|
||||
relay_token=token,
|
||||
teardown_timeout_ms=1_000,
|
||||
)
|
||||
running = asyncio.create_task(agent.run("solve", environment, None))
|
||||
reader, writer = await connected
|
||||
try:
|
||||
await reader.readline()
|
||||
writer.write(
|
||||
(
|
||||
__import__("json").dumps(
|
||||
{
|
||||
"token": token,
|
||||
"kind": "execute",
|
||||
"command": "/bin/true",
|
||||
"args": [],
|
||||
"credentials": {},
|
||||
"resultToken": "0" * 32,
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
).encode()
|
||||
)
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
await asyncio.wait_for(running, timeout=0.5)
|
||||
self.assertTrue(environment.release.is_set())
|
||||
finally:
|
||||
server.close()
|
||||
await server.wait_closed()
|
||||
|
||||
async def test_terminal_execution_wins_when_framework_cancellation_is_observed(self):
|
||||
relay = load_relay()
|
||||
environment = LiveScopeEnvironment()
|
||||
token = f"simultaneous-{os.getpid()}"
|
||||
connected = asyncio.get_running_loop().create_future()
|
||||
|
||||
async def accept(reader, writer):
|
||||
connected.set_result((reader, writer))
|
||||
|
||||
server = await asyncio.start_server(accept, "127.0.0.1", 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
agent = relay.RelayAgent(
|
||||
logs_dir=Path(tempfile.gettempdir()),
|
||||
relay_host="127.0.0.1",
|
||||
relay_port=port,
|
||||
relay_token=token,
|
||||
teardown_timeout_ms=110_000,
|
||||
)
|
||||
running = asyncio.create_task(agent.run("solve", environment, None))
|
||||
reader, writer = await connected
|
||||
try:
|
||||
ready = __import__("json").loads(await reader.readline())
|
||||
self.assertEqual(ready["kind"], "ready")
|
||||
self.assertEqual(ready["cwd"], "/workspace")
|
||||
writer.write(
|
||||
(
|
||||
__import__("json").dumps(
|
||||
{
|
||||
"token": token,
|
||||
"kind": "execute",
|
||||
"command": "/bin/true",
|
||||
"args": [],
|
||||
"credentials": {},
|
||||
"resultToken": "0" * 32,
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
).encode()
|
||||
)
|
||||
await writer.drain()
|
||||
environment.release.set()
|
||||
await environment.finished.wait()
|
||||
relay.request_host_teardown()
|
||||
running.cancel()
|
||||
executed = __import__("json").loads(await reader.readline())
|
||||
self.assertEqual(executed["termination"], "exited")
|
||||
self.assertEqual(executed["exitCode"], 0)
|
||||
# Cancellation found the subject already stopped, and a stopped
|
||||
# subject is reported as one -- so the verifier reads the same
|
||||
# environment it would have read without the cancellation. Tearing
|
||||
# its scope down here would edit that environment for exactly the
|
||||
# runs the framework happened to interrupt.
|
||||
self.assertEqual(
|
||||
[command for command in environment.commands if _is_teardown(command)],
|
||||
[],
|
||||
)
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await running
|
||||
finally:
|
||||
writer.close()
|
||||
with contextlib.suppress(ConnectionError, asyncio.CancelledError):
|
||||
await writer.wait_closed()
|
||||
server.close()
|
||||
await server.wait_closed()
|
||||
Path(f"/tmp/maka-eval-{token}.env").unlink(missing_ok=True)
|
||||
|
||||
async def test_framework_cancellation_remains_timeout_when_settlement_induces_exit(self):
|
||||
relay = load_relay()
|
||||
environment = SettleCompletionEnvironment()
|
||||
token = f"host-terminal-{os.getpid()}"
|
||||
connected = asyncio.get_running_loop().create_future()
|
||||
|
||||
async def accept(reader, writer):
|
||||
connected.set_result((reader, writer))
|
||||
|
||||
server = await asyncio.start_server(accept, "127.0.0.1", 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
agent = relay.RelayAgent(
|
||||
logs_dir=Path(tempfile.gettempdir()), relay_host="127.0.0.1",
|
||||
relay_port=port, relay_token=token, teardown_timeout_ms=1_000,
|
||||
)
|
||||
running = asyncio.create_task(agent.run("solve", environment, None))
|
||||
reader, writer = await connected
|
||||
try:
|
||||
await reader.readline()
|
||||
writer.write((__import__("json").dumps({
|
||||
"token": token, "kind": "execute", "command": "/bin/true", "args": [],
|
||||
"credentials": {}, "resultToken": "0" * 32,
|
||||
}) + "\n").encode())
|
||||
await writer.drain()
|
||||
await environment.started.wait()
|
||||
running.cancel()
|
||||
executed = __import__("json").loads(await asyncio.wait_for(reader.readline(), 0.5))
|
||||
self.assertEqual(executed["termination"], "framework_timeout")
|
||||
self.assertEqual(executed["exitCode"], 124)
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await running
|
||||
finally:
|
||||
writer.close()
|
||||
server.close()
|
||||
await server.wait_closed()
|
||||
|
||||
async def test_framework_timeout_survives_destroy_fallback(self):
|
||||
relay = load_relay()
|
||||
environment = FrameworkTimeoutEnvironment()
|
||||
token = f"framework-destroy-{os.getpid()}"
|
||||
connected = asyncio.get_running_loop().create_future()
|
||||
|
||||
async def accept(reader, writer):
|
||||
connected.set_result((reader, writer))
|
||||
|
||||
server = await asyncio.start_server(accept, "127.0.0.1", 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
agent = relay.RelayAgent(
|
||||
logs_dir=Path(tempfile.gettempdir()), relay_host="127.0.0.1",
|
||||
relay_port=port, relay_token=token, teardown_timeout_ms=50,
|
||||
)
|
||||
running = asyncio.create_task(agent.run("solve", environment, None))
|
||||
reader, writer = await connected
|
||||
try:
|
||||
await reader.readline()
|
||||
writer.write((__import__("json").dumps({
|
||||
"token": token, "kind": "execute", "command": "/bin/true", "args": [],
|
||||
"credentials": {}, "resultToken": "0" * 32,
|
||||
}) + "\n").encode())
|
||||
await writer.drain()
|
||||
await environment.started.wait()
|
||||
running.cancel()
|
||||
executed = __import__("json").loads(await asyncio.wait_for(reader.readline(), 0.5))
|
||||
self.assertEqual(executed["termination"], "framework_timeout")
|
||||
self.assertEqual(executed["exitCode"], 124)
|
||||
self.assertEqual(executed["diagnostic"]["category"], "result-frame-missing")
|
||||
self.assertTrue(environment.stopped)
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await running
|
||||
finally:
|
||||
writer.close()
|
||||
server.close()
|
||||
await server.wait_closed()
|
||||
|
||||
@unittest.skipUnless(shutil.which("setsid"), "requires GNU setsid")
|
||||
async def test_scope_waits_for_child_and_publishes_bounded_stdout(self):
|
||||
relay = load_relay()
|
||||
environment = LocalEnvironment()
|
||||
token = f"test-{os.getpid()}"
|
||||
scope_path = f"/tmp/maka-eval-{token}.pid"
|
||||
request = {
|
||||
"command": "/bin/sh",
|
||||
"args": ["-c", "sleep 0.1; printf result-json"],
|
||||
"credentials": {},
|
||||
"resultToken": "0" * 32,
|
||||
}
|
||||
try:
|
||||
command = await relay._prepare_command(environment, request, token, scope_path)
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=tempfile.gettempdir(),
|
||||
shell=True,
|
||||
check=False,
|
||||
timeout=2,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0)
|
||||
self.assertGreaterEqual(time.monotonic() - started, 0.09)
|
||||
self.assertEqual(result.stdout, "result-json")
|
||||
finally:
|
||||
Path(scope_path).unlink(missing_ok=True)
|
||||
|
||||
@unittest.skipUnless(shutil.which("setsid"), "requires GNU setsid")
|
||||
async def test_settle_kills_descendants_before_verification_boundary(self):
|
||||
relay = load_relay()
|
||||
environment = LocalEnvironment()
|
||||
token = f"descendant-{os.getpid()}"
|
||||
scope_path = f"/tmp/maka-eval-{token}.pid"
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
late_write = Path(directory) / "late-write"
|
||||
child_ready = Path(directory) / "child-ready"
|
||||
release_pipe = Path(directory) / "release"
|
||||
os.mkfifo(release_pipe)
|
||||
# Keep the FIFO open from the test side so the descendant can report
|
||||
# readiness and remain blocked until settlement has completed.
|
||||
release_fd = os.open(release_pipe, os.O_RDWR | os.O_NONBLOCK)
|
||||
child_pid = None
|
||||
execution = None
|
||||
request = {
|
||||
"command": sys.executable,
|
||||
"args": [
|
||||
"-c",
|
||||
"\n".join(
|
||||
[
|
||||
"import os,time",
|
||||
f"release_pipe={str(release_pipe)!r}",
|
||||
f"child_ready={str(child_ready)!r}",
|
||||
f"late_write={str(late_write)!r}",
|
||||
"child=os.fork()",
|
||||
"if child == 0:",
|
||||
" with open(release_pipe, 'rb', buffering=0) as pipe:",
|
||||
" ready=child_ready+'.tmp'",
|
||||
" with open(ready, 'w') as marker:",
|
||||
" marker.write(str(os.getpid()))",
|
||||
" os.replace(ready, child_ready)",
|
||||
" pipe.read(1)",
|
||||
" with open(late_write, 'w') as late:",
|
||||
" late.write('late')",
|
||||
" os._exit(0)",
|
||||
"deadline=time.monotonic()+5",
|
||||
"while not os.path.exists(child_ready) and time.monotonic() < deadline:",
|
||||
" time.sleep(0.01)",
|
||||
"os._exit(0 if os.path.exists(child_ready) else 2)",
|
||||
]
|
||||
),
|
||||
],
|
||||
"credentials": {},
|
||||
"resultToken": "0" * 32,
|
||||
"cwd": directory,
|
||||
}
|
||||
try:
|
||||
command = await relay._prepare_command(environment, request, token, scope_path)
|
||||
execution = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
subprocess.run,
|
||||
command,
|
||||
cwd=directory,
|
||||
shell=True,
|
||||
check=False,
|
||||
timeout=2,
|
||||
)
|
||||
)
|
||||
deadline = time.monotonic() + 1
|
||||
while True:
|
||||
scope = Path(scope_path)
|
||||
pid = scope.read_text().strip() if scope.exists() else ""
|
||||
if pid.isdigit():
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
self.fail("scope PID was not published")
|
||||
await asyncio.sleep(0.01)
|
||||
deadline = time.monotonic() + 1
|
||||
while not child_ready.exists():
|
||||
if time.monotonic() >= deadline:
|
||||
self.fail("descendant did not become ready")
|
||||
await asyncio.sleep(0.01)
|
||||
child_pid = int(child_ready.read_text())
|
||||
self.assertEqual(os.getpgid(child_pid), int(pid))
|
||||
completed = await execution
|
||||
self.assertEqual(completed.returncode, 0)
|
||||
result = await relay._settle(environment, directory, scope_path, execution)
|
||||
self.assertEqual(result.returncode, 0)
|
||||
# A broken quiescence check leaves the descendant alive. Releasing
|
||||
# it now makes that bug observable as a late workspace write.
|
||||
os.write(release_fd, b"x")
|
||||
deadline = time.monotonic() + 1
|
||||
while not late_write.exists() and time.monotonic() < deadline:
|
||||
await asyncio.sleep(0.01)
|
||||
self.assertFalse(late_write.exists())
|
||||
finally:
|
||||
with contextlib.suppress(OSError):
|
||||
os.write(release_fd, b"x")
|
||||
if child_pid is None and child_ready.exists():
|
||||
with contextlib.suppress(OSError, ValueError):
|
||||
child_pid = int(child_ready.read_text())
|
||||
if child_pid is not None:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.kill(child_pid, signal.SIGKILL)
|
||||
if execution is not None and not execution.done():
|
||||
with contextlib.suppress(Exception):
|
||||
await relay._signal(environment, directory, scope_path, "KILL")
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
await asyncio.wait_for(execution, timeout=1)
|
||||
os.close(release_fd)
|
||||
Path(scope_path).unlink(missing_ok=True)
|
||||
|
||||
async def test_an_exited_subject_is_left_alone_whatever_it_reported(self):
|
||||
# The verifier scores the environment the task was left in, so a subject
|
||||
# that stopped on its own keeps whatever it started — a service the task
|
||||
# was asked to run has to still be there. Reading the exit code here to
|
||||
# decide otherwise would make the measurement depend on the framework's
|
||||
# own classification, and only for the subjects it classifies as failed.
|
||||
for return_code, status in ((0, "completed"), (1, "failed")):
|
||||
with self.subTest(return_code=return_code):
|
||||
relay = load_relay()
|
||||
environment = RecordingEnvironment(return_code, status)
|
||||
token = f"exited-{return_code}-{os.getpid()}"
|
||||
connected = asyncio.get_running_loop().create_future()
|
||||
|
||||
async def accept(reader, writer):
|
||||
connected.set_result((reader, writer))
|
||||
|
||||
server = await asyncio.start_server(accept, "127.0.0.1", 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
agent = relay.RelayAgent(
|
||||
logs_dir=Path(tempfile.gettempdir()),
|
||||
relay_host="127.0.0.1",
|
||||
relay_port=port,
|
||||
relay_token=token,
|
||||
teardown_timeout_ms=1_000,
|
||||
)
|
||||
running = asyncio.create_task(agent.run("solve", environment, None))
|
||||
reader, writer = await connected
|
||||
try:
|
||||
await reader.readline()
|
||||
writer.write(
|
||||
(
|
||||
json.dumps(
|
||||
{
|
||||
"token": token,
|
||||
"kind": "execute",
|
||||
"command": "/bin/true",
|
||||
"args": [],
|
||||
"credentials": {},
|
||||
"resultToken": "0" * 32,
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
).encode()
|
||||
)
|
||||
await writer.drain()
|
||||
executed = json.loads(await reader.readline())
|
||||
self.assertEqual(executed["termination"], "exited")
|
||||
self.assertEqual(executed["exitCode"], return_code)
|
||||
writer.write(
|
||||
(json.dumps({"token": token, "kind": "verify"}) + "\n").encode()
|
||||
)
|
||||
await writer.drain()
|
||||
await asyncio.wait_for(running, timeout=2)
|
||||
finally:
|
||||
writer.close()
|
||||
with contextlib.suppress(Exception):
|
||||
await writer.wait_closed()
|
||||
server.close()
|
||||
await server.wait_closed()
|
||||
self.assertEqual(
|
||||
[command for command in environment.commands if _is_teardown(command)],
|
||||
[],
|
||||
)
|
||||
|
||||
@unittest.skipUnless(shutil.which("node"), "requires Node.js")
|
||||
def test_deepseek_harness_toolchain_patch_preserves_background_descendants(self):
|
||||
patcher = (
|
||||
Path(__file__).parent
|
||||
/ "deepseek-harness-toolchain"
|
||||
/ "patch-subprocess-local.mjs"
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
target = (
|
||||
Path(directory)
|
||||
/ "node_modules"
|
||||
/ "@deepseek-ai"
|
||||
/ "dsh-subprocess-local"
|
||||
/ "lib"
|
||||
/ "index.js"
|
||||
)
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text(
|
||||
"""\
|
||||
\tterminateForHostExit() {
|
||||
\t\tthis.forceStopDescendants();
|
||||
\t\tthis.forceStopShell();
|
||||
\t\tthis.forceStopDescendants();
|
||||
\t}
|
||||
\tasync closeOnce() {
|
||||
\t\tlet survivors = await this.stopDescendants();
|
||||
\t\tif (survivors.length > 0) throw new Error(`terminal cleanup failed; surviving pids: ${survivors.map((member) => member.pid).join(", ")}`);
|
||||
\t\tawait this.stopShell();
|
||||
\t\tsurvivors = await this.stopDescendants();
|
||||
\t\tif (survivors.length > 0) throw new Error(`terminal cleanup failed; surviving pids: ${survivors.map((member) => member.pid).join(", ")}`);
|
||||
\t\tthis.dataDisposable.dispose();
|
||||
\t\tthis.exitDisposable.dispose();
|
||||
\t}
|
||||
"""
|
||||
)
|
||||
subprocess.run(
|
||||
[shutil.which("node"), patcher, directory],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
patched = target.read_text()
|
||||
self.assertEqual(
|
||||
patched.count('process.env.DSH_PRESERVE_BACKGROUND_PROCESSES === "1"'),
|
||||
2,
|
||||
)
|
||||
self.assertIn("this.forceStopShell();\n\t\t\treturn;", patched)
|
||||
self.assertIn("await this.stopShell();", patched)
|
||||
|
||||
# The value of this patcher is that an upstream release it no longer
|
||||
# understands stops the build rather than producing a toolchain that
|
||||
# silently kills the task's services again. Running it over its own
|
||||
# output is the cheapest form of drift: the text it anchors on is
|
||||
# gone.
|
||||
drifted = subprocess.run(
|
||||
[shutil.which("node"), patcher, directory],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertNotEqual(drifted.returncode, 0)
|
||||
self.assertNotEqual(target.read_text(), "")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,79 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
MODULE_PATH = Path(__file__).with_name("run_trial.py")
|
||||
SPEC = importlib.util.spec_from_file_location("maka_eval_run_trial", MODULE_PATH)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
class RunTrialPolicyTest(unittest.TestCase):
|
||||
def test_forces_only_the_subject_phase_through_the_cell_proxy(self) -> None:
|
||||
agent = SimpleNamespace(network_mode=None, allowed_hosts=None)
|
||||
task = SimpleNamespace(config=SimpleNamespace(agent=agent))
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"MAKA_EVAL_EGRESS_REQUIRED": "1",
|
||||
"MAKA_EVAL_EGRESS_ALLOWED_HOST": "maka-eval-mitmproxy",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
MODULE.apply_subject_egress_policy(task)
|
||||
self.assertEqual(agent.network_mode, "allowlist")
|
||||
self.assertEqual(agent.allowed_hosts, ["maka-eval-mitmproxy"])
|
||||
|
||||
def test_required_egress_fails_closed_without_the_proxy_host(self) -> None:
|
||||
task = SimpleNamespace(config=SimpleNamespace(agent=SimpleNamespace()))
|
||||
with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}, clear=True):
|
||||
with self.assertRaisesRegex(RuntimeError, "proxy host is unavailable"):
|
||||
MODULE.apply_subject_egress_policy(task)
|
||||
|
||||
def test_invalid_framework_fails_before_framework_import(self) -> None:
|
||||
with patch.object(MODULE.importlib, "import_module") as imported:
|
||||
with self.assertRaisesRegex(RuntimeError, "harbor or pier"):
|
||||
asyncio.run(MODULE.run_trial("other", "1.0.0", Path("missing.json")))
|
||||
imported.assert_not_called()
|
||||
|
||||
def test_main_installs_the_argv_framework_before_the_trial(self) -> None:
|
||||
import eval_framework
|
||||
|
||||
installed: list[str] = []
|
||||
|
||||
async def fake_trial(framework: str, expected_version: str, config_file: Path) -> None:
|
||||
installed.append(eval_framework.selected())
|
||||
self.assertEqual(framework, "pier")
|
||||
self.assertEqual(expected_version, "1.2.3")
|
||||
|
||||
with patch.object(sys, "argv", ["run_trial.py", "pier", "1.2.3", "config.json"]):
|
||||
with patch.object(MODULE, "run_trial", fake_trial):
|
||||
asyncio.run(MODULE.main())
|
||||
self.assertEqual(installed, ["pier"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user