fix: use detected media type for screenshots instead of hard-coded image/png

browser_screenshot declared every capture as image/png regardless of the
actual bytes. On Chromium/Edge builds where captureVisibleTab returns JPEG
bytes, the stored attachment reference recorded mediaType "image/png" against
JPEG bytes, so dsh-attachment-local's read-time metadata verification fails
with ATTACHMENT_CORRUPT ("Stored attachment metadata does not match its
reference"), surfacing as code UNKNOWN and triggering endless LLM retries.

- image.ts: sniff the capture's magic bytes (PNG/JPEG/WebP/GIF) and declare
  that real media type to attachments.saveImage, instead of always "image/png".
- tools.ts: assemble the tool result from ref.mediaType (the store's verified
  type) rather than re-declaring "image/png" as const, and relax the output
  schema's const to the accepted media-type enum.
- tests: add a regression case asserting JPEG capture bytes surface as
  image/jpeg.

Co-Authored-By: DeepSeek Harness <noreply@deepseek.com>
This commit is contained in:
cyhano
2026-08-24 15:31:34 +08:00
co-authored by DeepSeek Harness
parent 4b6cdde168
commit 60fbf30e24
3 changed files with 124 additions and 11 deletions
+65 -9
View File
@@ -1,10 +1,11 @@
/**
* Screenshot delivery. The canonical tool value stays plain JSON (a PNG file
* path plus pixel metadata); when the host mounts a durable attachment store
* AND the calling route declares image input, the PNG bytes are additionally
* committed through `ctx.attachments` so the render step can attach the image
* itself to the tool result. Any uncertainty (no store, unknown route,
* text-only model) falls back to the path-only form.
* Screenshot delivery. The canonical tool value stays plain JSON (a screenshot
* file path plus pixel metadata); when the host mounts a durable attachment
* store AND the calling route declares image input, the capture bytes are
* additionally committed through `ctx.attachments` so the render step can
* attach the image itself to the tool result. Any uncertainty (no store,
* unknown route, text-only model, unrecognized bytes) falls back to the
* path-only form.
*/
import type { Context } from "@deepseek-ai/cordis";
@@ -33,6 +34,54 @@ interface LlmLike {
): Promise<{ inputModalities?: readonly string[] }>;
}
/**
* Sniff the image media type from magic bytes. The capture path can hand back
* JPEG (e.g. `chrome.tabs.captureVisibleTab` on some Chromium/Edge builds
* returning JPEG regardless of the requested format), so the declared type can
* never be trusted from the file extension alone.
* @param data - first bytes of the capture.
* @returns the detected media type, or undefined when unrecognized.
*/
function sniffImageMediaType(data: Uint8Array): string | undefined {
if (data.length >= 4) {
// PNG signature: 89 50 4E 47 (0D 0A 1A 0A)
if (data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) {
return "image/png";
}
// JPEG SOI: FF D8 FF
if (data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
return "image/jpeg";
}
// WebP: "RIFF" (52 49 46 46) + "WEBP" at offset 8
if (
data[0] === 0x52 &&
data[1] === 0x49 &&
data[2] === 0x46 &&
data[3] === 0x46 &&
data.length >= 12 &&
data[8] === 0x57 &&
data[9] === 0x45 &&
data[10] === 0x42 &&
data[11] === 0x50
) {
return "image/webp";
}
// GIF: "GIF87a" or "GIF89a"
if (
data[0] === 0x47 &&
data[1] === 0x49 &&
data[2] === 0x46 &&
data[3] === 0x38 &&
data.length >= 6 &&
(data[4] === 0x37 || data[4] === 0x39) &&
data[5] === 0x61
) {
return "image/gif";
}
}
return undefined;
}
/**
* Commit screenshot bytes to the host attachment store when the composition
* supports durable images on the current model route.
@@ -46,7 +95,14 @@ export async function trySaveScreenshot(
): Promise<ImageAttachmentRef | undefined> {
const attachments = ctx.get("attachments") as AttachmentLike | undefined;
if (attachments === undefined) return undefined;
if (!attachments.imageLimits.mediaTypes.includes("image/png")) return undefined;
// Declare the real media type from the bytes, never a hard-coded "image/png".
// Declaring the wrong type trips the store's IMAGE_TYPE_MISMATCH check and
// silently degrades to path-only; sniffing first lets JPEG captures (returned
// by some Chromium builds for `captureVisibleTab({ format: "png" })`) still
// inline correctly.
const mediaType = sniffImageMediaType(data);
if (mediaType === undefined) return undefined;
if (!attachments.imageLimits.mediaTypes.includes(mediaType)) return undefined;
if (
data.byteLength >
Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
@@ -55,9 +111,9 @@ export async function trySaveScreenshot(
}
if (!(await isImageCapableRoute(ctx, exec))) return undefined;
try {
return await attachments.saveImage({ data, mediaType: "image/png", name });
return await attachments.saveImage({ data, mediaType, name });
} catch {
// A store hiccup must not fail the tool call; the PNG path still works.
// A store hiccup must not fail the tool call; the path-only form still works.
return undefined;
}
}