mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix: use detected media type for screenshots instead of hard-coded image/png
browser_screenshot declared every capture as image/png regardless of the
actual bytes. On Chromium/Edge builds where captureVisibleTab returns JPEG
bytes, the stored attachment reference recorded mediaType "image/png" against
JPEG bytes, so dsh-attachment-local's read-time metadata verification fails
with ATTACHMENT_CORRUPT ("Stored attachment metadata does not match its
reference"), surfacing as code UNKNOWN and triggering endless LLM retries.
- image.ts: sniff the capture's magic bytes (PNG/JPEG/WebP/GIF) and declare
that real media type to attachments.saveImage, instead of always "image/png".
- tools.ts: assemble the tool result from ref.mediaType (the store's verified
type) rather than re-declaring "image/png" as const, and relax the output
schema's const to the accepted media-type enum.
- tests: add a regression case asserting JPEG capture bytes surface as
image/jpeg.
Co-Authored-By: DeepSeek Harness <noreply@deepseek.com>
This commit is contained in:
co-authored by
DeepSeek Harness
parent
4b6cdde168
commit
60fbf30e24
@@ -1,10 +1,11 @@
|
||||
/**
|
||||
* Screenshot delivery. The canonical tool value stays plain JSON (a PNG file
|
||||
* path plus pixel metadata); when the host mounts a durable attachment store
|
||||
* AND the calling route declares image input, the PNG bytes are additionally
|
||||
* committed through `ctx.attachments` so the render step can attach the image
|
||||
* itself to the tool result. Any uncertainty (no store, unknown route,
|
||||
* text-only model) falls back to the path-only form.
|
||||
* Screenshot delivery. The canonical tool value stays plain JSON (a screenshot
|
||||
* file path plus pixel metadata); when the host mounts a durable attachment
|
||||
* store AND the calling route declares image input, the capture bytes are
|
||||
* additionally committed through `ctx.attachments` so the render step can
|
||||
* attach the image itself to the tool result. Any uncertainty (no store,
|
||||
* unknown route, text-only model, unrecognized bytes) falls back to the
|
||||
* path-only form.
|
||||
*/
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
@@ -33,6 +34,54 @@ interface LlmLike {
|
||||
): Promise<{ inputModalities?: readonly string[] }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sniff the image media type from magic bytes. The capture path can hand back
|
||||
* JPEG (e.g. `chrome.tabs.captureVisibleTab` on some Chromium/Edge builds
|
||||
* returning JPEG regardless of the requested format), so the declared type can
|
||||
* never be trusted from the file extension alone.
|
||||
* @param data - first bytes of the capture.
|
||||
* @returns the detected media type, or undefined when unrecognized.
|
||||
*/
|
||||
function sniffImageMediaType(data: Uint8Array): string | undefined {
|
||||
if (data.length >= 4) {
|
||||
// PNG signature: 89 50 4E 47 (0D 0A 1A 0A)
|
||||
if (data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) {
|
||||
return "image/png";
|
||||
}
|
||||
// JPEG SOI: FF D8 FF
|
||||
if (data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
|
||||
return "image/jpeg";
|
||||
}
|
||||
// WebP: "RIFF" (52 49 46 46) + "WEBP" at offset 8
|
||||
if (
|
||||
data[0] === 0x52 &&
|
||||
data[1] === 0x49 &&
|
||||
data[2] === 0x46 &&
|
||||
data[3] === 0x46 &&
|
||||
data.length >= 12 &&
|
||||
data[8] === 0x57 &&
|
||||
data[9] === 0x45 &&
|
||||
data[10] === 0x42 &&
|
||||
data[11] === 0x50
|
||||
) {
|
||||
return "image/webp";
|
||||
}
|
||||
// GIF: "GIF87a" or "GIF89a"
|
||||
if (
|
||||
data[0] === 0x47 &&
|
||||
data[1] === 0x49 &&
|
||||
data[2] === 0x46 &&
|
||||
data[3] === 0x38 &&
|
||||
data.length >= 6 &&
|
||||
(data[4] === 0x37 || data[4] === 0x39) &&
|
||||
data[5] === 0x61
|
||||
) {
|
||||
return "image/gif";
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commit screenshot bytes to the host attachment store when the composition
|
||||
* supports durable images on the current model route.
|
||||
@@ -46,7 +95,14 @@ export async function trySaveScreenshot(
|
||||
): Promise<ImageAttachmentRef | undefined> {
|
||||
const attachments = ctx.get("attachments") as AttachmentLike | undefined;
|
||||
if (attachments === undefined) return undefined;
|
||||
if (!attachments.imageLimits.mediaTypes.includes("image/png")) return undefined;
|
||||
// Declare the real media type from the bytes, never a hard-coded "image/png".
|
||||
// Declaring the wrong type trips the store's IMAGE_TYPE_MISMATCH check and
|
||||
// silently degrades to path-only; sniffing first lets JPEG captures (returned
|
||||
// by some Chromium builds for `captureVisibleTab({ format: "png" })`) still
|
||||
// inline correctly.
|
||||
const mediaType = sniffImageMediaType(data);
|
||||
if (mediaType === undefined) return undefined;
|
||||
if (!attachments.imageLimits.mediaTypes.includes(mediaType)) return undefined;
|
||||
if (
|
||||
data.byteLength >
|
||||
Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
|
||||
@@ -55,9 +111,9 @@ export async function trySaveScreenshot(
|
||||
}
|
||||
if (!(await isImageCapableRoute(ctx, exec))) return undefined;
|
||||
try {
|
||||
return await attachments.saveImage({ data, mediaType: "image/png", name });
|
||||
return await attachments.saveImage({ data, mediaType, name });
|
||||
} catch {
|
||||
// A store hiccup must not fail the tool call; the PNG path still works.
|
||||
// A store hiccup must not fail the tool call; the path-only form still works.
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user