|
@@ -35,7 +35,7 @@ interface FetchDecompressionError extends Error {
|
|
|
path: string
|
|
path: string
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-export const SYNTHETIC_ATTACHMENT_PROMPT = "Attached image(s) from tool result:"
|
|
|
|
|
|
|
+export const SYNTHETIC_ATTACHMENT_PROMPT = "Attached media from tool result:"
|
|
|
export { isMedia }
|
|
export { isMedia }
|
|
|
|
|
|
|
|
export const OutputLengthError = namedSchemaError("MessageOutputLengthError", {})
|
|
export const OutputLengthError = namedSchemaError("MessageOutputLengthError", {})
|
|
@@ -734,25 +734,25 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
|
|
|
const result: UIMessage[] = []
|
|
const result: UIMessage[] = []
|
|
|
const toolNames = new Set<string>()
|
|
const toolNames = new Set<string>()
|
|
|
// Track media from tool results that need to be injected as user messages
|
|
// Track media from tool results that need to be injected as user messages
|
|
|
- // for providers that don't support media in tool results.
|
|
|
|
|
|
|
+ // for providers that don't support that media type in tool results.
|
|
|
//
|
|
//
|
|
|
// OpenAI-compatible APIs only support string content in tool results, so we need
|
|
// OpenAI-compatible APIs only support string content in tool results, so we need
|
|
|
- // to extract media and inject as user messages. Other SDKs (anthropic, google,
|
|
|
|
|
- // bedrock) handle type: "content" with media parts natively.
|
|
|
|
|
|
|
+ // to extract media and inject as user messages. Some SDKs only support a subset
|
|
|
|
|
+ // of media in tool results; e.g. Bedrock supports images but not PDFs there.
|
|
|
//
|
|
//
|
|
|
- // Only apply this workaround if the model actually supports image input -
|
|
|
|
|
- // otherwise there's no point extracting images.
|
|
|
|
|
- const supportsMediaInToolResults = (() => {
|
|
|
|
|
|
|
+ // Only apply this workaround if the model actually supports that media input -
|
|
|
|
|
+ // otherwise unsupportedParts() will turn it into a user-visible error.
|
|
|
|
|
+ const supportsMediaInToolResult = (attachment: { mime: string }) => {
|
|
|
if (model.api.npm === "@ai-sdk/anthropic") return true
|
|
if (model.api.npm === "@ai-sdk/anthropic") return true
|
|
|
if (model.api.npm === "@ai-sdk/openai") return true
|
|
if (model.api.npm === "@ai-sdk/openai") return true
|
|
|
- if (model.api.npm === "@ai-sdk/amazon-bedrock") return true
|
|
|
|
|
|
|
+ if (model.api.npm === "@ai-sdk/amazon-bedrock") return attachment.mime.startsWith("image/")
|
|
|
if (model.api.npm === "@ai-sdk/google-vertex/anthropic") return true
|
|
if (model.api.npm === "@ai-sdk/google-vertex/anthropic") return true
|
|
|
if (model.api.npm === "@ai-sdk/google") {
|
|
if (model.api.npm === "@ai-sdk/google") {
|
|
|
const id = model.api.id.toLowerCase()
|
|
const id = model.api.id.toLowerCase()
|
|
|
return id.includes("gemini-3") && !id.includes("gemini-2")
|
|
return id.includes("gemini-3") && !id.includes("gemini-2")
|
|
|
}
|
|
}
|
|
|
return false
|
|
return false
|
|
|
- })()
|
|
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
const toModelOutput = (options: { toolCallId: string; input: unknown; output: unknown }) => {
|
|
const toModelOutput = (options: { toolCallId: string; input: unknown; output: unknown }) => {
|
|
|
const output = options.output
|
|
const output = options.output
|
|
@@ -797,9 +797,9 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
|
|
|
role: "user",
|
|
role: "user",
|
|
|
parts: [],
|
|
parts: [],
|
|
|
}
|
|
}
|
|
|
- result.push(userMessage)
|
|
|
|
|
for (const part of msg.parts) {
|
|
for (const part of msg.parts) {
|
|
|
- if (part.type === "text" && !part.ignored)
|
|
|
|
|
|
|
+ // User message parts should never be empty
|
|
|
|
|
+ if (part.type === "text" && !part.ignored && part.text !== "")
|
|
|
userMessage.parts.push({
|
|
userMessage.parts.push({
|
|
|
type: "text",
|
|
type: "text",
|
|
|
text: part.text,
|
|
text: part.text,
|
|
@@ -834,11 +834,12 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
|
|
|
})
|
|
})
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
+ if (userMessage.parts.length > 0) result.push(userMessage)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
if (msg.info.role === "assistant") {
|
|
if (msg.info.role === "assistant") {
|
|
|
const differentModel = `${model.providerID}/${model.id}` !== `${msg.info.providerID}/${msg.info.modelID}`
|
|
const differentModel = `${model.providerID}/${model.id}` !== `${msg.info.providerID}/${msg.info.modelID}`
|
|
|
- const media: Array<{ mime: string; url: string }> = []
|
|
|
|
|
|
|
+ const media: Array<{ mime: string; url: string; filename?: string }> = []
|
|
|
|
|
|
|
|
if (
|
|
if (
|
|
|
msg.info.error &&
|
|
msg.info.error &&
|
|
@@ -864,11 +865,10 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
|
|
|
// a proxy, or a lower-level library, but preserving a non-empty separator
|
|
// a proxy, or a lower-level library, but preserving a non-empty separator
|
|
|
// here is the only safe replay point we have.
|
|
// here is the only safe replay point we have.
|
|
|
// Use a single space so the separator survives replay without changing
|
|
// Use a single space so the separator survives replay without changing
|
|
|
- // the neighboring signed reasoning blocks. Bedrock-hosted Claude stores
|
|
|
|
|
- // the same signature under the bedrock metadata namespace.
|
|
|
|
|
|
|
+ // the neighboring signed reasoning blocks.
|
|
|
const hasSignedReasoning = msg.parts.some((part) => {
|
|
const hasSignedReasoning = msg.parts.some((part) => {
|
|
|
if (part.type !== "reasoning") return false
|
|
if (part.type !== "reasoning") return false
|
|
|
- return part.metadata?.anthropic?.signature != null || part.metadata?.bedrock?.signature != null
|
|
|
|
|
|
|
+ return part.metadata?.anthropic?.signature != null
|
|
|
})
|
|
})
|
|
|
for (const part of msg.parts) {
|
|
for (const part of msg.parts) {
|
|
|
if (part.type === "text") {
|
|
if (part.type === "text") {
|
|
@@ -894,11 +894,11 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
|
|
|
// For providers that don't support media in tool results, extract media files
|
|
// For providers that don't support media in tool results, extract media files
|
|
|
// (images, PDFs) to be sent as a separate user message
|
|
// (images, PDFs) to be sent as a separate user message
|
|
|
const mediaAttachments = attachments.filter((a) => isMedia(a.mime))
|
|
const mediaAttachments = attachments.filter((a) => isMedia(a.mime))
|
|
|
- const nonMediaAttachments = attachments.filter((a) => !isMedia(a.mime))
|
|
|
|
|
- if (!supportsMediaInToolResults && mediaAttachments.length > 0) {
|
|
|
|
|
- media.push(...mediaAttachments)
|
|
|
|
|
|
|
+ const extractedMedia = mediaAttachments.filter((a) => !supportsMediaInToolResult(a))
|
|
|
|
|
+ if (extractedMedia.length > 0) {
|
|
|
|
|
+ media.push(...extractedMedia)
|
|
|
}
|
|
}
|
|
|
- const finalAttachments = supportsMediaInToolResults ? attachments : nonMediaAttachments
|
|
|
|
|
|
|
+ const finalAttachments = attachments.filter((a) => !isMedia(a.mime) || supportsMediaInToolResult(a))
|
|
|
|
|
|
|
|
const output =
|
|
const output =
|
|
|
finalAttachments.length > 0
|
|
finalAttachments.length > 0
|
|
@@ -988,6 +988,7 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
|
|
|
type: "file" as const,
|
|
type: "file" as const,
|
|
|
url: attachment.url,
|
|
url: attachment.url,
|
|
|
mediaType: attachment.mime,
|
|
mediaType: attachment.mime,
|
|
|
|
|
+ filename: attachment.filename,
|
|
|
})),
|
|
})),
|
|
|
],
|
|
],
|
|
|
})
|
|
})
|