import * as http from "node:http"; import * as https from "node:https"; import * as net from "node:net"; import * as os from "node:os"; import * as path from "node:path"; import { getEnvApiKey, type Model } from "@gajae-code/ai/core"; import { CODEX_BASE_URL, getCodexAccountId, OPENAI_HEADER_VALUES, OPENAI_HEADERS, URL_PATHS, } from "@gajae-code/ai/providers/openai-codex/constants"; import { $credentialEnv, isEnoent, parseImageMetadata, prompt, ptree, readSseJson, Snowflake, sanitizeHeaderComponent, untilAborted, } from "@gajae-code/utils"; import * as z from "zod/v4"; import packageJson from "../../package.json" with { type: "json" }; import { isAuthenticated, type ModelRegistry } from "../config/model-registry"; import { type ModelRoleSettings, resolveModelRoleValue } from "../config/model-resolver"; import type { CustomTool } from "../extensibility/custom-tools/types"; import imageGenDescription from "../prompts/tools/image-gen.md" with { type: "text" }; import { isPrivateOrSpecialAddress, validatePublicHttpUrl } from "../web/insane/url-guard"; import { resolveReadPath } from "./path-utils"; const DEFAULT_MODEL = "gemini-3-pro-image-preview"; const IMAGE_TIMEOUT = 3 * 60 * 1000; // 3 minutes const MAX_IMAGE_SIZE = 35 * 1024 * 1024; const MAX_IMAGE_REDIRECTS = 5; const MAX_IMAGE_HEADER_SIZE = 16 * 1024; const MAX_IMAGE_ERROR_PREVIEW_SIZE = 8 * 1024; const IMAGE_REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"; const OPENAI_IMAGE_OUTPUT_FORMAT = "webp"; const OPENAI_IMAGE_MIME_TYPE = "image/webp"; const MAX_PROVIDER_TEXT_LENGTH = 4096; const REDACTED_PROVIDER_SECRET = "[redacted]"; const MAX_PROVIDER_BODY_BYTES = 64 * 1024 * 1024; const MAX_PROVIDER_SSE_EVENT_BYTES = MAX_PROVIDER_BODY_BYTES; const PROVIDER_MALFORMED_RESPONSE = "Provider returned a malformed image response."; const PROVIDER_OVERSIZED_RESPONSE = "Provider image response exceeded the supported size."; function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } export function redactImageProviderText(value: unknown, activeApiKey?: string): string { let text = typeof value === "string" ? value : value === undefined || value === null ? "" : String(value); const key = activeApiKey?.trim(); if (key) { text = text.replace(new RegExp(escapeRegExp(key), "g"), REDACTED_PROVIDER_SECRET); const keyChars = [...key].filter(character => !/[\s\p{Cc}\p{Cf}]/u.test(character)); if (keyChars.length > 0) { const separatorTolerantKey = keyChars.map(escapeRegExp).join("[\\s\\p{Cc}\\p{Cf}]*"); text = text.replace(new RegExp(separatorTolerantKey, "gu"), REDACTED_PROVIDER_SECRET); } } text = text.replace(/[\p{Cc}\p{Cf}]/gu, " "); text = text .replace(/\b(?:bearer|basic)\s+[^\s,;]+/gi, match => `${match.split(/\s+/, 1)[0]} ${REDACTED_PROVIDER_SECRET}`) .replace( /(?:\b|["'])(?:authorization|api[-_ ]?key|access[-_ ]?token|refresh[-_ ]?token|token)["']?\s*[:=]\s*(?:"[^"]*"|'[^']*'|[^\s,;}]+)/gi, match => match.replace(/([:=]\s*).*/, `$1${REDACTED_PROVIDER_SECRET}`), ) .replace(/\b(?:sk|rk|pk|sess|ghp|gho|github_pat|xox[baprs])-[-A-Za-z0-9._]{8,}\b/gi, REDACTED_PROVIDER_SECRET) .replace(/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, REDACTED_PROVIDER_SECRET) .replace(/\b[A-Za-z0-9+/_-]{40,}={0,2}\b/g, REDACTED_PROVIDER_SECRET); return text.length > MAX_PROVIDER_TEXT_LENGTH ? `${text.slice(0, MAX_PROVIDER_TEXT_LENGTH - 1)}…` : text; } function tryParseProviderJson(rawText: string): T | undefined { if (Buffer.byteLength(rawText, "utf8") > MAX_PROVIDER_BODY_BYTES) return undefined; try { const parsed: unknown = JSON.parse(rawText); return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as T) : undefined; } catch { return undefined; } } function parseProviderJson(rawText: string, activeApiKey?: string): T { const parsed = tryParseProviderJson(rawText); if (parsed !== undefined) return parsed; const message = Buffer.byteLength(rawText, "utf8") > MAX_PROVIDER_BODY_BYTES ? PROVIDER_OVERSIZED_RESPONSE : PROVIDER_MALFORMED_RESPONSE; throw new Error(redactImageProviderText(message, activeApiKey)); } async function readProviderResponseText( response: Response, activeApiKey?: string, signal?: AbortSignal, ): Promise { const contentLength = response.headers.get("content-length"); if (contentLength !== null) { const declaredLength = Number(contentLength); if (Number.isFinite(declaredLength) && declaredLength > MAX_PROVIDER_BODY_BYTES) { await response.body?.cancel().catch(() => undefined); throw new Error(PROVIDER_OVERSIZED_RESPONSE); } } if (!response.body) return ""; const reader = response.body.getReader(); const decoder = new TextDecoder("utf-8"); const chunks: string[] = []; let totalBytes = 0; try { while (true) { const { done, value } = await reader.read(); if (done) break; if (!value || value.byteLength === 0) continue; if (totalBytes > MAX_PROVIDER_BODY_BYTES - value.byteLength) { await reader.cancel().catch(() => undefined); throw new Error(PROVIDER_OVERSIZED_RESPONSE); } totalBytes += value.byteLength; const decoded = decoder.decode(value, { stream: true }); if (decoded.length > 0) chunks.push(decoded); } const trailingText = decoder.decode(); if (trailingText.length > 0) chunks.push(trailingText); return chunks.join(""); } catch (error) { if (signal?.aborted) throw signal.reason ?? error; throw new Error(redactImageProviderText(error instanceof Error ? error.message : error, activeApiKey)); } finally { reader.releaseLock(); } } const PROVIDER_SSE_READ_OPTIONS = { maxEventBytes: MAX_PROVIDER_SSE_EVENT_BYTES, maxTotalBytes: MAX_PROVIDER_BODY_BYTES, } as const; async function fetchImageProvider( input: string | URL | Request, init: RequestInit, activeApiKey: string, ): Promise { try { return await fetch(input, init); } catch (error) { if (init.signal?.aborted) throw init.signal.reason ?? error; throw new Error(redactImageProviderText(error instanceof Error ? error.message : error, activeApiKey)); } } const ANTIGRAVITY_ENDPOINT = "https://daily-cloudcode-pa.sandbox.googleapis.com"; const ALIBABA_TOKEN_PLAN_HOST = "https://token-plan.ap-southeast-1.maas.aliyuncs.com"; const ALIBABA_IMAGE_GENERATION_URL = `${ALIBABA_TOKEN_PLAN_HOST}/api/v1/services/aigc/multimodal-generation/generation`; const IMAGE_SYSTEM_INSTRUCTION = "You are an AI image generator. Generate images based on user descriptions. Focus on creating high-quality, visually appealing images that match the user's request."; type ImageProvider = "alibaba" | "antigravity" | "gemini" | "openai" | "openai-codex" | "openrouter"; interface ImageApiKey { provider: ImageProvider; apiKey: string; projectId?: string; model?: Model; authCredentialType?: "api_key" | "oauth"; } export class UnsupportedImageProviderError extends Error { readonly code = "unsupported_image_provider" as const; constructor( readonly provider: string, readonly modelId: string, readonly api: Model["api"], ) { super(`Image generation is not supported for provider "${provider}" with transport "${api}".`); this.name = "UnsupportedImageProviderError"; } } const responseModalitySchema = z.enum(["IMAGE", "TEXT"] as const); const aspectRatioSchema = z.enum(["1:1", "3:4", "4:3", "9:16", "16:9"] as const).describe("aspect ratio"); const imageSizeSchema = z.enum(["1024x1024", "1536x1024", "1024x1536"] as const).describe("image size"); const inputImageSchema = z .object({ path: z.string().describe("input image path").optional(), data: z.string().describe("base64 image data").optional(), mime_type: z.string().describe("mime type").optional(), }) .strict(); const baseImageSchema = z .object({ subject: z.string().describe("main subject"), action: z.string().describe("what subject is doing").optional(), scene: z.string().describe("location or environment").optional(), composition: z.string().describe("camera angle and framing").optional(), lighting: z.string().describe("lighting setup").optional(), style: z.string().describe("artistic style").optional(), text: z.string().describe("text to render").optional(), changes: z.array(z.string()).describe("edits to make").optional(), aspect_ratio: aspectRatioSchema.optional(), image_size: imageSizeSchema.optional(), input: z.array(inputImageSchema).describe("input images").optional(), }) .strict(); export const imageGenSchema = baseImageSchema; export type ImageGenParams = z.infer; export type GeminiResponseModality = z.infer; /** * Assembles a structured prompt from the provided parameters. * For generation: builds "subject, action, scene. composition. lighting. camera. style." * For edits: appends change instructions and preserve directives. */ function assemblePrompt(params: ImageGenParams): string { const parts: string[] = []; // Core subject line: subject + action + scene const subjectParts = [params.subject]; if (params.action) subjectParts.push(params.action); if (params.scene) subjectParts.push(params.scene); parts.push(subjectParts.join(", ")); // Technical details as separate sentences if (params.composition) parts.push(params.composition); if (params.lighting) parts.push(params.lighting); if (params.style) parts.push(params.style); // Join with periods for sentence structure let prompt = `${parts.map(p => p.replace(/[.!,;:]+$/, "")).join(". ")}.`; // Text rendering specs if (params.text) { prompt += `\n\nText: ${params.text}`; } // Edit mode: changes and preserve directives if (params.changes?.length) { prompt += `\n\nChanges:\n${params.changes.map(c => `- ${c}`).join("\n")}`; } return prompt; } interface GeminiInlineData { data?: string; mimeType?: string; } interface GeminiPart { text?: string; inlineData?: GeminiInlineData; } interface GeminiCandidate { content?: { parts?: GeminiPart[] }; } interface GeminiSafetyRating { category?: string; probability?: string; } interface GeminiPromptFeedback { blockReason?: string; safetyRatings?: GeminiSafetyRating[]; } interface GeminiUsageMetadata { promptTokenCount?: number; candidatesTokenCount?: number; totalTokenCount?: number; } interface GeminiGenerateContentResponse { candidates?: GeminiCandidate[]; promptFeedback?: GeminiPromptFeedback; usageMetadata?: GeminiUsageMetadata; } interface OpenAIResponsesUsage { input_tokens?: number; output_tokens?: number; total_tokens?: number; } type ImageUsageMetadata = GeminiUsageMetadata | OpenAIResponsesUsage; type OpenAIImageAction = "edit" | "generate"; interface OpenAIInputTextContent { type: "input_text"; text: string; } interface OpenAIInputImageContent { type: "input_image"; detail: "auto"; image_url: string; } type OpenAIInputContent = OpenAIInputTextContent | OpenAIInputImageContent; interface OpenAIImageGenerationTool { type: "image_generation"; action: OpenAIImageAction; output_format: typeof OPENAI_IMAGE_OUTPUT_FORMAT; size?: string; } interface OpenAIHostedImageRequest { model: string; instructions?: string; input: Array<{ role: "user"; content: OpenAIInputContent[] }>; tools: OpenAIImageGenerationTool[]; tool_choice: { type: "image_generation" }; store: false; stream?: boolean; } interface OpenAIImageGenerationCall { id?: string; type: "image_generation_call"; result?: string; revised_prompt?: string; status?: string; } interface OpenAIOutputText { type: "output_text" | "refusal"; text?: string; refusal?: string; } interface OpenAIOutputMessage { id?: string; type: "message"; content?: OpenAIOutputText[]; } type OpenAIResponseOutput = OpenAIImageGenerationCall | OpenAIOutputMessage; interface OpenAIHostedImageResponse { output?: OpenAIResponseOutput[]; usage?: OpenAIResponsesUsage; error?: { code?: string; message?: string }; } interface OpenAISseEvent { type?: string; item?: OpenAIResponseOutput; response?: OpenAIHostedImageResponse; code?: string; message?: string; error?: { code?: string; message?: string }; } interface OpenAIHostedImageResult { images: InlineImageData[]; responseText?: string; revisedPrompt?: string; usage?: OpenAIResponsesUsage; } function redactOpenAIHostedImageResult(result: OpenAIHostedImageResult, activeApiKey: string): OpenAIHostedImageResult { return { ...result, responseText: result.responseText ? redactImageProviderText(result.responseText, activeApiKey) : undefined, revisedPrompt: result.revisedPrompt ? redactImageProviderText(result.revisedPrompt, activeApiKey) : undefined, }; } function redactGeminiPromptFeedback( promptFeedback: GeminiPromptFeedback | undefined, activeApiKey: string, ): GeminiPromptFeedback | undefined { if (!promptFeedback) return undefined; return { blockReason: promptFeedback.blockReason ? redactImageProviderText(promptFeedback.blockReason, activeApiKey) : undefined, safetyRatings: promptFeedback.safetyRatings?.map(rating => ({ category: rating.category ? redactImageProviderText(rating.category, activeApiKey) : undefined, probability: rating.probability ? redactImageProviderText(rating.probability, activeApiKey) : undefined, })), }; } interface OpenRouterImageUrl { url: string; } interface OpenRouterContentPart { type: "text" | "image_url"; text?: string; image_url?: OpenRouterImageUrl; } interface OpenRouterMessage { content?: string | OpenRouterContentPart[]; images?: Array; } interface OpenRouterChoice { message?: OpenRouterMessage; } interface OpenRouterResponse { choices?: OpenRouterChoice[]; } interface AntigravityRequest { project: string; model: string; request: { contents: Array<{ role: "user"; parts: Array<{ text?: string; inlineData?: InlineImageData }> }>; systemInstruction?: { parts: Array<{ text: string }> }; generationConfig?: { responseModalities?: GeminiResponseModality[]; imageConfig?: { aspectRatio?: string; imageSize?: string }; candidateCount?: number; }; safetySettings?: Array<{ category: string; threshold: string }>; }; requestType?: string; userAgent?: string; requestId?: string; } interface AntigravityResponseChunk { response?: { candidates?: Array<{ content?: { role: string; parts?: Array<{ text?: string; inlineData?: { mimeType?: string; data?: string }; }>; }; }>; usageMetadata?: GeminiUsageMetadata; }; } interface AlibabaImageContentPart { type?: string; text?: string; image?: string; } interface AlibabaImageResponse { code?: string; message?: string; output?: { choices?: Array<{ message?: { content?: AlibabaImageContentPart[] } }>; }; } /** Map the tool's image_size values onto wan2.7's 1K/2K size classes. */ export function resolveAlibabaImageSize(imageSize: string | undefined): string | undefined { switch (imageSize) { case "1024x1024": return "1K"; case "1536x1024": case "1024x1536": return "2K"; default: return undefined; } } /** * Build the Bailian multimodal-generation request body for wan2.7 image * generation/editing. Input images ride along as data URLs; generation-only * calls send just the prompt text. The sync endpoint returns OSS URLs inline. */ export function buildAlibabaImageRequest( model: string, promptText: string, inputImages: InlineImageData[], imageSize: string | undefined, ): { model: string; input: { messages: Array<{ role: "user"; content: Array<{ text: string } | { image: string }> }> }; parameters: { n: number; watermark: boolean; size?: string }; } { const content: Array<{ text: string } | { image: string }> = []; for (const image of inputImages) { content.push({ image: toDataUrl(image) }); } content.push({ text: promptText }); const size = resolveAlibabaImageSize(imageSize); return { model, input: { messages: [{ role: "user", content }] }, parameters: { n: 1, watermark: false, ...(size ? { size } : {}) }, }; } /** Collect OSS image URLs and any text parts from a Bailian image response. */ export function collectAlibabaImageResult(response: AlibabaImageResponse): { imageUrls: string[]; responseText?: string; } { const imageUrls: string[] = []; const textParts: string[] = []; for (const choice of response.output?.choices ?? []) { for (const part of choice.message?.content ?? []) { if (part.image) { imageUrls.push(part.image); } else if (part.text) { textParts.push(part.text); } } } const responseText = textParts.join("\n").trim(); return { imageUrls, responseText: responseText.length > 0 ? responseText : undefined }; } interface ImageGenToolDetails { provider: ImageProvider; model: string; imageCount: number; imagePaths: string[]; images: InlineImageData[]; responseText?: string; promptFeedback?: GeminiPromptFeedback; revisedPrompt?: string; usage?: ImageUsageMetadata; } interface ImageInput { path?: string; data?: string; mime_type?: string; } interface InlineImageData { data: string; mimeType: string; } function normalizeDataUrl(data: string): { data: string; mimeType?: string } { const match = data.match(/^data:([^;]+);base64,(.+)$/); if (!match) return { data }; return { data: match[2] ?? "", mimeType: match[1] }; } function resolveOpenRouterModel(model: string): string { return model.includes("/") ? model : `google/${model}`; } function toDataUrl(image: InlineImageData): string { return `data:${image.mimeType};base64,${image.data}`; } function withAbortSignal(operation: Promise, signal: AbortSignal | undefined): Promise { if (!signal) return operation; if (signal.aborted) return Promise.reject(signal.reason); const deferred = Promise.withResolvers(); const abort = () => deferred.reject(signal.reason); signal.addEventListener("abort", abort, { once: true }); operation.then(deferred.resolve, deferred.reject).finally(() => signal.removeEventListener("abort", abort)); return deferred.promise; } function normalizePeerAddress(address: string): string { const normalized = address.toLowerCase(); return normalized.startsWith("::ffff:") ? normalized.slice(7) : normalized; } function normalizeUrlHostname(hostname: string): string { return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; } function responseHeaderByteLength(response: http.IncomingMessage): number { let bytes = Buffer.byteLength( `HTTP/${response.httpVersion} ${response.statusCode ?? ""}${response.statusMessage ? ` ${response.statusMessage}` : ""}\r\n`, ); for (let index = 0; index < response.rawHeaders.length; index += 2) { bytes += Buffer.byteLength(`${response.rawHeaders[index] ?? ""}: ${response.rawHeaders[index + 1] ?? ""}\r\n`); } return bytes + Buffer.byteLength("\r\n"); } async function validateImageUrl(rawUrl: string, signal: AbortSignal | undefined) { return withAbortSignal(validatePublicHttpUrl(rawUrl), signal); } function openImageResponse( url: URL, addresses: string[], signal: AbortSignal | undefined, ): Promise { const hostname = normalizeUrlHostname(url.hostname); const approved = addresses.map(address => ({ address, family: net.isIP(address) })); const approvedPeers = new Set(approved.map(record => normalizePeerAddress(record.address))); const lookup: net.LookupFunction = (requestedHostname, options, callback) => { const requestedFamily = options.family === "IPv4" ? 4 : options.family === "IPv6" ? 6 : (options.family ?? 0); const matching = approved.filter(record => requestedFamily === 0 || record.family === requestedFamily); if (normalizeUrlHostname(requestedHostname) !== hostname || matching.length === 0) { const error = Object.assign(new Error("No approved address for image host"), { code: "ENOTFOUND" }); callback(error, options.all ? [] : "", 0); return; } if (options.all) callback(null, matching); else callback(null, matching[0].address, matching[0].family); }; const deferred = Promise.withResolvers(); const options: https.RequestOptions = { protocol: url.protocol, hostname, port: url.port || undefined, path: `${url.pathname}${url.search}`, method: "GET", headers: { Accept: "image/*", "Accept-Encoding": "identity", Connection: "close", Host: url.host }, agent: false, insecureHTTPParser: false, lookup, maxHeaderSize: MAX_IMAGE_HEADER_SIZE, signal, ...(url.protocol === "https:" ? { rejectUnauthorized: true, servername: net.isIP(hostname) === 0 ? hostname : undefined } : {}), }; const requestFn = url.protocol === "https:" ? https.request : http.request; const request = requestFn(options, response => { if (responseHeaderByteLength(response) > MAX_IMAGE_HEADER_SIZE) { response.destroy(); deferred.reject(new Error("Image response headers exceed the maximum size of 16 KiB")); return; } const peer = response.socket.remoteAddress; if (peer && (isPrivateOrSpecialAddress(peer) || !approvedPeers.has(normalizePeerAddress(peer)))) { response.destroy(); deferred.reject(new Error("Refusing image response from an unapproved connected peer")); return; } deferred.resolve(response); }); request.once("error", deferred.reject); const abort = () => { request.destroy(signal?.reason); deferred.reject(signal?.reason); }; if (signal?.aborted) abort(); else signal?.addEventListener("abort", abort, { once: true }); request.once("close", () => signal?.removeEventListener("abort", abort)); request.end(); return deferred.promise; } function validateImageResponseFraming(response: http.IncomingMessage): void { const rawContentLengths: string[] = []; let hasTransferEncoding = false; for (let index = 0; index < response.rawHeaders.length; index += 2) { const name = response.rawHeaders[index]?.toLowerCase(); if (name === "content-length") rawContentLengths.push(response.rawHeaders[index + 1] ?? ""); if (name === "transfer-encoding") hasTransferEncoding = true; } if (rawContentLengths.length > 1 || (rawContentLengths.length > 0 && hasTransferEncoding)) { response.destroy(); throw new Error("Image response has ambiguous framing"); } const contentLength = response.headers["content-length"]; if (contentLength === undefined) return; const declaredBytes = typeof contentLength === "string" && /^\d+$/.test(contentLength) ? Number(contentLength) : NaN; if (!Number.isSafeInteger(declaredBytes)) { response.destroy(); throw new Error("Image response has an invalid Content-Length"); } if (declaredBytes > MAX_IMAGE_SIZE) { response.destroy(); throw new Error("Image response exceeds the maximum size of 35 MiB"); } } async function readImageResponse( response: http.IncomingMessage, maxBytes: number, signal: AbortSignal | undefined, ): Promise { const abort = () => response.destroy(signal?.reason); if (signal?.aborted) { abort(); throw signal.reason; } signal?.addEventListener("abort", abort, { once: true }); const chunks: Buffer[] = []; let receivedBytes = 0; try { for await (const chunk of response) { const bytes = Buffer.from(chunk); receivedBytes += bytes.byteLength; if (receivedBytes > maxBytes) { response.destroy(); throw new Error( maxBytes === MAX_IMAGE_SIZE ? "Image response exceeds the maximum size of 35 MiB" : "Image download error response exceeded the preview limit", ); } chunks.push(bytes); } return Buffer.concat(chunks, receivedBytes); } finally { signal?.removeEventListener("abort", abort); } } async function loadImageFromUrl( imageUrl: string, signal?: AbortSignal, activeApiKey?: string, ): Promise { if (imageUrl.startsWith("data:")) { const normalized = normalizeDataUrl(imageUrl.trim()); if (!normalized.mimeType) { throw new Error("mime_type is required when providing raw base64 data."); } if (!normalized.data) { throw new Error("Image data is empty."); } return { data: normalized.data, mimeType: normalized.mimeType }; } let currentUrl = imageUrl; for (let redirectCount = 0; ; redirectCount++) { const guard = await validateImageUrl(currentUrl, signal); if (!guard.ok) { throw new Error(`Refusing image URL: target URL is not public HTTP(S): ${guard.reason}`); } const response = await openImageResponse(guard.url, guard.addresses, signal); const status = response.statusCode ?? 0; if (IMAGE_REDIRECT_STATUSES.has(status)) { if (redirectCount >= MAX_IMAGE_REDIRECTS) { response.destroy(); throw new Error("Too many redirects downloading image"); } const location = response.headers.location; response.destroy(); if (!location) throw new Error("Image redirect is missing a Location header"); currentUrl = new URL(location, guard.url).toString(); continue; } validateImageResponseFraming(response); if (status < 200 || status >= 300) { const preview = redactImageProviderText( (await readImageResponse(response, MAX_IMAGE_ERROR_PREVIEW_SIZE, signal)).toString("utf8"), activeApiKey, ); throw new Error(`Image download failed (${status}): ${preview || "provider returned an error"}`); } const rawContentType = response.headers["content-type"]; const contentType = typeof rawContentType === "string" ? rawContentType.split(";", 1)[0]?.trim().toLowerCase() : undefined; if (!contentType?.startsWith("image/")) { response.destroy(); throw new Error("Image response is missing a supported image Content-Type"); } const buffer = await readImageResponse(response, MAX_IMAGE_SIZE, signal); return { data: buffer.toBase64(), mimeType: contentType }; } } function collectOpenRouterResponseText(message: OpenRouterMessage | undefined): string | undefined { if (!message) return undefined; if (typeof message.content === "string") { const trimmed = message.content.trim(); return trimmed.length > 0 ? trimmed : undefined; } if (Array.isArray(message.content)) { const texts = message.content .filter(part => part.type === "text") .map(part => part.text) .filter((text): text is string => Boolean(text)); const combined = texts.join("\n").trim(); return combined.length > 0 ? combined : undefined; } return undefined; } function extractOpenRouterImageUrls(message: OpenRouterMessage | undefined): string[] { const urls: string[] = []; if (!message) return urls; for (const image of message.images ?? []) { if (typeof image === "string") { urls.push(image); continue; } if (image.image_url?.url) { urls.push(image.image_url.url); } } if (Array.isArray(message.content)) { for (const part of message.content) { if (part.type === "image_url" && part.image_url?.url) { urls.push(part.image_url.url); } } } return urls; } /** Provider → default image model mapping for fallbacks */ export const IMAGE_PROVIDER_DEFAULTS: Record = { openai: "gpt-image-2", alibaba: "wan2.7-image", "openai-codex": "gpt-image-2", antigravity: "gemini-3-pro-image", gemini: "gemini-3-pro-image-preview", openrouter: "google/gemini-3-pro-image-preview", }; /** * Resolve the image-generation model from the `modelRoles.image` settings entry. * Returns the resolved Model (with provider identity) or undefined when no * image role is configured / no matching model is available. */ export function resolveImageRoleModel( settings: ModelRoleSettings, modelRegistry: ModelRegistry, options?: { sessionId?: string; credentialSessionId?: string }, ): Model | undefined { const roleValue = settings.getModelRole("image"); if (!roleValue) return undefined; const availableModels = modelRegistry.getAvailable(); const resolved = resolveModelRoleValue(roleValue, availableModels, { settings, modelRegistry, sessionId: options?.sessionId, credentialSessionId: options?.credentialSessionId, }); return resolved.model; } /** Resolve the effective image model for a configured provider */ export function resolveImageModel(provider: string, modelOverride: string | null): string { if (modelOverride) return modelOverride; return IMAGE_PROVIDER_DEFAULTS[provider] ?? DEFAULT_MODEL; } interface ParsedAntigravityCredentials { accessToken: string; projectId?: string; } function parseAntigravityCredentials(raw: string): ParsedAntigravityCredentials | null { try { const parsed = JSON.parse(raw) as { token?: string; accessToken?: string; projectId?: string }; const token = parsed.token ?? parsed.accessToken; if (typeof token === "string" && token.trim().length > 0) { return { accessToken: token.trim(), projectId: parsed.projectId }; } // Parsed as JSON but no usable token field. return null; } catch { // Not JSON: treat the value as a raw bearer token. } const rawToken = raw.trim(); return rawToken.length > 0 ? { accessToken: rawToken } : null; } async function findAntigravityCredentials( modelRegistry: ModelRegistry, sessionId?: string, ): Promise { const oauthAccess = await modelRegistry.authStorage.getOAuthAccess("google-antigravity", sessionId); if (oauthAccess?.accessToken) { return { provider: "antigravity", apiKey: oauthAccess.accessToken, projectId: oauthAccess.projectId, }; } const apiKey = await modelRegistry.getApiKeyForProvider("google-antigravity", sessionId); if (!apiKey) return null; const parsed = parseAntigravityCredentials(apiKey); if (!parsed) return null; return { provider: "antigravity", apiKey: parsed.accessToken, projectId: parsed.projectId, }; } async function findAlibabaImageCredentials( modelRegistry: ModelRegistry | undefined, sessionId?: string, ): Promise { const envKey = getEnvApiKey("alibaba-token-plan"); if (envKey) return { provider: "alibaba", apiKey: envKey }; if (!modelRegistry) return null; const apiKey = await modelRegistry.getApiKeyForProvider("alibaba-token-plan", sessionId); if (!isAuthenticated(apiKey)) return null; return { provider: "alibaba", apiKey }; } /** * Google image-provider key fallback, from trusted environment sources only. * * `$env` merges the caller's `cwd/.env`, so reading the key there would let * repository content supply the credential these image requests authenticate * with. Provider credentials resolve from the launching shell plus GJC/user-owned * `.env` files, never the project `.env`. */ function googleImageApiKeyFromEnv(): string | undefined { return $credentialEnv("GOOGLE_API_KEY"); } /** Test seam: the Google image key fallback as resolved from trusted env. */ export function googleImageApiKeyFromEnvForTest(): string | undefined { return googleImageApiKeyFromEnv(); } /** * Resolve image API credentials from the `modelRoles.image` model. * * The resolved model's provider identity determines which credential path * is used. Returns null when no image role model is configured or the * corresponding credentials are unavailable. */ async function findImageApiKey( modelRegistry: ModelRegistry, settings: ModelRoleSettings, sessionId?: string, credentialSessionId?: string, ): Promise { const imageModel = resolveImageRoleModel(settings, modelRegistry, { sessionId, credentialSessionId }); if (!imageModel) return null; return resolveCredentialsForImageModel(imageModel, modelRegistry, sessionId); } /** * Map a resolved image model to its provider credentials. Transport and image * capability are validated before any credential lookup; unsupported models * fail without forwarding their provider credential to another endpoint. */ async function resolveCredentialsForImageModel( model: Model, modelRegistry: ModelRegistry, sessionId?: string, ): Promise { const provider: string = model.provider; // OpenAI-hosted image generation (gpt-image via Responses API, including // OpenAI-compatible proxies whose models declare output:image). if (isOpenAIHostedImageModel(model)) { const apiKey = await modelRegistry.getApiKey(model, sessionId); if (!isAuthenticated(apiKey)) return null; return { provider: getOpenAIHostedImageProvider(model), apiKey, model, authCredentialType: modelRegistry.getSessionCredentialType?.(model.provider, sessionId), }; } if ( (provider === "antigravity" || provider === "google-antigravity") && model.api === "google-gemini-cli" && model.output?.includes("image") ) { return await findAntigravityCredentials(modelRegistry, sessionId); } if ((provider === "alibaba" || provider === "alibaba-token-plan") && model.output?.includes("image")) { return await findAlibabaImageCredentials(modelRegistry, sessionId); } if (provider === "openrouter" && model.output?.includes("image")) { const openRouterKey = await modelRegistry.getApiKey(model, sessionId); if (isAuthenticated(openRouterKey)) return { provider: "openrouter", apiKey: openRouterKey }; const envKey = getEnvApiKey("openrouter"); return envKey ? { provider: "openrouter", apiKey: envKey } : null; } if (provider === "google" && model.api === "google-generative-ai" && model.output?.includes("image")) { const googleKey = await modelRegistry.getApiKey(model, sessionId); if (isAuthenticated(googleKey)) return { provider: "gemini", apiKey: googleKey }; const envKey = getEnvApiKey("google"); if (envKey) return { provider: "gemini", apiKey: envKey }; const fallbackKey = googleImageApiKeyFromEnv(); return fallbackKey ? { provider: "gemini", apiKey: fallbackKey } : null; } throw new UnsupportedImageProviderError(model.provider, model.id, model.api); } async function loadImageFromPath(imagePath: string, cwd: string): Promise { const resolved = resolveReadPath(imagePath, cwd); try { const buffer = await Bun.file(resolved).bytes(); if (buffer.length > MAX_IMAGE_SIZE) { throw new Error(`Image file too large: ${imagePath}`); } const metadata = parseImageMetadata(buffer); const mimeType = metadata?.mimeType; if (!mimeType) { throw new Error(`Unsupported image type: ${imagePath}`); } return { data: buffer.toBase64(), mimeType }; } catch (err) { if (isEnoent(err)) throw new Error(`Image file not found: ${imagePath}`); throw err; } } async function resolveInputImage(input: ImageInput, cwd: string): Promise { if (input.path) { return loadImageFromPath(input.path, cwd); } if (input.data) { const normalized = normalizeDataUrl(input.data.trim()); const mimeType = normalized.mimeType ?? input.mime_type; if (!mimeType) { throw new Error("mime_type is required when providing raw base64 data."); } if (!normalized.data) { throw new Error("Image data is empty."); } return { data: normalized.data, mimeType }; } throw new Error("input_images entries must include either path or data."); } function getExtensionForMime(mimeType: string): string { const map: Record = { "image/png": "png", "image/jpeg": "jpg", "image/gif": "gif", "image/webp": "webp", }; return map[mimeType] ?? "png"; } async function saveImageToTemp(image: InlineImageData): Promise { const ext = getExtensionForMime(image.mimeType); const filename = `gjc-image-${Snowflake.next()}.${ext}`; const filepath = path.join(os.tmpdir(), filename); await Bun.write(filepath, Buffer.from(image.data, "base64")); return filepath; } async function saveImagesToTemp(images: InlineImageData[]): Promise { return Promise.all(images.map(saveImageToTemp)); } function buildResponseSummary( provider: ImageProvider, model: string, imagePaths: string[], responseText: string | undefined, ): string { const lines = [`Provider: ${provider}`, `Model: ${model}`, `Generated ${imagePaths.length} image(s):`]; for (const p of imagePaths) { lines.push(` ${p}`); } if (responseText) { lines.push("", responseText.trim()); } return lines.join("\n"); } function collectResponseText(parts: GeminiPart[]): string | undefined { const texts = parts.map(part => part.text).filter((text): text is string => Boolean(text)); const combined = texts.join("\n").trim(); return combined.length > 0 ? combined : undefined; } function collectInlineImages(parts: GeminiPart[]): InlineImageData[] { const images: InlineImageData[] = []; for (const part of parts) { const data = part.inlineData?.data; const mimeType = part.inlineData?.mimeType; if (!data || !mimeType) continue; images.push({ data, mimeType }); } return images; } export function isOpenAIHostedImageModel(model: Model | undefined): boolean { if (!model) return false; // The hosted image_generation tool is only available over the Responses API. if (model.api !== "openai-responses" && model.api !== "openai-codex-responses") return false; // Declarative capability: any provider (e.g. an OpenAI-compatible proxy // fronting gpt-image) whose model advertises image output can drive // generate_image, routed to the model's own baseUrl with registry auth. if (model.output?.includes("image")) return true; // First-party heuristic: OpenAI/OpenAI code GPT and o3 models generate // images inline through the hosted tool without a declared output modality. if (model.provider === "openai" || model.provider === "openai-codex") { const modelId = model.id.toLowerCase(); return modelId.startsWith("gpt-") || modelId === "o3" || modelId.startsWith("o3-"); } return false; } function getOpenAIHostedImageProvider(model: Model): ImageProvider { return model.api === "openai-codex-responses" || model.provider === "openai-codex" ? "openai-codex" : "openai"; } function resolveOpenAIImageSize(aspectRatio: string | undefined, imageSize: string | undefined): string | undefined { if (imageSize) return imageSize; switch (aspectRatio) { case "1:1": return "1024x1024"; case "3:4": case "9:16": return "1024x1536"; case "4:3": case "16:9": return "1536x1024"; default: return undefined; } } function buildOpenAIHostedImageRequest( model: Model, promptText: string, params: ImageGenParams, inputImages: InlineImageData[], stream: boolean, ): OpenAIHostedImageRequest { const content: OpenAIInputContent[] = [{ type: "input_text", text: promptText }]; for (const image of inputImages) { content.push({ type: "input_image", detail: "auto", image_url: toDataUrl(image) }); } const size = resolveOpenAIImageSize(params.aspect_ratio, params.image_size); const tool: OpenAIImageGenerationTool = { type: "image_generation", action: inputImages.length > 0 ? "edit" : "generate", output_format: OPENAI_IMAGE_OUTPUT_FORMAT, ...(size ? { size } : {}), }; return { model: model.id, input: [{ role: "user", content }], tools: [tool], tool_choice: { type: "image_generation" }, store: false, ...(stream ? { instructions: "You are an AI image generator. Generate images based on user descriptions. Focus on creating high-quality, visually appealing images that match the user's request.", } : {}), ...(stream ? { stream: true } : {}), }; } function createOpenAIInlineImage(data: string): InlineImageData { const bytes = Buffer.from(data, "base64"); const mimeType = parseImageMetadata(bytes)?.mimeType ?? OPENAI_IMAGE_MIME_TYPE; return { data, mimeType }; } function collectOpenAIHostedImageResult(response: OpenAIHostedImageResponse): OpenAIHostedImageResult { const images: InlineImageData[] = []; const textParts: string[] = []; let revisedPrompt: string | undefined; for (const output of response.output ?? []) { if (output.type === "image_generation_call") { if (output.result) { images.push(createOpenAIInlineImage(output.result)); } if (output.revised_prompt) { revisedPrompt = output.revised_prompt; } continue; } for (const part of output.content ?? []) { if (part.type === "output_text" && part.text) { textParts.push(part.text); } else if (part.type === "refusal" && part.refusal) { textParts.push(part.refusal); } } } const responseText = textParts.join("\n").trim(); return { images, revisedPrompt, responseText: responseText.length > 0 ? responseText : undefined, usage: response.usage, }; } function getOpenAIResponseErrorMessage(rawText: string, activeApiKey?: string): string { const parsed = tryParseProviderJson<{ error?: { message?: string } }>(rawText); return redactImageProviderText(parsed?.error?.message ?? "provider returned an error", activeApiKey); } function getOpenAIBaseUrl(model: Model, authCredentialType?: "api_key" | "oauth"): string { if (model.api === "openai-codex-responses" || model.provider === "openai-codex") { return (model.baseUrl || CODEX_BASE_URL).replace(/\/+$/, ""); } if (authCredentialType === "oauth") return DEFAULT_OPENAI_BASE_URL; // Trusted sources only: this base URL becomes the endpoint for authenticated // image requests, and `$env` merges the caller's `cwd/.env`. const envBaseUrl = $credentialEnv("OPENAI_BASE_URL"); const configuredBaseUrl = model.baseUrl?.trim(); if (envBaseUrl && (!configuredBaseUrl || configuredBaseUrl.toLowerCase().includes("api.openai.com"))) { return envBaseUrl.replace(/\/+$/, ""); } return (configuredBaseUrl || envBaseUrl || DEFAULT_OPENAI_BASE_URL).replace(/\/+$/, ""); } /** Test seam: the image-request base URL as resolved from trusted env. */ export function getOpenAIImageBaseUrlForTest(model: Model, authCredentialType?: "api_key" | "oauth"): string { return getOpenAIBaseUrl(model, authCredentialType); } function getOpenAIResponsesUrl(model: Model, authCredentialType?: "api_key" | "oauth"): string { const baseUrl = getOpenAIBaseUrl(model, authCredentialType); if (model.api !== "openai-codex-responses" && model.provider !== "openai-codex") { return `${baseUrl}/responses`; } const baseWithSlash = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`; return new URL(URL_PATHS.RESPONSES.slice(1), baseWithSlash) .toString() .replace(URL_PATHS.RESPONSES, URL_PATHS.CODEX_RESPONSES); } function buildOpenAIImageHeaders(model: Model, apiKey: string, sessionId: string | undefined): Headers { const headers = new Headers(model.headers ?? {}); headers.set("Content-Type", "application/json"); headers.set("Authorization", `Bearer ${apiKey}`); if (model.api === "openai-codex-responses" || model.provider === "openai-codex") { const accountId = getCodexAccountId(apiKey); if (!accountId) { throw new Error("Failed to extract accountId from OpenAI Codex token"); } headers.delete("x-api-key"); headers.set(OPENAI_HEADERS.ACCOUNT_ID, accountId); headers.set(OPENAI_HEADERS.BETA, OPENAI_HEADER_VALUES.BETA_RESPONSES); headers.set(OPENAI_HEADERS.ORIGINATOR, OPENAI_HEADER_VALUES.ORIGINATOR_CODEX); headers.set( "User-Agent", `pi/${packageJson.version} (${sanitizeHeaderComponent(os.platform())} ${sanitizeHeaderComponent(os.release())}; ${sanitizeHeaderComponent(os.arch())})`, ); if (sessionId) { headers.set(OPENAI_HEADERS.CONVERSATION_ID, sessionId); headers.set(OPENAI_HEADERS.SESSION_ID, sessionId); } } return headers; } async function parseOpenAIHostedImageSse( response: Response, signal: AbortSignal | undefined, activeApiKey: string, ): Promise { if (!response.body) { throw new Error("No response body"); } const fallbackOutput: OpenAIResponseOutput[] = []; let completedResponse: OpenAIHostedImageResponse | undefined; try { for await (const event of readSseJson( response.body, signal, undefined, PROVIDER_SSE_READ_OPTIONS, )) { if (!event || typeof event !== "object" || Array.isArray(event)) throw new Error(PROVIDER_MALFORMED_RESPONSE); if (event.type === "error") { const message = event.error?.message ?? event.message ?? "OpenAI image request failed"; throw new Error(redactImageProviderText(message, activeApiKey)); } if (event.type === "response.failed") { const message = event.response?.error?.message ?? "OpenAI image request failed"; throw new Error(redactImageProviderText(message, activeApiKey)); } if (event.type === "response.output_item.done" && event.item) { fallbackOutput.push(event.item); } if ((event.type === "response.completed" || event.type === "response.done") && event.response) { completedResponse = event.response; } } } catch (error) { if (signal?.aborted) throw signal.reason ?? error; if (error instanceof Error && error.message !== PROVIDER_MALFORMED_RESPONSE) throw error; throw new Error(redactImageProviderText(PROVIDER_MALFORMED_RESPONSE, activeApiKey)); } return collectOpenAIHostedImageResult( completedResponse?.output?.length ? completedResponse : { output: fallbackOutput, usage: completedResponse?.usage }, ); } async function generateOpenAIHostedImage( apiKey: string, model: Model, params: ImageGenParams, inputImages: InlineImageData[], signal: AbortSignal | undefined, sessionId: string | undefined, options?: { authCredentialType?: "api_key" | "oauth" }, ): Promise { const promptText = assemblePrompt(params); const stream = model.api === "openai-codex-responses" || model.provider === "openai-codex"; const requestBody = buildOpenAIHostedImageRequest(model, promptText, params, inputImages, stream); const response = await fetchImageProvider( getOpenAIResponsesUrl(model, options?.authCredentialType), { method: "POST", headers: buildOpenAIImageHeaders(model, apiKey, sessionId), body: JSON.stringify(requestBody), signal, }, apiKey, ); if (!response.ok) { const errorText = await readProviderResponseText(response, apiKey, signal); throw new Error( `OpenAI image request failed (${response.status}): ${getOpenAIResponseErrorMessage(errorText, apiKey)}`, ); } const contentType = response.headers.get("content-type") ?? ""; if (stream || contentType.includes("text/event-stream")) { return redactOpenAIHostedImageResult(await parseOpenAIHostedImageSse(response, signal, apiKey), apiKey); } const rawText = await readProviderResponseText(response, apiKey, signal); const data = parseProviderJson(rawText, apiKey); return redactOpenAIHostedImageResult(collectOpenAIHostedImageResult(data), apiKey); } function combineParts(response: GeminiGenerateContentResponse): GeminiPart[] { const parts: GeminiPart[] = []; for (const candidate of response.candidates ?? []) { const candidateParts = candidate.content?.parts ?? []; parts.push(...candidateParts); } return parts; } function buildAntigravityRequest( prompt: string, model: string, projectId: string, aspectRatio: string | undefined, imageSize: string | undefined, inputImages: InlineImageData[], ): AntigravityRequest { const parts: Array<{ text?: string; inlineData?: InlineImageData }> = []; for (const image of inputImages) { parts.push({ inlineData: image }); } parts.push({ text: prompt }); const imageConfig = aspectRatio || imageSize ? { aspectRatio: aspectRatio, imageSize: imageSize } : undefined; return { project: projectId, model, request: { contents: [{ role: "user", parts }], systemInstruction: { parts: [{ text: IMAGE_SYSTEM_INSTRUCTION }] }, generationConfig: { responseModalities: ["IMAGE"], imageConfig, candidateCount: 1, }, safetySettings: [ { category: "HARM_CATEGORY_HARASSMENT", threshold: "BLOCK_ONLY_HIGH" }, { category: "HARM_CATEGORY_HATE_SPEECH", threshold: "BLOCK_ONLY_HIGH" }, { category: "HARM_CATEGORY_SEXUALLY_EXPLICIT", threshold: "BLOCK_ONLY_HIGH" }, { category: "HARM_CATEGORY_DANGEROUS_CONTENT", threshold: "BLOCK_ONLY_HIGH" }, { category: "HARM_CATEGORY_CIVIC_INTEGRITY", threshold: "BLOCK_ONLY_HIGH" }, ], }, requestType: "agent", requestId: `agent-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`, userAgent: "antigravity", }; } interface AntigravitySseResult { images: InlineImageData[]; text: string[]; usage?: GeminiUsageMetadata; } async function parseAntigravitySseForImage( response: Response, signal: AbortSignal | undefined, activeApiKey?: string, ): Promise { if (!response.body) { throw new Error("No response body"); } const textParts: string[] = []; const images: InlineImageData[] = []; let usage: GeminiUsageMetadata | undefined; let chunks: AntigravityResponseChunk[]; try { chunks = []; for await (const chunk of readSseJson( response.body, signal, undefined, PROVIDER_SSE_READ_OPTIONS, )) { if (!chunk || typeof chunk !== "object" || Array.isArray(chunk)) throw new Error(PROVIDER_MALFORMED_RESPONSE); chunks.push(chunk); } } catch (error) { if (signal?.aborted) throw signal.reason ?? error; throw new Error(redactImageProviderText(PROVIDER_MALFORMED_RESPONSE, activeApiKey)); } for (const chunk of chunks) { const responseData = chunk.response; if (!responseData || typeof responseData !== "object") continue; const candidates = Array.isArray(responseData.candidates) ? responseData.candidates : []; for (const candidate of candidates) { if (!candidate || typeof candidate !== "object") continue; const parts = Array.isArray(candidate.content?.parts) ? candidate.content.parts : []; for (const part of parts) { if (!part || typeof part !== "object") continue; if (typeof part.text === "string" && part.text.length > 0) { textParts.push(part.text); } const inlineData = part.inlineData; if (inlineData && typeof inlineData === "object" && inlineData.data && inlineData.mimeType) { images.push({ data: inlineData.data, mimeType: inlineData.mimeType }); } } } if (responseData.usageMetadata && typeof responseData.usageMetadata === "object") { usage = responseData.usageMetadata; } } return { images, text: textParts, usage }; } export const imageGenTool: CustomTool = { name: "generate_image", label: "GenerateImage", strict: false, description: prompt.render(imageGenDescription), parameters: imageGenSchema, async execute(_toolCallId, params, _onUpdate, ctx, signal) { return untilAborted(signal, async () => { const sessionId = ctx.credentialSessionId ?? ctx.sessionManager.getSessionId(); const settings = ctx.settings; if (!settings) { throw new Error("Image generation requires session settings to resolve the image model role."); } const apiKey = await findImageApiKey(ctx.modelRegistry, settings, sessionId, ctx.credentialSessionId); if (!apiKey) { throw new Error( "No image model configured. Set an image-capable model via /model (image role) to enable image generation.", ); } const provider = apiKey.provider; const imageModel = apiKey.model; const model = imageModel ? imageModel.id : provider === "antigravity" ? resolveImageModel("antigravity", null) : provider === "alibaba" ? resolveImageModel("alibaba", null) : provider === "openrouter" ? resolveImageModel("openrouter", null) : resolveImageModel("gemini", null); const resolvedModel = provider === "openrouter" ? resolveOpenRouterModel(model) : model; const cwd = ctx.sessionManager.getCwd(); const resolvedImages: InlineImageData[] = []; if (params.input?.length) { for (const input of params.input) { resolvedImages.push(await resolveInputImage(input, cwd)); } } const requestSignal = ptree.combineSignals(signal, IMAGE_TIMEOUT); if (provider === "openai" || provider === "openai-codex") { if (!apiKey.model) { throw new Error("Missing active GPT model for OpenAI image generation"); } const parsed = await generateOpenAIHostedImage( apiKey.apiKey, apiKey.model, params, resolvedImages, requestSignal, sessionId, { authCredentialType: apiKey.authCredentialType }, ); if (parsed.images.length === 0) { const messageText = parsed.responseText ? `\n\n${parsed.responseText}` : ""; return { content: [{ type: "text", text: `No image data returned.${messageText}` }], details: { provider, model, imageCount: 0, imagePaths: [], images: [], responseText: parsed.responseText, revisedPrompt: parsed.revisedPrompt, usage: parsed.usage, }, }; } const imagePaths = await saveImagesToTemp(parsed.images); return { content: [ { type: "text", text: buildResponseSummary(provider, model, imagePaths, parsed.responseText) }, ], details: { provider, model, imageCount: parsed.images.length, imagePaths, images: parsed.images, responseText: parsed.responseText, revisedPrompt: parsed.revisedPrompt, usage: parsed.usage, }, }; } if (provider === "antigravity") { if (!apiKey.projectId) { throw new Error( "Antigravity image generation requires a projectId, but the stored google-antigravity credential only contains an access token. Run the google-antigravity login flow again so the projectId is stored, then retry.", ); } const prompt = assemblePrompt(params); const requestBody = buildAntigravityRequest( prompt, model, apiKey.projectId, params.aspect_ratio, params.image_size, resolvedImages, ); const { getAntigravityUserAgent } = require("@gajae-code/ai/providers/google-gemini-headers") as typeof import("@gajae-code/ai/providers/google-gemini-headers"); const response = await fetchImageProvider( `${ANTIGRAVITY_ENDPOINT}/v1internal:streamGenerateContent?alt=sse`, { method: "POST", headers: { Authorization: `Bearer ${apiKey.apiKey}`, "Content-Type": "application/json", Accept: "text/event-stream", "User-Agent": getAntigravityUserAgent(), }, body: JSON.stringify(requestBody), signal: requestSignal, }, apiKey.apiKey, ); if (!response.ok) { const errorText = await readProviderResponseText(response, apiKey.apiKey, requestSignal); const parsed = tryParseProviderJson<{ error?: { message?: string } }>(errorText); const message = parsed?.error?.message; throw new Error( `Antigravity image request failed (${response.status}): ${redactImageProviderText(message ?? "provider returned an error", apiKey.apiKey)}`, ); } const parsed = await parseAntigravitySseForImage(response, requestSignal, apiKey.apiKey); const responseText = parsed.text.length > 0 ? redactImageProviderText(parsed.text.join(" "), apiKey.apiKey) : undefined; if (parsed.images.length === 0) { const messageText = responseText ? `\n\n${responseText}` : ""; return { content: [{ type: "text", text: `No image data returned.${messageText}` }], details: { provider, model, imageCount: 0, imagePaths: [], images: [], responseText, usage: parsed.usage, }, }; } const imagePaths = await saveImagesToTemp(parsed.images); return { content: [{ type: "text", text: buildResponseSummary(provider, model, imagePaths, responseText) }], details: { provider, model, imageCount: parsed.images.length, imagePaths, images: parsed.images, responseText, usage: parsed.usage, }, }; } if (provider === "openrouter") { const prompt = assemblePrompt(params); const contentParts: OpenRouterContentPart[] = [{ type: "text", text: prompt }]; for (const image of resolvedImages) { contentParts.push({ type: "image_url", image_url: { url: toDataUrl(image) } }); } const requestBody = { model: resolvedModel, messages: [{ role: "user" as const, content: contentParts }], }; const response = await fetchImageProvider( "https://openrouter.ai/api/v1/chat/completions", { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey.apiKey}`, "HTTP-Referer": "https://gaebal-gajae.dev/", "X-OpenRouter-Title": "Gajae Code", "X-OpenRouter-Categories": "cli-agent", }, body: JSON.stringify(requestBody), signal: requestSignal, }, apiKey.apiKey, ); const rawText = await readProviderResponseText(response, apiKey.apiKey, requestSignal); if (!response.ok) { const parsed = tryParseProviderJson<{ error?: { message?: string } }>(rawText); const message = parsed?.error?.message; throw new Error( `OpenRouter image request failed (${response.status}): ${redactImageProviderText(message ?? "provider returned an error", apiKey.apiKey)}`, ); } const data = parseProviderJson(rawText, apiKey.apiKey); const message = data.choices?.[0]?.message; const rawResponseText = collectOpenRouterResponseText(message); const responseText = rawResponseText ? redactImageProviderText(rawResponseText, apiKey.apiKey) : undefined; const imageUrls = extractOpenRouterImageUrls(message); const inlineImages: InlineImageData[] = []; for (const imageUrl of imageUrls) { inlineImages.push(await loadImageFromUrl(imageUrl, requestSignal, apiKey.apiKey)); } if (inlineImages.length === 0) { const messageText = responseText ? `\n\n${responseText}` : ""; return { content: [{ type: "text", text: `No image data returned.${messageText}` }], details: { provider, model: resolvedModel, imageCount: 0, imagePaths: [], images: [], responseText, }, }; } const imagePaths = await saveImagesToTemp(inlineImages); return { content: [ { type: "text", text: buildResponseSummary(provider, resolvedModel, imagePaths, responseText) }, ], details: { provider, model: resolvedModel, imageCount: inlineImages.length, imagePaths, images: inlineImages, responseText, }, }; } if (provider === "alibaba") { const requestBody = buildAlibabaImageRequest( model, assemblePrompt(params), resolvedImages, params.image_size, ); const response = await fetchImageProvider( ALIBABA_IMAGE_GENERATION_URL, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey.apiKey}`, }, body: JSON.stringify(requestBody), signal: requestSignal, }, apiKey.apiKey, ); const rawText = await readProviderResponseText(response, apiKey.apiKey, requestSignal); if (!response.ok) { const parsed = tryParseProviderJson<{ message?: string; error?: { message?: string } }>(rawText); const message = parsed?.error?.message ?? parsed?.message; throw new Error( `Alibaba image request failed (${response.status}): ${redactImageProviderText(message ?? "provider returned an error", apiKey.apiKey)}`, ); } const data = parseProviderJson(rawText, apiKey.apiKey); if (data.code) { throw new Error( `Alibaba image request failed: ${redactImageProviderText(`${data.code}: ${data.message ?? ""}`, apiKey.apiKey)}`, ); } const { imageUrls, responseText: rawResponseText } = collectAlibabaImageResult(data); const responseText = rawResponseText ? redactImageProviderText(rawResponseText, apiKey.apiKey) : undefined; // Result URLs are short-lived OSS-signed URLs (24h); download immediately. const inlineImages: InlineImageData[] = []; for (const imageUrl of imageUrls) { inlineImages.push(await loadImageFromUrl(imageUrl, requestSignal, apiKey.apiKey)); } if (inlineImages.length === 0) { const messageText = responseText ? `\n\n${responseText}` : ""; return { content: [{ type: "text", text: `No image data returned.${messageText}` }], details: { provider, model, imageCount: 0, imagePaths: [], images: [], responseText, }, }; } const imagePaths = await saveImagesToTemp(inlineImages); return { content: [{ type: "text", text: buildResponseSummary(provider, model, imagePaths, responseText) }], details: { provider, model, imageCount: inlineImages.length, imagePaths, images: inlineImages, responseText, }, }; } const parts = [] as Array<{ text?: string; inlineData?: InlineImageData }>; for (const image of resolvedImages) { parts.push({ inlineData: image }); } parts.push({ text: assemblePrompt(params) }); const generationConfig: { responseModalities: GeminiResponseModality[]; imageConfig?: { aspectRatio?: string; imageSize?: string }; } = { responseModalities: ["IMAGE"], }; if (params.aspect_ratio || params.image_size) { generationConfig.imageConfig = { aspectRatio: params.aspect_ratio, imageSize: params.image_size, }; } const requestBody = { contents: [{ role: "user" as const, parts }], generationConfig, }; const response = await fetchImageProvider( `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}:generateContent`, { method: "POST", headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey.apiKey, }, body: JSON.stringify(requestBody), signal: requestSignal, }, apiKey.apiKey, ); const rawText = await readProviderResponseText(response, apiKey.apiKey, requestSignal); if (!response.ok) { const parsed = tryParseProviderJson<{ error?: { message?: string } }>(rawText); const message = parsed?.error?.message; throw new Error( `Gemini image request failed (${response.status}): ${redactImageProviderText(message ?? "provider returned an error", apiKey.apiKey)}`, ); } const data = parseProviderJson(rawText, apiKey.apiKey); const responseParts = combineParts(data); const rawResponseText = collectResponseText(responseParts); const responseText = rawResponseText ? redactImageProviderText(rawResponseText, apiKey.apiKey) : undefined; const inlineImages = collectInlineImages(responseParts); if (inlineImages.length === 0) { const blocked = data.promptFeedback?.blockReason ? `Blocked: ${redactImageProviderText(data.promptFeedback.blockReason, apiKey.apiKey)}` : "No image data returned."; return { content: [{ type: "text", text: `${blocked}${responseText ? `\n\n${responseText}` : ""}` }], details: { provider, model, imageCount: 0, imagePaths: [], images: [], responseText, promptFeedback: redactGeminiPromptFeedback(data.promptFeedback, apiKey.apiKey), usage: data.usageMetadata, }, }; } const imagePaths = await saveImagesToTemp(inlineImages); return { content: [{ type: "text", text: buildResponseSummary(provider, model, imagePaths, responseText) }], details: { provider, model, imageCount: inlineImages.length, imagePaths, images: inlineImages, responseText, promptFeedback: redactGeminiPromptFeedback(data.promptFeedback, apiKey.apiKey), usage: data.usageMetadata, }, }; }); }, }; export async function getImageGenTools( modelRegistry?: ModelRegistry, settings?: ModelRoleSettings, ): Promise>> { // The tool is available when an image role model is configured and resolvable. if (!modelRegistry || !settings) return []; const imageModel = resolveImageRoleModel(settings, modelRegistry); if (!imageModel) return []; return [imageGenTool]; } export async function getImageGenToolsWithRegistry( modelRegistry: ModelRegistry, settings: ModelRoleSettings, ): Promise>> { const imageModel = resolveImageRoleModel(settings, modelRegistry); if (!imageModel) return []; return [imageGenTool]; }