/** * pi-provider-freellmapi — register the FreeLLM API gateway as a pi provider. * * This extension registers `https://freeapi.n.cofire.cn/v1` as three * OpenAI-compatible providers and auto-discovers the models your API key can * actually use: * * | Provider | API type | Endpoint | * |----------|----------|----------| * | `freellmapi` | `openai-completions` | `/v1/chat/completions` | * | `freellmapi-responses` | `openai-responses` | `/v1/responses` | * | `freellmapi-anthropic` | `anthropic-messages` | `/v1/messages` | * * It also registers a set of custom tools that wrap the other OpenAI-compatible * endpoints the gateway exposes, so the agent can use them directly: * * - `POST /embeddings` → `freellm_embed` * - `POST /images/generations` → `freellm_image` * - `POST /videos/generations` → `freellm_video` (tasks + polling) * - `POST /audio/speech` → `freellm_speech` * - `POST /audio/transcriptions` → `freellm_transcribe` * * Each tool is enabled by default and can be turned off (or given a default * model) via the `endpoints` block in the config file. * * ## Install * * ```bash * pi install npm:pi-provider-freellmapi * ``` * * Then create the config file `~/.pi/agent/freellmapi.json`: * * ```json * { * "name": "freellmapi", // provider id → --provider freellmapi * "label": "FreeLLM API", // display label in /model * "baseUrl": "https://freeapi.n.cofire.cn", // defaults to gateway; add /v1 if needed * "apiKey": "freellmapi-...", // literal, $ENV_VAR, or ${ENV_VAR} * "fetchModels": true, // discover models from {baseUrl}/models * "contextWindow": 131072, // fallback context window (tokens) * "maxTokens": 16384, // fallback max output tokens * "compat": { "supportsDeveloperRole": false }, * "models": [], // optional static models (override by id) * "endpoints": { * "embeddings": { "enabled": true, "model": "auto" }, * "images": { "enabled": true, "model": "auto" }, * "videos": { "enabled": true, "model": "inkling" }, * "speech": { "enabled": true, "model": "auto" }, * "transcriptions": { "enabled": true, "model": "whisper-1" } * } * } * ``` * * Config path override: `PI_FREELLMAPI_CONFIG=/path/to/config.json`. * API key fallback env var: `FREELLM_API_KEY` (used when the config file has * no `apiKey`). * * Only models the gateway reports as `available` (with a positive * `context_window`) are registered, so the list stays limited to models that * work with your key. Reasoning support is detected from the gateway's * `supported_parameters` (`reasoning_effort`) or the model name; vision models * get `input: ["text", "image"]`. */ import type { ExtensionAPI, ProviderModelConfig } from "@earendil-works/pi-coding-agent"; import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; import { Type } from "typebox"; import { execSync } from "node:child_process"; import { existsSync, readFileSync } from "node:fs"; import { readFile, writeFile, mkdir } from "node:fs/promises"; import { homedir } from "node:os"; import { basename, dirname, extname, resolve } from "node:path"; // ── Types ────────────────────────────────────────────────────────────────── interface RemoteModel { id: string; name?: string; context_window?: number | null; available?: boolean; supported_parameters?: string[]; } interface EndpointConfig { /** Whether the corresponding tool is registered (default: true). */ enabled?: boolean; /** Default model id used when a tool call omits `model`. */ model?: string; } interface EndpointsConfig { embeddings?: EndpointConfig | boolean; images?: EndpointConfig | boolean; videos?: EndpointConfig | boolean; speech?: EndpointConfig | boolean; transcriptions?: EndpointConfig | boolean; /** OpenAI Responses API (`/v1/responses`). */ responses?: EndpointConfig | boolean; /** Anthropic Messages API (`/v1/messages`). */ anthropic?: EndpointConfig | boolean; } interface ConfigFile { name?: string; label?: string; baseUrl?: string; apiKey?: string; fetchModels?: boolean; contextWindow?: number; maxTokens?: number; /** Extra request headers merged into every tool request. */ headers?: Record; compat?: Record; models?: ProviderModelConfig[]; endpoints?: EndpointsConfig; } type ToolContent = (TextContent | ImageContent)[]; interface HttpInit { method?: string; headers?: Record; body?: string | FormData; signal?: AbortSignal; } // ── Defaults ─────────────────────────────────────────────────────────────── const DEFAULT_CONFIG_PATH = resolve(homedir(), ".pi", "agent", "freellmapi.json"); const DEFAULTS = { name: "freellmapi", label: "FreeLLM API", baseUrl: "https://freeapi.n.cofire.cn", contextWindow: 131072, maxTokens: 16384, } as const; /** Fallback models when discovery fails (or fetchModels is off and no static list). */ const FALLBACK_MODELS: Array<{ id: string; name: string }> = [ { id: "auto", name: "Auto (router picks best available model)" }, { id: "fusion", name: "Fusion (panel of models + judge)" }, { id: "deepseek-v4-flash", name: "DeepSeek V4 Flash" }, { id: "qwen3.7-flash", name: "Qwen3.7 Flash" }, { id: "claude-sonnet-4-5", name: "Claude Sonnet slot" }, ]; const VIDEO_TERMINAL_STATUSES = new Set([ "completed", "succeeded", "success", "done", "failed", "error", "cancelled", "canceled", "killed", "rejected", ]); const VIDEO_SUCCESS_STATUSES = new Set(["completed", "succeeded", "success", "done"]); // ── Helpers ──────────────────────────────────────────────────────────────── function loadConfig(configPath: string): ConfigFile { if (!existsSync(configPath)) { return {}; } try { return JSON.parse(readFileSync(configPath, "utf-8")) as ConfigFile; } catch (err) { console.error(`[freellmapi] Failed to parse config: ${configPath}`, err); return {}; } } /** Normalize `endpoints.` which may be a boolean or an object. */ function normalizeEndpoint(raw: EndpointConfig | boolean | undefined): EndpointConfig { if (raw === undefined) return {}; if (typeof raw === "boolean") return { enabled: raw }; return raw; } /** * Resolve an apiKey value that may use pi's value syntax: * - `${ENV_VAR}` / `$ENV_VAR` (whole value) → environment variable * - `!command` → returned as-is (pi resolves it at request time) * - literal string → returned as-is */ function resolveApiKey(raw: string): string { if (raw.startsWith("!")) return raw; const braced = raw.match(/^\$\{([^}]+)\}$/); if (braced) return process.env[braced[1]] ?? ""; if (raw.startsWith("$") && !raw.startsWith("$$")) { return process.env[raw.slice(1)] ?? raw; } return raw; } /** Resolve an apiKey for a direct tool request, executing `!command` if needed. */ function resolveApiKeyValue(raw: string): string { if (raw.startsWith("!")) { try { return execSync(raw.slice(1), { encoding: "utf8", timeout: 30000 }).trim(); } catch (err) { throw new Error(`[freellmapi] Failed to execute apiKey command: ${raw.slice(1)}`); } } return resolveApiKey(raw); } function isVision(name: string, id: string): boolean { return /vision|vl|image|moondream/i.test(`${name} ${id}`); } /** A model supports thinking if the gateway advertises reasoning_effort, or the name hints at it. */ function isReasoning(name: string, id: string, params: string[]): boolean { return params.includes("reasoning_effort") || /reasoning|thinking/i.test(`${name} ${id}`); } async function fetchModelsFromEndpoint( baseUrl: string, apiKey?: string, ): Promise { const url = baseUrl.replace(/\/+$/, "") + "/models"; const headers: Record = { "Content-Type": "application/json" }; if (apiKey) headers["Authorization"] = `Bearer ${resolveApiKey(apiKey)}`; const response = await fetch(url, { headers }); if (!response.ok) { throw new Error(`HTTP ${response.status} from ${url}`); } const payload = (await response.json()) as { data?: RemoteModel[] }; return Array.isArray(payload.data) ? payload.data : []; } function mapRemoteModel( m: RemoteModel, contextWindow: number, maxTokens: number, ): ProviderModelConfig { const name = m.name?.trim() || m.id; const window = typeof m.context_window === "number" && m.context_window > 0 ? m.context_window : contextWindow; const params = m.supported_parameters ?? []; return { id: m.id, name, reasoning: isReasoning(name, m.id, params), input: isVision(name, m.id) ? ["text", "image"] : ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: window, maxTokens: Math.min(maxTokens, window), }; } function mergeModels( staticModels: ProviderModelConfig[], discoveredModels: ProviderModelConfig[], ): ProviderModelConfig[] { const map = new Map(); for (const m of discoveredModels) map.set(m.id, m); // Static models override discovered ones by id. for (const m of staticModels) map.set(m.id, m); return [...map.values()]; } /** Pick the first available model id matching any of the patterns, else the fallback. */ function pickDefaultModel(ids: string[], patterns: RegExp[], fallback: string): string { for (const pattern of patterns) { const match = ids.find((id) => pattern.test(id)); if (match) return match; } return fallback; } function guessMimeFromExt(nameOrUrl: string): string | undefined { const ext = extname(nameOrUrl.split("?")[0].split("#")[0]).toLowerCase(); const map: Record = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", ".gif": "image/gif", ".mp3": "audio/mpeg", ".wav": "audio/wav", ".ogg": "audio/ogg", ".oga": "audio/ogg", ".opus": "audio/opus", ".flac": "audio/flac", ".m4a": "audio/mp4", ".aac": "audio/aac", ".mp4": "video/mp4", ".webm": "video/webm", ".mov": "video/quicktime", ".mpeg": "video/mpeg", }; return map[ext]; } function guessVideoExt(url: string): string { const ext = extname(url.split("?")[0].split("#")[0]).toLowerCase(); return [".mp4", ".webm", ".mov", ".mpeg", ".mkv"].includes(ext) ? ext : ".mp4"; } function encodeBase64(bytes: Uint8Array): string { return Buffer.from(bytes).toString("base64"); } const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); async function readErrorBody(res: Response): Promise { const text = await res.text(); try { const json = JSON.parse(text) as { error?: { message?: string }; message?: string }; return json?.error?.message ?? json?.message ?? text; } catch { return text; } } /** Unified request helper: auth + config headers + JSON/multipart body handling. */ async function httpRequest( baseUrl: string, apiKey: string | undefined, extraHeaders: Record | undefined, path: string, init: HttpInit = {}, ): Promise { const reqHeaders = init.headers ?? {}; const isForm = typeof (init.body as FormData | undefined)?.append === "function"; const merged: Record = { ...(extraHeaders ?? {}) }; for (const [k, v] of Object.entries(reqHeaders)) merged[k] = v; const resolvedKey = apiKey ? resolveApiKeyValue(apiKey) : undefined; if (resolvedKey && !("Authorization" in merged) && !("authorization" in merged)) { merged["Authorization"] = `Bearer ${resolvedKey}`; } if (!isForm && !("Content-Type" in merged) && !("content-type" in merged)) { merged["Content-Type"] = "application/json"; } return fetch(baseUrl.replace(/\/+$/, "") + path, { ...init, headers: merged }); } async function writeBytes(targetPath: string, bytes: Uint8Array): Promise { await mkdir(dirname(targetPath), { recursive: true }); await writeFile(targetPath, bytes); } async function downloadBytes(url: string, signal?: AbortSignal): Promise { const res = await fetch(url, { signal }); if (!res.ok) throw new Error(`Download failed (HTTP ${res.status}): ${url}`); return new Uint8Array(await res.arrayBuffer()); } async function imageUrlToBase64(url: string): Promise<{ data: string; mimeType: string }> { const bytes = await downloadBytes(url); const mimeType = guessMimeFromExt(url) ?? "image/png"; return { data: encodeBase64(bytes), mimeType }; } /** Lenient scan of a video task response for a downloadable URL. */ function findVideoUrl(value: unknown): string | undefined { const seen = new Set(); const stack: unknown[] = [value]; while (stack.length) { const cur = stack.pop(); if (!cur || typeof cur !== "object" || seen.has(cur as object)) continue; seen.add(cur as object); if (Array.isArray(cur)) { stack.push(...cur); continue; } const record = cur as Record; for (const key of ["url", "download_url", "video_url", "file_url", "output_url", "play_url", "link", "href"]) { const v = record[key]; if (typeof v === "string" && /^https?:\/\//i.test(v)) return v; } for (const key of ["output", "outputs", "result", "results", "video", "videos", "data"]) { const v = record[key]; if (typeof v === "string" && /^https?:\/\//i.test(v)) return v; if (v && typeof v === "object") stack.push(v); } } return undefined; } function extractTaskId(payload: unknown): string | undefined { const p = payload as Record & { data?: Array | string>; }; const first = p?.data?.[0]; if (typeof first === "string") return first; if (first && typeof first === "object" && typeof first["id"] === "string") return first["id"]; if (typeof p?.id === "string") return p.id; if (typeof p?.task_id === "string") return p.task_id; if (typeof p?.video_id === "string") return p.video_id; return undefined; } function extractVideoStatus(payload: unknown): string { const p = payload as { status?: unknown; data?: Array<{ status?: unknown }>; }; const raw = p?.status ?? p?.data?.[0]?.status ?? ""; return String(raw ?? "").toLowerCase(); } async function resolveAudioFile( input: string, cwd: string, signal?: AbortSignal, ): Promise<{ bytes: Uint8Array; filename: string; mimeType: string }> { if (/^https?:\/\//i.test(input)) { const res = await fetch(input, { signal }); if (!res.ok) throw new Error(`Failed to fetch audio (HTTP ${res.status}): ${input}`); const filename = basename(new URL(input).pathname) || "audio.mp3"; return { bytes: new Uint8Array(await res.arrayBuffer()), filename, mimeType: guessMimeFromExt(filename) ?? "audio/mpeg", }; } const path = resolve(cwd, input); if (!existsSync(path)) throw new Error(`Audio file not found: ${path}`); const buffer = await readFile(path); return { bytes: new Uint8Array(buffer), filename: basename(path), mimeType: guessMimeFromExt(path) ?? "audio/mpeg", }; } // ── Extension Entry Point ────────────────────────────────────────────────── export default async function (pi: ExtensionAPI) { const configPath = process.env["PI_FREELLMAPI_CONFIG"] ?? DEFAULT_CONFIG_PATH; const config = loadConfig(configPath); const providerId = config.name ?? DEFAULTS.name; const rawBase = config.baseUrl ?? process.env["FREELLM_API_BASE_URL"] ?? DEFAULTS.baseUrl; // Ensure baseUrl always ends with /v1 (without duplicating it) so path // segments like /chat/completions compose correctly. const trimmedBase = rawBase.replace(/\/+$/, ""); const baseUrl = /\/v1$/i.test(trimmedBase) ? trimmedBase : `${trimmedBase}/v1`; const apiKey = config.apiKey ?? process.env["FREELLM_API_KEY"]; const contextWindow = config.contextWindow ?? DEFAULTS.contextWindow; const maxTokens = config.maxTokens ?? DEFAULTS.maxTokens; const staticModels = config.models ?? []; const extraHeaders = config.headers ?? {}; const endpoints: EndpointsConfig = config.endpoints ?? {}; const embeddingsCfg = normalizeEndpoint(endpoints.embeddings); const imagesCfg = normalizeEndpoint(endpoints.images); const videosCfg = normalizeEndpoint(endpoints.videos); const speechCfg = normalizeEndpoint(endpoints.speech); const transcriptionsCfg = normalizeEndpoint(endpoints.transcriptions); const responsesCfg = normalizeEndpoint(endpoints.responses); const anthropicCfg = normalizeEndpoint(endpoints.anthropic); const endpointEnabled = embeddingsCfg.enabled !== false || imagesCfg.enabled !== false || videosCfg.enabled !== false || speechCfg.enabled !== false || transcriptionsCfg.enabled !== false || responsesCfg.enabled !== false || anthropicCfg.enabled !== false; // ── Model discovery (used for both chat models and endpoint defaults) ──── let remote: RemoteModel[] = []; const shouldFetchModels = config.fetchModels !== false || endpointEnabled; if (shouldFetchModels) { try { remote = await fetchModelsFromEndpoint(baseUrl, apiKey); } catch (err) { console.warn( `[freellmapi] Could not fetch models from ${baseUrl}:`, err instanceof Error ? err.message : err, ); } } const availableIds = remote .filter((m) => m.available !== false) .map((m) => m.id); let discovered: ProviderModelConfig[] = []; if (config.fetchModels !== false) { discovered = remote .filter((m) => m.available !== false) .filter((m) => typeof m.context_window === "number" && m.context_window > 0) .map((m) => mapRemoteModel(m, contextWindow, maxTokens)); } let models = mergeModels(staticModels, discovered); if (models.length === 0) { console.warn(`[freellmapi] No models discovered; registering fallback list`); models = FALLBACK_MODELS.map(({ id, name }) => ({ id, name, reasoning: /reasoning|thinking/i.test(name), input: ["text"] as ("text" | "image")[], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow, maxTokens: Math.min(maxTokens, contextWindow), })); } // ── Register chat provider ─────────────────────────────────────────────── const providerConfig: Record = { name: config.label ?? DEFAULTS.label, baseUrl, apiKey, api: "openai-completions", models, compat: { supportsDeveloperRole: false, ...(config.compat ?? {}), }, }; pi.registerProvider(providerId, providerConfig as never); console.log( `[freellmapi] Registered "${providerId}" → ${baseUrl} (${models.length} model(s))`, ); // ── Register additional API-type providers ─────────────────────────────── // The gateway is OpenAI-compatible but also supports the Responses and // Anthropic Messages API shapes at /v1/responses and /v1/messages. if (responsesCfg.enabled !== false) { const responsesProviderId = `${providerId}-responses`; pi.registerProvider(responsesProviderId, { name: `${config.label ?? DEFAULTS.label} (Responses)`, baseUrl, apiKey, api: "openai-responses", models, compat: { supportsDeveloperRole: false, ...(config.compat ?? {}) }, } as never); console.log(`[freellmapi] Registered "${responsesProviderId}" → openai-responses (${models.length} model(s))`); } if (anthropicCfg.enabled !== false) { const anthropicProviderId = `${providerId}-anthropic`; pi.registerProvider(anthropicProviderId, { name: `${config.label ?? DEFAULTS.label} (Anthropic)`, baseUrl, apiKey, api: "anthropic-messages", models, compat: { supportsDeveloperRole: false, ...(config.compat ?? {}) }, } as never); console.log(`[freellmapi] Registered "${anthropicProviderId}" → anthropic-messages (${models.length} model(s))`); } // ── Register endpoint tools ────────────────────────────────────────────── // Default model per endpoint: explicit config wins, then discovery, then a // conventional OpenAI id (which may need overriding for this gateway). const embeddingsDefault = embeddingsCfg.model ?? pickDefaultModel(availableIds, [/embed|embedding/i], "auto"); const imagesDefault = imagesCfg.model ?? pickDefaultModel(availableIds, [/gemini.*image|flux|dall|sdxl|stable/i], "auto"); const videosDefault = videosCfg.model ?? pickDefaultModel(availableIds, [/video|inkling|sora|veo|kling|hailuo|wan|pixverse|runway|pika|ray/i], "inkling"); const speechDefault = speechCfg.model ?? pickDefaultModel(availableIds, [/tts|speech|voice|cosy|melotts|fish|bark/i], "auto"); const transcriptionsDefault = transcriptionsCfg.model ?? pickDefaultModel(availableIds, [/whisper|transcri|asr|sensevoice|paraformer|audio/i], "whisper-1"); if (embeddingsCfg.enabled !== false) { pi.registerTool({ name: "freellm_embed", label: "Embed text", description: "Generate vector embeddings via the FreeLLM gateway (POST /embeddings). " + "Useful for semantic search, clustering, similarity, or retrieval tasks.", promptSnippet: "Generate text embeddings via the FreeLLM gateway (POST /embeddings)", promptGuidelines: [ "Use freellm_embed when the task calls for semantic search, clustering, similarity, or retrieval embeddings rather than a chat completion.", ], parameters: Type.Object({ input: Type.Union([ Type.String({ description: "Single text to embed" }), Type.Array(Type.String({ description: "Text to embed" })), ]), model: Type.Optional(Type.String({ description: `Embedding model id (default: ${embeddingsDefault})` })), dimensions: Type.Optional(Type.Integer({ description: "Optional output dimension" })), encodingFormat: Type.Optional(Type.String({ description: '"float" or "base64"' })), outputFile: Type.Optional(Type.String({ description: "Path for the JSON result (default: freellm-embeddings-.json in cwd)" })), }), async execute(_toolCallId, params, signal, _onUpdate, ctx) { const model = params.model ?? embeddingsDefault; const body: Record = { model, input: params.input }; if (params.dimensions) body["dimensions"] = params.dimensions; if (params.encodingFormat) body["encoding_format"] = params.encodingFormat; const res = await httpRequest(baseUrl, apiKey, extraHeaders, "/embeddings", { method: "POST", body: JSON.stringify(body), signal, }); if (!res.ok) { throw new Error(`[freellmapi] /embeddings failed (HTTP ${res.status}): ${await readErrorBody(res)}`); } const payload = (await res.json()) as { data?: Array<{ embedding?: unknown[] }>; model?: string; usage?: unknown }; const data = payload.data ?? []; const dimension = data[0]?.embedding?.length ?? 0; const outFile = resolve(ctx.cwd, params.outputFile ?? `freellm-embeddings-${Date.now()}.json`); await writeBytes(outFile, new TextEncoder().encode(JSON.stringify(payload, null, 2))); return { content: [{ type: "text", text: `Generated ${data.length} embedding(s) of dimension ${dimension} using model "${payload.model ?? model}". Saved to ${outFile}`, }], details: { outputFile: outFile, count: data.length, dimension, usage: payload.usage }, }; }, }); } if (imagesCfg.enabled !== false) { pi.registerTool({ name: "freellm_image", label: "Generate image", description: "Generate images via the FreeLLM gateway (POST /images/generations). " + "Returns the image(s) inline so they can be viewed and described.", promptSnippet: "Generate images via the FreeLLM gateway (POST /images/generations)", promptGuidelines: [ "Use freellm_image when the user asks to create, draw, or generate an image.", ], parameters: Type.Object({ prompt: Type.String({ description: "Image prompt" }), model: Type.Optional(Type.String({ description: `Image model id (default: ${imagesDefault})` })), n: Type.Optional(Type.Integer({ description: "Number of images (1-10)" })), size: Type.Optional(Type.String({ description: 'e.g. "1024x1024"' })), quality: Type.Optional(Type.String({ description: 'e.g. "standard" or "hd"' })), style: Type.Optional(Type.String({ description: 'e.g. "vivid" or "natural"' })), responseFormat: Type.Optional(Type.String({ description: '"b64_json" (default) or "url"' })), }), async execute(_toolCallId, params, signal, _onUpdate, _ctx) { const model = params.model ?? imagesDefault; const body: Record = { model, prompt: params.prompt, n: params.n ?? 1, response_format: params.responseFormat ?? "b64_json", }; if (params.size) body["size"] = params.size; if (params.quality) body["quality"] = params.quality; if (params.style) body["style"] = params.style; const res = await httpRequest(baseUrl, apiKey, extraHeaders, "/images/generations", { method: "POST", body: JSON.stringify(body), signal, }); if (!res.ok) { throw new Error(`[freellmapi] /images/generations failed (HTTP ${res.status}): ${await readErrorBody(res)}`); } const payload = (await res.json()) as { data?: Array<{ b64_json?: string; url?: string; revised_prompt?: string }>; }; const content: ToolContent = []; const images: Array<{ mimeType: string; data: string; revisedPrompt?: string }> = []; const items = payload.data ?? []; for (const item of items) { let data = item.b64_json; let mimeType = "image/png"; if (!data && item.url) { try { const downloaded = await imageUrlToBase64(item.url); data = downloaded.data; mimeType = downloaded.mimeType; } catch { content.push({ type: "text", text: `Image URL: ${item.url}` }); continue; } } if (data) { content.push({ type: "image", data, mimeType }); images.push({ mimeType, data, revisedPrompt: item.revised_prompt }); } else if (item.url) { content.push({ type: "text", text: `Image URL: ${item.url}` }); } } const revised = images.map((i) => i.revisedPrompt).filter(Boolean); const summary = `Generated ${images.length} image(s)` + (revised.length ? ` (with revised prompts)` : "") + (images.length !== items.length ? `; ${items.length - images.length} returned only URLs` : ""); content.unshift({ type: "text", text: summary }); return { content, details: { images } }; }, }); } if (videosCfg.enabled !== false) { pi.registerTool({ name: "freellm_video", label: "Generate video", description: "Generate a video via the FreeLLM gateway (POST /videos/generations) with task polling. " + "Downloads the finished video to a local file and reports the path.", promptSnippet: "Generate videos via the FreeLLM gateway (POST /videos/generations)", promptGuidelines: [ "Use freellm_video when the user asks to create or generate a video. It blocks until the render finishes, so prefer it only for explicit video requests.", ], parameters: Type.Object({ prompt: Type.String({ description: "Video prompt" }), model: Type.Optional(Type.String({ description: `Video model id (default: ${videosDefault})` })), n: Type.Optional(Type.Integer({ description: "Number of videos" })), size: Type.Optional(Type.String({ description: 'e.g. "1280x720"' })), seconds: Type.Optional(Type.String({ description: 'e.g. "5"' })), pollTimeoutSeconds: Type.Optional(Type.Number({ description: "Max time to wait for completion (default 300)" })), }), async execute(_toolCallId, params, signal, onUpdate, ctx) { const model = params.model ?? videosDefault; const body: Record = { model, prompt: params.prompt }; if (params.n) body["n"] = params.n; if (params.size) body["size"] = params.size; if (params.seconds) body["seconds"] = params.seconds; const submit = await httpRequest(baseUrl, apiKey, extraHeaders, "/videos/generations", { method: "POST", body: JSON.stringify(body), signal, }); if (!submit.ok) { throw new Error(`[freellmapi] /videos/generations failed (HTTP ${submit.status}): ${await readErrorBody(submit)}`); } const submitted = (await submit.json()) as unknown; const taskId = extractTaskId(submitted); if (!taskId) { const directUrl = findVideoUrl(submitted); if (directUrl) { const bytes = await downloadBytes(directUrl, signal); const outFile = resolve(ctx.cwd, `freellm-video-${Date.now()}${guessVideoExt(directUrl)}`); await writeBytes(outFile, bytes); return { content: [{ type: "text", text: `Video ready and downloaded (${bytes.length} bytes) to ${outFile}. Source URL: ${directUrl}` }], details: { outputFile: outFile, url: directUrl }, }; } throw new Error(`[freellmapi] /videos/generations returned no task id: ${JSON.stringify(submitted).slice(0, 500)}`); } const timeoutSeconds = params.pollTimeoutSeconds ?? 300; const deadline = Date.now() + timeoutSeconds * 1000; let task: unknown = submitted; let status = extractVideoStatus(task); while (!VIDEO_TERMINAL_STATUSES.has(status) && Date.now() < deadline) { if (signal?.aborted) throw new Error("Video generation cancelled"); await sleep(3000); const poll = await httpRequest(baseUrl, apiKey, extraHeaders, `/videos/${encodeURIComponent(taskId)}`, { signal }); if (!poll.ok) { status = `error (poll HTTP ${poll.status})`; break; } task = (await poll.json()) as unknown; status = extractVideoStatus(task); if (!VIDEO_TERMINAL_STATUSES.has(status)) { onUpdate?.({ content: [{ type: "text", text: `Video ${taskId} status: ${status || "processing"}…` }], }); } } if (!VIDEO_SUCCESS_STATUSES.has(status)) { throw new Error( `[freellmapi] Video generation did not complete (status: ${status || "unknown"}). Last response: ${JSON.stringify(task).slice(0, 500)}`, ); } const url = findVideoUrl(task); if (!url) { throw new Error(`[freellmapi] Video completed but no URL found: ${JSON.stringify(task).slice(0, 1000)}`); } const bytes = await downloadBytes(url, signal); const outFile = resolve(ctx.cwd, `freellm-video-${taskId}${guessVideoExt(url)}`); await writeBytes(outFile, bytes); return { content: [{ type: "text", text: `Video ready (status ${status}). Downloaded ${bytes.length} bytes to ${outFile}. Source URL: ${url}`, }], details: { taskId, status, outputFile: outFile, url }, }; }, }); } if (speechCfg.enabled !== false) { pi.registerTool({ name: "freellm_speech", label: "Text to speech", description: "Synthesize speech from text via the FreeLLM gateway (POST /audio/speech). " + "Writes the audio to a local file and reports the path.", promptSnippet: "Synthesize speech via the FreeLLM gateway (POST /audio/speech)", promptGuidelines: [ "Use freellm_speech when the user asks to generate, read aloud, or synthesize speech/audio from text.", ], parameters: Type.Object({ text: Type.String({ description: "Text to synthesize" }), model: Type.Optional(Type.String({ description: `TTS model id (default: ${speechDefault})` })), voice: Type.Optional(Type.String({ description: 'Voice name (default: "alloy")' })), speed: Type.Optional(Type.Number({ description: "Playback speed (0.25-4.0)" })), format: Type.Optional(Type.String({ description: 'Audio format: "mp3" (default), "opus", "aac", "flac", "wav", "pcm"' })), outputFile: Type.Optional(Type.String({ description: `Output path (default: freellm-speech-. in cwd)` })), }), async execute(_toolCallId, params, signal, _onUpdate, ctx) { const model = params.model ?? speechDefault; const format = params.format ?? "mp3"; const body: Record = { model, input: params.text, voice: params.voice ?? "alloy", response_format: format, }; if (params.speed != null) body["speed"] = params.speed; const res = await httpRequest(baseUrl, apiKey, extraHeaders, "/audio/speech", { method: "POST", body: JSON.stringify(body), signal, }); if (!res.ok) { throw new Error(`[freellmapi] /audio/speech failed (HTTP ${res.status}): ${await readErrorBody(res)}`); } const bytes = new Uint8Array(await res.arrayBuffer()); const outFile = resolve(ctx.cwd, params.outputFile ?? `freellm-speech-${Date.now()}.${format}`); await writeBytes(outFile, bytes); return { content: [{ type: "text", text: `Synthesized ${bytes.length} bytes of ${format} audio with voice "${params.voice ?? "alloy"}" → ${outFile}`, }], details: { outputFile: outFile, format, bytes: bytes.length }, }; }, }); } if (transcriptionsCfg.enabled !== false) { pi.registerTool({ name: "freellm_transcribe", label: "Transcribe audio", description: "Transcribe an audio file via the FreeLLM gateway (POST /audio/transcriptions, multipart upload). " + "Accepts a local path or an http(s) URL.", promptSnippet: "Transcribe audio via the FreeLLM gateway (POST /audio/transcriptions)", promptGuidelines: [ "Use freellm_transcribe when the user asks to transcribe, caption, or get the text of an audio recording.", ], parameters: Type.Object({ file: Type.String({ description: "Local path or http(s) URL of the audio file" }), model: Type.Optional(Type.String({ description: `Transcription model id (default: ${transcriptionsDefault})` })), language: Type.Optional(Type.String({ description: 'Optional ISO-639-1 language, e.g. "en"' })), format: Type.Optional(Type.String({ description: 'Response format: "json" (default), "text", "srt", "verbose_json", "vtt"' })), prompt: Type.Optional(Type.String({ description: "Optional hint to guide transcription style/vocabulary" })), }), async execute(_toolCallId, params, signal, _onUpdate, ctx) { const model = params.model ?? transcriptionsDefault; const { bytes, filename, mimeType } = await resolveAudioFile(params.file, ctx.cwd, signal); const form = new FormData(); form.append("file", new Blob([bytes as BlobPart], { type: mimeType }), filename); form.append("model", model); if (params.language) form.append("language", params.language); if (params.format) form.append("response_format", params.format); if (params.prompt) form.append("prompt", params.prompt); const res = await httpRequest(baseUrl, apiKey, extraHeaders, "/audio/transcriptions", { method: "POST", body: form, signal, }); if (!res.ok) { throw new Error(`[freellmapi] /audio/transcriptions failed (HTTP ${res.status}): ${await readErrorBody(res)}`); } const raw = await res.text(); let text = raw; if (params.format === undefined || params.format === "json" || params.format === "verbose_json") { try { const json = JSON.parse(raw) as { text?: string }; text = json.text ?? raw; } catch { text = raw; } } return { content: [{ type: "text", text }], details: { raw } }; }, }); } }