import { once } from "node:events"; import { createServer } from "node:net"; import { dirname } from "node:path"; export async function withEmbeddingServer(options: { runtimeBinary: string; modelFile: string; vulkanDevice?: string; run(baseUrl: string): Promise; }): Promise { const port = await freePort(); const child = Bun.spawn( embeddingServerArgs({ runtimeBinary: options.runtimeBinary, modelFile: options.modelFile, port, vulkanDevice: options.vulkanDevice, }), { cwd: dirname(options.runtimeBinary), stdout: "ignore", stderr: "pipe", env: { ...processEnv(), LD_LIBRARY_PATH: [dirname(options.runtimeBinary), process.env.LD_LIBRARY_PATH] .filter(Boolean) .join(":"), }, }, ); try { await waitForServer(`http://127.0.0.1:${port}`, child); return await options.run(`http://127.0.0.1:${port}`); } finally { child.kill(); await child.exited; } } export function embeddingServerArgs(options: { runtimeBinary: string; modelFile: string; port: number; vulkanDevice?: string; }): string[] { return [ options.runtimeBinary, "-m", options.modelFile, ...(options.vulkanDevice ? ["--device", options.vulkanDevice] : []), "--host", "127.0.0.1", "--port", String(options.port), "--embeddings", "--pooling", "cls", "-ngl", "99", ]; } async function waitForServer(baseUrl: string, child: Bun.Subprocess): Promise { const deadline = Date.now() + 60_000; while (Date.now() < deadline) { if (child.exitCode !== null) { const stderr = await readStderr(child); throw new Error(`llama-server exited early: ${stderr.trim()}`); } try { const response = await fetch(`${baseUrl}/health`); if (response.ok) { return; } } catch { // Server is still starting up. } await Bun.sleep(250); } child.kill(); const stderr = await readStderr(child); throw new Error(`timed out waiting for llama-server: ${stderr.trim()}`); } async function freePort(): Promise { const server = createServer(); server.listen(0, "127.0.0.1"); await once(server, "listening"); const address = server.address(); server.close(); if (!address || typeof address === "string") { throw new Error("failed to allocate a local port for llama-server"); } return address.port; } function processEnv(): NodeJS.ProcessEnv { return process.env as NodeJS.ProcessEnv; } async function readStderr(child: Bun.Subprocess): Promise { if (!(child.stderr instanceof ReadableStream)) { return ""; } return await new Response(child.stderr).text(); }