/** * @artemiskit/sdk * Contract types for custom implementations * * These types define the contracts that custom adapters, evaluators, * and storage backends must implement to integrate with ArtemisKit. */ import type { Evaluator, EvaluatorContext, EvaluatorResult, Expected, GenerateOptions, GenerateResult, ModelCapabilities, ModelClient, StorageAdapter } from '@artemiskit/core'; /** * Contract for implementing custom model adapters * * @example * ```typescript * import { defineAdapter, type AdapterContract } from '@artemiskit/sdk' * * const myAdapter = defineAdapter({ * provider: 'my-provider', * async generate(options) { * // Implementation * return { id: '...', model: '...', text: '...', tokens: {...}, latencyMs: 0 } * }, * async capabilities() { * return { streaming: false, functionCalling: false, toolUse: false, maxContext: 4096 } * } * }) * ``` */ export interface AdapterContract extends ModelClient { /** * Provider identifier - must be unique */ readonly provider: string; /** * Generate a completion from the model */ generate(options: GenerateOptions): Promise; /** * Get the capabilities of this adapter */ capabilities(): Promise; /** * Optional: Stream a completion from the model */ stream?(options: GenerateOptions, onChunk: (chunk: string) => void): AsyncIterable; /** * Optional: Generate embeddings */ embed?(text: string, model?: string): Promise; /** * Optional: Close/cleanup resources */ close?(): Promise; } /** * Define a custom adapter with type checking * * @example * ```typescript * const myAdapter = defineAdapter({ * provider: 'my-llm', * async generate(options) { * const response = await myLLMClient.complete(options.prompt) * return { * id: response.id, * model: 'my-model', * text: response.text, * tokens: { prompt: 0, completion: 0, total: 0 }, * latencyMs: response.duration * } * }, * async capabilities() { * return { streaming: true, functionCalling: false, toolUse: false, maxContext: 8192 } * } * }) * ``` */ export declare function defineAdapter(adapter: T): T; /** * Contract for implementing custom evaluators * * @example * ```typescript * import { defineEvaluator, type EvaluatorContract } from '@artemiskit/sdk' * * const customEvaluator = defineEvaluator({ * type: 'custom-semantic', * async evaluate(response, expected, context) { * // Custom evaluation logic * const score = calculateSemanticScore(response, expected.value) * return { passed: score > 0.8, score, reason: `Semantic score: ${score}` } * } * }) * ``` */ export interface EvaluatorContract extends Evaluator { /** * Unique type identifier for this evaluator */ readonly type: string; /** * Evaluate a response against expected criteria * * @param response - The model's response text * @param expected - The expected result configuration * @param context - Optional context including model client and test case * @returns Evaluation result with pass/fail, score, and reason */ evaluate(response: string, expected: Expected, context?: EvaluatorContext): Promise; } /** * Define a custom evaluator with type checking * * @example * ```typescript * const wordCountEvaluator = defineEvaluator({ * type: 'word-count', * async evaluate(response, expected, _context) { * const wordCount = response.split(/\s+/).length * const targetCount = (expected as any).wordCount ?? 100 * const tolerance = (expected as any).tolerance ?? 10 * * const diff = Math.abs(wordCount - targetCount) * const passed = diff <= tolerance * const score = passed ? 1 : Math.max(0, 1 - diff / targetCount) * * return { * passed, * score, * reason: `Word count: ${wordCount}, target: ${targetCount} (±${tolerance})`, * details: { wordCount, targetCount, tolerance, diff } * } * } * }) * ``` */ export declare function defineEvaluator(evaluator: T): T; /** * Contract for implementing custom storage adapters * * **IMPORTANT: Sort Order Requirement** * * The `list()` method MUST return results sorted by `startTime` in descending order * (most recent first). This is required by `ArtemisKit.compare()` when using * `baseline: 'latest'` - it relies on `list({ limit: 1 })[0]` being the most recent run. * * If your storage backend doesn't natively support this sort order, you must * implement sorting in the adapter's `list()` method. * * @example * ```typescript * import { defineStorage, type StorageContract } from '@artemiskit/sdk' * * const redisStorage = defineStorage({ * async save(manifest) { * await redis.set(`run:${manifest.run_id}`, JSON.stringify(manifest)) * return manifest.run_id * }, * async load(runId) { * const data = await redis.get(`run:${runId}`) * return JSON.parse(data) * }, * async list(options) { * // IMPORTANT: Results must be sorted by startTime descending (most recent first) * const runs = await this.getAllRuns() * runs.sort((a, b) => b.startTime.getTime() - a.startTime.getTime()) * return runs.slice(0, options?.limit ?? runs.length) * }, * async delete(runId) { * await redis.del(`run:${runId}`) * } * }) * ``` */ export interface StorageContract extends StorageAdapter { } /** * Define a custom storage adapter with type checking * * @example * ```typescript * const s3Storage = defineStorage({ * async save(manifest) { * const key = `runs/${manifest.run_id}.json` * await s3.putObject({ Bucket: 'my-bucket', Key: key, Body: JSON.stringify(manifest) }) * return manifest.run_id * }, * async load(runId) { * const key = `runs/${runId}.json` * const { Body } = await s3.getObject({ Bucket: 'my-bucket', Key: key }) * return JSON.parse(Body) * }, * async list(options) { * // List objects from S3 * return [] * }, * async delete(runId) { * const key = `runs/${runId}.json` * await s3.deleteObject({ Bucket: 'my-bucket', Key: key }) * } * }) * ``` */ export declare function defineStorage(storage: T): T; /** * Factory function type for creating adapters */ export type AdapterFactory = (config: TConfig) => AdapterContract | Promise; /** * Factory function type for creating evaluators */ export type EvaluatorFactory = (config: TConfig) => EvaluatorContract; /** * Factory function type for creating storage adapters */ export type StorageFactory = (config: TConfig) => StorageContract | Promise; /** * Plugin interface for extending ArtemisKit */ export interface ArtemisKitPlugin { /** * Plugin name (unique identifier) */ name: string; /** * Plugin version */ version: string; /** * Optional: Custom adapters to register */ adapters?: Record; /** * Optional: Custom evaluators to register */ evaluators?: Record; /** * Optional: Custom storage adapters to register */ storage?: Record; /** * Optional: Initialize the plugin */ init?(): Promise; /** * Optional: Cleanup the plugin */ cleanup?(): Promise; } /** * Define a plugin with type checking */ export declare function definePlugin(plugin: ArtemisKitPlugin): ArtemisKitPlugin; //# sourceMappingURL=index.d.ts.map