import { ClientSDK, RequestOptions } from "../lib/sdks.js"; import * as components from "../models/components/index.js"; import * as operations from "../models/operations/index.js"; export declare class Evals extends ClientSDK { /** * Create an eval suite * * @remarks * Creates a team-scoped declarative eval suite. Dataset and Python grader files must be uploaded separately with purpose 'evals'. */ createSuite(request: operations.CreateEvalSuiteRequest, options?: RequestOptions): Promise; /** * List eval suites * * @remarks * Returns eval suites visible to the authenticated team context. */ listSuites(request: operations.ListEvalSuitesRequest, options?: RequestOptions): Promise; /** * Get an eval suite * * @remarks * Retrieves an eval suite by ID. */ getSuite(request: operations.GetEvalSuiteRequest, options?: RequestOptions): Promise; /** * Delete an eval suite * * @remarks * Soft-deletes an eval suite and all of its eval runs so they no longer appear in user-facing reads. */ deleteSuite(request: operations.DeleteEvalSuiteRequest, options?: RequestOptions): Promise; /** * Create an eval suite version * * @remarks * Creates an immutable suite version. Use this endpoint for edits to an existing eval suite. */ createSuiteVersion(request: operations.CreateEvalSuiteVersionRequest, options?: RequestOptions): Promise; /** * List eval suite versions * * @remarks * Returns immutable versions for an eval suite, including each version manifest. */ listSuiteVersions(request: operations.ListEvalSuiteVersionsRequest, options?: RequestOptions): Promise; /** * Get an eval suite version * * @remarks * Retrieves an immutable eval suite version and its manifest. */ getSuiteVersion(request: operations.GetEvalSuiteVersionRequest, options?: RequestOptions): Promise; /** * List leaderboard runs for an eval suite * * @remarks * Returns each ranking group's most recent completed runs for a suite in one request, with the group assignments in `groups`. A group is (model, targeted task) for task_ids-restricted runs and (model) for unrestricted runs (task_id null). Build boards from `groups`, not from run.models/run.task_ids — a returned run may name models or tasks whose groups it did not rank in. `truncated: true` means a server ceiling trimmed the aggregate; whole runs are kept in (best rank, first model, first task, id) order, so every group's newest runs survive before any group's fallbacks and no group is dropped for being old. Resume a truncated board by passing `next_cursor` back as `cursor` and unioning the responses. See the evals docs for grouping rationale and cost characteristics. */ listSuiteLeaderboardRuns(request: operations.ListEvalSuiteLeaderboardRunsRequest, options?: RequestOptions): Promise; /** * Create an eval run * * @remarks * Starts a durable eval run over the selected suite version, tasks, and models. */ createRun(request: operations.CreateEvalRunRequest, options?: RequestOptions): Promise; /** * List eval runs * * @remarks * Returns eval runs visible to the authenticated team context. suite_version is a per-suite ordinal and must be paired with suite_id; supplying it alone returns 400. */ listRuns(request: operations.ListEvalRunsRequest, options?: RequestOptions): Promise; /** * Get an eval run * * @remarks * Retrieves an eval run by ID. */ getRun(request: operations.GetEvalRunRequest, options?: RequestOptions): Promise; /** * Delete an eval run * * @remarks * Soft-deletes an eval run so it no longer appears in run lists, details, or score leaderboards. */ deleteRun(request: operations.DeleteEvalRunRequest, options?: RequestOptions): Promise; /** * Cancel an eval run * * @remarks * Requests cancellation for a queued or running eval run. */ cancelRun(request: operations.CancelEvalRunRequest, options?: RequestOptions): Promise; /** * Rerun failed eval samples * * @remarks * Queues a completed eval run to retry only samples with sample status 'failed'. Retried samples keep the same run ID and sample IDs, replacing errored sample results in place. */ rerunFailedSamples(request: operations.RerunFailedEvalSamplesRequest, options?: RequestOptions): Promise; /** * Retry an eval run * * @remarks * Queues a failed or cancelled eval run to retry in place. Completed samples keep their results, generated-but-unscored cancelled samples resume at scoring, unfinished samples are requeued, and runs with no persisted samples are prepared from scratch with the same run ID. */ retryFailedRun(request: operations.RetryFailedEvalRunRequest, options?: RequestOptions): Promise; /** * List eval samples * * @remarks * Returns prompt, target, model output, extraction, score, and judge details for an eval run. */ listSamples(request: operations.ListEvalSamplesRequest, options?: RequestOptions): Promise; /** * Get eval sample audio * * @remarks * Returns `{ object, audio, sample_index, task_id, model }` for a single transcription sample — the clip reference (a base64 `data:` URI or a URL) read from the un-redacted `dataset_row` (list responses redact inline `data:` blobs), plus the resolved task/model. Prefers the task's configured `audio_column`, falling back to a heuristic scan. The clip differs across tasks, so pass `task_id` when a run has more than one transcription task; if the index spans multiple tasks and `task_id` is omitted, returns 400 rather than guessing. `model` is optional (the clip is identical across models) and only extends the index prefix. Returns 404 when the sample or its audio is absent, 413 when the inline clip is too large. Returns 503 when object storage is briefly unreachable, as distinct from 404 for a clip that is genuinely gone. */ getSampleAudio(request: operations.GetEvalSampleAudioRequest, options?: RequestOptions): Promise; /** * Get eval run artifacts * * @remarks * Returns generated result and sample artifact file IDs for a completed eval run. */ getArtifacts(request: operations.GetEvalArtifactsRequest, options?: RequestOptions): Promise; /** * Import historical eval results from Hugging Face * * @remarks * Imports simplified historical samples.jsonl-style eval results from a Hugging Face dataset repository into a completed eval suite run shape. Aggregates are recomputed from samples and include canonical score metrics for leaderboards. */ importHistoricalResults(request: operations.ImportHistoricalEvalResultsRequest, options?: RequestOptions): Promise; /** * Create an eval schedule * * @remarks * Schedules an eval run to fire on a cron or interval cadence, with optional webhook notifications of run status. */ createSchedule(request: operations.CreateEvalScheduleRequest, options?: RequestOptions): Promise; /** * List eval schedules * * @remarks * Returns eval schedules visible to the authenticated team context. */ listSchedules(request: operations.ListEvalSchedulesRequest, options?: RequestOptions): Promise; /** * Get an eval schedule * * @remarks * Retrieves an eval schedule by ID, including the next scheduled fire time from Temporal when resolvable. */ getSchedule(request: operations.GetEvalScheduleRequest, options?: RequestOptions): Promise; /** * Update an eval schedule * * @remarks * Updates an eval schedule's cadence, run config, webhook, overlap policy, or enabled state. */ updateSchedule(request: operations.UpdateEvalScheduleRequest, options?: RequestOptions): Promise; /** * Delete an eval schedule * * @remarks * Soft-deletes an eval schedule and removes its Temporal schedule. Historical runs are preserved. */ deleteSchedule(request: operations.DeleteEvalScheduleRequest, options?: RequestOptions): Promise; /** * Trigger an eval schedule now * * @remarks * Fires the schedule immediately, in addition to its normal cadence. */ triggerSchedule(request: operations.TriggerEvalScheduleRequest, options?: RequestOptions): Promise; /** * List runs created by an eval schedule * * @remarks * Returns eval runs that were created by the given schedule. suite_version may be used without suite_id here, since the schedule already pins the suite; supplying a suite_id that is not the schedule's returns 400. */ listScheduleRuns(request: operations.ListEvalScheduleRunsRequest, options?: RequestOptions): Promise; } //# sourceMappingURL=evals.d.ts.map