// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.

import { APIResource } from '../../core/resource';
import * as EvaluationsAPI from './evaluations';
import * as DatasetsAPI from '../datasets';
import * as EvaluationItemsAPI from '../evaluation-items';
import * as InferenceAPI from '../inference';
import * as Shared from '../shared';
import * as CompletionsAPI from '../chat/completions';
import * as TasksAPI from './tasks';
import { TaskAddParams, TaskUpdateParams, Tasks } from './tasks';
import * as RubricsAPI from '../rubrics/rubrics';
import { APIPromise } from '../../core/api-promise';
import { CursorPage, type CursorPageParams, PagePromise } from '../../core/pagination';
import { RequestOptions } from '../../internal/request-options';
import { path } from '../../internal/utils/path';

export class Evaluations extends APIResource {
  tasks: TasksAPI.Tasks = new TasksAPI.Tasks(this._client);

  /**
   * Create an evaluation together with its items, optionally running test criteria
   * against them.
   *
   * Accepts three request shapes: standalone (inline `data`), from an existing
   * dataset (`dataset_id` with optional per-item references), or with a new reusable
   * dataset created inline from `data`. When the evaluation includes tasks that
   * require execution (for example an LLM judge or custom function), an async job
   * and a Temporal workflow are started and the evaluation is returned immediately
   * with status `running`; task results and `error_count` populate asynchronously.
   * When it includes only contributor tasks, taxonomy-only input, or no tasks, no
   * workflow runs and it is returned with status `completed`. Optional `tasks`,
   * `metadata`, `tags`, and `taxonomy_params` are persisted alongside the evaluation
   * and its items.
   */
  create(params: EvaluationCreateParams, options?: RequestOptions): APIPromise<Evaluation> {
    const { evaluation } = params;
    return this._client.post('/v5/evaluations', { body: evaluation, ...options });
  }

  /**
   * Retrieve a single evaluation by ID.
   *
   * Returns the evaluation with its datasets, async-job progress, metadata, and
   * task-error count. Archived evaluations are excluded unless `include_archived` is
   * set. Pass the `tasks` view to include the evaluation's task configurations in
   * the response.
   */
  retrieve(
    evaluationID: string,
    query: EvaluationRetrieveParams | null | undefined = {},
    options?: RequestOptions,
  ): APIPromise<Evaluation> {
    return this._client.get(path`/v5/evaluations/${evaluationID}`, { query, ...options });
  }

  /**
   * Update an evaluation's mutable fields, or restore it from the archive.
   *
   * The action is selected by the request body: a restore request un-archives the
   * evaluation and cascades the restore to its items and dashboards, while any other
   * body applies a partial update to fields such as name, description, tags, and
   * metadata (metadata is applied as an RFC 7396 merge patch). Updating an
   * already-archived evaluation is rejected — restore it first. The evaluation row
   * is locked for the duration of the write to avoid concurrent-update races.
   */
  update(
    evaluationID: string,
    params: EvaluationUpdateParams,
    options?: RequestOptions,
  ): APIPromise<Evaluation> {
    const { evaluation } = params;
    return this._client.patch(path`/v5/evaluations/${evaluationID}`, { body: evaluation, ...options });
  }

  /**
   * List evaluations for the account, with pagination.
   *
   * Supports filtering by case-insensitive name substring and by tags; archived
   * evaluations are excluded unless `include_archived` is set. Pass the `tasks` view
   * to include each evaluation's task configurations in the response. Use this for
   * simple name or tag lookups; to filter on metadata key-value pairs or status, use
   * the filter endpoint instead.
   */
  list(
    query: EvaluationListParams | null | undefined = {},
    options?: RequestOptions,
  ): PagePromise<EvaluationsCursorPage, Evaluation> {
    return this._client.getAPIList('/v5/evaluations', CursorPage<Evaluation>, { query, ...options });
  }

  /**
   * Archive (soft-delete) an evaluation.
   *
   * Sets the evaluation's archived timestamp rather than permanently deleting it,
   * and cascades the archive to the evaluation's items and dashboards while removing
   * it from any evaluation groups. The evaluation can later be brought back with a
   * restore request to the update endpoint.
   */
  archive(evaluationID: string, options?: RequestOptions): APIPromise<Evaluation> {
    return this._client.delete(path`/v5/evaluations/${evaluationID}`, options);
  }

  /**
   * Filter evaluations by metadata, status, and tags.
   *
   * Accepts up to 10 filters combined with AND logic, each comparing a key against a
   * value with an operator (`==`, `!=`, `>=`, `<=`, `IN`, `NOT_IN`). Filter on
   * metadata keys returned by the metadata-keys endpoint, plus the built-in `status`
   * and `tag` keys. Archived evaluations are excluded unless `include_archived` is
   * set, and the `tasks` view includes task configurations in each result. Use this
   * for metadata or status filtering; for simple name or tag lookups the list
   * endpoint is sufficient.
   */
  filter(
    params: EvaluationFilterParams,
    options?: RequestOptions,
  ): PagePromise<EvaluationsCursorPage, Evaluation> {
    const { ending_before, include_archived, limit, sort_by, sort_order, starting_after, views, ...body } =
      params;
    return this._client.getAPIList('/v5/evaluations/filter', CursorPage<Evaluation>, {
      query: { ending_before, include_archived, limit, sort_by, sort_order, starting_after, views },
      body,
      method: 'post',
      ...options,
    });
  }

  /**
   * Describe the data schema of an evaluation's items.
   *
   * Inspects the item `data` and task-result fields and returns each discovered
   * field with its flattened key path, JSON type, source, and the number of items
   * containing it, ordered alphabetically by field name. For large evaluations the
   * schema may be inferred from a sample of items, in which case `is_sampled` is set
   * and `sample_size` reports how many were analyzed. Set `include_archived` to
   * include archived items in the analysis.
   */
  retrieveSchema(
    evaluationID: string,
    query: EvaluationRetrieveSchemaParams | null | undefined = {},
    options?: RequestOptions,
  ): APIPromise<EvaluationSchemaResponse> {
    return this._client.get(path`/v5/evaluations/${evaluationID}/schema`, { query, ...options });
  }

  /**
   * Get the taxonomy JSON for an evaluation's contributor question tasks.
   *
   * Returns the raw taxonomy document stored for the evaluation. Responds with a
   * not-found error if the evaluation has no taxonomy.
   */
  retrieveTaxonomy(
    evaluationID: string,
    options?: RequestOptions,
  ): APIPromise<EvaluationRetrieveTaxonomyResponse> {
    return this._client.get(path`/v5/evaluations/${evaluationID}/taxonomy`, options);
  }
}

export type EvaluationsCursorPage = CursorPage<Evaluation>;

export interface AndEvaluationRunCondition {
  operands: Array<unknown>;

  op?: 'and';
}

export interface AutoEvaluationAgentTaskRequestWithItemLocator {
  definition: string;

  name: string;

  output_rules: Array<string>;

  data_fields?: Array<string>;

  designated_to?:
    | AutoEvaluationAgentTaskRequestWithItemLocator.ApeAgent
    | AutoEvaluationAgentTaskRequestWithItemLocator.IfAgent
    | AutoEvaluationAgentTaskRequestWithItemLocator.TruthfulnessAgent
    | AutoEvaluationAgentTaskRequestWithItemLocator.BaseAgent;

  output_type?: 'text' | 'integer' | 'float' | 'boolean';

  output_values?: Array<string | number | boolean>;

  rubric_id?: string;

  rubric_version?: number;
}

export namespace AutoEvaluationAgentTaskRequestWithItemLocator {
  export interface ApeAgent {
    config: ApeAgent.Config;

    agent_name?: 'APEAgent';
  }

  export namespace ApeAgent {
    export interface Config {
      model?: string;

      temperature?: number;
    }
  }

  export interface IfAgent {
    config: IfAgent.Config;

    agent_name?: 'IFAgent';
  }

  export namespace IfAgent {
    export interface Config {
      model?: string;
    }
  }

  export interface TruthfulnessAgent {
    config: TruthfulnessAgent.Config;

    agent_name?: 'TruthfulnessAgent';
  }

  export namespace TruthfulnessAgent {
    export interface Config {
      model?: string;
    }
  }

  export interface BaseAgent {
    config: BaseAgent.Config;

    agent_name?: 'BaseAgent';
  }

  export namespace BaseAgent {
    export interface Config {
      model?: string;
    }
  }
}

export interface EqEvaluationRunCondition {
  left: unknown;

  right: unknown;

  op?: 'eq';
}

export interface Evaluation {
  /**
   * The unique identifier of the entity.
   */
  id: string;

  /**
   * The date and time when the entity was created in ISO format.
   */
  created_at: string;

  /**
   * The identity that created the entity.
   */
  created_by: Shared.Identity;

  datasets: Array<DatasetsAPI.Dataset> | null;

  name: string;

  status: 'failed' | 'completed' | 'running';

  /**
   * The tags associated with the entity
   */
  tags: Array<string> | null;

  /**
   * The date and time when the entity was archived in ISO format.
   */
  archived_at?: string;

  description?: string;

  /**
   * Number of task errors across all items in this evaluation.
   */
  error_count?: number;

  /**
   * Metadata key-value pairs for the evaluation
   */
  metadata?: { [key: string]: unknown };

  object?: 'evaluation';

  /**
   * Progress of the evaluation's underlying async job
   */
  progress?: EvaluationTasksProgressSchema;

  /**
   * Reason for evaluation status
   */
  status_reason?: string;

  /**
   * Tasks executed during evaluation. Populated with optional `task` view.
   */
  tasks?: Array<EvaluationTask>;
}

/**
 * Schema information for an evaluation's item data structure
 */
export interface EvaluationSchemaResponse {
  /**
   * The ID of the evaluation
   */
  evaluation_id: string;

  /**
   * List of all discovered fields, ordered alphabetically by field_name
   */
  fields: Array<EvaluationSchemaResponse.Field>;

  /**
   * Total number of evaluation items
   */
  total_items: number;

  /**
   * Whether schema was computed from a sample of items (for large evaluations)
   */
  is_sampled?: boolean;

  object?: 'evaluation_schema';

  /**
   * Number of items sampled for schema inference, if applicable
   */
  sample_size?: number;
}

export namespace EvaluationSchemaResponse {
  /**
   * Schema information for a single field in evaluation item data
   */
  export interface Field {
    /**
     * JSON type: 'string', 'number', 'boolean', 'object', 'array', or 'null'
     */
    data_type: string;

    /**
     * The flattened JSON key path (e.g., 'metadata.category')
     */
    field_name: string;

    /**
     * Number of evaluation items containing this field
     */
    item_count: number;

    /**
     * The source of the field: 'data' or 'task_result_cache'
     */
    source: 'data' | 'task_result_cache';

    object?: 'field_schema';
  }
}

export type EvaluationTask =
  | EvaluationTask.ChatCompletionEvaluationTask
  | EvaluationTask.GenericInferenceEvaluationTask
  | EvaluationTask.ApplicationVariantV1EvaluationTask
  | EvaluationTask.AgentexOutputEvaluationTask
  | EvaluationTask.MetricEvaluationTask
  | EvaluationTask.AutoEvaluationQuestionTask
  | EvaluationTask.AutoEvaluationGuidedDecodingEvaluationTask
  | EvaluationTask.AutoEvaluationAgentEvaluationTask
  | EvaluationTask.ContributorEvaluationQuestionTask
  | EvaluationTask.CustomFunctionEvaluationTask;

export namespace EvaluationTask {
  export interface ChatCompletionEvaluationTask {
    configuration: ChatCompletionEvaluationTask.Configuration;

    /**
     * Alias to title the results column. Defaults to the `chat_completion`
     */
    alias?: string;

    task_type?: 'chat_completion';
  }

  export namespace ChatCompletionEvaluationTask {
    export interface Configuration {
      /**
       * openai standard message format
       */
      messages: Array<{ [key: string]: unknown }> | EvaluationsAPI.ItemLocator;

      /**
       * model specified as `model_vendor/model`, for example `openai/gpt-4o`
       */
      model: string;

      /**
       * Parameters for audio output. Required when audio output is requested with
       * modalities: ['audio'].
       */
      audio?: { [key: string]: unknown } | EvaluationsAPI.ItemLocator;

      /**
       * Number between -2.0 and 2.0. Positive values penalize new tokens based on their
       * existing frequency in the text so far.
       */
      frequency_penalty?: number | EvaluationsAPI.ItemLocator;

      /**
       * Deprecated in favor of tool_choice. Controls which function is called by the
       * model.
       */
      function_call?: { [key: string]: unknown } | EvaluationsAPI.ItemLocator;

      /**
       * Deprecated in favor of tools. A list of functions the model may generate JSON
       * inputs for.
       */
      functions?: Array<{ [key: string]: unknown }> | EvaluationsAPI.ItemLocator;

      /**
       * Modify the likelihood of specified tokens appearing in the completion. Maps
       * tokens to bias values from -100 to 100.
       */
      logit_bias?: { [key: string]: number } | EvaluationsAPI.ItemLocator;

      /**
       * Whether to return log probabilities of the output tokens or not.
       */
      logprobs?: boolean | EvaluationsAPI.ItemLocator;

      /**
       * An upper bound for the number of tokens that can be generated, including visible
       * output tokens and reasoning tokens.
       */
      max_completion_tokens?: number | EvaluationsAPI.ItemLocator;

      /**
       * Deprecated in favor of max_completion_tokens. The maximum number of tokens to
       * generate.
       */
      max_tokens?: number | EvaluationsAPI.ItemLocator;

      /**
       * Developer-defined tags and values used for filtering completions in the
       * dashboard.
       */
      metadata?: { [key: string]: string } | EvaluationsAPI.ItemLocator;

      /**
       * Output types that you would like the model to generate for this request.
       */
      modalities?: Array<string> | EvaluationsAPI.ItemLocator;

      /**
       * How many chat completion choices to generate for each input message.
       */
      n?: number | EvaluationsAPI.ItemLocator;

      /**
       * Whether to enable parallel function calling during tool use.
       */
      parallel_tool_calls?: boolean | EvaluationsAPI.ItemLocator;

      /**
       * Static predicted output content, such as the content of a text file being
       * regenerated.
       */
      prediction?: { [key: string]: unknown } | EvaluationsAPI.ItemLocator;

      /**
       * Number between -2.0 and 2.0. Positive values penalize tokens based on whether
       * they appear in the text so far.
       */
      presence_penalty?: number | EvaluationsAPI.ItemLocator;

      /**
       * For o1 models only. Constrains effort on reasoning. Values: low, medium, high.
       */
      reasoning_effort?: string;

      /**
       * An object specifying the format that the model must output.
       */
      response_format?: { [key: string]: unknown } | EvaluationsAPI.ItemLocator;

      /**
       * If specified, system will attempt to sample deterministically for repeated
       * requests with same seed.
       */
      seed?: number | EvaluationsAPI.ItemLocator;

      /**
       * Up to 4 sequences where the API will stop generating further tokens.
       */
      stop?: string | Array<string>;

      /**
       * Whether to store the output for use in model distillation or evals products.
       */
      store?: boolean | EvaluationsAPI.ItemLocator;

      /**
       * What sampling temperature to use. Higher values make output more random, lower
       * more focused.
       */
      temperature?: number | EvaluationsAPI.ItemLocator;

      /**
       * Controls which tool is called by the model. Values: none, auto, required, or
       * specific tool.
       */
      tool_choice?: string | { [key: string]: unknown };

      /**
       * A list of tools the model may call. Currently, only functions are supported. Max
       * 128 functions.
       */
      tools?: Array<{ [key: string]: unknown }> | EvaluationsAPI.ItemLocator;

      /**
       * Only sample from the top K options for each subsequent token
       */
      top_k?: number | EvaluationsAPI.ItemLocator;

      /**
       * Number of most likely tokens to return at each position, with associated log
       * probability.
       */
      top_logprobs?: number | EvaluationsAPI.ItemLocator;

      /**
       * Alternative to temperature. Only tokens comprising top_p probability mass are
       * considered.
       */
      top_p?: number | EvaluationsAPI.ItemLocator;

      [k: string]: unknown;
    }
  }

  export interface GenericInferenceEvaluationTask {
    configuration: GenericInferenceEvaluationTask.Configuration;

    /**
     * Alias to title the results column. Defaults to the `inference`
     */
    alias?: string;

    task_type?: 'inference';
  }

  export namespace GenericInferenceEvaluationTask {
    export interface Configuration {
      /**
       * model specified as `vendor/name` (ex. openai/gpt-5)
       */
      model: string;

      /**
       * Arguments passed into model
       */
      args?: { [key: string]: unknown } | EvaluationsAPI.ItemLocator;

      /**
       * Vendor specific configuration
       */
      inference_configuration?: InferenceAPI.LaunchInferenceConfiguration | EvaluationsAPI.ItemLocator;
    }
  }

  export interface ApplicationVariantV1EvaluationTask {
    configuration: ApplicationVariantV1EvaluationTask.Configuration;

    /**
     * Alias to title the results column. Defaults to the `application_variant`
     */
    alias?: string;

    task_type?: 'application_variant';
  }

  export namespace ApplicationVariantV1EvaluationTask {
    export interface Configuration {
      application_variant_id: string;

      /**
       * Input data for the application. For agents service variants, you must provide
       * inputs as a mapping from `{input_name: input_value}`. For V0 variants, you must
       * specify the node your input should be passed to, structuring your input as
       * `{node_id: {input_name: input_value}}`.
       */
      inputs: { [key: string]: unknown } | EvaluationsAPI.ItemLocator;

      /**
       * History of the application
       */
      history?: Array<Configuration.ApplicationRequestResponsePairArray> | EvaluationsAPI.ItemLocator;

      /**
       * Arbitrary user-defined metadata that can be attached to the process operations
       * and will be registered in the interaction.
       */
      operation_metadata?: { [key: string]: unknown } | EvaluationsAPI.ItemLocator;

      /**
       * Optional overrides for the application
       */
      overrides?:
        | Configuration.AgenticApplicationOverrides
        | { [key: string]: Configuration.KnowledgeBaseNodeOverride }
        | EvaluationsAPI.ItemLocator;
    }

    export namespace Configuration {
      export interface ApplicationRequestResponsePairArray {
        /**
         * Request inputs
         */
        request: string;

        /**
         * Response outputs
         */
        response: string;

        /**
         * Session data corresponding to the request response pair
         */
        session_data?: { [key: string]: unknown };
      }

      /**
       * Execution override options for agentic applications
       */
      export interface AgenticApplicationOverrides {
        concurrent?: boolean;

        initial_state?: AgenticApplicationOverrides.InitialState;

        partial_trace?: Array<AgenticApplicationOverrides.PartialTrace>;

        return_span?: boolean;

        use_channels?: boolean;
      }

      export namespace AgenticApplicationOverrides {
        export interface InitialState {
          current_node: string;

          state: { [key: string]: unknown };
        }

        export interface PartialTrace {
          duration_ms: number;

          node_id: string;

          operation_input: string;

          operation_output: string;

          operation_type: string;

          start_timestamp: string;

          workflow_id: string;

          operation_metadata?: { [key: string]: unknown };
        }
      }

      export interface KnowledgeBaseNodeOverride {
        artifact_ids_filter?: Array<string>;

        artifact_name_regex?: Array<string>;

        type?: 'knowledge_base_schema';
      }
    }
  }

  export interface AgentexOutputEvaluationTask {
    configuration: AgentexOutputEvaluationTask.Configuration;

    /**
     * Alias to title the results column. Defaults to the `agentex_output`
     */
    alias?: string;

    task_type?: 'agentex_output';
  }

  export namespace AgentexOutputEvaluationTask {
    export interface Configuration {
      /**
       * The ID of the Agentex agent to use
       */
      agentex_agent_id: string;

      /**
       * The dataset column to use as input for the agent
       */
      input_column: string | { [key: string]: unknown } | Array<unknown>;

      /**
       * Extra params merged into the Agentex `task/create` call's `params` object and
       * forwarded verbatim to the agent. Required by agents that demand configuration at
       * task creation -- the golden agent, for example, rejects any task whose params
       * omit `config_id`. SGP always pins `is_eval: true`; a caller-supplied
       * `description` overrides the SGP default. Nested `item.`-prefixed strings and
       * `{{item.x}}` templates are resolved per evaluation item, so a per-row
       * `config_id` can come from a dataset column.
       */
      agent_task_params?: { [key: string]: unknown } | EvaluationsAPI.ItemLocator;

      /**
       * How the agent's first turn is judged finished. `first_message` (the default)
       * grades the first non-empty agent text message after the input, which is cheap
       * but grades a streaming harness on whatever text block streamed first.
       * `turn_quiescence` keeps listening while the agent is still producing messages
       * and grades once at least one agent text message exists and nothing new has
       * arrived for `quiescence_seconds` -- the right choice for tool-using agents.
       * Neither mode requires the agent to mark the task complete; a terminal task
       * status always ends the wait, and `timeout_seconds` always bounds it.
       */
      completion_mode?: 'first_message' | 'turn_quiescence';

      /**
       * Optional Agentex deployment ID to pin the eval to a specific deployment. When
       * set, RPC traffic routes through
       * /agents/{agent_id}/deployments/{deployment_id}/rpc. When unset, traffic uses the
       * agent's default RPC endpoint, which resolves through the agent's current routing
       * rules on the Agentex side.
       */
      deployment_id?: string;

      /**
       * Whether to include trace data in the evaluation results. Traces are read from
       * SGP's own span store for the agent's trace, not from Agentex.
       */
      include_traces?: boolean | EvaluationsAPI.ItemLocator;

      /**
       * How the resolved `input_column` is delivered to the agent. `text` (the default)
       * sends a TextContent message with the value stringified. `data` sends a
       * DataContent message whose `data` is the value as a JSON object; the resolved
       * value must be an object, or a string that parses to one. Most agents accept text
       * only and reject `data`.
       */
      input_mode?: 'text' | 'data';

      /**
       * Seconds of no new messages before `completion_mode: turn_quiescence` considers
       * the turn finished. Ignored in `first_message` mode. Should exceed the agent's
       * longest expected gap between messages (a slow tool call), or the turn is graded
       * early.
       */
      quiescence_seconds?: number | EvaluationsAPI.ItemLocator;

      /**
       * Maximum seconds to wait for the agent's first-turn response per item. If not
       * set, the server-side default of 600s applies. Capped at 1500s to stay within the
       * evaluation item activity's 1800s start-to-close budget.
       */
      timeout_seconds?: number | EvaluationsAPI.ItemLocator;
    }
  }

  export interface MetricEvaluationTask {
    configuration:
      | MetricEvaluationTask.BleuScorerConfigWithItemLocator
      | MetricEvaluationTask.MeteorScorerConfigWithItemLocator
      | MetricEvaluationTask.CosineSimilarityScorerConfigWithItemLocator
      | MetricEvaluationTask.F1ScorerConfigWithItemLocator
      | MetricEvaluationTask.RougeScorer1ConfigWithItemLocator
      | MetricEvaluationTask.RougeScorer2ConfigWithItemLocator
      | MetricEvaluationTask.RougeScorerLConfigWithItemLocator;

    /**
     * Alias to title the results column. Defaults to the metric type specified in the
     * configuration
     */
    alias?: string;

    task_type?: 'metric';
  }

  export namespace MetricEvaluationTask {
    export interface BleuScorerConfigWithItemLocator {
      candidate: string;

      reference: string;

      type: 'bleu';
    }

    export interface MeteorScorerConfigWithItemLocator {
      candidate: string;

      reference: string;

      type: 'meteor';
    }

    export interface CosineSimilarityScorerConfigWithItemLocator {
      candidate: string;

      reference: string;

      type: 'cosine_similarity';
    }

    export interface F1ScorerConfigWithItemLocator {
      candidate: string;

      reference: string;

      type: 'f1';
    }

    export interface RougeScorer1ConfigWithItemLocator {
      candidate: string;

      reference: string;

      type: 'rouge1';
    }

    export interface RougeScorer2ConfigWithItemLocator {
      candidate: string;

      reference: string;

      type: 'rouge2';
    }

    export interface RougeScorerLConfigWithItemLocator {
      candidate: string;

      reference: string;

      type: 'rougeL';
    }
  }

  export interface AutoEvaluationQuestionTask {
    configuration: AutoEvaluationQuestionTask.Configuration;

    /**
     * Alias to title the results column. Defaults to the `auto_evaluation_question`
     */
    alias?: string;

    task_type?: 'auto_evaluation.question';
  }

  export namespace AutoEvaluationQuestionTask {
    export interface Configuration {
      /**
       * model specified as `model_vendor/model_name`
       */
      model: string;

      prompt: string;

      /**
       * question to be evaluated
       */
      question_id: string;
    }
  }

  export interface AutoEvaluationGuidedDecodingEvaluationTask {
    configuration:
      | AutoEvaluationGuidedDecodingEvaluationTask.AutoEvaluationStructuredOutputTaskRequestWithItemLocator
      | AutoEvaluationGuidedDecodingEvaluationTask.AutoEvaluationGuidedDecodingTaskRequestWithItemLocator
      | EvaluationsAPI.AutoEvaluationAgentTaskRequestWithItemLocator;

    /**
     * Alias to title the results column. Defaults to the
     * `auto_evaluation_guided_decoding`
     */
    alias?: string;

    task_type?: 'auto_evaluation.guided_decoding';
  }

  export namespace AutoEvaluationGuidedDecodingEvaluationTask {
    export interface AutoEvaluationStructuredOutputTaskRequestWithItemLocator {
      /**
       * model specified as `model_vendor/model_name`
       */
      model: string;

      prompt: string;

      /**
       * JSON schema used for structuring the model response
       */
      response_format: { [key: string]: unknown };

      /**
       * Additional arguments to pass to the inference request
       */
      inference_args?: { [key: string]: unknown };

      run_condition?:
        | AutoEvaluationStructuredOutputTaskRequestWithItemLocator.ConstEvaluationRunCondition
        | AutoEvaluationStructuredOutputTaskRequestWithItemLocator.VarEvaluationRunCondition
        | EvaluationsAPI.EqEvaluationRunCondition
        | EvaluationsAPI.NeEvaluationRunCondition
        | EvaluationsAPI.LtEvaluationRunCondition
        | EvaluationsAPI.LteEvaluationRunCondition
        | EvaluationsAPI.GtEvaluationRunCondition
        | EvaluationsAPI.GteEvaluationRunCondition
        | EvaluationsAPI.AndEvaluationRunCondition
        | EvaluationsAPI.OrEvaluationRunCondition
        | EvaluationsAPI.InEvaluationRunCondition
        | EvaluationsAPI.NotInEvaluationRunCondition
        | EvaluationsAPI.NotEvaluationRunCondition
        | EvaluationsAPI.IsNullEvaluationRunCondition
        | EvaluationsAPI.IsNotNullEvaluationRunCondition;

      system_prompt?: string;
    }

    export namespace AutoEvaluationStructuredOutputTaskRequestWithItemLocator {
      export interface ConstEvaluationRunCondition {
        op?: 'const';

        value?: string | number | boolean;
      }

      export interface VarEvaluationRunCondition {
        path: string;

        op?: 'var';
      }
    }

    export interface AutoEvaluationGuidedDecodingTaskRequestWithItemLocator {
      /**
       * Choices array cannot be empty
       */
      choices: Array<string>;

      /**
       * model specified as `model_vendor/model_name`
       */
      model: string;

      prompt: string;

      /**
       * Additional arguments to pass to the inference request
       */
      inference_args?: { [key: string]: unknown };

      run_condition?:
        | AutoEvaluationGuidedDecodingTaskRequestWithItemLocator.ConstEvaluationRunCondition
        | AutoEvaluationGuidedDecodingTaskRequestWithItemLocator.VarEvaluationRunCondition
        | EvaluationsAPI.EqEvaluationRunCondition
        | EvaluationsAPI.NeEvaluationRunCondition
        | EvaluationsAPI.LtEvaluationRunCondition
        | EvaluationsAPI.LteEvaluationRunCondition
        | EvaluationsAPI.GtEvaluationRunCondition
        | EvaluationsAPI.GteEvaluationRunCondition
        | EvaluationsAPI.AndEvaluationRunCondition
        | EvaluationsAPI.OrEvaluationRunCondition
        | EvaluationsAPI.InEvaluationRunCondition
        | EvaluationsAPI.NotInEvaluationRunCondition
        | EvaluationsAPI.NotEvaluationRunCondition
        | EvaluationsAPI.IsNullEvaluationRunCondition
        | EvaluationsAPI.IsNotNullEvaluationRunCondition;

      system_prompt?: string;
    }

    export namespace AutoEvaluationGuidedDecodingTaskRequestWithItemLocator {
      export interface ConstEvaluationRunCondition {
        op?: 'const';

        value?: string | number | boolean;
      }

      export interface VarEvaluationRunCondition {
        path: string;

        op?: 'var';
      }
    }
  }

  export interface AutoEvaluationAgentEvaluationTask {
    configuration: EvaluationsAPI.AutoEvaluationAgentTaskRequestWithItemLocator;

    /**
     * Alias to title the results column. Defaults to the `auto_evaluation_agent`
     */
    alias?: string;

    task_type?: 'auto_evaluation.agent';
  }

  export interface ContributorEvaluationQuestionTask {
    configuration: ContributorEvaluationQuestionTask.Configuration;

    /**
     * Alias to title the results column. Defaults to the
     * `contributor_evaluation_question`
     */
    alias?: string;

    task_type?: 'contributor_evaluation.question';
  }

  export namespace ContributorEvaluationQuestionTask {
    export interface Configuration {
      layout: EvaluationItemsAPI.Container;

      question_id: string;

      /**
       * Dataset column to prefill contributor question task result
       */
      prefill_from?: string;

      /**
       * The contributor annotation queue to include this task in. Defaults to `default`
       */
      queue_id?: string;

      /**
       * Whether the question is required to be answered
       */
      required?: boolean;

      /**
       * ID of the rubric to use for scoring this evaluation question
       */
      rubric_id?: string;
    }
  }

  export interface CustomFunctionEvaluationTask {
    /**
     * Configuration for a custom Python function evaluation task.
     */
    configuration: CustomFunctionEvaluationTask.Configuration;

    /**
     * Alias to title the results column. Defaults to the function name.
     */
    alias?: string;

    task_type?: 'custom_function';
  }

  export namespace CustomFunctionEvaluationTask {
    /**
     * Configuration for a custom Python function evaluation task.
     */
    export interface Configuration {
      /**
       * Python function source code
       */
      function_source: string;

      /**
       * Mapping of function parameter names to item locators (e.g. item.field).
       * Auto-derived from function signature if not provided.
       */
      arg_mapping?: { [key: string]: string };

      /**
       * Literal argument values for function parameters, such as thresholds or RNG
       * seeds. Serialized JSON must be at most 10000 characters.
       */
      config_args?: { [key: string]: unknown };

      /**
       * Optional output paths to materialize as separate result columns. If omitted, the
       * function return value is stored only under the task alias/data key.
       */
      outputs?: Array<Configuration.Output>;
    }

    export namespace Configuration {
      export interface Output {
        /**
         * Dot path in the custom function return value to materialize.
         */
        path: string;

        /**
         * Result column alias. Defaults to path with dots replaced by underscores.
         */
        alias?: string;
      }
    }
  }
}

export interface EvaluationTasksProgressSchema {
  items?: EvaluationTasksProgressSchema.Items;

  workflows?: EvaluationTasksProgressSchema.Workflows;
}

export namespace EvaluationTasksProgressSchema {
  export interface Items {
    failed: number;

    pending: number;

    successful: number;

    total: number;

    failed_items?: Array<Items.FailedItem>;
  }

  export namespace Items {
    export interface FailedItem {
      item_id: string;

      error?: string;

      error_type?: string;
    }
  }

  export interface Workflows {
    completed: number;

    failed: number;

    pending: number;

    total: number;
  }
}

export type EvaluationViews = 'tasks';

export interface GtEvaluationRunCondition {
  left: unknown;

  right: unknown;

  op?: 'gt';
}

export interface GteEvaluationRunCondition {
  left: unknown;

  right: unknown;

  op?: 'gte';
}

export interface InEvaluationRunCondition {
  left: unknown;

  operands: Array<unknown>;

  op?: 'in';
}

export interface IsNotNullEvaluationRunCondition {
  operands: Array<unknown>;

  op?: 'is_not_null';
}

export interface IsNullEvaluationRunCondition {
  operands: Array<unknown>;

  op?: 'is_null';
}

export type ItemLocator = string;

export type ItemLocatorTemplate = string;

export interface LtEvaluationRunCondition {
  left: unknown;

  right: unknown;

  op?: 'lt';
}

export interface LteEvaluationRunCondition {
  left: unknown;

  right: unknown;

  op?: 'lte';
}

export interface NeEvaluationRunCondition {
  left: unknown;

  right: unknown;

  op?: 'ne';
}

export interface NotEvaluationRunCondition {
  operands: Array<unknown>;

  op?: 'not';
}

export interface NotInEvaluationRunCondition {
  left: unknown;

  operands: Array<unknown>;

  op?: 'not_in';
}

export interface OrEvaluationRunCondition {
  operands: Array<unknown>;

  op?: 'or';
}

export interface PaginatedListEvaluation {
  /**
   * Whether there are more items left to be fetched.
   */
  has_more: boolean;

  items: Array<Evaluation>;

  /**
   * The total of items that match the query. This is greater than or equal to the
   * number of items returned.
   */
  total: number;

  /**
   * The maximum number of items to return.
   */
  limit?: number;

  object?: 'list';
}

export type EvaluationRetrieveTaxonomyResponse = { [key: string]: unknown };

export interface EvaluationCreateParams {
  evaluation:
    | EvaluationCreateParams.EvaluationStandaloneCreateRequest
    | EvaluationCreateParams.EvaluationFromDatasetCreateRequest
    | EvaluationCreateParams.EvaluationWithDatasetCreateRequest;
}

export namespace EvaluationCreateParams {
  export interface EvaluationStandaloneCreateRequest {
    /**
     * Items to be evaluated
     */
    data: Array<{ [key: string]: unknown }>;

    name: string;

    description?: string;

    /**
     * Files to be associated to the evaluation
     */
    files?: Array<{ [key: string]: string }>;

    /**
     * Optional metadata key-value pairs for the evaluation
     */
    metadata?: { [key: string]: unknown };

    /**
     * Do not queue a contributor task for prefilled questions
     */
    skip_prefilled_rows?: boolean;

    /**
     * The tags associated with the evaluation
     */
    tags?: Array<string>;

    /**
     * Tasks allow you to augment and evaluate your data
     */
    tasks?: Array<EvaluationsAPI.EvaluationTask>;

    /**
     * Taxonomy params from the task builder. When provided, stores directly as
     * evaluation taxonomy.
     */
    taxonomy_params?: { [key: string]: unknown };
  }

  export interface EvaluationFromDatasetCreateRequest {
    /**
     * The ID of the dataset containing the items referenced by the `data` field
     */
    dataset_id: string;

    name: string;

    /**
     * Items to be evaluated, including references to the input dataset
     */
    data?: Array<EvaluationFromDatasetCreateRequest.Data>;

    description?: string;

    /**
     * Optional metadata key-value pairs for the evaluation
     */
    metadata?: { [key: string]: unknown };

    /**
     * Do not queue a contributor task for prefilled questions
     */
    skip_prefilled_rows?: boolean;

    /**
     * The tags associated with the evaluation
     */
    tags?: Array<string>;

    /**
     * Tasks allow you to augment and evaluate your data
     */
    tasks?: Array<EvaluationsAPI.EvaluationTask>;

    /**
     * Taxonomy params from the task builder. When provided, stores directly as
     * evaluation taxonomy.
     */
    taxonomy_params?: { [key: string]: unknown };
  }

  export namespace EvaluationFromDatasetCreateRequest {
    export interface Data {
      dataset_item_id: string;

      [k: string]: unknown;
    }
  }

  export interface EvaluationWithDatasetCreateRequest {
    /**
     * Items to be evaluated
     */
    data: Array<{ [key: string]: unknown }>;

    /**
     * Create a reusable dataset from items in the `data` field
     */
    dataset: EvaluationWithDatasetCreateRequest.Dataset;

    name: string;

    description?: string;

    /**
     * Files to be associated to the evaluation
     */
    files?: Array<{ [key: string]: string }>;

    /**
     * Optional metadata key-value pairs for the evaluation
     */
    metadata?: { [key: string]: unknown };

    /**
     * Do not queue a contributor task for prefilled questions
     */
    skip_prefilled_rows?: boolean;

    /**
     * The tags associated with the evaluation
     */
    tags?: Array<string>;

    /**
     * Tasks allow you to augment and evaluate your data
     */
    tasks?: Array<EvaluationsAPI.EvaluationTask>;

    /**
     * Taxonomy params from the task builder. When provided, stores directly as
     * evaluation taxonomy.
     */
    taxonomy_params?: { [key: string]: unknown };
  }

  export namespace EvaluationWithDatasetCreateRequest {
    /**
     * Create a reusable dataset from items in the `data` field
     */
    export interface Dataset {
      name: string;

      description?: string;

      /**
       * Keys from items in the `data` field that should be included in the dataset. If
       * not provided, all keys will be included.
       */
      keys?: Array<string>;

      /**
       * The tags associated with the entity
       */
      tags?: Array<string>;
    }
  }
}

export interface EvaluationRetrieveParams {
  include_archived?: boolean;

  views?: Array<EvaluationViews>;
}

export interface EvaluationUpdateParams {
  evaluation: EvaluationUpdateParams.PartialEvaluationUpdateRequest | RubricsAPI.RestoreRequest;
}

export namespace EvaluationUpdateParams {
  export interface PartialEvaluationUpdateRequest {
    description?: string;

    /**
     * Optional metadata key-value pairs for the evaluation
     */
    metadata?: { [key: string]: unknown };

    name?: string;

    /**
     * The tags associated with the evaluation
     */
    tags?: Array<string>;
  }
}

export interface EvaluationListParams extends CursorPageParams {
  include_archived?: boolean;

  name?: string;

  sort_by?: string;

  sort_order?: CompletionsAPI.SortOrder;

  tags?: Array<string>;

  views?: Array<EvaluationViews>;
}

export interface EvaluationFilterParams extends CursorPageParams {
  /**
   * Body param: List of metadata filters to apply (maximum 10)
   */
  filters: Array<EvaluationFilterParams.Filter>;

  /**
   * Query param
   */
  include_archived?: boolean;

  /**
   * Query param
   */
  sort_by?: string;

  /**
   * Query param
   */
  sort_order?: CompletionsAPI.SortOrder;

  /**
   * Query param
   */
  views?: Array<EvaluationViews>;
}

export namespace EvaluationFilterParams {
  /**
   * Individual metadata filter specification
   */
  export interface Filter {
    /**
     * The metadata key to filter on
     */
    key: string;

    /**
     * The comparison operator to use
     */
    operator: '==' | '!=' | '>=' | '<=' | 'IN' | 'NOT_IN';

    /**
     * The value to compare against (string for all types)
     */
    value: string;

    object?: 'metadata_filter';
  }
}

export interface EvaluationRetrieveSchemaParams {
  /**
   * Include archived items in schema analysis
   */
  include_archived?: boolean;
}

Evaluations.Tasks = Tasks;

export declare namespace Evaluations {
  export {
    type AndEvaluationRunCondition as AndEvaluationRunCondition,
    type AutoEvaluationAgentTaskRequestWithItemLocator as AutoEvaluationAgentTaskRequestWithItemLocator,
    type EqEvaluationRunCondition as EqEvaluationRunCondition,
    type Evaluation as Evaluation,
    type EvaluationSchemaResponse as EvaluationSchemaResponse,
    type EvaluationTask as EvaluationTask,
    type EvaluationTasksProgressSchema as EvaluationTasksProgressSchema,
    type EvaluationViews as EvaluationViews,
    type GtEvaluationRunCondition as GtEvaluationRunCondition,
    type GteEvaluationRunCondition as GteEvaluationRunCondition,
    type InEvaluationRunCondition as InEvaluationRunCondition,
    type IsNotNullEvaluationRunCondition as IsNotNullEvaluationRunCondition,
    type IsNullEvaluationRunCondition as IsNullEvaluationRunCondition,
    type ItemLocator as ItemLocator,
    type ItemLocatorTemplate as ItemLocatorTemplate,
    type LtEvaluationRunCondition as LtEvaluationRunCondition,
    type LteEvaluationRunCondition as LteEvaluationRunCondition,
    type NeEvaluationRunCondition as NeEvaluationRunCondition,
    type NotEvaluationRunCondition as NotEvaluationRunCondition,
    type NotInEvaluationRunCondition as NotInEvaluationRunCondition,
    type OrEvaluationRunCondition as OrEvaluationRunCondition,
    type PaginatedListEvaluation as PaginatedListEvaluation,
    type EvaluationRetrieveTaxonomyResponse as EvaluationRetrieveTaxonomyResponse,
    type EvaluationsCursorPage as EvaluationsCursorPage,
    type EvaluationCreateParams as EvaluationCreateParams,
    type EvaluationRetrieveParams as EvaluationRetrieveParams,
    type EvaluationUpdateParams as EvaluationUpdateParams,
    type EvaluationListParams as EvaluationListParams,
    type EvaluationFilterParams as EvaluationFilterParams,
    type EvaluationRetrieveSchemaParams as EvaluationRetrieveSchemaParams,
  };

  export { Tasks as Tasks, type TaskUpdateParams as TaskUpdateParams, type TaskAddParams as TaskAddParams };
}