import { BaseClient, RequestMethod } from '../base-client'; import { getSdkLogger } from 'galileo-generated'; import { Routes } from '../../types/routes.types'; import { ScorerTypes, ValidateCodeScorerResponse, RegisteredScorerTaskResultResponse, TaskStatus, ResultType, ValidateRegisteredScorerResult, CreateScorerRequest, CreateScorerRequestOpenAPI, ScorerResponseOpenAPI, ListScorersRequestOpenAPI, ScorerResponse, ListScorersRequest, ListScorersResponseOpenAPI, ListScorersResponse, BaseScorerVersionResponseOpenAPI, BaseScorerVersionResponse, CreateLlmScorerVersionRequest, CreateLlmScorerVersionRequestOpenAPI, DeleteScorerResponse, DeleteScorerResponseOpenAPI } from '../../types/scorer.types'; import { StepType } from '../../types/logging/step.types'; const sdkLogger = getSdkLogger(); export class ScorerService extends BaseClient { constructor(apiUrl: string, token: string) { super(); this.apiUrl = apiUrl; this.token = token; this.initializeClient(); } /** * Retrieves a list of scorers, optionally filtered by scorer type. * * @param type - (Optional) The type of scorer to filter by. * @returns A promise that resolves to an array of {@link Scorer} objects. */ public getScorers = async (options?: { type?: ScorerTypes; names?: string[]; }): Promise => { const payload: ListScorersRequest = { filters: [] }; if (options?.type) { payload.filters!.push({ name: 'scorer_type', value: options.type, operator: 'eq' }); } if (options?.names && options.names.length === 1) { payload.filters!.push({ name: 'name', value: options.names[0], operator: 'eq' }); } else if (options?.names && options.names.length > 1) { payload.filters!.push({ name: 'name', value: options.names, operator: 'one_of' }); } const request = this.convertToSnakeCase< ListScorersRequest, ListScorersRequestOpenAPI >(payload); const response = await this.makeRequest( RequestMethod.POST, Routes.scorers, request ); return this.convertToCamelCase< ListScorersResponseOpenAPI, ListScorersResponse >(response); }; /** * Retrieves a page of scorers with pagination support. */ public getScorersPage = async (options?: { name?: string; names?: string[]; types?: ScorerTypes[]; startingToken?: number; limit?: number; }): Promise => { const payload: ListScorersRequest = { filters: [] }; if (options?.types && options.types.length === 1) { payload.filters!.push({ name: 'scorer_type', value: options.types[0], operator: 'eq' }); } else if (options?.types && options.types.length > 1) { payload.filters!.push({ name: 'scorer_type', value: options.types, operator: 'one_of' }); } if (options?.name) { payload.filters!.push({ name: 'name', value: options.name, operator: 'eq' }); } if (options?.names && options.names.length === 1) { payload.filters!.push({ name: 'name', value: options.names[0], operator: 'eq' }); } else if (options?.names && options.names.length > 1) { payload.filters!.push({ name: 'name', value: options.names, operator: 'one_of' }); } const request = this.convertToSnakeCase< ListScorersRequest, ListScorersRequestOpenAPI >(payload); const response = await this.makeRequest( RequestMethod.POST, Routes.scorers, request, { starting_token: options?.startingToken ?? 0, limit: options?.limit ?? 100 } ); return this.convertToCamelCase< ListScorersResponseOpenAPI, ListScorersResponse >(response); }; /** * Retrieves a page of scorers filtered by label. * Uses ScorerLabelFilter with optional strict mode and case sensitivity. * * @param options.labels - Labels to search for. * @param options.strict - When false, also matches by scorer name as fallback. * @param options.caseSensitive - Whether label matching is case-sensitive. * @param options.startingToken - Pagination starting token. * @param options.limit - Maximum results per page. */ public getScorersPageByLabels = async (options: { labels: string[]; strict?: boolean; caseSensitive?: boolean; startingToken?: number; limit?: number; }): Promise => { if (options.labels.length === 0) { return { scorers: [], startingToken: 0 }; } const payload: ListScorersRequest = { filters: [] }; payload.filters!.push({ name: 'label', operator: options.labels.length === 1 ? 'eq' : 'one_of', value: options.labels.length === 1 ? options.labels[0] : options.labels, caseSensitive: options.caseSensitive ?? false, strict: options.strict }); const request = this.convertToSnakeCase< ListScorersRequest, ListScorersRequestOpenAPI >(payload); const response = await this.makeRequest( RequestMethod.POST, Routes.scorers, request, { starting_token: options.startingToken ?? 0, limit: options.limit ?? 100 } ); return this.convertToCamelCase< ListScorersResponseOpenAPI, ListScorersResponse >(response); }; /** * Retrieves a page of scorers filtered by ID. * Uses ScorerIDFilter. * * @param options.ids - Scorer IDs to search for. * @param options.startingToken - Pagination starting token. * @param options.limit - Maximum results per page. */ public getScorersPageByIds = async (options: { ids: string[]; startingToken?: number; limit?: number; }): Promise => { if (options.ids.length === 0) { return { scorers: [], startingToken: 0 }; } const payload: ListScorersRequest = { filters: [] }; payload.filters!.push({ name: 'id' as const, operator: options.ids.length === 1 ? 'eq' : 'one_of', value: options.ids.length === 1 ? options.ids[0] : options.ids }); const request = this.convertToSnakeCase< ListScorersRequest, ListScorersRequestOpenAPI >(payload); const response = await this.makeRequest( RequestMethod.POST, Routes.scorers, request, { starting_token: options.startingToken ?? 0, limit: options.limit ?? 100 } ); return this.convertToCamelCase< ListScorersResponseOpenAPI, ListScorersResponse >(response); }; /** * Retrieves a specific version of a scorer by its ID and version number. * * @param scorerId - The unique identifier of the scorer. * @param version - The version number of the scorer to retrieve. * @returns A promise that resolves to the requested {@link BaseScorerVersionResponse}. */ public getScorerVersion = async ( scorerId: string, version: number ): Promise => { const response = await this.makeRequest( RequestMethod.GET, Routes.scorerVersion, undefined, { scorer_id: scorerId, version: version } ); return this.convertToCamelCase< BaseScorerVersionResponseOpenAPI, BaseScorerVersionResponse >(response); }; /** * Creates a new scorer with the specified parameters. * @param options - The scorer creation options. * @param options.name - The name of the scorer. * @param options.scorerType - The type of the scorer. * @param options.description - (Optional) A description for the scorer. * @param options.tags - (Optional) Tags to associate with the scorer. * @param options.defaults - (Optional) Default settings for the scorer. Required for LLM scorers. * @param options.modelType - (Optional) The model type for the scorer. * @param options.defaultVersionId - (Optional) The default version ID for the scorer. * @param options.scoreableNodeTypes - (Optional) The node types that can be scored. * @param options.outputType - (Optional) The output type for the scorer. * @param options.inputType - (Optional) The input type for the scorer. * @returns A promise that resolves to the created scorer. */ public createScorer = async ( options: CreateScorerRequest ): Promise => { const requestBody: CreateScorerRequestOpenAPI = this.convertToSnakeCase< CreateScorerRequest, CreateScorerRequestOpenAPI >(options); const response = await this.makeRequest( RequestMethod.POST, Routes.scorer, requestBody ); return this.convertToCamelCase( response ); }; /** * Creates a new LLM scorer version for a given scorer. * * @param scorerId - The unique identifier of the scorer. * @param instructions - Instructions for the scorer version. * @param chainPollTemplate - The chain poll template for the scorer version. * @param userPrompt - (Optional) The user prompt for the scorer version. * @param scoreableNodeTypes - (Optional) The node level for the scorer version. Defaults to ['llm']. * @param cotEnabled - (Optional) Whether chain of thought is enabled. Defaults to * @param modelName - (Optional) The model name to use. * @param numJudges - (Optional) The number of judges to use. * @param outputType - (Optional) The output type for the scorer version. * @returns A promise that resolves to the created {@link ScorerVersion}. */ public createLLMScorerVersion = async ( scorerId: string, options: CreateLlmScorerVersionRequest ): Promise => { const request = this.convertToSnakeCase< CreateLlmScorerVersionRequest, CreateLlmScorerVersionRequestOpenAPI >(options); const response = await this.makeRequest( RequestMethod.POST, Routes.llmScorerVersion, request, { scorer_id: scorerId } ); return this.convertToCamelCase< BaseScorerVersionResponseOpenAPI, BaseScorerVersionResponse >(response); }; /** * Deletes a scorer by its unique identifier. * * @param id - The unique identifier of the scorer to delete. * @returns A promise that resolves when the scorer is deleted. */ public deleteScorer = async (id: string): Promise => { const response = await this.makeRequest( RequestMethod.DELETE, Routes.scorerId, undefined, { scorer_id: id } ); return this.convertToCamelCase< DeleteScorerResponseOpenAPI, DeleteScorerResponse >(response); }; public createCodeScorerVersion = async ( scorerId: string, codeContent: string, validationResult?: string ): Promise => { sdkLogger.info(`Creating metric version: ${scorerId}`); sdkLogger.debug(`Code content length: ${codeContent.length} bytes`); // Create FormData with the code content as a file const formData = new FormData(); const blob = new Blob([codeContent], { type: 'text/x-python' }); formData.append('file', blob, 'scorer.py'); // Add validation result if provided if (validationResult) { sdkLogger.debug(`Including validation result in request`); formData.append('validation_result', validationResult); } const result = await this.makeRequest( RequestMethod.POST, Routes.codeScorerVersion, formData, { scorer_id: scorerId } ); sdkLogger.info(`Metric version created: ${result.id}`); return this.convertToCamelCase< BaseScorerVersionResponseOpenAPI, BaseScorerVersionResponse >(result); }; /** * Validates code scorer content by submitting it for validation. * * @param codeContent - The Python code content to validate. * @param scoreableNodeTypes - The node types that this scorer can score. * @returns A promise that resolves to the validation task ID. */ public validateCodeScorer = async ( codeContent: string, scoreableNodeTypes: StepType[], requiredScorers?: string[] ): Promise => { sdkLogger.info(`Submitting code for validation...`); sdkLogger.debug(`Step type(s): ${JSON.stringify(scoreableNodeTypes)}`); const formData = new FormData(); const blob = new Blob([codeContent], { type: 'text/x-python' }); formData.append('file', blob, 'scorer.py'); formData.append('scoreable_node_types', JSON.stringify(scoreableNodeTypes)); if (requiredScorers && requiredScorers.length > 0) { formData.append('required_scorers', JSON.stringify(requiredScorers)); } const response = await this.makeRequest( RequestMethod.POST, Routes.codeScorerValidate, formData ); sdkLogger.info(`Validation task created: ${response.task_id}`); return response; }; /** * Gets the result of a code scorer validation task. * * @param taskId - The ID of the validation task. * @returns A promise that resolves to the validation result. */ public getCodeScorerValidationResult = async ( taskId: string ): Promise => { const path = Routes.codeScorerValidateResult.replace('{task_id}', taskId); return await this.makeRequest( RequestMethod.GET, path as Routes ); }; /** * Validates code scorer and waits for the result. * Polls the validation endpoint at specified intervals until complete or timeout. * * @param codeContent - The Python code content to validate. * @param scoreableNodeTypes - The node types that this scorer can score. * @param timeoutMs - Maximum time to wait for validation (default: 60000ms). * @param pollIntervalMs - Interval between polling attempts (default: 1000ms). * @returns A promise that resolves to the validation result. * @throws Error if validation fails or times out. */ public validateCodeScorerAndWait = async ( codeContent: string, scoreableNodeTypes: StepType[], timeoutMs: number = 60000, pollIntervalMs: number = 1000, requiredScorers?: string[] ): Promise => { // Submit validation request const { task_id } = await this.validateCodeScorer( codeContent, scoreableNodeTypes, requiredScorers ); const startTime = Date.now(); let pollCount = 0; // Poll for result while (Date.now() - startTime < timeoutMs) { pollCount++; const response = await this.getCodeScorerValidationResult(task_id); if (response.status === TaskStatus.COMPLETE) { sdkLogger.info(`Validation completed successfully`); // Parse result if it's a string let result: ValidateRegisteredScorerResult | null = null; if (typeof response.result === 'string') { try { result = JSON.parse( response.result ) as ValidateRegisteredScorerResult; } catch (err) { throw new Error( `Failed to parse validation result as JSON: ${err instanceof Error ? err.message : String(err)}` ); } } else { result = response.result; } if (result === null) { throw new Error('Validation completed but result is empty'); } // Check if result is invalid if (result.result.result_type === ResultType.INVALID) { sdkLogger.warn( `Validation result: INVALID - ${result.result.error_message}` ); throw new Error( `Code metric validation failed: ${result.result.error_message}` ); } sdkLogger.debug(` Score type: ${result.result.score_type}`); sdkLogger.debug( ` Step type(s): ${JSON.stringify(result.result.scoreable_node_types)}` ); return result; } if (response.status === TaskStatus.FAILED) { const errorMessage = typeof response.result === 'string' ? response.result : 'Validation task failed'; sdkLogger.error(`Validation task failed: ${errorMessage}`); throw new Error(`Code metric validation failed: ${errorMessage}`); } // Wait before next poll await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); } sdkLogger.warn( `Validation timed out after ${timeoutMs / 1000} seconds (${pollCount} polls)` ); throw new Error( `Code scorer validation timed out after ${timeoutMs / 1000} seconds` ); }; }