import React, { useEffect, useState } from "react"; import { Button } from "@/src/components/ui/button"; import { FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage, } from "@/src/components/ui/form"; import { Form } from "@/src/components/ui/form"; import { Textarea } from "@/src/components/ui/textarea"; import { ModelParameters } from "@/src/components/ModelParameters"; import { InputCommandEmpty, InputCommandGroup, InputCommandInput, InputCommandList, InputCommand, InputCommandItem, } from "@/src/components/ui/input-command"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/src/components/ui/select"; import { cn } from "@/src/utils/tailwind"; import { Popover, PopoverContent, PopoverTrigger, } from "@/src/components/ui/popover"; import { ChevronDown, CheckIcon, Info, CircleCheck, Loader2, } from "lucide-react"; import { api } from "@/src/utils/api"; import { Card, CardDescription, CardHeader, CardTitle, } from "@/src/components/ui/card"; import { showErrorToast } from "@/src/features/notifications/showErrorToast"; import { useModelParams } from "@/src/features/playground/page/hooks/useModelParams"; import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess"; import { Input } from "@/src/components/ui/input"; import { DialogHeader, DialogTitle, DialogDescription, Dialog, DialogContent, DialogBody, DialogFooter, } from "@/src/components/ui/dialog"; import Link from "next/link"; import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture"; import { TemplateSelector } from "@/src/features/evals/components/template-selector"; import { EvaluatorForm } from "@/src/features/evals/components/evaluator-form"; import { useEvaluatorDefaults } from "@/src/features/experiments/hooks/useEvaluatorDefaults"; import { useExperimentEvaluatorData } from "@/src/features/experiments/hooks/useExperimentEvaluatorData"; import { zodResolver } from "@hookform/resolvers/zod"; import { useForm } from "react-hook-form"; import { getFinalModelParams } from "@/src/utils/getFinalModelParams"; import { useExperimentNameValidation } from "@/src/features/experiments/hooks/useExperimentNameValidation"; import { useExperimentPromptData } from "@/src/features/experiments/hooks/useExperimentPromptData"; import { CreateExperimentData, type CreateExperiment, } from "@/src/features/experiments/types"; import { Skeleton } from "@/src/components/ui/skeleton"; export const PromptExperimentsForm = ({ projectId, setFormOpen, defaultValues = {}, promptDefault, handleExperimentSettled, handleExperimentSuccess, setShowPromptForm, showSDKRunInfoPage, }: { projectId: string; setFormOpen: (open: boolean) => void; defaultValues?: Partial; promptDefault?: { name: string; version: number; }; handleExperimentSuccess?: (data?: { success: boolean; datasetId: string; runId: string; runName: string; }) => Promise; handleExperimentSettled?: (data?: { success: boolean; datasetId: string; runId: string; runName: string; }) => Promise; setShowPromptForm: (open: boolean) => void; showSDKRunInfoPage?: boolean; }) => { const capture = usePostHogClientCapture(); const [open, setOpen] = useState(false); const [selectedPromptName, setSelectedPromptName] = useState( promptDefault?.name ?? "", ); const [selectedPromptVersion, setSelectedPromptVersion] = useState< number | null >(promptDefault?.version ?? null); const hasEvalReadAccess = useHasProjectAccess({ projectId, scope: "evalJob:read", }); const form = useForm({ resolver: zodResolver(CreateExperimentData), defaultValues: { promptId: "", datasetId: "", modelConfig: {}, ...defaultValues, }, }); const datasetId = form.watch("datasetId"); const evaluators = api.evals.jobConfigsByTarget.useQuery( { projectId, targetObject: "dataset" }, { enabled: hasEvalReadAccess && !!datasetId, }, ); const evalTemplates = api.evals.allTemplates.useQuery( { projectId }, { enabled: hasEvalReadAccess, }, ); const { createDefaultEvaluator } = useEvaluatorDefaults(); const { activeEvaluators, inActiveEvaluators, selectedEvaluatorData, showEvaluatorForm, handleConfigureEvaluator, handleCloseEvaluatorForm, handleEvaluatorSuccess, handleSelectEvaluator, } = useExperimentEvaluatorData({ datasetId, createDefaultEvaluator, evaluatorsData: evaluators.data, evalTemplatesData: evalTemplates.data, refetchEvaluators: evaluators.refetch, }); const { modelParams, updateModelParamValue, setModelParamEnabled, availableModels, providerModelCombinations, availableProviders, } = useModelParams(); useExperimentNameValidation({ projectId, datasetId, form, }); // TODO: show a warning if someone has multiple configs defined for the same template that target this dataset. Do not let them submit the form to create experiment. // Prompt them to delete duplicate running evaluators. // Watch model config changes and update form useEffect(() => { form.setValue("modelConfig", { provider: modelParams.provider.value, model: modelParams.model.value, modelParams: getFinalModelParams(modelParams), }); // Clear errors when valid values are set if (modelParams.provider.value && modelParams.model.value) { form.clearErrors("modelConfig"); } }, [modelParams, form]); const { promptId, promptsByName, expectedColumns } = useExperimentPromptData({ projectId, form, }); const experimentMutation = api.experiments.createExperiment.useMutation({ onSuccess: handleExperimentSuccess ?? (() => {}), onError: (error) => { showErrorToast( error.message || "Failed to trigger dataset run", "Please try again.", ); }, onSettled: handleExperimentSettled ?? (() => {}), }); const validationResult = api.experiments.validateConfig.useQuery( { projectId, promptId: promptId as string, datasetId: datasetId as string, }, { enabled: Boolean(promptId && datasetId), }, ); const datasets = api.datasets.allDatasetMeta.useQuery( { projectId }, { trpc: { context: { skipBatch: true, }, }, refetchOnMount: false, refetchOnWindowFocus: false, refetchOnReconnect: false, staleTime: Infinity, enabled: true, }, ); const onSubmit = async (data: CreateExperiment) => { capture("dataset_run:new_form_submit"); const experiment = { ...data, projectId, }; await experimentMutation.mutateAsync(experiment); form.reset(); setFormOpen(false); }; if ( !promptsByName || !datasets.data || (hasEvalReadAccess && !!datasetId && !evaluators.data) ) { return ; } return ( <> {showSDKRunInfoPage && ( )} New Dataset Run Start a dataset run to test a prompt version on a dataset. See{" "} documentation {" "} to learn more.
( Dataset run name (optional) )} /> ( Description (optional)