import { runCode } from "std::agency"
import { Feedback, failOpenFeedback } from "std::agents/lib/feedback"
import { AGENT_SPILL_OVER, llmOptions } from "std::agents/lib/shared"
import { readOnlyFileTools, readOnlyGitTools, shellTools } from "std::agents/lib/toolkits"
import { guarded } from "std::decorators"
import { agentSkill } from "std::skills"
import { systemMessage, threadIsNew } from "std::thread"

/** @module
@summary Run an Agency program and judge its result against a task.

The verifier checks the returned value and can inspect files the program
created. A program that cannot run produces an error finding.
Use `agencyReviewAgent` to check source without executing it.

A stopped run may return saved partial findings. A failed judgment returns
no findings, so an empty list alone does not prove the program passed.
*/

static const systemPrompt = """You judge whether a program's actual result satisfies a task.

The value the program returned is only part of its result. When the task asks for a file, inspect what the program actually wrote with your tools and judge that too: a program that returns "done" and writes nothing has not done the task.

For each finding, return a Feedback item: error=true when it fails the task (name the specific problem), error=false for a satisfied point."""

static const skills = agentSkill("agency/verifier")

// The verifier reads and measures what the program left behind. It has no
// write tools, because a verifier that can fix things is no longer a
// verifier.
export def buildTools(): any[] {
  """
  Return the Agency verifier's file inspection and command execution tools.
  """
  return [...readOnlyFileTools(), ...readOnlyGitTools(), ...shellTools(), skills]
}

def judgeResult(
  task: string,
  data: any,
  model: string,
  provider: string,
  extraTools: any[],
): Feedback[] {
  if (threadIsNew()) {
    systemMessage(systemPrompt)
  }
  const prompt = """
Task:
${task}

The program ran and produced this result:
${data}
"""
  // Return position so a model saveDraft propagates as partial findings.
  return llm(
    prompt,
    llmOptions(
      model: model,
      provider: provider,
      tools: buildTools().concat(extraTools).concat([saveDraft]),
      spillToolResultsOver: AGENT_SPILL_OVER,
    ),
  )
}

// Returns findings, never a Result: a program that cannot run is itself an
// error finding, so callers can always read findings. Keeping this plain
// also lets the caller return the guard Result straight through.
def runAndJudge(
  source: string,
  task: string,
  model: string,
  provider: string,
  session: string,
  extraTools: any[],
): Feedback[] {
  const ran = runCode(source)
  if (ran is failure(err)) {
    return [{ error: true, feedback: "The program failed to run: ${err}" }]
  }
  // Return position, so the findings the model saves reach the guard.
  thread(label: "agencyVerifierAgent", session: session) {
    return judgeResult(
      task: task,
      data: ran.value,
      model: model,
      provider: provider,
      extraTools: extraTools,
    )
  }
}

export def agencyVerifierAgent(
  source: string,
  task: string,
  maxCost: number = $20.00,
  maxTime: number = 15m,
  model: string = "",
  provider: string = "",
  session: string = "",
  extraTools: any[] = [],
): Result<Feedback[]> {
  """
  Run an Agency program and judge its result against a task.
  Run failures become error findings, and a failed judgment returns no findings.

  @param maxCost - Cost budget in USD.
  @param maxTime - Time budget in milliseconds.
  @param model - Model override for the judgment, or "" for the ambient model.
  @param provider - Provider for the judgment model, ignored when model is empty.
  @param session - Name a judgment conversation to reuse, or "" for an isolated thread.
  @param extraTools - Additional tools for judging the program's result.
  """
  return failOpenFeedback(agencyVerifyUnderBudget(
    source: source,
    task: task,
    model: model,
    provider: provider,
    session: session,
    extraTools: extraTools,
    maxCost: maxCost,
    maxTime: maxTime,
  ))
}

// The guarded half of agencyVerifierAgent, a def of its own because the
// exported def post-processes the guard's Result. Return position: when
// the budget runs out, the findings the model saved reach the guard and
// come back as the result.
@guarded(cost: maxCost, time: maxTime, label: "agencyVerifierAgent")
def agencyVerifyUnderBudget(
  source: string,
  task: string,
  model: string,
  provider: string,
  session: string,
  extraTools: any[],
  maxCost: number,
  maxTime: number,
): Result<Feedback[]> {
  return runAndJudge(
    source: source,
    task: task,
    model: model,
    provider: provider,
    session: session,
    extraTools: extraTools,
  )
}