import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";

import { afterEach, expect, test } from "vitest";

import { multiAgentProfiles, toolComparisonSuite } from "./shared.mjs";
import { runEvaluation } from "#src/core/runner.js";

const temporaryDirectories: string[] = [];

afterEach(async () =>
{
    await Promise.all(temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })));
});

test("evaluates three agent profiles and calculates every agent profile pair", async () =>
{
    const root = await mkdtemp(path.join(tmpdir(), "pi-eval-multi-profile-example-"));
    temporaryDirectories.push(root);
    const runDirectory = await runEvaluation({
        suite: toolComparisonSuite,
        benchmarkPreset: "hard",
        agentProfiles: multiAgentProfiles({ scripted: true }),
        resultsDirectory: path.join(root, "results"),
        workspacesDirectory: path.join(root, "workspaces"),
        runId: "scripted-multi-profile-example",
        live: process.env.PI_INTEGRATION_TEST_LIVE === "1",
    });
    const summary = JSON.parse(await readFile(path.join(runDirectory, "summary.json"), "utf8")) as {
        readonly agentProfiles: Record<string, { readonly trials: number; readonly means: Record<string, number>; }>;
        readonly comparisons: readonly { readonly left: string; readonly right: string; readonly completePairs: number; }[];
        readonly trials: readonly { readonly agentProfile: string; readonly effectiveTools: unknown; }[];
    };

    expect(Object.keys(summary.agentProfiles)).toEqual(["filesystem-tools", "bash-only", "vanilla"]);
    expect(Object.values(summary.agentProfiles).map((profile) => profile.trials)).toEqual([1, 1, 1]);
    expect(summary.comparisons.map(({ left, right, completePairs }) => ({ left, right, completePairs }))).toEqual([
        { left: "filesystem-tools", right: "bash-only", completePairs: 1 },
        { left: "filesystem-tools", right: "vanilla", completePairs: 1 },
        { left: "bash-only", right: "vanilla", completePairs: 1 },
    ]);
    expect(summary.trials.find((trial) => trial.agentProfile === "vanilla")?.effectiveTools).toBe("all");
    expect(summary.agentProfiles.vanilla!.means.reward).toBe(1);
});