import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";

import { afterEach, expect, test } from "vitest";

import { rebuildEvalReport } from "#src/reporting/report.js";
import { runEvaluation } from "#src/core/runner.js";
import { toolAgentProfiles, toolComparisonSuite } from "./shared.mjs";

const temporaryDirectories: string[] = [];
const benchmarkPresets = ["easy", "medium", "hard"] as const;

afterEach(async () =>
{
    await Promise.all(temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })));
});

test("compares agent profiles across easy, medium, and hard benchmark presets", async () =>
{
    const root = await mkdtemp(path.join(tmpdir(), "pi-eval-tool-profile-example-"));
    temporaryDirectories.push(root);
    const globalSkillPath = path.join(root, "skills", "global", "SKILL.md");
    const profileSkillPath = path.join(root, "skills", "profile", "SKILL.md");
    await mkdir(path.dirname(globalSkillPath), { recursive: true });
    await mkdir(path.dirname(profileSkillPath), { recursive: true });
    await writeFile(globalSkillPath, "---\nname: global-skill\ndescription: Global evaluation guidance.\n---\nUse the global guidance.\n");
    await writeFile(profileSkillPath, "---\nname: profile-skill\ndescription: Profile-specific evaluation guidance.\n---\nUse the profile guidance.\n");
    const agentProfiles = toolAgentProfiles({ scripted: true }).map((agentProfile) =>
        agentProfile.id === "filesystem-tools"
            ? {
                ...agentProfile,
                model: "scripted/scripted-model",
                thinking: "high" as const,
                skills: [profileSkillPath],
                systemPrompt: "Profile system instructions.",
                appendSystemPrompt: ["Profile appended instructions."],
            }
            : agentProfile);
    const globalSettings = {
        model: "scripted/scripted-model",
        thinking: "off" as const,
        skills: [globalSkillPath],
        systemPrompt: "Global system instructions.",
        appendSystemPrompt: ["Global appended instructions."],
    };
    const runs = new Map<string, { readonly directory: string; readonly summary: Summary; }>();

    for (const benchmarkPreset of benchmarkPresets)
    {
        const directory = await runEvaluation({
            suite: toolComparisonSuite,
            benchmarkPreset,
            agentProfiles,
            ...globalSettings,
            resultsDirectory: path.join(root, "results"),
            workspacesDirectory: path.join(root, "workspaces"),
            runId: `scripted-${benchmarkPreset}`,
            live: process.env.PI_INTEGRATION_TEST_LIVE === "1",
            metricCalculators: [{
                id: "single-call",
                calculate: ({ agentMetrics }) => agentMetrics.toolCalls === 1,
            }],
        });
        const summary = JSON.parse(await readFile(path.join(directory, "summary.json"), "utf8")) as Summary;
        runs.set(benchmarkPreset, { directory, summary });
    }

    for (const [benchmarkPreset, run] of runs)
    {
        expect(Object.keys(run.summary.agentProfiles)).toEqual(["filesystem-tools", "bash-only"]);
        expect(run.summary.trials).toHaveLength(2);
        expect(run.summary.trials.every((trial) => trial.score.passed && trial.score.reward === 1)).toBe(true);
        expect(run.summary.trials.every((trial) => trial.difficulty === benchmarkPreset)).toBe(true);
        expect(run.summary.comparisons).toEqual([
            expect.objectContaining({ left: "filesystem-tools", right: "bash-only", completePairs: 1 }),
        ]);
        expect(run.summary.agentProfiles["bash-only"]!.means["custom.single-call"]).toBe(1);
        expect(run.summary.agentProfiles["filesystem-tools"]!.means["custom.single-call"]).toBe(0);
    }

    expect(runs.get("easy")!.summary.trials[0]!.fileCount).toBe(1);
    expect(runs.get("medium")!.summary.trials[0]!.fileCount).toBe(2);
    expect(runs.get("hard")!.summary.trials[0]!.fileCount).toBe(3);

    const hardRun = runs.get("hard")!;
    for (const trial of hardRun.summary.trials)
    {
        const systemPrompt = await readFile(path.join(
            hardRun.directory,
            "trials",
            `${trial.pairId}__p${trial.position}__${trial.agentProfile}`,
            "agent",
            "system-prompt.txt",
        ), "utf8");
        if (trial.agentProfile === "filesystem-tools")
        {
            expect(systemPrompt).toContain("Profile system instructions.");
            expect(systemPrompt).toContain("Profile appended instructions.");
            expect(systemPrompt).toContain("profile-skill");
            expect(systemPrompt).not.toContain("global-skill");
            expect(systemPrompt).not.toContain("Global appended instructions.");
        }
        else
        {
            expect(systemPrompt).toContain("Global system instructions.");
            expect(systemPrompt).toContain("Global appended instructions.");
            expect(systemPrompt).not.toContain("profile-skill");

        }
    }
    const manifest = JSON.parse(await readFile(path.join(hardRun.directory, "manifest.json"), "utf8")) as {
        readonly agentProfileSettings: Record<string, {
            readonly model: string | null;
            readonly thinking: string | null;
            readonly skills: readonly string[];
            readonly extensions: readonly string[];
        }>;
    };
    expect(manifest.agentProfileSettings["filesystem-tools"]!.model).toBe("scripted/scripted-model");
    expect(manifest.agentProfileSettings["filesystem-tools"]!.thinking).toBe("high");
    expect(manifest.agentProfileSettings["filesystem-tools"]!.skills).toEqual([profileSkillPath]);
    expect(manifest.agentProfileSettings["bash-only"]!.model).toBe("scripted/scripted-model");
    expect(manifest.agentProfileSettings["bash-only"]!.thinking).toBe("off");
    expect(manifest.agentProfileSettings["bash-only"]!.skills).toEqual([globalSkillPath]);
    expect(manifest.agentProfileSettings["filesystem-tools"]!.extensions.some(
        (extension) => extension.endsWith("/disable-thinking.js"),
    )).toBe(false);
    expect(manifest.agentProfileSettings["bash-only"]!.extensions.some(
        (extension) => extension.endsWith("/disable-thinking.js"),
    )).toBe(true);

    const bashTrial = hardRun.summary.trials.find((trial) => trial.agentProfile === "bash-only")!;
    const providerControls = (await readFile(path.join(
        hardRun.directory,
        "trials",
        `${bashTrial.pairId}__p${bashTrial.position}__${bashTrial.agentProfile}`,
        "agent",
        "provider-control.jsonl",
    ), "utf8"))
        .trim()
        .split("\n")
        .map((line) => JSON.parse(line) as Record<string, unknown>);
    expect(providerControls).toContainEqual(expect.objectContaining({
        kind: "session",
        thinkingLevel: "off",
        modelReasoning: false,
    }));

    let rendererSawCustomMetric = false;
    await rebuildEvalReport(runs.get("hard")!.directory, (data) =>
    {
        rendererSawCustomMetric = data.agentProfiles["bash-only"]!.means["custom.single-call"] === 1;
        return {
            terminal: "CUSTOM REPORT\n",
            files: { "custom-report.txt": String(data.comparisons.length) },
        };
    });
    expect(rendererSawCustomMetric).toBe(true);
    expect(await readFile(path.join(runs.get("hard")!.directory, "custom-report.txt"), "utf8")).toBe("1");
});

interface Summary
{
    readonly agentProfiles: Record<string, {
        readonly trials: number;
        readonly means: Record<string, number>;
    }>;
    readonly comparisons: readonly {
        readonly left: string;
        readonly right: string;
        readonly completePairs: number;
    }[];
    readonly trials: readonly {
        readonly agentProfile: string;
        readonly pairId: string;
        readonly position: number;
        readonly difficulty: string;
        readonly fileCount: number;
        readonly score: { readonly reward: number; readonly passed: boolean; };
    }[];
}