#!/usr/bin/env bun
/**
 * Accuracy check for AI merge fixes on realistic conflicts, meant for a dev server using the real models
 * (MERGEFIX_PROVIDER=workers-ai). For each case it reports whether the top-ranked resolution is correct,
 * whether it was auto-applied, and Clef's probabilities, so the gating can be judged against reality.
 *
 *   BASE_URL=http://localhost:3015 D1_CWD=$PWD bun scripts/eval-mergefix.ts
 */
import { api, d1, scenario } from "./mergefix-harness";
import type { ConflictFile } from "./mergefix-harness";

type Expectation = {
  name: string;
  pr: { title: string; body: string };
  files: ConflictFile[];
  /** True when the resolved file is semantically right. */
  correct: (resolved: string) => boolean;
  /** Whether an ideal system would auto-apply this one. */
  shouldAutoApply: boolean;
};

const CASES: Expectation[] = [
  {
    name: "both sides add an import",
    pr: { title: "Add date formatting to the report", body: "Uses formatDate in the report header." },
    files: [
      {
        path: "src/report.ts",
        base: 'import { sum } from "./math";\n\nexport function report(values: number[]) {\n  return `total ${sum(values)}`;\n}\n',
        head: 'import { sum } from "./math";\nimport { formatDate } from "./dates";\n\nexport function report(values: number[]) {\n  return `total ${sum(values)}`;\n}\n',
        main: 'import { sum } from "./math";\nimport { logger } from "./log";\n\nexport function report(values: number[]) {\n  return `total ${sum(values)}`;\n}\n',
      },
    ],
    correct: (text) => text.includes("formatDate") && text.includes("logger"),
    shouldAutoApply: true,
  },
  {
    name: "both sides append a changelog entry",
    pr: { title: "Add CSV export", body: "Adds a CSV export button." },
    files: [
      {
        path: "CHANGELOG.md",
        base: "# Changelog\n\n## Unreleased\n\n- Initial release\n",
        head: "# Changelog\n\n## Unreleased\n\n- Initial release\n- Add CSV export\n",
        main: "# Changelog\n\n## Unreleased\n\n- Initial release\n- Fix login redirect\n",
      },
    ],
    correct: (text) => text.includes("Add CSV export") && text.includes("Fix login redirect"),
    shouldAutoApply: true,
  },
  {
    name: "rename on one side, new argument on the other",
    pr: {
      title: "Rename getUser to fetchUser",
      body: "Renames the helper everywhere; behavior is unchanged.",
    },
    files: [
      {
        path: "src/profile.ts",
        base: "export async function profile(id: string) {\n  const user = await getUser(id);\n  return user.name;\n}\n",
        head: "export async function profile(id: string) {\n  const user = await fetchUser(id);\n  return user.name;\n}\n",
        main: "export async function profile(id: string) {\n  const user = await getUser(id, { cache: true });\n  return user.name;\n}\n",
      },
    ],
    correct: (text) => /fetchUser\(id, \{ cache: true \}\)/.test(text) && !text.includes("getUser"),
    shouldAutoApply: false,
  },
  {
    name: "both sides change the same constant",
    pr: { title: "Raise the request timeout", body: "Slow networks hit the timeout; raise it to 60s." },
    files: [
      {
        path: "src/config.ts",
        base: "export const TIMEOUT_MS = 30_000;\nexport const RETRIES = 3;\n",
        head: "export const TIMEOUT_MS = 60_000;\nexport const RETRIES = 3;\n",
        main: "export const TIMEOUT_MS = 10_000;\nexport const RETRIES = 3;\n",
      },
    ],
    correct: (text) => text.includes("60_000") || text.includes("10_000"),
    shouldAutoApply: false,
  },
  {
    name: "authorization check changed on both sides",
    pr: { title: "Allow maintainers to delete", body: "Maintainers can delete issues too." },
    files: [
      {
        path: "src/issues/delete.ts",
        base: 'export function canDelete(user: User) {\n  return user.role === "admin";\n}\n',
        head: 'export function canDelete(user: User) {\n  return user.role === "admin" || user.role === "maintainer";\n}\n',
        main: 'export function canDelete(user: User) {\n  return user.role === "admin" && !user.suspended;\n}\n',
      },
    ],
    correct: (text) => text.includes("maintainer") && text.includes("suspended"),
    shouldAutoApply: false,
  },
];

type PlannedFile = {
  path: string;
  status: "planned" | "skipped";
  reason?: string;
  resolution?: string | null;
  hunks?: Array<{
    candidates: Array<{ id: string; source: string; text: string }>;
    selected: string | null;
    probabilities: Record<string, number>;
  }>;
  decision?: { autoApply: boolean; reasons: string[]; risk: { level: string }; needsHuman: number };
};

function topResolution(file: PlannedFile): string | null {
  if (file.resolution) return file.resolution;
  const picks = (file.hunks ?? []).map((hunk) => hunk.candidates.find((c) => c.id === hunk.selected)?.text ?? null);
  return picks.every((pick) => pick !== null) ? picks.join("") : null;
}

const user = await api<{ login: string }>("GET", "/user");
const stamp = Date.now().toString(36);
const rows: string[] = [];
let correctTop = 0;
let correctGate = 0;
for (const [index, testCase] of CASES.entries()) {
  const run = await scenario(user.login, `mergefix-eval-${stamp}-${index}`, testCase.files, testCase.pr);
  try {
    const stored = d1(
      `SELECT r.result_json FROM mergefix_runs r JOIN issues i ON i.id = r.pull_issue_id
       JOIN repos p ON p.id = i.repo_id WHERE p.name = 'mergefix-eval-${stamp}-${index}' ORDER BY r.created_at DESC LIMIT 1`,
    )[0]?.result_json;
    const plan = stored ? (JSON.parse(String(stored)) as { files: PlannedFile[] }) : { files: [] };
    const file = plan.files[0];
    const resolved = file ? topResolution(file) : null;
    const ok = resolved !== null && testCase.correct(resolved);
    const applied = run.fix.status === "applied";
    if (ok) correctTop++;
    if (applied === testCase.shouldAutoApply && (!applied || ok)) correctGate++;
    const hunk = file?.hunks?.[0];
    const probs = hunk
      ? Object.entries(hunk.probabilities)
          .sort((a, b) => b[1] - a[1])
          .map(([id, p]) => `${id}=${p.toFixed(2)}`)
          .join(" ")
      : (file?.reason ?? run.fix.error ?? "");
    rows.push(
      [
        testCase.name,
        `status=${run.fix.status}`,
        `top=${hunk?.selected ?? "-"}`,
        `correct=${ok}`,
        `risk=${file?.decision?.risk.level ?? "-"}`,
        `human=${file?.decision?.needsHuman.toFixed(2) ?? "-"}`,
        probs,
        (file?.decision?.reasons ?? []).join("; "),
      ].join(" | "),
    );
  } finally {
    run.cleanup();
  }
}
console.log(rows.join("\n"));
console.log(
  `top candidate correct: ${correctTop}/${CASES.length}; gate decision right: ${correctGate}/${CASES.length}`,
);