"""Analytic metrics and protocol-aware reporting regression tests."""
import json
import itertools
from functools import lru_cache
from pathlib import Path
import tempfile
import unittest

import numpy as np
import pandas as pd

from pace.metrics import evaluate, tune_thresholds, report_results


class EvaluateTests(unittest.TestCase):
    def test_analytic_probability_metrics_and_separate_thresholds(self):
        got = evaluate([0, 1, 0, 1], [.1, .8, .4, .6],
                       thresholds={"f1": .5, "f2": .7})
        self.assertAlmostEqual(got["ap"], 1)
        self.assertAlmostEqual(got["auc"], 1)
        self.assertAlmostEqual(got["brier"], .0925)
        self.assertAlmostEqual(got["ece15"], .275)
        self.assertAlmostEqual(got["f1"], 1)
        self.assertAlmostEqual(got["f2"], 5 / 9)
        self.assertEqual((got["tp"], got["fp"], got["tn"], got["fn"]), (2, 0, 2, 0))
        self.assertEqual((got["f2_tp"], got["f2_fn"]), (1, 1))
        self.assertEqual(got["f2_recall"], .5)

    def test_exact_budget_ties_use_row_identity_independent_of_input_order(self):
        ids = np.array([f"r{i:02}" for i in range(20)])
        y = np.array([1, 0, 1, 1] + [0] * 16)
        first = evaluate(y, np.repeat(.5, 20), row_ids=ids)
        permutation = np.random.default_rng(7).permutation(20)
        second = evaluate(y[permutation], np.repeat(.5, 20), row_ids=ids[permutation])
        for budget, count, positives in [(5, 1, 1), (10, 2, 1), (20, 4, 3)]:
            self.assertEqual(first[f"alerts_{budget}pct"], count)
            self.assertEqual(first[f"tp_at_{budget}pct"], positives)
            self.assertEqual(first[f"precision_at_{budget}pct"], second[f"precision_at_{budget}pct"])
        self.assertEqual(evaluate([1] * 7, [.5] * 7)["alerts_20pct"], 2)

    def test_no_positive_and_single_class_have_explicit_reasons(self):
        got = evaluate([0, 0], [.2, .7])
        self.assertTrue(np.isnan(got["ap"]))
        self.assertTrue(np.isnan(got["auc"]))
        self.assertTrue(np.isnan(got["f1"]))
        self.assertIn("positive", got["ap_reason"])
        self.assertIn("positive", got["recall_reason"])
        self.assertAlmostEqual(got["brier"], .265)
        positive = evaluate([1, 1], [.2, .7])
        self.assertEqual(positive["ap"], 1)
        self.assertIn("negative", positive["auc_reason"])
        empty = evaluate([], [])
        self.assertEqual(empty["n"], 0)
        self.assertEqual(empty["ap_reason"], "empty_input")
        self.assertTrue(np.isnan(empty["brier"]))

    def test_invalid_probabilities_and_duplicate_row_ids_are_rejected(self):
        for y, p in [([0], [1.1]), ([0], [np.nan]), ([2], [.2]), ([0, 1], [.2])]:
            with self.assertRaises(ValueError):
                evaluate(y, p)
        with self.assertRaises(ValueError):
            evaluate([0, 1], [.2, .8], row_ids=["same", "same"])

    def test_tuning_optimizes_each_f_score_and_handles_constants(self):
        got = tune_thresholds([1, 0, 0, 0, 1], [.9, .8, .7, .6, .1])
        self.assertEqual(got, {"f1": .9, "f2": .1})
        self.assertEqual(tune_thresholds([0, 1], [.3, .3]), {"f1": .3, "f2": .3})
        self.assertEqual(tune_thresholds([0, 0], [.2, .8]), {"f1": .5, "f2": .5})
        self.assertEqual(tune_thresholds([], []), {"f1": .5, "f2": .5})


@lru_cache(maxsize=1)
def positive_rankings():
    ranks = np.asarray(list(itertools.combinations(range(1, 41), 4)))
    ap = (np.arange(1, 5) / ranks).mean(axis=1)
    return ranks, ap


def scores_near_ap(target):
    """Real, attainable AP near target; shared labels retain row alignment."""
    ranks, ap = positive_rankings()
    selected = ranks[int(np.argmin(np.abs(ap - target)))] - 1
    y = np.asarray([int(i in (1, 11, 21, 31)) for i in range(40)])
    order = np.empty(40, dtype=int)
    order[selected] = np.flatnonzero(y)
    remainder = np.ones(40, dtype=bool)
    remainder[selected] = False
    order[remainder] = np.flatnonzero(y == 0)
    p = np.empty(40)
    p[order] = np.linspace(.99, .01, 40)
    return y, p


def make_rows(candidates=None, faculties=("A", "B"), folds=(1,), seeds=(0, 1)):
    """Aligned canonical predictions; reported AP is computed from real scores."""
    candidates = candidates or [("gru", "joint", .62)]
    results, predictions = [], []
    for faculty in faculties:
        for fold in folds:
            for seed in seeds:
                specs = [("H0", "none", "none", .5), ("H1", "none", "none", .51)]
                for arch, task, ap in candidates:
                    specs += [("HR", arch, "none", .52), ("H2", arch, task, ap)]
                for arm, arch, task, target_ap in specs:
                    base = dict(faculty=faculty, fold=fold, seed=seed, arm=arm, arch=arch, task=task)
                    y, p = scores_near_ap(target_ap)
                    ids = [f"{faculty}:{fold}:{i:02}" for i in range(len(y))]
                    results.append({**base, **evaluate(y, p, row_ids=ids)})
                    for i in range(len(y)):
                        predictions.append({**base, "row_id": ids[i],
                                            "student_id": f"{faculty}:{i}", "y": y[i], "p": p[i],
                                            "n_history": [0, 2, 10, 30][i % 4], "n_semesters": i % 4,
                                            "oov_rate": [0., .1, .5, 1.][i % 4]})
    return pd.DataFrame(results), pd.DataFrame(predictions)


class ReportTests(unittest.TestCase):
    def test_seed_then_equal_faculty_fold_aggregation_and_full_tables(self):
        rows = pd.DataFrame([
            dict(faculty="A", fold=1, seed=0, arm="H0", arch="none", task="none", ap=.9),
            dict(faculty="A", fold=1, seed=1, arm="H0", arch="none", task="none", ap=.7),
            dict(faculty="B", fold=1, seed=0, arm="H0", arch="none", task="none", ap=.2),
        ])
        logs = []
        with tempfile.TemporaryDirectory() as directory:
            decision = report_results(rows, pd.DataFrame(), directory, logs.append, bootstrap=40)
            macro = pd.read_csv(Path(directory) / "macro_metrics.csv")
            self.assertAlmostEqual(macro.iloc[0]["ap"], .5)
            self.assertEqual(macro.iloc[0]["n_cells_ap"], 2)
            self.assertIsNone(decision["winner_arch"])
            self.assertEqual(len(pd.read_csv(Path(directory) / "results_by_seed.csv")), 3)
            report = (Path(directory) / "report.md").read_text()
            self.assertIn("correlated", report)
            self.assertIn("undefined", report.lower())
            self.assertTrue(any("A" in line and "B" in line for line in logs))

    def test_development_winner_and_promotion_never_use_retrospective_rows(self):
        dev_r, dev_p = make_rows([("gru", "joint", .62), ("transformer", "course", .56)])
        retro_r, retro_p = make_rows([("gru", "joint", .1), ("transformer", "course", .99)], folds=(6, 7))
        with tempfile.TemporaryDirectory() as directory:
            decision = report_results(pd.concat([dev_r, retro_r]), pd.concat([dev_p, retro_p]),
                                      directory, lambda _: None, bootstrap=40)
            self.assertEqual((decision["winner_arch"], decision["winner_task"]), ("gru", "joint"))
            self.assertTrue(decision["promoted"])
            target_ap = dev_r.loc[(dev_r.arm == "H2") & (dev_r.arch == "gru"), "ap"].mean()
            strongest = dev_r.loc[dev_r.arm.isin(["H0", "H1", "HR"])].groupby("arm").ap.mean().max()
            self.assertAlmostEqual(decision["worst_control_gain"], target_ap - strongest)
            self.assertEqual(decision["positive_cell_fraction"], 1)
            self.assertFalse(decision["holdout_positive_evidence"])
            macro = pd.read_csv(Path(directory) / "macro_metrics.csv")
            self.assertEqual(set(macro["stage"]), {"development", "retrospective"})
            saved = json.loads((Path(directory) / "decision.json").read_text())
            self.assertEqual(saved["winner_arch"], "gru")
            self.assertIn("did not alter", (Path(directory) / "report.md").read_text())

    def test_prediction_support_mismatch_excludes_pair_and_blocks_promotion(self):
        rows, preds = make_rows()
        mask = (preds["arm"] == "HR") & (preds["faculty"] == "A") & (preds["seed"] == 0)
        preds.loc[mask, "row_id"] = preds.loc[mask, "row_id"] + "wrong"
        with tempfile.TemporaryDirectory() as directory:
            decision = report_results(rows, preds, directory, lambda _: None, bootstrap=0)
            self.assertFalse(decision["promoted"])
            self.assertFalse(decision["complete_support"])
            audit = pd.read_csv(Path(directory) / "support_audit.csv")
            self.assertIn("row_ids_mismatch", set(audit["support_status"]))

    def test_missing_development_cell_cannot_be_hidden_by_seed_average(self):
        rows, preds = make_rows()
        mask = (rows.arm == "H2") & (rows.faculty == "B")
        rows = rows.loc[~mask]
        preds = preds.loc[~((preds.arm == "H2") & (preds.faculty == "B"))]
        with tempfile.TemporaryDirectory() as directory:
            decision = report_results(rows, preds, directory, lambda _: None, bootstrap=20)
            self.assertFalse(decision["promoted"])
            self.assertFalse(decision["complete_support"])
            self.assertEqual(decision["expected_cells"], 2)

    def test_no_positive_cells_are_undefined_not_failed_training(self):
        rows, preds = make_rows()
        rows.loc[rows.faculty == "B", "ap"] = np.nan
        rows.loc[rows.faculty == "B", "ap_reason"] = "no_positive_labels"
        preds.loc[preds.faculty == "B", "y"] = 0
        with tempfile.TemporaryDirectory() as directory:
            decision = report_results(rows, preds, directory, lambda _: None, bootstrap=20)
            self.assertTrue(decision["promoted"])
            self.assertEqual(decision["expected_cells"], 1)
            diagnostics = pd.read_csv(Path(directory) / "history_diagnostics.csv")
            self.assertTrue(diagnostics["ap_reason"].fillna("").str.contains("positive").any())

    def test_neural_paired_contrasts_and_faculty_harms_are_reported(self):
        rows, preds = make_rows()
        extras_r, extras_p = [], []
        for arm, gain in [("N0", -.2), ("N1", 0), ("N2", .03), ("N3", -.01)]:
            generated_r, generated_p = make_rows([("gru", "joint", .5 + gain)])
            r = generated_r.loc[generated_r.arm == "H2"].copy()
            p = generated_p.loc[generated_p.arm == "H2"].copy()
            r["arm"], p["arm"] = arm, arm
            extras_r.append(r)
            extras_p.append(p)
        with tempfile.TemporaryDirectory() as directory:
            report_results(pd.concat([rows, *extras_r]), pd.concat([preds, *extras_p]),
                           directory, lambda _: None, bootstrap=20)
            contrasts = pd.read_csv(Path(directory) / "paired_contrasts.csv")
            self.assertIn("N2_vs_N1", set(contrasts["contrast"]))
            self.assertIn("N3_vs_N1", set(contrasts["contrast"]))
            harm = pd.read_csv(Path(directory) / "faculty_wins_harms.csv")
            self.assertTrue(((harm.contrast == "N3_vs_N1") & (harm.harm_cells > 0)).any())
            report = (Path(directory) / "report.md").read_text()
            self.assertTrue("Frozen SSL has higher mean AP than supervised history" in report)
            self.assertTrue("Fine-tuning has lower mean AP than frozen SSL" in report)
            self.assertTrue("No-history AP is undefined" in report)

    def test_missing_seed_and_missing_result_coverage_are_explicit(self):
        rows, preds = make_rows()
        rows = rows.loc[~((rows.arm == "H2") & (rows.faculty == "B") & (rows.seed == 1))]
        preds = preds.loc[~((preds.arm == "H2") & (preds.faculty == "B") & (preds.seed == 1))]
        with tempfile.TemporaryDirectory() as directory:
            decision = report_results(rows, preds, directory, lambda _: None, bootstrap=0)
            self.assertFalse(decision["promoted"])
            coverage = pd.read_csv(Path(directory) / "coverage_audit.csv")
            missing = coverage.loc[(coverage.arm == "H2") & (coverage.faculty == "B") & (coverage.seed == 1)]
            self.assertEqual(len(missing), 1)
            self.assertFalse(missing.iloc[0].result_present)

    def test_retrospective_only_candidate_never_becomes_development_winner(self):
        dev_r, dev_p = make_rows()
        retro_r, retro_p = make_rows([("transformer", "course", .99)], folds=(6,))
        with tempfile.TemporaryDirectory() as directory:
            decision = report_results(pd.concat([dev_r, retro_r]), pd.concat([dev_p, retro_p]),
                                      directory, lambda _: None, bootstrap=0)
            self.assertEqual(decision["winner_arch"], "gru")
            self.assertEqual(len(decision["candidates"]), 1)
            self.assertFalse(decision["holdout_positive_evidence"])

    def test_diagnostic_strata_reuse_supplied_thresholds_without_tuning(self):
        rows, preds = make_rows()
        rows["threshold_f1"] = 1.
        rows["threshold_f2"] = 0.
        with tempfile.TemporaryDirectory() as directory:
            report_results(rows, preds, directory, lambda _: None, bootstrap=0)
            strata = pd.read_csv(Path(directory) / "no_history_diagnostics.csv")
            self.assertEqual(set(strata.threshold_f1), {1.})
            self.assertEqual(set(strata.threshold_f2), {0.})
            self.assertTrue((strata.tp == 0).all())
            self.assertTrue((strata.f2_predicted_positive == strata.n).all())

    def test_task_and_architecture_ablations_are_paired_on_identical_rows(self):
        candidates = [("mlp", "grade", .55), ("mlp", "course", .54), ("mlp", "joint", .60),
                      ("gru", "grade", .58), ("gru", "course", .56), ("gru", "joint", .63),
                      ("transformer", "grade", .60), ("transformer", "course", .57),
                      ("transformer", "joint", .65)]
        rows, preds = make_rows(candidates)
        keys = ["faculty", "fold", "seed", "arm", "arch", "task"]
        rows = rows.drop_duplicates(keys)
        preds = preds.drop_duplicates(keys + ["row_id"])
        # One grade fit used different rows: its task comparison must be excluded
        # for that seed, even though its result table has a finite AP.
        mismatch = (preds.arm == "H2") & (preds.arch == "mlp") & (preds.task == "grade") & (preds.faculty == "A") & (preds.seed == 0)
        preds.loc[mismatch, "row_id"] += "wrong"
        with tempfile.TemporaryDirectory() as directory:
            report_results(rows, preds, directory, lambda _: None, bootstrap=0)
            contrasts = pd.read_csv(Path(directory) / "paired_contrasts.csv")
            def contrast(name, arch, task):
                selected = contrasts.loc[(contrasts.contrast == name) & (contrasts.arch == arch) & (contrasts.task == task)]
                self.assertFalse(selected.empty, name)
                return selected.iloc[0]
            def expected(arch, task, baseline_arch, baseline_task):
                a = rows.loc[(rows.arm == "H2") & (rows.arch == arch) & (rows.task == task), "ap"].mean()
                b = rows.loc[(rows.arm == "H2") & (rows.arch == baseline_arch) & (rows.task == baseline_task), "ap"].mean()
                return a - b
            self.assertAlmostEqual(contrast("H2_joint_vs_grade", "mlp", "joint").ap_delta, expected("mlp", "joint", "mlp", "grade"))
            self.assertAlmostEqual(contrast("H2_joint_vs_course", "mlp", "joint").ap_delta, expected("mlp", "joint", "mlp", "course"))
            self.assertAlmostEqual(contrast("H2_grade_vs_course", "mlp", "grade").ap_delta, expected("mlp", "grade", "mlp", "course"))
            self.assertAlmostEqual(contrast("H2_gru_vs_mlp", "gru", "joint").ap_delta, expected("gru", "joint", "mlp", "joint"))
            self.assertAlmostEqual(contrast("H2_transformer_vs_gru", "transformer", "joint").ap_delta, expected("transformer", "joint", "gru", "joint"))
            self.assertEqual(contrast("H2_joint_vs_grade", "mlp", "joint").n_paired_seeds, 3)
            audit = pd.read_csv(Path(directory) / "support_audit.csv")
            bad = audit.loc[(audit.contrast == "H2_joint_vs_grade") & (audit.arch == "mlp") & (audit.faculty == "A") & (audit.seed == 0)]
            self.assertEqual(bad.iloc[0].support_status, "row_ids_mismatch")

    def test_stale_reported_ap_cannot_drive_promotion(self):
        rows, preds = make_rows([("gru", "joint", .5)])
        rows.loc[rows.arm == "H2", "ap"] = .99
        with tempfile.TemporaryDirectory() as directory:
            decision = report_results(rows, preds, directory, lambda _: None, bootstrap=0)
            self.assertFalse(decision["promoted"])
            self.assertLess(decision["worst_control_gain"], 0)
            audit = pd.read_csv(Path(directory) / "metric_integrity_audit.csv")
            h2 = audit.loc[audit.arm == "H2"]
            self.assertEqual(set(h2.ap_consistency_status), {"reported_ap_mismatch_recomputed"})
            self.assertTrue(np.allclose(h2.ap, .5))

    def test_duplicate_predictions_for_whole_cell_cannot_shrink_required_support(self):
        for reported_undefined in (False, True):
            with self.subTest(reported_undefined=reported_undefined):
                rows, preds = make_rows()
                affected = rows.faculty == "B"
                if reported_undefined:
                    rows.loc[affected, "ap"] = np.nan
                preds = pd.concat([preds, preds.loc[preds.faculty == "B"]], ignore_index=True)
                with tempfile.TemporaryDirectory() as directory:
                    decision = report_results(rows, preds, directory, lambda _: None, bootstrap=0)
                    self.assertFalse(decision["promoted"])
                    self.assertFalse(decision["complete_support"])
                    self.assertEqual(decision["expected_cells"], 2)
                    self.assertEqual(decision["candidates"][0]["n_expected_seeds"], 4)
                    audit = pd.read_csv(Path(directory) / "metric_integrity_audit.csv")
                    invalid = audit.loc[audit.faculty == "B"]
                    self.assertTrue(invalid.ap_support_required.all())
                    self.assertEqual(set(invalid.ap_consistency_status), {"duplicate_prediction_row_ids"})

    def test_duplicate_predictions_for_whole_seed_cannot_shrink_required_support(self):
        rows, preds = make_rows()
        affected = (preds.faculty == "B") & (preds.seed == 1)
        preds = pd.concat([preds, preds.loc[affected]], ignore_index=True)
        with tempfile.TemporaryDirectory() as directory:
            decision = report_results(rows, preds, directory, lambda _: None, bootstrap=0)
            self.assertFalse(decision["promoted"])
            self.assertFalse(decision["complete_support"])
            self.assertEqual(decision["expected_cells"], 2)
            self.assertEqual(decision["candidates"][0]["n_expected_seeds"], 4)
            self.assertEqual(decision["candidates"][0]["n_paired_seeds"], 3)

    def test_stage_disagreement_and_duplicate_result_identities_are_rejected(self):
        rows, preds = make_rows()
        rows["stage"] = "retrospective"
        with tempfile.TemporaryDirectory() as directory:
            with self.assertRaises(ValueError):
                report_results(rows, preds, directory, lambda _: None)
            rows = rows.drop(columns="stage")
            with self.assertRaises(ValueError):
                report_results(pd.concat([rows, rows.iloc[:1]]), preds, directory, lambda _: None)


if __name__ == "__main__":
    unittest.main()
