"""Behavioral contracts for matched evaluation and safe reference alignment."""
import importlib
import contextlib
import io
import json
import tempfile
import unittest
from pathlib import Path

import numpy as np
import pandas as pd

try:
    mm = importlib.import_module("mint_metrics")
except ModuleNotFoundError:
    mm = None


class MetricsTests(unittest.TestCase):
    def setUp(self):
        self.assertIsNotNone(mm, "mint_metrics is not implemented")

    def test_known_ranking_calibration_and_warning_budget(self):
        # Catches pooled/thresholded AP, incorrect F2, and score-bin weighting.
        y = np.array([1, 0, 1, 0] + [0] * 16)
        score = np.array([.9, .8, .7, .1] + [.05] * 16)
        got = mm.metrics(y, score)
        self.assertAlmostEqual(got["ap"], 5 / 6)
        self.assertAlmostEqual(got["auc"], 35 / 36)
        self.assertAlmostEqual(got["brier"], .0395)
        self.assertAlmostEqual(got["ece10"], .105)
        self.assertAlmostEqual(got["f2_at_10"], .5)
        self.assertAlmostEqual(got["f1_at_15"], .8)
        self.assertAlmostEqual(got["f2_at_15"], 10 / 11)
        self.assertAlmostEqual(got["f2_at_20"], 10 / 12)
        self.assertAlmostEqual(got["f2_at_25"], 10 / 13)
        self.assertEqual(got["flagged_at_20"], 4)
        self.assertEqual(got["rows"], 20)
        self.assertEqual(got["positives"], 2)

    def test_missing_scores_report_support_without_imputation(self):
        got = mm.metrics([0, 1, 1], [.2, .8, np.nan])
        self.assertEqual((got["rows"], got["total_rows"], got["missing_scores"]), (2, 3, 1))
        self.assertAlmostEqual(got["ap"], 1.)
        self.assertAlmostEqual(got["brier"], .04)

    def test_invalid_labels_and_probabilities_fail_loudly(self):
        for labels, score in [([0, 2], [.2, .8]), ([0, np.nan], [.2, .8]),
                              ([0, 1], [.2, 1.2]), ([0, 1], [.2, np.inf])]:
            with self.subTest(labels=labels, score=score):
                with self.assertRaises(ValueError):
                    mm.metrics(labels, score)

    def test_one_class_and_empty_support_keep_calibration_defined(self):
        one = mm.metrics([0, 0], [.1, .2])
        self.assertTrue(np.isnan(one["ap"]))
        self.assertTrue(np.isnan(one["auc"]))
        self.assertAlmostEqual(one["brier"], .025)
        empty = mm.metrics([0, 1], [np.nan, np.nan])
        self.assertEqual(empty["rows"], 0)
        self.assertTrue(np.isnan(empty["f2_at_20"]))

    def test_macro_ap_weights_cells_equally_instead_of_pooling(self):
        frame = pd.DataFrame({"fold": [6] * 10, "faculty": ["A"] * 2 + ["B"] * 8,
                              "actual": [1, 0] + [0] * 7 + [1],
                              "score": [.9, .8, .7, .6, .5, .4, .3, .2, .1, .05]})
        self.assertAlmostEqual(mm.macro_ap(frame, "score"), 9 / 16)
        self.assertAlmostEqual(mm.macro_ap(frame.drop(columns="fold"), "score"), 9 / 16)


def keyed_frame():
    return pd.DataFrame({
        "fold": [6] * 4, "faculty": ["IT"] * 4,
        "sample_id": ["e0", "e1", "e2", "e3"],
        "student_id": ["s0", "s1", "s2", "s3"],
        "course_id": ["c0", "c0", "c1", "c1"],
        "semester": [6070] * 4, "actual": [0, 1, 0, 1],
        "committee": [.3, .6, .2, .7], "suite26_mint": [.1, .9, .2, .8]})


class ReferenceTests(unittest.TestCase):
    def setUp(self):
        self.assertTrue(hasattr(mm, "add_references"), "Keyed reference loader is not implemented")
        self.temp = tempfile.TemporaryDirectory()
        self.addCleanup(self.temp.cleanup)
        self.root = Path(self.temp.name)

    def save(self, source, filename="IT_fold_6_test_results.csv"):
        path = self.root / filename
        path.parent.mkdir(parents=True, exist_ok=True)
        source.to_csv(path, index=False)
        return path

    def test_reordered_keys_align_even_when_label_arrays_are_identical(self):
        # The old label-array check would accept this order but assign wrong scores.
        frame = keyed_frame()
        source = frame.drop(columns=["committee", "suite26_mint"])
        source["final"] = [.11, .91, .21, .81]
        self.save(source.iloc[[2, 3, 0, 1]])
        wide, audit = mm.add_references(frame, {"suite22": self.root})
        np.testing.assert_allclose(wide.suite22, [.11, .91, .21, .81])
        self.assertEqual(audit.iloc[0].status, "loaded")
        self.assertEqual(audit.iloc[0].matched_rows, 4)
        self.assertEqual(audit.iloc[0]["keys"], "sample_id")
        self.assertNotIn("suite22", frame)

    def test_label_only_reference_is_skipped_despite_matching_labels(self):
        self.save(pd.DataFrame({"actual": [0, 1, 0, 1], "final": [.2, .8, .3, .7]}))
        wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root})
        self.assertTrue(wide.suite22.isna().all())
        self.assertEqual(audit.iloc[0].status, "rejected")
        self.assertIn("identifier", audit.iloc[0].detail.lower())

    def test_matching_sample_id_does_not_override_conflicting_ids_or_labels(self):
        for column, replacement in [("student_id", "wrong-student"), ("course_id", "wrong-course"),
                                    ("semester", 6069), ("actual", 1)]:
            with self.subTest(column=column):
                source = keyed_frame()
                source["final"] = .5
                source.loc[0, column] = replacement
                self.save(source)
                wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root})
                self.assertTrue(wide.suite22.isna().all())
                self.assertEqual(audit.iloc[0].status, "rejected")
                self.assertIn(column, audit.iloc[0].detail)

    def test_legacy_ids_and_raw_semester_codes_match_ordinal_semesters(self):
        source = keyed_frame().drop(columns=["sample_id", "semester"])
        source = source.rename(columns={"student_id": "IDSV", "course_id": "IDMH",
                                        "actual": "actual_failure_lt5"})
        source["target_NHHK"] = 20232
        source["failure_probability_lt5"] = [.11, .91, .21, .81]
        self.save(source.iloc[::-1])
        wide, audit = mm.add_references(keyed_frame(), {"v2": self.root})
        np.testing.assert_allclose(wide.v2, [.11, .91, .21, .81])
        self.assertEqual(audit.iloc[0]["keys"], "student_id,course_id,semester")

    def test_student_course_keys_require_fixed_fold_file_and_uniqueness(self):
        source = keyed_frame()[["student_id", "course_id", "actual"]].copy()
        source["final"] = [.11, .91, .21, .81]
        self.save(source.iloc[::-1])
        wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root})
        np.testing.assert_allclose(wide.suite22, [.11, .91, .21, .81])
        self.assertEqual(audit.iloc[0]["keys"], "student_id,course_id")
        (self.root / "IT_fold_6_test_results.csv").unlink()
        self.save(source, "predictions.csv")
        wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root})
        self.assertTrue(wide.suite22.isna().all())
        self.assertEqual(audit.iloc[0].status, "rejected")

    def test_duplicate_keys_are_rejected_and_missing_files_are_audited(self):
        source = keyed_frame()[["sample_id", "actual"]].copy()
        source["sample_id"] = "duplicate"
        source["final"] = .5
        self.save(source)
        wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root,
                                                               "suite25": self.root / "absent"})
        self.assertTrue(wide[["suite22", "suite25"]].isna().all().all())
        self.assertEqual(audit.set_index("reference").loc["suite22", "status"], "rejected")
        self.assertEqual(audit.set_index("reference").loc["suite25", "status"], "missing")

    def test_partial_coverage_preserves_nan_and_counts_invalid_scores(self):
        source = keyed_frame().iloc[[0, 1, 2]].copy()
        source["final"] = [.2, .8, "broken"]
        self.save(source)
        wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root})
        self.assertEqual(wide.suite22.notna().sum(), 2)
        self.assertEqual(audit.iloc[0].status, "partial")
        self.assertEqual(audit.iloc[0].matched_rows, 3)
        self.assertEqual(audit.iloc[0].scored_rows, 2)
        self.assertEqual(audit.iloc[0].invalid_scores, 1)
        self.assertEqual(audit.iloc[0].missing_rows, 2)


class ReportTests(unittest.TestCase):
    def setUp(self):
        self.assertTrue(hasattr(mm, "write_report"), "Report writer is not implemented")
        self.temp = tempfile.TemporaryDirectory()
        self.addCleanup(self.temp.cleanup)
        self.root = Path(self.temp.name)

    def report(self, frame, dirname="report", **kwargs):
        with contextlib.redirect_stdout(io.StringIO()):
            return mm.write_report(frame, self.root / dirname, **kwargs)

    def table(self, dirname, filename):
        return pd.read_csv(self.root / dirname / filename)

    def test_fixed_threshold_is_selected_on_dev_and_confusion_is_correct(self):
        dev = keyed_frame().assign(fold=1, actual=[0, 0, 1, 1], suite26_mint=[.1, .3, .6, .9])
        holdout = keyed_frame().assign(suite26_mint=[.7, .8, .2, .4])
        frame = pd.concat([dev, holdout], ignore_index=True)
        report = self.report(frame, bootstrap=0)
        thresholds = self.table("report", "thresholds.csv")
        selected = thresholds[thresholds.model == "suite26_mint"].iloc[0]
        self.assertAlmostEqual(selected.threshold, .6)
        self.assertEqual(selected.development_rows, 4)
        operational = self.table("report", "operational_metrics.csv")
        row = operational[(operational.model == "suite26_mint") &
                          (operational.fold.astype(str) == "6")].iloc[0]
        self.assertEqual((row.tn, row.fp, row.fn, row.tp), (1, 1, 1, 1))
        self.assertAlmostEqual(row.f2, .5)
        self.assertEqual(set(operational.policy), {"fixed"})
        it = self.table("report", "it_confusion.csv")
        self.assertEqual(len(it), len(operational))
        changed = frame.copy()
        changed.loc[changed.fold == 6, "actual"] = [1, 0, 1, 0]
        self.report(changed, "changed", bootstrap=0)
        changed_thresholds = self.table("changed", "thresholds.csv")
        pd.testing.assert_frame_equal(thresholds, changed_thresholds)
        self.assertTrue(Path(report["paths"]["report"]).is_file())

    def test_one_model_one_class_without_development_remains_auditable(self):
        frame = keyed_frame().drop(columns="committee").assign(actual=0)
        summary = self.report(frame, bootstrap=4)
        cells = self.table("report", "per_cell_metrics.csv")
        self.assertEqual(cells.rows.sum(), 4)
        self.assertTrue(cells.ap.isna().all())
        thresholds = self.table("report", "thresholds.csv")
        self.assertTrue(thresholds.threshold.isna().all())
        self.assertEqual(set(thresholds.status), {"no_development_scores"})
        operational = self.table("report", "operational_metrics.csv")
        self.assertTrue(operational.threshold.isna().all())
        self.assertTrue(operational.tp.isna().all())
        self.assertEqual(summary["models"], ["suite26_mint"])
        # JSON must be interoperable: undefined results are null, never NaN.
        raw = (self.root / "report" / "report.json").read_text()
        self.assertNotIn("NaN", raw)
        self.assertNotIn("Infinity", raw)
        self.assertEqual(json.loads(raw)["rows"], 4)

    def test_paired_bootstrap_uses_cell_macro_and_repeats_student_weights_across_folds(self):
        first = keyed_frame().assign(actual=[1, 0, 1, 0],
                                     suite26_mint=[.9, .8, .7, .1], committee=[.8, .9, .1, .7])
        second = keyed_frame().assign(fold=7, actual=[1, 0, 0, 0],
                                      suite26_mint=[.1, .4, .3, .2], committee=[.9, .8, .7, .6])
        self.report(pd.concat([first, second], ignore_index=True), bootstrap=10, seed=11)
        boots = self.table("report", "bootstrap_ap.csv")
        row = boots[(boots.model_a == "suite26_mint") & (boots.model_b == "committee")].iloc[0]
        # Cell differences: (5/6 - 1/2) and (1/4 - 1), mean = -5/24.
        self.assertAlmostEqual(row.delta_ap, -5 / 24)
        self.assertEqual((row.cells, row.rows, row.students, row.valid_replicates), (2, 8, 4, 5))
        self.assertAlmostEqual(row.ci_low, -17 / 60)
        self.assertAlmostEqual(row.ci_high, -11 / 240)
        draws = self.table("report", "bootstrap_replicates.csv")
        draws = draws[(draws.model_a == "suite26_mint") & (draws.model_b == "committee")]
        # Hand-computed weighted AP for the first ten seed-11 cluster draws.
        expected = {0: -1/12, 2: -1/24, 5: -5/24, 7: -1/8, 8: -7/24}
        for _, draw in draws.iterrows():
            if int(draw.replicate) in expected:
                self.assertAlmostEqual(draw.delta_ap, expected[int(draw.replicate)])
            else:
                self.assertTrue(np.isnan(draw.delta_ap))
        self.assertEqual(row.statistic, "macro_faculty_fold_ap_delta")

    def test_partial_reference_comparisons_use_exactly_shared_rows(self):
        frame = keyed_frame()
        frame["suite22"] = [.7, .6, np.nan, np.nan]
        self.report(frame, bootstrap=0)
        paired = self.table("report", "paired_metrics.csv")
        row = paired[(paired.model_a == "suite26_mint") & (paired.model_b == "suite22")].iloc[0]
        self.assertEqual((row.rows, row.total_rows), (2, 4))
        self.assertAlmostEqual(row.ap_a, 1.)
        self.assertAlmostEqual(row.ap_b, .5)
        self.assertAlmostEqual(row.delta_ap, .5)
        cells = self.table("report", "per_cell_metrics.csv")
        self.assertEqual(cells[cells.model == "suite22"].iloc[0].missing_scores, 2)

    def test_components_compare_seed_tree_memory_and_mint_in_order(self):
        frame = keyed_frame().assign(suite26_seed_forest=[.5, .4, .3, .2],
                                     suite26_tree_only=[.4, .5, .3, .2],
                                     suite26_memory=[.3, .5, .4, .2])
        self.report(frame, bootstrap=0)
        components = self.table("report", "component_uplifts.csv")
        pairs = set(zip(components.model_a, components.model_b))
        self.assertEqual(pairs, {("suite26_tree_only", "suite26_seed_forest"),
                                 ("suite26_memory", "suite26_tree_only"),
                                 ("suite26_mint", "suite26_memory")})
        self.assertTrue(np.isfinite(components.delta_ap).all())

    def test_development_only_and_empty_reports_keep_all_output_schemas(self):
        for name, frame in [("dev_only", keyed_frame().assign(fold=1)),
                            ("empty", keyed_frame().iloc[:0])]:
            with self.subTest(name=name):
                summary = self.report(frame, name, bootstrap=3)
                self.assertEqual(summary["rows"], len(frame))
                operational = self.table(name, "operational_metrics.csv")
                self.assertEqual(len(operational), 0)
                self.assertIn("threshold", operational.columns)
                it = self.table(name, "it_confusion.csv")
                self.assertIn("fold", it.columns)
                boots = self.table(name, "bootstrap_ap.csv")
                self.assertTrue((boots.status == "no_holdout_rows").all())

    def test_development_holdout_overlap_is_rejected(self):
        with self.assertRaisesRegex(ValueError, "overlap"):
            self.report(keyed_frame(), dev_folds=(1, 6), holdout_folds=(6, 7), bootstrap=0)


if __name__ == "__main__":
    unittest.main()
