"""Behavioral contracts for matched evaluation and safe reference alignment."""
import importlib
import contextlib
import io
import json
import tempfile
import unittest
from pathlib import Path

import numpy as np
import pandas as pd

try:
    mm = importlib.import_module("evaluation")
except ModuleNotFoundError:
    mm = None


class MetricsTests(unittest.TestCase):
    def setUp(self):
        self.assertIsNotNone(mm, "evaluation is not implemented")

    def test_known_ranking_calibration_and_warning_budget(self):
        # Catches pooled/thresholded AP, incorrect F2, and score-bin weighting.
        y = np.array([1, 0, 1, 0] + [0] * 16)
        score = np.array([.9, .8, .7, .1] + [.05] * 16)
        got = mm.metrics(y, score)
        self.assertAlmostEqual(got["ap"], 5 / 6)
        self.assertAlmostEqual(got["auc"], 35 / 36)
        self.assertAlmostEqual(got["brier"], .0395)
        self.assertAlmostEqual(got["ece10"], .105)
        self.assertAlmostEqual(got["f2_at_10"], .5)
        self.assertAlmostEqual(got["f1_at_15"], .8)
        self.assertAlmostEqual(got["f2_at_15"], 10 / 11)
        self.assertAlmostEqual(got["f2_at_20"], 10 / 12)
        self.assertAlmostEqual(got["f2_at_25"], 10 / 13)
        self.assertEqual(got["flagged_at_20"], 4)
        self.assertEqual(got["rows"], 20)
        self.assertEqual(got["positives"], 2)

    def test_missing_scores_report_support_without_imputation(self):
        got = mm.metrics([0, 1, 1], [.2, .8, np.nan])
        self.assertEqual((got["rows"], got["total_rows"], got["missing_scores"]), (2, 3, 1))
        self.assertAlmostEqual(got["ap"], 1.)
        self.assertAlmostEqual(got["brier"], .04)

    def test_invalid_labels_and_probabilities_fail_loudly(self):
        for labels, score in [([0, 2], [.2, .8]), ([0, np.nan], [.2, .8]),
                              ([0, 1], [.2, 1.2]), ([0, 1], [.2, np.inf])]:
            with self.subTest(labels=labels, score=score):
                with self.assertRaises(ValueError):
                    mm.metrics(labels, score)

    def test_one_class_and_empty_support_keep_calibration_defined(self):
        one = mm.metrics([0, 0], [.1, .2])
        self.assertTrue(np.isnan(one["ap"]))
        self.assertTrue(np.isnan(one["auc"]))
        self.assertAlmostEqual(one["brier"], .025)
        empty = mm.metrics([0, 1], [np.nan, np.nan])
        self.assertEqual(empty["rows"], 0)
        self.assertTrue(np.isnan(empty["f2_at_20"]))

    def test_macro_ap_weights_cells_equally_instead_of_pooling(self):
        frame = pd.DataFrame({"fold": [6] * 10, "faculty": ["A"] * 2 + ["B"] * 8,
                              "actual": [1, 0] + [0] * 7 + [1],
                              "score": [.9, .8, .7, .6, .5, .4, .3, .2, .1, .05]})
        self.assertAlmostEqual(mm.macro_ap(frame, "score"), 9 / 16)
        self.assertAlmostEqual(mm.macro_ap(frame.drop(columns="fold"), "score"), 9 / 16)


def keyed_frame():
    return pd.DataFrame({
        "fold": [6] * 4, "faculty": ["IT"] * 4,
        "sample_id": ["e0", "e1", "e2", "e3"],
        "student_id": ["s0", "s1", "s2", "s3"],
        "course_id": ["c0", "c0", "c1", "c1"],
        "semester": [6070] * 4, "actual": [0, 1, 0, 1],
        "committee": [.3, .6, .2, .7], "suite27_mcgb": [.1, .9, .2, .8]})


class ReferenceTests(unittest.TestCase):
    def setUp(self):
        self.assertTrue(hasattr(mm, "add_references"), "Keyed reference loader is not implemented")
        self.temp = tempfile.TemporaryDirectory()
        self.addCleanup(self.temp.cleanup)
        self.root = Path(self.temp.name)

    def save(self, source, filename="IT_fold_6_test_results.csv"):
        path = self.root / filename
        path.parent.mkdir(parents=True, exist_ok=True)
        source.to_csv(path, index=False)
        return path

    def test_reordered_keys_align_even_when_label_arrays_are_identical(self):
        # The old label-array check would accept this order but assign wrong scores.
        frame = keyed_frame()
        source = frame.drop(columns=["committee", "suite27_mcgb"])
        source["final"] = [.11, .91, .21, .81]
        self.save(source.iloc[[2, 3, 0, 1]])
        wide, audit = mm.add_references(frame, {"suite22": self.root})
        np.testing.assert_allclose(wide.suite22, [.11, .91, .21, .81])
        self.assertEqual(audit.iloc[0].status, "loaded")
        self.assertEqual(audit.iloc[0].matched_rows, 4)
        self.assertEqual(audit.iloc[0]["keys"], "sample_id")
        self.assertNotIn("suite22", frame)

    def test_label_only_reference_is_skipped_despite_matching_labels(self):
        self.save(pd.DataFrame({"actual": [0, 1, 0, 1], "final": [.2, .8, .3, .7]}))
        wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root})
        self.assertTrue(wide.suite22.isna().all())
        self.assertEqual(audit.iloc[0].status, "rejected")
        self.assertIn("identifier", audit.iloc[0].detail.lower())

    def test_matching_sample_id_does_not_override_conflicting_ids_or_labels(self):
        for column, replacement in [("student_id", "wrong-student"), ("course_id", "wrong-course"),
                                    ("semester", 6069), ("actual", 1)]:
            with self.subTest(column=column):
                source = keyed_frame()
                source["final"] = .5
                source.loc[0, column] = replacement
                self.save(source)
                wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root})
                self.assertTrue(wide.suite22.isna().all())
                self.assertEqual(audit.iloc[0].status, "rejected")
                self.assertIn(column, audit.iloc[0].detail)

    def test_legacy_ids_and_raw_semester_codes_match_ordinal_semesters(self):
        source = keyed_frame().drop(columns=["sample_id", "semester"])
        source = source.rename(columns={"student_id": "IDSV", "course_id": "IDMH",
                                        "actual": "actual_failure_lt5"})
        source["target_NHHK"] = 20232
        source["failure_probability_lt5"] = [.11, .91, .21, .81]
        self.save(source.iloc[::-1])
        wide, audit = mm.add_references(keyed_frame(), {"v2": self.root})
        np.testing.assert_allclose(wide.v2, [.11, .91, .21, .81])
        self.assertEqual(audit.iloc[0]["keys"], "student_id,course_id,semester")

    def test_student_course_keys_require_fixed_fold_file_and_uniqueness(self):
        source = keyed_frame()[["student_id", "course_id", "actual"]].copy()
        source["final"] = [.11, .91, .21, .81]
        self.save(source.iloc[::-1])
        wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root})
        np.testing.assert_allclose(wide.suite22, [.11, .91, .21, .81])
        self.assertEqual(audit.iloc[0]["keys"], "student_id,course_id")
        (self.root / "IT_fold_6_test_results.csv").unlink()
        self.save(source, "predictions.csv")
        wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root})
        self.assertTrue(wide.suite22.isna().all())
        self.assertEqual(audit.iloc[0].status, "rejected")

    def test_duplicate_keys_are_rejected_and_missing_files_are_audited(self):
        source = keyed_frame()[["sample_id", "actual"]].copy()
        source["sample_id"] = "duplicate"
        source["final"] = .5
        self.save(source)
        wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root,
                                                               "suite25": self.root / "absent"})
        self.assertTrue(wide[["suite22", "suite25"]].isna().all().all())
        self.assertEqual(audit.set_index("reference").loc["suite22", "status"], "rejected")
        self.assertEqual(audit.set_index("reference").loc["suite25", "status"], "missing")

    def test_partial_coverage_preserves_nan_and_counts_invalid_scores(self):
        source = keyed_frame().iloc[[0, 1, 2]].copy()
        source["final"] = [.2, .8, "broken"]
        self.save(source)
        wide, audit = mm.add_references(keyed_frame(), {"suite22": self.root})
        self.assertEqual(wide.suite22.notna().sum(), 2)
        self.assertEqual(audit.iloc[0].status, "partial")
        self.assertEqual(audit.iloc[0].matched_rows, 3)
        self.assertEqual(audit.iloc[0].scored_rows, 2)
        self.assertEqual(audit.iloc[0].invalid_scores, 1)
        self.assertEqual(audit.iloc[0].missing_rows, 2)


class ReportTests(unittest.TestCase):
    def setUp(self):
        self.assertTrue(hasattr(mm, "write_report"), "Report writer is not implemented")
        self.temp = tempfile.TemporaryDirectory()
        self.addCleanup(self.temp.cleanup)
        self.root = Path(self.temp.name)

    def report(self, frame, dirname="report", **kwargs):
        with contextlib.redirect_stdout(io.StringIO()):
            return mm.write_report(frame, self.root / dirname, **kwargs)

    def table(self, dirname, filename):
        return pd.read_csv(self.root / dirname / filename)

    def test_fixed_threshold_is_selected_on_dev_and_confusion_is_correct(self):
        dev = keyed_frame().assign(fold=1, semester=6060, actual=[0, 0, 1, 1], suite27_mcgb=[.1, .3, .6, .9])
        holdout = keyed_frame().assign(suite27_mcgb=[.7, .8, .2, .4])
        frame = pd.concat([dev, holdout], ignore_index=True)
        report = self.report(frame, bootstrap=0)
        thresholds = self.table("report", "thresholds.csv")
        selected = thresholds[thresholds.model == "suite27_mcgb"].iloc[0]
        self.assertAlmostEqual(selected.threshold, .6)
        self.assertEqual(selected.development_rows, 4)
        operational = self.table("report", "operational_metrics.csv")
        row = operational[(operational.model == "suite27_mcgb") &
                          (operational.fold.astype(str) == "6")].iloc[0]
        self.assertEqual((row.tn, row.fp, row.fn, row.tp), (1, 1, 1, 1))
        self.assertAlmostEqual(row.f2, .5)
        self.assertEqual(set(operational.policy), {"fixed"})
        it = self.table("report", "it_confusion.csv")
        self.assertEqual(len(it), len(operational))
        changed = frame.copy()
        changed.loc[changed.fold == 6, "actual"] = [1, 0, 1, 0]
        self.report(changed, "changed", bootstrap=0)
        changed_thresholds = self.table("changed", "thresholds.csv")
        pd.testing.assert_frame_equal(thresholds, changed_thresholds)
        self.assertTrue(Path(report["paths"]["report"]).is_file())

    def test_one_model_one_class_without_development_remains_auditable(self):
        frame = keyed_frame().drop(columns="committee").assign(actual=0)
        summary = self.report(frame, bootstrap=4)
        cells = self.table("report", "per_cell_metrics.csv")
        self.assertEqual(cells.rows.sum(), 4)
        self.assertTrue(cells.ap.isna().all())
        thresholds = self.table("report", "thresholds.csv")
        self.assertTrue(thresholds.threshold.isna().all())
        self.assertEqual(set(thresholds.status), {"no_development_scores"})
        operational = self.table("report", "operational_metrics.csv")
        self.assertTrue(operational.threshold.isna().all())
        self.assertTrue(operational.tp.isna().all())
        self.assertEqual(summary["models"], ["suite27_mcgb"])
        # JSON must be interoperable: undefined results are null, never NaN.
        raw = (self.root / "report" / "report.json").read_text()
        self.assertNotIn("NaN", raw)
        self.assertNotIn("Infinity", raw)
        self.assertEqual(json.loads(raw)["rows"], 4)

    def test_paired_bootstrap_uses_cell_macro_and_repeats_student_weights_across_folds(self):
        first = keyed_frame().assign(actual=[1, 0, 1, 0],
                                     suite27_mcgb=[.9, .8, .7, .1], committee=[.8, .9, .1, .7])
        second = keyed_frame().assign(fold=7, actual=[1, 0, 0, 0],
                                      suite27_mcgb=[.1, .4, .3, .2], committee=[.9, .8, .7, .6])
        self.report(pd.concat([first, second], ignore_index=True), bootstrap=10, seed=11)
        boots = self.table("report", "bootstrap_ap.csv")
        row = boots[(boots.model_a == "suite27_mcgb") & (boots.model_b == "committee")].iloc[0]
        # Cell differences: (5/6 - 1/2) and (1/4 - 1), mean = -5/24.
        self.assertAlmostEqual(row.delta_ap, -5 / 24)
        self.assertEqual((row.cells, row.rows, row.students, row.valid_replicates), (2, 8, 4, 5))
        self.assertAlmostEqual(row.ci_low, -17 / 60)
        self.assertAlmostEqual(row.ci_high, -11 / 240)
        draws = self.table("report", "bootstrap_replicates.csv")
        draws = draws[(draws.model_a == "suite27_mcgb") & (draws.model_b == "committee")]
        # Hand-computed weighted AP for the first ten seed-11 cluster draws.
        expected = {0: -1/12, 2: -1/24, 5: -5/24, 7: -1/8, 8: -7/24}
        for _, draw in draws.iterrows():
            if int(draw.replicate) in expected:
                self.assertAlmostEqual(draw.delta_ap, expected[int(draw.replicate)])
            else:
                self.assertTrue(np.isnan(draw.delta_ap))
        self.assertEqual(row.statistic, "macro_faculty_fold_ap_delta")

    def test_partial_reference_comparisons_use_exactly_shared_rows(self):
        frame = keyed_frame()
        frame["suite22"] = [.7, .6, np.nan, np.nan]
        self.report(frame, bootstrap=0)
        paired = self.table("report", "paired_metrics.csv")
        row = paired[(paired.model_a == "suite27_mcgb") & (paired.model_b == "suite22")].iloc[0]
        self.assertEqual((row.rows, row.total_rows), (2, 4))
        self.assertAlmostEqual(row.ap_a, 1.)
        self.assertAlmostEqual(row.ap_b, .5)
        self.assertAlmostEqual(row.delta_ap, .5)
        cells = self.table("report", "per_cell_metrics.csv")
        self.assertEqual(cells[cells.model == "suite22"].iloc[0].missing_scores, 2)

    def test_components_compare_all_six_variants_in_prespecified_order(self):
        frame = keyed_frame().assign(suite27_stock=[.6, .4, .3, .2],
                                     suite27_native=[.5, .4, .3, .2],
                                     suite27_deterministic=[.4, .5, .2, .3],
                                     suite27_path=[.4, .5, .3, .2],
                                     suite27_global=[.3, .5, .4, .2])
        self.report(frame, bootstrap=0)
        components = self.table("report", "component_uplifts.csv")
        pairs = set(zip(components.model_a, components.model_b))
        self.assertEqual(pairs, {("suite27_native", "suite27_stock"),
                                 ("suite27_deterministic", "suite27_native"),
                                 ("suite27_global", "suite27_deterministic"),
                                 ("suite27_path", "suite27_global"),
                                 ("suite27_mcgb", "suite27_path")})
        self.assertTrue(np.isfinite(components.delta_ap).all())

    def test_development_only_and_empty_reports_keep_all_output_schemas(self):
        for name, frame in [("dev_only", keyed_frame().assign(fold=1)),
                            ("empty", keyed_frame().iloc[:0])]:
            with self.subTest(name=name):
                summary = self.report(frame, name, bootstrap=3)
                self.assertEqual(summary["rows"], len(frame))
                operational = self.table(name, "operational_metrics.csv")
                self.assertEqual(len(operational), 0)
                self.assertIn("threshold", operational.columns)
                it = self.table(name, "it_confusion.csv")
                self.assertIn("fold", it.columns)
                boots = self.table(name, "bootstrap_ap.csv")
                self.assertTrue((boots.status == "no_holdout_rows").all())
                workload = self.table(name, "student_workload.csv")
                self.assertEqual(tuple(workload.columns), mm.WORKLOAD_COLUMNS)
                self.assertEqual(len(workload), 0)

    def test_workload_counts_student_burden_across_courses_and_folds(self):
        frame = keyed_frame().assign(student_id=['s0', 's0', 's1', 's2'])
        development = keyed_frame().assign(fold=1, semester=6060)
        self.report(pd.concat([development, frame], ignore_index=True), bootstrap=0)
        workload = self.table('report', 'student_workload.csv')
        fixed = workload[(workload.model == 'suite27_mcgb') & (workload.fold.astype(str) == '6') & (workload.policy == 'fixed')].iloc[0]
        self.assertEqual((fixed.students, fixed.flagged_courses, fixed.flagged_students), (3, 2, 2))
        self.assertAlmostEqual(fixed.student_flag_rate, 2 / 3)

    def test_console_prints_every_fold_faculty_operational_and_budget_column(self):
        frame = pd.concat([keyed_frame().assign(fold=1, semester=6060), keyed_frame(), keyed_frame().assign(fold=7, faculty='BUS')], ignore_index=True)
        output = io.StringIO()
        with contextlib.redirect_stdout(output):
            mm.write_report(frame, self.root / 'console', bootstrap=2)
        console = output.getvalue()
        for text in ('Per Cell Metrics', 'Per Fold Metrics', 'Per Faculty Metrics', 'Operational Metrics',
                     'Bootstrap Ap', 'Student Workload', 'BUS', 'IT', 'tp_at_10', 'tn_at_25'):
            self.assertIn(text, console)

    def test_development_holdout_overlap_is_rejected(self):
        with self.assertRaisesRegex(ValueError, "overlap"):
            self.report(keyed_frame(), dev_folds=(1, 6), holdout_folds=(6, 7), bootstrap=0)

    def test_future_development_semester_is_rejected_before_report_files(self):
        frame = pd.concat([keyed_frame().assign(fold=7, semester=63), keyed_frame().assign(semester=60)], ignore_index=True)
        with self.assertRaisesRegex(ValueError, 'chronology violated'):
            self.report(frame, 'future', dev_folds=(7,), holdout_folds=(6,), bootstrap=0)
        self.assertFalse((self.root / 'future').exists())
        with self.assertRaisesRegex(ValueError, 'chronology violated'):
            mm._operational_tables(frame, ['suite27_mcgb'], (7,), (6,))

    def test_equal_semesters_reject_but_fold_fallback_is_explicit_when_missing(self):
        frame = pd.concat([keyed_frame().assign(fold=1), keyed_frame()], ignore_index=True)
        with self.assertRaisesRegex(ValueError, 'chronology violated'):
            self.report(frame, 'equal', bootstrap=0)
        summary = self.report(frame.drop(columns='semester'), 'legacy', bootstrap=0)
        self.assertIn('numeric fold order', summary['conventions']['threshold_chronology'])
        with self.assertRaisesRegex(ValueError, 'chronology violated'):
            self.report(frame.drop(columns='semester'), 'reverse_legacy', dev_folds=(6,), holdout_folds=(1,), bootstrap=0)


if __name__ == "__main__":
    unittest.main()
