"""Matched-cell effects, earlier-development policies and bounded decisions."""
from __future__ import annotations
import numpy as np
import pandas as pd
from s34_data import json_hash, sha256_file
from s34_metrics import select_threshold, policy_metrics

PAIRS = [("A1", "A0"), ("A1", "A0M"), ("A2", "A1"), ("A2", "A0"),
         ("A2", "A0M"), ("A2", "P1"), ("A3", "A2"), ("A2", "HGB"),
         ("A2", "CacheReference")]
ARCHIVED = [
    ("Suite25 Committee blend", .369526, "fourteen_faculty"),
    ("Suite22 safe blend", .369159, "fourteen_faculty"),
    ("Suite31 full-history HGB", .353807, "fourteen_faculty"),
    ("Committee", .350152, "fourteen_faculty"),
    ("Suite32 local common / Suite33 S0", .346168, "fourteen_faculty"),
    ("Suite33 S4", .343516, "fourteen_faculty"),
    ("Council", .364204, "thirteen_faculty_excluding_English"),
    ("Council IT", .444927, "IT_only"),
]


def summarize_metrics(metrics):
    if metrics.empty:
        return pd.DataFrame()
    columns = [c for c in ["ap", "auc", "bce", "brier", "ece", "prevalence"] if c in metrics]
    cells = metrics.groupby(["stage", "arm", "faculty", "fold"], as_index=False)[columns].mean()
    summary = cells.groupby(["stage", "arm"], as_index=False)[columns].mean()
    support = cells.groupby(["stage", "arm"]).agg(cells=("fold", "size"),
                                                 defined_ap_cells=("ap", "count")).reset_index()
    return summary.merge(support, on=["stage", "arm"])


def paired_contrasts(metrics, bootstrap=500, bootstrap_seed=34029):
    if metrics.empty:
        return pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
    if metrics.duplicated(["stage", "faculty", "fold", "seed", "arm"]).any():
        raise ValueError("Duplicate model-cell metrics")
    pivot = metrics.pivot(index=["stage", "faculty", "fold", "seed"], columns="arm", values="ap")
    rows, summaries, intervals = [], [], []
    for a, b in PAIRS:
        if a not in pivot or b not in pivot:
            continue
        p = (pivot[a]-pivot[b]).rename("delta_ap").reset_index()
        p["contrast"] = a+"-"+b
        rows.append(p)
        for stage, group in p.groupby("stage"):
            values = group.delta_ap.dropna().to_numpy()
            if not len(values):
                continue
            cell = group.groupby(["faculty", "fold"], as_index=False).delta_ap.mean()
            defined = cell.dropna(subset=["delta_ap"])
            faculty_effects = defined.groupby("faculty").delta_ap.mean()
            fold_effects = defined.groupby("fold").delta_ap.mean()
            seed_effects = group.groupby("seed").delta_ap.mean()
            summaries.append(dict(stage=stage, contrast=a+"-"+b,
                mean_delta_ap=float(defined.delta_ap.mean()), median_delta_ap=float(np.median(values)),
                sd_delta_ap=float(np.std(values, ddof=1)) if len(values) > 1 else 0.,
                wins=int((values > 1e-12).sum()), losses=int((values < -1e-12).sum()),
                ties=int((np.abs(values) <= 1e-12).sum()), paired_seed_cells=len(values),
                paired_cells=len(defined), best_delta=float(values.max()), worst_delta=float(values.min()),
                positive_faculties=int((faculty_effects > 0).sum()),
                positive_folds=int((fold_effects > 0).sum()),
                positive_seeds=int((seed_effects > 0).sum()),
                worst_faculty_delta=float(faculty_effects.min())))
            faculties = sorted(defined.faculty.unique())
            interval = dict(stage=stage, contrast=a+"-"+b, clusters=len(faculties),
                paired_cells=len(defined), replicates=int(bootstrap),
                unit="PAIRED_FACULTY_CLUSTER_FIXED_PERIODS_SEED_AVERAGED",
                ci_low=np.nan, ci_high=np.nan)
            if bootstrap and faculties:
                rng = np.random.default_rng(bootstrap_seed)
                clusters = {f: defined.loc[defined.faculty == f, "delta_ap"].to_numpy() for f in faculties}
                draws = [np.concatenate([clusters[f] for f in rng.choice(faculties, len(faculties))]).mean()
                         for _ in range(int(bootstrap))]
                interval.update(ci_low=float(np.quantile(draws, .025)),
                                ci_high=float(np.quantile(draws, .975)))
            intervals.append(interval)
    return (pd.DataFrame(summaries), pd.concat(rows, ignore_index=True) if rows else pd.DataFrame(),
            pd.DataFrame(intervals))


def operational_tables(index):
    thresholds, operational = [], []
    if index.empty:
        return pd.DataFrame(), pd.DataFrame()
    for (faculty, arm, seed), records in index.groupby(["faculty", "arm", "seed"], sort=True):
        frames = {int(r.fold): pd.read_csv(r.path) for r in records.itertuples()}
        paths = {int(r.fold): r.path for r in records.itertuples()}
        for fold, target in sorted(frames.items()):
            prior_folds = [f for f in sorted(frames) if f < min(fold, 6)]
            prior = pd.concat([frames[f] for f in prior_folds], ignore_index=True) if prior_folds else None
            for beta in (1, 2):
                policy = select_threshold(prior.y, prior.p, beta) if prior is not None else select_threshold([], [], beta)
                threshold = policy["threshold"]
                base = dict(faculty=faculty, arm=arm, seed=int(seed), fold=fold,
                            stage="development" if fold <= 5 else "retrospective", beta=beta)
                supporting_hash = json_hash([sha256_file(paths[f]) for f in prior_folds])
                thresholds.append(dict(**base, **policy, source_folds=",".join(map(str, prior_folds)),
                                       support_sha256=supporting_hash,
                                       tie_rule="MAXIMUM_THRESHOLD", frozen_for_retrospective=fold >= 6))
                operational.append(dict(**base, threshold=threshold, status=policy["status"],
                                        **policy_metrics(target.y, target.p, threshold)))
    return pd.DataFrame(thresholds), pd.DataFrame(operational)


def promotion_decision(metrics, operational, primary=False):
    dev = metrics[metrics.fold <= 5].copy()
    if not primary:
        return dict(research_promotion="INCONCLUSIVE", hypothesis="INCONCLUSIVE",
                    reason="Restricted, smoke, synthetic or nonprimary protocol; full matrix required.",
                    promoted_for_deployment=False, primary_candidate="A2")
    arms, seeds = {"A0", "A0M", "A1", "A2", "P1"}, {29, 129, 229}
    expected = dev[dev.arm.isin(arms)]
    complete = (dev.faculty.nunique() == 14 and set(expected.fold) == {1, 2, 3, 4, 5}
                and set(expected.seed) == seeds and len(expected) == 14*5*3*5
                and not expected.duplicated(["faculty", "fold", "seed", "arm"]).any())
    if not complete:
        return dict(research_promotion="INCONCLUSIVE", hypothesis="INCONCLUSIVE",
                    reason="Incomplete principal development matrix.", promoted_for_deployment=False,
                    primary_candidate="A2")
    summary, deltas, _ = paired_contrasts(dev, bootstrap=0)
    effects = {r.contrast: float(r.mean_delta_ap) for r in summary.itertuples()}
    d = deltas[deltas.contrast == "A2-A1"]
    faculty_effect = d.groupby("faculty").delta_ap.mean()
    fold_effect = d.groupby("fold").delta_ap.mean()
    seed_effect = d.groupby("seed").delta_ap.mean()
    gates = dict(
        defined_principal_ap=bool(np.isfinite(expected.ap).all()),
        A2_A1_min_010=effects.get("A2-A1", -np.inf) >= .010,
        A2_A0_min_010=effects.get("A2-A0", -np.inf) >= .010,
        A2_A0M_positive=effects.get("A2-A0M", -np.inf) > 0.,
        A2_P1_min_005=effects.get("A2-P1", -np.inf) >= .005,
        positive_folds=int((fold_effect > 0).sum()) >= 3,
        positive_faculties=int((faculty_effect > 0).sum()) >= 8,
        positive_seeds=int((seed_effect > 0).sum()) >= 2,
        faculty_harm_bounded=float(faculty_effect.min()) >= -.030)
    op = operational[(operational.fold.between(2, 5)) & operational.arm.isin(["A1", "A2"])] if not operational.empty else operational
    for beta in (1, 2):
        if op.empty:
            gates[f"operational_beta{beta}"] = False
            continue
        p = op[op.beta == beta].pivot(index=["faculty", "fold", "seed"], columns="arm",
                                     values=["f1" if beta == 1 else "f2", "alert_fraction"])
        metric = "f1" if beta == 1 else "f2"
        valid = (len(p) == 14*4*3 and (metric, "A1") in p and (metric, "A2") in p)
        if valid:
            delta = p[(metric, "A2")]-p[(metric, "A1")]
            load = p[("alert_fraction", "A2")]-p[("alert_fraction", "A1")]
            valid = bool(np.isfinite(delta).all() and np.isfinite(load).all()
                         and delta.mean() >= -.010 and abs(load.mean()) <= .050)
        gates[f"operational_beta{beta}"] = bool(valid)
    passed = all(gates.values())
    hypothesis = ("SUPPORTED_ON_DEVELOPMENT" if passed else
                  "UNSUPPORTED" if effects.get("A2-A1", 0) <= 0 or effects.get("A2-P1", 0) <= 0
                  else "INCONCLUSIVE")
    return dict(research_promotion="PASS" if passed else "FAIL", hypothesis=hypothesis,
                gates={k: bool(v) for k, v in gates.items()}, effects=effects,
                positive_faculties=int((faculty_effect > 0).sum()),
                positive_folds=int((fold_effect > 0).sum()),
                positive_seeds=int((seed_effect > 0).sum()), worst_faculty=float(faculty_effect.min()),
                primary_candidate="A2", promoted_for_deployment=False,
                reason="Frozen development gates; retrospective outcomes do not enter this decision.")


def archived_comparisons(primary_population):
    return pd.DataFrame([dict(model=name, retrospective_ap=score, population=population,
        comparison="METRICS_ONLY_DIFFERENT_RECIPES_NO_PAIRED_CLAIM",
        full_population_available=bool(primary_population)) for name, score, population in ARCHIVED])
