
# -*- coding: utf-8 -*-
"""Suite18 — chronology-safe multiscale FMM representations on top of Suite17.

This is intentionally a thin derivative of Suite17:
  * Committee remains immutable.
  * Suite17 R/M/L construction, Newton-HGB conditional learner, robust temporal
    admission, dense OOF ledger, eta grid, stage count and alignment are reused.
  * The only deployable architectural change is raw-view + FMM-descriptor
    concatenation for each R ablation, M and L.
  * A separate common/flat FMM encoder is diagnostic only and supports the
    paper-inspired FMM -> centroid/CF -> CART/RF branch.
"""
from __future__ import annotations

import argparse
import builtins
import gc
import hashlib
import json
import os
import re
import shutil
import sys
from dataclasses import asdict
from pathlib import Path

import joblib
import numpy as np
import pandas as pd

# Resolve Suite17 from Suite18's own location, not from the shell CWD.
#
# Supported layouts include:
#   30072026/
#     suite17/
#       train_suite17_robust_forward_specialist_boosting.py
#     suite18/
#       train_suite18_fmm_forward_specialist_boosting.py
#
#   30072026/
#     suite17_robust_forward_specialist_boosting/
#       train_suite17_robust_forward_specialist_boosting.py
#     suite18/
#
# and the older layout where Suite17 .py files live directly under 30072026/.
_HERE = Path(__file__).resolve().parent
_PROJECT_ROOT = _HERE.parent
_SUITE17_ENTRY = "train_suite17_robust_forward_specialist_boosting.py"


def _suite17_search_dirs():
    candidates = []

    env_dir = os.environ.get("SUITE17_CODE_DIR")
    if env_dir:
        env_path = Path(env_dir).expanduser()
        candidates.append(env_path)
        # Be forgiving if the user points SUITE17_CODE_DIR at project root.
        if env_path.exists() and env_path.is_dir():
            try:
                candidates.extend(
                    d for d in sorted(env_path.iterdir())
                    if d.is_dir() and d.name.lower().startswith("suite17")
                )
            except OSError:
                pass

    # File-relative locations are authoritative, so `cd suite18` works.
    candidates += [
        _HERE,
        _PROJECT_ROOT,
        _PROJECT_ROOT / "suite17",
        _PROJECT_ROOT / "Suite17",
    ]

    # Accept any sibling Suite17 directory name.
    if _PROJECT_ROOT.exists():
        try:
            candidates.extend(
                d for d in sorted(_PROJECT_ROOT.iterdir())
                if d.is_dir() and d.name.lower().startswith("suite17")
            )
        except OSError:
            pass

    # CWD is only a fallback; correctness does not depend on it.
    cwd = Path.cwd()
    candidates += [cwd, cwd.parent]
    for base in (cwd, cwd.parent):
        if base.exists():
            try:
                candidates.extend(
                    d for d in sorted(base.iterdir())
                    if d.is_dir() and d.name.lower().startswith("suite17")
                )
            except OSError:
                pass

    seen = set()
    out = []
    for d in candidates:
        try:
            d = d.resolve()
        except Exception:
            continue
        key = str(d)
        if key not in seen:
            seen.add(key)
            out.append(d)
    return out


_SUITE17_DIR = None
_SUITE17_SEARCHED = []
for _candidate in _suite17_search_dirs():
    _SUITE17_SEARCHED.append(str(_candidate))
    if (_candidate / _SUITE17_ENTRY).is_file():
        _SUITE17_DIR = _candidate
        if str(_candidate) in sys.path:
            sys.path.remove(str(_candidate))
        sys.path.insert(0, str(_candidate))
        break

if _SUITE17_DIR is None:
    _searched_text = "\n  - ".join(_SUITE17_SEARCHED)
    raise ImportError(
        "Suite18 could not locate the Suite17 source entry file "
        f"'{_SUITE17_ENTRY}'.\n"
        "Searched:\n  - " + _searched_text + "\n"
        "Expected the Suite17 source folder to be beside suite18 (its directory "
        "name may be suite17 or any name beginning with suite17). "
        "As an override, set SUITE17_CODE_DIR=/absolute/path/to/suite17."
    )

try:
    import train_suite17_robust_forward_specialist_boosting as s17
except ImportError as exc:
    raise ImportError(
        f"Suite18 located Suite17 at '{_SUITE17_DIR}', but Suite17 itself could "
        "not be imported. Make sure its dependency files (suite17_pipeline.py, "
        "suite17_training.py, suite17_reporting.py, etc.) are in that source tree."
    ) from exc

from suite18_reporting import (
    suite18_title,
    suite18_display_frame,
    prequential_summary_rows,
    feature_selection_preview_rows,
    hyperbox_summary_for_log,
    reference_head_to_head,
)

from suite18_fmm import (
    FMMConfig,
    build_prequential_fmm_representation,
    paper_hybrid_predictions,
    hyperbox_scale_summary,
)

SUITE18_RAW = "suite18_hgb_raw"
SUITE18_FINAL = "suite18_hgb_final"
SUITE17_REFERENCE = "suite17_hgb_final_reference"
FMM_STANDALONE = "fmm_common_standalone"
FMM_CART = "fmm_common_cart_diagnostic"
FMM_RF = "fmm_common_rf_diagnostic"
FMM_CODE_VERSION = 1

# Capture pristine Suite17 hooks before monkey-patching them.
_ORIG_LOAD_FOLD_BUNDLE = s17.load_fold_bundle
_ORIG_BUILD_STRUCTURED_VIEWS = s17.build_or_load_structured_views
_ORIG_BUILD_LOCAL = s17.build_or_load_local
_ORIG_PRINT_TABLE = getattr(s17, 'print_table', None)

_FOLD_STATE: dict[int, dict] = {}


def _suite18_module_print(*args, **kwargs):
    """Translate inherited Suite17 experiment labels without touching values."""
    vals = [
        suite18_title(x) if isinstance(x, str) else x
        for x in args
    ]
    builtins.print(*vals, **kwargs)


def _suite18_print_table(title, df, cols=None, max_rows=None):
    z = suite18_display_frame(df)
    use_cols = cols
    if cols is not None:
        mapping = {
            c: suite18_display_frame(pd.DataFrame(columns=[c])).columns[0]
            for c in cols
        }
        use_cols = [mapping[c] for c in cols]
    if _ORIG_PRINT_TABLE is not None:
        return _ORIG_PRINT_TABLE(suite18_title(title), z, use_cols, max_rows)

    # Defensive fallback for stripped/minimal Suite17 source copies.
    builtins.print("\n" + suite18_title(title))
    builtins.print("-" * min(180, max(80, len(str(title)) + 4)))
    show = z if use_cols is None else z[[c for c in use_cols if c in z.columns]]
    if max_rows:
        show = show.head(int(max_rows))
    builtins.print(show.to_string(index=False, float_format=lambda x: f"{x:.4f}"))


def _suite18_print_win_rates(meanf, candidate=SUITE18_FINAL):
    """Suite17-compatible pairwise audit with truthful Suite18 labeling."""
    h = reference_head_to_head(meanf, candidate)
    builtins.print(f"\nSUITE18 WIN-RATE / DELTA AUDIT — candidate={candidate}")
    if h.empty:
        builtins.print("  no overlapping reference models available")
        return
    for r in h.to_dict("records"):
        bits = []
        if "mean_delta_ap" in r:
            bits.append(
                f"AP: {int(r.get('ap_wins',0))}/{int(r['faculties'])} wins "
                f"meanΔ={r.get('mean_delta_ap',np.nan):+.4f} "
                f"medΔ={r.get('median_delta_ap',np.nan):+.4f} "
                f"worstΔ={r.get('worst_delta_ap',np.nan):+.4f}"
            )
        if "mean_delta_f2" in r:
            bits.append(
                f"F2@20: {int(r.get('f2_wins',0))}/{int(r['faculties'])} wins "
                f"meanΔ={r.get('mean_delta_f2',np.nan):+.4f} "
                f"medΔ={r.get('median_delta_f2',np.nan):+.4f} "
                f"worstΔ={r.get('worst_delta_f2',np.nan):+.4f}"
            )
        builtins.print(f"  vs {r['reference']}: " + " | ".join(bits))


def _state(fold):
    return _FOLD_STATE.setdefault(int(fold), {
        "results": {},
        "raw_dims": {},
        "common_names": None,
        "paper": None,
        "target_y": None,
        "target_faculty_idx": None,
        "target_faculty_names": None,
    })


def _config(args):
    return FMMConfig(
        top_k=int(args.fmm_top_k),
        thetas=tuple(float(x) for x in args.fmm_thetas),
        gamma=float(args.fmm_gamma),
        confidence_k=float(args.fmm_confidence_k),
        min_history_rows=int(args.fmm_min_history_rows),
        min_history_failures=int(args.fmm_min_history_failures),
        chunk_size=int(args.fmm_chunk_size),
        random_state=int(args.seed),
    ).validate()


def _view_signature(view_name, Xh, Xt, names, blocks, cfg):
    b = np.asarray(blocks, float)
    finite = b[np.isfinite(b)]
    payload = {
        "suite18_fmm_code_version": FMM_CODE_VERSION,
        "view": str(view_name),
        "hist_shape": list(np.asarray(Xh).shape),
        "target_shape": list(np.asarray(Xt).shape),
        "feature_names": list(names),
        "block_min": float(np.min(finite)) if len(finite) else None,
        "block_max": float(np.max(finite)) if len(finite) else None,
        "unique_blocks": int(len(np.unique(finite))) if len(finite) else 0,
        "config": asdict(cfg),
    }
    return hashlib.sha1(json.dumps(payload, sort_keys=True, default=str).encode()).hexdigest()


def _safe_view_name(name):
    return re.sub(r"[^A-Za-z0-9_.-]+", "_", str(name))


def _load_or_build_fmm(view_name, Xh, Xt, names, hist, args, fold, out):
    cfg = _config(args)
    fd = Path(out) / f"fold_{int(fold)}"
    cache_dir = fd / "fmm_cache"
    cache_dir.mkdir(parents=True, exist_ok=True)
    path = cache_dir / f"{_safe_view_name(view_name)}.joblib"
    sig = _view_signature(view_name, Xh, Xt, names, hist.blocks, cfg)

    result = None
    if path.exists() and not args.force_fmm:
        try:
            obj = joblib.load(path)
            if obj.get("signature") == sig:
                result = obj["result"]
                print(f"[FMM CACHE] reuse {view_name}: {path}")
        except Exception as exc:
            print(f"[FMM CACHE WARN] could not reuse {path}: {exc}")

    if result is None:
        print(
            f"[FMM BUILD] {view_name}: raw={np.asarray(Xh).shape[1]} "
            f"top_k={min(cfg.top_k,np.asarray(Xh).shape[1])} "
            f"theta={cfg.thetas} strict=query-before-update"
        )
        result = build_prequential_fmm_representation(
            Xh, hist.y, hist.blocks, Xt, names, cfg, view_name=view_name
        )
        joblib.dump({"signature": sig, "result": result}, path, compress=3)

    st = _state(fold)
    st["results"][view_name] = result
    st["raw_dims"][view_name] = int(np.asarray(Xh).shape[1])

    selected = (
        len(result.preprocessor.selected_indices_)
        if result.preprocessor is not None else 0
    )
    print(
        f"[FMM READY] {view_name}: init_block={result.initialization_block} "
        f"selected={selected} descriptor={result.hist_descriptor.shape[1]} "
        f"hist_available={(np.any(result.hist_descriptor!=0,axis=1)).mean():.1%}"
    )
    for row in hyperbox_scale_summary(result):
        if row["class"] == 1:
            print(
                f"  theta={row['theta']:.2f} failure_boxes={row['boxes']} "
                f"support={row['support']} support_med={row['support_median']:.1f} "
                f"CF={row['confidence_mean']:.3f} width={row['width_mean']:.3f}"
            )
    return result


def _capture_load_fold_bundle(*args, **kwargs):
    # Signature in Suite17: cache_dir, structured_dir, faculties, fold, common_names=None
    r = _ORIG_LOAD_FOLD_BUNDLE(*args, **kwargs)
    fold = kwargs.get("fold", args[3] if len(args) > 3 else None)
    if fold is not None:
        _state(int(fold))["common_names"] = list(r[2])
    return r


def _suite18_build_structured_views(hist, target, raw_index, args, fold, out):
    """Return the original Suite17 views with only FMM descriptors appended."""
    z = _ORIG_BUILD_STRUCTURED_VIEWS(hist, target, raw_index, args, fold, out)

    # Common/flat FMM is diagnostic only: it does NOT alter Committee q and does
    # not alter Suite17's local-predictor construction.
    common_names = _state(fold).get("common_names")
    if not common_names or len(common_names) != hist.flat.shape[1]:
        common_names = [f"flat_{i}" for i in range(hist.flat.shape[1])]
    common = _load_or_build_fmm(
        "COMMITTEE_FLAT", hist.flat, target.flat, common_names,
        hist, args, fold, out
    )
    paper = paper_hybrid_predictions(
        common,
        theta=float(args.fmm_paper_theta),
        cart_max_depth=(None if int(args.fmm_cart_max_depth) <= 0 else int(args.fmm_cart_max_depth)),
        rf_trees=int(args.fmm_rf_trees),
        seed=int(args.seed) + int(fold) * 1009,
    )
    st = _state(fold)
    st["paper"] = paper
    st["target_y"] = np.asarray(target.y, int)
    st["target_faculty_idx"] = np.asarray(target.faculty_idx, int)
    st["target_faculty_names"] = list(target.faculty_names)

    # Each R ablation receives its OWN encoder.  This is important: using the
    # R_FULL FMM representation for R_RELATION would leak ablated curriculum/
    # latent information into the ablation.
    for view_name in tuple(s17.R_VARIANTS) + ("M",):
        raw_h = np.asarray(z["hist"][view_name], np.float32)
        raw_t = np.asarray(z["target"][view_name], np.float32)
        raw_names = list(z["names"][view_name])
        fmm = _load_or_build_fmm(
            view_name, raw_h, raw_t, raw_names, hist, args, fold, out
        )
        z["hist"][view_name] = np.column_stack(
            [raw_h, fmm.hist_descriptor]
        ).astype(np.float32)
        z["target"][view_name] = np.column_stack(
            [raw_t, fmm.target_descriptor]
        ).astype(np.float32)
        z["names"][view_name] = raw_names + list(fmm.feature_names)

    return z


def _suite18_build_local(hist, target, args, fold, out):
    """Build Suite17 L unchanged, then append an L-specific FMM representation."""
    z = _ORIG_BUILD_LOCAL(hist, target, args, fold, out)
    raw_h = np.asarray(z["hist"], np.float32)
    raw_t = np.asarray(z["target"], np.float32)
    raw_names = list(z["names"])
    fmm = _load_or_build_fmm("L", raw_h, raw_t, raw_names, hist, args, fold, out)
    z["hist"] = np.column_stack([raw_h, fmm.hist_descriptor]).astype(np.float32)
    z["target"] = np.column_stack([raw_t, fmm.target_descriptor]).astype(np.float32)
    z["names"] = raw_names + list(fmm.feature_names)
    return z


def _rename_suite17_stage_keys(score_dict):
    for key in list(score_dict):
        if key.startswith("suite17_stage"):
            new = "suite18" + key[len("suite17"):]
            score_dict[new] = score_dict.pop(key)


def _load_suite17_reference(args, fold, faculty, y):
    root = getattr(args, "suite17_results_dir", None)
    if not root:
        return None
    p = Path(root) / f"fold_{int(fold)}" / f"{faculty}_scores.csv"
    if not p.exists():
        return None
    df = pd.read_csv(p, low_memory=False)
    if "suite17_hgb_final" not in df.columns:
        return None
    if len(df) != len(y):
        raise ValueError(f"Suite17 reference row mismatch: {p}")
    if "actual" in df.columns and not np.array_equal(
        df["actual"].to_numpy(int), np.asarray(y, int)
    ):
        raise ValueError(f"Suite17 reference target alignment mismatch: {p}")
    return df["suite17_hgb_final"].to_numpy(float)


def _write_fold_fmm_artifacts(args, out, fold):
    st = _state(fold)
    fd = Path(out) / f"fold_{int(fold)}"
    fd.mkdir(parents=True, exist_ok=True)

    fs, ba, hb, hs, vs = [], [], [], [], []
    for view, result in st["results"].items():
        fs += [dict(fold=int(fold), **r) for r in result.feature_selection_rows]
        ba += [dict(fold=int(fold), **r) for r in result.block_audit]
        hs += [dict(fold=int(fold), **r) for r in hyperbox_scale_summary(result)]
        if args.save_fmm_hyperboxes:
            hb += [dict(fold=int(fold), **r) for r in result.hyperbox_rows]
        raw_dim = st["raw_dims"][view]
        vs.append({
            "fold": int(fold),
            "view": view,
            "raw_dim": int(raw_dim),
            "fmm_dim": int(result.hist_descriptor.shape[1]),
            "augmented_dim": int(raw_dim + result.hist_descriptor.shape[1]),
            "selected_raw_features": (
                len(result.preprocessor.selected_indices_)
                if result.preprocessor is not None else 0
            ),
            "initialization_block": result.initialization_block,
            "historical_descriptor_nonzero_rate": float(
                np.mean(np.any(result.hist_descriptor != 0, axis=1))
            ),
            "target_descriptor_nonzero_rate": float(
                np.mean(np.any(result.target_descriptor != 0, axis=1))
            ),
        })
    pd.DataFrame(fs).to_csv(fd / "fmm_feature_selection.csv", index=False)
    pd.DataFrame(ba).to_csv(fd / "fmm_prequential_audit.csv", index=False)
    pd.DataFrame(hs).to_csv(fd / "fmm_hyperbox_summary.csv", index=False)
    pd.DataFrame(vs).to_csv(fd / "fmm_view_summary.csv", index=False)
    if args.save_fmm_hyperboxes:
        pd.DataFrame(hb).to_csv(fd / "fmm_hyperboxes.csv", index=False)

    paper = st.get("paper") or {}
    rows = []
    if paper.get("available"):
        yall = st["target_y"]
        facidx = st["target_faculty_idx"]
        facnames = st["target_faculty_names"]
        mapping = [
            (FMM_STANDALONE, paper["fmm_standalone"]),
            (FMM_CART, paper["fmm_cart"]),
            (FMM_RF, paper["fmm_rf"]),
        ]
        for fi, faculty in enumerate(facnames):
            m = facidx == fi
            yy = yall[m]
            for model, score in mapping:
                ss = np.asarray(score)[m]
                rows.append({
                    "fold": int(fold),
                    "faculty": faculty,
                    "model": model,
                    "rows": int(m.sum()),
                    "failures": int(yy.sum()),
                    "ap": s17.safe_ap(yy, ss),
                    "auc": s17.safe_auc(yy, ss),
                    "theta": paper.get("theta"),
                    "hyperbox_training_rows": paper.get("hyperbox_training_rows"),
                    "mean_confidence_factor": paper.get("mean_confidence_factor"),
                    "diagnostic_only": True,
                })
    pd.DataFrame(rows).to_csv(fd / "fmm_paper_hybrid_diagnostics.csv", index=False)


def run_fold(args, faculties, fold, out, raw_index):
    """Run the original Suite17 fold after installing Suite18 representation hooks."""
    st = _state(fold)
    result = s17.run_fold(args, faculties, fold, out, raw_index)
    records, metrics, audit, audit_fac, candidates, stages, corr, usage, align, summary = result

    # Rename inherited intermediate score labels.  Final/raw already use patched
    # constants, but Suite17's stage prefix is a literal string in its writer.
    for row in metrics:
        if str(row.get("model", "")).startswith("suite17_stage"):
            row["model"] = "suite18" + row["model"][len("suite17"):]

    paper = st.get("paper") or {}
    paper_arrays = ({
        FMM_STANDALONE: paper.get("fmm_standalone"),
        FMM_CART: paper.get("fmm_cart"),
        FMM_RF: paper.get("fmm_rf"),
    } if paper.get("available") else {})

    facidx = st.get("target_faculty_idx")
    facnames = st.get("target_faculty_names") or []
    for rec in records:
        _rename_suite17_stage_keys(rec["scores"])
        faculty = rec["faculty"]
        y = np.asarray(rec["y"], int)
        if faculty in facnames and facidx is not None:
            fi = facnames.index(faculty)
            mask = np.asarray(facidx) == fi
            for model, whole in paper_arrays.items():
                if whole is None:
                    continue
                score = np.asarray(whole, float)[mask]
                rec["scores"][model] = score
                metrics.append(s17.metric_row(fold, faculty, model, y, score, args.threshold_beta))

        ref = _load_suite17_reference(args, fold, faculty, y)
        if ref is not None:
            rec["scores"][SUITE17_REFERENCE] = ref
            metrics.append(
                s17.metric_row(fold, faculty, SUITE17_REFERENCE, y, ref, args.threshold_beta)
            )

        # Enrich the per-faculty score artifact written by Suite17.
        sf_path = Path(out) / f"fold_{int(fold)}" / f"{faculty}_scores.csv"
        if sf_path.exists():
            sf = pd.read_csv(sf_path, low_memory=False)
            ren = {
                c: "suite18" + c[len("suite17"):]
                for c in sf.columns if c.startswith("suite17_stage")
            }
            if ren:
                sf = sf.rename(columns=ren)
            for model in (FMM_STANDALONE, FMM_CART, FMM_RF, SUITE17_REFERENCE):
                if model in rec["scores"]:
                    sf[model] = rec["scores"][model]
            sf.to_csv(sf_path, index=False)

    _write_fold_fmm_artifacts(args, out, fold)
    return records, metrics, audit, audit_fac, candidates, stages, corr, usage, align, summary


def _concat_fold_csv(out, filename, output_name=None):
    parts = []
    for fd in sorted(Path(out).glob("fold_*")):
        p = fd / filename
        if p.exists():
            try:
                z = pd.read_csv(p)
                if len(z):
                    parts.append(z)
            except pd.errors.EmptyDataError:
                pass
    if not parts:
        return pd.DataFrame()
    df = pd.concat(parts, ignore_index=True)
    df.to_csv(Path(out) / (output_name or filename), index=False)
    return df


def _suite18_vs_suite17_final(out):
    p = Path(out) / "mean_fold_holdout.csv"
    if not p.exists():
        return pd.DataFrame()
    df = pd.read_csv(p)
    a = df[df.model == SUITE18_FINAL].copy()
    b = df[df.model == SUITE17_REFERENCE].copy()
    if a.empty or b.empty:
        return pd.DataFrame()
    cols = [
        "faculty", "mean_fold_ap", "mean_fold_auc",
        "mean_fold_precision_at_20", "mean_fold_recall_at_20",
        "mean_fold_f1_at_20", "mean_fold_f2_at_20",
    ]
    a = a[[c for c in cols if c in a]].rename(
        columns={c: f"suite18_{c}" for c in cols if c != "faculty"}
    )
    b = b[[c for c in cols if c in b]].rename(
        columns={c: f"suite17_{c}" for c in cols if c != "faculty"}
    )
    z = a.merge(b, on="faculty", how="inner")
    for metric in [
        "mean_fold_ap", "mean_fold_auc", "mean_fold_precision_at_20",
        "mean_fold_recall_at_20", "mean_fold_f1_at_20", "mean_fold_f2_at_20"
    ]:
        x, y = f"suite18_{metric}", f"suite17_{metric}"
        if x in z and y in z:
            z[f"delta_{metric}"] = z[x] - z[y]
    z.to_csv(Path(out) / "suite18_vs_suite17.csv", index=False)
    return z


def _suite18_vs_suite17_specialists(args, out):
    root = getattr(args, "suite17_results_dir", None)
    newp = Path(out) / "standalone_specialist_audit.csv"
    oldp = Path(root) / "standalone_specialist_audit.csv" if root else None
    if not root or not newp.exists() or not oldp.exists():
        return pd.DataFrame()
    new = pd.read_csv(newp)
    old = pd.read_csv(oldp)
    key = [c for c in ("fold", "specialist") if c in new.columns and c in old.columns]
    if len(key) != 2:
        return pd.DataFrame()
    keep = [
        "fold", "specialist", "meta_score", "meta_mean_delta_ap",
        "meta_median_delta_ap", "meta_recent_mean_delta_ap",
        "meta_win_rate", "target_macro_delta_ap", "target_wins",
    ]
    n = new[[c for c in keep if c in new]].rename(
        columns={c: f"suite18_{c}" for c in keep if c not in key}
    )
    o = old[[c for c in keep if c in old]].rename(
        columns={c: f"suite17_{c}" for c in keep if c not in key}
    )
    z = n.merge(o, on=key, how="inner")
    for metric in (
        "meta_score", "meta_mean_delta_ap", "meta_median_delta_ap",
        "meta_recent_mean_delta_ap", "meta_win_rate", "target_macro_delta_ap",
    ):
        a, b = f"suite18_{metric}", f"suite17_{metric}"
        if a in z and b in z:
            z[f"delta_{metric}"] = z[a] - z[b]
    z.to_csv(Path(out) / "suite18_vs_suite17_specialist_audit.csv", index=False)
    return z


def postprocess_extra(args, out):
    # Aggregate every FMM-specific artifact across folds.
    fsel = _concat_fold_csv(out, "fmm_feature_selection.csv")
    preq = _concat_fold_csv(out, "fmm_prequential_audit.csv")
    hbox = _concat_fold_csv(out, "fmm_hyperbox_summary.csv")
    view = _concat_fold_csv(out, "fmm_view_summary.csv")
    paper = _concat_fold_csv(out, "fmm_paper_hybrid_diagnostics.csv")

    # Suite17's postprocessor has two legacy filenames with "suite17" embedded.
    for old, new in [
        ("suite17_vs_council_faculty.csv", "suite18_vs_council_faculty.csv"),
        ("suite17_vs_council_summary.csv", "suite18_vs_council_summary.csv"),
    ]:
        src = Path(out) / old
        if src.exists():
            shutil.copy2(src, Path(out) / new)

    final_cmp = _suite18_vs_suite17_final(out)
    spec_cmp = _suite18_vs_suite17_specialists(args, out)

    if len(preq):
        pq = pd.DataFrame(prequential_summary_rows(preq))
        print("\nSUITE18 FMM PREQUENTIAL / LEAKAGE AUDIT")
        print("-" * 140)
        print(pq.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
        violations = int(pq["chronology_violations"].sum()) if "chronology_violations" in pq else 0
        print(f"[FMM CHRONOLOGY CHECK] violations={violations} (required: 0)")
        if violations:
            raise RuntimeError(f"FMM chronology audit detected {violations} violations")

    if len(fsel):
        fp = feature_selection_preview_rows(fsel, top_n=10)
        if len(fp):
            print("\nSUITE18 FMM SELECTED RAW FEATURES — TOP 10 PER VIEW/FOLD")
            print("-" * 160)
            print(fp.to_string(index=False, float_format=lambda x: f"{x:.4g}"))

    if len(hbox):
        hb = hyperbox_summary_for_log(hbox)
        if len(hb):
            print("\nSUITE18 FMM HYPERBOX GEOMETRY / CONFIDENCE AUDIT")
            print("-" * 180)
            print(hb.to_string(index=False, float_format=lambda x: f"{x:.4f}"))

    if len(view):
        print("\nSUITE18 FMM REPRESENTATION AUDIT")
        print("-" * 120)
        show = [c for c in [
            "fold", "view", "raw_dim", "fmm_dim", "augmented_dim",
            "selected_raw_features", "initialization_block",
            "historical_descriptor_nonzero_rate"
        ] if c in view]
        print(view[show].to_string(index=False, float_format=lambda x: f"{x:.4f}"))

    if len(final_cmp):
        print("\nSUITE18 vs SUITE17 — HOLDOUT MEAN-WITHIN-FOLD")
        print("-" * 120)
        show = [c for c in [
            "faculty", "suite17_mean_fold_ap", "suite18_mean_fold_ap",
            "delta_mean_fold_ap", "suite17_mean_fold_f2_at_20",
            "suite18_mean_fold_f2_at_20", "delta_mean_fold_f2_at_20"
        ] if c in final_cmp]
        print(final_cmp[show].to_string(index=False, float_format=lambda x: f"{x:.4f}"))

    mean_path = Path(out) / "mean_fold_holdout.csv"
    if mean_path.exists():
        meanf = pd.read_csv(mean_path)
        broad = reference_head_to_head(meanf, SUITE18_FINAL)
        if len(broad):
            broad.to_csv(Path(out) / "suite18_all_reference_head_to_head.csv", index=False)
            print("\nSUITE18 vs ALL AVAILABLE EARLIER VERSIONS — HOLDOUT FACULTY HEAD-TO-HEAD")
            print("-" * 180)
            show = [c for c in [
                "reference","faculties",
                "candidate_macro_ap","reference_macro_ap","mean_delta_ap","median_delta_ap","worst_delta_ap","ap_wins","ap_win_rate",
                "candidate_macro_auc","reference_macro_auc","mean_delta_auc","median_delta_auc","worst_delta_auc","auc_wins","auc_win_rate",
                "candidate_macro_f2","reference_macro_f2","mean_delta_f2","median_delta_f2","worst_delta_f2","f2_wins","f2_win_rate",
            ] if c in broad.columns]
            print(broad[show].to_string(index=False, float_format=lambda x: f"{x:+.4f}"))

    if len(spec_cmp):
        h = spec_cmp[spec_cmp.fold.isin(args.holdout_folds)].copy()
        if len(h):
            print("\nSUITE18 FMM SPECIALIST COMPLEMENTARITY vs SUITE17 — HOLDOUT")
            print("-" * 120)
            cols = [c for c in [
                "fold", "specialist", "delta_meta_mean_delta_ap",
                "delta_meta_recent_mean_delta_ap", "delta_target_macro_delta_ap"
            ] if c in h]
            print(h[cols].to_string(index=False, float_format=lambda x: f"{x:+.4f}"))

    if len(paper):
        print("\n[PAPER HYBRID DIAGNOSTIC — NEVER USED FOR SUITE18 SELECTION]")
        print("-" * 120)
        h = paper[paper.fold.isin(args.holdout_folds)] if "fold" in paper else paper
        if len(h):
            print(
                h.groupby("model")[["ap", "auc"]].mean()
                .reset_index().to_string(index=False, float_format=lambda x: f"{x:.4f}")
            )


def build_parser():
    p = s17.build_parser()
    p.description = (
        "Suite18: Suite17 robust forward specialist boosting + strict "
        "prequential multiscale fuzzy-min-max representations."
    )
    p.add_argument("--fmm_top_k", type=int, default=32)
    p.add_argument("--fmm_thetas", default="0.15,0.30,0.50")
    p.add_argument("--fmm_gamma", type=float, default=1.0)
    p.add_argument("--fmm_confidence_k", type=float, default=0.50)
    p.add_argument("--fmm_min_history_rows", type=int, default=5000)
    p.add_argument("--fmm_min_history_failures", type=int, default=50)
    p.add_argument("--fmm_chunk_size", type=int, default=1024)
    p.add_argument("--fmm_paper_theta", type=float, default=0.30)
    p.add_argument(
        "--fmm_cart_max_depth", type=int, default=0,
        help="0 means an unpruned CART diagnostic."
    )
    p.add_argument("--fmm_rf_trees", type=int, default=200)
    p.add_argument("--force_fmm", action="store_true")
    p.add_argument("--save_fmm_hyperboxes", action="store_true")
    p.add_argument(
        "--suite17_results_dir", default=None,
        help="Optional Suite17 result root for direct Suite18-vs-Suite17 comparisons."
    )
    return p


def _run_config(args, faculties):
    return {
        "experiment": "Suite18",
        "research_change": "Suite17 raw specialist views + chronology-safe multiscale FMM descriptors",
        "committee": "immutable Suite17 Committee",
        "specialist_learner": "unchanged Suite17 Newton-HGB",
        "forward_selection": "unchanged Suite17 robust chronology-safe selector",
        "faculties": list(faculties),
        "folds": list(args.folds),
        "dev_folds": list(args.dev_folds),
        "holdout_folds": list(args.holdout_folds),
        "fmm": asdict(_config(args)),
        "paper_hybrid": {
            "diagnostic_only": True,
            "theta": float(args.fmm_paper_theta),
            "cart_max_depth": (
                None if int(args.fmm_cart_max_depth) <= 0 else int(args.fmm_cart_max_depth)
            ),
            "rf_trees": int(args.fmm_rf_trees),
            "centroid_as_tree_sample": True,
            "confidence_factor_as_sample_weight": True,
        },
        "suite17_results_dir": args.suite17_results_dir,
    }


def _install_hooks():
    # FINAL/RAW constants are looked up dynamically by Suite17 run_fold/postprocess.
    s17.RAW = SUITE18_RAW
    s17.FINAL = SUITE18_FINAL
    s17.load_fold_bundle = _capture_load_fold_bundle
    s17.build_or_load_structured_views = _suite18_build_structured_views
    s17.build_or_load_local = _suite18_build_local
    # Display-only hooks: metrics remain untouched, but inherited report headers
    # and suite17_* display labels become truthful Suite18 labels.
    s17.print_table = _suite18_print_table
    s17.print_win_rates = _suite18_print_win_rates
    # Suite17 defines print calls in its own module globals, so replace that
    # module's print symbol only; Python's global builtins are not modified.
    s17.print = _suite18_module_print


def main(argv=None):
    args = build_parser().parse_args(argv)
    args.folds = s17.parse_ints(args.folds)
    args.dev_folds = s17.parse_ints(args.dev_folds)
    args.holdout_folds = s17.parse_ints(args.holdout_folds)
    args.eta_grid = s17.parse_floats(args.eta_grid)
    args.hgb_iter_candidates = s17.parse_iters(args.hgb_iter_candidates)
    args.fmm_thetas = tuple(float(x.strip()) for x in str(args.fmm_thetas).split(",") if x.strip())
    _config(args)  # validate early

    args.raw_index_dir = str(Path(args.structured_cache_dir) / "raw_index")
    out = Path(args.out_dir)
    out.mkdir(parents=True, exist_ok=True)
    available = s17.discover_faculties(args.cache_dir, args.folds[0])
    faculties = s17.parse_faculties(args.faculties, available)
    raw_dir = s17.ensure_structured_and_raw(args, faculties, args.folds)
    args.raw_index_dir = str(raw_dir)
    raw_index = s17.RawEvidenceIndex(raw_dir)
    s17.seed_all(args.seed)
    _install_hooks()

    (out / "suite18_run_config.json").write_text(
        json.dumps(_run_config(args, faculties), indent=2, default=str),
        encoding="utf-8",
    )

    print("=" * 180)
    print("SUITE18 — MULTISCALE FUZZY-MIN-MAX REPRESENTATION + ROBUST FORWARD SPECIALIST BOOSTING")
    print(
        "Immutable Committee | Suite17 R/M/L semantics unchanged | "
        "raw views preserved | FMM auxiliary descriptors only"
    )
    print(
        f"FMM top_k={args.fmm_top_k} theta={args.fmm_thetas} gamma={args.fmm_gamma:g} "
        f"CF_k={args.fmm_confidence_k:g} strict prequential query-before-update"
    )
    print(
        f"Suite17 eta_grid={args.eta_grid} deploy_stages={args.max_deploy_stages} "
        f"correction_bound={args.correction_bound}; CART/RF branch diagnostic-only"
    )
    print("=" * 180)

    records=[];metrics=[];audit=[];audit_fac=[];candidates=[];stages=[];corr=[];usage=[];align=[];fs=[]
    for fold in args.folds:
        a,b,c,d,e,f,g,h,i,j = run_fold(args, faculties, fold, out, raw_index)
        records += a; metrics += b; audit += c; audit_fac += d
        candidates += e; stages += f; corr += g; usage += h; align += i; fs += j
        _FOLD_STATE.pop(int(fold), None)
        gc.collect()

    # Reuse the exact Suite17 evaluator and operational policy machinery.
    s17.postprocess(
        args, out, faculties, records, metrics, audit, audit_fac,
        candidates, stages, corr, usage, align, fs
    )
    postprocess_extra(args, out)
    print(f"\n[DONE] Suite18 results: {out}")


if __name__ == "__main__":
    main()
