"""E4: handling the start of a recording, on the best recipe (low-pass 45 Hz input, 217 features, `dist_w` head).

Why: 67.8% of the recordings start with a ~0 sample (switch-on transient), 41% of the channel-windows are bad in the first 30 s, the
first minute of the public test recordings is cleaner than in train (p = 0.0014), and a recurrent state can count the windows since
the start, which correlates with the target in the public data (recordings are cut at about 25 minutes).

Variants (all trained with the protocol of the earlier `dist_w` runs: 85% of the people, 25-epoch schedule, epoch chosen on the other
15%; validation always on the full, unmasked recordings with the true metric weights):
  base     no special handling (re-run in the same session as a control)
  mask     (1) the first second (128 samples) of every recording is set to zero BEFORE the low-pass filter, features and CNN input
               (causal: it is the first second of the stream, identical at inference);
           (2) the first window of a recording gets loss weight 0 (it is still predicted at evaluation, where it counts)
  rstart   random start: in each epoch each training recording is, with probability 0.5, cropped to start at a random window in
           [0, 36] (0-3 minutes) so the GRU starts mid-recording; the metric weights are recomputed on the cropped targets
  both     mask + rstart
Evaluation is the usual one on full recordings, plus two probes of the start: the MAE over the first minute (12 windows) of every
held-out recording, and the crop-start probe (the first 0-50% of each held-out recording removed, context restarted).
Reference: `dist_w` of experiments/dist_head (seeds 0-2) and, for the public split, single `val25` models of experiments/ensemble.
"""
import argparse
import os
import sys
import time

import mne
import numpy as np
import torch
from torch import nn

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(os.path.dirname(HERE))
for p in ("experiments", "experiments/iir_filter", "experiments/dist_head", "experiments/ensemble"):
    sys.path.insert(0, os.path.join(ROOT, p))
import cvharness as H  # noqa: E402
import run_dist_head as D  # noqa: E402
import run_iir_filter as I  # noqa: E402
from ens_utils import median_decode  # noqa: E402

C, R, log = H.C, H.R, H.log
IIR = os.path.join(ROOT, "experiments/iir_filter")
CACHE = os.path.join(HERE, "cache")
WIN = R.WIN
ZERO_SAMPLES = 128                      # 1 s
VARIANTS = {"base": dict(mask=0, zero=False, p_crop=0.0), "mask": dict(mask=1, zero=True, p_crop=0.0),
            "rstart": dict(mask=0, zero=False, p_crop=0.5), "both": dict(mask=1, zero=True, p_crop=0.5)}
MAX_CROP = 36                           # windows = 3 minutes


def build_masked(names, win, rec, X0):
    paths = (f"{CACHE}/mask_X.npy", f"{CACHE}/mask_raw_f16.npy")
    if all(os.path.exists(p) for p in paths):
        return
    os.makedirs(CACHE, exist_ok=True)
    art = [i for i, n in enumerate(names) if n.startswith(("clip_frac_", "log_maxabs_")) or n == "n_bad"]
    sos = I.design(None)
    counts = win.groupby("rec").size(); n_all = len(win)
    Xs = np.empty((n_all, X0.shape[1]), np.float32); raws = np.empty((n_all, 4, WIN), np.float16)
    pos, t0 = 0, time.time()
    for i, r in rec.iterrows():
        vhdr = f"{R.BIDS}/{r.subject}/{r.session_id}/eeg/{r.rec_id}_task-sleeponset_eeg.vhdr"
        x = mne.io.read_raw_brainvision(vhdr, preload=True, verbose="ERROR").get_data(picks=R.CH) * 1e6
        n = int(counts[i]); x = x[:, : n * WIN].copy()
        x[:, :ZERO_SAMPLES] = 0.0                                          # first second of the recording
        xf, _ = I.filter_stream(sos, x)
        xw = xf.reshape(4, n, WIN).transpose(1, 0, 2)
        Xs[pos:pos + n], _ = R.window_features(xw)
        raws[pos:pos + n] = xw.astype(np.float16)
        pos += n
    assert pos == n_all
    Xs[:, art] = X0[:, art]                                                # artifact features from the untouched raw signal
    np.save(paths[0], Xs); np.save(paths[1], raws)
    log(f"E4 masked-start caches built in {time.time() - t0:.0f} s")


def weights_for(y):
    return R.metric_weights(y, np.zeros(len(y), dtype=np.int64))


def train_e4(spec, store, tr, va, seed, epochs, lr=2e-3, batch_recs=16):
    """Same as cvharness.train_generic (validation on full unmasked recordings), with a loss mask on the first windows and random starts."""
    cfg = spec["e4"]
    torch.manual_seed(seed)
    rng = np.random.default_rng(seed)
    model = spec["make_model"](store, tr).to(H.DEV)
    steps = epochs * int(np.ceil(len(tr) / batch_recs))
    opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-2)
    sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=0.1)
    best, best_ep, best_state, hist = 1e9, -1, None, []
    va_w = [store.w[store.idx[r]].cpu().numpy() for r in va]
    va_y = [store.y[store.idx[r]].cpu().numpy() for r in va]
    for ep in range(epochs):
        model.train()
        order = rng.permutation(len(tr))
        tl = 0.0
        for i in range(0, len(order), batch_recs):
            recs = [tr[j] for j in order[i:i + batch_recs]]
            crop = {}
            for r in recs:
                n = len(store.idx[r])
                if cfg["p_crop"] > 0 and rng.random() < cfg["p_crop"]:
                    crop[r] = int(rng.integers(0, min(MAX_CROP, n - 12) + 1))
            xr, f, y, w, idxs = store.batch(recs, True, True, crop)
            if crop or cfg["mask"]:
                w = torch.zeros_like(w)
                for k, r in enumerate(recs):
                    n_k = len(idxs[k])
                    wk = weights_for(y[k, :n_k].cpu().numpy())               # weights of the (possibly cropped) recording
                    if cfg["mask"] and crop.get(r, 0) == 0:
                        wk[:cfg["mask"]] = 0.0                               # first window of the true start: not trained on
                    w[k, :n_k] = torch.from_numpy(wk.astype(np.float32)).to(w.device)
            out, _ = model(xr, f)
            loss = spec["loss"](out, y, w, model)
            opt.zero_grad(set_to_none=True)
            (loss * spec.get("loss_scale", 1.0)).backward()
            nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            opt.step(); sched.step()
            tl += loss.item() * len(recs)
        preds = H.predict_generic(model, store, va, spec)
        v = float(np.mean([(w_ * np.abs(np.clip(p, 0, 600) - y_)).sum() for w_, y_, p in zip(va_w, va_y, preds)]))
        hist.append((tl / len(tr), v))
        if v < best:
            best, best_ep = v, ep
            best_state = {k: t.detach().clone() for k, t in model.state_dict().items()}
    model.load_state_dict(best_state)
    return model, best, best_ep, hist


def make_specs(win, variants):
    y = win.y.to_numpy(); w = R.metric_weights(y, win.rec.to_numpy())
    X = np.load(f"{IIR}/cache/lp45_X.npy")
    base = D.build_specs(win, X, None)["dist_w"]
    specs = {}
    for v in variants:
        cfg = VARIANTS[v]
        if cfg["zero"]:
            Fx = np.load(f"{CACHE}/mask_X.npy"); raw = f"{CACHE}/mask_raw_f16.npy"
        else:
            Fx = X; raw = f"{IIR}/cache/lp45_raw_f16.npy"
        specs[v] = dict(base, make_store=lambda raw=raw, Fx=Fx: C.Store(np.load(raw), Fx.astype(np.float64), y, w, win),
                        e4=cfg, train=train_e4)
    return specs


def public_reference(win, rec, seeds):
    y = win.y.to_numpy(); wrec = win.rec.to_numpy(); idx = win.groupby("rec").indices
    ite = np.concatenate([idx[r] for r in rec.index[rec.split == "test"]])
    out = []
    for s in seeds:
        p = np.load(os.path.join(ROOT, f"experiments/ensemble/preds/val25_public_f0_s{s}_test40.npy")).astype(np.float32)
        out.append(float(np.mean(list(R.score(y[ite], median_decode(p), wrec[ite])[0].values()))))
    return out


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--variants", nargs="+", default=list(VARIANTS))
    ap.add_argument("--seeds", type=int, nargs="+", default=[0, 1, 2])
    ap.add_argument("--public-seeds", type=int, nargs="+", default=[42, 43, 44, 45, 46])
    ap.add_argument("--epochs", type=int, default=25)
    ap.add_argument("--smoke", action="store_true")
    args = ap.parse_args()
    if args.smoke:
        args.seeds, args.public_seeds, args.epochs = [0], [42], 2
    res = os.path.join(HERE, "results_smoke" if args.smoke else "results"); os.makedirs(res, exist_ok=True)
    t0 = time.time()
    X0, names, win, rec = R.extract()
    build_masked(list(names), win, rec, X0)
    specs = make_specs(win, args.variants)
    a = argparse.Namespace(folds=2 if args.smoke else 5, seeds=args.seeds, epochs=args.epochs, smoke=args.smoke,
                           skip_cv=False, public_seeds=[])
    fold, per = H.run_cv(specs, X0, win, rec, a, res)
    H.summarize(res, fold, per, a, "E4: xử lý phần đầu bản ghi trên `cnn_feat_gru` (low-pass 45 Hz, head `dist_w`)",
                "`base` chạy lại cùng phiên làm đối chứng. `mask`: đặt 1 s đầu bản ghi bằng 0 trước bộ lọc và cho cửa sổ đầu trọng số loss 0; "
                "`rstart`: mỗi epoch, xác suất 0.5 cắt đầu bản ghi huấn luyện ngẫu nhiên 0-3 phút; `both`: cả hai. "
                "Mốc cũ `dist_w` là lần chạy trước của cùng cấu hình.",
                [("dist_w (cũ)", os.path.join(ROOT, "experiments/dist_head/results"), "dist_w")])
    b = argparse.Namespace(folds=5, seeds=args.public_seeds, epochs=args.epochs, smoke=args.smoke, skip_cv=True, public_seeds=None)
    res42 = os.path.join(res, "public"); os.makedirs(res42, exist_ok=True)
    fold42, _ = H.run_cv(specs, X0, win, rec, b, res42)
    fold42.to_csv(os.path.join(res42, "scores_by_fold.csv"), index=False)
    L = ["", "## So cặp với `base` (CV, trung bình qua seed, trên từng bản ghi), ± SE", ""]
    pr = per[per.split == "cv"].pivot_table(index=["fold", "group", "rec"], columns="variant", values="wbmae", aggfunc="mean")
    pc = per[per.split == "cv"].pivot_table(index=["fold", "group", "rec"], columns="variant", values="wbmae_cropstart", aggfunc="mean")
    if "base" in pr:
        for v in [c for c in pr.columns if c != "base"]:
            d = (pr[v] - pr["base"]).dropna(); dc = (pc[v] - pc["base"]).dropna()
            L.append(f"- {v} − base: {d.mean():+.2f} ± {d.std() / np.sqrt(len(d)):.2f} s (tốt hơn ở {100 * (d < 0).mean():.0f}% bản ghi-nhóm); "
                     f"khi cắt đầu bản ghi: {dc.mean():+.2f} ± {dc.std() / np.sqrt(len(dc)):.2f} s")
    cv = fold[fold.split == "cv"]
    L += ["", "## Phút đầu của bản ghi giữ ngoài (CV): MAE trên 12 cửa sổ đầu, gộp mọi cửa sổ", "",
          "| Biến thể | MAE phút đầu (s) | W-bMAE | W-bMAE khi cắt đầu bản ghi |", "|---|---|---|---|"]
    for v in args.variants:
        g = cv[cv.variant == v]
        L.append(f"| {v} | {g.mae_first60s.mean():.1f} | {g.pivot_table(index=['fold','seed'], columns='group', values='wbmae').mean(1).mean():.2f} | "
                 f"{g.pivot_table(index=['fold','seed'], columns='group', values='wbmae_cropstart').mean(1).mean():.2f} |")
    ref = public_reference(win, rec, args.public_seeds if not args.smoke else [42])
    L += ["", f"## Test công khai: seed {args.public_seeds[0]} và các seed kế tiếp (train 500 bản ghi, chấm 40 bản ghi; SE khoảng 4 s)", "",
          "| Biến thể | " + " | ".join(f"seed {s}" for s in args.public_seeds) + " | Trung bình | **Seed 42** | Khi cắt đầu bản ghi (TB) | MAE phút đầu (TB) |",
          "|---|" + "---|" * (len(args.public_seeds) + 4),
          "| `val25` đơn (mốc cũ) | " + " | ".join(f"{x:.1f}" for x in ref) + f" | {np.mean(ref):.1f} | **{ref[0]:.1f}** | | |"]
    for v in args.variants:
        g = fold42[fold42.variant == v].sort_values("seed")
        L.append(f"| {v} | " + " | ".join(f"{x:.1f}" for x in g.wbmae) + f" | {g.wbmae.mean():.1f} | **{g.wbmae.iloc[0]:.1f}** | "
                 f"{g.wbmae_cropstart.mean():.1f} | {g.mae_first60s.mean():.1f} |")
    open(os.path.join(res, "summary.md"), "a").write("\n".join(L) + "\n")
    print("\n".join(L))
    log(f"done in {(time.time() - t0) / 60:.1f} min")
