"""One experiment = one decoder x one preprocessing, 5 LODO folds, then switch-detection scoring.

  python run_experiment.py --decoder wf --prep py
Fold outputs are cached in folds/<exp>/<held-out>.pkl (resume after a crash); final results in results/<exp>.json.
"""
import argparse
import json
import os
import sys
import time

import numpy as np
import torch

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from common_exp import CFG, EXP, EXTERNAL_DATA, GLIMPSE_THR, LAB, evidence_from_features, fold_file, load_trial, resolve_model, rows_of, save_pickle  # noqa: E402
from common import DS_KEYS  # noqa: E402
import decoders  # noqa: E402
from score import score_experiment  # noqa: E402


def log_to(path):
    fh = open(path, "a", buffering=1)

    def log(msg):
        line = f"{time.strftime('%Y-%m-%d %H:%M:%S')} {msg}"
        print(line, flush=True)
        fh.write(line + "\n")
    return log


EXTERNAL = EXTERNAL_DATA


def load_external(names, prep, log):
    """Extra TRAINING trials from converted external datasets (external_data/<name>/cache/<prep>/*.npz, same
    compliant pipeline). Never used as held-out data; the 5 LODO test datasets stay unchanged."""
    out = []
    for nm in names:
        d = os.path.join(EXTERNAL, nm, "cache", prep)
        fs_ = sorted(f for f in os.listdir(d) if f.endswith(".npz"))
        for f in fs_:
            z = np.load(os.path.join(d, f))
            ef = os.path.join(d, f[:-4] + "_eeg.npy")      # large sets (SparrKULee): float16 EEG, memory-mapped
            eeg = np.load(ef, mmap_mode="r") if os.path.exists(ef) else z["eeg"]
            tr = dict(eeg=eeg, env=z["env"], att=z["att"], t0=float(z["t0"]), key=f[:-4], _grp=nm)
            g = os.path.join(EXTERNAL, nm, "cache", f"glimpse{GLIMPSE_THR}", f)
            if os.path.exists(g):                        # glimpse targets (build_glimpse_fug.py)
                zg = np.load(g)
                tr.update({k: zg[k] for k in zg.files if k != "m"})
            out.append(tr)
        log(f"  extra training data {nm}: {len(fs_)} trials")
    return out


def init_net(dec, init_from, log):
    """S1: start from a pretrained network (e.g. SparrKULee); head columns are matched by target name."""
    init_from = resolve_model(init_from)
    st, cur = torch.load(init_from, map_location=decoders.DEV), dec.net.state_dict()
    hw = st.get("head.weight")
    tg, src = decoders.D.get("targets", ["env"]), decoders.D.get("init_targets", ["env", "onset"])
    if hw is not None and (hw.shape != cur["head.weight"].shape or tg[:hw.shape[1]] != src[:hw.shape[1]]):
        # the head columns are matched BY TARGET NAME (src = targets of the pretrained network): shared targets
        # keep their pretrained column, the others (e.g. dom: SparrKULee is single-talker) are freshly initialised
        assert hw.shape[0] == cur["head.weight"].shape[0] and hw.shape[1] == len(src), (hw.shape, src)
        new = cur["head.weight"].clone()
        kept = [f for f in tg if f in src]
        for f in kept:
            new[:, tg.index(f)] = hw[:, src.index(f)]
        st["head.weight"] = new
        log(f"  head: pretrained columns kept for {kept}, newly initialised for {[f for f in tg if f not in src]}")
    dec.net.load_state_dict(st)
    log(f"  initialised from {init_from}")


def run_fold(dec_name, prep, exp, held, log, max_epochs=None, warmup_epochs=None, save_recon=False, extra=(), train_frac=1.0,
             init_from=None):
    if max_epochs is not None:
        decoders.D["max_epochs"] = max_epochs
    if warmup_epochs is not None:
        decoders.D["warmup_epochs"] = warmup_epochs
    rng = np.random.default_rng(CFG["seed"] + DS_KEYS.index(held))
    train, val, meta = [], [], []
    is_dnn = dec_name in decoders.DNN
    for ds in DS_KEYS:
        if ds == held:
            continue
        rows = rows_of(ds)
        val_subj = rng.choice(sorted(rows.subject.unique())) if is_dnn else None   # early-stopping subject
        for _, r in rows.iterrows():
            tr = load_trial(prep, r["key"])
            if is_dnn and r["subject"] == val_subj:
                val.append(tr)
            else:
                train.append(tr)
                meta.append(r["subj_uid"] if dec_name == "wf_riemann" else (int(r["did"]), int(r["subject"])))
    if train_frac < 1.0:                              # learning curve: keep a random fraction of training SUBJECTS per dataset
        sub_rng = np.random.default_rng(CFG["seed"] + 1000 + DS_KEYS.index(held))
        keep = set()
        for did in sorted({m[0] for m in meta}):
            subs = sorted({m for m in meta if m[0] == did})
            k = max(1, int(round(train_frac * len(subs))))
            keep |= set(map(tuple, np.array(subs)[sub_rng.permutation(len(subs))[:k]].tolist()))
        idx = [i for i, m in enumerate(meta) if tuple(m) in keep]
        train, meta = [train[i] for i in idx], [meta[i] for i in idx]
        log(f"  train_frac {train_frac}: kept {len(keep)} subjects, {len(train)} trials")
    if extra:
        ext = load_external(extra, prep, log)
        train += ext
        meta += [(-1, -1)] * len(ext)
    C = train[0]["eeg"].shape[1]
    log(f"  fold held-out={held}: {len(train)} train trials, {len(val)} val trials, {C} channels")
    t0 = time.time()
    dec = decoders.make(dec_name, C, log)
    if init_from:
        init_net(dec, init_from, log)
    best = dec.fit(train, val, meta)
    crf = dec.fit_sequence(train, val) if is_dnn and decoders.D.get("seq_loss") else None    # A1 end-to-end phase
    t_fit = time.time() - t0
    del train, val
    modes = ["causal", "offline"] if is_dnn else ["causal"]
    evidence = {m: {} for m in modes}
    recon = {}
    for _, r in rows_of(held).iterrows():
        tr = load_trial(prep, r["key"])
        for m in modes:
            A, B0, B1 = dec.features(tr, m) if is_dnn else dec.features(tr)
            evidence[m][r["key"]] = evidence_from_features(A, B0, B1, tr["att"], tr["t0"])
            if save_recon and m == "causal":
                recon[r["key"]] = np.asarray(A, np.float32)          # causal reconstruction at 32 Hz (for msm.py)
    info = dict(decoder=dec_name, prep=prep, held_out=held, fit_s=t_fit, total_s=time.time() - t0,
                lookahead_s=dec.lookahead_s, offline_lookahead_s=getattr(dec, "offline_lookahead_s", None),
                best_val=best, channels=C, **({"crf": crf} if crf else {}))
    save_pickle(dict(evidence=evidence, info=info, **({"recon": recon} if save_recon else {})), fold_file(exp, held))
    e = evidence["causal"]
    d = np.concatenate([(v["r0"] - v["r1"])[::4] * np.where(v["att"][::4] == 0, 1, -1) for v in e.values()])
    log(f"  fold held-out={held} done in {info['total_s']:.0f}s: 1-s acc {np.mean(d > 0):.3f}, pooled SNR1 {d.mean() / d.std():.3f}")
    if is_dnn:
        del dec
        torch.cuda.empty_cache()


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--decoder", required=True)
    ap.add_argument("--prep", required=True, choices=list(CFG["preps"]))
    ap.add_argument("--tag", default="")
    ap.add_argument("--max-epochs", type=int, default=None)
    ap.add_argument("--window-s", type=float, default=None, help="DNN input window in s (default config dnn.window_s)")
    ap.add_argument("--model-override", default=None, help='JSON dict of model_args overriding the package yaml, e.g. {"use_skip": true}')
    ap.add_argument("--train-seed", type=int, default=None, help="DNN: seed for init + window sampling (step 0)")
    ap.add_argument("--save-recon", action="store_true", help="also store the causal reconstructions (for msm.py)")
    ap.add_argument("--train-frac", type=float, default=1.0, help="learning curve: fraction of training subjects kept per dataset")
    ap.add_argument("--extra-train", default="", help="comma list of external datasets added to TRAINING only (external_data/<name>)")
    ap.add_argument("--seq-loss", default=None, choices=["filter", "crf"],
                    help="A1: after the normal fit, fine-tune decoder + CRF unary weights END-TO-END on synthetic-switch "
                         "sequences (config dnn_seq): 'filter' = causal filtering cross-entropy, 'crf' = path NLL")
    ap.add_argument("--seq-v-norm", action="store_true", help="A1: learn only the MIX of the targets (softmax), keep the "
                    "total LLR scale of the scoring HMM")
    ap.add_argument("--evidence-weights", default="equal", choices=["equal", "lda", "crf"],
                    help="how the targets' correlations are combined into the evidence: equal mean (default) or LDA "
                         "weights fitted on the other four datasets' out-of-fold evidence (score.lda_weights)")
    ap.add_argument("--extra-frac", default="", help="comma list NAME=FRAC: sampling share of an extra dataset's windows (S2)")
    ap.add_argument("--init-targets", default="env,onset", help="targets (head columns) of the --init-from network")
    ap.add_argument("--init-from", default=None, help="DNN: state_dict to initialise the network from (S1 fine-tuning)")
    ap.add_argument("--loss", default=None, choices=["contrastive", "infonce", "pearson"], help="DNN training loss (val stays contrastive)")
    ap.add_argument("--targets", default=None, help="DNN targets, comma list: env (default), onset")
    ap.add_argument("--set", nargs="*", default=[], help="hyper-parameter overrides of config dnn: key=value ...")
    ap.add_argument("--aug-set", nargs="*", default=[], help="overrides of the augmentation settings: key=value ...")
    ap.add_argument("--adv", default=None, choices=["dataset", "subject"], help="DNN: adversarial loss (gradient reversal) "
                    "that removes dataset / subject information from the backbone features")
    ap.add_argument("--adv-lambda", type=float, default=0.05)
    ap.add_argument("--augment", action="store_true", help="DNN: training augmentation with config dnn_augment")
    ap.add_argument("--aug-variant", default=None, help="DNN: dnn_augment + config aug_variants[NAME] (implies --augment)")
    ap.add_argument("--warmup-epochs", type=int, default=None, help="DNN: LR warm-up epochs without early stopping")
    args = ap.parse_args()
    exp = f"{args.decoder}__{args.prep}{args.tag}"
    if args.window_s is not None:
        decoders.D["window_s"] = args.window_s
    if args.loss:
        decoders.D["loss"] = args.loss
    if args.targets:
        decoders.D["targets"] = args.targets.split(",")
    if args.extra_frac:
        decoders.D["group_frac"] = {k: float(v) for k, v in (x.split("=") for x in args.extra_frac.split(","))}
    decoders.D["init_targets"] = args.init_targets.split(",")
    if args.seq_loss:
        decoders.D.update(CFG["dnn_seq"], seq_loss=args.seq_loss, seq_v_norm=args.seq_v_norm)
    if args.train_seed is not None:
        decoders.D["train_seed"] = args.train_seed
    if args.model_override:
        decoders.MODEL_OVERRIDES[args.decoder] = json.loads(args.model_override)
    if args.adv:
        decoders.D.update(adv=args.adv, adv_lambda=args.adv_lambda)
    if args.augment or args.aug_variant:
        decoders.D["augment"] = {**CFG["dnn_augment"], **(CFG["aug_variants"][args.aug_variant] if args.aug_variant else {})}
    decoders.apply_overrides(args.set, args.aug_set)
    log_path = os.path.join(EXP, "logs", f"{exp}.log")
    if not any(os.path.exists(fold_file(exp, h)) for h in DS_KEYS):
        # fresh start: drop the logs of an earlier aborted attempt (the queue appends stdout, so truncate fd 1 too)
        open(log_path, "w").close()
        if not sys.stdout.isatty():
            try:
                os.ftruncate(sys.stdout.fileno(), 0)
            except OSError:
                pass
    log = log_to(log_path)
    torch.set_num_threads(4)
    log(f"=== experiment {exp} ===")
    for held in DS_KEYS:
        if os.path.exists(fold_file(exp, held)):
            log(f"  fold held-out={held}: cached")
            continue
        run_fold(args.decoder, args.prep, exp, held, log, args.max_epochs, args.warmup_epochs, args.save_recon, [x for x in args.extra_train.split(',') if x], args.train_frac,
                 args.init_from)
    lookahead = decoders.LINEAR[args.decoder].lookahead_s if args.decoder in decoders.LINEAR else \
        CFG["dnn"]["evidence_lookahead_s"]
    out = dict(experiment=exp, decoder=args.decoder, prep=args.prep, warmup_epochs=decoders.D.get("warmup_epochs", 0), window_s=decoders.D["window_s"], augment=decoders.D.get("augment"), model_override=decoders.MODEL_OVERRIDES.get(args.decoder), train_seed=decoders.D.get("train_seed", CFG["seed"]), targets=decoders.D.get("targets", ["env"]), loss=decoders.D.get("loss", "contrastive"), extra_train=args.extra_train, extra_frac=args.extra_frac, init_from=args.init_from, train_frac=args.train_frac, prep_note=CFG["preps"][args.prep]["note"],
               switch=f"HMM p_stay={CFG['switch']['p_stay']} per 0.25-s step, SYNTHETIC switches, LODO",
               evidence_weights=args.evidence_weights, seq_loss=args.seq_loss, lookahead_s=lookahead,
               causal=score_experiment(exp, "causal", lookahead, args.evidence_weights))
    if args.decoder in decoders.DNN:
        W = decoders.D["window_s"]
        out["offline_reference_NONCOMPLIANT"] = score_experiment(exp, "offline", W - W / 4, args.evidence_weights)
    for k in ("causal",):
        m = out[k]["mean"]
        log(f"  MEAN over held-out datasets: Acc {m['Acc']:.2f}  SDT {m['SDT']:.2f}  L {m['L']:.3f}  H {m['H']:.3f}  "
            f"(acc1 {m['acc1']:.3f}, acc10 {m['acc10']:.3f}, SNR1 {m['snr1']:.3f})")
    with open(os.path.join(EXP, "results", f"{exp}.json"), "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    log(f"=== done {exp} ===")


if __name__ == "__main__":
    main()
