"""Permutation importance of the 217 features, inside each fold, for the top-K variants of F1 (FEATURE_EXPERIMENTS.md).

For each CV fold (and the public split) one `base` model (seed 0) is trained exactly like in run_augment.py, on the inner-training
people of the fold's training set; the importance of a column is the rise of the W-bMAE on the inner-validation people of that
fold when the column is shuffled across their windows (mean of 2 shuffles). Held-out recordings are never used.
Output: cache/f1_importance.json, {split key: {"train": fold training recordings, "order": columns by decreasing importance, ...}}.
"""
import json
import os
import sys
import time

import numpy as np
import torch

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import run_augment as A  # noqa: E402

H, R, log = A.H, A.R, A.log
REPEATS = 2


def va_score(model, store, va, spec):
    preds = H.predict_generic(model, store, va, spec)
    return float(np.mean([(store.w[store.idx[r]].cpu().numpy() * np.abs(np.clip(p, 0, 600) - store.y[store.idx[r]].cpu().numpy())).sum()
                          for r, p in zip(va, preds)]))


if __name__ == "__main__":
    t0 = time.time()
    X0, names, win, rec = R.extract()
    spec = A.make_specs(win, rec, ["base"], names)["base"]
    store = spec["make_store"]()
    widx = win.groupby("rec").indices
    folds = R.make_folds(rec, 5, seed=0)
    splits = {f"cv_f{k}": (k, list(fd["train"])) for k, fd in enumerate(folds)}
    splits["public"] = (0, list(rec.index[rec.split == "train"]))
    out = {}
    for key, (fold_id, train) in splits.items():
        tr, va = R.inner_split(train, rec, np.random.default_rng(1000 + 0 + 17 * fold_id))
        store.standardise(np.concatenate([widx[r] for r in train]))
        model, best, best_ep, _ = A.train_aug(spec, store, tr, va, 0, 25)
        base = va_score(model, store, va, spec)
        rows = torch.cat([store.idx[r] for r in va])
        rng = torch.Generator(device=store.f.device).manual_seed(0)
        imp = np.zeros(len(names))
        f0 = store.f.clone()
        for j in range(len(names)):
            for _ in range(REPEATS):
                perm = rows[torch.randperm(len(rows), generator=rng, device=rows.device)]
                store.f[rows, j] = f0[perm, j]
                imp[j] += (va_score(model, store, va, spec) - base) / REPEATS
                store.f[rows, j] = f0[rows, j]
        order = [int(i) for i in np.argsort(-imp)]
        out[key] = dict(train=[int(r) for r in train], order=order, importance=[float(x) for x in imp], base_val=base,
                        best_epoch=best_ep + 1, top20=[str(names[i]) for i in order[:20]])
        log(f"{key}: inner-val {base:.2f} (epoch {best_ep + 1}); top 10: {', '.join(str(names[i]) for i in order[:10])}; "
            f"{int((imp > 0).sum())} columns with positive importance ({time.time() - t0:.0f} s)")
        del model
        torch.cuda.empty_cache()
    os.makedirs(os.path.dirname(A.IMPORTANCE), exist_ok=True)
    json.dump(out, open(A.IMPORTANCE, "w"))
    log(f"wrote {A.IMPORTANCE} in {(time.time() - t0) / 60:.1f} min")
