"""Deterministic masked denoising and chronological downstream training.

All preprocessing and vocabulary construction belong to the caller. Validation
only selects an epoch budget; fixed_epochs always builds a fresh training run.
"""

import copy
import hashlib
import random
from dataclasses import dataclass
from typing import Any, Dict, List

import numpy as np
import torch
from torch import nn
from torch.nn import functional as F
from sklearn.metrics import average_precision_score

from .models import HistoryEncoder, MaskedEventModel, NeuralRiskModel
from .data import IndexedHistory,history_rows


@dataclass
class SSLResult:
    encoder: HistoryEncoder
    epochs: int
    history: List[Dict[str, Any]]
    diagnostics: Dict[str, Any]
    model: MaskedEventModel


@dataclass
class NeuralResult:
    model: NeuralRiskModel
    epochs: int
    history: List[Dict[str, Any]]


def _seed(seed):
    random.seed(seed)
    np.random.seed(seed % (2 ** 32))
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False


def _tensors(batch):
    if batch is None:
        return None
    if isinstance(batch,IndexedHistory):
        return IndexedHistory(_tensors(batch.unique),batch.row_index)
    output = {k: (v.detach().cpu() if torch.is_tensor(v) else torch.as_tensor(v)) for k, v in batch.items()}
    needed = {"course", "grade_band", "numeric", "event_mask", "semester_mask", "gaps"}
    if not needed.issubset(output):
        raise ValueError("history batch missing: " + ", ".join(sorted(needed - output.keys())))
    shape = output["course"].shape
    if len(shape) != 3:
        raise ValueError("course must have shape [N,S,C]")
    for key in ("grade_band", "event_mask"):
        if output[key].shape != shape:
            raise ValueError(key + " shape does not match course")
    if output["numeric"].shape != (*shape, 3):
        raise ValueError("numeric must have shape [N,S,C,3]")
    for key in ("semester_mask", "gaps"):
        if output[key].shape != shape[:2]:
            raise ValueError(key + " must have shape [N,S]")
    return output


def _take(batch, indices, device):
    if batch is None:
        return None
    if isinstance(batch,IndexedHistory):
        return {k:torch.as_tensor(v).to(device) for k,v in batch.take(indices).items()}
    return {k: v.index_select(0, indices).to(device) for k, v in batch.items()}


def _observed(batch):
    return batch["event_mask"].bool() & batch["semester_mask"].bool().unsqueeze(-1)


def _optimizer_step(loss, model, optimizer, clip, context):
    """Fail before updating weights if either loss or gradient norm is invalid."""
    if not torch.isfinite(loss).all():
        raise FloatingPointError(f"non-finite {context} loss")
    optimizer.zero_grad(set_to_none=True)
    loss.backward()
    try:
        nn.utils.clip_grad_norm_(model.parameters(), float(clip), error_if_nonfinite=True)
    except RuntimeError as error:
        if "non-finite" in str(error):
            raise FloatingPointError(f"non-finite {context} gradient norm") from error
        raise
    optimizer.step()


def corrupt_batch(batch, task, rate=.2, generator=None, masks=None):
    """Clone and corrupt only known, observed grade/course targets.

    Masks are sampled independently. Each eligible minibatch gets at least one
    selected token per active task, including at a zero rate. Explicit masks
    allow validation corruption to remain identical across epochs and batches.
    Course corruption hides credits and attempt as well as course identity.
    """
    if task not in {"grade", "course", "joint"}:
        raise ValueError("task must be grade, course, or joint")
    if not 0 <= rate <= 1:
        raise ValueError("mask rate must be between zero and one")
    observed = _observed(batch)
    grade_eligible = observed & (batch["grade_band"] >= 3) & (batch["grade_band"] <= 7)
    course_eligible = observed & (batch["course"] >= 3)

    def sample(eligible, active):
        if not active:
            return torch.zeros_like(eligible)
        chosen = (torch.rand(eligible.shape, generator=generator, device="cpu").to(eligible.device) < rate) & eligible
        if eligible.any() and not chosen.any():
            locations = eligible.flatten().nonzero(as_tuple=False).flatten()
            pick = int(torch.randint(len(locations), (1,), generator=generator).item())
            chosen.flatten()[locations[pick]] = True
        return chosen

    if masks is None:
        grade_mask = sample(grade_eligible, task in {"grade", "joint"})
        course_mask = sample(course_eligible, task in {"course", "joint"})
    else:
        grade_mask = masks[0].to(observed.device).bool() & grade_eligible
        course_mask = masks[1].to(observed.device).bool() & course_eligible
        if task == "grade":
            course_mask = torch.zeros_like(course_mask)
        if task == "course":
            grade_mask = torch.zeros_like(grade_mask)
    corrupted = {k: v.clone() for k, v in batch.items()}
    corrupted["grade_band"][grade_mask] = 2
    corrupted["numeric"][..., 0][grade_mask] = 0.
    corrupted["course"][course_mask] = 2
    corrupted["numeric"][..., 1:][course_mask] = 0.
    return corrupted, grade_mask, course_mask


def _ssl_batch_size(cfg):
    return max(1, int(cfg.get("ssl_batch_size", cfg.get("batch_size", 64))))


def _fixed_masks(batch, task, cfg, seed):
    """Fix corruption once and force a target in each eligible eval minibatch."""
    generator = torch.Generator().manual_seed(int(seed) + 104729)
    gm = torch.zeros_like(batch["event_mask"], dtype=torch.bool)
    cm = torch.zeros_like(gm)
    width = _ssl_batch_size(cfg)
    for start in range(0, len(gm), width):
        stop = min(start + width, len(gm))
        subset = {k: v[start:stop] for k, v in batch.items()}
        _, gm[start:stop], cm[start:stop] = corrupt_batch(subset, task, float(cfg.get("mask_rate", .2)), generator)
    return gm, cm


def _target_eligibility(batch):
    observed = _observed(batch)
    total = int(observed.sum())
    unknown_course = int((observed & (batch["course"] == 1)).sum())
    return {"observed_events": total,
            "grade_targets": int((observed & (batch["grade_band"] >= 3) & (batch["grade_band"] <= 7)).sum()),
            "course_targets": int((observed & (batch["course"] >= 3)).sum()),
            "unknown_course_events": unknown_course,
            "unknown_course_fraction": unknown_course / total if total else None,
            "unknown_grade_events": int((observed & (batch["grade_band"] == 1)).sum()),
            "course_exclusion_policy": "UNK/PAD/MASK course identities are excluded from course reconstruction targets"}


def _require_active_ssl_targets(eligibility, task, split):
    for name in (("grade", "course") if task == "joint" else (task,)):
        if eligibility[name + "_targets"] == 0:
            target = "known-course" if name == "course" else "observed grade"
            raise ValueError(
                f"SSL {split} has no eligible {target} reconstruction targets for active task '{name}'. "
                "Provide more historical overlap/observed grades or configure an explicit alternate split. "
                "UNK/PAD/MASK tokens are excluded from reconstruction targets; an absent task cannot be scored as zero loss."
            )


def _frequency_baselines(batch, vocab_size):
    observed = _observed(batch)
    grade_valid = observed & (batch["grade_band"] >= 3) & (batch["grade_band"] <= 7)
    course_valid = observed & (batch["course"] >= 3)
    course_values = batch["course"][course_valid].long()
    if course_values.numel() and int(course_values.max()) >= vocab_size:
        raise ValueError("training course index exceeds supplied vocabulary")
    grade_counts = torch.bincount((batch["grade_band"][grade_valid] - 3).long(), minlength=5).numpy()
    course_counts = torch.bincount(course_values - 3, minlength=max(vocab_size - 3, 1)).numpy()
    # Laplace smoothing is fitted only on training counts; zero-shot validation
    # classes remain explicitly visible as zero in the unsmoothed counts.
    grade_prob = (grade_counts + 1.) / (grade_counts.sum() + len(grade_counts))
    course_prob = (course_counts + 1.) / (course_counts.sum() + len(course_counts))
    return {
        "grade_counts": grade_counts.tolist(), "course_counts": course_counts.tolist(),
        "grade_probabilities": grade_prob.tolist(), "course_probabilities": course_prob.tolist(),
        "grade_majority_class": int(grade_counts.argmax()),
        "course_majority_class": int(course_counts.argmax()),
        "grade_training_tokens": int(grade_counts.sum()), "course_training_tokens": int(course_counts.sum()),
        "smoothing": "Laplace +1, training counts only",
    }


def _task_summary(targets, predictions, losses, baselines, name, history_lengths):
    if not targets:
        return {"tokens": 0, "loss": None, "accuracy": None, "reason": "no observed masked targets"}
    y = np.concatenate(targets).astype(int)
    p = np.concatenate(predictions).astype(int)
    losses = np.concatenate(losses)
    length = np.concatenate(history_lengths)
    counts = np.array(baselines[name + "_counts"])
    probabilities = np.array(baselines[name + "_probabilities"])
    frequency = counts[y]
    majority = baselines[name + "_majority_class"]

    def subset(mask):
        return {"tokens": int(mask.sum()), "accuracy": float((y[mask] == p[mask]).mean()) if mask.any() else None,
                "loss": float(losses[mask].mean()) if mask.any() else None}

    result = {
        "tokens": int(len(y)), "loss": float(losses.mean()), "accuracy": float((y == p).mean()),
        "frequency_baseline_loss": float(-np.log(probabilities[y]).mean()),
        "frequency_baseline_accuracy": float((y == probabilities.argmax()).mean()),
        "majority_baseline_accuracy": float((y == majority).mean()),
        "history_strata": {"1_4_events": subset(length <= 4), "5_16_events": subset((length > 4) & (length <= 16)),
                           "17_plus_events": subset(length > 16)},
        "training_frequency_strata": {"unseen": subset(frequency == 0), "1_4": subset((frequency > 0) & (frequency < 5)),
                                      "5_19": subset((frequency >= 5) & (frequency < 20)), "20_plus": subset(frequency >= 20)},
    }
    if name == "grade":
        result["per_band_recall"] = {str(k + 3): float((p[y == k] == k).mean()) if (y == k).any() else None for k in range(5)}
        result["band_labels"] = {"3": "[0]", "4": "(0,4)", "5": "[4,5)", "6": "[5,7)", "7": "[7,10]"}
    return result


def _ssl_pass(model, batch, task, cfg, device, generator, baseline, optimizer=None, fixed_masks=None):
    training = optimizer is not None
    model.train(training)
    size = len(batch["course"])
    batch_size = _ssl_batch_size(cfg)
    order = torch.randperm(size, generator=generator) if training else torch.arange(size)
    sums = {"grade": 0., "course": 0.}
    totals = {"grade": 0, "course": 0}
    correct = {"grade": 0, "course": 0}
    collected = {name: {key: [] for key in ("targets", "predictions", "losses", "lengths")} for name in sums}
    for start in range(0, size, batch_size):
        indices = order[start:start + batch_size]
        original = _take(batch, indices, device)
        masks = tuple(m.index_select(0, indices) for m in fixed_masks) if fixed_masks is not None else None
        corrupted, grade_mask, course_mask = corrupt_batch(original, task, float(cfg.get("mask_rate", .2)), generator, masks)
        with torch.set_grad_enabled(training):
            logits = model(corrupted, grade_mask=grade_mask, course_mask=course_mask)
            objective = None
            lengths = _observed(original).sum((1, 2)).view(-1, 1, 1).expand_as(grade_mask)
            for name, mask, target_key in (("grade", grade_mask, "grade_band"), ("course", course_mask, "course")):
                count = int(mask.sum())
                if count == 0:
                    continue
                target = original[target_key][mask].long() - 3
                individual = F.cross_entropy(logits[name], target, reduction="none")
                if not torch.isfinite(individual).all():
                    raise FloatingPointError("non-finite SSL loss")
                prediction = logits[name].argmax(-1)
                term = float(cfg.get(name + "_weight", 1.)) * individual.mean()
                objective = term if objective is None else objective + term
                sums[name] += float(individual.detach().sum())
                totals[name] += count
                correct[name] += int((prediction == target).sum())
                if not training:
                    for key, value in (("targets", target), ("predictions", prediction), ("losses", individual), ("lengths", lengths[mask])):
                        collected[name][key].append(value.detach().cpu().numpy())
            if training and objective is not None:
                _optimizer_step(objective, model, optimizer, cfg.get("clip_grad", 1.), "SSL")
    stats = {}
    for name in sums:
        stats[name + "_loss"] = sums[name] / totals[name] if totals[name] else 0.
        stats[name + "_tokens"] = totals[name]
        stats[name + "_accuracy"] = correct[name] / totals[name] if totals[name] else None
        stats[name + "_normalized_ce"] = stats[name + "_loss"] / np.log(max(len(baseline[name + "_counts"]), 2))
    stats["loss"] = sum(float(cfg.get(name + "_weight", 1.)) * stats[name + "_loss"] for name in sums)
    diagnostics = {}
    if not training:
        for name, values in collected.items():
            diagnostics[name] = _task_summary(values["targets"], values["predictions"], values["losses"], baseline, name, values["lengths"])
    return stats, diagnostics


def train_ssl(train_batch, val_batch, vocab_size, arch, task, cfg, seed, device, log, fixed_epochs=None):
    """Select SSL epochs by fixed-mask held-out token loss, or refit fresh."""
    if task not in {"grade", "course", "joint"}:
        raise ValueError("task must be grade, course, or joint")
    if fixed_epochs is None and val_batch is None:
        raise ValueError("validation batch required unless fixed_epochs supplied")
    epochs = int(fixed_epochs if fixed_epochs is not None else cfg.get("ssl_epochs", 20))
    if epochs < 1:
        raise ValueError("at least one SSL epoch required")
    _seed(int(seed))
    device = torch.device(device)
    train, validation = _tensors(train_batch), _tensors(val_batch) if fixed_epochs is None else None
    if train is None or len(train["course"]) == 0:
        raise ValueError("SSL training requires at least one history row")
    if validation is not None and len(validation["course"]) == 0:
        raise ValueError("SSL validation requires at least one history row")
    eligibility = {"training": _target_eligibility(train)}
    _require_active_ssl_targets(eligibility["training"], task, "training")
    if validation is not None:
        eligibility["validation"] = _target_eligibility(validation)
        _require_active_ssl_targets(eligibility["validation"], task, "validation")
    encoder = HistoryEncoder(vocab_size, arch, int(cfg.get("dim", 128)), int(cfg.get("layers", 2)),
                             int(cfg.get("heads", 4)), float(cfg.get("dropout", .1)))
    model = MaskedEventModel(encoder).to(device)
    optimizer = torch.optim.AdamW(model.parameters(), lr=float(cfg.get("ssl_lr", 3e-4)), weight_decay=float(cfg.get("weight_decay", 1e-3)))
    generator = torch.Generator().manual_seed(int(seed) + 17)
    baseline = _frequency_baselines(train, vocab_size)
    masks = None
    mask_hash = None
    if validation is not None:
        gm, cm = _fixed_masks(validation, task, cfg, seed)
        masks = (gm, cm)
        mask_hash = hashlib.sha256(gm.numpy().tobytes() + cm.numpy().tobytes()).hexdigest()
    log(f"SSL arch={arch} task={task} parameters={sum(p.numel() for p in model.parameters())} rows={len(train['course'])} fixed_epochs={fixed_epochs}")
    history, best_state = [], None
    best_loss, best_epoch, stale = float("inf"), epochs, 0
    for epoch in range(1, epochs + 1):
        train_stats, _ = _ssl_pass(model, train, task, cfg, device, generator, baseline, optimizer=optimizer)
        row = {"epoch": epoch, **{"train_" + k: v for k, v in train_stats.items()}}
        if validation is not None:
            val_stats, _ = _ssl_pass(model, validation, task, cfg, device, generator, baseline, fixed_masks=masks)
            row.update({"val_" + k: v for k, v in val_stats.items()})
            if val_stats["loss"] < best_loss - 1e-12:
                best_loss, best_epoch, stale = val_stats["loss"], epoch, 0
                best_state = copy.deepcopy(model.state_dict())
            else:
                stale += 1
        history.append(row)
        val_message = f" val_loss={row['val_loss']:.6f}" if validation is not None else ""
        log(f"SSL epoch={epoch} train_loss={train_stats['loss']:.6f}{val_message} grade_tokens={train_stats['grade_tokens']} course_tokens={train_stats['course_tokens']}")
        if validation is not None and stale >= int(cfg.get("ssl_patience", 4)):
            break
    if best_state is not None:
        model.load_state_dict(best_state)
    model.eval()
    diagnostics = {"frequency_baselines": baseline, "validation_mask_hash": mask_hash,
                   "selection": "fixed-mask validation token CE" if validation is not None else "fresh fixed-epoch refit",
                   "objective": "weighted sum of per-task mean masked-token CE", "normalized_ce_is_diagnostic_only": True,
                   "parameters": sum(p.numel() for p in model.parameters()),
                   "encoder_parameters": sum(p.numel() for p in encoder.parameters()),
                   "target_eligibility": eligibility,
                   "head_projection": "selected masked targets only; inactive task heads skipped",
                   "ssl_batch_size": _ssl_batch_size(cfg)}
    if validation is not None:
        _, diagnostics["validation"] = _ssl_pass(model, validation, task, cfg, device, generator, baseline, fixed_masks=masks)
    else:
        # Refit diagnostics use an independent fixed corruption, never select an epoch.
        gm, cm = _fixed_masks(train, task, cfg, seed)
        _, diagnostics["training"] = _ssl_pass(model, train, task, cfg, device, generator, baseline, fixed_masks=(gm, cm))
    return SSLResult(encoder, best_epoch, history, diagnostics, model)


def _xy(x, y, name):
    x = torch.as_tensor(np.asarray(x), dtype=torch.float32)
    y = torch.as_tensor(np.asarray(y), dtype=torch.float32).reshape(-1)
    if x.ndim != 2 or len(x) != len(y) or len(y) == 0:
        raise ValueError(name + " needs nonempty, aligned [N,D] features and [N] labels")
    if not torch.isfinite(x).all() or not torch.isfinite(y).all() or not ((y == 0) | (y == 1)).all():
        raise ValueError(name + " needs finite features and binary labels")
    return x, y


def fit_neural(xtr, ytr, btr, xval, yval, bval, encoder, mode, cfg, seed, device, log, fixed_epochs=None):
    """Fit BCE risk models, select by validation AP, or run a fresh refit."""
    if mode not in {"none", "scratch", "frozen", "finetune"}:
        raise ValueError("mode must be none, scratch, frozen, or finetune")
    if mode != "none" and encoder is None:
        raise ValueError("encoder required for " + mode)
    if fixed_epochs is None and (xval is None or yval is None):
        raise ValueError("validation data required unless fixed_epochs supplied")
    _seed(int(seed))
    device = torch.device(device)
    xtr, ytr = _xy(xtr, ytr, "training")
    train_batch = _tensors(btr) if mode != "none" else None
    if mode != "none" and (train_batch is None or history_rows(train_batch) != len(ytr)):
        raise ValueError("training history rows must align with features")
    if fixed_epochs is None:
        xval, yval = _xy(xval, yval, "validation")
        if xtr.shape[1] != xval.shape[1]:
            raise ValueError("training and validation feature widths differ")
        val_batch = _tensors(bval) if mode != "none" else None
        if mode != "none" and (val_batch is None or history_rows(val_batch) != len(yval)):
            raise ValueError("validation history rows must align with features")
    else:
        val_batch = None
    if mode == "none":
        trained_encoder = None
    elif mode == "scratch":
        trained_encoder = HistoryEncoder(encoder.vocab_size, encoder.arch, encoder.dim, encoder.layers, encoder.heads, encoder.dropout)
    else:
        trained_encoder = copy.deepcopy(encoder)
        trained_encoder.requires_grad_(mode == "finetune")
    model = NeuralRiskModel(xtr.shape[1], trained_encoder, mode, int(cfg.get("hidden_dim", 128)), float(cfg.get("dropout", .1))).to(device)
    frozen_embeddings=None
    if mode=='frozen':
        from .runner import encode
        frozen_embeddings=torch.as_tensor(encode(model.encoder,train_batch,device,int(cfg.get('batch_size',64))))
    optimizer = torch.optim.AdamW((p for p in model.parameters() if p.requires_grad),
                                 lr=float(cfg.get("neural_lr", 3e-4)), weight_decay=float(cfg.get("weight_decay", 1e-3)))
    epochs = int(fixed_epochs if fixed_epochs is not None else cfg.get("neural_epochs", 30))
    if epochs < 1:
        raise ValueError("at least one neural epoch required")
    batch_size = max(1, int(cfg.get("batch_size", 64)))
    generator = torch.Generator().manual_seed(int(seed) + 41)
    history, best_state, best_score, best_epoch, stale = [], None, -float("inf"), epochs, 0
    log(f"neural mode={mode} parameters={sum(p.numel() for p in model.parameters())} trainable={sum(p.numel() for p in model.parameters() if p.requires_grad)} fixed_epochs={fixed_epochs}")
    for epoch in range(1, epochs + 1):
        model.train()
        order = torch.randperm(len(ytr), generator=generator)
        total = 0.
        for start in range(0, len(ytr), batch_size):
            indices = order[start:start + batch_size]
            if frozen_embeddings is None:
                logits = model(xtr[indices].to(device), _take(train_batch, indices, device))
            else:
                logits=model.head(torch.cat((xtr[indices].to(device),frozen_embeddings[indices].to(device)),-1)).squeeze(-1)
            loss = F.binary_cross_entropy_with_logits(logits, ytr[indices].to(device))
            _optimizer_step(loss, model, optimizer, cfg.get("clip_grad", 1.), "neural")
            total += float(loss.detach()) * len(indices)
        row = {"epoch": epoch, "train_loss": total / len(ytr)}
        if fixed_epochs is None:
            p = predict_neural(model, xval.numpy(), val_batch, device, batch_size)
            yy = yval.numpy()
            val_ap = float(average_precision_score(yy, p)) if (yy == 1).any() else None
            val_loss = float(-(yy * np.log(np.clip(p, 1e-7, 1 - 1e-7)) + (1 - yy) * np.log(np.clip(1 - p, 1e-7, 1 - 1e-7))).mean())
            # AP has no definition without positives. Record a transparent
            # loss fallback instead of returning a random final checkpoint.
            score = val_ap if val_ap is not None else -val_loss
            row.update(val_ap=val_ap, val_loss=val_loss,
                       selection_metric="AP" if val_ap is not None else "BCE (AP undefined: no positives)")
            if score > best_score + 1e-12:
                best_score, best_epoch, stale = score, epoch, 0
                best_state = copy.deepcopy(model.state_dict())
            else:
                stale += 1
        history.append(row)
        val_message = f" val_ap={row['val_ap']}" if fixed_epochs is None else ""
        log(f"neural epoch={epoch} train_bce={row['train_loss']:.6f}{val_message}")
        if fixed_epochs is None and stale >= int(cfg.get("neural_patience", 5)):
            break
    if best_state is not None:
        model.load_state_dict(best_state)
    # Frozen encoders stay in eval mode while the head trains. Preserve this
    # invariant for callers inspecting the fitted model as well.
    model.eval()
    return NeuralResult(model, best_epoch, history)


def predict_neural(model, x, batch, device, batch_size=256):
    """Return aligned probabilities; no labels or fitted preprocessing used."""
    device = torch.device(device)
    features = torch.as_tensor(np.asarray(x), dtype=torch.float32)
    if features.ndim != 2:
        raise ValueError("prediction features must have shape [N,D]")
    history = _tensors(batch) if model.encoder is not None else None
    if model.encoder is not None and (history is None or history_rows(history) != len(features)):
        raise ValueError("prediction history rows must align with features")
    model.to(device)
    model.eval()
    embeddings=None
    if isinstance(history,IndexedHistory):
        from .runner import encode
        embeddings=torch.as_tensor(encode(model.encoder,history,device,batch_size))
    output = []
    with torch.no_grad():
        for start in range(0, len(features), max(1, int(batch_size))):
            indices = torch.arange(start, min(start + max(1, int(batch_size)), len(features)))
            if embeddings is None:
                logits = model(features[indices].to(device), _take(history, indices, device))
            else:
                logits=model.head(torch.cat((features[indices].to(device),embeddings[indices].to(device)),-1)).squeeze(-1)
            if not torch.isfinite(logits).all():
                raise FloatingPointError("non-finite neural prediction logits")
            output.append(logits.sigmoid().cpu().numpy())
    return np.concatenate(output) if output else np.empty(0, dtype=np.float32)
