from __future__ import annotations

import copy
import inspect
import math
import pickle
from pathlib import Path
from typing import Dict, Optional

import numpy as np
from sklearn.base import clone
from sklearn.ensemble import ExtraTreesClassifier, HistGradientBoostingClassifier, RandomForestClassifier
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import RobustScaler, StandardScaler


class ClassAwareHeterogeneousStackingClassifier:
    """Leakage-controlled heterogeneous stack for an imbalanced binary target.

    The active base learners are conventional balanced tree ensembles. The old
    ARL feature-subspace classifier and custom RL gradient-boosting classifier
    are intentionally not part of this implementation.

    ``classification_threshold_`` is an *internal* threshold used by the joint
    grade model. The deployable warning threshold should be tuned outside this
    class on pooled temporal development-fold predictions.
    """

    def __init__(
        self,
        estimators: Dict[str, object],
        positive_class: int = 1,
        meta_estimator=None,
        blender_fraction: float = 0.50,
        threshold_beta: float = 2.0,
        threshold_min_precision: float = 0.10,
        threshold_max_warning_ratio: float = 3.0,
        threshold_min_flags: int = 5,
        random_state: int = 42,
        verbose: bool = True,
        include_raw_features: bool = False,
        raw_scaler: str = "standard",
        meta_C: float = 0.5,
        meta_prune_correlation: Optional[float] = None,
        meta_variance_tolerance: float = 1e-12,
    ):
        if not estimators:
            raise ValueError("estimators cannot be empty")
        if raw_scaler not in ("standard", "robust"):
            raise ValueError("raw_scaler must be 'standard' or 'robust'")
        if meta_prune_correlation is not None and not 0.0 < meta_prune_correlation <= 1.0:
            raise ValueError("meta_prune_correlation must be in (0, 1]")
        if not 0.0 < blender_fraction < 1.0:
            raise ValueError("blender_fraction must be in (0, 1)")
        if threshold_beta <= 0.0:
            raise ValueError("threshold_beta must be positive")
        if not 0.0 <= threshold_min_precision <= 1.0:
            raise ValueError("threshold_min_precision must be in [0, 1]")
        if threshold_max_warning_ratio <= 0.0:
            raise ValueError("threshold_max_warning_ratio must be positive")
        if threshold_min_flags < 1:
            raise ValueError("threshold_min_flags must be >= 1")

        self.estimators = dict(estimators)
        self.positive_class = positive_class
        self.meta_estimator = meta_estimator
        self.blender_fraction = float(blender_fraction)
        self.threshold_beta = float(threshold_beta)
        self.threshold_min_precision = float(threshold_min_precision)
        self.threshold_max_warning_ratio = float(threshold_max_warning_ratio)
        self.threshold_min_flags = int(threshold_min_flags)
        self.random_state = int(random_state)
        self.verbose = bool(verbose)
        self.include_raw_features = bool(include_raw_features)
        self.raw_scaler = str(raw_scaler)
        # Inverse L2 strength of the meta logistic regression. Wide meta matrices
        # (include_raw_features) are fitted on the small blender split and need a
        # much smaller C than the stacking-only default.
        self.meta_C = float(meta_C)
        # None disables pruning entirely, keeping the meta matrix byte-identical to
        # the unpruned baseline. A float is the |r| above which the later of a
        # correlated pair is dropped.
        self.meta_prune_correlation = (
            None if meta_prune_correlation is None else float(meta_prune_correlation)
        )
        self.meta_variance_tolerance = float(meta_variance_tolerance)
        self._reset()

    def _reset(self) -> None:
        self.classes_ = None
        self.positive_class_ = None
        self.negative_class_ = None
        self.positive_class_index_ = None
        self.negative_class_index_ = None
        self.n_features_in_ = None
        self.fitted_estimators_ = {}
        self.base_model_metrics_ = {}
        self.meta_estimator_ = None
        self.meta_feature_names_ = []
        self.raw_scaler_ = None
        self.raw_feature_names_ = []
        self.meta_keep_mask_ = None
        self.meta_prune_report_ = []
        self.classification_threshold_ = 0.5
        self.validation_metrics_ = None
        self.is_fitted_ = False

    @staticmethod
    def _prepare_X(X):
        if hasattr(X, "to_numpy"):
            X = X.to_numpy()
        X = np.asarray(X, dtype=np.float64)
        if X.ndim != 2:
            raise ValueError(f"X must be 2D, got {X.shape}")
        return np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)

    @staticmethod
    def _prepare_y(y):
        if hasattr(y, "to_numpy"):
            y = y.to_numpy()
        return np.asarray(y).reshape(-1)

    @staticmethod
    def _safe_clone(estimator):
        try:
            return clone(estimator)
        except Exception:
            return copy.deepcopy(estimator)

    def _init_classes(self, y) -> None:
        self.classes_ = np.unique(y)
        if len(self.classes_) != 2:
            raise ValueError("binary classification only")
        if self.positive_class not in self.classes_:
            raise ValueError("positive class not found")
        self.positive_class_ = self.positive_class
        self.negative_class_ = self.classes_[self.classes_ != self.positive_class_][0]
        self.positive_class_index_ = int(np.where(self.classes_ == self.positive_class_)[0][0])
        self.negative_class_index_ = int(np.where(self.classes_ == self.negative_class_)[0][0])

    def _fit_estimator(self, estimator, X, y, X_selector, y_selector):
        params = inspect.signature(estimator.fit).parameters
        if "X_val" in params and "y_val" in params:
            estimator.fit(X, y, X_val=X_selector, y_val=y_selector)
        else:
            estimator.fit(X, y)
        return estimator

    def _positive_probability(self, estimator, X):
        probability = np.asarray(estimator.predict_proba(X), dtype=np.float64)
        classes = np.asarray(estimator.classes_)
        index = np.where(classes == self.positive_class_)[0]
        if len(index) != 1:
            raise RuntimeError("base estimator does not expose the positive class")
        return np.clip(probability[:, int(index[0])], 1e-7, 1.0 - 1e-7)

    @staticmethod
    def _metrics(y_true, y_pred, positive_class):
        positive = y_true == positive_class
        predicted_positive = y_pred == positive_class
        tp = int(np.sum(positive & predicted_positive))
        fp = int(np.sum(~positive & predicted_positive))
        fn = int(np.sum(positive & ~predicted_positive))
        tn = int(np.sum(~positive & ~predicted_positive))
        precision = tp / max(tp + fp, 1)
        recall = tp / max(tp + fn, 1)
        specificity = tn / max(tn + fp, 1)
        f1 = 2.0 * precision * recall / max(precision + recall, 1e-15)
        balanced_accuracy = 0.5 * (recall + specificity)
        flag_rate = (tp + fp) / max(len(y_true), 1)
        return {
            "accuracy": float((tp + tn) / max(len(y_true), 1)),
            "minority_precision": float(precision),
            "minority_recall": float(recall),
            "majority_recall": float(specificity),
            "minority_f1": float(f1),
            "balanced_accuracy": float(balanced_accuracy),
            "gmean": float(math.sqrt(max(recall, 0.0) * max(specificity, 0.0))),
            "flag_rate": float(flag_rate),
            "true_positive": tp,
            "false_positive": fp,
            "false_negative": fn,
            "true_negative": tn,
        }

    def _fbeta(self, precision: float, recall: float) -> float:
        beta_sq = self.threshold_beta ** 2
        denominator = beta_sq * precision + recall
        if denominator <= 0.0:
            return 0.0
        return float((1.0 + beta_sq) * precision * recall / denominator)

    @staticmethod
    def _candidate_thresholds(probability: np.ndarray, maximum: int = 600) -> np.ndarray:
        probability = np.asarray(probability, dtype=np.float64).reshape(-1)
        unique = np.unique(probability)
        if len(unique) <= maximum:
            return unique
        quantiles = np.linspace(0.001, 0.999, maximum)
        return np.unique(np.quantile(probability, quantiles))

    def _best_threshold(self, y, probability):
        y = np.asarray(y).reshape(-1)
        probability = np.asarray(probability, dtype=np.float64).reshape(-1)
        prevalence = float(np.mean(y == self.positive_class_))
        max_flag_rate = min(0.99, self.threshold_max_warning_ratio * max(prevalence, 1e-8))
        candidates = []

        for threshold in self._candidate_thresholds(probability):
            prediction = np.where(
                probability >= threshold,
                self.positive_class_,
                self.negative_class_,
            )
            metrics = self._metrics(y, prediction, self.positive_class_)
            metrics["fbeta"] = self._fbeta(
                metrics["minority_precision"],
                metrics["minority_recall"],
            )
            candidates.append((float(threshold), metrics))

        def enough_flags(item):
            return item[1]["flag_rate"] * len(y) >= self.threshold_min_flags

        strict = [
            item for item in candidates
            if enough_flags(item)
            and item[1]["minority_precision"] >= self.threshold_min_precision
            and item[1]["flag_rate"] <= max_flag_rate
        ]
        ratio_only = [
            item for item in candidates
            if enough_flags(item) and item[1]["flag_rate"] <= max_flag_rate
        ]
        eligible = strict or ratio_only or [item for item in candidates if enough_flags(item)] or candidates
        threshold, metrics = max(
            eligible,
            key=lambda item: (
                item[1]["fbeta"],
                item[1]["minority_recall"],
                item[1]["minority_precision"],
                -item[1]["flag_rate"],
            ),
        )
        return threshold, metrics

    def _make_raw_scaler(self):
        # RobustScaler suits the skewed count features better; StandardScaler is
        # the default so behaviour matches the usual z-score expectation.
        if self.raw_scaler == "robust":
            return RobustScaler()
        return StandardScaler()

    def _scaled_raw(self, X_raw):
        """Normalised raw design matrix, or an empty block when disabled."""
        if not self.include_raw_features:
            return np.empty((0, 0))
        if X_raw is None:
            raise ValueError("include_raw_features=True requires the raw design matrix")
        X_raw = self._prepare_X(X_raw)
        if self.raw_scaler_ is None:
            raise RuntimeError("raw scaler is not fitted")
        if X_raw.shape[1] != self.n_features_in_:
            raise ValueError(
                f"raw matrix has {X_raw.shape[1]} columns, expected {self.n_features_in_}"
            )
        return np.nan_to_num(
            np.asarray(self.raw_scaler_.transform(X_raw), dtype=np.float64),
            nan=0.0,
            posinf=0.0,
            neginf=0.0,
        )

    def _fit_meta_prune(self, Z, names):
        """Drop constant and near-duplicate meta columns.

        Unsupervised and deterministic: the target is never consulted, and of a
        correlated pair the *later* column is dropped. Because the stacking block
        is laid out first, that rule keeps ensemble signal in preference to raw
        columns, and keeps base probabilities in preference to the order
        statistics derived from them.
        """
        Z = np.asarray(Z, dtype=np.float64)
        keep = np.ones(Z.shape[1], dtype=bool)
        report = []

        deviation = Z.std(axis=0)
        for column in np.where(deviation <= self.meta_variance_tolerance)[0]:
            keep[column] = False
            report.append({
                "dropped": names[column],
                "reason": "constant",
                "kept_instead": "",
                "correlation": float("nan"),
            })

        surviving = np.where(keep)[0]
        if len(surviving) > 1:
            block = Z[:, surviving]
            block = (block - block.mean(axis=0)) / block.std(axis=0)
            correlation = np.nan_to_num(np.abs(np.corrcoef(block, rowvar=False)))
            anchors = []
            for position in range(len(surviving)):
                duplicate_of = next(
                    (
                        anchor
                        for anchor in anchors
                        if correlation[position, anchor] > self.meta_prune_correlation
                    ),
                    None,
                )
                if duplicate_of is None:
                    anchors.append(position)
                    continue
                keep[surviving[position]] = False
                report.append({
                    "dropped": names[surviving[position]],
                    "reason": "correlated",
                    "kept_instead": names[surviving[duplicate_of]],
                    "correlation": float(correlation[position, duplicate_of]),
                })

        self.meta_keep_mask_ = keep
        self.meta_prune_report_ = report
        return keep

    def _apply_meta_prune(self, Z):
        if self.meta_keep_mask_ is None:
            return Z
        return np.asarray(Z, dtype=np.float64)[:, self.meta_keep_mask_]

    def _meta_features(self, probability_matrix, X_raw=None, fit_names: bool = False):
        p = np.clip(np.asarray(probability_matrix, dtype=np.float64), 1e-7, 1.0 - 1e-7)
        logits = np.log(p / (1.0 - p))
        mean = p.mean(axis=1, keepdims=True)
        minimum = p.min(axis=1, keepdims=True)
        maximum = p.max(axis=1, keepdims=True)
        std = p.std(axis=1, keepdims=True)
        spread = maximum - minimum
        vote_fraction = (p >= 0.5).mean(axis=1, keepdims=True)
        entropy = -(mean * np.log(mean + 1e-12) + (1.0 - mean) * np.log(1.0 - mean + 1e-12))
        if fit_names:
            names = list(self.fitted_estimators_)
            self.meta_feature_names_ = (
                [f"probability__{name}" for name in names]
                + [f"logit__{name}" for name in names]
                + [
                    "probability_mean",
                    "probability_min",
                    "probability_max",
                    "probability_std",
                    "probability_spread",
                    "vote_fraction",
                    "ensemble_entropy",
                ]
            )
            if self.include_raw_features:
                self.meta_feature_names_ = self.meta_feature_names_ + [
                    f"raw__{name}" for name in self.raw_feature_names_
                ]
        blocks = [p, logits, mean, minimum, maximum, std, spread, vote_fraction, entropy]
        if self.include_raw_features:
            blocks.append(self._scaled_raw(X_raw))
        return np.column_stack(blocks)

    def _default_meta(self):
        return Pipeline([
            ("imputer", SimpleImputer(strategy="median")),
            ("scaler", StandardScaler()),
            (
                "classifier",
                LogisticRegression(
                    class_weight="balanced",
                    C=self.meta_C,
                    max_iter=3000,
                    solver="liblinear",
                    random_state=self.random_state,
                ),
            ),
        ])

    def fit(self, X, y, X_val=None, y_val=None, feature_names=None):
        if X_val is None or y_val is None:
            raise ValueError("X_val and y_val are required")
        # Column names must be read before _prepare_X drops them.
        if feature_names is None and hasattr(X, "columns"):
            feature_names = list(X.columns)
        X = self._prepare_X(X)
        y = self._prepare_y(y)
        X_val = self._prepare_X(X_val)
        y_val = self._prepare_y(y_val)
        self._reset()
        self.n_features_in_ = X.shape[1]
        self._init_classes(y)
        self.raw_feature_names_ = (
            list(feature_names)
            if feature_names is not None
            else [f"feature_{i}" for i in range(self.n_features_in_)]
        )
        if len(self.raw_feature_names_) != self.n_features_in_:
            raise ValueError(
                f"feature_names has {len(self.raw_feature_names_)} entries, "
                f"expected {self.n_features_in_}"
            )
        if self.include_raw_features:
            # Fitted on the 90% base-training split, not on the ~5% blender: the
            # scaler is meant to be reusable by downstream models (e.g. a deep net)
            # and the blender is far too small to estimate per-column statistics.
            self.raw_scaler_ = self._make_raw_scaler().fit(X)

        splitter = StratifiedShuffleSplit(
            n_splits=1,
            test_size=self.blender_fraction,
            random_state=self.random_state,
        )
        selector_idx, blender_idx = next(splitter.split(X_val, y_val))
        X_selector, y_selector = X_val[selector_idx], y_val[selector_idx]
        X_blender, y_blender = X_val[blender_idx], y_val[blender_idx]

        if self.verbose:
            print(f"Selector rows={len(selector_idx)} | Blender rows={len(blender_idx)}")

        blender_probabilities = []
        for name, prototype in self.estimators.items():
            estimator = self._safe_clone(prototype)
            try:
                estimator = self._fit_estimator(estimator, X, y, X_selector, y_selector)
                probability = self._positive_probability(estimator, X_blender)
            except Exception as error:
                if self.verbose:
                    print(f"  {name}: skipped ({type(error).__name__}: {error})")
                continue
            threshold, metrics = self._best_threshold(y_blender, probability)
            self.fitted_estimators_[name] = estimator
            self.base_model_metrics_[name] = {"threshold": threshold, **metrics}
            blender_probabilities.append(probability)
            if self.verbose:
                print(
                    f"  {name:24s} F{self.threshold_beta:g}={metrics['fbeta']:.4f} "
                    f"threshold={threshold:.5f} P={metrics['minority_precision']:.4f} "
                    f"R={metrics['minority_recall']:.4f} F1={metrics['minority_f1']:.4f} "
                    f"flag={metrics['flag_rate']:.4f}"
                )

        if len(self.fitted_estimators_) < 2:
            raise RuntimeError("at least two base estimators must fit successfully")

        matrix = np.column_stack(blender_probabilities)
        meta_X = self._meta_features(matrix, X_raw=X_blender, fit_names=True)
        raw_width = self.n_features_in_ if self.include_raw_features else 0
        full_width = meta_X.shape[1]
        if self.meta_prune_correlation is not None:
            keep = self._fit_meta_prune(meta_X, list(self.meta_feature_names_))
            self.meta_feature_names_ = [
                name for name, kept in zip(self.meta_feature_names_, keep) if kept
            ]
            meta_X = self._apply_meta_prune(meta_X)
        if self.verbose:
            print(
                f"Meta input width={full_width} "
                f"(stacking={full_width - raw_width} + raw={raw_width}"
                + (f", scaler={self.raw_scaler}" if raw_width else "")
                + f") on {len(y_blender):,} blender rows"
            )
            if self.meta_prune_correlation is not None:
                constant = sum(1 for r in self.meta_prune_report_ if r["reason"] == "constant")
                correlated = len(self.meta_prune_report_) - constant
                print(
                    f"  pruned |r|>{self.meta_prune_correlation:g}: "
                    f"-{constant} constant, -{correlated} correlated "
                    f"-> {meta_X.shape[1]} columns kept"
                )
        self.meta_estimator_ = (
            self._default_meta()
            if self.meta_estimator is None
            else self._safe_clone(self.meta_estimator)
        )
        self.meta_estimator_.fit(meta_X, y_blender)
        meta_probability = self._positive_probability(self.meta_estimator_, meta_X)
        threshold, metrics = self._best_threshold(y_blender, meta_probability)
        self.classification_threshold_ = float(threshold)
        self.validation_metrics_ = metrics
        self.is_fitted_ = True
        if self.verbose:
            print(f"Internal stacking threshold={threshold:.6f}")
            print(f"Internal stacking validation metrics={metrics}")
        return self

    def _check_fitted(self):
        if not self.is_fitted_:
            raise RuntimeError("model is not fitted")

    def transform_meta_matrix(self, X):
        """Build the exact matrix the meta learner consumes.

        Returns ``(Z, names)``. With ``include_raw_features`` the layout is
        ``[stacking block | normalised raw block]``, so a downstream model — a
        deep net, for instance — can train on the identical representation
        without re-deriving the normalisation.
        """
        self._check_fitted()
        X = self._prepare_X(X)
        matrix = np.column_stack([
            self._positive_probability(estimator, X)
            for estimator in self.fitted_estimators_.values()
        ])
        Z = self._apply_meta_prune(self._meta_features(matrix, X_raw=X))
        return Z, list(self.meta_feature_names_)

    def predict_positive_proba(self, X):
        self._check_fitted()
        meta_X, _ = self.transform_meta_matrix(X)
        return self._positive_probability(self.meta_estimator_, meta_X)

    def predict_proba(self, X):
        p = self.predict_positive_proba(X)
        result = np.zeros((len(p), 2), dtype=np.float64)
        result[:, self.positive_class_index_] = p
        result[:, self.negative_class_index_] = 1.0 - p
        return result

    def predict(self, X, threshold: Optional[float] = None):
        threshold = self.classification_threshold_ if threshold is None else float(threshold)
        probability = self.predict_positive_proba(X)
        return np.where(probability >= threshold, self.positive_class_, self.negative_class_)

    def evaluate_binary(self, X, y, threshold=None):
        y = self._prepare_y(y)
        return self._metrics(y, self.predict(X, threshold), self.positive_class_)

    def get_base_model_report(self):
        self._check_fitted()
        return [{"model": name, **metrics} for name, metrics in self.base_model_metrics_.items()]

    def get_meta_prune_report(self):
        """Which meta columns were dropped, why, and in favour of what."""
        self._check_fitted()
        return list(self.meta_prune_report_)

    def get_meta_coefficients(self):
        self._check_fitted()
        classifier = (
            self.meta_estimator_.named_steps.get("classifier", self.meta_estimator_)
            if hasattr(self.meta_estimator_, "named_steps")
            else self.meta_estimator_
        )
        coefficients = getattr(classifier, "coef_", None)
        if coefficients is None:
            return []
        coefficients = np.asarray(coefficients).reshape(-1)
        return sorted(
            [
                {
                    "meta_feature": name,
                    "coefficient": float(value),
                    "absolute_coefficient": float(abs(value)),
                }
                for name, value in zip(self.meta_feature_names_, coefficients)
            ],
            key=lambda row: row["absolute_coefficient"],
            reverse=True,
        )

    def save_model(self, path):
        path = Path(path)
        path.parent.mkdir(parents=True, exist_ok=True)
        with path.open("wb") as file:
            pickle.dump(self, file)

    @staticmethod
    def load_model(path):
        with Path(path).open("rb") as file:
            return pickle.load(file)


def build_standard_base_estimators(
    random_state: int = 42,
    include_lightgbm: bool = True,
):
    """Build the reduced classifier community.

    Active models:
      * balanced Random Forest
      * balanced Extra Trees
      * Histogram Gradient Boosting
      * optional LightGBM
    """
    estimators = {
        "random_forest_balanced": RandomForestClassifier(
            n_estimators=500,
            min_samples_leaf=4,
            max_features="sqrt",
            class_weight="balanced_subsample",
            bootstrap=True,
            n_jobs=-1,
            random_state=random_state,
        ),
        "extra_trees_balanced": ExtraTreesClassifier(
            n_estimators=500,
            min_samples_leaf=3,
            max_features=0.70,
            class_weight="balanced",
            n_jobs=-1,
            random_state=random_state + 1,
        ),
        "hist_gradient_boosting": HistGradientBoostingClassifier(
            learning_rate=0.05,
            max_iter=250,
            max_leaf_nodes=31,
            min_samples_leaf=20,
            l2_regularization=1.0,
            early_stopping=True,
            random_state=random_state + 2,
        ),
    }

    if include_lightgbm:
        try:
            from lightgbm import LGBMClassifier

            estimators["lightgbm_balanced"] = LGBMClassifier(
                objective="binary",
                n_estimators=500,
                learning_rate=0.03,
                num_leaves=31,
                min_child_samples=20,
                subsample=0.85,
                colsample_bytree=0.80,
                reg_alpha=0.10,
                reg_lambda=1.0,
                class_weight="balanced",
                n_jobs=-1,
                random_state=random_state + 3,
                verbosity=-1,
            )
        except ImportError:
            pass

    return estimators
