import os
import argparse
import polars as pl
import numpy as np
import pandas as pd
import random
from pathlib import Path
import xgboost as xgb
import matplotlib.pyplot as plt
from scipy.constants import yocto
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
from sklearn.metrics import mean_absolute_error, r2_score
from typing import Union
from imblearn.over_sampling import SMOTE

import pickle
from pathlib import Path
from xgboost import XGBClassifier


N_RATE = 0.1
N_EST = 20
LOW_GRADE_THRESHOLD = 0.7

ROOT = Path(__file__).resolve().parents[1]

random.seed(42)

DROP_MISSING_COLUMNS = [
    "course_key",
    "hist_same_course_attempts",
    "hist_same_course_mean_grade10",
    "hist_same_course_best_grade10",
    "hist_same_course_last_grade10",
    "hist_same_course_pass_rate",
    "prior_fail_count_in_target_course",
    "hist_same_course_component_count",
    "hist_same_course_component_mean",
    "hist_same_course_component_min",
    "hist_component_B1_mean",
    "hist_component_B1_low_rate",
]

DROP_CONSTANT_COLUMNS = [
    "course_grade_cong_nghe_phan_mem",
    "course_grade_marketing_ky_thuat_so",
    "course_grade_thuc_hanh_tieng_anh_b1",
]

# ----------------------------
# Step 2. Regression Tree Node Types
# ----------------------------
# ----------------------------
# Node Classes
# ----------------------------
class Leaf:
    def __init__(self, value):
        self.value = value

    def predict(self, X):
        # Return a constant prediction (leaf value) for all samples
        return np.full(X.shape[0], self.value, dtype=float)

class Split:
    def __init__(self, attribute, threshold, left, right, default_value):
        self.attribute = attribute        # feature index
        self.threshold = threshold
        self.left = left
        self.right = right
        self.default_value = default_value

    def predict(self, X):
        mask = np.isfinite(X[:, self.attribute]) & (X[:, self.attribute] <= self.threshold)
        y_pred = np.full(X.shape[0], self.default_value, dtype=float)

        if mask.any():
            y_pred[mask] = self.left.predict(X[mask])
        if (~mask).any():
            y_pred[~mask] = self.right.predict(X[~mask])
        return y_pred

DecisionTree = Union[Leaf, Split]

# ----------------------------
# Robust Tree Builder
# ----------------------------
def build_tree_id3_regression(X, y, feature_indices=None, min_samples=0, max_depth=5, depth=0) -> DecisionTree:
    # Clean input
    mask = np.isfinite(y) & np.all(np.isfinite(X), axis=1)
    X, y = X[mask], y[mask]

    if len(y) == 0:
        return Leaf(0.0)

    # Stop if depth limit reached or samples too small
    if len(y) <= min_samples or np.all(y == y[0]) or depth >= max_depth:
        return Leaf(np.mean(y))

    if feature_indices is None:
        feature_indices = range(X.shape[1])

    best_feature, best_threshold, best_loss = None, None, float("inf")

    for feature in feature_indices:
        col = X[:, feature]
        if np.all(~np.isfinite(col)) or np.all(col == col[0]):
            continue

        thresholds = np.unique(col[np.isfinite(col)])

        for t in thresholds:
            left_mask = col <= t
            #print("left_mask: ", left_mask)
            right_mask = ~left_mask

            # Skip invalid splits
            if left_mask.sum() == 0 or right_mask.sum() == 0:
                continue

            left_y = y[left_mask]
            #print("left_y: ", left_y)
            right_y = y[right_mask]

            # Skip NaN targets
            if len(left_y) == 0 or len(right_y) == 0:
                continue

            # Variance-based loss
            loss = np.var(left_y) * len(left_y) + np.var(right_y) * len(right_y)
            if not np.isnan(loss) and loss < best_loss:
                best_feature, best_threshold, best_loss = feature, t, loss
                #print("best_feature: ", best_feature)
                #print("best_threshold: ", best_threshold)


    if best_feature is None:
        return Leaf(np.mean(y))

    left_mask = X[:, best_feature] <= best_threshold
    right_mask = ~left_mask

    # Recursive calls
    left_tree = build_tree_id3_regression(X[left_mask], y[left_mask],
                                          feature_indices, min_samples, max_depth, depth + 1)
    right_tree = build_tree_id3_regression(X[right_mask], y[right_mask],
                                           feature_indices, min_samples, max_depth, depth + 1)

    return Split(best_feature, best_threshold, left_tree, right_tree, np.mean(y))

# ----------------------------
# Gradient Boosting (Safe)
# ----------------------------
class RL_GradientBoostingRegressorScratch:
    """
    Gradient Boosting Regressor with a tabular Q-learning agent.

    The RL agent selects a learning-rate multiplier at each
    boosting iteration.

    State:
        Current normalized mean absolute residual.

    Action:
        Multiplier applied to the base learning rate.

    Reward:
        Relative reduction in MAE after adding the new tree.
    """

    def __init__(
        self,
        learning_rate=0.1,
        n_estimators=50,
        min_samples=2,
        max_depth=5,
        actions=None,
        epsilon=0.20,
        epsilon_decay=0.98,
        min_epsilon=0.02,
        alpha_q=0.1,
        gamma_q=0.9,
        random_state=42,
        verbose=True,
    ):
        self.learning_rate = float(learning_rate)
        self.n_estimators = int(n_estimators)
        self.min_samples = int(min_samples)
        self.max_depth = int(max_depth)

        # Learning-rate multipliers
        if actions is None:
            actions = [0.25, 0.5, 1.0, 1.5, 2.0]

        self.actions = np.asarray(actions, dtype=np.float64)

        if np.any(self.actions < 0):
            raise ValueError("All action multipliers must be non-negative.")

        self.num_states = 5
        self.num_actions = len(self.actions)

        # Q-learning parameters
        self.epsilon = float(epsilon)
        self.initial_epsilon = float(epsilon)
        self.epsilon_decay = float(epsilon_decay)
        self.min_epsilon = float(min_epsilon)
        self.alpha_q = float(alpha_q)
        self.gamma_q = float(gamma_q)

        self.random_state = random_state
        self.verbose = verbose

        self.rng = np.random.default_rng(random_state)

        # Model attributes
        self.Q = np.zeros((self.num_states, self.num_actions), dtype=np.float64)

        self.trees = []
        self.step_sizes = []
        self.action_history = []
        self.state_history = []
        self.reward_history = []
        self.training_mae_history = []
        self.evaluation_mae_history = []

        self.init_value = None
        self.target_min = None
        self.target_max = None
        self.target_range = None
        self.is_fitted = False

    def get_state(self, residuals):
        """
        Convert the current residual magnitude into one of five states.

        State 0: very low error
        State 1: low error
        State 2: moderate error
        State 3: high error
        State 4: very high error
        """
        residuals = np.asarray(residuals, dtype=np.float64)

        if residuals.size == 0:
            return 0

        mean_absolute_residual = np.mean(np.abs(residuals))

        target_range = max(float(self.target_range), 1e-12)

        normalized_error = (mean_absolute_residual / target_range)

        state_boundaries = np.array([0.025, 0.05, 0.10, 0.20], dtype=np.float64)

        state = np.digitize(normalized_error, bins=state_boundaries, right=True)

        return int(np.clip(state,0, self.num_states - 1))

    def select_action(self, state, training=True):
        """
        Select an action using epsilon-greedy exploration.

        During prediction no action selection is needed because
        the selected step size is stored for every trained tree.
        """
        if training and self.rng.random() < self.epsilon:
            return int(self.rng.integers(0, self.num_actions))

        state_q_values = self.Q[state]
        maximum_q = np.max(state_q_values)

        # Random tie-breaking prevents action zero from always
        # being selected when all Q-values are initially equal.
        best_actions = np.flatnonzero(
            np.isclose(
                state_q_values,
                maximum_q,
            )
        )

        return int(self.rng.choice(best_actions))

    def _prepare_X(self, X):
        X = np.asarray(X, dtype=np.float64)

        if X.ndim != 2:
            raise ValueError(f"X must be two-dimensional; got {X.shape}.")

        return np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)

    def _prepare_y(self, y):
        return np.asarray(y, dtype=np.float64).reshape(-1)

    def fit(self, X, y, X_val=None, y_val=None):
        """
        Fit the RL-enhanced gradient boosting model.

        When validation data is provided, the Q-learning reward is
        calculated from validation MAE. Otherwise, training MAE is used.

        Parameters
        ----------
        X : array-like
            Training features.

        y : array-like
            Training targets.

        X_val : array-like or None
            Optional validation features.

        y_val : array-like or None
            Optional validation targets.

        Returns
        -------
        self
        """
        X = self._prepare_X(X)
        y = self._prepare_y(y)

        if len(X) != len(y):
            raise ValueError(f"X and y lengths differ: {len(X)} != {len(y)}.")

        valid_mask = np.isfinite(y)

        X = X[valid_mask]
        y = y[valid_mask]

        if len(y) == 0:
            raise ValueError(
                "No valid training targets were found."
            )

        use_validation = (X_val is not None and y_val is not None)

        if use_validation:
            X_val = self._prepare_X(X_val)
            y_val = self._prepare_y(y_val)

            if len(X_val) != len(y_val):
                raise ValueError("X_val and y_val have different lengths.")

            valid_val_mask = np.isfinite(y_val)
            X_val = X_val[valid_val_mask]
            y_val = y_val[valid_val_mask]

            if len(y_val) == 0:
                raise ValueError("Validation data contains no valid targets.")

        # Reset the model when fit is called again.
        self.trees = []
        self.step_sizes = []
        self.action_history = []
        self.state_history = []
        self.reward_history = []
        self.training_mae_history = []
        self.evaluation_mae_history = []

        self.Q.fill(0.0)
        self.epsilon = self.initial_epsilon

        self.target_min = float(np.min(y))
        self.target_max = float(np.max(y))
        self.target_range = max(self.target_max - self.target_min, 1e-12,)

        # Weighted mean can be substituted here later if needed.
        self.init_value = float(np.mean(y))

        y_pred = np.full(len(y), self.init_value, dtype=np.float64)

        if use_validation:
            val_pred = np.full(len(y_val),self.init_value,dtype=np.float64)

            previous_evaluation_mae = (mean_absolute_error(y_val,val_pred))
        else:
            previous_evaluation_mae = (mean_absolute_error(y,y_pred))

        for estimator_index in range(self.n_estimators):
            # Gradient of squared-error loss corresponds to residuals.
            residuals = y - y_pred

            state = self.get_state(residuals)

            action_index = self.select_action(state, training=True)

            multiplier = float(self.actions[action_index])

            step_size = (self.learning_rate * multiplier)

            # Bootstrap indices preserve feature-target alignment.
            bootstrap_indices = self.rng.integers(low=0, high=len(X), size=len(X))

            X_bootstrap = X[bootstrap_indices]

            residuals_bootstrap = residuals[bootstrap_indices]

            tree = build_tree_id3_regression(
                X_bootstrap,
                residuals_bootstrap,
                min_samples=self.min_samples,
                max_depth=self.max_depth,
            )

            train_update = tree.predict(X)

            train_update = np.nan_to_num(train_update, nan=0.0, posinf=0.0, neginf=0.0)

            # Correct gradient boosting operation: ADD the tree.
            candidate_train_pred = (y_pred + step_size * train_update)

            # Keep predictions in the known target range.
            candidate_train_pred = np.clip(candidate_train_pred, self.target_min, self.target_max)

            if use_validation:
                val_update = tree.predict(X_val)

                val_update = np.nan_to_num(val_update, nan=0.0, posinf=0.0, neginf=0.0)

                candidate_val_pred = (val_pred + step_size * val_update)

                candidate_val_pred = np.clip(candidate_val_pred, self.target_min, self.target_max)

                current_evaluation_mae = (mean_absolute_error(y_val, candidate_val_pred))
            else:
                current_evaluation_mae = ( mean_absolute_error( y, candidate_train_pred))

            # Relative improvement gives rewards a more stable scale.
            denominator = max(previous_evaluation_mae, 1e-12,)

            reward = (previous_evaluation_mae - current_evaluation_mae) / denominator

            next_residuals = (y - candidate_train_pred)

            next_state = self.get_state(next_residuals)

            old_q = self.Q[state, action_index,]

            future_q = np.max(self.Q[next_state])

            temporal_difference_target = (reward + self.gamma_q * future_q)

            self.Q[state, action_index] = (old_q + self.alpha_q * (temporal_difference_target - old_q))

            # Keep the tree and its individual step size.
            self.trees.append(tree)
            self.step_sizes.append(step_size)
            self.state_history.append(state)
            self.action_history.append(action_index)
            self.reward_history.append(reward)

            y_pred = candidate_train_pred

            if use_validation:
                val_pred = candidate_val_pred

            training_mae = mean_absolute_error(y, y_pred)

            self.training_mae_history.append(training_mae)

            self.evaluation_mae_history.append(current_evaluation_mae)

            previous_evaluation_mae = (current_evaluation_mae)

            self.epsilon = max(self.min_epsilon, self.epsilon * self.epsilon_decay)

            if self.verbose:
                print(
                    f"Estimator {estimator_index + 1:03d} | "
                    f"state={state} | "
                    f"action={multiplier:.2f} | "
                    f"step={step_size:.5f} | "
                    f"train MAE={training_mae:.6f} | "
                    f"evaluation MAE="
                    f"{current_evaluation_mae:.6f} | "
                    f"reward={reward:+.6f} | "
                    f"epsilon={self.epsilon:.4f}"
                )

        self.is_fitted = True

        return self

    def predict(self, X):
        """
        Predict using additive boosting updates.

        Each tree uses the effective step size selected by
        the RL agent during training.
        """
        if not self.is_fitted:
            raise RuntimeError("The model must be fitted before prediction.")

        X = self._prepare_X(X)

        prediction = np.full(X.shape[0], self.init_value, dtype=np.float64)

        for tree, step_size in zip(self.trees, self.step_sizes):

            update = tree.predict(X)

            update = np.nan_to_num(update, nan=0.0, posinf=0.0, neginf=0.0,)

            prediction += step_size * update

        prediction = np.clip(prediction, self.target_min, self.target_max)

        return prediction

    def save_model(self, filepath):
        filepath = Path(filepath)
        filepath.parent.mkdir(parents=True, exist_ok=True)

        with open(filepath, "wb") as file:
            pickle.dump(self, file)

    @classmethod
    def load_model(cls, filepath):
        filepath = Path(filepath)

        with open(filepath, "rb") as file:
            model = pickle.load(file)

        if not isinstance(model, cls):
            raise TypeError(
                f"Expected {cls.__name__}, got {type(model).__name__}"
            )
        return model


import numpy as np

from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.metrics import (
    mean_absolute_error,
    mean_squared_error,
    r2_score,
    accuracy_score,
    precision_score,
    recall_score,
    f1_score,
    confusion_matrix,
    classification_report,
)

def evaluate_low_grade_classifier(classifier, X, y, boundary=1.0,):
    """
    Evaluate the binary classifier for predicting grade < boundary.

    Class 1 : grade < boundary
    Class 0 : grade >= boundary
    """

    # Binary labels
    #y_train_binary = (y_train < boundary).astype(int)
    #y_test_binary = (y_test < boundary).astype(int)

    # Predictions
    y_pred = classifier.predict(X)
    #test_pred = classifier.predict(X_test)

    # ---------- TRAIN ----------
    print("=" * 70)
    print("PERFORMANCE")
    print("=" * 70)

    print(f"Accuracy : {accuracy_score(y, y_pred):.4f}")
    print(f"Precision: {precision_score(y, y_pred, zero_division=0):.4f}")
    print(f"Recall   : {recall_score(y, y_pred, zero_division=0):.4f}")
    print(f"F1-score : {f1_score(y, y_pred, zero_division=0):.4f}")

    print("\nConfusion Matrix")
    print(confusion_matrix(y, y_pred))

    print("\nClassification Report")
    print(classification_report(y, y_pred,
            target_names=[
                "Grade >= 1.5",
                "Grade < 1.5",
            ],
            digits=4,
            zero_division=0,
        )
    )

class BelowOneRLGradeRegressor:
    """
    Two-stage grade model.

    Stage 1
    -------
    Binary classification:

        class 1: y < 1.0
        class 0: y >= 1.0

    Stage 2
    -------
    RL_GradientBoostingRegressorScratch is trained only on samples
    whose target is between 1.0 and 4.0.

    Final prediction
    ----------------
    If P(y < 1.0) >= low_grade_threshold:
        predict low_grade_value

    Otherwise:
        predict with the RL regressor and clip to [1.0, 4.0].
    """

    def __init__(
        self,
        boundary=1.0,
        low_grade_value=0.0,
        low_grade_threshold=0.70,
        low_class_weight=5.0,

        classifier_learning_rate=0.03,
        classifier_n_estimators=300,
        classifier_max_depth=5,
        classifier_min_child_weight=3,
        classifier_subsample=0.8,
        classifier_colsample_bytree=0.8,
        classifier_reg_alpha=0.1,
        classifier_reg_lambda=3.0,

        rl_learning_rate=0.1,
        rl_n_estimators=100,
        rl_min_samples=10,
        rl_max_depth=4,
        rl_actions=None,
        rl_epsilon=0.30,
        rl_epsilon_decay=0.98,
        rl_min_epsilon=0.02,
        rl_alpha_q=0.1,
        rl_gamma_q=0.9,

        random_state=42,
        verbose=True,
    ):
        self.boundary = float(boundary)
        self.low_grade_value = float(low_grade_value)
        self.low_grade_threshold = float(low_grade_threshold)
        self.low_class_weight = float(low_class_weight)

        self.random_state = random_state
        self.verbose = verbose

        # Stage 1: y <= 1 versus y > 1
        # self.low_grade_classifier = (
        #     HistGradientBoostingClassifier(
        #         learning_rate=classifier_learning_rate,
        #         max_iter=classifier_max_iter,
        #         max_leaf_nodes=classifier_max_leaf_nodes,
        #         min_samples_leaf=classifier_min_samples_leaf,
        #         l2_regularization=classifier_l2_regularization,
        #         random_state=random_state,
        #     )
        # )

        self.low_grade_classifier = XGBClassifier(
            objective="binary:logistic",
            eval_metric="logloss",

            n_estimators=classifier_n_estimators,
            learning_rate=classifier_learning_rate,

            max_depth=classifier_max_depth,
            min_child_weight=classifier_min_child_weight,

            subsample=classifier_subsample,
            colsample_bytree=classifier_colsample_bytree,

            reg_alpha=classifier_reg_alpha,
            reg_lambda=classifier_reg_lambda,

            tree_method="hist",
            random_state=random_state,
            n_jobs=-1,
        )

        if rl_actions is None:
            rl_actions = [0.25, 0.5, 1.0, 1.5, 2.0,]

        # Stage 2: regression only for y >= 1
        self.regressor = (
            RL_GradientBoostingRegressorScratch(
                learning_rate=rl_learning_rate,
                n_estimators=rl_n_estimators,
                min_samples=rl_min_samples,
                max_depth=rl_max_depth,
                actions=rl_actions,
                epsilon=rl_epsilon,
                epsilon_decay=rl_epsilon_decay,
                min_epsilon=rl_min_epsilon,
                alpha_q=rl_alpha_q,
                gamma_q=rl_gamma_q,
                random_state=random_state,
                verbose=verbose,
            )
        )

        self.regression_min = None
        self.regression_max = None
        self.is_fitted = False

    @staticmethod
    def _prepare_X(X):
        X = np.asarray(X, dtype=np.float64,)

        if X.ndim != 2:
            raise ValueError(
                "X must be a two-dimensional array. "
                f"Received shape: {X.shape}"
            )

        return np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0,)

    @staticmethod
    def _prepare_y(y):
        return np.asarray(y, dtype=np.float64,).reshape(-1)

    def fit(self,  X, y, X_val=None, y_val=None,):
        X = self._prepare_X(X)
        y = self._prepare_y(y)

        X_val = self._prepare_X(X_val)
        y_val = self._prepare_y(y_val)

        if len(X) != len(y):
            raise ValueError("X and y have different lengths: " 
                             f"{len(X)} != {len(y)}")

        valid_mask = np.isfinite(y)
        X = X[valid_mask]
        y = y[valid_mask]

        val_mask = np.isfinite(y_val)
        X_val = X_val[val_mask]
        y_val = y_val[val_mask]

        if len(y) == 0:
            raise ValueError("No valid training samples were found.")

        # =====================================================
        # Stage 1: classify y < boundary
        # =====================================================
        low_mask = y < self.boundary
        regular_mask = ~low_mask

        val_low_mask = y_val < self.boundary
        val_regular_mask = ~val_low_mask

        low_count = int(np.sum(low_mask))
        regular_count = int(np.sum(regular_mask))

        if low_count == 0:
            raise ValueError(f"No training targets are below {self.boundary}.")

        if regular_count == 0:
            raise ValueError(f"No training targets are at least {self.boundary}.")

        # Binary target:
        # 1 means grade below 1.5
        # 0 means grade from 1.5 to 4.0
        np.set_printoptions(threshold=np.inf)

        y_is_low = low_mask.astype(np.int64)

        val_y_is_low = val_low_mask.astype(np.int64)

        #print("y_is_low: ", y_is_low)

        #print("val_y_is_low: ", val_y_is_low)

        print("X.shape, y_is_low.shape: ", X.shape, y_is_low.shape)

        print("X_val.shape, val_y_is_low.shape: ", X_val.shape, val_y_is_low.shape)

        positive_count = np.sum(y_is_low == 1)
        negative_count = np.sum(y_is_low == 0)

        scale_pos_weight = (negative_count/ max(positive_count, 1))
        #scale_pos_weight = min(negative_count / positive_count, 10)

        print("Positive samples:", positive_count)
        print("Negative samples:", negative_count)
        print("Raw scale_pos_weight:", scale_pos_weight)
        # Update the classifier parameter before fitting
        self.low_grade_classifier.set_params(scale_pos_weight=scale_pos_weight)

        # Do not pass scale_pos_weight to fit()
        self.low_grade_classifier.fit(X, y_is_low)

        evaluate_low_grade_classifier(
            classifier=self.low_grade_classifier,
            X=X,
            y=y_is_low,
            boundary=1.0,
        )

        evaluate_low_grade_classifier(
            classifier=self.low_grade_classifier,
            X=X_val,
            y=val_y_is_low,
            boundary=1.0,
        )

        # =====================================================
        # Stage 2: regression for y >= boundary only
        # =====================================================
        X_regular = X[regular_mask]
        y_regular = y[regular_mask]

        self.regression_min = max(self.boundary, float(np.min(y_regular)))

        self.regression_max = float(np.max(y_regular))

        X_val_regular = None
        y_val_regular = None

        if X_val is not None and y_val is not None:
            X_val = self._prepare_X(X_val)
            y_val = self._prepare_y(y_val)

            if len(X_val) != len(y_val):
                raise ValueError("X_val and y_val have different lengths: " 
                                 f"{len(X_val)} != {len(y_val)}")

            valid_val_mask = np.isfinite(y_val)

            X_val = X_val[valid_val_mask]
            y_val = y_val[valid_val_mask]

            validation_regular_mask = (y_val >= self.boundary)

            if validation_regular_mask.any():
                X_val_regular = X_val[validation_regular_mask]
                y_val_regular = y_val[validation_regular_mask]
        
        print("X_val_regular.shape, y_val_regular: ", X_val_regular.shape, y_val_regular.shape)
        
        self.regressor.fit(
            X_regular,
            y_regular,
            X_val=X_val_regular,
            y_val=y_val_regular,
        )

        self.is_fitted = True

        if self.verbose:
            print("\nBelow-one RL grade model fitted")
            print(
                f"Classification boundary: "
                f"{self.boundary:.2f}"
            )
            print(
                f"Samples below boundary: {low_count}"
            )
            print(
                f"Samples at/above boundary: "
                f"{regular_count}"
            )
            print(
                f"Low-grade class weight: "
                f"{self.low_class_weight:.2f}"
            )
            print(
                f"Regression range: "
                f"[{self.regression_min:.2f}, "
                f"{self.regression_max:.2f}]"
            )

        return self

    def predict_low_probability(self, X):
        """
        Return P(y < boundary).
        """
        if not self.is_fitted:
            raise RuntimeError("The model must be fitted before prediction.")

        X = self._prepare_X(X)

        probabilities = (self.low_grade_classifier.predict_proba(X))

        low_class_position = np.where(self.low_grade_classifier.classes_ == 1)[0]

        if len(low_class_position) == 0:
            return np.zeros(X.shape[0], dtype=np.float64,)

        return probabilities[:, low_class_position[0],]

    def predict_regular_grade(self, X):
        """
        Predict grades for the 1.0-to-4.0 regression branch.
        """
        if not self.is_fitted:
            raise RuntimeError("The model must be fitted before prediction.")

        X = self._prepare_X(X)

        prediction = self.regressor.predict(X)

        return np.clip(prediction, self.regression_min, self.regression_max)

    def predict(self, X, return_components=False):
        """
        Produce final two-stage predictions.
        """
        low_probability = (self.predict_low_probability(X))

        regular_prediction = (self.predict_regular_grade(X))

        low_decision = (low_probability>= self.low_grade_threshold)

        final_prediction = np.where(low_decision, self.low_grade_value, regular_prediction,)

        if return_components:
            return {
                "prediction": final_prediction,
                "low_probability": low_probability,
                "regular_prediction": regular_prediction,
                "low_decision": low_decision,
            }

        return final_prediction

    def save_model(self, filepath):
        filepath = Path(filepath)
        filepath.parent.mkdir(parents=True, exist_ok=True)

        with open(filepath, "wb") as file:
            pickle.dump(self, file)

    @classmethod
    def load_model(cls, filepath):
        filepath = Path(filepath)

        with open(filepath, "rb") as file:
            model = pickle.load(file)

        if not isinstance(model, cls):
            raise TypeError(
                f"Expected {cls.__name__}, got {type(model).__name__}"
            )
        return model

def compute_rmse(y_true, y_pred):
    return np.sqrt(np.mean((np.array(y_true) - np.array(y_pred)) ** 2))


def evaluate_model(y_true, y_pred):
    mae = mean_absolute_error(y_true, y_pred)
    rmse = compute_rmse(y_true, y_pred)
    r2 = r2_score(y_true, y_pred)
    return mae, rmse, r2


def convert_grade10_to_grade4(y10):
    is_scalar = np.isscalar(y10)
    y10_arr = np.atleast_1d(y10)
    y4 = np.zeros_like(y10_arr, dtype=float)
    y4 = np.where(y10_arr >= 9.0, 4.0, y4)
    y4 = np.where((y10_arr >= 8.5) & (y10_arr < 9.0), 3.7, y4)
    y4 = np.where((y10_arr >= 8.0) & (y10_arr < 8.5), 3.5, y4)
    y4 = np.where((y10_arr >= 7.0) & (y10_arr < 8.0), 3.0, y4)
    y4 = np.where((y10_arr >= 6.0) & (y10_arr < 7.0), 2.5, y4)
    y4 = np.where((y10_arr >= 5.5) & (y10_arr < 6.0), 2.0, y4)
    y4 = np.where((y10_arr >= 5.0) & (y10_arr < 5.5), 1.5, y4)
    y4 = np.where((y10_arr >= 4.0) & (y10_arr < 5.0), 1.0, y4)
    y4 = np.where(y10_arr < 4.0, 0.0, y4)
    if is_scalar:
        return float(y4[0])
    return y4


def train_on_split(train_path, test_path, args, fac_key, fold_name, models_dir, plots_dir, exclude_cols):
    if not train_path.exists() or not test_path.exists():
        print(f"Skipping {fold_name} - Split files not found.")
        return None

    train_df = pl.read_csv(train_path)
    test_df = pl.read_csv(test_path)

    # Cast target columns to Float64 explicitly, parsing empty strings or invalid strings to null
    train_df = train_df.with_columns([
        pl.col("target_grade10").cast(pl.Float64, strict=False),
        pl.col("target_grade4").cast(pl.Float64, strict=False)
    ])
    test_df = test_df.with_columns([
        pl.col("target_grade10").cast(pl.Float64, strict=False),
        pl.col("target_grade4").cast(pl.Float64, strict=False)
    ])

    # Filter out rows with null values in target column
    train_df = train_df.filter(pl.col(args.target).is_not_null() & pl.col("target_grade4").is_not_null())
    test_df = test_df.filter(pl.col(args.target).is_not_null() & pl.col("target_grade4").is_not_null())

    # Check size threshold
    if len(train_df) < 50:
        print(f"Skipping {fold_name} - Insufficient training samples ({len(train_df)}). Minimum threshold is 50.")
        return None

    # Identify feature columns
    feature_cols = [col for col in train_df.columns if col not in exclude_cols]
    print(f"  {fold_name} Size - Train: {len(train_df)} | Test: {len(test_df)} | Features: {len(feature_cols)}")

    # Cast any string/null feature columns to Float64
    for col in feature_cols:
        if train_df[col].dtype in (pl.String, pl.Null):
            train_df = train_df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
            test_df = test_df.with_columns(pl.col(col).cast(pl.Float64, strict=False))

    # Detect any categorical columns
    cat_cols = [col for col in feature_cols if train_df[col].dtype in (pl.String, pl.Categorical)]

    X_train_pd = pd.DataFrame(train_df.select(feature_cols).to_dict(as_series=False))

    X_test_pd = pd.DataFrame(test_df.select(feature_cols).to_dict(as_series=False))

    # Force convert all non-categorical feature columns to float in Pandas
    for col in feature_cols:
        if col not in cat_cols:
            X_train_pd[col] = pd.to_numeric(X_train_pd[col], errors='coerce').astype(float)
            X_test_pd[col] = pd.to_numeric(X_test_pd[col], errors='coerce').astype(float)

    # DROP_COLUMNS = sorted(set(DROP_MISSING_COLUMNS + DROP_CONSTANT_COLUMNS))
    #
    # print(f"Removing {len(DROP_COLUMNS)} columns")
    #
    # X_train_pd = X_train_pd.drop(columns=DROP_COLUMNS, errors="ignore")
    # X_test_pd = X_test_pd.drop(columns=DROP_COLUMNS, errors="ignore")

    y_train = train_df[args.target].to_numpy()
    y_test = test_df[args.target].to_numpy()
    y_train_g4 = train_df["target_grade4"].to_numpy()
    y_test_g4 = test_df["target_grade4"].to_numpy()

    # Apply one-hot encoding if categorical features are detected
    if cat_cols:
        encoder = OneHotEncoder(handle_unknown='ignore', sparse_output=False)
        for c in cat_cols:
            X_train_pd[c] = X_train_pd[c].fillna("missing")
            X_test_pd[c] = X_test_pd[c].fillna("missing")

        encoded_train = encoder.fit_transform(X_train_pd[cat_cols])
        encoded_test = encoder.transform(X_test_pd[cat_cols])

        encoded_cols = encoder.get_feature_names_out(cat_cols)
        encoded_train_df = pd.DataFrame(encoded_train, columns=encoded_cols, index=X_train_pd.index)
        encoded_test_df = pd.DataFrame(encoded_test, columns=encoded_cols, index=X_test_pd.index)

        X_train_pd = X_train_pd.drop(columns=cat_cols).join(encoded_train_df)
        X_test_pd = X_test_pd.drop(columns=cat_cols).join(encoded_test_df)
        feature_cols = list(X_train_pd.columns)

    # Determine number of estimators
    #n_est = args.n_estimators if args.n_estimators is not None else (100 if args.val_ratio > 0.0 else 25)

    model = BelowOneRLGradeRegressor(
        boundary=1.0,
        low_grade_threshold=LOW_GRADE_THRESHOLD,
        low_class_weight=5.0,

        classifier_learning_rate=0.02,
        classifier_n_estimators=500,
        classifier_max_depth=4,
        classifier_min_child_weight=5,
        classifier_subsample=0.8,
        classifier_colsample_bytree=0.8,
        classifier_reg_alpha=0.2,
        classifier_reg_lambda=5.0,

        # RL regressor settings
        rl_learning_rate=N_RATE,
        rl_n_estimators=N_EST,
        rl_min_samples=10,
        rl_max_depth=4,
        rl_actions=[0.25, 0.5, 1.0, 1.5, 2.0],

        random_state=42,
        verbose=True,
    )

    # Model training and fitting
    if args.val_ratio > 0.0:
        if args.target == "target_grade10" and args.convert_to_grade4:
            X_tr, X_val, y_tr, y_val, y_tr_g4, y_val_g4 = train_test_split(
                X_train_pd, y_train, y_train_g4, test_size=args.val_ratio, random_state=42
            )
        else:
            X_tr, X_val, y_tr, y_val = train_test_split(
                X_train_pd, y_train, test_size=args.val_ratio, random_state=42
            )

        model.fit(X_train_pd, y_train)

        # Predictions
        y_pred_tr = model.predict(X_tr)
        y_pred_val = model.predict(X_val)
        y_pred_te = model.predict(X_test_pd)

        # Metrics
        mae_tr, rmse_tr, r2_tr = evaluate_model(y_tr, y_pred_tr)
        mae_val, rmse_val, r2_val = evaluate_model(y_val, y_pred_val)
        mae_te, rmse_te, r2_te = evaluate_model(y_test, y_pred_te)

        # Converted Grade 4 Metrics
        if args.target == "target_grade10" and args.convert_to_grade4:
            y_pred_tr_g4 = convert_grade10_to_grade4(y_pred_tr)
            y_pred_val_g4 = convert_grade10_to_grade4(y_pred_val)
            y_pred_te_g4 = convert_grade10_to_grade4(y_pred_te)

            mae_tr_g4, rmse_tr_g4, r2_tr_g4 = evaluate_model(y_tr_g4, y_pred_tr_g4)
            mae_val_g4, rmse_val_g4, r2_val_g4 = evaluate_model(y_val_g4, y_pred_val_g4)
            mae_te_g4, rmse_te_g4, r2_te_g4 = evaluate_model(y_test_g4, y_pred_te_g4)
        else:
            mae_tr_g4, rmse_tr_g4, r2_tr_g4 = np.nan, np.nan, np.nan
            mae_val_g4, rmse_val_g4, r2_val_g4 = np.nan, np.nan, np.nan
            mae_te_g4, rmse_te_g4, r2_te_g4 = np.nan, np.nan, np.nan

        val_size = len(X_val)
        train_size = len(X_tr)

    else:
        X_train_pd.to_csv("X_train_pd.csv", index=False)

        X_test_pd.to_csv("X_test_pd.csv", index=False)

        feature_std = X_train_pd.std()

        print("Number of features:", len(feature_std))

        print("Zero-variance features:",(feature_std == 0).sum())

        print("Features with std < 1e-6:",(feature_std < 1e-6).sum())

        print(feature_std.describe())

        np.savetxt("y_train_g4.csv", y_train_g4, delimiter=",")

        np.savetxt("y_test_g4.csv", y_test_g4, delimiter=",")

        model.fit(X_train_pd, y_train_g4, X_test_pd, y_test_g4)

        # Predictions
        y_pred_tr = model.predict(X_train_pd)

        y_pred_te = model.predict(X_test_pd)

        # Metrics
        mae_tr, rmse_tr, r2_tr = evaluate_model(y_train_g4, y_pred_tr)
        mae_val, rmse_val, r2_val = np.nan, np.nan, np.nan
        mae_te, rmse_te, r2_te = evaluate_model(y_test_g4, y_pred_te)

        # Converted Grade 4 Metrics
        if args.target == "target_grade10" and args.convert_to_grade4:
            y_pred_tr_g4 = convert_grade10_to_grade4(y_pred_tr)
            y_pred_te_g4 = convert_grade10_to_grade4(y_pred_te)

            mae_tr_g4, rmse_tr_g4, r2_tr_g4 = evaluate_model(y_train_g4, y_pred_tr_g4)
            mae_val_g4, rmse_val_g4, r2_val_g4 = np.nan, np.nan, np.nan
            mae_te_g4, rmse_te_g4, r2_te_g4 = evaluate_model(y_test_g4, y_pred_te_g4)
        else:
            mae_tr_g4, rmse_tr_g4, r2_tr_g4 = np.nan, np.nan, np.nan
            mae_val_g4, rmse_val_g4, r2_val_g4 = np.nan, np.nan, np.nan
            mae_te_g4, rmse_te_g4, r2_te_g4 = np.nan, np.nan, np.nan

        val_size = 0
        train_size = len(X_train_pd)

    # Save model weights
    fold_suffix = f"_{fold_name}" if fold_name != "standard" else ""
    model_path = models_dir / f"{fac_key}{fold_suffix}_model.json"
    model.save_model(str(model_path))


    # Print metrics to console
    print(f"    Evaluation Metrics for {fold_name}:")
    print(f"      Grade 10 Scale:")
    print(f"        Train - MAE: {mae_tr:.4f} | RMSE: {rmse_tr:.4f} | R2: {r2_tr:.4f}")
    if args.val_ratio > 0.0:
        print(f"        Val   - MAE: {mae_val:.4f} | RMSE: {rmse_val:.4f} | R2: {r2_val:.4f}")
    print(f"        Test  - MAE: {mae_te:.4f} | RMSE: {rmse_te:.4f} | R2: {r2_te:.4f}")

    if args.target == "target_grade10" and args.convert_to_grade4:
        print(f"      Converted Grade 4 Scale:")
        print(f"        Train - MAE: {mae_tr_g4:.4f} | RMSE: {rmse_tr_g4:.4f} | R2: {r2_tr_g4:.4f}")
        if args.val_ratio > 0.0:
            print(f"        Val   - MAE: {mae_val_g4:.4f} | RMSE: {rmse_val_g4:.4f} | R2: {r2_val_g4:.4f}")
        print(f"        Test  - MAE: {mae_te_g4:.4f} | RMSE: {rmse_te_g4:.4f} | R2: {r2_te_g4:.4f}")

    # Generate predictions on full train set if saving or plotting train distribution
    if args.plot_train_dist or args.save_train_results:
        y_pred_tr_full = model.predict(X_train_pd)
        if args.target == "target_grade10" and args.convert_to_grade4:
            y_pred_tr_full_g4 = convert_grade10_to_grade4(y_pred_tr_full)

    # Plot train distribution
    if args.plot_train_dist:
        if args.target == "target_grade10" and args.convert_to_grade4:
            fig, axes = plt.subplots(2, 2, figsize=(14, 11))

            # Row 0: Grade 10
            bins10 = np.linspace(0, 10, 41)
            axes[0, 0].hist(y_train, bins=bins10, alpha=0.5, label='Actual', color='#2ca02c', edgecolor='k',
                            linewidth=0.5)
            axes[0, 0].hist(y_pred_tr_full, bins=bins10, alpha=0.5, label='Predicted', color='#1f77b4', edgecolor='k',
                            linewidth=0.5)
            axes[0, 0].set_xlabel("Grade (0-10)")
            axes[0, 0].set_ylabel("Count")
            axes[0, 0].set_title("Grade 10 Distribution Comparison (Train)", fontsize=11, fontweight='bold')
            axes[0, 0].legend()
            axes[0, 0].grid(axis='y', linestyle='--', alpha=0.5)

            axes[0, 1].scatter(y_train, y_pred_tr_full, alpha=0.3, color='#1f77b4', edgecolors='none', s=20)
            axes[0, 1].plot([0, 10], [0, 10], 'r--', label='Ideal Fit (y = x)')
            axes[0, 1].set_xlabel("Actual Grade 10")
            axes[0, 1].set_ylabel("Predicted Grade 10")
            axes[0, 1].set_xlim(-0.2, 10.2)
            axes[0, 1].set_ylim(-0.2, 10.2)
            axes[0, 1].set_title("Grade 10 Actual vs Predicted (Train)", fontsize=11, fontweight='bold')
            axes[0, 1].legend()
            axes[0, 1].grid(linestyle='--', alpha=0.5)

            # Row 1: Grade 4
            bins4 = np.linspace(0, 4, 17)
            axes[1, 0].hist(y_train_g4, bins=bins4, alpha=0.5, label='Actual', color='#2ca02c', edgecolor='k',
                            linewidth=0.5)
            axes[1, 0].hist(y_pred_tr_full_g4, bins=bins4, alpha=0.5, label='Predicted', color='#1f77b4', edgecolor='k',
                            linewidth=0.5)
            axes[1, 0].set_xlabel("Grade (0-4)")
            axes[1, 0].set_ylabel("Count")
            axes[1, 0].set_title("Grade 4 Distribution Comparison (Train)", fontsize=11, fontweight='bold')
            axes[1, 0].legend()
            axes[1, 0].grid(axis='y', linestyle='--', alpha=0.5)

            # Add small random jitter for discrete grade4 visualization
            jitter_actual = y_train_g4 + np.random.normal(0, 0.05, size=len(y_train_g4))
            jitter_pred = y_pred_tr_full_g4 + np.random.normal(0, 0.05, size=len(y_pred_tr_full_g4))
            axes[1, 1].scatter(jitter_actual, jitter_pred, alpha=0.3, color='#1f77b4', edgecolors='none', s=20)
            axes[1, 1].plot([0, 4], [0, 4], 'r--', label='Ideal Fit (y = x)')
            axes[1, 1].set_xlabel("Actual Grade 4 (Jittered)")
            axes[1, 1].set_ylabel("Predicted Grade 4 (Jittered)")
            axes[1, 1].set_xlim(-0.2, 4.2)
            axes[1, 1].set_ylim(-0.2, 4.2)
            axes[1, 1].set_title("Grade 4 Actual vs Predicted (Train)", fontsize=11, fontweight='bold')
            axes[1, 1].legend()
            axes[1, 1].grid(linestyle='--', alpha=0.5)

            plt.suptitle(
                f"Train Prediction Analysis - {fac_key} ({fold_name})\nTarget: {args.target} (with Grade 4 conversion)",
                fontsize=13, fontweight='bold')
            plt.tight_layout()
        else:
            fig, axes = plt.subplots(1, 2, figsize=(14, 6))
            target_name = args.target
            is_g10 = (target_name == "target_grade10")
            max_val = 10.0 if is_g10 else 4.0
            bins = np.linspace(0, max_val, 41)

            # Hist
            axes[0].hist(y_train, bins=bins, alpha=0.5, label='Actual', color='#2ca02c', edgecolor='k', linewidth=0.5)
            axes[0].hist(y_pred_tr_full, bins=bins, alpha=0.5, label='Predicted', color='#1f77b4', edgecolor='k',
                         linewidth=0.5)
            axes[0].set_xlabel(f"Grade ({'0-10' if is_g10 else '0-4'})")
            axes[0].set_ylabel("Count")
            axes[0].set_title("Distribution Comparison (Train)", fontsize=11, fontweight='bold')
            axes[0].legend()
            axes[0].grid(axis='y', linestyle='--', alpha=0.5)

            # Scatter
            axes[1].scatter(y_train, y_pred_tr_full, alpha=0.3, color='#1f77b4', edgecolors='none', s=20)
            axes[1].plot([0, max_val], [0, max_val], 'r--', label='Ideal Fit (y = x)')
            axes[1].set_xlabel("Actual Grade")
            axes[1].set_ylabel("Predicted Grade")
            axes[1].set_xlim(-0.2, max_val + 0.2)
            axes[1].set_ylim(-0.2, max_val + 0.2)
            axes[1].set_title("Actual vs Predicted (Train)", fontsize=11, fontweight='bold')
            axes[1].legend()
            axes[1].grid(linestyle='--', alpha=0.5)

            plt.suptitle(
                f"Train Prediction Analysis - {fac_key} ({fold_name})\nTarget: {target_name} | RMSE: {compute_rmse(y_train, y_pred_tr_full):.4f}",
                fontsize=13, fontweight='bold')
            plt.tight_layout()

        dist_plot_path = plots_dir / f"{fac_key}{fold_suffix}_train_dist.png"
        plt.savefig(dist_plot_path, dpi=150)
        plt.close()
        print(f"      Saved train distribution plot to {dist_plot_path}")

    # Save train results
    if args.save_train_results:
        train_results_df = pl.DataFrame({
            "IDSV": train_df["IDSV"],
            "IDMH": train_df["IDMH"],
            "target_NHHK": train_df["target_NHHK"],
            "target_grade10": train_df["target_grade10"],
            "target_grade4": train_df["target_grade4"]
        })
        if args.target == "target_grade10":
            train_results_df = train_results_df.with_columns([
                pl.Series("prediction_grade10", y_pred_tr_full)
            ])
            if args.convert_to_grade4:
                train_results_df = train_results_df.with_columns([
                    pl.Series("prediction_grade4", y_pred_tr_full_g4)
                ])
        else:
            train_results_df = train_results_df.with_columns([
                pl.Series("prediction_grade4", y_pred_tr_full)
            ])

        train_results_path = models_dir / f"{fac_key}{fold_suffix}_train_results.csv"
        train_results_df.write_csv(train_results_path)
        print(f"      Saved train results to {train_results_path}")

    # Save test results
    test_results_df = pl.DataFrame({
        "IDSV": test_df["IDSV"],
        "IDMH": test_df["IDMH"],
        "target_NHHK": test_df["target_NHHK"],
        "target_grade10": test_df["target_grade10"],
        "target_grade4": test_df["target_grade4"]
    })

    if args.target == "target_grade10":
        test_results_df = test_results_df.with_columns([
            pl.Series("prediction_grade10", y_pred_te)
        ])
        if args.convert_to_grade4:
            test_results_df = test_results_df.with_columns([
                pl.Series("prediction_grade4", y_pred_te_g4)
            ])
    else:
        test_results_df = test_results_df.with_columns([
            pl.Series("prediction_grade4", y_pred_te)
        ])

    test_results_path = models_dir / f"{fac_key}{fold_suffix}_test_results.csv"
    test_results_df.write_csv(test_results_path)
    print(f"      Saved test results to {test_results_path}")

    return {
        "fold": fold_name,
        "train_size": train_size,
        "val_size": val_size,
        "test_size": len(X_test_pd),
        "train_mae": mae_tr,
        "train_rmse": rmse_tr,
        "train_r2": r2_tr,
        "val_mae": mae_val,
        "val_rmse": rmse_val,
        "val_r2": r2_val,
        "test_mae": mae_te,
        "test_rmse": rmse_te,
        "test_r2": r2_te,
        "train_mae_g4": mae_tr_g4,
        "train_rmse_g4": rmse_tr_g4,
        "train_r2_g4": r2_tr_g4,
        "val_mae_g4": mae_val_g4,
        "val_rmse_g4": rmse_val_g4,
        "val_r2_g4": r2_val_g4,
        "test_mae_g4": mae_te_g4,
        "test_rmse_g4": rmse_te_g4,
        "test_r2_g4": r2_te_g4
    }


def run_training_for_config(strategy, exclude_t_minus_1, args, exclude_cols):
    t_minus_1_dir = "exclude_t_1" if exclude_t_minus_1 else "include_t_1"

    splits_base = Path(ROOT) / args.splits_dir / t_minus_1_dir / strategy
    models_dir = ROOT / args.models_dir / t_minus_1_dir / strategy
    plots_dir = ROOT / args.plots_dir / t_minus_1_dir / strategy

    os.makedirs(models_dir, exist_ok=True)
    os.makedirs(plots_dir, exist_ok=True)

    print("-" * 60)
    print(f"Executing XGBoost Model Training configuration:")
    print(f"  Strategy: {strategy}")
    print(f"  Splits Base: {splits_base}")
    print(f"  Models Dir: {models_dir}")
    print(f"  Exclude T-1: {exclude_t_minus_1}")

    if not splits_base.exists():
        print(f"Error: Splits directory not found at {splits_base}")
        return

    # Identify all unique faculty names in splits
    fold_dirs = list(splits_base.glob("fold_*"))
    if not fold_dirs:
        print(f"No fold subdirectories found in {splits_base}")
        return
    train_files = list(fold_dirs[0].glob("*_train.csv"))
    faculties = {f.name.replace("_train.csv", "") for f in train_files}

    if args.faculty != "all":
        if args.faculty not in faculties:
            print(f"Warning: Faculty key '{args.faculty}' not found in splits. Skipping this configuration.")
            return
        faculties = {args.faculty}

    print(f"Processing faculties: {list(faculties)}")

    results = []

    for fac_key in faculties:
        print(f"  Faculty: {fac_key}")
        fac_results = []

        fold_dirs = sorted(list(splits_base.glob("fold_*")), key=lambda x: int(x.name.split("_")[1]))
        for fold_dir in fold_dirs:
            fold_name = fold_dir.name
            train_path = fold_dir / f"{fac_key}_train.csv"
            test_path = fold_dir / f"{fac_key}_test.csv"

            res = train_on_split(train_path, test_path, args, fac_key, fold_name, models_dir, plots_dir, exclude_cols)
            if res:
                fac_results.append(res)

        if not fac_results:
            continue

        df_res = pl.DataFrame(fac_results)
        metrics_path = models_dir / f"{fac_key}_metrics_summary.csv"
        df_res.write_csv(metrics_path)

        for r in fac_results:
            r["faculty"] = fac_key
            results.append(r)

    if results:
        summary_df = pl.DataFrame(results)
        summary_path = models_dir / "overall_evaluation_summary.csv"

        if summary_path.exists():
            print(f"  Existing performance summary found at {summary_path}. Performing non-destructive update...")
            try:
                existing_df = pl.read_csv(summary_path)
                trained_facs = summary_df["faculty"].unique().to_list()
                existing_df_filtered = existing_df.filter(~pl.col("faculty").is_in(trained_facs))
                summary_df = pl.concat([existing_df_filtered, summary_df], how="diagonal")
            except Exception as e:
                print(f"  Warning: Could not merge with existing summary: {e}. Overwriting instead.")

        summary_df.write_csv(summary_path)
        print(f"Consolidated performance summary saved to {summary_path}")


def parse_args():
    parser = argparse.ArgumentParser(
        description="XGBoost Training and Evaluation Pipeline (supports expanding and sliding CV)")
    parser.add_argument(
        "--strategy",
        type=str,
        default="all",
        choices=["all", "expanding", "sliding"],
        help="Temporal partitioning strategy of splits to train on or 'all' to run both."
    )
    parser.add_argument(
        "--exclude_t_minus_1",
        action="store_true",
        help="Train models on splits that excluded the T-1 semester."
    )
    parser.add_argument(
        "--splits_dir",
        type=str,
        default="dataset/splits",
        help="Input splits folder containing train/test splits."
    )
    parser.add_argument(
        "--faculty",
        type=str,
        default="khoa_cong_nghe_thong_tin",
        help="Faculty key to train (e.g. 'khoa_cong_nghe_thong_tin') or 'all' to train for all faculties."
    )
    parser.add_argument(
        "--target",
        type=str,
        default="target_grade10",
        choices=["target_grade10", "target_grade4"],
        help="Target column variable for regression."
    )
    parser.add_argument(
        "--convert_to_grade4",
        action="store_true",
        help="Convert grade 10 predictions to grade 4 and evaluate on target_grade4."
    )
    parser.add_argument(
        "--val_ratio",
        type=float,
        default=0.0,
        help="Fraction of training data to randomly hold out for validation/early stopping. Set to 0 to disable."
    )
    parser.add_argument(
        "--n_estimators",
        type=int,
        default=None,
        help="Number of trees. Defaults to 1000 if using early stopping, or 150 if not."
    )
    parser.add_argument(
        "--early_stopping",
        type=int,
        default=50,
        help="Number of early stopping rounds."
    )
    parser.add_argument(
        "--models_dir",
        type=str,
        default="results/rl_gb/" + f"{N_RATE}/{N_EST}/v7_4/{LOW_GRADE_THRESHOLD}",
        help="Directory where trained models and summaries will be saved."
    )
    parser.add_argument(
        "--plots_dir",
        type=str,
        default="results/plots",
        help="Directory where feature importance plots will be saved."
    )
    parser.add_argument(
        "--no_run_all",
        action="store_true",
        help="Disable training models sequentially on all configurations."
    )
    parser.add_argument(
        "--plot_train_dist",
        action="store_true",
        help="Plot the distribution of actual vs predicted target values for the train data."
    )
    parser.add_argument(
        "--save_train_results",
        action="store_true",
        help="Save the numerical prediction results of the train data to a CSV file."
    )
    return parser.parse_args()


def main():
    args = parse_args()

    exclude_cols = {
        "dataset_split",
        "student_faculty_key",
        "student_faculty_name",
        "IDSV",
        "target_NHHK",
        "target_semester_ordinal",
        "is_summer_target",
        "IDKQDK",
        "IDToHoc",
        "IDMH",
        "target_IDCongDiem",
        "target_grade10",
        "target_grade4",
        "target_status",
        "base_row_id",
        "course_cluster_label"
    }

    # Check if a custom parameter was specified
    is_custom_splits = args.splits_dir != "dataset/splits"
    is_custom_models = args.models_dir != "results"
    is_custom_plots = args.plots_dir != "results/plots"
    is_single_run = args.exclude_t_minus_1 or is_custom_splits or is_custom_models or is_custom_plots or args.no_run_all or args.strategy != "all"

    if is_single_run:
        strategy_to_run = "expanding" if args.strategy == "all" else args.strategy
        run_training_for_config(
            strategy=strategy_to_run,
            exclude_t_minus_1=args.exclude_t_minus_1,
            args=args,
            exclude_cols=exclude_cols
        )
    else:
        # Run all combinations sequentially (2 strategies x 2 lag options)
        combinations = [
            ("expanding", False),
            ("expanding", True),
            ("sliding", False),
            ("sliding", True)
        ]

        for strategy, excl_t in combinations:
            print("\n" + "=" * 80)
            print(f"Executing Model Training: Strategy={strategy}, T-1={excl_t}")
            print("=" * 80)
            run_training_for_config(
                strategy=strategy,
                exclude_t_minus_1=excl_t,
                args=args,
                exclude_cols=exclude_cols
            )


if __name__ == "__main__":
    main()

