# -*- coding: utf-8 -*-
"""
convert_cnd_to_superhuge.py
===========================

Convert a folder of CND-format AAD datasets (dataSub<N>.mat / dataStim<N>.mat,
as stored under e.g. ``C:/Users/sean/Downloads/data-Yuanming``) into the
*SuperHuge* dataset layout (the on-disk format of ``E:/derivatives/SuperHuge``).

SuperHuge layout produced here. Both SuperHuge stage directories are written
by default (each dataset keeps the same dataset_id across stages)::

    <out_root>/raw/eeg/dataset-<DDD>-subject-<SSS>-trial-<TTT>.npy
    <out_root>/raw/stimuli/env/dataset-<DDD>-subject-<SSS>-trial-<TTT>_env.npy
    <out_root>/raw/meta/metadata.pkl
    <out_root>/preprocessed/...            (same layout, dataset names *_preprocessed)

* The ``raw`` stage is produced from the ``dataSub<N>.mat``/``dataStim<N>.mat``
  files; whenever a ``dataCND_resampled`` sibling exists (USTC, Ultra-HD) that
  resampled version is used.
* The ``preprocessed`` stage is produced from the ``pre_dataSub<N>.mat`` /
  ``pre_dataStim<N>.mat`` files that ship next to the raw files. These are the
  pre-processed CND variants (1-9 Hz band-pass + downsampling to fs = 32 Hz by
  the KUL CND toolchain); their actual fs is stored verbatim in the metadata.
  No extra EEG cleaning is applied by this tool.

SuperHuge data conventions honoured by this converter
-----------------------------------------------------
* EEG arrays are stored as ``float`` NumPy arrays of shape ``(T, C)``
  (time-major) written in **Fortran order** (as on disk in SuperHuge).
* Envelope arrays are stored as ``(T, 1, K)`` arrays (``K`` = number of
  concurrent speakers / streams).
* **The attended envelope is always written to column 0** of the envelope
  array (``env[:, 0, 0]``), exactly as in the existing SuperHuge files
  (verified: for static-attention trials ``corr(col0, attended) ~ 1.0``).
  Columns 1..K-1 keep their original source order.
* Trial-level metadata entries strictly follow the SuperHuge ``metadata.pkl``
  row schema of the same dataset name (raw / preprocessed reference rows):
  ``dataset_id / subject_id / trial_id / signal_length / num_channel / fs /
  dataset_name / channel_infos / num_speakers / extra_info``, ``label`` (when
  the reference row has one), ``stimuli_name / compet_stimuli_name`` and the
  feature availability flags. No extra fields are added.

Feature flags & scaling files
-----------------------------
* Only the envelope feature is produced here. The feature flag keys present in
  the reference rows are always written with truthful values
  (``wav:0, env:1, env_fs:<fs>, mel:0, wav2vec2:0``); ``<feature>_fs`` keys
  are written only for features that are actually provided (never here).
* ``stimuli_name`` / ``compet_stimuli_name`` are kept as **empty-string
  placeholders** (the CND files do not expose story names).
* ``extra_info`` is strictly ``{}``; per-sample attention ground truth stays in
  the source CND files and is NOT stored in metadata.
* ``scaling_factor.pkl`` and ``channelwise_scaling_factor.pkl`` are **not**
  produced: they are normalisation statistics computed by the SuperHuge build
  pipeline over the full corpus, not part of the raw conversion.

Label conventions
-----------------
The trial-level ``label`` is derived from the *attended azimuth at trial onset*
(the azimuth of the attended stream at sample 0), which reproduces the
existing SuperHuge labels for static paradigms (e.g. azimuth -90 -> "left"/"L")
and is well defined for trials whose azimuth pans across +/-90 (e.g.
AV-GC/DLAAD style stimuli). The label string format is controlled per dataset
(``label_style``): ``left_right`` ("left"/"right"), ``LR`` ("L"/"R") or
``azimuth_str`` (e.g. "-90"). For datasets that have no meaningful attended
azimuth (``label_style="none"``), the ``label`` field is omitted, matching the
existing KUL-DLAAD entries. Label values and channel names keep the source CND
values (no re-labelling / re-naming).

Source CND variants handled
---------------------------
* MATLAB v5/v7 ``.mat`` (scipy.io) and MATLAB v7.3 HDF5 ``.mat`` (h5py).
* Per-subject ``dataStim<N>.mat`` files or a single shared ``dataStim.mat``.
* ``dataCND`` folders whose native fs != target fs (e.g. USTC @ 250 Hz or the
  Ultra-HD files @ 250 Hz) are skipped in favour of the sibling
  ``dataCND_resampled`` folder when present (target fs = 128 Hz by default).
* v7.3 EEG trials are stored channel-major ``(C, T)`` by MATLAB and are
  transposed to time-major ``(T, C)`` automatically.

Dataset registry / numbering
----------------------------
Every source folder is mapped through :data:`DATASETS`. Dataset ids here are a
*fresh, independent* numbering (1..N in registry order) and are NOT the ids
used by the existing ``E:/derivatives/SuperHuge`` metadata. ``orig_dataset_id``
records which existing SuperHuge dataset the folder corresponds to (purely
informational provenance). Names are written as ``<name>_<stage>``, e.g.
``KUL_raw`` (stage "raw"), exactly like existing SuperHuge dataset names.

USAGE
-----
    python convert_cnd_to_superhuge.py --source-root <src> --out-root <dst>
    python convert_cnd_to_superhuge.py --datasets DTU2018,KUL2016   # subset
    python convert_cnd_to_superhuge.py --stage preprocessed         # only pre_* files
    python convert_cnd_to_superhuge.py --subjects 1,2              # subset of subjects
    python convert_cnd_to_superhuge.py --metadata-only --overwrite # re-write metadata.pkl
    python convert_cnd_to_superhuge.py --overwrite

Dependencies: numpy, scipy (for classic .mat), h5py (for v7.3 .mat).
"""

from __future__ import annotations

import argparse
import os
import pickle
import re
import sys
from typing import Any, Dict, List, Optional, Sequence, Tuple

import numpy as np

# ---------------------------------------------------------------------------
# 0. Configuration
# ---------------------------------------------------------------------------

#: Files produced for a single trial are written as
#: ``<stage>/eeg/dataset-%03d-subject-%03d-trial-%03d.npy``.
FILE_KEY = "dataset-{did:03d}-subject-{sid:03d}-trial-{tid:03d}"

# ---------------------------------------------------------------------------
# DATASETS
# ---------------------------------------------------------------------------
# Every entry describes one folder under the source root:
#   folder             : sub-folder name of the CND data (under <source-root>)
#   dataset_id         : fresh independent dataset id used for the output
#   name               : base dataset name (stage suffix "_raw" is appended)
#   orig_dataset_id    : informational only - id of the *same* dataset in the
#                        existing E:/derivatives/SuperHuge registry
#   label_style        : "left_right" | "LR" | "azimuth_str" | "none"
#   drop_channels      : channel names (as in the CND chanlocs) to remove from
#                        EEG + channel_infos (e.g. external/extra electrodes)
#   dtype              : numpy dtype used for eeg and env output (float32 to
#                        halve disk usage, float64 to match existing SuperHuge)
#   candidate_subdirs  : CND sub-folders probed in order; the first one whose
#                        neural fs == target fs is used.
#
# Subject ids are ALWAYS renumbered contiguously (1..N in file-number order),
# matching the reference SuperHuge rows whose subject_id is contiguous 1..N.
# NOTE: names/ids below are *defaults*. Adjust freely before running - the
# converter is driven exclusively by this table plus the CLI arguments.
DATASETS: List[Dict[str, Any]] = [
    dict(folder="AVGCAAD2024",         dataset_id=1, name="KUL-AV-GC",
         orig_dataset_id=5,  label_style="left_right",
         drop_channels=[], dtype=np.float64,
         candidate_subdirs=("dataCND",)),
    dict(folder="DLAAD2021",           dataset_id=2, name="KUL-DLAAD",
         orig_dataset_id=15, label_style="none",
         drop_channels=[], dtype=np.float64,
         candidate_subdirs=("dataCND",)),
    dict(folder="DTU2018",             dataset_id=3, name="DTU",
         orig_dataset_id=2,  label_style="LR",
         drop_channels=[], dtype=np.float64,
         candidate_subdirs=("dataCND",)),
    dict(folder="KUL2016",             dataset_id=4, name="KUL",
         orig_dataset_id=4,  label_style="left_right",
         drop_channels=[], dtype=np.float64,
         candidate_subdirs=("dataCND",)),
    dict(folder="KUL_AAD_UltraHD2021", dataset_id=5, name="KUL-HighDensity",
         orig_dataset_id=14, label_style="LR",
         drop_channels=["XtraL", "XtraR"], dtype=np.float64,
         candidate_subdirs=("dataCND_resampled", "dataCND")),
    dict(folder="NJUSUS2026",          dataset_id=6,
         name="NJU-SUSTech-ICASSP2027GC-GC-AS-BlindTestSet",
         orig_dataset_id=13, label_style="azimuth_str",
         drop_channels=[], dtype=np.float64,
         candidate_subdirs=("dataCND",)),
    dict(folder="USTC2026",            dataset_id=7, name="USTC",
         orig_dataset_id=12, label_style="LR", subject_renumber=False,
         drop_channels=[], dtype=np.float64,
         candidate_subdirs=("dataCND_resampled", "dataCND")),
]

STAGES = ("raw", "preprocessed")   # SuperHuge stage dirs written by default
TARGET_FS = 128                    # expected rate of the (non pre_) CND data
SUBJECT_FILE_RE = re.compile(r"^dataSub(\d+)\.mat$")
PRE_SUBJECT_FILE_RE = re.compile(r"^pre_dataSub(\d+)\.mat$")

# ---------------------------------------------------------------------------
# 1. Small helpers
# ---------------------------------------------------------------------------


def log(msg: str) -> None:
    print(msg, flush=True)


def _decode_mat_char(a: Any) -> str:
    """Decode a MATLAB char payload (uint16 row/column vector or bytes)."""
    a = np.asarray(a)
    if a.size == 0:
        return ""
    if a.dtype.kind == "S":
        return bytes(a.ravel()).decode("utf-8", errors="replace").rstrip("\x00").strip()
    if a.dtype.kind in ("U", "u") and a.dtype.itemsize == 2:
        s = a.ravel().view("<u2")
        return s.tobytes().decode("utf-16-le", errors="replace").strip("\x00 ").strip()
    if a.dtype.kind == "U":
        # scipy decodes MATLAB char to unicode (single string or char matrix)
        return "".join(str(x) for x in a.ravel()).strip()
    if a.dtype.kind == "u" and a.dtype.itemsize == 1:
        return bytes(a.ravel()).decode("utf-8", errors="replace").strip("\x00 ").strip()
    return str(a)


def mat_version(path: str) -> str:
    """Return '73' for MATLAB v7.3 (HDF5) files, 'classic' otherwise."""
    with open(path, "rb") as fh:
        head = fh.read(128)
    if head.startswith(b"MATLAB 7.3 MAT-file"):
        return "73"
    return "classic"


def _ravel1(x: Any) -> np.ndarray:
    return np.ravel(np.asarray(x))


def _mode_int(values: np.ndarray) -> Optional[int]:
    """Most frequent integer value (NaN-safe); None when empty."""
    v = _ravel1(values)
    if v.size == 0:
        return None
    fv = v[~np.isnan(v.astype(float))]
    if fv.size == 0:
        return None
    if fv.dtype.kind in "iu":
        return int(np.bincount(fv).argmax())
    r = np.rint(fv).astype(np.int64)
    return int(r.min() + np.bincount(r - r.min()).argmax())


# ---------------------------------------------------------------------------
# 2. MATLAB loaders
# ---------------------------------------------------------------------------

def _load_classic(path: str) -> Dict[str, Any]:
    """Load a classic (<= v7) .mat; keep cell dims, convert mat_struct to dict.

    Cells are returned as ``numpy.ndarray(dtype=object)`` so their MATLAB
    layout (e.g. ``(nStreams, nTrials)``) survives; structs become dicts.
    """
    import scipy.io as sio

    raw = sio.loadmat(path, squeeze_me=False, struct_as_record=False)

    def conv(obj: Any) -> Any:
        if obj is None:
            return None
        if isinstance(obj, np.ndarray) and obj.dtype == object:
            # A 1x1 object array whose element is a struct/scalar is a MATLAB
            # scalar struct (e.g. neural (1,1)) -> collapse to the struct dict.
            if obj.size == 1 and not isinstance(obj.flat[0], np.ndarray):
                return conv(obj.flat[0])
            out = np.empty(obj.shape, dtype=object)
            for idx in np.ndindex(obj.shape):
                out[idx] = conv(obj[idx])
            return out
        if isinstance(obj, np.ndarray):
            # matlab 0-d scalars -> keep as scalar
            if obj.size == 1 and obj.ndim == 0:
                return obj.item()
            return obj
        if isinstance(obj, np.generic):
            return obj.item()
        if isinstance(obj, str):
            return obj
        if isinstance(obj, bytes):
            return obj.decode("utf-8", errors="replace")
        if isinstance(obj, np.void):
            return obj.tolist()
        # scipy mat_struct
        names = [f for f in dir(obj) if not f.startswith("_")]
        return {f: conv(getattr(obj, f)) for f in names}

    out: Dict[str, Any] = {}
    for k, v in raw.items():
        if k.startswith("__"):
            continue
        out[k] = conv(v)
    return out


def _load_v73(path: str) -> Dict[str, Any]:
    """Load a MATLAB v7.3 (HDF5) .mat.

    Nested h5py objects are mapped to plain data:
      * numeric / char datasets  -> numpy arrays
      * reference (cell) datasets -> object-dtype numpy arrays with the same
        shape, each element resolved through its reference
      * groups (structs / containers) -> dicts of converted children
    The top-level ``#refs#`` bookkeeping group is skipped.
    """
    import h5py

    out: Dict[str, Any] = {}

    def conv_obj(obj: Any) -> Any:
        if isinstance(obj, h5py.Dataset):
            dt = obj.dtype
            if dt == h5py.ref_dtype:
                res = np.empty(obj.shape, dtype=object)
                flat = obj[...].ravel()
                for i, ref in enumerate(flat):
                    res.ravel()[i] = conv_obj(obj.file[ref])
                return res
            return np.asarray(obj[...])
        if isinstance(obj, h5py.Group):
            d: Dict[str, Any] = {}
            for name, child in obj.items():
                if name == "#refs#":
                    continue
                d[name] = conv_obj(child)
            return d
        if isinstance(obj, h5py.Reference):
            return conv_obj(obj.file[obj])
        return obj

    with h5py.File(path, "r") as f:
        for k in f.keys():
            if k == "#refs#":
                continue
            out[k] = conv_obj(f[k])
    return out


# ---------------------------------------------------------------------------
# 3. CND semantic extraction (works for both classic and v7.3 payloads)
# ---------------------------------------------------------------------------

def _chan_names_values(field: Any) -> List[Any]:
    """Return per-channel values of one chanlocs field (any storage form)."""
    if isinstance(field, np.ndarray) and field.dtype == object:
        # (n,1) / (n,) cell of per-channel values (may hold arrays to decode)
        vals: List[Any] = []
        for el in field.ravel():
            if isinstance(el, np.ndarray):
                vals.append(_decode_mat_char(el) if el.dtype.itemsize in (1, 2) and el.size < 32 else el)
            elif isinstance(el, (str, bytes, int, float)):
                vals.append(el.decode() if isinstance(el, bytes) else el)
            else:
                vals.append(el)
        return vals
    if isinstance(field, np.ndarray):
        if field.size and field.dtype.itemsize in (1, 2) and field.ndim <= 2:
            return [_decode_mat_char(field)]
        arr = np.asarray(field)
        if arr.ndim == 2 and 1 in arr.shape:
            return [arr.ravel()[i] for i in range(arr.shape[0] if arr.shape[0] > 1 else arr.shape[1])]
        return list(arr.ravel())
    if isinstance(field, list):
        return [_decode_mat_char(x) if isinstance(x, np.ndarray) else x for x in field]
    return [field]


def _parse_chanlocs(chanlocs: Any) -> List[Dict[str, Any]]:
    """Normalise neural.chanlocs to a list of per-channel dicts with 'name'.

    Handles three storage styles seen in the CND files:
      1. classic v7 struct array -> list of dicts (labels = channel names)
      2. v7.3 scalar struct whose fields are per-channel cells/columns
      3. v7.3 plain struct-array group -> list of per-element dicts
    """
    if chanlocs is None:
        return []

    # style 1 / 3: list-like (object array or python list) of per-channel dicts
    if isinstance(chanlocs, np.ndarray) and chanlocs.dtype == object:
        items = list(chanlocs.ravel())
        if items and isinstance(items[0], dict):
            recs = []
            for it in items:
                rec = {"name": ""}
                for k, v in it.items():
                    if k in ("labels", "name"):
                        rec["name"] = str(_decode_mat_char(v)).strip() if not isinstance(v, str) else str(v).strip()
                    elif isinstance(v, (int, float, str)) or (isinstance(v, np.ndarray) and v.size == 1):
                        rec[k.lower()] = v.item() if isinstance(v, np.ndarray) else v
                recs.append(rec)
            return recs

    if isinstance(chanlocs, dict):
        fields = {k: v for k, v in chanlocs.items() if k != "#refs#"}
        # find per-channel length
        n = 0
        for v in fields.values():
            if isinstance(v, np.ndarray) and v.ndim:
                n = max(n, v.shape[0] if v.shape[0] > 1 else v.shape[-1])
            elif isinstance(v, list):
                n = max(n, len(v))
        if n == 0:
            # scalar struct (single channel info) or empty
            return [{"name": str(_decode_mat_char(fields.get("labels", ""))).strip()}] \
                if fields.get("labels") is not None else []
        recs: List[Dict[str, Any]] = []
        for i in range(n):
            rec: Dict[str, Any] = {"name": ""}
            for k, v in fields.items():
                vals = _chan_names_values(v)
                if not vals:
                    continue
                val = vals[i] if len(vals) == n else vals[0]
                if k in ("labels", "name"):
                    nm = str(_decode_mat_char(val)).strip() if not isinstance(val, str) else str(val).strip()
                    rec["name"] = nm
                elif isinstance(val, (int, float)):
                    rec[k.lower()] = val
            recs.append(rec)
        return recs

    if isinstance(chanlocs, list):
        return _parse_chanlocs({"labels": [c.get("labels", c.get("name", "")) for c in chanlocs]})

    return []


def _extract_neural(neural: Dict[str, Any], source_path: str) -> Dict[str, Any]:
    """Normalise the ``neural`` struct into plain numpy-based fields."""
    fs_raw = neural.get("fs")
    fs = float(_ravel1(fs_raw)[0]) if fs_raw is not None else float("nan")

    data_cell = neural.get("data")
    trials: List[np.ndarray] = []
    if data_cell is not None:
        arr = data_cell if isinstance(data_cell, np.ndarray) else np.asarray(data_cell, dtype=object)
        if arr.dtype != object:
            trials = [np.asarray(arr)]          # a single 2-D block
        else:
            trials = [np.asarray(arr[i]) for i in np.ndindex(arr.shape)]

    chanlocs = _parse_chanlocs(neural.get("chanlocs"))

    def cell_series(name: str) -> List[Optional[np.ndarray]]:
        c = neural.get(name)
        if c is None:
            return [None] * len(trials)
        arr = c if isinstance(c, np.ndarray) else np.asarray(c, dtype=object)
        if arr.dtype != object:
            return [_ravel1(arr)]
        out: List[Optional[np.ndarray]] = []
        for idx in np.ndindex(arr.shape):
            el = arr[idx]
            out.append(_ravel1(el) if el is not None else None)
        return out

    attended_stream = cell_series("attendedStream")
    attended_azimuth = cell_series("attendedAzimuth")

    orig_pos = None
    if neural.get("origTrialPosition") is not None:
        arr = _ravel1(neural["origTrialPosition"])
        orig_pos = [int(x) for x in arr] if arr.size else None

    return {
        "fs": fs,
        "trials": trials,
        "chanlocs": chanlocs,
        "attended_stream": attended_stream,
        "attended_azimuth": attended_azimuth,
        "orig_trial_position": orig_pos,
        "data_type": _decode_mat_char(neural.get("dataType")) if neural.get("dataType") is not None else None,
        "device_name": _decode_mat_char(neural.get("deviceName")) if neural.get("deviceName") is not None else None,
        "source_path": source_path,
    }


def _extract_stim(stim: Dict[str, Any], n_trials: int,
                  source_path: str) -> Dict[str, Any]:
    """Normalise ``stim`` into {fs, per_trial: [[env_1, ..., env_K], ...]}."""
    fs_raw = stim.get("fs")
    fs = float(_ravel1(fs_raw)[0]) if fs_raw is not None else float("nan")

    cell = stim.get("data")
    per_trial: List[List[np.ndarray]] = [[] for _ in range(n_trials)]
    if cell is not None:
        arr = cell if isinstance(cell, np.ndarray) else np.asarray(cell, dtype=object)
        if arr.dtype == object:
            if arr.ndim == 2:
                # (nStreams, nTrials) classic  OR  (nTrials, nStreams) v7.3
                if arr.shape[1] == n_trials:
                    for s in range(arr.shape[0]):
                        for t in range(n_trials):
                            per_trial[t].append(_ravel1(arr[s, t]).astype(float))
                elif arr.shape[0] == n_trials:
                    for t in range(n_trials):
                        for s in range(arr.shape[1]):
                            per_trial[t].append(_ravel1(arr[t, s]).astype(float))
                else:
                    flat = arr.ravel()
                    per_trial = [np.asarray(e).ravel().astype(float) for e in flat]
                    per_trial = [[] for _ in range(n_trials)]
                    k = len(flat) // n_trials
                    for t in range(n_trials):
                        for s in range(k):
                            per_trial[t].append(_ravel1(flat[t * k + s]).astype(float))
            else:
                flat = arr.ravel()
                k = len(flat) // n_trials
                for t in range(n_trials):
                    for s in range(k):
                        if t * k + s < len(flat):
                            per_trial[t].append(_ravel1(flat[t * k + s]).astype(float))
    if not any(per_trial):
        raise ValueError(f"stim.data could not be parsed ({source_path})")

    names: List[str] = []
    if stim.get("names") is not None:
        nm_arr = stim["names"]
        arr = nm_arr if isinstance(nm_arr, np.ndarray) else np.asarray(nm_arr, dtype=object)
        for el in arr.ravel():
            if isinstance(el, str):
                names.append(el)
            elif isinstance(el, bytes):
                names.append(el.decode("utf-8", errors="replace"))
            else:
                names.append(_decode_mat_char(el))
    return {"fs": fs, "per_trial": per_trial, "names": names, "source_path": source_path}


# ---------------------------------------------------------------------------
# 4. Per-trial conversion
# ---------------------------------------------------------------------------

def _label_from_azimuth(azimuth: Optional[np.ndarray], style: str) -> Optional[str]:
    """SuperHuge-style trial label from the attended azimuth at trial onset.

    Falls back to the modal azimuth when sample 0 is not informative. Returns
    None for "none" style / uninformative azimuths.
    """
    if azimuth is None or azimuth.size == 0:
        return None
    az0 = float(azimuth[0])
    if np.isnan(az0) or np.isinf(az0):
        m = _mode_int(azimuth)
        az0 = float(m) if m is not None else az0
    if style == "azimuth_str":
        return str(int(round(az0)))
    if style == "LR":
        return "R" if az0 > 0 else ("L" if az0 < 0 else None)
    if style == "left_right":
        return "right" if az0 > 0 else ("left" if az0 < 0 else None)
    return None


def _attended_env(envs: Sequence[np.ndarray],
                  attended_stream: Optional[np.ndarray],
                  length: int) -> Tuple[np.ndarray, Optional[int]]:
    """Build the ``(T, 1, K)`` envelope array, attended stream in column 0.

    The attended column is the stream most frequently labelled attended across
    the trial (1-based CND indices). For constant-attention trials this is
    exact; for attention-switch trials the column-0 stream is the
    majority-attended one and the per-sample ground truth remains available in
    ``extra_info["attention"]`` so switch locations stay recoverable.
    """
    K = len(envs)
    trimmed: List[np.ndarray] = []
    for e in envs:
        v = _ravel1(e)
        if v.size < length - 1:
            raise ValueError(
                f"envelope length {v.size} is shorter than the EEG length "
                f"{length} by more than 1 sample"
            )
        # Existing SuperHuge env files equal the EEG length and are aligned to
        # the EEG onset (head-trimmed), verified against the Ultra-HD data
        # whose stored envelopes are 640 samples longer than the EEG.
        trimmed.append(v[:length].astype(np.float64, copy=False))

    attended_idx: Optional[int] = None
    if attended_stream is not None and attended_stream.size:
        win = attended_stream[:length]
        mode = _mode_int(win)
        if mode is not None and 1 <= mode <= K:
            attended_idx = int(mode)

    order = list(range(K))
    if attended_idx is not None:
        order = [attended_idx - 1] + [j for j in range(K) if j != attended_idx - 1]

    mat = np.stack([trimmed[j] for j in order], axis=1)      # (T, K)
    return mat.reshape(length, 1, K), attended_idx          # (T, 1, K)


def _drop_channels(eeg: np.ndarray,
                   chanlocs: List[Dict[str, Any]],
                   drop: Sequence[str]) -> Tuple[np.ndarray, List[Dict[str, Any]]]:
    """Remove channels by name (case-insensitive) from EEG columns + chanlocs."""
    if not drop:
        return eeg, chanlocs
    drop_low = {d.lower() for d in drop}
    keep_idx = [i for i, c in enumerate(chanlocs) if c["name"].lower() not in drop_low]
    dropped = [c["name"] for i, c in enumerate(chanlocs) if i not in keep_idx]
    if dropped:
        log(f"      dropped channels: {dropped}")
    return eeg[:, keep_idx], [chanlocs[i] for i in keep_idx]


# ---------------------------------------------------------------------------
# 5. Metadata entry builder
# ---------------------------------------------------------------------------

# Reference-shaped template of one metadata entry. The key set / ordering
# mirrors the reference SuperHuge rows (feature flags are always present;
# "<feature>_fs" keys are only written when the feature is actually provided,
# which never happens here because this converter produces env only).
# extra_info is strictly empty and stimuli names are empty-string placeholders
# (no private additions are made to the reference format).
def _build_entry(dataset: Dict[str, Any], stage: str, subject_id: int,
                 trial_id: int, fs: float, signal_length: int,
                 chanlocs: List[Dict[str, Any]], n_streams: int,
                 label: Optional[str]) -> Dict[str, Any]:
    """Assemble one metadata.pkl entry strictly following the SuperHuge
    reference rows of the same dataset."""
    entry: Dict[str, Any] = {
        "dataset_id": int(dataset["dataset_id"]),
        "subject_id": int(subject_id),
        "trial_id": int(trial_id),
        "signal_length": int(signal_length),
        "num_channel": int(len(chanlocs)),
        "fs": int(round(fs)),
        "dataset_name": f"{dataset['name']}_{stage}",
        "channel_infos": {i + 1: {"name": c["name"]} for i, c in enumerate(chanlocs)},
        "num_speakers": int(n_streams),
        "extra_info": {},
    }
    if label is not None:
        entry["label"] = label
    entry["stimuli_name"] = ""
    entry["compet_stimuli_name"] = ""
    entry["wav"] = 0          # only env is produced by this converter
    entry["env"] = 1
    entry["env_fs"] = int(round(fs))
    entry["mel"] = 0
    entry["wav2vec2"] = 0
    return entry


# ---------------------------------------------------------------------------
# 6. Dataset converter
# ---------------------------------------------------------------------------

def _find_subject_subdir(dataset_dir: str, candidates: Sequence[str],
                         target_fs: float) -> Optional[Tuple[str, float]]:
    """Pick the CND sub-folder whose neural.fs == target_fs (in order)."""
    for sub in candidates:
        d = os.path.join(dataset_dir, sub)
        if not os.path.isdir(d):
            continue
        files = sorted(n for n in os.listdir(d) if SUBJECT_FILE_RE.match(n))
        if not files:
            continue
        p = os.path.join(d, files[0])
        try:
            fs = _read_neural_fs(p)
        except Exception as exc:  # noqa: BLE001
            log(f"    [warn] could not read fs from {p}: {exc}")
            continue
        if abs(float(fs) - target_fs) < 1e-6:
            return sub, float(fs)
        log(f"    [info] {sub}: fs={fs:g} Hz != target {target_fs:g} Hz, skipping")
    return None


def _read_neural_fs(path: str) -> float:
    """Read ``neural.fs`` without loading the full EEG."""
    if mat_version(path) == "73":
        import h5py

        with h5py.File(path, "r") as f:
            return float(_ravel1(f["neural"]["fs"][...])[0])
    import scipy.io as sio

    m = sio.loadmat(path, squeeze_me=True, struct_as_record=False,
                    variable_names=["neural"])
    return float(_ravel1(m["neural"].fs)[0])


def _load_pair(sub_dir: str, sub_num: int, prefix: str = "",
               ) -> Tuple[Dict[str, Any], str, Dict[str, Any], str]:
    """Load and normalise the (neural, stim) pair of one subject file number.

    ``prefix`` selects the CND file family: "" for ``dataSub/dataStim`` (raw)
    and ``"pre_"`` for ``pre_dataSub/pre_dataStim`` (the pre-processed CND
    variant shipped next to the raw files in the source folders)."""
    p_sub = os.path.join(sub_dir, f"{prefix}dataSub{sub_num}.mat")
    p_stim = os.path.join(sub_dir, f"{prefix}dataStim{sub_num}.mat")
    if not os.path.exists(p_stim):
        p_stim = os.path.join(sub_dir, f"{prefix}dataStim.mat")
    if not os.path.exists(p_sub):
        raise FileNotFoundError(p_sub)
    if not os.path.exists(p_stim):
        raise FileNotFoundError(p_stim)
    raw_n = _load_v73(p_sub) if mat_version(p_sub) == "73" else _load_classic(p_sub)
    raw_s = _load_v73(p_stim) if mat_version(p_stim) == "73" else _load_classic(p_stim)
    neural = _extract_neural(raw_n.get("neural", raw_n), p_sub)
    stim = _extract_stim(raw_s.get("stim", raw_s), len(neural["trials"]), p_stim)
    return neural, p_sub, stim, p_stim


def convert_dataset(dataset: Dict[str, Any], source_root: str, out_root: str,
                    stage: str, variant: str = "data",
                    write_files: bool = True, overwrite: bool = False,
                    subjects_only: Optional[Sequence[int]] = None,
                    target_fs: float = TARGET_FS) -> Dict[str, Any]:
    """Convert one CND source folder into one SuperHuge stage directory.

    ``variant`` selects the CND file family:
      * ``"data"`` -> ``dataSub/dataStim`` files (written to ``stage`` = "raw")
      * ``"pre"``  -> ``pre_dataSub/pre_dataStim`` files (stage "preprocessed")
    Both variants keep the same dataset_id / subject / trial numbering; the
    pre-processed CND files carry their own fs (typically 32 Hz after the
    1-9 Hz band-pass + downsampling of the KUL CND toolchain), which is stored
    verbatim in the metadata of the "preprocessed" stage.
    With ``write_files=False`` only the metadata entries are recomputed (the
    eeg/env .npy files are assumed to exist already).
    """
    folder = dataset["folder"]
    did = int(dataset["dataset_id"])
    ds_dir = os.path.join(source_root, folder)
    if not os.path.isdir(ds_dir):
        raise FileNotFoundError(f"dataset folder not found: {ds_dir}")

    prefix = "pre_" if variant == "pre" else ""
    sub_re = PRE_SUBJECT_FILE_RE if variant == "pre" else SUBJECT_FILE_RE

    # The CND sub-folder is always selected from the raw (non pre_) data so
    # that raw + preprocessed stages of one dataset share the same source dir
    # (dataCND, or dataCND_resampled when present).
    candidates = list(dataset.get("candidate_subdirs") or ())
    if os.path.isdir(os.path.join(ds_dir, "dataCND_resampled")):
        candidates = ["dataCND_resampled"] + [c for c in candidates
                                              if c != "dataCND_resampled"]
    found = _find_subject_subdir(ds_dir, candidates, target_fs)
    if found is None:
        raise RuntimeError(
            f"{folder}: no CND sub-folder with fs == {target_fs:g} Hz found in "
            f"{candidates}")
    sub_name, _raw_fs = found
    sub_dir = os.path.join(ds_dir, sub_name)
    log(f"=== {folder} [{stage}] (dataset_id={did}, dir={sub_name}) ===")

    subject_nums = sorted(
        int(m.group(1)) for n in os.listdir(sub_dir) if (m := sub_re.match(n))
    )
    if variant == "pre" and not subject_nums:
        log(f"    [info] no {prefix}dataSub<N>.mat files - skipping this stage")
        return {"folder": folder, "dataset_id": did, "stage": stage,
                "dataset_name": f"{dataset['name']}_{stage}",
                "trials": 0, "channels": None, "fs": None,
                "subjects": [], "metadata": {}, "skipped": True}
    if not subject_nums:
        raise RuntimeError(
            f"{folder}: no {prefix}dataSub<N>.mat files in {sub_dir}")
    if subjects_only:
        allowed = set(subjects_only)
        subject_nums = [n for n in subject_nums if n in allowed]
        missing = sorted(allowed - set(subject_nums))
        if missing:
            log(f"    [warn] requested subject file number(s) not found: {missing}")
    if not subject_nums:
        raise RuntimeError(f"{folder}: no {prefix}dataSub<N>.mat files to process")
    # Subject ids are contiguous 1..N in file-number order (reference rows use
    # contiguous subject ids for every dataset).
    subject_ids = {num: i + 1 for i, num in enumerate(subject_nums)}

    eeg_dir = os.path.join(out_root, stage, "eeg")
    env_dir = os.path.join(out_root, stage, "stimuli", "env")
    meta_dir = os.path.join(out_root, stage, "meta")
    for d in (eeg_dir, env_dir, meta_dir):
        os.makedirs(d, exist_ok=True)

    dtype = np.dtype(dataset.get("dtype", np.float64))
    metadata_entries: Dict[str, Any] = {}
    total_trials = 0
    n_chan_out: Optional[int] = None
    fs_values: List[float] = []

    for sub_num in sorted(subject_nums):
        sid = subject_ids[sub_num]
        try:
            neural, p_sub, stim, p_stim = _load_pair(sub_dir, sub_num, prefix)
        except FileNotFoundError as exc:
            log(f"    [warn] skipping subject {sub_num}: {exc}")
            continue
        fs_sub = float(neural["fs"])
        if np.isnan(fs_sub):
            raise RuntimeError(f"sub{sub_num}: could not read neural.fs")
        if abs(stim["fs"] - fs_sub) > 1e-6:
            raise RuntimeError(
                f"sub{sub_num}: stim.fs {stim['fs']:g} != neural fs {fs_sub:g}")
        if variant == "data" and abs(fs_sub - target_fs) > 1e-6:
            raise RuntimeError(
                f"sub{sub_num}: neural.fs {fs_sub:g} != expected {target_fs:g}")
        fs_values.append(fs_sub)

        n_tr = len(neural["trials"])
        chanlocs_all = neural["chanlocs"]
        if len(stim["per_trial"]) != n_tr:
            raise RuntimeError(
                f"sub{sub_num}: {len(stim['per_trial'])} stim trials vs "
                f"{n_tr} neural trials")
        log(f"    subject {sid} (file {sub_num}): {n_tr} trials, "
            f"{len(chanlocs_all)} channels, fs={fs_sub:g} Hz")

        for t in range(n_tr):
            tid = t + 1
            key = FILE_KEY.format(did=did, sid=sid, tid=tid)

            eeg = np.asarray(neural["trials"][t])
            n_src = len(chanlocs_all)
            if eeg.ndim != 2:
                raise ValueError(f"{key}: expected 2-D EEG, got {eeg.shape}")
            # v7 classic stores (T, C); v7.3 stores (C, T). Resolve by shape.
            if eeg.shape[0] == n_src and eeg.shape[1] != n_src:
                eeg = eeg.T
            if eeg.shape[1] != n_src:
                raise ValueError(
                    f"{key}: EEG columns {eeg.shape[1]} != chanlocs count {n_src}")
            eeg, chanlocs = _drop_channels(eeg, chanlocs_all,
                                           dataset.get("drop_channels", []))
            env_streams = stim["per_trial"][t]
            # Analysis window = intersection of EEG and stimulus envelope
            # (SuperHuge files keep env[:,0,:] and eeg of identical length).
            # If the envelope is longer, its head is trimmed to the EEG length;
            # if the EEG is longer (trailing samples beyond the stimulus), the
            # EEG is trimmed to the envelope length.
            env_len = min(int(_ravel1(e).size) for e in env_streams)
            length = int(min(eeg.shape[0], env_len))
            if length < eeg.shape[0]:
                eeg = eeg[:length]
            env, _ = _attended_env(env_streams,
                                   neural["attended_stream"][t], length)

            label = None
            if dataset["label_style"] != "none":
                label = _label_from_azimuth(neural["attended_azimuth"][t],
                                            dataset["label_style"])

            entry = _build_entry(
                dataset=dataset, stage=stage, subject_id=sid, trial_id=tid,
                fs=fs_sub, signal_length=length, chanlocs=chanlocs,
                n_streams=len(env_streams), label=label)

            eeg_path = os.path.join(eeg_dir, f"{key}.npy")
            env_path = os.path.join(env_dir, f"{key}_env.npy")
            if write_files:
                if (os.path.exists(eeg_path) or os.path.exists(env_path)) and not overwrite:
                    raise FileExistsError(
                        f"output files already exist for {key}; use --overwrite")
                np.save(eeg_path, np.asfortranarray(eeg.astype(dtype, copy=False)))
                np.save(env_path, np.asfortranarray(env.astype(dtype, copy=False)))
            metadata_entries[key] = entry
            total_trials += 1
            n_chan_out = len(chanlocs)

    fs_out = round(float(np.mean(fs_values))) if fs_values else None
    log(f"    -> wrote {total_trials} trial entries x {n_chan_out} ch "
        f"(fs={fs_out} Hz, files={'yes' if write_files else 'kept'})")

    fs_out = round(float(np.mean(fs_values))) if fs_values else None
    log(f"    -> wrote {total_trials} trials x {n_chan_out} ch (fs={fs_out} Hz)")
    return {
        "folder": folder, "dataset_id": did,
        "dataset_name": f"{dataset['name']}_{stage}",
        "trials": total_trials, "channels": n_chan_out,
        "fs": fs_out,
        "subjects": [subject_ids[n] for n in sorted(subject_nums)],
        "metadata": metadata_entries,
        "stage": stage,
        "skipped": False,
    }


def main(argv: Optional[Sequence[str]] = None) -> int:
    parser = argparse.ArgumentParser(
        description="Convert CND AAD datasets (dataSub/dataStim .mat and their "
                    "pre_dataSub/pre_dataStim pre-processed variants) into the "
                    "SuperHuge on-disk layout (raw + preprocessed stages).",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--source-root",
                        default=r"C:/Users/sean/Downloads/data-Yuanming",
                        help="root folder holding one CND sub-folder per dataset")
    parser.add_argument("--out-root", default=r"E:/ICASSP2027-EEG-sAAD-GC/supehuge",
                        help="output root; <stage>/eeg|stimuli|meta tree is created inside")
    parser.add_argument("--stage", default="both", choices=("both", "raw", "preprocessed"),
                        help="which stage(s) to write: raw uses dataSub/dataStim, "
                             "preprocessed uses pre_dataSub/pre_dataStim "
                             "(this tool stores the CND content as-is; it applies "
                             "no extra EEG cleaning)")
    parser.add_argument("--target-fs", type=float, default=TARGET_FS,
                        help="expected sampling rate of the raw (non pre_) CND data")
    parser.add_argument("--datasets", default="",
                        help="comma-separated source folders to convert "
                             "(default: all in DATASETS)")
    parser.add_argument("--subjects", default="",
                        help="comma-separated dataSub<N>/pre_dataSub<N> file numbers "
                             "to convert (default: all subject files of each dataset)")
    parser.add_argument("--metadata-only", action="store_true",
                        help="recompute metadata entries only; keep the existing "
                             "eeg/env .npy files untouched")
    parser.add_argument("--overwrite", action="store_true",
                        help="overwrite existing trial .npy / metadata.pkl")
    args = parser.parse_args(argv)

    subjects_only: Optional[List[int]] = None
    if args.subjects:
        subjects_only = [int(s.strip()) for s in args.subjects.split(",") if s.strip()]

    if args.stage == "both":
        stages: List[Tuple[str, str]] = [("raw", "data"), ("preprocessed", "pre")]
    else:
        variant = "data" if args.stage == "raw" else "pre"
        stages = [(args.stage, variant)]

    datasets = list(DATASETS)
    if args.datasets:
        wanted = {s.strip() for s in args.datasets.split(",") if s.strip()}
        datasets = [d for d in datasets if d["folder"] in wanted]
        missing = wanted - {d["folder"] for d in datasets}
        if missing:
            log(f"[error] unknown dataset folder(s): {sorted(missing)}")
            return 2
    if not datasets:
        log("[error] no datasets selected")
        return 2

    report: List[Dict[str, Any]] = []
    for ds in datasets:
        for stage, variant in stages:
            try:
                rep = convert_dataset(
                    dataset=ds, source_root=args.source_root,
                    out_root=args.out_root, stage=stage, variant=variant,
                    target_fs=args.target_fs,
                    write_files=not args.metadata_only,
                    overwrite=args.overwrite,
                    subjects_only=subjects_only)
                if not rep.get("skipped"):
                    report.append(rep)
            except Exception as exc:  # noqa: BLE001 - isolate dataset failures
                log(f"[error] {ds['folder']} [{stage}]: {exc}")
                report.append({"folder": ds["folder"], "stage": stage,
                               "error": str(exc)})

    # Write one merged metadata.pkl per stage (SuperHuge stores a single
    # metadata.pkl under <stage>/meta covering every dataset of that stage).
    merged: Dict[str, Dict[str, Any]] = {}
    for r in report:
        if "error" in r or "metadata" not in r:
            continue
        stage = r["stage"]
        for k, e in r["metadata"].items():
            if k in merged.setdefault(stage, {}):
                raise RuntimeError(f"duplicate trial key in stage {stage}: {k}")
            merged[stage][k] = e
    for stage, entries in merged.items():
        meta_path = os.path.join(args.out_root, stage, "meta", "metadata.pkl")
        if os.path.exists(meta_path) and not args.overwrite:
            raise SystemExit(
                f"{meta_path} already exists; use --overwrite to merge into it")
        # With --overwrite, merge into a pre-existing file (e.g. an earlier
        # partial run); without it the file simply holds the current run.
        if os.path.exists(meta_path) and args.overwrite:
            with open(meta_path, "rb") as fh:
                old = pickle.load(fh)
            entries = {**old, **entries}
        with open(meta_path, "wb") as fh:
            pickle.dump(entries, fh, protocol=4)
        log(f"[meta] {len(entries)} trial entries -> {meta_path}")

    log("")
    log("=== conversion summary ===")
    failed = 0
    skipped = 0
    for r in report:
        if "error" in r:
            failed += 1
            log(f"  {r['folder']} [{r.get('stage','')}]: FAILED - {r['error']}")
        elif r.get("skipped"):
            skipped += 1
            log(f"  {r['folder']} [{r['stage']}]: skipped (no {r['dataset_name']} files)")
        else:
            log(f"  {r['folder']} [{r['stage']}] -> {r['dataset_name']} "
                f"(dataset_id={r['dataset_id']}): {r['trials']} trials, "
                f"{r['channels']} ch, fs={r['fs']} Hz")
    if failed:
        log(f"{failed} dataset/stage(s) failed; see messages above.")
        return 1
    log(f"done. ({len(report)} dataset/stage conversions, {skipped} skipped)")
    return 0


if __name__ == "__main__":
    sys.exit(main())
