Ë
    Â�¬jì°  ã                  óv  — U d Z ddlmZ ddlZddlZddlZddlZddlZddlm	Z	m
Z
mZmZmZmZ ddlZdZ edddd	d
g ej&                  d¬«       edddddg ej&                  d¬«       edddddg ej&                  d¬«       eddddd
g ej&                  d¬«       edd	dddddgej&                  d¬«       edd d!d"d#g ej&                  d¬«       ed$d%d&d'dd(g ej&                  d¬)«	      gZd*ed+<   d,Zd-Z ej0                  d.«      Z ej0                  d/«      ZdGd0„ZdHd1„ZdId2„ZdJd3„ZdKd4„ZdLd5„Z dLd6„Z!dMd7„Z"dNd8„Z#dOd9„Z$	 	 	 	 dPd:„Z%dQd;„Z&	 	 	 	 	 	 dRd<„Z'	 	 	 	 	 	 dSd=„Z(	 	 	 	 	 	 	 	 	 	 	 	 	 	 dTd>„Z)	 	 	 	 dUd?„Z*dVd@„Z+dW	 dXdA„Z,dBdCd(def	 	 	 	 	 	 	 	 	 	 	 	 	 dYdD„Z-dZd[dE„Z.e/dFk(  r ej`                   e.«       «       yy)\a¢  
convert_cnd_to_superhuge.py
===========================

Convert a folder of CND-format AAD datasets (dataSub<N>.mat / dataStim<N>.mat,
as stored under e.g. ``C:/Users/sean/Downloads/data-Yuanming``) into the
*SuperHuge* dataset layout (the on-disk format of ``E:/derivatives/SuperHuge``).

SuperHuge layout produced here. Both SuperHuge stage directories are written
by default (each dataset keeps the same dataset_id across stages)::

    <out_root>/raw/eeg/dataset-<DDD>-subject-<SSS>-trial-<TTT>.npy
    <out_root>/raw/stimuli/env/dataset-<DDD>-subject-<SSS>-trial-<TTT>_env.npy
    <out_root>/raw/meta/metadata.pkl
    <out_root>/preprocessed/...            (same layout, dataset names *_preprocessed)

* The ``raw`` stage is produced from the ``dataSub<N>.mat``/``dataStim<N>.mat``
  files; whenever a ``dataCND_resampled`` sibling exists (USTC, Ultra-HD) that
  resampled version is used.
* The ``preprocessed`` stage is produced from the ``pre_dataSub<N>.mat`` /
  ``pre_dataStim<N>.mat`` files that ship next to the raw files. These are the
  pre-processed CND variants (1-9 Hz band-pass + downsampling to fs = 32 Hz by
  the KUL CND toolchain); their actual fs is stored verbatim in the metadata.
  No extra EEG cleaning is applied by this tool.

SuperHuge data conventions honoured by this converter
-----------------------------------------------------
* EEG arrays are stored as ``float`` NumPy arrays of shape ``(T, C)``
  (time-major) written in **Fortran order** (as on disk in SuperHuge).
* Envelope arrays are stored as ``(T, 1, K)`` arrays (``K`` = number of
  concurrent speakers / streams).
* **The attended envelope is always written to column 0** of the envelope
  array (``env[:, 0, 0]``), exactly as in the existing SuperHuge files
  (verified: for static-attention trials ``corr(col0, attended) ~ 1.0``).
  Columns 1..K-1 keep their original source order.
* Trial-level metadata entries strictly follow the SuperHuge ``metadata.pkl``
  row schema of the same dataset name (raw / preprocessed reference rows):
  ``dataset_id / subject_id / trial_id / signal_length / num_channel / fs /
  dataset_name / channel_infos / num_speakers / extra_info``, ``label`` (when
  the reference row has one), ``stimuli_name / compet_stimuli_name`` and the
  feature availability flags. No extra fields are added.

Feature flags & scaling files
-----------------------------
* Only the envelope feature is produced here. The feature flag keys present in
  the reference rows are always written with truthful values
  (``wav:0, env:1, env_fs:<fs>, mel:0, wav2vec2:0``); ``<feature>_fs`` keys
  are written only for features that are actually provided (never here).
* ``stimuli_name`` / ``compet_stimuli_name`` are kept as **empty-string
  placeholders** (the CND files do not expose story names).
* ``extra_info`` is strictly ``{}``; per-sample attention ground truth stays in
  the source CND files and is NOT stored in metadata.
* ``scaling_factor.pkl`` and ``channelwise_scaling_factor.pkl`` are **not**
  produced: they are normalisation statistics computed by the SuperHuge build
  pipeline over the full corpus, not part of the raw conversion.

Label conventions
-----------------
The trial-level ``label`` is derived from the *attended azimuth at trial onset*
(the azimuth of the attended stream at sample 0), which reproduces the
existing SuperHuge labels for static paradigms (e.g. azimuth -90 -> "left"/"L")
and is well defined for trials whose azimuth pans across +/-90 (e.g.
AV-GC/DLAAD style stimuli). The label string format is controlled per dataset
(``label_style``): ``left_right`` ("left"/"right"), ``LR`` ("L"/"R") or
``azimuth_str`` (e.g. "-90"). For datasets that have no meaningful attended
azimuth (``label_style="none"``), the ``label`` field is omitted, matching the
existing KUL-DLAAD entries. Label values and channel names keep the source CND
values (no re-labelling / re-naming).

Source CND variants handled
---------------------------
* MATLAB v5/v7 ``.mat`` (scipy.io) and MATLAB v7.3 HDF5 ``.mat`` (h5py).
* Per-subject ``dataStim<N>.mat`` files or a single shared ``dataStim.mat``.
* ``dataCND`` folders whose native fs != target fs (e.g. USTC @ 250 Hz or the
  Ultra-HD files @ 250 Hz) are skipped in favour of the sibling
  ``dataCND_resampled`` folder when present (target fs = 128 Hz by default).
* v7.3 EEG trials are stored channel-major ``(C, T)`` by MATLAB and are
  transposed to time-major ``(T, C)`` automatically.

Dataset registry / numbering
----------------------------
Every source folder is mapped through :data:`DATASETS`. Dataset ids here are a
*fresh, independent* numbering (1..N in registry order) and are NOT the ids
used by the existing ``E:/derivatives/SuperHuge`` metadata. ``orig_dataset_id``
records which existing SuperHuge dataset the folder corresponds to (purely
informational provenance). Names are written as ``<name>_<stage>``, e.g.
``KUL_raw`` (stage "raw"), exactly like existing SuperHuge dataset names.

USAGE
-----
    python convert_cnd_to_superhuge.py --source-root <src> --out-root <dst>
    python convert_cnd_to_superhuge.py --datasets DTU2018,KUL2016   # subset
    python convert_cnd_to_superhuge.py --stage preprocessed         # only pre_* files
    python convert_cnd_to_superhuge.py --subjects 1,2              # subset of subjects
    python convert_cnd_to_superhuge.py --metadata-only --overwrite # re-write metadata.pkl
    python convert_cnd_to_superhuge.py --overwrite

Dependencies: numpy, scipy (for classic .mat), h5py (for v7.3 .mat).
é    )ÚannotationsN)ÚAnyÚDictÚListÚOptionalÚSequenceÚTuplez3dataset-{did:03d}-subject-{sid:03d}-trial-{tid:03d}ÚAVGCAAD2024é   z	KUL-AV-GCé   Ú
left_right)ÚdataCND)ÚfolderÚ
dataset_idÚnameÚorig_dataset_idÚlabel_styleÚdrop_channelsÚdtypeÚcandidate_subdirsÚ	DLAAD2021é   z	KUL-DLAADé   ÚnoneÚDTU2018é   ÚDTUÚLRÚKUL2016é   ÚKULÚKUL_AAD_UltraHD2021zKUL-HighDensityé   ÚXtraLÚXtraR)ÚdataCND_resampledr   Ú
NJUSUS2026é   z+NJU-SUSTech-ICASSP2027GC-GC-AS-BlindTestSeté   Úazimuth_strÚUSTC2026é   ÚUSTCé   F)	r   r   r   r   r   Úsubject_renumberr   r   r   úList[Dict[str, Any]]ÚDATASETS)ÚrawÚpreprocessedé€   z^dataSub(\d+)\.mat$z^pre_dataSub(\d+)\.mat$c                ó   — t        | d¬«       y )NT)Úflush)Úprint)Úmsgs    úZ/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/tools/convert_cnd_to_superhuge.pyÚlogr:   ¸   s   € Ü	ˆ#�TÖó    c                óÊ  — t        j                  | «      } | j                  dk(  ry| j                  j                  dk(  rGt        | j                  «       «      j                  dd¬«      j                  d«      j                  «       S | j                  j                  dv rv| j                  j                  d	k(  r]| j                  «       j                  d
«      }|j                  «       j                  dd¬«      j                  d«      j                  «       S | j                  j                  dk(  r4dj                  d„ | j                  «       D «       «      j                  «       S | j                  j                  dk(  r`| j                  j                  dk(  rGt        | j                  «       «      j                  dd¬«      j                  d«      j                  «       S t        | «      S )zADecode a MATLAB char payload (uint16 row/column vector or bytes).r   Ú ÚSúutf-8Úreplace©ÚerrorsÚ )ÚUÚur   z<u2z	utf-16-lez  rD   c              3  ó2   K  — | ]  }t        |«      –— Œ y ­w©N)Ústr)Ú.0Úxs     r9   Ú	<genexpr>z#_decode_mat_char.<locals>.<genexpr>È   s   è ø€ Ð1¡y !”s˜1—v¡yùs   ‚rE   r   )ÚnpÚasarrayÚsizer   ÚkindÚbytesÚravelÚdecodeÚrstripÚstripÚitemsizeÚviewÚtobytesÚjoinrH   )ÚaÚss     r9   Ú_decode_mat_charr[   ¼   sZ  € ä
�
‰
�1‹€AØ‡v�v�‚{ØØ‡w�w‡|�|�sÒÜ�Q—W‘W“YÓ×&Ñ& w°yÐ&ÓA×HÑHÈÓP×VÑVÓXÐXØ‡w�w‡|�|�zÑ! a§g¡g×&6Ñ&6¸!Ò&;Ø�G‰G‹I�N‰N˜5Ó!ˆØ�y‰y‹{×!Ñ! +°iÐ!Ó@×FÑFÀwÓO×UÑUÓWÐWØ‡w�w‡|�|�sÒà�w‰wÑ1 q§w¡w¤yÓ1Ó1×7Ñ7Ó9Ð9Ø‡w�w‡|�|�sÒ˜qŸw™w×/Ñ/°1Ò4Ü�Q—W‘W“YÓ×&Ñ& w°yÐ&ÓA×GÑGÈÓP×VÑVÓXÐXÜˆq‹6€Mr;   c                óŒ   — t        | d«      5 }|j                  d«      }ddd«       j                  d«      ryy# 1 sw Y   ŒxY w)z>Return '73' for MATLAB v7.3 (HDF5) files, 'classic' otherwise.Úrbr4   Ns   MATLAB 7.3 MAT-fileÚ73Úclassic)ÚopenÚreadÚ
startswith)ÚpathÚfhÚheads      r9   Úmat_versionrf   Î   s<   € ä	ˆd�DÔ	˜RØ�w‰w�s‹|ˆ÷ 
à‡�Ð-Ô.ØØ÷	 
Ð	ús	   �:ºAc                óR   — t        j                  t        j                  | «      «      S rG   )rL   rQ   rM   )rJ   s    r9   Ú_ravel1rh   ×   s   € Ü�8‰8”B—J‘J˜q“MÓ"Ð"r;   c                ó8  — t        | «      }|j                  dk(  ry|t        j                  |j	                  t
        «      «          }|j                  dk(  ry|j                  j                  dv r,t        t        j                  |«      j                  «       «      S t        j                  |«      j	                  t        j                  «      }t        |j                  «       t        j                  ||j                  «       z
  «      j                  «       z   «      S )z8Most frequent integer value (NaN-safe); None when empty.r   NÚiu)rh   rN   rL   ÚisnanÚastypeÚfloatr   rO   ÚintÚbincountÚargmaxÚrintÚint64Úmin)ÚvaluesÚvÚfvÚrs       r9   Ú	_mode_intrx   Û   s¿   € ä�‹€AØ‡v�v�‚{ØØ	
ŒB�H‰H�Q—X‘Xœe“_Ó%Ð%Ñ	&€BØ	‡w�w�!‚|ØØ	‡x�x‡}�}˜ÑÜ”2—;‘;˜r“?×)Ñ)Ó+Ó,Ð,Ü
�‰�‹×Ñœ2Ÿ8™8Ó$€AÜˆq�u‰u‹wœŸ™ Q¨¯©«¡[Ó1×8Ñ8Ó:Ñ:Ó;Ð;r;   c                ó¶   ‡— ddl m} |j                  | dd¬«      }dˆfd„Ši }|j                  «       D ]"  \  }}|j	                  d«      rŒ ‰|«      ||<   Œ$ |S )zâLoad a classic (<= v7) .mat; keep cell dims, convert mat_struct to dict.

    Cells are returned as ``numpy.ndarray(dtype=object)`` so their MATLAB
    layout (e.g. ``(nStreams, nTrials)``) survives; structs become dicts.
    r   NF)Ú
squeeze_meÚstruct_as_recordc           
     óü  •— | €y t        | t        j                  «      r·| j                  t        k(  r¤| j
                  dk(  r<t        | j                  d   t        j                  «      s ‰| j                  d   «      S t        j                  | j                  t        ¬«      }t        j                  | j                  «      D ]  } ‰| |   «      ||<   Œ |S t        | t        j                  «      r0| j
                  dk(  r| j                  dk(  r| j                  «       S | S t        | t        j                  «      r| j                  «       S t        | t        «      r| S t        | t        «      r| j                  dd¬«      S t        | t        j                   «      r| j#                  «       S t%        | «      D �cg c]  }|j'                  d«      rŒ|‘Œ }}|D �ci c]  }| ‰t)        | |«      «      “Œ c}S c c}w c c}w )Nr   r   ©r   r?   r@   rA   Ú_)Ú
isinstancerL   Úndarrayr   ÚobjectrN   ÚflatÚemptyÚshapeÚndindexÚndimÚitemÚgenericrH   rP   rR   ÚvoidÚtolistÚdirrb   Úgetattr)ÚobjÚoutÚidxÚfÚnamesÚconvs        €r9   r’   z_load_classic.<locals>.conv÷   s~  ø€ Øˆ;ØÜ�cœ2Ÿ:™:Ô&¨3¯9©9¼Ò+>ð �x‰x˜1Š}¤Z°·±¸±¼R¿Z¹ZÔ%HÙ˜CŸH™H Q™KÓ(Ð(Ü—(‘(˜3Ÿ9™9¬FÔ3ˆCÜ—z‘z #§)¡)Ö,�Ù  C¡›>��C’ð -àˆJÜ�cœ2Ÿ:™:Ô&à�x‰x˜1Š} §¡¨Q¢Ø—x‘x“zÐ!ØˆJÜ�cœ2Ÿ:™:Ô&Ø—8‘8“:ÐÜ�cœ3ÔØˆJÜ�cœ5Ô!Ø—:‘:˜g¨i�:Ó8Ð8Ü�cœ2Ÿ7™7Ô#Ø—:‘:“<Ðä œHÓ>™H�q¨A¯L©L¸Õ,=’˜HˆÐ>Ù27Ó8±%¨Q�‘4œ  Q›Ó(Ñ(°%Ñ8Ð8ùò ?ùÚ8s   Æ6G4ÇG4ÇG9Ú__©r�   r   Úreturnr   )Úscipy.ioÚioÚloadmatÚitemsrb   )rc   Úsior2   rŽ   Úkru   r’   s         @r9   Ú_load_classicrœ   í   s_   ø€ õ à
�+‰+�d u¸uˆ+Ó
E€Cõ9ð: €CØ—	‘	–‰ˆˆ1Ø�<‰<˜ÔØÙ�a“ˆˆAŠð ð €Jr;   c                óÈ   ‡‡— ddl Ši }dˆˆfd„Š‰j                  | d«      5 }|j                  «       D ]  }|dk(  rŒ	 ‰||   «      ||<   Œ 	 ddd«       |S # 1 sw Y   |S xY w)a�  Load a MATLAB v7.3 (HDF5) .mat.

    Nested h5py objects are mapped to plain data:
      * numeric / char datasets  -> numpy arrays
      * reference (cell) datasets -> object-dtype numpy arrays with the same
        shape, each element resolved through its reference
      * groups (structs / containers) -> dicts of converted children
    The top-level ``#refs#`` bookkeeping group is skipped.
    r   Nc                ó\  •— t        | ‰
j                  «      r¦| j                  }|‰
j                  k(  rst	        j
                  | j                  t        ¬«      }| d   j                  «       }t        |«      D ]+  \  }} ‰	| j                  |   «      |j                  «       |<   Œ- |S t	        j                  | d   «      S t        | ‰
j                  «      r-i }| j                  «       D ]  \  }}|dk(  rŒ ‰	|«      ||<   Œ |S t        | ‰
j                  «      r ‰	| j                  |    «      S | S )Nr}   .ú#refs#)r   ÚDatasetr   Ú	ref_dtyperL   rƒ   r„   r�   rQ   Ú	enumerateÚfilerM   ÚGroupr™   Ú	Reference)r�   ÚdtÚresr‚   ÚiÚrefÚdr   ÚchildÚconv_objÚh5pys            €€r9   r¬   z_load_v73.<locals>.conv_obj*  sü   ø€ Ü�c˜4Ÿ<™<Ô(Ø—‘ˆBØ�T—^‘^Ò#Ü—h‘h˜sŸy™y´Ô7�Ø˜3‘x—~‘~Ó'�Ü'¨žo‘F�A�sÙ%-¨c¯h©h°s©mÓ%<�C—I‘I“K ’Nð .à�
Ü—:‘:˜c #™hÓ'Ð'Ü�c˜4Ÿ:™:Ô&Ø "ˆAØ"Ÿy™yž{‘��eØ˜8Ò#ØÙ" 5›/��$’ð  +ð ˆHÜ�c˜4Ÿ>™>Ô*Ù˜CŸH™H S™MÓ*Ð*Øˆ
r;   rw   rŸ   r”   )r­   ÚFileÚkeys)rc   rŽ   r�   r›   r¬   r­   s       @@r9   Ú	_load_v73r°     sh   ù€ ó à€Cöð* 
�‰�4˜Ô	 Ø—‘–ˆAØ�HŠ}ØÙ˜a ™d“^ˆC�ŠFñ ÷ 
ð
 €J÷ 
ð
 €Jús   ¢*AÁA!c                ó¬  — t        | t        j                  «      rí| j                  t        k(  rÚg }| j                  «       D ]Ã  }t        |t        j                  «      rD|j                  |j                  j                  dv r|j                  dk  rt        |«      n|«       Œat        |t        t        t        t        f«      r2|j                  t        |t        «      r|j                  «       n|«       Œ³|j                  |«       ŒÅ |S t        | t        j                  «      râ| j                  r3| j                  j                  dv r| j                  dk  rt        | «      gS t        j                   | «      }|j                  dk(  rfd|j"                  v rXt%        |j"                  d   dkD  r|j"                  d   n|j"                  d   «      D �cg c]  }|j                  «       |   ‘Œ c}S t'        |j                  «       «      S t        | t&        «      r5| D �cg c])  }t        |t        j                  «      rt        |«      n|‘Œ+ c}S | gS c c}w c c}w )zCReturn per-channel values of one chanlocs field (any storage form).)r   r   é    r   r   r   )r   rL   r€   r   r�   rQ   ÚappendrU   rN   r[   rH   rP   rn   rm   rR   r†   rM   r„   ÚrangeÚlist)ÚfieldÚvalsÚelÚarrr¨   rJ   s         r9   Ú_chan_names_valuesrº   K  sÀ  € ä�%œŸ™Ô$¨¯©¼Ò)>àˆØ—+‘+–-ˆBÜ˜"œbŸj™jÔ)Ø—‘°B·H±H×4EÑ4EÈÑ4OÐTV×T[ÑT[Ð^`ÒT`Ô,¨RÔ0ÐfhÕiÜ˜B¤¤e¬S´%Ð 8Ô9Ø—‘¬:°b¼%Ô+@˜BŸI™IœKÀbÕIà—‘˜B•ð  ð ˆÜ�%œŸ™Ô$Ø�:Š:˜%Ÿ+™+×.Ñ.°&Ñ8¸U¿Z¹ZÈ1º_Ü$ UÓ+Ð,Ð,Ü�j‰j˜ÓˆØ�8‰8�qŠ=˜Q #§)¡)™^Ü,1À#Ç)Á)ÈAÁ,ÐQRÒBR°#·)±)¸A²,ÐX[×XaÑXaÐbcÑXdÔ,eÓfÑ,e q�C—I‘I“K “NÐ,eÑfÐfÜ�C—I‘I“KÓ Ð Ü�%œÔÙQVÓWÑQVÈA¤z°!´R·Z±ZÔ'@Ô  Ô#ÀaÑGÐQVÑWÐWØˆ7€Nùò	 gùò Xs   ÇIÈ.Ic                ó®  — | €g S t        | t        j                  «      �rT| j                  t        k(  �r@t        | j                  «       «      }|�r$t        |d   t        «      �rg }|D �]  }ddi}|j                  «       D ]Û  \  }}|dv rOt        |t        «      s"t        t        |«      «      j                  «       nt        |«      j                  «       |d<   ŒYt        |t        t        t        f«      s+t        |t        j                  «      sŒ�|j                  dk(  sŒŸt        |t        j                  «      r|j                  «       n|||j!                  «       <   ŒÝ |j#                  |«       �Œ	 |S t        | t        «      �rú| j                  «       D ��ci c]  \  }}|dk7  sŒ||“Œ }}}d}|j%                  «       D ]‰  }t        |t        j                  «      rG|j&                  r;t)        ||j*                  d   dkD  r|j*                  d   n|j*                  d   «      }Œdt        |t
        «      sŒut)        |t-        |«      «      }Œ‹ |dk(  rH|j/                  d«      �5dt        t        |j/                  dd«      «      «      j                  «       igS g S g }t1        |«      D ]Ô  }	ddi}|j                  «       D ]ª  \  }}t3        |«      }
|
sŒt-        |
«      |k(  r|
|	   n|
d   }|dv rQt        |t        «      s"t        t        |«      «      j                  «       nt        |«      j                  «       }||d<   Œ�t        |t        t        f«      sŒ˜|||j!                  «       <   Œ¬ |j#                  |«       ŒÖ |S t        | t
        «      r;t5        d| D �cg c]$  }|j/                  d|j/                  dd«      «      ‘Œ& c}i«      S g S c c}}w c c}w )	aY  Normalise neural.chanlocs to a list of per-channel dicts with 'name'.

    Handles three storage styles seen in the CND files:
      1. classic v7 struct array -> list of dicts (labels = channel names)
      2. v7.3 scalar struct whose fields are per-channel cells/columns
      3. v7.3 plain struct-array group -> list of per-element dicts
    r   r   r=   )Úlabelsr   r   rŸ   éÿÿÿÿr¼   )r   rL   r€   r   r�   rµ   rQ   Údictr™   rH   r[   rT   rn   rm   rN   r‡   Úlowerr³   rt   r†   Úmaxr„   ÚlenÚgetr´   rº   Ú_parse_chanlocs)Úchanlocsr™   ÚrecsÚitÚrecr›   ru   ÚfieldsÚnr¨   r·   ÚvalÚnmÚcs                 r9   rÃ   rÃ   d  s=  € ð ÐØˆ	ô �(œBŸJ™JÕ'¨H¯N©N¼fÓ,DÜ�X—^‘^Ó%Ó&ˆÚ”Z  a¡¬$Õ/ØˆDÜ�Ø˜r�l�ØŸH™HžJ‘D�A�qØÐ.Ñ.ÜNXÐYZÔ\_ÔN`¤cÔ*:¸1Ó*=Ó&>×&DÑ&DÔ&FÔfiÐjkÓfl×frÑfrÓft˜˜FšÜ# A¬¬U´CÐ'8Ô9¼jÈÌBÏJÉJÕ>WÐ\]×\bÑ\bÐfgÓ\gÜ5?ÀÄ2Ç:Á:Ô5N¨¯©¬ÐTU˜˜AŸG™G›Išð	 'ð
 —‘˜CÖ ð ð ˆKä�(œDÕ!Ø#+§>¡>Ô#3ÔEÑ#3™4˜1˜a°q¸H³}�!�Q‘$Ð#3ˆÑEàˆØ—‘–ˆAÜ˜!œRŸZ™ZÔ(¨Q¯VªVÜ˜¨¯©°©°aª˜1Ÿ7™7 1š:¸Q¿W¹WÀR¹[ÓI‘Ü˜AœtÕ$Ü˜œ3˜q›6“N‘ð	 !ð
 �Š6ð —:‘:˜hÓ'Ð3ð œSÔ!1°&·*±*¸XÀrÓ2JÓ!KÓL×RÑRÓTÐUÐVð <Ø9;ð<à%'ˆÜ�q–ˆAØ#)¨2 ,ˆCØŸ™ž‘��1Ü)¨!Ó,�ÙØÜ!$ T£¨a¢�d˜1’g°T¸!±W�ØÐ*Ñ*ÜCMÈcÔSVÔCWœÔ-¨cÓ2Ó3×9Ñ9Ô;Ô]`ÐadÓ]e×]kÑ]kÓ]m�BØ"$�C˜’KÜ ¤c¬5 \Õ2Ø%(�C˜Ÿ™›	’Nð 'ð �K‰K˜Õð ð ˆä�(œDÔ!Ü ÑW_Ó*`ÑW_ÐRS¨1¯5©5°¸1¿5¹5ÀÈÓ;LÕ+MÐW_Ñ*`ÐaÓbÐbà€Iùó? Fùò: +as   ÆOÆ&OÎ)O
c                ó²  ‡ ‡— ‰ j                  d«      }|�t        t        |«      d   «      n
t        d«      }‰ j                  d«      }g Š|�¤t        |t        j
                  «      r|nt	        j                  |t        ¬«      }|j                  t        k7  rt	        j                  |«      gŠnCt	        j                  |j                  «      D �cg c]  }t	        j                  ||   «      ‘Œ c}Št        ‰ j                  d«      «      }dˆ ˆfd„} |d	«      }	 |d
«      }
d}‰ j                  d«      �5t        ‰ d   «      }|j                  r|D �cg c]  }t        |«      ‘Œ c}nd}|‰||	|
|‰ j                  d«      �t        ‰ j                  d«      «      nd‰ j                  d«      �t        ‰ j                  d«      «      |dœ	S d|dœ	S c c}w c c}w )z>Normalise the ``neural`` struct into plain numpy-based fields.ÚfsNr   ÚnanÚdatar}   rÄ   c                óŠ  •— ‰j                  | «      }|€d gt        ‰«      z  S t        |t        j                  «      r|nt        j
                  |t        ¬«      }|j                  t        k7  rt        |«      gS g }t        j                  |j                  «      D ]%  }||   }|j                  |�t        |«      nd «       Œ' |S )Nr}   )rÂ   rÁ   r   rL   r€   rM   r�   r   rh   r…   r„   r³   )r   rÌ   r¹   rŽ   r�   r¸   ÚneuralÚtrialss         €€r9   Úcell_seriesz$_extract_neural.<locals>.cell_series±  s    ø€ Ø�J‰J�tÓˆØˆ9Ø�6œC ›KÑ'Ð'Ü˜a¤§¡Ô,‰a´"·*±*¸QÄfÔ2MˆØ�9‰9œÒÜ˜C“L�>Ð!Ø*,ˆÜ—:‘:˜cŸi™iÖ(ˆCØ�S‘ˆBØ�J‰J b n”w˜r”{¸$Õ?ð )ð ˆ
r;   ÚattendedStreamÚattendedAzimuthÚorigTrialPositionÚdataTypeÚ
deviceName)	rÎ   rÓ   rÄ   Úattended_streamÚattended_azimuthÚorig_trial_positionÚ	data_typeÚdevice_nameÚsource_path)r   rH   r•   zList[Optional[np.ndarray]])rÂ   rm   rh   r   rL   r€   rM   r�   r   r…   r„   rÃ   rN   rn   r[   )rÒ   rß   Úfs_rawrÎ   Ú	data_cellr¹   r¨   rÄ   rÔ   rÚ   rÛ   Úorig_posrJ   rÓ   s   `            @r9   Ú_extract_neuralrã   ¡  s®  ù€ à�Z‰Z˜Ó€FØ&,Ð&8ŒŒw�v‹˜qÑ!Ô	"¼eÀE»l€Bà—
‘
˜6Ó"€IØ!€FØÐÜ% i´·±Ô<‰iÄ"Ç*Á*ÈYÔ^dÔBeˆØ�9‰9œÒÜ—j‘j “oÐ&‰Fä24·*±*¸S¿Y¹YÔ2GÓHÑ2G¨Q”b—j‘j  Q¡Õ(Ð2GÑHˆFä˜vŸz™z¨*Ó5Ó6€Höñ "Ð"2Ó3€OÙ"Ð#4Ó5Ðà€HØ‡z�zÐ%Ó&Ð2Ü�fÐ0Ñ1Ó2ˆØ,/¯HªH¡CÓ(¡C˜q”C˜•F CÒ(¸$ˆð ØØØ*Ø,Ø'ØAGÇÁÈJÓAWÐAcÔ% f§j¡j°Ó&<Ô=ÐimØEKÇZÁZÐP\ÓE]ÐEiÔ'¨¯
©
°<Ó(@ÓAØ"ñ
ð 
ð ptØ"ñ
ð 
ùò3 Iùò. )s   ÃGÅGc           	     ó  — | j                  d«      }|�t        t        |«      d   «      n
t        d«      }| j                  d«      }t        |«      D �cg c]  }g ‘Œ }}|��¦t	        |t
        j                  «      r|nt        j                  |t        ¬«      }|j                  t        k(  �r[|j                  dk(  �r¿|j                  d   |k(  rdt        |j                  d   «      D ]G  }	t        |«      D ]7  }
||
   j                  t        ||	|
f   «      j                  t        «      «       Œ9 ŒI �nÕ|j                  d   |k(  rdt        |«      D ]T  }
t        |j                  d   «      D ]7  }	||
   j                  t        ||
|	f   «      j                  t        «      «       Œ9 ŒV �n_|j                  «       }|D �cg c]8  }t        j                  |«      j                  «       j                  t        «      ‘Œ: }}t        |«      D �cg c]  }g ‘Œ }}t        |«      |z  }t        |«      D ]K  }
t        |«      D ];  }	||
   j                  t        ||
|z  |	z      «      j                  t        «      «       Œ= ŒM nŒ|j                  «       }t        |«      |z  }t        |«      D ]`  }
t        |«      D ]P  }	|
|z  |	z   t        |«      k  sŒ||
   j                  t        ||
|z  |	z      «      j                  t        «      «       ŒR Œb t!        |«      st#        d|› d	�«      ‚g }| j                  d
«      �À| d
   }t	        |t
        j                  «      r|nt        j                  |t        ¬«      }|j                  «       D ]q  }t	        |t$        «      r|j                  |«       Œ%t	        |t&        «      r#|j                  |j)                  dd¬«      «       ŒX|j                  t+        |«      «       Œs ||||dœS c c}w c c}w c c}w )zDNormalise ``stim`` into {fs, per_trial: [[env_1, ..., env_K], ...]}.rÎ   r   rÏ   rÐ   r}   r   r   zstim.data could not be parsed (Ú)r‘   r?   r@   rA   )rÎ   Ú	per_trialr‘   rß   )rÂ   rm   rh   r´   r   rL   r€   rM   r�   r   r†   r„   r³   rl   rQ   rÁ   ÚanyÚ
ValueErrorrH   rP   rR   r[   )ÚstimÚn_trialsrß   rà   rÎ   Úcellr~   ræ   r¹   rZ   Útr‚   Úer›   r‘   Únm_arrr¸   s                    r9   Ú_extract_stimrï   Ó  sP  € ð �X‰X�d‹^€FØ&,Ð&8ŒŒw�v‹˜qÑ!Ô	"¼eÀE»l€Bà�8‰8�FÓ€DÜ5:¸8´_Ó(E±_°ª°_€IÐ(EØÑÜ  ¤r§z¡zÔ2‰d¼¿
¹
À4ÌvÔ8VˆØ�9‰9œÓØ�x‰x˜1‹}à—9‘9˜Q‘< 8Ò+Ü" 3§9¡9¨Q¡<Ö0˜Ü!& x¦˜AØ% a™L×/Ñ/´¸¸A¸q¸D¹	Ó0B×0IÑ0IÌ%Ó0PÕQñ "1ò 1ð —Y‘Y˜q‘\ XÒ-Ü" 8ž_˜Ü!& s§y¡y°¡|Ö!4˜AØ% a™L×/Ñ/´¸¸A¸q¸D¹	Ó0B×0IÑ0IÌ%Ó0PÕQñ "5ò -ð Ÿ9™9›;�DÙNRÓ SÉdÈ¤§¡¨A£×!4Ñ!4Ó!6×!=Ñ!=¼eÕ!DÈd�IÐ SÜ-2°8¬_Ó =©_¨¢¨_�IÐ =Ü˜D›	 XÑ-�AÜ" 8ž_˜Ü!& q¦˜AØ% a™L×/Ñ/´¸¸QÀ¹UÀQ¹Y¹Ó0H×0OÑ0OÔPUÓ0VÕWñ "*ñ -ð —y‘y“{�Ü˜“I Ñ)�Ü˜xž�AÜ" 1žX˜Ø˜q™5 1™9¤s¨4£yÓ0Ø% a™L×/Ñ/´¸¸QÀ¹UÀQ¹Y¹Ó0H×0OÑ0OÔPUÓ0VÕWñ &ð )ô ˆyŒ>ÜÐ:¸;¸-ÀqÐIÓJÐJà€EØ‡x�x�ÓÐ$Ø�g‘ˆÜ" 6¬2¯:©:Ô6‰f¼B¿J¹JÀvÔU[Ô<\ˆØ—)‘)–+ˆBÜ˜"œcÔ"Ø—‘˜RÕ Ü˜B¤Ô&Ø—‘˜RŸY™Y w°y˜YÓAÕBà—‘Ô-¨bÓ1Õ2ð ð  9°uÈ[ÑYÐYùòW )Fùò  !TùÚ =s   Á	O<Æ==PÈ		Pc                ód  — | �| j                   dk(  ryt        | d   «      }t        j                  |«      st        j                  |«      rt        | «      }|�t        |«      n|}|dk(  rt        t        t        |«      «      «      S |dk(  r|dkD  rdS |dk  rdS dS |dk(  r|dkD  rdS |dk  rd	S dS y)
zÍSuperHuge-style trial label from the attended azimuth at trial onset.

    Falls back to the modal azimuth when sample 0 is not informative. Returns
    None for "none" style / uninformative azimuths.
    Nr   r*   r   ÚRÚLr   ÚrightÚleft)	rN   rm   rL   rk   Úisinfrx   rH   rn   Úround)ÚazimuthÚstyleÚaz0Úms       r9   Ú_label_from_azimuthrû     s¸   € ð €˜'Ÿ,™,¨!Ò+ØÜ
�˜‘
Ó
€CÜ	‡x�x�„}œŸ™ œÜ�gÓˆØ˜-Œe�AŒh¨SˆØ�ÒÜ”3”u˜S“z“?Ó#Ð#Ø�‚}Ø˜A’gˆsÐ=¨3°ª7 CÐ=¸Ð=Ø�ÒØ š'ˆwÐD°#¸²'¨ÐD¸tÐDØr;   c                ó„  — t        | «      }g }| D ]n  }t        |«      }|j                  |dz
  k  rt        d|j                  › d|› d�«      ‚|j	                  |d| j                  t        j                  d¬«      «       Œp d}|�7|j                  r+|d| }t        |«      }	|	�d|	cxk  r|k  rn nt        |	«      }t        t        |«      «      }
|�)|dz
  gt        |«      D �cg c]  }||dz
  k7  sŒ|‘Œ c}z   }
t        j                  |
D �cg c]  }||   ‘Œ	 c}d¬«      }|j                  |d|«      |fS c c}w c c}w )	a»  Build the ``(T, 1, K)`` envelope array, attended stream in column 0.

    The attended column is the stream most frequently labelled attended across
    the trial (1-based CND indices). For constant-attention trials this is
    exact; for attention-switch trials the column-0 stream is the
    majority-attended one and the per-sample ground truth remains available in
    ``extra_info["attention"]`` so switch locations stay recoverable.
    r   zenvelope length z  is shorter than the EEG length z by more than 1 sampleNF©Úcopy)Úaxis)rÁ   rh   rN   rè   r³   rl   rL   Úfloat64rx   rn   rµ   r´   ÚstackÚreshape)ÚenvsrÚ   ÚlengthÚKÚtrimmedrí   ru   Úattended_idxÚwinÚmodeÚorderÚjÚmats                r9   Ú_attended_envr  !  sQ  € ô 	ˆD‹	€AØ "€GÛˆÜ�A‹JˆØ�6‰6�F˜Q‘JÒÜØ" 1§6¡6 (Ð*JØ�(Ð0ð2óð ð 	�‰�q˜˜&�z×(Ñ(¬¯©¸%Ð(Ó@ÕAð ð #'€LØÐ" ×';Ò';Ø˜g˜vÐ&ˆÜ˜‹~ˆØÐ  T¤¨Q¥Ü˜t›9ˆLä”�q“‹N€EØÐØ Ñ!Ð"´°q´Ó%S±¨A¸QÀ,ÐQRÑBRÓ=R¢a°Ñ%SÑSˆä
�(‰(©Ó.© 1�G˜A“J¨Ñ.°QÔ
7€CØ�;‰;�v˜q !Ó$ lÐ2Ð2ùò &Tùâ.s   Ã$D8Ã2D8ÄD=c                ó|  — |s| |fS |D �ch c]  }|j                  «       ’Œ }}t        |«      D ��cg c]  \  }}|d   j                  «       |vsŒ|‘Œ }}}t        |«      D ��cg c]  \  }}||vsŒ|d   ‘Œ }}}|rt        d|› �«       | dd…|f   |D �cg c]  }||   ‘Œ	 c}fS c c}w c c}}w c c}}w c c}w )zGRemove channels by name (case-insensitive) from EEG columns + chanlocs.r   z      dropped channels: N)r¿   r¢   r:   )	ÚeegrÄ   Údroprª   Údrop_lowr¨   rÌ   Úkeep_idxÚdroppeds	            r9   Ú_drop_channelsr  I  sÐ   € ñ Ø�Hˆ}ÐÙ#'Ó(¡4˜a�—‘•	 4€HÐ(Ü'¨Ô1ÔWÑ1‘d�a˜°Q°v±Y·_±_Ó5FÈhÒ5V’Ð1€HÑWÜ%.¨xÔ%8ÔNÑ%8™T˜Q ¸AÀXÒ<Mˆq�‹yÐ%8€GÑNÙÜÐ& w iÐ0Ô1ØŠq�(ˆ{Ñ±8Ó<±8¨a˜h q›k°8Ñ<Ð<Ð<ùò )ùÛWùÛNùò =s"   ‹B(²B-ÁB-Á%B3Á2B3ÂB9c	                ó¦  — t        | d   «      t        |«      t        |«      t        |«      t        t        |«      «      t        t        |«      «      | d   › d|› �t        |«      D �	�
ci c]  \  }	}
|	dz   d|
d   i“Œ c}
}	t        |«      i dœ
}|�||d<   d|d<   d|d	<   d
|d<   d|d<   t        t        |«      «      |d<   d
|d<   d
|d<   |S c c}
}	w )zhAssemble one metadata.pkl entry strictly following the SuperHuge
    reference rows of the same dataset.r   r   r~   r   )
r   Ú
subject_idÚtrial_idÚsignal_lengthÚnum_channelrÎ   Údataset_nameÚchannel_infosÚnum_speakersÚ
extra_infoÚlabelr=   Ústimuli_nameÚcompet_stimuli_namer   ÚwavÚenvÚenv_fsÚmelÚwav2vec2)rn   rÁ   rö   r¢   )ÚdatasetÚstager  r  rÎ   r  rÄ   Ú	n_streamsr  r¨   rÌ   Úentrys               r9   Ú_build_entryr*  a  sø   € ô ˜' ,Ñ/Ó0Ü˜*“oÜ˜“MÜ˜]Ó+Üœ3˜x›=Ó)Ü”%˜“)‹nØ" 6™?Ð+¨1¨U¨GÐ4ÜAJÈ8ÔATÔUÑAT¹¸¸A˜!˜a™% &¨!¨F©)Ð!4Ñ4ÐATÒUÜ˜I›Øñ€Eð ÐØˆˆg‰Ø€Eˆ.ÑØ#%€EÐ
Ñ Ø€Eˆ%�LØ€Eˆ%�LÜœ% ›)“n€Eˆ(�OØ€Eˆ%�LØ€Eˆ*ÑØ€Lùó Vs   Á)Cc           
     ó  — |D ]Ù  }t         j                  j                  | |«      }t         j                  j                  |«      sŒCt	        d„ t        j
                  |«      D «       «      }|sŒkt         j                  j                  ||d   «      }	 t        |«      }t        t        |«      |z
  «      dk  r|t        |«      fc S t        d|› d|d	›d
|d	›d�«       ŒÛ y# t        $ r}t        d|› d|› �«       Y d}~Œüd}~ww xY w)z@Pick the CND sub-folder whose neural.fs == target_fs (in order).c              3  óL   K  — | ]  }t         j                  |«      sŒ|–— Œ y ­wrG   )ÚSUBJECT_FILE_REÚmatch)rI   rÉ   s     r9   rK   z'_find_subject_subdir.<locals>.<genexpr>Š  s   è ø€ ÐL¡-˜Q´?×3HÑ3HÈÕ3K”q¡-ùs   ‚$�$r   z"    [warn] could not read fs from ú: Nç�íµ ÷Æ°>z    [info] z: fs=Úgz Hz != target z Hz, skipping)Úosrc   rX   ÚisdirÚsortedÚlistdirÚ_read_neural_fsÚ	Exceptionr:   Úabsrm   )	Údataset_dirÚ
candidatesÚ	target_fsÚsubrª   ÚfilesÚprÎ   Úexcs	            r9   Ú_find_subject_subdirr@  ƒ  sñ   € ó ˆÜ�G‰G�L‰L˜ cÓ*ˆÜ�w‰w�}‰}˜QÔØÜÑL¤"§*¡*¨Q¤-ÓLÓLˆÙØÜ�G‰G�L‰L˜˜E !™HÓ%ˆð	Ü  Ó#ˆBô Œu�R‹y˜9Ñ$Ó%¨Ò,Øœ˜b›	�>Ò!Üˆk˜#˜˜e B q 6¨¸	À!°}ÀMÐRÕSð ð  øô ò 	ÜÐ4°Q°C°r¸#¸Ð?Ô@Üûð	ús   ÂC Ã 	DÃ)C?Ã?Dc                ó6  — t        | «      dk(  r@ddl}|j                  | d«      5 }t        t	        |d   d   d   «      d   «      cddd«       S ddlm} |j                  | dd	dg¬
«      }t        t	        |d   j                  «      d   «      S # 1 sw Y   ŒIxY w)z0Read ``neural.fs`` without loading the full EEG.r^   r   Nrw   rÒ   rÎ   .TF)rz   r{   Úvariable_names)	rf   r­   r®   rm   rh   r–   r—   r˜   rÎ   )rc   r­   r�   rš   rú   s        r9   r6  r6  ™  s‘   € ä�4Ó˜DÒ Ûà�Y‰Y�t˜SÔ! QÜœ  8¡¨TÑ!2°3Ñ!7Ó8¸Ñ;Ó<÷ "Ñ!åà�‰�D T¸EØ$, :ð 	ó 	/€Aä”˜˜8™Ÿ™Ó(¨Ñ+Ó,Ð,÷ "Ð!ús   ¥ BÂBc                óì  — t         j                  j                  | |› d|› d�«      }t         j                  j                  | |› d|› d�«      }t         j                  j                  |«      s#t         j                  j                  | |› d�«      }t         j                  j                  |«      st	        |«      ‚t         j                  j                  |«      st	        |«      ‚t        |«      dk(  rt        |«      n
t        |«      }t        |«      dk(  rt        |«      n
t        |«      }t        |j                  d|«      |«      }t        |j                  d|«      t        |d   «      |«      }||||fS )	a"  Load and normalise the (neural, stim) pair of one subject file number.

    ``prefix`` selects the CND file family: "" for ``dataSub/dataStim`` (raw)
    and ``"pre_"`` for ``pre_dataSub/pre_dataStim`` (the pre-processed CND
    variant shipped next to the raw files in the source folders).ÚdataSubz.matÚdataStimzdataStim.matr^   rÒ   ré   rÓ   )r2  rc   rX   ÚexistsÚFileNotFoundErrorrf   r°   rœ   rã   rÂ   rï   rÁ   )	Úsub_dirÚsub_numÚprefixÚp_subÚp_stimÚraw_nÚraw_srÒ   ré   s	            r9   Ú
_load_pairrO  §  s+  € ô �G‰G�L‰L˜ V H¨G°G°9¸DÐ"AÓB€EÜ�W‰W�\‰\˜' f X¨X°g°Y¸dÐ#CÓD€FÜ�7‰7�>‰>˜&Ô!Ü—‘—‘˜g¨&¨°Ð'>Ó?ˆÜ�7‰7�>‰>˜%Ô Ü Ó&Ð&Ü�7‰7�>‰>˜&Ô!Ü Ó'Ð'Ü +¨EÓ 2°dÒ :ŒI�eÔÄÈeÓ@T€EÜ!,¨VÓ!4¸Ò!<ŒI�fÔÄ-ÐPVÓBW€EÜ˜UŸY™Y x°Ó7¸Ó?€FÜ˜Ÿ™ 6¨5Ó1´3°v¸hÑ7GÓ3HÈ&ÓQ€DØ�5˜$ Ð&Ð&r;   rÐ   Tc	                óº  ‡=‡>— | d   }	t        | d   «      }
t        j                  j                  ||	«      }t        j                  j	                  |«      st        d|› �«      ‚|dk(  rdnd}|dk(  rt        nt        Š>t        | j                  d«      xs d«      }t        j                  j	                  t        j                  j                  |d	«      «      rd	g|D �cg c]	  }|d	k7  r|‘Œ c}z   }t        |||«      }|€t        |	› d|d›d|› �«      ‚|\  }}t        j                  j                  ||«      }t        d|	› d|› d|
› d|› d�	«       t        ˆ=ˆ>fd„t        j                  |«      D «       «      }|dk(  r&|s$t        d|› d�«       |	|
|| d   › d|› �dd
d
g i ddœ
S |st        |	› d|› d|› �«      ‚|rFt        |«      }|D �cg c]	  }||v sŒ|‘Œ }}t        |t        |«      z
  «      }|rt        d|› �«       |st        |	› d|› d�«      ‚t!        |«      D ��ci c]  \  }}||dz   “Œ }}}t        j                  j                  ||d «      }t        j                  j                  ||d!d"«      }t        j                  j                  ||d#«      }|||fD ]  }t        j"                  |d¬$«       Œ t%        j&                  | j                  d%t$        j(                  «      «      }i }d} d
}!g }"t        |«      D �]æ  }#||#   }$	 t+        ||#|«      \  }%}&}'}(t-        |%d(   «      }*t%        j.                  |*«      rt        d)|#› d*�«      ‚t1        |'d(   |*z
  «      d+kD  rt        d)|#› d,|'d(   d›d-|*d›�«      ‚|d.k(  r't1        |*|z
  «      d+kD  rt        d)|#› d/|*d›d0|d›�«      ‚|"j3                  |*«       t5        |%d1   «      }+|%d2   },t5        |'d3   «      |+k7  r!t        d)|#› d't5        |'d3   «      › d4|+› d5�«      ‚t        d6|$› d7|#› d8|+› d9t5        |,«      › d:|*d›d;�«       t7        |+«      D �]¶  }-|-dz   }.t8        j;                  |
|$|.¬<«      }/t%        j<                  |%d1   |-   «      }0t5        |,«      }1|0j>                  d=k7  rtA        |/› d>|0jB                  › �«      ‚|0jB                  d   |1k(  r|0jB                  d   |1k7  r|0jD                  }0|0jB                  d   |1k7  r tA        |/› d?|0jB                  d   › d@|1› �«      ‚tG        |0|,| j                  dAg «      «      \  }0}2|'d3   |-   }3tI        dB„ |3D «       «      }4t        tI        |0jB                  d   |4«      «      }5|5|0jB                  d   k  r|0d
|5 }0tK        |3|%dC   |-   |5«      \  }6}7d
}8| dD   dEk7  rtM        |%dF   |-   | dD   «      }8tO        | ||$|.|*|5|2t5        |3«      |8¬G«	      }9t        j                  j                  ||/› dH�«      }:t        j                  j                  ||/› dI�«      };|rÃt        j                  jQ                  |:«      st        j                  jQ                  |;«      r|stS        dJ|/› dK�«      ‚t%        jT                  |:t%        jV                  |0jY                  |dL¬M«      «      «       t%        jT                  |;t%        jV                  |6jY                  |dL¬M«      «      «       |9||/<   | dz  } t5        |2«      }!�Œ¹ �Œé |"r't[        t-        t%        j\                  |"«      «      «      nd
}<t        dN| › dO|!› dP|<› dQ|rdRndS› dT�	«       |"r't[        t-        t%        j\                  |"«      «      «      nd
}<t        dN| › dU|!› dP|<› dV�«       |	|
| d   › d|› �| |!|<t        |«      D �cg c]  }||   ‘Œ	 c}||dLdWœ
S c c}w c c}w c c}}w # t
        $ r})t        d&|#› d'|)› �«       Y d
})~)�ŒÐd
})~)ww xY wc c}w )Xa¶  Convert one CND source folder into one SuperHuge stage directory.

    ``variant`` selects the CND file family:
      * ``"data"`` -> ``dataSub/dataStim`` files (written to ``stage`` = "raw")
      * ``"pre"``  -> ``pre_dataSub/pre_dataStim`` files (stage "preprocessed")
    Both variants keep the same dataset_id / subject / trial numbering; the
    pre-processed CND files carry their own fs (typically 32 Hz after the
    1-9 Hz band-pass + downsampling of the KUL CND toolchain), which is stored
    verbatim in the metadata of the "preprocessed" stage.
    With ``write_files=False`` only the metadata entries are recomputed (the
    eeg/env .npy files are assumed to exist already).
    r   r   zdataset folder not found: ÚpreÚpre_r=   r   © r&   Nz: no CND sub-folder with fs == r1  z Hz found in z=== ú [z] (dataset_id=z, dir=z) ===c              3  óz   •K  — | ]2  }‰j                  |«      xŠsŒt        ‰j                  d «      «      –— Œ4 y­w)r   N)r.  rn   Úgroup)rI   rÉ   rú   Úsub_res     €€r9   rK   z"convert_dataset.<locals>.<genexpr>ç  s4   øè ø€ ð Ù!4˜A¸f¿l¹lÈ1»oÐ9M¸Ñ9MŒˆA�G‰G�A‹J�Ñ!4ùs   ƒ;œ;z    [info] no z*dataSub<N>.mat files - skipping this stager   r~   r   T)
r   r   r'  r  rÓ   ÚchannelsrÎ   ÚsubjectsÚmetadataÚskippedz: no zdataSub<N>.mat files in z7    [warn] requested subject file number(s) not found: zdataSub<N>.mat files to processr   r  Ústimulir"  Úmeta)Úexist_okr   z    [warn] skipping subject r/  rÎ   r<  z: could not read neural.fsr0  z
: stim.fs z != neural fs rÐ   z: neural.fs z != expected rÓ   rÄ   ræ   z stim trials vs z neural trialsz    subject z (file ú): ú	 trials, z channels, fs=ú Hz)ÚdidÚsidÚtidr   z: expected 2-D EEG, got z: EEG columns z != chanlocs count r   c              3  óX   K  — | ]"  }t        t        |«      j                  «      –— Œ$ y ­wrG   )rn   rh   rN   )rI   rí   s     r9   rK   z"convert_dataset.<locals>.<genexpr><  s   è ø€ ÐD¹°1œ#œg a›jŸo™o×.¹ùs   ‚(*rÚ   r   r   rÛ   )	r&  r'  r  r  rÎ   r  rÄ   r(  r  z.npyz_env.npyzoutput files already exist for z; use --overwriteFrý   z    -> wrote z trial entries x z ch (fs=z Hz, files=ÚyesÚkeptrå   z
 trials x z Hz))
r   r   r  rÓ   rX  rÎ   rY  rZ  r'  r[  )/rn   r2  rc   rX   r3  rG  ÚPRE_SUBJECT_FILE_REr-  rµ   rÂ   r@  ÚRuntimeErrorr:   r4  r5  Úsetr¢   ÚmakedirsrL   r   r   rO  rm   rk   r8  r³   rÁ   r´   ÚFILE_KEYÚformatrM   r†   rè   r„   ÚTr  rs   r  rû   r*  rF  ÚFileExistsErrorÚsaveÚasfortranarrayrl   rö   Úmean)?r&  Úsource_rootÚout_rootr'  ÚvariantÚwrite_filesÚ	overwriteÚsubjects_onlyr;  r   rb  Úds_dirrJ  r:  rÌ   ÚfoundÚsub_nameÚ_raw_fsrH  Úsubject_numsÚallowedrÉ   Úmissingr¨   ÚnumÚsubject_idsÚeeg_dirÚenv_dirÚmeta_dirrª   r   Úmetadata_entriesÚtotal_trialsÚ
n_chan_outÚ	fs_valuesrI  rc  rÒ   rK  ré   rL  r?  Úfs_subÚn_trÚchanlocs_allrì   rd  Úkeyr  Ún_srcrÄ   Úenv_streamsÚenv_lenr  r"  r~   r  r)  Úeeg_pathÚenv_pathÚfs_outrú   rW  s?                                                                @@r9   Úconvert_datasetr“  ½  s  ù€ ð" �XÑ€FÜ
ˆg�lÑ#Ó
$€CÜ�W‰W�\‰\˜+ vÓ.€FÜ�7‰7�=‰=˜Ô ÜÐ"<¸V¸HÐ EÓFÐFà %Ò'‰V¨R€FØ$+¨uÒ$4Õ ¼/€Fô
 �g—k‘kÐ"5Ó6Ò<¸"Ó=€JÜ	‡w�w‡}�}”R—W‘W—\‘\ &Ð*=Ó>Ô?Ø)Ð*¹ó .K¹°AØ12Ð6IÒ1Iò /0¸ñ .Kñ Kˆ
ä  ¨°YÓ?€EØ€}ÜØˆhÐ5°iÀ°]À-Øˆlðóð 	ð Ñ€HˆgÜ�g‰g�l‰l˜6 8Ó,€GÜˆ$ˆvˆh�b˜˜˜~¨c¨U°&¸¸
À%ÐHÔIäô Ü!#§¡¨GÔ!4óó €Lð �%Ò¡Üˆn˜V˜HÐ$NÐOÔPØ °¸eØ#*¨6¡?Ð"3°1°U°GÐ <Ø¨°TØ¨B¸4ñAð 	Añ ÜØˆh�e˜F˜8Ð#;¸G¸9ÐEóGð 	GáÜ�mÓ$ˆÙ#/Ó@¡<˜a°1¸²<š <ˆÐ@Ü˜¤3 |Ó#4Ñ4Ó5ˆÙÜÐIÈ'ÈÐSÔTÙÜ˜f˜X U¨6¨(Ð2QÐRÓSÐSô -6°lÔ,CÔDÑ,C¡& ! S�3˜˜A™‘:Ð,C€KÑDä�g‰g�l‰l˜8 U¨EÓ2€GÜ�g‰g�l‰l˜8 U¨I°uÓ=€GÜ�w‰w�|‰|˜H e¨VÓ4€HØ�w Ó)ˆÜ
�‰�A Ö%ð *ô �H‰H�W—[‘[ ¬"¯*©*Ó5Ó6€EØ')ÐØ€LØ $€JØ€Iä˜,×'ˆØ˜'Ñ"ˆð	Ü*4°W¸gÀvÓ*NÑ'ˆF�E˜4 ô �v˜d‘|Ó$ˆÜ�8‰8�FÔÜ  W IÐ-GÐHÓIÐIÜˆt�D‰z˜FÑ"Ó# dÒ*ÜØ�g�Y˜j¨¨d©°A¨°nÀVÈAÀJÐOóQð Qà�fÒ¤ V¨iÑ%7Ó!8¸4Ò!?ÜØ�g�Y˜l¨6°!¨*°MÀ)ÈAÀÐOóQð Qà×Ñ˜Ô ä�6˜(Ñ#Ó$ˆØ˜jÑ)ˆÜˆt�KÑ Ó! TÒ)ÜØ�g�Y˜b¤ T¨+Ñ%6Ó!7Ð 8Ð8HØ�&˜ð(ó)ð )ô 	ˆl˜3˜%˜w w i¨s°4°&¸	Ü�<Ó Ð! °°q¨z¸ð>ô 	?ô �t—ˆAØ�a‘%ˆCÜ—/‘/ c¨s¸�/Ó<ˆCä—*‘*˜V HÑ-¨aÑ0Ó1ˆCÜ˜Ó%ˆEØ�x‰x˜1Š}Ü  C 5Ð(@ÀÇÁÀÐ!LÓMÐMà�y‰y˜‰|˜uÒ$¨¯©°1©¸Ò)>Ø—e‘e�Ø�y‰y˜‰|˜uÒ$Ü Ø�e˜>¨#¯)©)°A©,¨Ð7JÈ5È'ÐRóTð Tä*¨3°Ø+2¯;©;°ÈÓ+KóM‰MˆC�à˜{Ñ+¨AÑ.ˆKô ÑD¹ÓDÓDˆGÜœ˜SŸY™Y q™\¨7Ó3Ó4ˆFØ˜Ÿ	™	 !™Ò$Ø˜'˜6�l�Ü" ;Ø#)Ð*;Ñ#<¸QÑ#?ÀóI‰FˆC�ð ˆEØ�}Ñ%¨Ò/Ü+¨FÐ3EÑ,FÀqÑ,IØ,3°MÑ,BóD�ô !Ø u¸ÀsØ¨¸(Ü˜kÓ*°%ô9ˆEô
 —w‘w—|‘| G°¨u°D¨\Ó:ˆHÜ—w‘w—|‘| G°¨u°HÐ-=Ó>ˆHÙÜ—G‘G—N‘N 8Ô,´·±·±¸xÔ0HÑR[Ü)Ø9¸#¸Ð>OÐPóRð Rä—‘˜¤"×"3Ñ"3°C·J±J¸uÈ5°JÓ4QÓ"RÔSÜ—‘˜¤"×"3Ñ"3°C·J±J¸uÈ5°JÓ4QÓ"RÔSØ$)Ð˜SÑ!Ø˜AÑˆLÜ˜X›ŠJòc ð7 (ñ\ 2;ŒU”5œŸ™ Ó+Ó,Ô-À€FÜˆ-˜�~Ð%6°z°lð CØˆh�k©;¡%¸FÐ!CÀ1ð	Fô Gñ 2;ŒU”5œŸ™ Ó+Ó,Ô-À€FÜˆ-˜�~ Z°
¨|¸8ÀFÀ8È4ÐPÔQà¨Ø" 6™?Ð+¨1¨U¨GÐ4Ø¨JØÜ-3°LÔ-AÓBÑ-A¨�[ “^Ð-AÑBØ$ØØñ	ð 	ùòG.Kùò2 Aùó Eøô$ !ò 	ÜÐ.¨w¨i°r¸#¸Ð?Ô@Ýûð	üòj Cs6   Ã^ Ç
	^%Ç^%È"^*Ì
^0Þ_Þ0	_Þ9_ß_c                ó&  — t        j                  dt         j                  ¬«      }|j                  ddd¬«       |j                  ddd	¬«       |j                  d
ddd¬«       |j                  dt        t
        d¬«       |j                  ddd¬«       |j                  ddd¬«       |j                  ddd¬«       |j                  ddd¬«       |j                  | «      }d }|j                  rP|j                  j                  d«      D �cg c],  }|j                  «       sŒt        |j                  «       «      ‘Œ. }}|j                  dk(  rddg}n"|j                  d k(  rd!nd"}|j                  |fg}t        t        «      }|j                  rŽ|j                  j                  d«      D �ch c]#  }|j                  «       sŒ|j                  «       ’Œ% }}|D �	cg c]  }	|	d#   |v sŒ|	‘Œ }}	||D �	ch c]  }	|	d#   ’Œ	 c}	z
  }
|
rt        d$t!        |
«      › �«       y%|st        d&«       y%g }|D ]v  }|D ]o  \  }}	 t#        ||j$                  |j&                  |||j(                  |j*                   |j,                  |¬'«	      }|j/                  d(«      s|j1                  |«       Œq Œx i }|D ]X  }d-|v sd.|vrŒ|d/   }|d.   j7                  «       D ]2  \  }}||j9                  |i «      v rt;        d0|› d1|› �«      ‚|||   |<   Œ4 ŒZ |j7                  «       D �]  \  }}t<        j>                  jA                  |j&                  |d2d3«      }t<        j>                  jC                  |«      r|j,                  stE        |› d4�«      ‚t<        j>                  jC                  |«      r<|j,                  r0tG        |d5«      5 }tI        jJ                  |«      }d d d «       i ¥|¥}tG        |d6«      5 }tI        jL                  ||d7¬8«       d d d «       t        d9tO        |«      › d:|› �«       �Œ t        d«       t        d;«       d<}d<}|D ]¡  }d-|v r0|d=z  }t        d>|d#   › d*|j/                  d/d«      › d?|d-   › �«       Œ7|j/                  d(«      r$|d=z  }t        d>|d#   › d*|d/   › d@|dA   › dB�«       Œlt        d>|d#   › d*|d/   › dC|dA   › dD|dE   › dF|dG   › dH|dI   › dJ|dK   › dL�«       Œ£ |rt        |› dM�«       y=t        dNtO        |«      › dO|› dP�«       y<c c}w c c}w c c}	w c c}	w # t2        $ rC}t        d)|d#   › d*|› d+|› �«       |j1                  |d#   |t5        |«      d,œ«       Y d }~�Œ<d }~ww xY w# 1 sw Y   �ŒªxY w# 1 sw Y   �Œ„xY w)QNz©Convert CND AAD datasets (dataSub/dataStim .mat and their pre_dataSub/pre_dataStim pre-processed variants) into the SuperHuge on-disk layout (raw + preprocessed stages).)ÚdescriptionÚformatter_classz--source-rootz%C:/Users/sean/Downloads/data-Yuanmingz2root folder holding one CND sub-folder per dataset)ÚdefaultÚhelpz
--out-rootz"E:/ICASSP2027-EEG-sAAD-GC/supehugez<output root; <stage>/eeg|stimuli|meta tree is created insidez--stageÚboth)r™  r2   r3   z©which stage(s) to write: raw uses dataSub/dataStim, preprocessed uses pre_dataSub/pre_dataStim (this tool stores the CND content as-is; it applies no extra EEG cleaning))r—  Úchoicesr˜  z--target-fsz5expected sampling rate of the raw (non pre_) CND data)Útyper—  r˜  z
--datasetsr=   zDcomma-separated source folders to convert (default: all in DATASETS)z
--subjectszncomma-separated dataSub<N>/pre_dataSub<N> file numbers to convert (default: all subject files of each dataset)z--metadata-onlyÚ
store_truezOrecompute metadata entries only; keep the existing eeg/env .npy files untouched)Úactionr˜  z--overwritez,overwrite existing trial .npy / metadata.pklÚ,)r2   rÐ   )r3   rQ  r2   rÐ   rQ  r   z#[error] unknown dataset folder(s): r   z[error] no datasets selected)	r&  rs  rt  r'  ru  r;  rv  rw  rx  r[  z[error] rT  z]: )r   r'  ÚerrorrŸ  rZ  r'  zduplicate trial key in stage r/  r]  zmetadata.pklz1 already exists; use --overwrite to merge into itr]   Úwbr    )Úprotocolz[meta] z trial entries -> z=== conversion summary ===r   r   z  z]: FAILED - z]: skipped (no r  z files)z] -> z (dataset_id=r   r_  rÓ   r`  rX  z ch, fs=rÎ   ra  z- dataset/stage(s) failed; see messages above.zdone. (z dataset/stage conversions, z	 skipped))(ÚargparseÚArgumentParserÚArgumentDefaultsHelpFormatterÚadd_argumentrm   Ú	TARGET_FSÚ
parse_argsrY  ÚsplitrT   rn   r'  rµ   r1   Údatasetsr:   r4  r“  rs  rt  r;  Úmetadata_onlyrw  rÂ   r³   r7  rH   r™   Ú
setdefaultri  r2  rc   rX   rF  Ú
SystemExitr`   ÚpickleÚloadÚdumprÁ   )ÚargvÚparserÚargsrx  rZ   Ústagesru  r©  Úwantedrª   r  ÚreportÚdsr'  Úrepr?  Úmergedrw   r›   rí   ÚentriesÚ	meta_pathrd   ÚoldÚfailedr[  s                             r9   Úmainr½  k  sà  € Ü×$Ñ$ðLô !×>Ñ>ô	@€Fð
 ×Ñ˜Ø HØQð ô Sð ×Ñ˜Ð.SØ[ð ô ]à
×Ñ˜	¨6Ð;Zð6ð ô 7ð
 ×Ñ˜¬E¼9ØTð ô Và
×Ñ˜¨bð:ð ô ;ð ×Ñ˜¨bðWð ô Xð ×ÑÐ)°,ð<ð ô =ð ×Ñ˜¨lØKð ô Mà×Ñ˜TÓ"€Dà)-€MØ‡}‚}Ø15·±×1DÑ1DÀSÔ1IÓWÑ1I¨AÈQÏWÉWÍYœ˜QŸW™W›Y�Ð1IˆÐWà‡z�z�VÒØ)8Ð:QÐ(R‰à ŸJ™J¨%Ò/‘&°UˆØ—:‘:˜wÐ'Ð(ˆä”H‹~€HØ‡}‚}Ø%)§]¡]×%8Ñ%8¸Ô%=ÓKÑ%= ÀÇÁÅ�!—'‘'•)Ð%=ˆÐKÙ'ÓA™x˜!¨1¨X©;¸&Ò+@’A˜xˆÐAØ±Ó:±¨A˜A˜h›K°Ñ:Ñ:ˆÙÜÐ5´f¸W³oÐ5FÐGÔHØÙÜÐ*Ô+Øà#%€FÛˆÛ$‰NˆE�7ð3Ü%Ø¨D×,<Ñ,<Ø!Ÿ]™]°%ÀØ"Ÿn™nØ$(×$6Ñ$6Ð 6Ø"Ÿn™nØ"/ô1�ð —w‘w˜yÔ)Ø—M‘M #Ô&øñ %ð ð& )+€FÛˆØ�a‰<˜:¨QÑ.ØØ�'‘
ˆØ�j‘M×'Ñ'Ö)‰DˆAˆqØ�F×%Ñ% e¨RÓ0Ñ0Ü"Ð%BÀ5À'ÈÈAÈ3Ð#OÓPÐPØ ˆF�5‰M˜!Òñ *ð	 ð !Ÿ,™,Ÿ.‰ˆˆwÜ—G‘G—L‘L §¡°°v¸~ÓNˆ	Ü�7‰7�>‰>˜)Ô$¨T¯^ª^ÜØ�+ÐNÐOóQð Qô �7‰7�>‰>˜)Ô$¨¯ªÜ�i Ô&¨"Ü—k‘k "“o�÷ 'à(˜Ð( Ð(ˆGÜ�)˜TÔ" bÜ�K‰K˜ ¨aÕ0÷ #äˆg”c˜'“l�^Ð#5°i°[ÐAÖBð )ô ˆ„GÜÐ$Ô%Ø€FØ€GÛˆØ�a‰<Ø�a‰KˆFÜ�"�Q�x‘[�M  A§E¡E¨'°"Ó$5Ð#6°lÀ1ÀWÁ:À,ÐOÕPØ�U‰U�9ÔØ�q‰LˆGÜ�"�Q�x‘[�M  A g¡J <¨¸qÀÑ?PÐ>QÐQXÐYÕZä�"�Q�x‘[�M  A g¡J <¨u°Q°~Ñ5FÐ4Gð HØ  ™Ð/¨s°1°X±;°-¸yØ�Z‘=�/ ¨!¨D©'¨°#ð7õ 8ð ñ ÜˆvˆhÐCÐDÔEØÜˆ'”#�f“+�Ð:¸7¸)À9ÐMÔNØùòg Xùò LùÚAùÚ:øô* ò 3Ü�h˜r (™|˜n¨B¨u¨g°S¸¸Ð>Ô?Ø—‘¨¨H©ÀÜ(+¨C«ñ2÷ 3ò 3ûð3ú÷0 'Ñ&ú÷ #Ñ"ús[   ÄTÄ"TÆ-TÇTÇT Ç(T Ç3T%È<A)T*Î;U9Ï+VÔ*	U6Ô38U1Õ1U6Õ9V	ÖV	Ú__main__)r8   rH   r•   ÚNone)rY   r   r•   rH   )rc   rH   r•   rH   )rJ   r   r•   ú
np.ndarray)rt   rÀ  r•   zOptional[int])rc   rH   r•   úDict[str, Any])r¶   r   r•   z	List[Any])rÄ   r   r•   r0   )rÒ   rÁ  rß   rH   r•   rÁ  )ré   rÁ  rê   rn   rß   rH   r•   rÁ  )r÷   úOptional[np.ndarray]rø   rH   r•   úOptional[str])r  zSequence[np.ndarray]rÚ   rÂ  r  rn   r•   z Tuple[np.ndarray, Optional[int]])r  rÀ  rÄ   r0   r  úSequence[str]r•   z'Tuple[np.ndarray, List[Dict[str, Any]]])r&  rÁ  r'  rH   r  rn   r  rn   rÎ   rm   r  rn   rÄ   r0   r(  rn   r  rÃ  r•   rÁ  )r9  rH   r:  rÄ  r;  rm   r•   zOptional[Tuple[str, float]])rc   rH   r•   rm   )r=   )rH  rH   rI  rn   rJ  rH   r•   z/Tuple[Dict[str, Any], str, Dict[str, Any], str])r&  rÁ  rs  rH   rt  rH   r'  rH   ru  rH   rv  Úboolrw  rÅ  rx  zOptional[Sequence[int]]r;  rm   r•   rÁ  rG   )r°  zOptional[Sequence[str]]r•   rn   )1Ú__doc__Ú
__future__r   r¢  r2  r­  ÚreÚsysÚtypingr   r   r   r   r   r	   ÚnumpyrL   rl  r¾   r   r1   Ú__annotations__ÚSTAGESr¦  Úcompiler-  rh  r:   r[   rf   rh   rx   rœ   r°   rº   rÃ   rã   rï   rû   r  r  r*  r@  r6  rO  r“  r½  Ú__name__ÚexitrS  r;   r9   Ú<module>rÑ     sÜ  ðòbõH #ã Û 	Û Û 	Û 
ß =× =ã ð A€ñ0 	�°!¸+Ø¨Ø §¡Ø'ô)ñ 	�°!¸+Ø¨Ø §¡Ø'ô)ñ 	�	°!¸%Ø¨Ø §¡Ø'ô)ñ 	�	°!¸%Ø¨Ø §¡Ø'ô)ñ 	Ð%°!Ð:KØ¨Ø Ð)°·±Ø;ô=ñ 	�°!Ø;Ø¨Ø §¡Ø'ô	)ñ
 	�
°!¸&Ø¨ÀØ §¡Ø;ô=ð5"€Ð
ó ð@ 
!€Ø€	Ø�"—*‘*Ð3Ó4€Ø �b—j‘jÐ!;Ó<Ð óóó$ó#ó<ó$,ó^(ó^ó2:óz/ðd2ZØ"ð2ZØ'5ó2Zórð*%3Ø#7ð%3àð%3à"Bó%3ðP=Ø1ð=à&ð=à+Ró=ð0ØðØ$)ðØ:=ðà/ðà<?ðð &ðð +9óðDØ$)ðØ.Ióó,-ð'ØCó'ð. 06Ø(,ÀØ=AØ'0ð	kØðkØ),ðkà!%ðkà9=ðkð $;ðkð  %ð	kð 6Dó	kô\tðn ˆzÒØ€C‡H�H‰T‹VÕð r;   