Ë
    Ï—¬j»B  ã                   ó¾   — d dl Z d dlmZ d dlmZ d dlmZmZmZ d dl	Z	d dl
Zd dlZd dlZd dlmZ d dlmZ ddlmZ d	d
lmZmZmZ d	dlmZ d	dlmZ  G d„ de«      Zy)é    N)ÚMapping)Úceil)ÚLiteralÚSequenceÚoverride)ÚNDArray)ÚDataseté   ©Úvalidate_kwargsé   )ÚMetadataÚMetadataElementÚMetadataField)ÚTransformComposer)ÚStatisticalTransformc                   óx  — e Zd ZU eZeed<   ee   ed<   eed<   e	e
z  ed<   e	e
z  ed<   e	ed<   edz  ed<   ee   ed	<   eee
e
f      ed
<   eee
e
f      dz  ed<   ed   ed<   dZed   dz  ed<   dZe	dz  ed<   ee	ef   ed<   	 d„ Zd„ Zd„ Zdeez  fd„Zed„ «       Z	 d-d
eee
e
f      dz  deee
e
f      dz  fd„Zd„ Zed„ «       Zd„ Zde	fd„Zd„ Zed„ «       Z d e	d!e!ed"   ee"z  f   fd#„Z#e$d e	d!e!ed"   ee"z  f   fd$„«       Z%d%e&jN                  d&efd'„Z(d e	fd(„Z)d)ee   d*ee   fd+„Z*ed!e
fd,„«       Z+y).Ú
EegDatasetÚeeg_pathÚ_input_files_listÚmetadataÚfsÚwindow_lengthÚoverlapNÚ	transformÚmetadata_fieldsÚaccept_rangeÚreject_range©ÚtrainÚvalÚtestÚ_stage)TFÚ_save_on_memoryÚ_disk_usageÚmemoryc                 ó   — g d¢}| j                  t        |j                  «       «      |«       |d   | _        |d   | _        |d   | _        |d   | _        |d   | _        |j                  dd«      | _	        |j                  d	d
«      | _
        |d   | _        | j                  |j                  dd«      |j                  dd
«      ¬«       | j                  «        | j                  «        | `|j                  dd
«      }|dvrt        d|› d�«      ‚|| _        | j#                  «        t%        «       | _        |j                  dd
«      | _        y
)aä  
        Initialize the EEG Dataset.

        Args:
            eeg_path (str): Path to the dataset folder.
            files (list): List of file names.
            metadata (dict): Metadata containing information for each trial.
            window_length (int): Length of each signal segment.
            overlap (int): Overlap ratio, determines the stride for segment slicing.
            transform (TransformComposer | None): Transformations applied to samples.
            metadata_fields (list): Metadata fields to record.
            accept_range (tuple[float, float] | None): Range of valid signal segments (start, end).
            reject_range (tuple[float, float] | None): Range of rejected signal segments (start, end).
        )r   Úfilesr   r   r   r(   r   r   r   r   é   r   Nr   r   ©ç        ç      ð?r   )r   r   Ústager   zInvalid stage 'z,'. Must be one of 'train', 'val', or 'test'.Úsave_on_memory)Ú_validate_kwargsÚlistÚkeysr   r   r   r   r   Úgetr   r   r   Ú_validate_rangesÚ_validate_filesÚ_cache_and_prepare_segmentsÚ
ValueErrorr#   Ú_update_and_fit_stats_transformÚdictr&   r$   )ÚselfÚkwargsÚrequired_keysr#   s       úl/root/disk_3tb/EEG-sAAD-Grand-Challenge-Training-Dataset/SuperHugeAAD/superhuge/data/datasets/eeg_dataset.pyÚ__init__zEegDataset.__init__3   sK  € ò MˆØ×Ñœd 6§;¡;£=Ó1°=ÔAà˜zÑ*ˆŒØ!'¨¡ˆÔØ˜zÑ*ˆŒØ˜‘,ˆŒØ# OÑ4ˆÔØ—z‘z )¨QÓ/ˆŒØŸ™ K°Ó6ˆŒØ%Ð&7Ñ8ˆÔà×ÑØŸ™ N°JÓ?ØŸ™ N°DÓ9ð 	ô 	
ð 	×ÑÔð 	×(Ñ(Ô*ð Ð"à—‘˜G TÓ*ˆØÐ1Ñ1ÜØ! & Ð)UÐVóð ð !ˆDŒKØ×,Ñ,Ô.ä“fˆŒà%Ÿz™zÐ*:¸DÓAˆÕó    c                 óÊ   — d}| j                   D ]Q  }t        j                  j                  | j                  |dz   «      }|t        j                  j                  |«      z  }ŒS |S )Nr   ú.npy)r(   ÚosÚpathÚjoinr   Úgetsize)r9   Útotal_size_bytesÚ	file_nameÚ	file_paths       r<   Ú_get_file_disk_usagezEegDataset._get_file_disk_usagei   sS   € ØÐØŸœˆIÜŸ™Ÿ™ T§]¡]°IÀÑ4FÓGˆIØ¤§¡§¡°	Ó :Ñ:Ñð $ð  Ðr>   c           	      óN  — t        | j                  t        «      �r t        | j                  j                  D �cg c]  }t        |t
        «      ‘Œ c}«      rÄ| j                  dk(  r´| j                  j                  D �cg c]  }t        |t
        «      r|‘Œ }}| j                  D ]Z  }t        j                  t        j                  j                  | j                  |dz   «      «      }|D ]  }|j                  |«       Œ Œ\ |D ]  }|j                  «        Œ yyyyc c}w c c}w )zZ
        Update the transform to include metadata fields and fit it if necessary.
        r    r@   N)Ú
isinstancer   r   ÚanyÚ
transformsr   r#   r(   ÚnpÚloadrA   rB   rC   r   ÚupdateÚfit)r9   ÚtÚupdatable_transformsrF   Úeegs        r<   r7   z*EegDataset._update_and_fit_stats_transformq   sý   € ô
 �t—~‘~Ô'8Õ9ÜØ>B¿n¹n×>WÒ>WÓXÑ>W¸”˜AÔ3Õ4Ð>WÑXôð —‘˜wÒ&ð Ÿ™×2Ò2ó$á2�AÜ˜aÔ!5Ô6ò Ø2ð !ð $ð
 "ŸZœZ�	Ü—g‘gœbŸg™gŸl™l¨4¯=©=¸)ÀfÑ:LÓMÓN�Û-�AØ—H‘H˜S•Mñ .ð (ó *�Ø—‘•ñ *ð 'ðð :ùâXùò$s   ¹DÁ>D"Úsource_transformc                 óœ  — |€J d«       ‚| j                   €J d«       ‚t        |t        «      rw| j                   j                  D ]]  }t        |t	        |«      «      sŒ|j
                  |j
                  k(  sŒ3|j                  |j                  k(  sŒM|j                  |_        Œ_ yt        |t        «      r™|j                  D ]‰  }t        |t        «      sŒ| j                   j                  D ]]  }t        |t	        |«      «      sŒ|j
                  |j
                  k(  sŒ3|j                  |j                  k(  sŒM|j                  |_        Œ_ Œ‹ yy)z÷
        Synchronize the statistics of the transform with another StatisticalTransform or TransformComposer.

        Args:
            source_transform (StatisticalTransform | TransformComposer): The source transform to synchronize with.
        Nz Source transform cannot be None.z Target transform cannot be None.)	r   rJ   r   rL   ÚtypeÚwhomÚwhenÚstatr   )r9   rT   rQ   Úsource_ts       r<   Úsync_transform_statszEegDataset.sync_transform_statsˆ   s
  € ð  Ð+ÐOÐ-OÓOÐ+Ø�~‰~Ð)ÐMÐ+MÓMÐ)ÜÐ&Ô(<Ô=Ø—^‘^×.Ô.�ä˜q¤$Ð'7Ó"8Õ9ØŸ™Ð"2×"7Ñ"7Ó7ØŸ™Ð"2×"7Ñ"7Ó7à-×2Ñ2�A•Fñ /ô Ð(Ô*;Ô<Ø,×7Ô7�Ü˜hÔ(<Õ=Ø!Ÿ^™^×6Ô6˜ä& q¬$¨x«.Õ9Ø !§¡¨(¯-©-Ó 7Ø !§¡¨(¯-©-Ó 7à%-§]¡]˜A�Fñ 7ñ 8ð =r>   c                 ó6   — | j                   j                  «       S )zt
        Access the file names.

        Returns:
            list[str]: A list of file names (as strings).
        )Ú_file_namesÚtolist©r9   s    r<   r(   zEegDataset.files¦   s   € ð ×Ñ×&Ñ&Ó(Ð(r>   c                 óŠ   — dt         t        t        t        f      fd„}|€dg} ||«       |� ||«       || _        || _        y)zH
        Validate the accept_range and reject_range attributes.
        Ú	range_seqc                 ó^   — | D ](  }d|d   cxk  r|d   cxk  rdk  rŒJ d«       ‚ J d«       ‚ y )Nr+   r   r)   r,   zREach range must be a tuple of two floats in the range [0.0, 1.0] with start < end.© )ra   Úrs     r<   Úvalidate_rangez3EegDataset._validate_ranges.<locals>.validate_range¹   sJ   € Û�à˜1˜Q™4Ô- ! A¡$Ô-¨#Ó-ðhàgóhÙ-ðhàgóhØ-ñ r>   Nr*   )r   ÚtupleÚfloatr   r   )r9   r   r   re   s       r<   r3   zEegDataset._validate_ranges°   sX   € ð	h¤h¬u´U¼E°\Ñ/BÑ&Có 	hð ÐàðˆLñ 	�|Ô$àÐ#Ù˜<Ô(à(ˆÔØ(ˆÕr>   c                 óþ   — t        | j                  «      j                  | j                  j	                  «       «      sAJ dt        | j                  «      t        | j                  j	                  «       «      z
  › �«       ‚y)zS
        Ensure that the files attribute is a subset of the metadata keys.
        zDSome files specified in `files` do not have corresponding metadata: N)Úsetr   Úissubsetr   r1   r_   s    r<   r4   zEegDataset._validate_filesË   s”   € ô �4×)Ñ)Ó*×3Ñ3Ø�M‰M×ÑÓ ô
ð 	LàQÔRUÐVZ×VlÑVlÓRmÔpsÐtx÷  uBñ  uB÷  uGñ  uGó  uIó  qJñ  SJð  RKð  Ló	Lñ 
r>   c                 óH   — | j                   | j                  | j                  fS )a`  
        Access the cached valid segments.

        Returns:
            tuple: A tuple containing:
                - np.ndarray: A numpy array of file names (as strings).
                - np.ndarray: A numpy array of offsets mapping file indices to valid segments.
                - np.ndarray: A single array of valid segment start indices.
        )r]   Ú$_file_to_valid_start_indices_offsetsÚ_valid_start_indicesr_   s    r<   Úvalid_segments_cachezEegDataset.valid_segments_cacheÓ   s)   € ð ×ÑØ×5Ñ5Ø×%Ñ%ð
ð 	
r>   c                 ól  — g }t        j                  t        | j                  «      dz   t         j                  ¬«      }g }g }d}t        | j                  «      D ]²  \  }}| j                  |   j                  | j                  |   j                  z  | j                  z  }|sJ d|› d�«       ‚| j                  |«      }	|j                  |«       |||<   |j                  |	«       |t        |	«      z  }|j                  t        |	«      «       Œ´ ||t        | j                  «      <   t        j                  |t         j                  j                  ¬«      | _        || _        t        j                  |t         j                  ¬«      | _        t        j                  |t         j                  ¬«      | _        y)zH
        Cache valid segments for all files using numpy arrays.
        r)   )Údtyper   zMetadata for file z is missing signal_length.N)rM   ÚzerosÚlenr   Úint32Ú	enumerater   Úsignal_lengthr   Ú_find_valid_start_idxÚappendÚextendÚarrayÚdtypesÚStrDTyper]   rl   rm   Ú_per_file_sample_count)
r9   Ú
file_namesÚ#file_to_valid_start_indices_offsetsÚvalid_start_indicesÚper_file_sample_countÚcurrent_indexÚfile_idxÚfileÚtrial_lengthÚstart_indicess
             r<   r5   z&EegDataset._cache_and_prepare_segmentsä   sv  € ð ˆ
Ü.0¯h©hÜ�×&Ñ&Ó'¨!Ñ+´2·8±8ô/
Ð+ð !ÐØ "Ðàˆä'¨×(>Ñ(>Ö?‰NˆH�dà—‘˜dÑ#×1Ñ1°D·M±MÀ$Ñ4G×4JÑ4JÑJÈTÏWÉWÑTð ñ  ÐVÐ#5°d°VÐ;UÐ!VÓV�<Ø ×6Ñ6°|ÓDˆMð ×Ñ˜dÔ#Ø<IÐ/°Ñ9Ø×&Ñ& }Ô5ØœS Ó/Ñ/ˆMØ!×(Ñ(¬¨]Ó);Õ<ð @ð LYÐ+¬C°×0FÑ0FÓ,GÑHäŸ8™8 J´b·i±i×6HÑ6HÔIˆÔØ4WˆÔ1Ü$&§H¡HÐ-@ÌÏÉÔ$QˆÔ!Ü&(§h¡hÐ/DÌBÏHÉHÔ&UˆÕ#r>   r„   c                 ó   — t        | j                  | j                  z  | j                  z  «      }g }| j                  D ]Í  }t        |d   |z  «      }t        |d   |z  «      }t        |||«      D ]™  }|| j                  | j                  z  z   }||kD  r ŒX| j                  rWd}	| j                  D ]C  }
t        |
d   |z  «      }t        |
d   |z  «      }||cxk  r|k  sn ||cxk  r|k  sŒ>n ŒAd}	 n |	rŒ‰|j                  |«       Œ› ŒÏ |S )zÚ
        Compute valid segments for a given trial length.

        Args:
            trial_length (int): The length of the trial.

        Returns:
            list[int]: A list of valid segment start indices.
        r   r)   FT)	r   r   r   r   r   ÚintÚranger   rw   )r9   r„   Ústrider…   r   Úaccept_startÚ
accept_endÚ	start_idxÚend_idxÚreject_flagr   Úreject_startÚ
reject_ends                r<   rv   z EegDataset._find_valid_start_idx  s  € ô �d—g‘g × 2Ñ 2Ñ2°d·l±lÑBÓCˆØˆà ×-Ô-ˆLÜ˜|¨A™°Ñ=Ó>ˆLÜ˜\¨!™_¨|Ñ;Ó<ˆJä" <°¸VÖD�	Ø# d§g¡g°×0BÑ0BÑ&BÑB�Ø˜ZÒ'ÙØ×$Ò$Ø"'�KØ(,×(9Ô(9˜Ü'*¨<¸©?¸\Ñ+IÓ'J˜Ü%(¨°a©¸<Ñ)GÓ%H˜
à(¨IÔB¸
ÔBØ+¨gÔC¸ÖCà*.˜KÙ!ð ):ñ #Ø Ø×$Ñ$ YÕ/ñ# Eð	 .ð, Ðr>   c                 ó    — | j                   d   S )Néÿÿÿÿ)rl   r_   s    r<   Ú__len__zEegDataset.__len__,  s   € Ø×8Ñ8¸Ñ<Ð<r>   c                 ó   — t        | «      S ©N)rr   r_   s    r<   rr   zEegDataset.len/  s   € ä�4‹yÐr>   ÚidxÚreturn©ÚmetarS   c                 ó†  — | j                   €:| j                  «       t        j                  «       j                  dz  k  rd| _         n| j                   €d| _         | j                   rH|| j
                  v r:ddhj                  t        | j
                  |   «      «      r| j
                  |   }|S | j                  |«      \  }}| j                  |   }t        j                  j                  | j                  |dz   «      }| j                  |   } | j                  di |j!                  «       ¤Ž}t#        j$                  |dd¬«      }| j'                  ||«       | j(                  r&| j)                  |j+                  «       |d	d¬
«      \  }}|||| j,                  | j.                  z  z    }	| j(                  r| j)                  |	|dd¬
«      \  }	}t1        |	t"        j2                  «      sJ dt5        |	«      j6                  › d�«       ‚||	j9                  t"        j:                  «      dœ}| j                   r|| j
                  |<   |S )Ngš™™™™™é?TFr™   rS   r@   rd   )Ú	mmap_modeÚallow_pickleÚbefore_slicing)r™   rX   rW   Úbefore_returningz+EEG segment must be a numpy array, but got Ú.r˜   rc   )r$   rH   ÚpsutilÚvirtual_memoryÚ	availabler&   rj   ri   Ú_map_idx_to_file_and_segmentr(   rA   rB   rC   r   r   Úmetadata_clsÚ
model_dumprM   rN   Ú_validate_eeg_shaper   Úcopyr   r   rJ   ÚndarrayrV   Ú__name__ÚastypeÚfloat32)
r9   r–   Údatar‚   rŒ   rF   rG   r™   rS   Úeeg_segs
             r<   Ú	load_datazEegDataset.load_data3  s  € ð × Ñ Ð(Ø×)Ñ)Ó+¬f×.CÑ.CÓ.E×.OÑ.OÐRUÑ.UÒUà#'ˆDÕ Ø×!Ñ!Ð)Ø#(ˆDÔ à× Ò Ø�t—{‘{Ñ"Ø˜�×(Ñ(¬¨T¯[©[¸Ñ-=Ó)>Ô?à—;‘;˜sÑ#ˆDðN ˆðK #'×"CÑ"CÀCÓ"HÑˆH�iØŸ
™
ØñˆIô Ÿ™Ÿ™ T§]¡]°IÀÑ4FÓGˆIà—=‘= Ñ+ˆDØ$�4×$Ñ$Ñ9 t§¡Ó'8Ñ9ˆDä*,¯'©'Ø S°uô+ˆCð ×$Ñ$ S¨)Ô4à�~Š~Ø ŸN™N¨3¯8©8«:¸DÐGWÐ^c˜NÓd‘	��Tà˜) i°$×2DÑ2DÀtÇwÁwÑ2NÑ&NÐOˆGà�~Š~Ø $§¡Ø $Ð-?Àeð !/ó !‘�˜ô ØœŸ™ôð Wà<¼TÀ'»]×=SÑ=SÐ<TÐTUÐVóWð ð
 Ø—~‘~¤b§j¡jÓ1ñMˆDð
 ×#Ò#Ø#'�—‘˜CÑ àˆr>   c                 óV   — | j                  |«      }|d   j                  «       |d   dœS )Nr™   rS   r˜   )r®   r¥   )r9   r–   Úitems      r<   Ú__getitem__zEegDataset.__getitem__k  s1   € à�~‰~˜cÓ"ˆà˜‘L×+Ñ+Ó-Ø˜‘;ñ
ð 	
r>   rS   rF   c                 óV  — |j                   d   | j                  |   j                  k(  s5J d|j                   d   › d| j                  |   j                  › d|› d�«       ‚|j                  dk(  sJ d|j                  › d|› d�«       ‚t	        | j                  |   d	«      r‹| j                  |   j
                  rq|j                   d
   t        | j                  |   j
                  «      k(  s>J d|j                   d
   › dt        | j                  |   j
                  «      › d|› d�«       ‚yyy)a  
        Validate the shape of the loaded EEG data.

        Args:
            eeg (np.ndarray): The loaded EEG data.
            file_name (str): The name of the file.

        Raises:
            AssertionError: If the shape of the EEG data is invalid.
        r   zLoaded data shape z' does not match expected signal_length z
 for file rŸ   r   zLoaded data is not 2D, but zD for file Úchannel_infosr)   zNumber of channels z does not match expected N)Úshaper   ru   ÚndimÚhasattrr³   rr   )r9   rS   rF   s      r<   r¦   zEegDataset._validate_eeg_shapes  sl  € ð �I‰I�a‰L˜DŸM™M¨)Ñ4×BÑBÒBð	Tà §	¡	¨!¡˜~Ð-TÐUY×UbÑUbÐclÑUm×U{ÑU{ÐT|ð  }Gð  HQð  GRð  RSð  Tó	TØBð �H‰H˜ŠMð	Kà(¨¯©¨
°+¸i¸[ÈÐJó	KØô �D—M‘M )Ñ,¨oÔ>Ø—‘˜iÑ(×6Ò6à—9‘9˜Q‘<¤3Ø—‘˜iÑ(×6Ñ6ó$ò ð Pà$ S§Y¡Y¨q¡\ NÐ2KÌCÐPT×P]ÑP]Ð^gÑPh×PvÑPvÓLwÐKxð  yCð  DMð  CNð  NOð  PóPñ ð 7ð ?r>   c                 ó(  — | j                   \  }}}t        j                  ||d¬«      dz
  }|dk  s|t        |«      k\  rt	        d|› d|d   › d�«      ‚|||   z
  }|dk  s|||dz      ||   z
  k\  rt	        d	|› d
||   › d�«      ‚||||   |z      fS )a  
        Map a global index to a specific file and segment index.

        Args:
            idx (int): Global index.

        Returns:
            tuple[int,int]: File index and start_idx

        Raises:
            IndexError: If the index is out of range.
        Úright)Úsider)   r   zIndex z8 is out of range for the dataset. Total valid segments: r’   rŸ   zLocal index z is out of range for file )rn   rM   Úsearchsortedrr   Ú
IndexError)r9   r–   r}   r~   Úvalid_segmentsr‚   Ú	local_idxs          r<   r£   z'EegDataset._map_idx_to_file_and_segmentŒ  sü   € ð ×%Ñ%ñ 	Hˆ
Ð7¸ô �O‰OÐ?ÀÈ7ÔSÐVWÑWð 	ð �aŠ<˜8¤s¨:£Ò6ÜØ˜˜ÐUÐVyÐz|ÑV}ÐU~Ð~ð  Aóð ð
 Ð=¸hÑGÑGˆ	Ø�qŠ=˜IØ/°¸1±Ñ=Ø1°(Ñ;ñ<ò
ô Ø˜y˜kÐ)CÀJÈxÑDXÐCYÐYZÐ[óð ð
 ØÐ>¸xÑHÈ9ÑTÑUð
ð 	
r>   r:   r;   c                 ó   — t        ||«       y)a  
        Validate the provided keyword arguments.

        Args:
            kwargs (Sequence[str]): The provided keyword arguments.
            required_keys (Sequence[str]): The required keys.

        Raises:
            AssertionError: If any required key is missing.
        Nr   )r9   r:   r;   s      r<   r/   zEegDataset._validate_kwargsµ  s   € ô 	˜ Õ.r>   c                 ó4   — | j                   | j                  z  S r•   )r   r   r_   s    r<   Úsegment_lengthzEegDataset.segment_lengthÂ  s   € à�w‰w˜×+Ñ+Ñ+Ð+r>   r•   ),r©   Ú
__module__Ú__qualname__r   r¤   ÚstrÚ__annotations__r   r   r‡   rg   r   r0   r   rf   r   r$   r%   r8   r=   rH   r7   r   r[   Úpropertyr(   r3   r4   rn   r5   rv   r“   rr   r   r   r®   r   r±   rM   r¨   r¦   r£   r/   rÀ   rc   r>   r<   r   r      s6  … Ø"€LØƒMØ ‘}Ó$ØÓØˆe‰ƒOØ˜‘;ÓØƒLØ  4Ñ'Ó'Ø˜-Ñ(Ó(Ø˜5 ¨ Ñ.Ñ/Ó/Ø˜5 ¨ Ñ.Ñ/°$Ñ6Ó6ØÐ*Ñ+Ó+Ø37€O�W˜[Ñ)¨DÑ0Ó7Ø"€K��t‘Ó"Ø��d�‰OÓðò4Bòl òð.3Ø 4Ð7HÑ Hó3ð< ñ)ó ð)ð >Bñ)à˜u U¨E \Ñ2Ñ3°dÑ:ð)ð ˜u U¨E \Ñ2Ñ3°dÑ:ó)ò6Lð ñ
ó ð
ò !VðF#°#ó #òJ=ð ñó ðð6Øð6à	�˜Ñ'¨¸7Ñ)BÐBÑ	Có6ðp ð
˜sð 
 w¨w°}Ñ/EÀtÈgÁ~Ð/UÑ'Vò 
ó ð
ðP r§z¡zð P¸có Pð2'
°ó '
ðR/ x°¡}ð /ÀXÈcÁ]ó /ð ð, ò ,ó ñ,r>   r   )rA   Úcollections.abcr   Úmathr   Útypingr   r   r   ÚattrÚnumpyrM   r    ÚtqdmÚnumpy.typingr   Útorch.utils.datar	   Úutils.validater   Úmetadata_processing.datar   r   r   Útransforms.composerr   Útransforms.stats_abcr   r   rc   r>   r<   Ú<module>rÒ      sC   ðÛ 	Ý #Ý ß .Ñ .ã Û Û Û Ý  Ý $å -ß OÑ OÝ 3Ý 7ô
o,�õ o,r>   