§
    4P„jD  ã                   óp   — d Z ddlZddlZddlmZmZ ddlmZmZm	Z	 ddl
mZmZmZ  G d„ de¬	¦  «        ZdS )
z+
The base class for all types of datasets.
é    N)ÚABCMetaÚabstractmethod)ÚDictÚListÚOptionalé   )ÚSACREBLEU_DIRÚdownload_fileÚ
smart_openc                   ó  — e Zd Zddddeeee         f         fdedeee                  dee         dee         deee                  f
d„Zd„ Ze	d	„ ¦   «         Z
d
„ Zd„ Zd„ Zeddd„¦   «         Zdee         fd„Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ ZdS )ÚDatasetNÚnameÚdataÚdescriptionÚcitationÚmd5c                 ó  — || _         || _        || _        || _        || _        || _        || _        t          j         	                    t          | j         ¦  «        | _        t          j         	                    | j        d¦  «        | _        dS )av  
        Params come from the values in DATASETS.

        :param name: Name of the dataset.
        :param data: URL of the raw data of the dataset.
        :param description: Description of the dataset.
        :param citation: Citation for the dataset.
        :param md5: MD5 checksum of the dataset.
        :param langpairs: List of available language pairs.
        ÚrawN)r   r   r   r   r   Ú	langpairsÚkwargsÚosÚpathÚjoinr	   Ú_outdirÚ_rawdir)Úselfr   r   r   r   r   r   r   s           ú[/root/disk_3tb/conda_envs/Huy-visual/lib/python3.11/site-packages/sacrebleu/dataset/base.pyÚ__init__zDataset.__init__   sn   € ð( ˆŒ	ØˆŒ	Ø&ˆÔØ ˆŒØˆŒØ"ˆŒØˆŒõ ”w—|’|¥M°4´9Ñ=Ô=ˆŒÝ”w—|’| D¤L°%Ñ8Ô8ˆŒˆˆó    c                 ó^  — t          j        | j        d¬¦  «         | j        r| j        ndgt	          | j        ¦  «        z  }t          | j        |¦  «        D ]U\  }}t           j                             | j        |  	                    |¦  «        ¦  «        }t          ||| j        |¬¦  «         ŒVdS )zÏ
        If the dataset isn't downloaded, use utils/download_file()
        This can be implemented here in the base class. It should write
        to ~/.sacreleu/DATASET/raw exactly as it does now.
        T)Úexist_okN)Ú
extract_toÚexpected_md5)r   Úmakedirsr   r   Úlenr   Úzipr   r   Ú_get_tarball_filenamer
   )r   Úexpected_checksumsÚurlr#   Útarballs        r   Úmaybe_downloadzDataset.maybe_download0   s´   € õ 	Œ�D”L¨4Ð0Ñ0Ô0Ð0à)-¬ÐN˜TœX˜X¸°vÅÀDÄIÁÄÑ7NÐå!$ T¤YÐ0BÑ!CÔ!Cð 	ð 	ÑˆC�Ý”g—l’l 4¤<°×1KÒ1KÈCÑ1PÔ1PÑQÔQˆGåØ�W¨¬ÀLðñ ô ð ð ð	ð 	r   c                 óR   — t          j        dd|                      ¦   «         ¦  «        S )z¾
        Removes trailing and leading spaces and collapses multiple consecutive internal spaces to a single one.

        :param s: The string.
        :return: A cleaned-up string.
        z\s+ú )ÚreÚsubÚstrip)Úss    r   Ú_cleanzDataset._cleanA   s    € õ Œv�f˜c 1§7¢7¡9¤9Ñ-Ô-Ð-r   c                 ó~   — | j                              dd¦  «        dz   t          j                             |¦  «        z   S )z´
        Produces a local filename for tarball.
        :param url: The url to download.
        :return: A name produced from the dataset identifier and the URL basename.
        ú/Ú_ú.)r   Úreplacer   r   Úbasename)r   r)   s     r   r'   zDataset._get_tarball_filenameK   s6   € ð Œy× Ò   cÑ*Ô*¨SÑ0µ2´7×3CÒ3CÀCÑ3HÔ3HÑHÐHr   c                 ó¾   — | j                              dd¦  «        }|                     dd¦  «        }t          j                             | j        |› d|› d|› �¦  «        S )a#  
        Given the language pair and fieldname, return the path to the text file.
        The format is: ~/.sacrebleu/DATASET/DATASET.LANGPAIR.FIELDNAME

        :param langpair: The language pair.
        :param fieldname: The fieldname.
        :return: The path to the text file.
        r4   r5   ú:ú-r6   )r   r7   r   r   r   r   )r   ÚlangpairÚ	fieldnamer   s       r   Ú_get_txt_file_pathzDataset._get_txt_file_pathS   s^   € ð Œy× Ò   cÑ*Ô*ˆà×%Ò% c¨3Ñ/Ô/ˆ	ÝŒw�|Š|˜DœL¨TÐ*JÐ*J°HÐ*JÐ*J¸yÐ*JÐ*JÑKÔKÐKr   c                 ó~   — |€| j         }n2|| j         vrt          d| j        › d|› �¦  «        ‚|| j         |         i}|S )a  
        Given a language pair, return the metadata for that language pair.
        Deal with errors if the language pair is not available.

        :param langpair: The language pair. e.g. "en-de"
        :return: Dict format which is same as self.langpairs.
        NzNo such language pair r4   )r   Ú	Exceptionr   )r   r<   r   s      r   Ú_get_langpair_metadatazDataset._get_langpair_metadatab   sX   € ð ÐØœˆIˆIØ˜Tœ^Ð+Ð+ÝÐK°T´YÐKÐKÀÐKÐKÑLÔLÐLà! 4¤>°(Ô#;Ð<ˆIàÐr   Úreturnc                 ó   — dS )z�Processes raw files to plain text files.

        :param langpair: The language pair to process. e.g. "en-de". If None, all files will be processed.
        N© ©r   r<   s     r   Úprocess_to_textzDataset.process_to_texts   s	   € ð 	ˆr   c                 ó
   — ddgS )aŒ  
        Return a list of all the field names. For most source, this is just
        the source and the reference. For others, it might include the document
        ID for each line, or the original language (origLang).

        get_files() should return the same number of items as this.

        :param langpair: The language pair (e.g., "de-en")
        :return: a list of field names
        ÚsrcÚrefrD   rE   s     r   Ú
fieldnameszDataset.fieldnames{   s   € ð �uˆ~Ðr   c              #   ól   K  — |                       |¦  «        }d„ |D ¦   «         }t          |Ž D ]}|V — ŒdS )zs
        Iterates over all fields (source, references, and other metadata) defined
        by the dataset.
        c                 ó,   — g | ]}t          |¦  «        ‘ŒS rD   ©r   ©Ú.0Úfs     r   ú
<listcomp>z$Dataset.__iter__.<locals>.<listcomp>Ž   ó   € Ð5Ð5Ð5 a•J˜q‘M”MÐ5Ð5Ð5r   N)Ú	get_filesr&   )r   r<   Ú	all_filesÚall_finsÚitems        r   Ú__iter__zDataset.__iter__ˆ   sT   è è € ð
 —N’N 8Ñ,Ô,ˆ	Ø5Ð5¨9Ð5Ñ5Ô5ˆå˜�Nð 	ð 	ˆDØˆJˆJˆJˆJð	ð 	r   c              #   ó¼   K  — |                       |¦  «        }t          |¦  «        5 }|D ]}|                     ¦   «         V — Œ	 ddd¦  «         dS # 1 swxY w Y   dS )z;
        Return an iterable over the source lines.
        N)Úget_source_filer   r0   )r   r<   Úsource_fileÚfinÚlines        r   ÚsourcezDataset.source“   sµ   è è € ð ×*Ò*¨8Ñ4Ô4ˆÝ˜Ñ$Ô$ð 	#¨Øð #ð #�Ø—j’j‘l”lÐ"Ð"Ð"Ð"ð#ð	#ð 	#ð 	#ñ 	#ô 	#ð 	#ð 	#ð 	#ð 	#ð 	#ð 	#ð 	#øøøð 	#ð 	#ð 	#ð 	#ð 	#ð 	#s   §AÁAÁAc              #   ól   K  — |                       |¦  «        }d„ |D ¦   «         }t          |Ž D ]}|V — ŒdS )z9
        Return an iterable over the references.
        c                 ó,   — g | ]}t          |¦  «        ‘ŒS rD   rM   rN   s     r   rQ   z&Dataset.references.<locals>.<listcomp>¡   rR   r   N)Úget_reference_filesr&   )r   r<   Ú	ref_filesÚref_finsrV   s        r   Ú
referenceszDataset.referencesœ   sV   è è € ð ×,Ò,¨XÑ6Ô6ˆ	Ø5Ð5¨9Ð5Ñ5Ô5ˆå˜�Nð 	ð 	ˆDØˆJˆJˆJˆJð	ð 	r   c                 ó�   — |                       |¦  «        }|                      |¦  «        }|                     d¦  «        }||         S )NrH   )rS   rJ   Úindex)r   r<   rT   Ú
all_fieldsre   s        r   rY   zDataset.get_source_file¦   sB   € Ø—N’N 8Ñ,Ô,ˆ	Ø—_’_ XÑ.Ô.ˆ
Ø× Ò  Ñ'Ô'ˆØ˜ÔÐr   c                 óŽ   — |                       |¦  «        }|                      |¦  «        }d„ t          ||¦  «        D ¦   «         }|S )Nc                 óB   — g | ]\  }}|                      d ¦  «        ¯|‘ŒS )rI   )Ú
startswith)rO   rP   Úfields      r   rQ   z/Dataset.get_reference_files.<locals>.<listcomp>¯   s?   € ð 
ð 
ð 
Ù�!�U¸E×<LÒ<LÈUÑ<SÔ<Sð
Øð
ð 
ð 
r   )rS   rJ   r&   )r   r<   rT   rf   ra   s        r   r`   zDataset.get_reference_files¬   sT   € Ø—N’N 8Ñ,Ô,ˆ	Ø—_’_ XÑ.Ô.ˆ
ð
ð 
Ý! )¨ZÑ8Ô8ð
ñ 
ô 
ˆ	ð Ðr   c                 óÄ   ‡ ‡— ‰                       ‰¦  «        }ˆˆ fd„|D ¦   «         }|D ]6}t          j                             |¦  «        s‰                      ‰¦  «         Œ7|S )aE  
        Returns the path of the source file and all reference files for
        the provided test set / language pair.
        Downloads the references first if they are not already local.

        :param langpair: The language pair (e.g., "de-en")
        :return: a list of the source file and all reference files
        c                 ó<   •— g | ]}‰                      ‰|¦  «        ‘ŒS rD   )r>   )rO   rj   r<   r   s     €€r   rQ   z%Dataset.get_files.<locals>.<listcomp>¾   s)   ø€ ÐNÐNÐN¸e�×(Ò(¨°5Ñ9Ô9ÐNÐNÐNr   )rJ   r   r   ÚexistsrF   )r   r<   ÚfieldsÚfilesÚfiles   ``   r   rS   zDataset.get_files´   sr   øø€ ð —’ Ñ*Ô*ˆØNÐNÐNÐNÐNÀvÐNÑNÔNˆàð 	/ð 	/ˆDÝ”7—>’> $Ñ'Ô'ð /Ø×$Ò$ XÑ.Ô.Ð.øØˆr   )N)rB   N)Ú__name__Ú
__module__Ú__qualname__r   Ústrr   r   r   r+   Ústaticmethodr2   r'   r>   rA   r   rF   rJ   rW   r]   rc   rY   r`   rS   rD   r   r   r   r      s‘  € € € € € ð %)Ø%)Ø"&Ø#'Ø�s˜D œI�~Ô&ð!9ð !9àð!9ð �t˜C”yÔ!ð!9ð ˜c”]ð	!9ð
 ˜3”-ð!9ð �d˜3”iÔ ð!9ð !9ð !9ð !9ðFð ð ð" ð.ð .ñ „\ð.ðIð Ið IðLð Lð Lðð ð ð" ðð ð ð ñ „^ðð d¨3¤ið ð ð ð ð	ð 	ð 	ð#ð #ð #ðð ð ð ð  ð  ðð ð ðð ð ð ð r   r   )Ú	metaclass)Ú__doc__r   r.   Úabcr   r   Útypingr   r   r   Úutilsr	   r
   r   r   rD   r   r   ú<module>r{      s»   ððð ð 
€	€	€	Ø 	€	€	€	Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'à <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ð <ðwð wð wð wð w˜ð wñ wô wð wð wð wr   