§
    4P„jY  ã                   ó²   — d Z ddlmZmZmZmZmZ ddlmZ ddl	m
Z
 ddlmZmZmZ ddlmZ  G d	„ d
e¦  «        Z G d„ de¦  «        Z G d„ de¦  «        ZdS )z;The implementation of the TER metric (Snover et al., 2006).é    )ÚListÚDictÚSequenceÚOptionalÚAnyé   )ÚTercomTokenizer)Úsum_of_listsé   )ÚScoreÚ	SignatureÚMetric)Útranslation_edit_ratec                   ó(   ‡ — e Zd ZdZdefˆ fd„Zˆ xZS )ÚTERSignaturez›A convenience class to represent the reproducibility signature for TER.

    :param args: key-value dictionary passed from the actual metric instance.
    Úargsc                 ó  •— t          ¦   «                              |¦  «         | j                             ddddddœ¦  «         | j                             |d         rdnd	|d
         |d         |d          |d         dœ¦  «         dS )z`TERSignature` initializer.ÚcÚtÚnrÚpnÚas)ÚcaseÚtokÚnormÚpunctÚasianÚcase_sensitiveÚmixedÚlcÚtokenizer_signatureÚ
normalizedÚno_punctÚasian_supportN)ÚsuperÚ__init__Ú_abbrÚupdateÚinfo)Úselfr   Ú	__class__s     €úZ/root/disk_3tb/conda_envs/Huy-visual/lib/python3.11/site-packages/sacrebleu/metrics/ter.pyr&   zTERSignature.__init__   s´   ø€ å‰Œ×Ò˜ÑÔÐØŒ
×ÒØØØØØð
ð 
ñ 	ô 	ð 	ð 	Œ	×ÒØ#Ð$4Ô5Ð?�G�G¸4ØÐ-Ô.Ø˜Ô&Ø˜jÔ)Ð)Ø˜/Ô*ð
ð 
ñ 	ô 	ð 	ð 	ð 	ó    )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Údictr&   Ú__classcell__©r+   s   @r,   r   r      sN   ø€ € € € € ðð ð˜Tð ð ð ð ð ð ð ð ð ð r-   r   c                   ó0   ‡ — e Zd ZdZdededefˆ fd„Zˆ xZS )ÚTERScorezÉA convenience class to represent TER scores.

    :param score: The TER score.
    :param num_edits: The cumulative number of edits.
    :param ref_length: The cumulative average reference length.
    ÚscoreÚ	num_editsÚ
ref_lengthc                 ó‚   •— t          ¦   «                              d|¦  «         t          |¦  «        | _        || _        dS )z`TERScore` initializer.ÚTERN)r%   r&   Úintr8   r9   )r*   r7   r8   r9   r+   s       €r,   r&   zTERScore.__init__:   s5   ø€ å‰Œ×Ò˜ Ñ&Ô&Ð&Ý˜Y™œˆŒØ$ˆŒˆˆr-   )r.   r/   r0   r1   Úfloatr&   r3   r4   s   @r,   r6   r6   3   s\   ø€ € € € € ðð ð%˜eð %°ð %À5ð %ð %ð %ð %ð %ð %ð %ð %ð %ð %r-   r6   c                   ó  ‡ — e Zd ZdZeZ	 	 	 	 	 ddededededeeee	                           f
ˆ fd	„Z
d
e	de	fd„Zdee         defd„Zdeee                  defd„Zde	dedee         fd„Zdee	         dee	ef         fd„Zˆ xZS )r;   a  Translation edit rate (TER). A near-exact reimplementation of the Tercom
    algorithm, produces identical results on all "sane" outputs.

    Tercom original implementation: https://github.com/jhclark/tercom

    The beam edit distance algorithm uses a slightly different approach (we stay
    around the diagonal which is faster, at least in Python) so in some
    (extreme) corner cases, the output could differ.

    Caching in the edit distance is based partly on the PyTer package by Hiroyuki
    Tanaka (MIT license). (https://github.com/aflc/pyter)

    :param normalized: Enable character normalization. By default, normalizes a couple of things such as
        newlines being stripped, retrieving XML encoded characters, and fixing tokenization for punctuation. When
        'asian_support' is enabled, also normalizes specific Asian (CJK) character sequences, i.e.
        split them down to the character level.
    :param no_punct: Remove punctuation. Can be used in conjunction with 'asian_support' to also remove typical
        punctuation markers in Asian languages (CJK).
    :param asian_support: Enable special treatment of Asian characters. This option only has an effect when
        'normalized' and/or 'no_punct' is enabled. If 'normalized' is also enabled, then Asian (CJK)
        characters are split down to the character level. If 'no_punct' is enabled alongside 'asian_support',
        specific unicode ranges for CJK and full-width punctuations are also removed.
    :param case_sensitive: If `True`, does not lowercase sentences.
    :param references: A sequence of reference documents with document being
        defined as a sequence of reference strings. If given, the reference info
        will be pre-computed and cached for faster re-computation across many systems.
    FNr"   r#   r$   r   Ú
referencesc                 óP  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        t          | j        | j        | j        | j        ¬¦  «        | _        | j                             ¦   «         | _	        |�|  
                    |¦  «        | _        dS dS )z`TER` initializer.)r"   r#   r$   r   N)r%   r&   r#   r"   r$   r   r	   Ú	tokenizerÚ	signaturer!   Ú_cache_referencesÚ
_ref_cache)r*   r"   r#   r$   r   r?   r+   s         €r,   r&   zTER.__init__`   s¤   ø€ õ 	‰Œ×ÒÑÔÐà ˆŒØ$ˆŒØ*ˆÔØ,ˆÔå(Ø”Ø”]ØÔ,ØÔ.ð	
ñ 
ô 
ˆŒð $(¤>×#;Ò#;Ñ#=Ô#=ˆÔ àÐ!Ø"×4Ò4°ZÑ@Ô@ˆDŒOˆOˆOð "Ð!r-   ÚsentÚreturnc                 óP   — |                       |                     ¦   «         ¦  «        S )zœGiven a sentence, apply tokenization if enabled.

        :param sent: The input sentence string.
        :return: The pre-processed output string.
        )rA   Úrstrip)r*   rE   s     r,   Ú_preprocess_segmentzTER._preprocess_segmentx   s   € ð �~Š~˜dŸkšk™mœmÑ,Ô,Ð,r-   Ústatsc                 óx   — |d         |d         }}|dk    r||z  }n|dk    rd}nd}t          d|z  ||¦  «        S )zµComputes the final score from already aggregated statistics.

        :param stats: A list or numpy array of segment-level statistics.
        :return: A `TERScore` object.
        r   r   g      ð?g        éd   )r6   )r*   rJ   Útotal_editsÚsum_ref_lengthsr7   s        r,   Ú_compute_score_from_statszTER._compute_score_from_stats€   sW   € ð (-¨Q¤x°°q´�_ˆà˜QÒÐØ /Ñ1ˆEˆEØ˜1Š_ˆ_ØˆEˆEàˆEå˜˜e™ [°/ÑBÔBÐBr-   c                 óF   — |                       t          |¦  «        ¦  «        S )z±Computes the final TER score given the pre-computed corpus statistics.

        :param stats: A list of segment-level statistics
        :return: A `TERScore` instance.
        )rO   r
   )r*   rJ   s     r,   Ú_aggregate_and_computezTER._aggregate_and_compute‘   s    € ð ×-Ò-­l¸5Ñ.AÔ.AÑBÔBÐBr-   Ú
hypothesisÚ
ref_kwargsc                 óÒ   — d}t          d¦  «        }|                     ¦   «         }|d         }|D ]"}t          ||¦  «        \  }}	||	z  }||k     r|}Œ#|t          |¦  «        z  }
||
gS )aà  Given a (pre-processed) hypothesis sentence and already computed
        reference words, returns the segment statistics required to compute
        the full TER score.

        :param hypothesis: Hypothesis sentence.
        :param ref_kwargs: A dictionary with `ref_words` key which is a list
        where each sublist contains reference words.
        :return: A two-element list that contains the 'minimum number of edits'
        and 'the average reference length'.
        r   g €à7yÃACÚ	ref_words)r<   Úsplitr   Úlen)r*   rR   rS   Úref_lengthsÚbest_num_editsÚ	words_hyprU   Ú	words_refr8   Úref_lenÚavg_ref_lens              r,   Ú_compute_segment_statisticszTER._compute_segment_statistics™   sŠ   € ð ˆÝ˜T™œˆà×$Ò$Ñ&Ô&ˆ	ð ˜{Ô+ˆ	Ø"ð 	+ð 	+ˆIÝ!6°yÀ)Ñ!LÔ!LÑˆI�wØ˜7Ñ"ˆKØ˜>Ò)Ð)Ø!*�øà!¥C¨	¡N¤NÑ2ˆØ Ð,Ð,r-   Úrefsc                 óŒ   — g }|D ]<}|                      |                      |¦  «                             ¦   «         ¦  «         Œ=d|iS )a.  Given a list of reference segments, applies pre-processing & tokenization
        and returns list of tokens for each reference.

        :param refs: A sequence of strings.
        :return: A dictionary that will be passed to `_compute_segment_statistics()`
        through keyword arguments.
        rU   )ÚappendrI   rV   )r*   r_   rU   Úrefs       r,   Ú_extract_reference_infozTER._extract_reference_info¶   sV   € ð ˆ	àð 	Dð 	DˆCØ×Ò˜T×5Ò5°cÑ:Ô:×@Ò@ÑBÔBÑCÔCÐCÐCà˜YÐ'Ð'r-   )FFFFN)r.   r/   r0   r1   r   Ú_SIGNATURE_TYPEÚboolr   r   Ústrr&   rI   r   r=   r6   rO   rQ   r   r^   r   rc   r3   r4   s   @r,   r;   r;   A   s„  ø€ € € € € ðð ð8 #€Oà*/Ø"'Ø',Ø(-ØAEð	Að A 4ð AØðAà $ðAð "&ðAð & h¨x¸¬}Ô&=Ô>ð	Að Að Að Að Að Að0-¨ð -°ð -ð -ð -ð -ðC¨t°E¬{ð C¸xð Cð Cð Cð Cð"C¨D°°e´Ô,=ð CÀ(ð Cð Cð Cð Cð-Ø!ð-Ø/3ð-Ø8<¸U¼ð-ð -ð -ð -ð:(¨H°S¬Mð (¸dÀ3ÈÀ8¼nð (ð (ð (ð (ð (ð (ð (ð (r-   r;   N)r1   Útypingr   r   r   r   r   Útokenizers.tokenizer_terr	   Úutilsr
   Úbaser   r   r   Úlib_terr   r   r6   r;   © r-   r,   ú<module>rm      s  ðØ AÐ Að" 7Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6à 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø  Ð  Ð  Ð  Ð  Ð  Ø *Ð *Ð *Ð *Ð *Ð *Ð *Ð *Ð *Ð *Ø *Ð *Ð *Ð *Ð *Ð *ðð ð ð ð �9ñ ô ð ð2%ð %ð %ð %ð %ˆuñ %ô %ð %ðB(ð B(ð B(ð B(ð B(ˆ&ñ B(ô B(ð B(ð B(ð B(r-   