§
    4P„j#	  ã            
       ó°   — d Z ddlmZ ddlmZmZ dedededeeef         fd„Zd	ee         d
edefd„Z	dded
ede
defd„Z	 ddedede
dee         fd„ZdS )zCVarious utility functions for word and character n-gram extraction.é    )ÚCounter)ÚListÚTupleÚlineÚ	min_orderÚ	max_orderÚreturnc           
      ó<  — g }|                       ¦   «         }t          ||dz   ¦  «        D ]U}t          dt          |¦  «        |z
  dz   ¦  «        D ]/}|                     t	          ||||z   …         ¦  «        ¦  «         Œ0ŒVt          |¦  «        t          |¦  «        fS )a  Extracts all ngrams (min_order <= n <= max_order) from a sentence.

    :param line: A string sentence.
    :param min_order: Minimum n-gram order.
    :param max_order: Maximum n-gram order.
    :return: a Counter object with n-grams counts and the sequence length.
    é   r   )ÚsplitÚrangeÚlenÚappendÚtupler   )r   r   r   ÚngramsÚtokensÚnÚis          ú^/root/disk_3tb/conda_envs/Huy-visual/lib/python3.11/site-packages/sacrebleu/metrics/helpers.pyÚextract_all_word_ngramsr      s£   € ð €FØ�ZŠZ‰\Œ\€Få�9˜i¨!™mÑ,Ô,ð 3ð 3ˆÝ�q�#˜f™+œ+¨™/¨AÑ-Ñ.Ô.ð 	3ð 	3ˆAØ�MŠM�%  q¨!¨a©% xÔ 0Ñ1Ô1Ñ2Ô2Ð2Ð2ð	3õ �6‰?Œ?�C ™KœKÐ'Ð'ó    r   r   c                 ó~   ‡ ‡— t          ˆˆ fd„t          t          ‰ ¦  «        ‰z
  dz   ¦  «        D ¦   «         ¦  «        S )z¸Extracts n-grams with order `n` from a list of tokens.

    :param tokens: A list of tokens.
    :param n: The order of n-grams.
    :return: a Counter object with n-grams counts.
    c                 óP   •— g | ]"}d                       ‰||‰z   …         ¦  «        ‘Œ#S )ú )Újoin)Ú.0r   r   r   s     €€r   ú
<listcomp>z'extract_word_ngrams.<locals>.<listcomp>!   s1   ø€ ÐRÐRÐR°!�C—H’H˜V A a¨!¡e Gœ_Ñ-Ô-ÐRÐRÐRr   r   )r   r   r   )r   r   s   ``r   Úextract_word_ngramsr      sC   øø€ õ ÐRÐRÐRÐRÐRµu½SÀ¹[¼[È1¹_ÈqÑ=PÑ7QÔ7QÐRÑRÔRÑSÔSÐSr   FÚinclude_whitespacec                 óÐ   ‡ ‡— |s'd                      ‰                      ¦   «         ¦  «        Š t          ˆ ˆfd„t          t	          ‰ ¦  «        ‰z
  dz   ¦  «        D ¦   «         ¦  «        S )a&  Yields counts of character n-grams from a sentence.

    :param line: A segment containing a sequence of words.
    :param n: The order of the n-grams.
    :param include_whitespace: If given, will not strip whitespaces from the line.
    :return: a dictionary containing ngrams and counts
    Ú c                 ó*   •— g | ]}‰||‰z   …         ‘ŒS © r#   ©r   r   r   r   s     €€r   r   z'extract_char_ngrams.<locals>.<listcomp>/   s%   ø€ ÐDÐDÐD a�D˜˜1˜q™5˜”MÐDÐDÐDr   r   )r   r   r   r   r   )r   r   r   s   `` r   Úextract_char_ngramsr%   $   sd   øø€ ð ð %Ø�wŠw�t—z’z‘|”|Ñ$Ô$ˆåÐDÐDÐDÐDÐD­5µ°T±´¸Q±ÀÑ1BÑ+CÔ+CÐDÑDÔDÑEÔEÐEr   c           	      ó.  ‡ ‡— g }|s'd                      ‰                      ¦   «         ¦  «        Š t          d|dz   ¦  «        D ]SŠt          ˆ ˆfd„t          t	          ‰ ¦  «        ‰z
  dz   ¦  «        D ¦   «         ¦  «        }|                     |¦  «         ŒT|S )aH  Extracts all character n-grams at once for convenience.

    :param line: A segment containing a sequence of words.
    :param max_order: The maximum order of the n-grams.
    :param include_whitespace: If given, will not strip whitespaces from the line.
    :return: a list of Counter objects containing ngrams and counts.
    r!   r   c                 ó*   •— g | ]}‰||‰z   …         ‘ŒS r#   r#   r$   s     €€r   r   z+extract_all_char_ngrams.<locals>.<listcomp>B   s%   ø€ ÐJÐJÐJ¨A˜$˜q  Q¡˜wœ-ÐJÐJÐJr   )r   r   r   r   r   r   )r   r   r   Úcountersr   r   s   `    @r   Úextract_all_char_ngramsr)   2   sŸ   øø€ ð €Hàð %Ø�wŠw�t—z’z‘|”|Ñ$Ô$ˆå�1�i !‘mÑ$Ô$ð  ð  ˆÝÐJÐJÐJÐJÐJµµs¸4±y´yÀ1±}ÀqÑ7HÑ1IÔ1IÐJÑJÔJÑKÔKˆØ�Š˜ÑÔÐÐà€Or   N)F)Ú__doc__Úcollectionsr   Útypingr   r   ÚstrÚintr   r   Úboolr%   r)   r#   r   r   ú<module>r0      s1  ðØ IÐ Ià Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð ð( #ð (°#ð (À#ð (È%ÐPWÐY\ÐP\ÔJ]ð (ð (ð (ð (ð&T  S¤	ð T¨cð T°gð Tð Tð Tð TðFð F˜cð F cð F¸tð FÐPWð Fð Fð Fð Fð ?Dðð ØðØ!ðØ7;ðØHLÈWÌðð ð ð ð ð r   