§
    4P„j.o  ã                   ó  — d Z ddlZddlZddlZddlZddlZddlZddlmZ e	€Ye
dk    rS ej        e¦  «                             ¦   «         j        d         Zej                             d ee¦  «        ¦  «         dZ	ddlmZ ddlmZ dd	lmZmZmZmZ dd
lmZmZmZmZ ddlm Z m!Z!m"Z"m#Z# ddlm$Z$m%Z% ddl&m'Z(  ej)        d¦  «        Z*	 ddl+m,Z, ddl+m+Z+m-Z-  e+e,e-¦  «         n# e.$ r Y nw xY wd„ Z/d„ Z0e
dk    r e0¦   «          dS dS )ap  
SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores.
Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text.
It also knows all the standard test sets and handles downloading, processing, and tokenization for you.

See the [README.md] file for more information.
é    N)ÚdefaultdictÚ__main__é   Ú	sacrebleu)ÚDATASETS)ÚMETRICS)Ú
smart_openÚfilter_subsetÚget_langpairs_for_testsetÚget_available_testsets)Úprint_test_setÚprint_subset_resultsÚget_reference_filesÚdownload_test_set)Úargs_to_dictÚsanity_check_lengthsÚprint_results_tableÚprint_single_results)Ú#get_available_testsets_for_langpairÚColor)Ú__version__)ÚSIGPIPE)ÚsignalÚSIG_DFLc                  óø  — t          j        dt           j        ¬¦  «        } |                      ddddd¬¦  «         |                      d	ddd
¬¦  «         |                      ddt          d d¬¦  «         |                      dddd d¬¦  «         |                      dddd d¬¦  «         |                      ddd d¬¦  «         |                      dt          d d¬¦  «         |                      ddt          d d¬ ¦  «         |                      d!d"t          d#d d$¬%¦  «         |                      d&d#g d'¬(¦  «         |                      d)d*t
          d+d,¬¦  «         |                      d-d.t          d/d0¬¦  «         d1„ t          D ¦   «         }|                      d2d3|dd4gd5¬6¦  «         |                      d7d8dd9¬:¦  «         |                      d;¦  «        }|                     d<d=t          d>         j         	                    ¦   «         d?d@dA¬B¦  «         |                     dCdDt          d dEdFt          d>         j        dG         › dHt          d>         j        dI         › dJ�¬K¦  «         |                     dLdMt          d>         j        d dNdO¬B¦  «         |                     dPdQdRdddS¬T¦  «         |                     dUdddVdW¬X¦  «         |                      dY¦  «        }|                     dZd[t
          t          d\         j        d]¬¦  «         |                     d^d_t
          t          d\         j        d`¬¦  «         |                     dat
          t          d\         j        db¬¦  «         |                     dcdddd¬e¦  «         |                     dfdddg¬e¦  «         |                     dhdddi¬e¦  «         |                      dj¦  «        }|                     dkddl¬:¦  «         |                     dmddn¬:¦  «         |                     doddp¬:¦  «         |                     dqddr¬:¦  «         |                      ds¦  «        }|                     dtduddv¬:¦  «         |                     dwdxt
          dydz¬¦  «         |                      d{¦  «        }|                     ¦   «         }|                     d|d}dd~¬:¦  «         |                     dd€dd�¬:¦  «         |                     d‚dƒt
          d„d…¬¦  «         |                     d†d‡t
          dydˆ¬¦  «         |                     d‰dŠt
          d+d‹¬¦  «         |                      dŒ¦  «        }|                     d�dŽddd�¬¦  «         |                     d�d‘ddd’¬¦  «         |                     d“d”ddd•¬¦  «         |                     d–d—t
          d+d˜¬¦  «         |                     d™dšddd›¬¦  «         |                     dœd�ddž¬:¦  «         g dŸ¢}	|                     d d¡d¢|	d£¬¤¦  «         |                      d¥d¦d§d¨                     t"          ¦  «        ¬©¦  «         |                      ¦   «         }
dªt&          j        v r/t&          j        dª                              ¦   «         }||	v r||
_        |
S )«Nz³sacreBLEU: Hassle-free computation of shareable BLEU scores.
Quick usage: score your detokenized output against WMT'14 EN-DE:
    cat output.detok.de | sacrebleu -t wmt14 -l en-de)ÚdescriptionÚformatter_classz
--citationz--citeFÚ
store_truez"Dump the bibtex citation and quit.)ÚdefaultÚactionÚhelpz--listz(Print a list of all available test sets.z
--test-setz-tz`The test set to use (see also --list) or a comma-separated list of test sets to be concatenated.)Útyper   r!   z--language-pairz-lÚlangpairz4Source-target language pair (2-char ISO639-1 codes).)Údestr   r!   z
--origlangz-olÚoriglangzoUse a subset of sentences with a given original language (2-char ISO639-1 codes), "non-" prefix means negation.z--subsetÚsubsetzkUse a subset of sentences whose document annotation matches a given regex (see SUBSETS in the source code).z
--downloadzDownload a test set and quit.z--echoú+ax  Output the source (src), reference (ref), or other available field (docid, ref:A, ref:1 for example) to STDOUT and quit. You can get available fields with options `--list` and `-t`For example: `sacrebleu -t wmt21 --list`. If multiple fields are given, they are outputted with tsv format in the order they are given.You can also use `--echo all` to output all available fields.)Únargsr"   r   r!   z--inputz-iÚ*z)Read input from file(s) instead of STDIN.)r"   r(   r   r!   ÚrefszQOptional list of references. If given, it should preceed the -i/--input argument.)r(   r   r!   z
--num-refsz-nrr   z[Split the reference stream on tabs, and expect this many references. (Default: %(default)s)z
--encodingz-eúutf-8z>Open text files with specified encoding (Default: %(default)s)c                 ó6   — g | ]}|                      ¦   «         ‘ŒS © )Úlower)Ú.0Úms     úX/root/disk_3tb/conda_envs/Huy-visual/lib/python3.11/site-packages/sacrebleu/sacrebleu.pyú
<listcomp>zparse_args.<locals>.<listcomp>e   s    € Ð0Ð0Ð0 1�Q—W’W‘Y”YÐ0Ð0Ð0ó    z	--metricsz-mÚbleuz:Space-delimited list of metrics to compute (Default: bleu))Úchoicesr(   r   r!   z--sentence-levelz-slz!Compute metric for each sentence.)r    r!   zBLEU related argumentsz--smooth-methodz-sÚBLEUÚexpÚbleu_smooth_methodz�Smoothing method: exponential decay, floor (increment zero counts), add-k (increment num/denom by k for n>1), or none. (Default: %(default)s))r5   r   r$   r!   z--smooth-valuez-svÚbleu_smooth_valuezGThe smoothing value. Only valid for floor and add-k. (Defaults: floor: Úfloorz	, add-k: zadd-kú))r"   r   r$   r!   z
--tokenizez-tokÚbleu_tokenizez Tokenization method to use for BLEU. If not provided, defaults to `zh` for Chinese, `ja-mecab` for Japanese, `ko-mecab` for Korean and `13a` (mteval) otherwise.z--lowercasez-lcÚbleu_lowercasez;If True, enables case-insensitivity. (Default: %(default)s))r$   r    r   r!   z--forceÚ
bleu_forcez9Insist that your tokenized input is actually detokenized.)r   r    r$   r!   zchrF related argumentsz--chrf-char-orderz-ccÚCHRFz.Character n-gram order. (Default: %(default)s)z--chrf-word-orderz-cwz^Word n-gram order (Default: %(default)s). If equals to 2, the metric is referred to as chrF++.z--chrf-betazJDetermine the importance of recall w.r.t precision. (Default: %(default)s)z--chrf-whitespacezMInclude whitespaces when extracting character n-grams. (Default: %(default)s))r    r   r!   z--chrf-lowercasez1Enable case-insensitivity. (Default: %(default)s)z--chrf-eps-smoothingz|Enables epsilon smoothing similar to chrF++.py, NLTK and Moses; instead of effective order smoothing. (Default: %(default)s)z@TER related arguments (The defaults replicate TERCOM's behavior)z--ter-case-sensitivez0Enables case sensitivity. (Default: %(default)s)z--ter-asian-supportzEEnables special treatment of Asian characters. (Default: %(default)s)z--ter-no-punctz+Removes punctuation. (Default: %(default)s)z--ter-normalizedzDApplies basic normalization and tokenization. (Default: %(default)s)z@Confidence interval (CI) estimation for single-system evaluationz--confidencez-ciz6Report confidence interval using bootstrap resampling.z--confidence-nz-ciniè  zOSet the number of bootstrap resamples for CI estimation (Default: %(default)s).z7Paired significance testing for multi-system evaluationz--paired-arz-parz„Perform paired test using approximate randomization (AR). This option is mutually exclusive with --paired-bs (Default: %(default)s).z--paired-bsz-pbszzPerform paired test using bootstrap resampling. This option is mutually exclusive with --paired-ar (Default: %(default)s).z--paired-ar-nz-parni'  zKNumber of trials for approximate randomization test (Default: %(default)s).z--paired-bs-nz-pbsnzZNumber of bootstrap resamples for paired bootstrap resampling test (Default: %(default)s).z--paired-jobsz-jz™If 0, launches as many workers as the number of systems. If > 0, sets the number of workers manually. This feature is currently not supported on Windows.zReporting related argumentsz--quietz-qzSuppress verbose messages.z--shortz-shz2Produce a shorter (less human readable) signature.z--score-onlyz-bzPrint only the computed score.z--widthz-wz,Floating point width (Default: %(default)s).z--detailz-dz?Print detailed information (split test sets based on origlang).z
--no-colorz-ncz.Disable the occasional use of terminal colors.)ÚjsonÚtextÚlatexz--formatz-fr@   zþSet the output format. `latex` is only valid for multi-system mode whereas `json` and `text` apply to single-system mode only. This flag is overridden if the SACREBLEU_FORMAT environment variable is set to one of the valid choices (Default: %(default)s).)r   r5   r!   z	--versionz-VÚversionz%(prog)s {})r    rC   ÚSACREBLEU_FORMAT)ÚargparseÚArgumentParserÚRawDescriptionHelpFormatterÚadd_argumentÚstrÚintr   Úadd_argument_groupÚSMOOTH_DEFAULTSÚkeysÚfloatÚ
TOKENIZERSÚ
CHAR_ORDERÚ
WORD_ORDERÚBETAÚadd_mutually_exclusive_groupÚformatÚVERSIONÚ
parse_argsÚosÚenvironr.   )Ú
arg_parserÚavail_metricsÚ	bleu_argsÚ	chrf_argsÚter_argsÚ	sign_argsÚ	pair_argsÚpair_args_choiceÚreport_argsÚoutput_formatsÚargsÚ
_new_values               r1   rV   rV   >   s/	  € ÝÔ(ðLõ !Ô<ð	>ñ >ô >€Jð ×Ò˜L¨(¸EÈ,Ø!Eð ñ Gô Gð Gà×Ò˜H¨e¸LØ!Kð ñ Mô Mð Mà×Ò˜L¨$µSÀ$ð "Dð ñ Eô Eð Eà×ÒÐ-¨t¸*ÈdØ!Wð ñ Yô Yð Yà×Ò˜L¨%°jÈ$ð "Sð ñ Tô Tð Tà×Ò˜J¨X¸tð "Oð ñ Pô Pð Pà×Ò˜L­s¸DØ!@ð ñ Bô Bð Bà×Ò˜H¨CµcÀ4ð"að ñ bô bð bð ×Ò˜I tµ#¸SÈ$Ø!Lð ñ Nô Nð Nà×Ò˜F¨#°rØ!tð ñ vô vð và×Ò˜L¨%µcÀ1Ø!~ð ñ @ô @ð @à×Ò˜L¨$µSÀ'Ø!að ñ cô cð cð 1Ð0­Ð0Ñ0Ô0€MØ×Ò˜K¨°}ÈCÐZ`ÐYaØ!]ð ñ _ô _ð _à×ÒÐ.°¸lÐQtÐÑuÔuÐuð ×-Ò-Ð.FÑGÔG€Ià×ÒÐ,¨d½GÀF¼OÔ<[×<`Ò<`Ñ<bÔ<bÐlqØ 4ð !pð ñ qô qð qð ×ÒÐ+¨U½ÈØ 3ð!VÝ5<¸V´_Ô5TÐU\Ô5]ð!Vð !Vå*1°&¬/Ô*IÈ'Ô*Rð!Vð !Vð !Vð ñ Wô Wð Wð
 ×Ò˜<¨½À¼Ô9SÐ]aØ /ð!oð ñ pô pð pð ×Ò˜=¨%Ð6FÈ|ÐejØ ]ð ñ _ô _ð _à×Ò˜9¨e¸LØ ,Ð3nð ñ pô pð pð ×-Ò-Ð.FÑGÔG€IØ×ÒÐ.°½CÍÐQWÌÔIcØ Pð ñ Rô Rð Rà×ÒÐ.°½CÍÐQWÌÔIcð !Að ñ Bô Bð Bà×Ò˜=­s½GÀF¼OÔ<PØ lð ñ nô nð nà×ÒÐ.°|ÈUØ oð ñ qô qð qà×ÒÐ-°lÈEØ Sð ñ Uô Uð Uà×ÒÐ1¸,ÐPUð !_ð ñ `ô `ð `ð ×,Ò,Ð-oÑpÔp€HØ×ÒÐ0¸ØQð ñ Sô Sð Sà×ÒÐ/¸Øfð ñ hô hð hà×ÒÐ*°<ØLð ñ Nô Nð Nà×ÒÐ,°\Øeð ñ gô gð gð ×-Ò-Ð.pÑqÔq€IØ×Ò˜>¨5¸Ø Xð ñ Zô Zð Zà×ÒÐ+¨V½#ÀtØ qð ñ sô sð sð ×-Ò-Ð.gÑhÔh€IØ ×=Ò=Ñ?Ô?Ðà×!Ò! -°Àð(eð "ñ fô fð fð ×!Ò! -°Àð(eð "ñ fô fð fð ×Ò˜?¨G½#ÀuØ mð ñ oô oð oð ×Ò˜?¨G½#ÀtØ |ð ñ ~ô ~ð ~ð ×Ò˜?¨DµsÀAð!Vð ñ Wô Wð Wð
 ×/Ò/Ð0MÑNÔN€KØ×Ò˜Y¨°eÀLØ">ð ñ @ô @ð @à×Ò˜Y¨°uÀ\Ø"Vð ñ Xô Xð Xà×Ò˜^¨T¸5ÈØ"Bð ñ Dô Dð Dà×Ò˜Y¨µ3ÀØ"Pð ñ Rô Rð Rà×Ò˜Z¨°uÀ\Ø"cð ñ eô eð eà×Ò˜\¨5¸Ø"Rð ñ Tô Tð Tð /Ð.Ð.€NØ×Ò˜Z¨°vÀ~ð#Eð ñ Fô Fð Fð
 ×Ò˜K¨°iÈ×I]ÒI]Õ^eÑIfÔIfÐÑgÔgÐgà× Ò Ñ"Ô"€Dð �RœZÐ'Ð'Ý”ZÐ 2Ô3×9Ò9Ñ;Ô;ˆ
Ø˜Ð'Ð'Ø$ˆDŒKà€Kr3   c                  ó#  ‡.‡/‡0— t          ¦   «         Š.‰.j        p‰.j        } t          t          j                             ¦   «         dddd¬¦  «        t          _        t          t          j                             ¦   «         ddd¬¦  «        t          _        t          j	         
                    dd	¦  «        s‰.j        rd	t          _        nd
d l}|                     ¦   «          ‰.j        s t#          j        t"          j        d¬¦  «         ‰.j        r.t+          ‰.j        ‰.j        ¦  «         t	          j        d
¦  «         ‰.j        �r*‰.j        rs‰.j        r‰.j        gnt5          ‰.j        ¦  «        D ]N}t6          ‰.j                                      |¦  «        }t;          |› dd                     |¦  «        › �¦  «         ŒOnœ‰.j        r-t;          d‰.j        › d�¦  «         t?          ‰.j        ¦  «        }nt;          d¦  «         tA          ¦   «         }tC          |¦  «        D ];}t6          |         j"         #                    ¦   «         }t;          |d›d|› �¦  «         Œ<t	          j        d
¦  «         ‰.j$        rFtK          ‰.j&        ¦  «        dk    r.tN           (                    d¦  «         t	          j        d¦  «         ‰.j)        r²‰.j        s.tN           (                    d¦  «         t	          j        d¦  «         ‰.j         *                    d¦  «        D ]N}dt6          |         vrtN           (                    d|› �¦  «         Œ/t;          t6          |         j)        ¦  «         ŒOt	          j        d
¦  «         ‰.j+        dk    r�‰.j        €tK          ‰.j,        ¦  «        dk    rbtN           (                    d¦  «         tN           (                    d¦  «         tN           (                    d¦  «         t	          j        d¦  «         ‰.j        �q‰.j         *                    d¦  «        D ]V}|t6          vrKtN           (                    d|›�¦  «         tN           (                    d¦  «         t	          j        d¦  «         ŒW‰.j        €ˆtK          ‰.j,        ¦  «        d
k    rntN           (                    d¦  «         tN           (                    d¦  «         tN           (                    tA          ¦   «         ¦  «         t	          j        d¦  «         �n(tK          ‰.j,        ¦  «        d
k    r/tN           (                    d ¦  «         t	          j        d¦  «         ná‰.j        €/tN           (                    d!¦  «         t	          j        d¦  «         n«‰.j         *                    d¦  «        D ]�}t5          |¦  «        }‰.j        |vrvtN           (                    d"‰.j        ›�¦  «         tN           (                    d#|›d�¦  «         |D ]}	tN           (                    d$|	› �¦  «         Œ t	          j        d¦  «         Œ‘‰.j-        r”‰.j        �‰.j        €.tN           .                    d%¦  «         t	          j        d¦  «         ‰.j         *                    d¦  «        D ])}t_          |‰.j        ‰.j-        ‰.j0        ‰.j1        ¦  «         Œ*t	          j        d
¦  «         ‰.j        r%‰.j         *                    d&¦  «        d         ‰._2        ‰.j        �?‰.j3        d'k    r4tN           .                    d(¦  «         tN           .                    d)¦  «         g }
‰.j        €|
 4                    ‰.j,        ¦  «         n€‰.j         *                    d¦  «        D ]e}tk          |‰.j        ¦  «        }tK          |¦  «        d
k    r&tN           .                    d*|› d+‰.j        › d,�¦  «         |
 4                    |¦  «         Œfd-„ tm          to          tK          |
d
         ¦  «        ‰.j+        ¦  «        ¦  «        D ¦   «         }|
D �]0}tq          |¦  «        D �]\  }}tq          ts          |‰.j:        ¬.¦  «        d¦  «        D ]ñ\  }}| ;                    ¦   «         }‰.j+        dk    r||          4                    |¦  «         Œ@| *                    d/‰.j+        dz
  ¬0¦  «        }tK          |¦  «        ‰.j+        k    rJtN           (                    d1|› d2‰.j+        › d3tK          |¦  «        › d,�¦  «         t	          j        d4¦  «         tq          |¦  «        D ] \  }}||          4                    |¦  «         Œ!Œò�Œ�Œ2tK          |¦  «        ‰._+        g g c}Š/‰.j<        �€Ht{          j>        t          j        j?        ‰.j:        ¬.¦  «        }| @                    ¦   «          ;                    ¦   «          *                    d/¦  «        }tK          |¦  «        }d5„ |D ¦   «         }d6„ tm          |¦  «        D ¦   «         Š/|D ]¬}| ;                    ¦   «          *                    d/¦  «        }tK          |¦  «        |k    r.tN           (                    d7¦  «         t	          j        d4¦  «         tq          |¦  «        D ]2\  }}||          4                    | ;                    ¦   «         ¦  «         Œ3Œ­�n‰.j<        D ]ì}|}|‰/v rx| r+|‰/d
         k    rtN           A                    d8|›d9�¦  «         Œ5tN           (                    |›d:�¦  «         tN           (                    d;¦  «         t	          j        d¦  «         g }ts          |‰.j:        ¬.¦  «        D ])}| 4                    | ;                    ¦   «         ¦  «         Œ*| 4                    |¦  «         ‰/ 4                    |¦  «         ŒítK          ‰/¦  «        }| r6‰.j<        €!d<gd=„ tm          |dz
  ¦  «        D ¦   «         z   Š/nd>‰/d
         › �‰/d
<   ‰.j$        rr|dk    r.tN           (                    d?¦  «         t	          j        d¦  «         ‰.jB        s| r.tN           (                    d@¦  «         t	          j        d¦  «         d‰._C        | r4|dk    r.tN           (                    dA¦  «         t	          j        d¦  «         |dk    r5‰.jB        r.tN           (                    dB¦  «         t	          j        d¦  «         t‰          g |¢|¢‰.j        ‰.j        ‰.j0        ‰.j1        ¦  «        }|d |…         ||d …         cŠ0}‰0D ]¨}tK          |¦  «        d
k    r|dC‰.j        ›dD�}‰.j0        €‰.j1        �5|dEz  }‰.j0        r|dF‰.j0        › �z  }‰.j1        r|dG‰.j1        › �‰.j1        z   z  }tN           (                    |¦  «         t	          j        d¦  «         t‹          ||‰.j        ¬H¦  «         Œ©i }‰.j&        D ]T}t�          ‰.| G                    ¦   «         d¬I¦  «        } || dJ<   | H                    ¦   «         }t“          |         dXi | ¤Ž||<   ŒU‰.j$        rÅt1          | J                    ¦   «         ¦  «        d
         ‰0d
         }}!t—          |g|¢R Ž D ]t^}"}#|! L                    |"|#¦  «        }$|! M                    ¦   «          N                    ‰.jO        ¦  «        }%t;          |$ N                    ‰.jP        ‰.jQ        |%¦  «        ¦  «         Œut	          j        d
¦  «         ‰.jR        r‰.jN        dKk    rdL‰._N        |dk    rðg }&tC          |¦  «        D ]«}||          S                    |d ‰.jB        r‰.jT        nd¬M¦  «        }$||          M                    ¦   «          N                    ‰.jN        dKk    r‰.jO        nd	¦  «        }%|& 4                    |$ N                    ‰.jP        ‰.jQ        |%‰.jN        dKk    ¦  «        ¦  «         Œ¬t«          |&‰.¦  «         ‰.jR        rt­          ||d
         |‰.¦  «         d S d S ˆ/ˆ0fdN„tm          |¦  «        D ¦   «         }'tN           A                    dO|› dP�¦  «         | sÆi }(t¯          t0          ¦  «        })‰/|)dQ<   |'D ]§\  }}tC          |¦  «        D ]’}||          S                    |d ¬R¦  «        }$||          M                    ¦   «          N                    ‰.jO        ¦  «        |(|$jX        <   |)|$jX                  4                    |$ N                    ‰.jP        d¦  «        ¦  «         Œ“Œ¨nuddSlYmZ}* ‰.j        rdTndU}+‰.j        r‰.j[        n‰.j\        }, |*|'|d |+|,‰.j]        ¬V¦  «        }-|-j^        ‰.__         |-¦   «         \  }(})ˆ.fdW„|( `                    ¦   «         D ¦   «         }(tÃ          |)|(‰.¦  «         d S )YNÚrr+   Tú
)ÚmodeÚencodingÚ	bufferingÚnewlineÚw)rh   ri   rj   ÚNO_COLORFr   zsacreBLEU: %(message)s)ÚlevelrT   z: z, zThe available test sets for z are:zThe available test sets are:z<30r   z3Only one metric can be used in sentence-level mode.zI need a test set (-t).ú,ÚcitationzNo citation found for zfThe --num-refs argument allows you to provide any number of tab-delimited references in a single file.zWYou can only use it with externally provided references, however (i.e., not with `-t`),z5and you cannot then provide multiple reference files.zUnknown test set z6Please run with --list to see the available test sets.zhIf manual references given, make sure to provide them before the -i/--input argument to avoid confusion.zEOtherwise, I need a predefined test set (-t) from the following list:zPI need exactly one of (a) a predefined test set (-t) or (b) a list of referenceszZI need a language pair (-l). Use --list to see available language pairs for this test set.zNo such language pair zAvailable language pairs for z > z9--echo requires a test set (--t) and a language pair (-l)ú-Únonez\You are turning off BLEU's internal tokenizer presumably to supply your own tokenized files.z9Published numbers will not be comparable to other papers.z!No references found for test set ú/ú.c                 ó   — g | ]}g ‘ŒS r-   r-   )r/   Úxs     r1   r2   zmain.<locals>.<listcomp>]  s   € ÐQÐQÐQ˜�ÐQÐQÐQr3   )ri   ú	)ÚsepÚmaxsplitzFATAL: line z: expected z fields, but found é   c                 ó   — g | ]}|g‘ŒS r-   r-   )r/   Úss     r1   r2   zmain.<locals>.<listcomp>�  s   € Ð,Ð,Ð, ˜˜Ð,Ð,Ð,r3   c                 ó   — g | ]
}d |dz   › �‘ŒS ©zSystem r   r-   ©r/   Úis     r1   r2   zmain.<locals>.<listcomp>„  s%   € Ð?Ð?Ð?¨1Ð&˜q 1™uÐ&Ð&Ð?Ð?Ð?r3   zRFATAL: the number of tab-delimited fields in the input stream differ across lines.z	Ignoring z& as it was also given as the baseline.z already used to name a system.z7Make sure to have a different basename for each system.ÚBaselinec                 ó   — g | ]
}d |dz   › �‘ŒS r~   r-   r   s     r1   r2   zmain.<locals>.<listcomp>®  s%   € Ð'VÐ'VÐ'V¸aÐ(9°!°a±%Ð(9Ð(9Ð'VÐ'VÐ'Vr3   z
Baseline: z8Only one system can be evaluated in sentence-level mode.z9Statistical tests are unavailable in sentence-level mode.zBPaired tests require multiple input systems given to --input (-i).z1Use paired tests (--paired) for multiple systems.z	Test set z contains no sentencez withz
 origlang=z subset=)Útest_set)Ústrip_prefixÚ
referencesr@   rA   )r…   Ún_bootstrapc                 ó0   •— g | ]}‰|         ‰|         f‘ŒS r-   r-   )r/   r€   Ú	sys_namesÚsystemss     €€r1   r2   zmain.<locals>.<listcomp>  s&   ø€ ÐLÐLÐL¸˜) Aœ,¨°¬
Ð3ÐLÐLÐLr3   zFound z	 systems.ÚSystem)r…   )Ú
PairedTestÚbsÚar)r…   Ú	test_typeÚ	n_samplesÚn_jobsc                 óL   •— i | ] \  }}||                      ‰j        ¦  «        “Œ!S r-   )rT   Úshort)r/   ÚkÚvrc   s      €r1   ú
<dictcomp>zmain.<locals>.<dictcomp>9  s-   ø€ ÐEÐEÐE±°°1�A�q—x’x ¤
Ñ+Ô+ÐEÐEÐEr3   r-   )brV   Ú	paired_bsÚ	paired_arÚopenÚsysÚstdinÚfilenoÚstdoutrW   rX   ÚgetÚno_colorr   ÚENABLE_COLORSÚcoloramaÚinitÚquietÚloggingÚbasicConfigÚINFOÚdownloadr   r#   ÚexitÚlistrƒ   r   r   Ú
fieldnamesÚprintÚjoinr   r   Úsortedr   ÚstripÚsentence_levelÚlenÚmetricsÚsacreloggerÚerrorrp   ÚsplitÚnum_refsr*   ÚechoÚwarningr   r%   r&   Úbleu_trg_langr<   Úappendr   ÚrangeÚmaxÚ	enumerater	   ri   ÚrstripÚinputÚioÚTextIOWrapperÚbufferÚreadlineÚinfoÚ
confidenceÚbleu_effective_orderr
   r   r   r.   Úupperr   ÚvaluesÚzipÚsentence_scoreÚget_signaturerT   r’   ÚwidthÚ
score_onlyÚdetailÚcorpus_scoreÚconfidence_nr   r   r   ÚnameÚsignificancer‹   Úpaired_bs_nÚpaired_ar_nÚpaired_jobsr�   Úpaired_nÚitemsr   )1Úpaired_test_moder    ÚpairÚfieldsÚtestsetsÚtestsetÚdescrƒ   Ú	langpairsÚlpÚconcat_ref_filesÚ	ref_filesÚ	full_refsÚrefnoÚref_fileÚlinenoÚliner*   ÚrefÚfull_systemsÚinputfhÚnum_sysÚsys_idxÚsentÚfnameÚsys_nameÚlinesÚoutputsÚsystemÚmessager°   rÏ   Úmetric_argsÚmetricÚ
hypothesisr…   ÚscoreÚsigÚresultsÚnamed_systemsÚsigsÚscoresr‹   rŽ   r�   Úpsrc   rˆ   r‰   s1                                                 @@@r1   Úmainrû   Ú   sâ  øøø€ Ý‰<Œ<€Dð ”~Ð7¨¬Ðõ •S”Y×%Ò%Ñ'Ô'¨c¸GÈtÐ]aÐbÑbÔb…C„IÝ•c”j×'Ò'Ñ)Ô)°¸gÐQUÐVÑVÔV…C„Jå	„z‡~‚~�j %Ñ(Ô(ð ¨D¬Mð Ø#�ÔÐð 	ˆˆˆØ�Š‰ŒˆàŒ:ð QÝÔ¥'¤,Ð7OÐPÑPÔPÐPà„}ð Ý˜$œ-¨¬Ñ7Ô7Ð7ÝŒ�‰Œˆà„yñ ØŒ=ð 	0Ø+/¬=Ðf˜œ˜˜Õ>WÐX\ÔXeÑ>fÔ>fð 6ð 6�Ý! $¤-Ô0×;Ò;¸DÑAÔA�Ý˜Ð4Ð4 §¢¨6Ñ!2Ô!2Ð4Ð4Ñ5Ô5Ð5Ð5ð6ð Œ}ð 4ÝÐI°T´]ÐIÐIÐIÑJÔJÐJÝ>¸t¼}ÑMÔM��åÐ4Ñ5Ô5Ð5Ý1Ñ3Ô3�Ý! (Ñ+Ô+ð 0ð 0�Ý Ô(Ô4×:Ò:Ñ<Ô<�Ý˜Ð.Ð.Ð.¨Ð.Ð.Ñ/Ô/Ð/Ð/ÝŒ�‰ŒˆàÔð �s 4¤<Ñ0Ô0°1Ò4Ð4Ý×ÒÐOÑPÔPÐPÝŒ�‰Œˆà„}ð 	ØŒ}ð 	Ý×ÒÐ7Ñ8Ô8Ð8ÝŒH�Q‰KŒKˆKØœ×+Ò+¨CÑ0Ô0ð 	3ð 	3ˆHØ¥¨(Ô!3Ð3Ð3Ý×!Ò!Ð"E¸8Ð"EÐ"EÑFÔFÐFÐFå•h˜xÔ(Ô1Ñ2Ô2Ð2Ð2ÝŒ�‰Œˆà„}˜ÒÐ˜tœ}Ð8½CÀÄ	¹N¼NÈQÒ<NÐ<NÝ×Òð  Cñ  	Dô  	Dð  	DÝ×ÒÐsÑtÔtÐtÝ×ÒÐQÑRÔRÐRÝŒ�‰Œˆà„}Ð Øœ×+Ò+¨CÑ0Ô0ð 	ð 	ˆHØ�xÐ'Ð'Ý×!Ò!Ð"B°hÐ"BÐ"BÑCÔCÐCÝ×!Ò!Ð"ZÑ[Ô[Ð[Ý”˜‘”�øà„}ÐÝˆtŒy‰>Œ>˜QÒÐÝ×Òð Sñ Tô Tð Tå×ÒÐeÑfÔfÐfÝ×ÒÕ4Ñ6Ô6Ñ7Ô7Ð7ÝŒH�Q‰KŒKˆKùÝ	ˆTŒY‰Œ˜!Ò	Ð	Ý×ÒÐlÑmÔmÐmÝŒ�‰ŒˆˆØ	ŒÐ	Ý×ÒÐvÑwÔwÐwÝŒ�‰Œˆˆàœ×+Ò+¨CÑ0Ô0ð 	ð 	ˆHÝ1°(Ñ;Ô;ˆIØŒ} IÐ-Ð-Ý×!Ò!Ð"L¸4¼=Ð"LÐ"LÑMÔMÐMÝ×!Ò!Ð"SÀ(Ð"SÐ"SÐ"SÑTÔTÐTØ#ð 2ð 2�BÝ×%Ò% j¨B j jÑ1Ô1Ð1Ð1Ý”˜‘”�øà„yð ØŒ=Ð  D¤MÐ$9Ý×ÒÐ [Ñ\Ô\Ð\ÝŒH�Q‰KŒKˆKØœ×+Ò+¨CÑ0Ô0ð 	[ð 	[ˆHÝ˜8 T¤]°D´I¸t¼}ÈdÌkÑZÔZÐZÐZÝŒ�‰Œˆð „}ð 9Ø!œ]×0Ò0°Ñ5Ô5°aÔ8ˆÔà„}Ð  TÔ%7¸6Ò%AÐ%AÝ×Òð=ñ	>ô 	>ð 	>õ 	×ÒØGñ	Iô 	Ið 	Ið ÐØ„}ÐØ×Ò ¤	Ñ*Ô*Ð*Ð*ð œ×+Ò+¨CÑ0Ô0ð 	/ð 	/ˆHÝ+¨H°d´mÑDÔDˆIÝ�9‰~Œ~ Ò"Ð"Ý×#Ò#ØS¸ÐSÐSÀ4Ä=ÐSÐSÐSñUô Uð Uà×#Ò# IÑ.Ô.Ð.Ð.ð
 RÐQ�U¥3¥sÐ+;¸AÔ+>Ñ'?Ô'?ÀÄÑ#OÔ#OÑPÔPÐQÑQÔQ€IØ%ð 5ñ 5ˆ	Ý(¨Ñ3Ô3ð 	5ñ 	5‰OˆE�8Ý )­*°XÈÌÐ*VÑ*VÔ*VÐXYÑ ZÔ Zð 5ð 5‘�˜Ø—{’{‘}”}�Ø”= AÒ%Ð%Ø˜eÔ$×+Ò+¨DÑ1Ô1Ð1Ð1àŸ:š:¨$¸¼ÈÑ9J˜:ÑKÔK�Dõ ˜4‘y”y D¤MÒ1Ð1Ý#×)Ò)Ð*z¸Ð*zÐ*zÈDÌMÐ*zÐ*zÕnqÐrvÑnwÔnwÐ*zÐ*zÐ*zÑ{Ô{Ð{Ýœ ™œ˜Ý&/°¡o¤oð 5ð 5™
˜˜sØ! %Ô(×/Ò/°Ñ4Ô4Ð4Ð4ð5ñ5ñ	5õ" ˜	‘N”N€D„Mð ! "Ð€L�)à„zÑåÔ"¥3¤9Ô#3¸d¼mÐLÑLÔLˆð ×!Ò!Ñ#Ô#×*Ò*Ñ,Ô,×2Ò2°4Ñ8Ô8ˆõ �f‘+”+ˆð -Ð, VÐ,Ñ,Ô,ˆð @Ð?µ°g±´Ð?Ñ?Ô?ˆ	ð ð 	<ð 	<ˆDØ—[’[‘]”]×(Ò(¨Ñ.Ô.ˆFÝ�6‰{Œ{˜gÒ%Ð%Ý×!Ò!Ð"vÑwÔwÐwÝ”˜‘”�å!*¨6Ñ!2Ô!2ð <ð <‘�˜Ø˜WÔ%×,Ò,¨T¯[ª[©]¬]Ñ;Ô;Ð;Ð;ð<ñ	<ð ”Zð 	'ð 	'ˆEØˆHà˜9Ð$Ð$Ø#ð  ¨°I¸a´LÒ(@Ð(@å×$Ò$Ð%c°Ð%cÐ%cÐ%cÑdÔdÐdØõ  ×%Ò%¨Ð&TÐ&TÐ&TÑUÔUÐUÝ×%Ò%Ð&_Ñ`Ô`Ð`Ý”H˜Q‘K”K�Kð ˆEÝ" 5°4´=ÐAÑAÔAð ,ð ,�Ø—’˜TŸ[š[™]œ]Ñ+Ô+Ð+Ð+Ø×Ò Ñ&Ô&Ð&Ø×Ò˜XÑ&Ô&Ð&Ð&õ �i‘.”.ˆð ð 7ØŒ:Ðà#˜Ð'VÐ'VÅ5ÈÐSTÉÑCUÔCUÐ'VÑ'VÔ'VÑVˆIˆIð 7¨	°!¬Ð6Ð6ˆI�a‰LàÔð )Ø�QŠ;ˆ;Ý×ÒÐXÑYÔYÐYÝŒH�Q‰KŒKˆKØŒ?ð 	Ð.ð 	Ý×ÒÐYÑZÔZÐZÝŒH�Q‰KŒKˆKð
 %)ˆÔ!àð ˜G qšL˜LÝ×ÒÐ^Ñ_Ô_Ð_ÝŒ�‰Œˆà�‚{€{�t”€{Ý×ÒÐMÑNÔNÐNÝŒ�‰Œˆõ Ø#ˆ,Ð#˜Ð# T¤]°D´MØŒ�t”{ñ$ô $€Gð
 ˜H˜W˜HÔ% w¨w¨x¨xÔ'8€M€GˆTð ð Cð CˆÝˆv‰;Œ;˜!ÒÐØH $¤-ÐHÐHÐHˆGØŒ}Ð(¨D¬KÐ,CØ˜7Ñ"�Ø”=ð <ØÐ;¨D¬MÐ;Ð;Ñ;�GØ”;ð FØÐ7¨$¬+Ð7Ð7¸$¼+ÑEÑE�GÝ×Ò˜gÑ&Ô&Ð&ÝŒH�Q‰KŒKˆKõ 	˜V T°D´MÐBÑBÔBÐBÐBð €GØ”ð 5ð 5ˆõ # 4¨¯ª©¬ÀDÐIÑIÔIˆð %)ˆ�LÑ!ð �zŠz‰|Œ|ˆÝ œÐ4Ð4¨Ð4Ð4ˆ�‰ˆð Ôð 	å˜gŸnšnÑ.Ô.Ñ/Ô/°Ô2°G¸A´J�ˆå'*¨6Ð'9°DÐ'9Ð'9Ð'9ð 	Bð 	BÐ#ˆJ˜Ø×)Ò)¨*°jÑAÔAˆEØ×&Ò&Ñ(Ô(×/Ò/°´
Ñ;Ô;ˆCÝ�%—,’,˜tœz¨4¬?¸CÑ@Ô@ÑAÔAÐAÐAåŒ�‰Œˆà„{ð �t”{ fÒ,Ð,àˆŒð
 �!‚|€|àˆÝ˜7‘O”Oð 		Wð 		WˆDà˜D”M×.Ò.Ø 4Ø15´ÐG˜DÔ-Ð-Àað /ñ Iô IˆEð ˜$”-×-Ò-Ñ/Ô/×6Ò6Ø"œk¨VÒ3Ð3�”
�
¸ñ@ô @ˆCà�NŠNØ—’˜TœZ¨¬¸#¸t¼{ÈfÒ?TÑUÔUñWô Wð Wð Wõ 	˜W dÑ+Ô+Ð+ð Œ;ð 	LÝ  ¨,°q¬/¸9ÀdÑKÔKÐKÐKÐKð	Lð 	Lð MÐLÐLÐLÐL½UÀ7¹^¼^ÐLÑLÔLˆÝ×ÒÐ4 'Ð4Ð4Ð4Ñ5Ô5Ð5àð 	FàˆDÝ ¥Ñ&Ô&ˆFØ(ˆF�8Ñà$1ð Nð NÑ �˜&Ý" 7™OœOð Nð N�DØ# DœM×6Ò6°vÈ$Ð6ÑOÔO�EØ'.¨t¤}×'BÒ'BÑ'DÔ'D×'KÒ'KÈDÌJÑ'WÔ'W�D˜œÑ$Ø˜5œ:Ô&×-Ò-¨e¯lªl¸4¼:ÀtÑ.LÔ.LÑMÔMÐMÐMðNðNð 1Ð0Ð0Ð0Ð0Ð0ð !%¤Ð8˜˜°DˆIØ,0¬NÐP˜Ô(Ð(ÀÔ@PˆIà�˜M¨7¸tØ&/¸9Ø#'Ô#3ð5ñ 5ô 5ˆBð
 œLˆDŒMð ˜2™4œ4‰LˆD�&ð FÐEÐEÐE¸¿
º
¹¼ÐEÑEÔEˆDõ 	˜F D¨$Ñ/Ô/Ð/Ð/Ð/r3   )1Ú__doc__r¾   rW   r™   r£   ÚpathlibrE   Úcollectionsr   Ú__package__Ú__name__ÚPathÚ__file__ÚabsoluteÚparentsÚparentÚpathÚinsertrI   Údatasetr   r°   r   Úutilsr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   Ú r   rU   Ú	getLoggerr±   r   r   r   ÚImportErrorrV   rû   r-   r3   r1   ú<module>r     s/  ðð ð ð 
€	€	€	Ø 	€	€	€	Ø 
€
€
€
Ø €€€Ø €€€Ø €€€Ø #Ð #Ð #Ð #Ð #Ð #ð
 Ð˜8 zÒ1Ð1ØˆWŒ\˜(Ñ#Ô#×,Ò,Ñ.Ô.Ô6°qÔ9€FØ„H‡O‚O�A�s�s˜6‘{”{Ñ#Ô#Ð#Ø€Kà Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ø _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ð _Ø `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ø =Ð =Ð =Ð =Ð =Ð =Ð =Ð =à $Ð $Ð $Ð $Ð $Ð $àˆgÔ Ñ,Ô,€ð	àÐÐÐÐÐð 'Ð&Ð&Ð&Ð&Ð&Ð&Ð&Ø
€Fˆ7�GÑÔÐÐøØð 	ð 	ð 	Ø€Dð	øøøðYð Yð Yðxb0ð b0ð b0ðJ ˆzÒÐØ€D�F„F€F€F€Fð Ðs   ÃC& Ã&C.Ã-C.