7,jz6UddlmZddlZddlmZddlmZejdkrddlm Z n ddl m Z n#e $rdZ YnwxYwdd l m Z mZmZmZmZmZmZmZmZmZmZmZmZdd lmZmZmZmZmZm Z m!Z!eezezezezZ"d e#d <e Gd dZ$GddZ%e Gdde%Z&e Gdde%Z'e Gdde%Z(e Gdde%Z)e Gdde%Z*e Gdde%Z+e Gdde%Z,e Gdd e%Z-e Gd!d"e%Z.ed#$d4d*Z/ed+$ d5d6d3Z0dS)7) annotationsN) lru_cache) getLogger))finalc|S)N)clss EC:\PYTHON\GemmaClient\venv\Lib\site-packages\charset_normalizer/md.pyrrsJ) COMMON_CJK_CHARACTERSCOMMON_SAFE_ASCII_CHARACTERSTRACEUNICODE_SECONDARY_RANGE_KEYWORD _ACCENTUATED_ARABIC_ARABIC_ISOLATED_FORM_CJK_HANGUL _HIRAGANA _KATAKANA_LATIN_THAI)_character_flags is_emoticonis_punctuation is_separator is_symbol remove_accent unicode_rangeint _GLYPH_MASKc&eZdZdZdZd dZd dZd S) CharInfou{Pre-computed character properties shared across all detectors. Instantiated once and reused via :meth:`update` on every character in the hot loop so that redundant calls to str methods (``isalpha``, ``isupper``, …) and cached utility functions (``_character_flags``, ``is_punctuation``, …) are avoided when several plugins need the same information.  character printablealphaupperlowerspacedigitis_ascii case_variableflags accentuatedlatinis_cjk is_arabicis_glyphpunctsymreturnNonecd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ d|_ d|_d|_d|_dS)NFrr'selfs r __init__zCharInfo.__init__Ns $          # #( !&  ! $#   r r(strc||_t|}|dkr@d|_d|_d|_d|_d|_d|cxkrdkrWnnTd|_d|_d|_ d|_ d|_ d|_ d|_ t|_d|_d|_d|_dSd|cxkrdkrWnnTd|_d|_d|_ d|_ d|_ d|_ d|_ t|_d|_d|_d|_dSd|cxkrd krRnnOd|_d|_d|_ d|_ d|_ d|_ d|_ d |_d|_d|_d|_dS|d ksd |cxkrd krVnnSd|_d|_d|_ d|_ d|_ |d k|_ d|_ d |_d|_d|_d|_dS||_ d|_d|_d|_ d|_ d|_ d|_ d |_d|_|j rt)|nd|_|j rt+|nd|_dSd|_||_ ||_||_||_ ||_ ||_ |j |jk|_ |jrt7|}nd }||_t9|t:z|_t9|tz|_t9|t<z|_t9|t>z|_t9|t@z|_|j rt)|nd|_|j rt+|nd|_dS)zBUpdate all properties for *character* (called once per character).TFAZaz09r N)!r(ordr/r2r4r5r6r*r+r,r-r.r)r0rr1r3r7r8 isprintablerr isalphaisupperislowerisspaceisdigitrboolrrrr$)r>r(or1s r updatezCharInfo.updateas"Y s77 DM$D DK"DN!DMQ}}}}"}}}}}! ! " " " !%%)"# ! "  qC! " ! " " !%%)"# ! "  qB" " " " ! !%%*" " "  bQ!\\\\r\\\\\" " " ! " !"b%*" " "  "+!6!6!8!8" " " " " %*" " :>.S^I666e 37>L9Y///u"DM&2244DN"**,,DJ"**,,DJ"**,,DJ"**,,DJ"**,,DJ!%tz!9D z (33DJ#EL$899D efn--DJut|,,DK!%'/22DN !455DM7;nO 222%DJ/3~Hy+++5DHHHr Nr9r:)r(r@r9r:)__name__ __module__ __qualname____doc__ __slots__r?rUr r r r&r&/sYI(&jIjIjIjIjIjIr r&c>eZdZdZdZdd Zdd Zedd Zd S)MessDetectorPluginzy Base abstract class used for mess detection plugins. All detectors MUST extend and implement given methods. r r(r@infor&r9r:ct)z The main routine to be executed upon character. Insert the logic in witch the text would be considered chaotic. NotImplementedErrorr>r(r^s r feed_infozMessDetectorPlugin.feed_infos "!r ct)zB Permit to reset the plugin to the initial state. r`r=s r resetzMessDetectorPlugin.resets "!r floatct)z Compute the chaos ratio based on what your feed() has seen. Must NOT be lower than 0.; No restriction gt 0. r`r=s r ratiozMessDetectorPlugin.ratios "!r Nr(r@r^r&r9r:rVr9rf) rWrXrYrZr[rcrepropertyrhr r r r]r]sm I"""""""" """X"""r r]cBeZdZdZddZdd Zdd Zedd Zd S) TooManySymbolOrPunctuationPlugin_punctuation_count _symbol_count_character_count_last_printable_char_frenzy_symbol_in_wordr9r:cLd|_d|_d|_d|_d|_dSNrFrnr=s r r?z)TooManySymbolOrPunctuationPlugin.__init__s0'("#%&04!,1###r r(r@r^r&c|xjdz c_||jkrN|tvrE|jr|xjdz c_n-|js&|jrt|s|xjdz c_||_dS)1Optimized feed using pre-computed character info.rN) rqrrrr7ror.r8rrprbs r rcz*TooManySymbolOrPunctuationPlugin.feed_infos " 2 2 2!===z (''1,'''Z (DH ([5K5K (""a'""$-!!!r c0d|_d|_d|_dSNr)rorqrpr=s r rez&TooManySymbolOrPunctuationPlugin.reset s "# !r rfc^|jdkrdS|j|jz|jz }|dkr|ndS)Nr333333?)rqrorp)r>ratio_of_punctuations r rhz&TooManySymbolOrPunctuationPlugin.ratiosK  A % %3  #d&8 8  !'"(ratio_of_accentuations r rhzTooManyAccentuatedPlugin.ratio1s<  1 $ $3'+'>AV'V(=(E(E$$3Nr NrVrirjrr r r rrs{:I))))))))$$$$OOOXOOOr rcBeZdZdZddZdd Zdd Zedd Zd S)UnprintablePlugin_unprintable_countrqr9r:c"d|_d|_dSrzrr=s r r?zUnprintablePlugin.__init__>s'(%&r r(r@r^r&cz|js#|js|dkr|dkr|xjdz c_|xjdz c_dS)rwurN)r-r)rrqrbs r rczUnprintablePlugin.feed_infoBs^  )N )V##X%%  # #q ( # # "r cd|_dSrz)rr=s r rezUnprintablePlugin.resetMs"#r rfc@|jdkrdS|jdz|jz S)Nrr|r)rqrr=s r rhzUnprintablePlugin.ratioPs+  A % %3'!+t/DDDr NrVrirjrr r r rr:s{:I'''' # # # #$$$$EEEXEEEr rcBeZdZdZddZdd Zdd Zedd Zd S)SuspiciousDuplicateAccentPlugin_successive_countrq_last_latin_character_last_was_accentuatedr9r:c>d|_d|_d|_d|_dSrurr=s r r?z(SuspiciousDuplicateAccentPlugin.__init__as(&'%&15"+0"""r r(r@r^r&c@|xjdz c_|js|jrl|jre|jr)|jr|xjdz c_t|t|jkr|xjdz c_||_|j|_dSr)rqrr2rr+rOrr!rbs r rcz)SuspiciousDuplicateAccentPlugin.feed_infohs "  & 2  3* 3z ,d8@@BB ,&&!+&&Y''=9S+T+TTT&&!+&&%."%)%5"""r c>d|_d|_d|_d|_dSrurr=s r rez%SuspiciousDuplicateAccentPlugin.resetws(!" !%)"%*"""r rfc@|jdkrdS|jdz|jz S)Nrr|rx)rqrr=s r rhz%SuspiciousDuplicateAccentPlugin.ratio}s+  A % %3&*d.CCCr NrVrirjrr r r rrXs}I1111 6 6 6 6++++ DDDXDDDr rcBeZdZdZddZdd Zdd Zedd Zd S)SuspiciousRange"_suspicious_successive_range_countrq_last_printable_seen_last_printable_ranger9r:c>d|_d|_d|_d|_dSrzrr=s r r?zSuspiciousRange.__init__s(78/%&04!15"""r r(r@r^r&cD|xjdz c_|js|js |tvrd|_d|_dS|j||_t ||_dS|j}t |}t||r|xjdz c_||_||_dSr) rqr-r7rrrr" is_suspiciously_successive_ranger)r>r(r^unicode_range_aunicode_range_bs r rczSuspiciousRange.feed_infos " :  y4P'P'P(,D %)-D & F  $ ,(1D %)6y)A)AD & F&*&@&3I&>&> +O_ M M 9  3 3q 8 3 3$-!%4"""r c>d|_d|_d|_d|_dSrz)rqrrrr=s r rezSuspiciousRange.resets( !23/$(!%)"""r rfcD|jdkrdS|jdz|jz }|S)NrKr|rx)rqr)r>ratio_of_suspicious_range_usages r rhzSuspiciousRange.ratios8  B & &3  3a 7  !2"'/.r NrVrirjrr r r rrswI6666 5555.**** ///X///r rcBeZdZdZddZdd Zdd Zedd Zd S)SuperWeirdWordPlugin _word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchrq_bad_character_count_buffer_length_buffer_last_char_buffer_last_char_accentuated_buffer_accent_count_buffer_glyph_count_buffer_upper_countr9r:cd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ dSrurr=s r r?zSuperWeirdWordPlugin.__init__sp !$%() */!). %&)*!#$-138*)*!() ()   r r(r@r^r&c|jr|xjdz c_||_|jr|xjdz c_|j|_|jr|xjdz c_|js|j r|jr|j sd|_|j r|xj dz c_ dS|jsdS|j s|j st|rH|xjdz c_|j}|xj|z c_|dkr{|j|z dkrd|_ne|jr<|jr#|j|kr|xjdz c_d|_n"|j dkrd|_|xjdz c_|dkr9|jr2|jdko |j|z dk}|s|xjdz c_d|_|jr'|xjdz c_|xj|z c_d |_d |_d|_d|_d |_d|_d|_ d|_dS|d vr5|js0|jr+d|_|xjdz c_||_d |_dSdSdSdS) rwrTN?rr}F>_-<=>|~)r*rrr+rr2rrrr3r6rr-r7rrrqrrOrrrr.r8)r>r(r^ buffer_lengthprobable_camel_caseds r rczSuperWeirdWordPlugin.feed_infos :    1 $  %.D "z .((A-((151AD . /))Q.)), 0 0'+'7 0  0 ,0(} .((A-(( F"  F :1 71 7|I'>'>1 7    !  !%!4M  ! !] 2 ! !!!,}<CC04D--6 2.6688 20MAA,,1,,04D---2204D-,,1,,""t'?",q0H0=@CG% ,5,,1,,04D-( 2$$)$$))]:)),1)',D $"#D %)D "16D .()D %'(D $'(D $ $ $ @ @ @J A A)-D %   1 $  %.D "16D . . . A @ @ @ @ @r cd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ dSru) rrrrrrrrqrrrrrr=s r rezSuperWeirdWordPlugin.reset,sp!%-2*$)!#(   !$%!#$ $%!#$ #$   r rfcP|jdkr |jdkrdS|j|jz S)N rr|)rrrrqr=s r rhzSuperWeirdWordPlugin.ratio;s3  r ! !d&>!&C&C3(4+@@@r NrVrirjrr r r rrsI ****$I7I7I7I7V % % % %AAAXAAAr rcFeZdZdZdZddZdd Zdd Zedd Z dS)CjkUncommonPluginz< Detect messy CJK text that probably means nothing. rq_uncommon_countr9r:c"d|_d|_dSrzrr=s r r?zCjkUncommonPlugin.__init__Ks%&$%r r(r@r^r&c\|xjdz c_|tvr|xjdz c_dSdSr)rqrrrbs r rczCjkUncommonPlugin.feed_infoOsD " 1 1 1  A %  2 1r c"d|_d|_dSrzrr=s r rezCjkUncommonPlugin.resetVs ! r rfcT|jdkrdS|j|jz }|dkr|dz ndS)Nrr|rrr)r>uncommon_form_usages r rhzCjkUncommonPlugin.ratioZsC  1 $ $3%)%9Dr(r^ is_concerned chunk_seps r rcz!ArchaicUpperLowerPlugin.feed_infos!Z>D,> **  =AA4:: ;0;8868823D .34D 0$(D !DI  ! !Q & ! !'+D $ F  # -DM -',D $  , "t: "  "#: "9%66!;66 %DII $DII!  " ,,1,, )&*j#&*j###r cd|_d|_d|_d|_d|_d|_d|_d|_d|_dS)NrFT) rqrrrrrrrrr=s r rezArchaicUpperLowerPlugin.resetsO !/0,-.*340 $&+#&+# #'   r rfc:|jdkrdS|j|jz S)Nrr|)rqrr=s r rhzArchaicUpperLowerPlugin.ratios&  A % %37$:OOOr NrVrirjrr r r rrfs~ I . . . .+1+1+1+1Z ( ( ( (PPPXPPPr rcBeZdZdZddZddZdd Zedd Zd S)ArabicIsolatedFormPluginrq_isolated_form_countr9r:c"d|_d|_dSrzrr=s r r?z!ArabicIsolatedFormPlugin.__init__s%&)*!!!r c"d|_d|_dSrzrr=s r rezArabicIsolatedFormPlugin.resets !$%!!!r r(r@r^r&ch|xjdz c_|jtzr|xjdz c_dSdSr)rqr1rrrbs r rcz"ArabicIsolatedFormPlugin.feed_infosJ " :- - +  % % * % % % % + +r rfc>|jdkrdS|j|jz }|S)Nrr|r)r>isolated_form_usages r rhzArabicIsolatedFormPlugin.ratios,  1 $ $3%)%>AV%V""r NrVrirj) rWrXrYr[r?rercrkrhr r r rrsu/Ii... Jj j k j   j   j  j j   / / / E 0  D $ $ $ Jj j k j   j   j  j j   =/00  515 5 5HW 5 5!2 5 5    R<< JJuG0@"0EGG H H H JJuG.>suu.EGG H H HtUD$dDI = =B JJu;;;; < < < < ! $ $$r )rrrrr9rS)rF)rr@rrfrrSr9rf)1 __future__rsys functoolsrloggingr version_infotypingrtyping_extensions ImportErrorconstantrrrrrrrrrrrrrutilsrrrrr r!r"r$__annotations__r&r]rmrrrrrrrrrr)r r r r5s""""""" v+++++++       '>I- 9EA AAAA[I[I[I[I[I[I[I[I|""""""""<.L.L.L.L.L'9.L.L.LbOOOOO1OOO6EEEEE*EEE:)D)D)D)D)D&8)D)D)DX4/4/4/4/4/(4/4/4/nBABABABABA-BABABAJNNNNN*NNNDYPYPYPYPYP0YPYPYPx#####1###8 4FFFFR 4IN}%}%}%}%}%}%}%s 2==