7,j1FUddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZddlmZdd lmZd d lmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$e e dVdZ%e e dWdZ&e e dXdZ'e(dej)DZ*de+d<de*DZ,de+d<e e dYdZ-e e dWdZ.e e dWdZ/e e dWdZ0e e dWd Z1e e dWd!Z2e e dWd"Z3e e dWd#Z4e e dWd$Z5e e dWd%Z6e e dWd&Z7e e dWd'Z8e e dWd(Z9e e dWd)Z:e e dWd*Z;e e d[d\d2Z?e d3 d]d5Z@d^d7ZAd_d9ZBd`dad=ZCdbdAZDdcdBZEdCejFdDfdddHZG dedfdUZHdS)g) annotationsN) bisect_right)IncrementalDecoder)aliases) lru_cache)findall) Generator)MultibyteIncrementalDecoder)ENCODING_MARKSIANA_SUPPORTED_SIMILARRE_POSSIBLE_ENCODING_INDICATIONUNICODE_RANGES_COMBINEDUNICODE_SECONDARY_RANGE_KEYWORDUTF8_MAXIMAL_ALLOCATIONCOMMON_CJK_CHARACTERS_LATIN_CJK_HANGUL _KATAKANA _HIRAGANA_THAI_ARABIC_ARABIC_ISOLATED_FORM_ACCENT_KEYWORDS _ACCENTUATED)maxsize characterstrreturnintcl tj|}n#t$rYdSwxYwd}d|vr |tz}d|vr |tz}d|vr |t z}d|vr |t z}d|vr |tz}d|vr |tz}d|vr|tz}d |vr |tz}tD]}||vr |tz}n|S) zRCompute all name-based classification flags with a single unicodedata.name() call.rLATINCJKHANGULKATAKANAHIRAGANATHAIARABICz ISOLATED FORM) unicodedataname ValueErrorrrrrrrrrrr)rdescflagskws HC:\PYTHON\GemmaClient\venv\Lib\site-packages\charset_normalizer/utils.py_character_flagsr1&s$Y// qqE$  }}  4 T T  ~~ 4  d " " * *E :: \ !E E  Ls  %%boolcJtt|tzSN)r2r1rrs r0is_accentuatedr6Is  ++l: ; ;;ctj|}|s|S|d}tt |ddS)N r)r* decompositionsplitchrr!)r decomposedcodess r0 remove_accentr@NsO!/ ::J !'',,E s58R  ! !!r7c#:K|]\}}|j|j|fVdSr4)startstop).0r+ ord_ranges r0 rF[sE<<i_ind+<<<<<<r7zlist[tuple[int, int, str]]_UNICODE_RANGES_SORTEDcg|] }|d S)r)rDes r0 rK_s#I#I#IQAaD#I#I#Ir7z list[int]_UNICODE_RANGE_STARTS str | Nonect|}tt|dz }|dkrt|\}}}||kr|SdS)zK Retrieve the Unicode range official name from a single character. r rN)ordrrLrG)r character_ordidxrBrCr+s r0 unicode_rangerRbsU YM ,m < |+<>TZ>PcPdPo)isspacer*rY)rr[s r0 is_separatorrrsOi+AAAt)29== $ $ P(:>P(PPr7cV||kSr4)islowerisupperr5s r0is_case_variablervs%     )"3"3"5"5 55r7cJtt|tzSr4)r2r1rr5s r0is_cjkrxs  ++d2 3 33r7cJtt|tzSr4)r2r1rr5s r0 is_hiraganarz  ++i7 8 88r7cJtt|tzSr4)r2r1rr5s r0 is_katakanar}r{r7cJtt|tzSr4)r2r1rr5s r0 is_hangulr  ++g5 6 66r7cJtt|tzSr4)r2r1rr5s r0is_thairs  ++e3 4 44r7cJtt|tzSr4)r2r1rr5s r0 is_arabicrrr7cJtt|tzSr4)r2r1rr5s r0is_arabic_isolated_formrs  ++.CC D DDr7c|tvSr4)rr5s r0is_cjk_uncommonrs 1 11r7 range_namecDtfdtDS)Nc3 K|]}|vV dSr4rI)rDkeywordrs r0rFz-is_unicode_range_secondary..s(TTw*$TTTTTTr7)anyr)rs`r0is_unicode_range_secondaryrs' TTTT4STTT T TTr7cr|duo!|duo |dko|dkS)NFu)rq isprintabler5s r0is_unprintablersR u$ "  ! ! # #u , "   "  ! r7 sequencebytes | bytearray search_zonec t|ttfstt |}t t |dt||dd}t |dkrdS|D][}| dd}tj D]\}}||kr|ccS||kr|ccS\dS)zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. Nasciiignoreerrorsr-_) isinstancebytes bytearray TypeErrorlenrrmindecodelowerreplaceritems)rrseq_lenresultsspecified_encodingencoding_alias encoding_ianas r0any_specified_encodingrs h 2 3 3x==G ',3w ,,,-44WX4NNG  7||qt% % %/5577??SII .5]__ % % )NM!333$$$$$$ 222$$$$$$3 % 4r7r+ch|dvp.ttjd|jtS)zQ Verify is a specific encoding is a multi byte one based on it IANA name > utf_7utf_8utf_16utf_32 utf_16_be utf_16_le utf_32_be utf_32_le utf_8_sig encodings.) issubclass importlib import_modulerr )r+s r0is_multi_byte_encodingr sG      3T 3 344G#   r7tuple[str | None, bytes]ctD]I}t|}t|tr|g}|D]}||r||fccS JdS)z9 Identify and extract SIG/BOM in given sequence. )Nr7)r rr startswith)r iana_encodingmarksmarks r0identify_sig_or_bomrs (++ %3M%B eU # # GE + +D""4(( +$d****** + + 9r7rc |dvS)N>rrrI)rs r0should_strip_sig_or_bomr0s  4 44r7Tcp_namestrictc|dd}tjD]\}}|||fvr|cS|rt d|d|S)zIReturns the Python normalized encoding name (Not the IANA official name).rrzUnable to retrieve IANA for '')rrrrr,)rrrrs r0 iana_namer4smmoo%%c3//G *1!!% ~}5 5 5  6ECCCCDDD Nr7 iana_name_a iana_name_bfloatct|st|rdStjd|j}tjd|j}|d}|d}d}t dD]C}t |g}||||kr|dz }D|dz S)Ngrrrrr )rrrrrangerr) rr decoder_a decoder_bid_aid_bcharacter_match_counti to_be_decodeds r0 cp_similarityrEsk**.D[.Q.Qs'(B[(B(BCCVI'(B[(B(BCCVI(y999D(y999D!" 3ZZ''$aSzz ;;} % %])C)C C C !Q & ! 3 &&r7c2|tvo|t|vS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r )rrs r0 is_cp_similarrYs% -- ? 1+> >r7charset_normalizerz)%(asctime)s | %(levelname)s | %(message)slevel format_stringNonectj|}||tj}|tj|||dSr4)logging getLoggersetLevel StreamHandler setFormatter Formatter addHandler)r+rrloggerhandlers r0set_logging_handlerrdsm  t $ $F OOE#%%G *=99::: gr7 sequencesroffsetsr chunk_sizebom_or_sig_availablestrip_sig_or_bom sig_payloadris_multi_byte_decoderdecoded_payloadGenerator[str, None, None]c #K|r!|dur|D]} || | |z} | sdS| VdS|D]} | |z} | t|dzkr|| | |z} |r |dur|| z} | ||rdnd} |ru| dkrot|d} |r]| d| |vrQt| | dz d D]<}||| } |r |dur|| z} | |d} | d| |vrn=| VdS) NFrrrrr:)rrrr)rrrrrrrrrrchunk chunk_end cut_sequencechunk_partial_size_chkjs r0cut_sequence_chunksrqs*0E99  A#AJ$67E KKKK    # # AJI3y>>A---$QZ%78L# :(8E(A(A*\9  ''#8Fxxh(E% "Q.1*b.A.A&$ "5556oMM"1a!eR00 " "'09'= /F4D4M4M+6+EL , 3 3M( 3 S S !8"8!89_LL!EMKKKKG# # r7)rrr r!)rrr r2)rrr r)rrr rM)rrr r2)r)rrrr!r rM)r+rr r2)rrr r)rrr r2)T)rrrr2r r)rrrrr r)rrrrr r2)r+rrr!rrr rr4)rrrrrrrr!rr2rr2rrrr2rrMr r)I __future__rrrr*bisectrcodecsrencodings.aliasesr functoolsrrertypingr _multibytecodecr constantr r rrrrrrrrrrrrrrrr1r6r@sortedrrG__annotations__rLrRrTr]rcrgrrrvrxrzr}rrrrrrrrrrrrrrrINFOrrrIr7r0r s"""""""%%%%%%%%%%%%* *+++,+D *+++<<<,+< *+++""",+"6