bBFje PUdZddlmZddlZddlZddlZddlmZddl m Z m Z m Z m Z mZmZmZmZmZddlmZddlmZmZmZdZejZejZejZejZejZejZej Z ej!Z!ej"Z"ej#Z#ej$Z$ej%Z%ej&Z&Gd d e Z'ej(d Z)iZ*ee e+e+fe'fe,d <d ddZ-dZ.d d ddZ/dZ0dZ1dZ2e0de2zdzZ3e0de2zdzZ4e0dZ5e0dZ6dZ7Gdde Z8Gdde8Z9Gd d!Z:d"Z;d#Zd+Z?dd$d,d-d.e e=d'e e+e+fd/ee+d(e e+e+fd)e e9f d0Z@d1ZAeBd2krhejCd3ZDeEeD5ZFeFGZHdddn #1swxYwYe>eHed45D]ZIeJeIdSdS)6a This tokenizer has been copied from the ``tokenize.py`` standard library tokenizer. The reason was simple: The standard library tokenizer fails if the indentation is not right. To make it possible to do error recovery the tokenizer needed to be rewritten. Basically this is a stripped down version of the standard library module, so you can read the documentation there. Additionally we included some speed and memory optimizations here. )absolute_importN)BOM_UTF8) NamedTupleTupleIteratorIterableListDictPatternSetAny)PythonTokenTypes) split_linesPythonVersionInfoparse_version_stringu􏿿ceZdZUeed<eeed<eeed<eeefed<eed<eeefed<eeed<dS) TokenCollection pseudo_token single_quoted triple_quotedendpats whitespacefstring_pattern_mapalways_break_tokensN)__name__ __module__ __qualname__r __annotations__r strr CC:\PYTHON\_runtimes\venv\Lib\site-packages\parso/python/tokenize.pyrr+ss8s8 #w, c3h'''S!!!!!r!rzutf-8_token_collection_cacheFcapturecR|rJd}|s|dz }|d|zdzS)N(z?:|))join)r%choiceskwargsstarts r"groupr.:s?: E    388G$$ $s **r!ct|dzS)N?)r.)r+s r"mayber1Cs '?S  r!include_fstring only_fstringcd}gd}dh}|rddg}|r|}t}n||z }n|rtS|D]7}tj|D] }|||!8|S)Nc3nKtjd|DD]}d|VdS)Nc:g|]}||fSr )upper).0cs r" zI_all_string_prefixes..different_case_versions..Js$%E%E%Eq!''))n%E%E%Er!) _itertoolsproductr*)prefixss r"different_case_versionsz5_all_string_prefixes..different_case_versionsIsO#%E%Ef%E%E%EF  A''!**      r!)brubrr<ffr)setr= permutationsupdate)r3r4rAvalid_string_prefixesresultrFr?ts r"_all_string_prefixesrNHs 211TF $K  '$% !UUFF !Q & ! ! uu (66(00 6 6A MM11!44 5 5 5 5 6 Mr!c@tj|tjSN)recompileUNICODE)exprs r"_compilerUes :dBJ ' ''r!c tt|S#t$r+t|xtt|<}|cYSwxYwrP)r#tupleKeyError_create_token_collection) version_inforLs r"_get_token_collectionr[is`&u\':':;;  $\ 2 2 3l 3 34v s2AAz"[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*z(?:\{\{|\}\}|\\N\{z*\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+z\}|\\[^N]|[^{}\\])+z(?:\\(?:\r\n?|\n)|[^{}\r\n])+z[^{}]+c d}t|}d}dtzdz}d}d}d}d}t||||} d } td d t| z} d | z} t| | } td | dz}t|| | }t }t|}tt d}t dd}t|}d}d}d}d}t|dz|dz}tddddddd}d}gd }|d!kr|d"d#n|d"d$t|}t|||}t|d%ztd&d'z|d(ztd)d'z}||g} d*}!|r| |t|!ztd+g| R}"t|d,t|"||||d,z}#i}$|D]V}%t||$|%d&z<t||$|%d)z<t||$|%dz<t||$|%dz<Wt}&t}'i}(|D]<})d-D]}*|&|)|*zd.D]}*|'|)|*z=|D]})|!D] }*|*|(|)|*z< d/}+t|#},t|,|&|'|$||(t|+S)0Nz[ \f\t]*z #[^\r\n]*u([A-Za-z_0-9€-z]+)z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]T)r3r2z(?:\\.|[^'\\])*'z(?:\\.|[^"\\])*"z(?:\\.|'(?!'')|[^'\\])*'''z(?:\\.|"(?!"")|[^"\\])*"""'''"""z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@`|^!=<>]=?~z[][(){}])z\.\.\.z\r\n?z\nz[;.,@])rz:=?:z'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*'z\\(?:\r\n?|\n)z"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*")rdrcr^r]z\\(?:\r\n?|\n)|\Zr$)rdrc)r^r]);importclassdeftryexceptfinallywhilewithreturncontinuebreakdelpassglobalassertnonlocal) rU MAX_UNICODEr.r1rNinsertappendrHaddr)-rZ WhitespacerCommentName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent PointfloatExpfloat Floatnumber ImagnumberNumberpossible_prefixes StringPrefixStringPrefixWithFfstring_prefixes FStringStartSingleDoubleSingle3Double3TripleOperatorBracket special_argsSpecialFunnyContStrpseudo_extra_pool all_quotes PseudoExtras PseudoTokenr_prefixrrrrMquoteALWAYS_BREAK_TOKENSpseudo_token_compileds- r"rYrY~sJ*%%JG ; . 6D*I#I$I2IiIyAAI,HC-//16xAJ#h.H H--K/w1FGGJ :{I 6 6F-..+,L3DIIIJ+DtTTT*+L!F F+G+G $u,.?%.G H HF Ye+H G:::LvAu%%%%As###\"G (GW - -EL#EEC!2334 #EEC!233455G!&)'JD  z0B!BCCC-B0ABBBL D111 lFE7D$GGGHK G$55!)&!1!1# !)&!1!1# #+G#4#4% #+G#4#4%  EEMEEM )) ) )E   a%i ( ( ( (! ) )E   a%i ( ( ( ( )33 3 3E-2 E * * 3S%[11 }mW'-@)A)A  r!cteZdZUeed<eed<eeefed<eed<edeeeffdZ dS)Tokentypestring start_posr?rnct|j}t|dkr"|jdt|zdz dfS|jd|jdt|jzfSNr)rrlenr)selfliness r"end_posz Token.end_possjDK(( u::>>>!$s5zz1A5q8 8>!$dnQ&7#dk:J:J&JJ Jr!N) rrrrrrrintpropertyrr r!r"rrs|  KKKS#X KKK KsCxKKKXKKKr!rceZdZdZdS) PythonTokencHd||jjzS)Nz6TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r))r)_replacernamers r"__repr__zPythonToken.__repr__s$H 49> 223 4r!N)rrrrr r!r"rrs#44444r!rc2eZdZdZdZdZdZdZdZdS) FStringNodecL||_d|_d|_d|_d|_dS)Nrr<)rparentheses_countprevious_lineslast_string_start_posformat_spec_count)rrs r"__init__zFStringNode.__init__s1 !" *.""#r!c&|xjdz c_dS)Nr)rr characters r"open_parentheseszFStringNode.open_parentheses s !#r!cN|xjdzc_|jdkr d|_dSdSrrrrs r"close_parentheseszFStringNode.close_parentheses s: !#  !Q & &%&D " " " ' &r!c2t|jdkS)Nr`)rrrs r"allow_multilinezFStringNode.allow_multilines4:!##r!c"|j|jkSrPrrs r" is_in_exprzFStringNode.is_in_exprs%(>>>r!c:| o|jSrP)rrrs r"is_in_format_speczFStringNode.is_in_format_specs??$$$?)??r!N) rrrrrrrrrr r!r"rrss###$$$''' $$$???@@@@@r!rc t|D]\}}|}t|t|z }||jrYt t |j|||zf||d|z} d}|jrJ||d=| dt|j|zfcSd|dfS)Nr?r<r) enumeratelstripr startswithrr FSTRING_ENDr) fstring_stackrline_nrcolumnadditional_prefixfstring_stack_indexnodelstripped_string len_lstriptokens r"_close_fstring_if_necessaryrs%.}%=%= ; ;!T!==??[[3'7#8#88  & &tz 2 2 ; &:-.(  )<< E !# * * **1223"c$*oo :: : : : ; "A %%r!c|d}|}|r|rt}nt}n|rt}nt }|||}| |j|fS|js ||f|_| d} |D]T} || j | } | 0| ddt| j  } U|} | t| z } | ds| dr|xj| z c_d} n |j| z} | | fS)Nr  r<) rrfstring_format_spec_multi_linefstring_format_spec_single_linefstring_string_multi_linefstring_string_single_linematchrrr.rrendswith) rrlinelnumpostosrregexrrfstring_stack_node end_matchnew_poss r"_find_fstring_stringr.sw  C))++O  /  42EE3EE  /-EE.E KKc " "E }!3&&  0%)3K! [[^^F+II.45;;FCC  __Q''(G#.@.F*G*G)G(GHFG s6{{Gt- 5 5- f$#f, 7?r!)rr)rcoderZrrncHt|d}t|||S)z0Generate tokens from a the source code (string).T)keepends)rZr)rtokenize_lines)rrZrrs r"tokenizerVs+ t , , ,E %li P P PPr!cfd}|S)zL A small helper function to help debug the tokenize_lines function. c?HK|i|D]}t||VdSrP)print)argsr,rfuncs r"wrapperz_print_tokens..wrapperbsET4*6**  E %LLLKKKK  r!r )rrs` r" _print_tokensr^s# Nr!T)indentsris_first_tokenrrc #./K./fd}t|\}}}} } } } d} dgd}d}d}d}d}d}|ddz .g}|D]}.dz .d}t|}|r]|tr t}|dd}t|}d|dz|z}|d}||dz }d }|r`||}|r>|d}t t||d|z||Vd}d}n ||z}||z}||kr|r|d }|sGt| ||.|\}}|r't t||j d Vd|_ d||krn||d}t||.||\}}}||z }||V|r|} |D]}!|!j}"| |"||}#|#k|#d}$t|$t|"z |zt| kr#|d||$dt|" z} || |}%n|||}%|%rb||%dz}d}|%d \}&}.|&f/|%d }'|'dkr|sJ|}n?|'d}(nH| ||})||)}(|)}&.|&f/|rc|(d vr_|(dks|%Wd }| dkrO|sM|&}*|*d kr.t t"d/dV|*||*Ed{V|%s| ||})|r+| dkr%|s#||)Ed{V|)}d }t t&||.|f||)dzVd}|dz } |(|vs|(dkr'|'dkr!|'dkrt t(|'/|Vn|%d|'| vrO|s| rKg|dd<d} t+jd|d|&}+|+#||+Ed{V|'rt t.|'/|Vnt1|'/|Ed{Vn|(dvrZt3d|Dr||s"| dkr|st t6|'/|Vn||'z}d}n|(dkro|'ds|'drJ|r:|d r t t&|(/|V|&dz}n:||'z}n3|'|vrj| |'}|||}|r:|d}||&|}'t t|'/|VnՉ/}||&d}|}n|(|vs|'dd |vs |'dd|vr|'d dvr].|&f}| |(p5| |'dp| |'d }||&d}|}nIt t|'/|Vn'|'| vrB|t=| |'t t>|'/|Vn|(dkr||&ddvr||||&dzz }n|'dvr$|r|d  |'n| dz } n}|'dvr&|r|d !|'n[| r| dz} nS|'dr>|r<|d j"|d j#z dkr|d xj#dz c_#d}'|&dz}t tH|'/|V||k|rEt t&|||V|ds|drd}|r3|d }|j r$t t|j |j d V.|f},ddD]/}-%t tLd|,dV0t tNd|,|VdS)a) A heavily modified Python standard library tokenizer. Additionally to the default information, yields also the prefix of each token. This idea comes from lib2to3. The prefix contains all information that is irrelevant for the parser like newlines in parentheses or comments. c3K|dkri|dkr"ttd|fdV|d<dSttddV|dkgdSdS)Nrr<)r ERROR_DEDENTpopDEDENT)r-rrsposs r"dedent_if_necessaryz+tokenize_lines..dedent_if_necessaryzsgbk!!wr{""!,T5M2FFFFF#  KKMMMfb$33 3 3 3 gbk!!!!!!r!rN 0123456789r<Tr^Frrz #\.z...r`z [ \f\t]*$z c3@K|]}| VdSrP)r)r9rFs r" z!tokenize_lines..s1FF11,,...FFFFFFr!#rr)z\ z\ z\ z([{z)]}rb)(r[rrBOM_UTF8_STRINGrendrSTRINGrrFSTRING_STRINGrrrrr.spanINDENTrx ERRORTOKENNUMBERrQ isidentifierNAME_split_illegal_unicode_nameanyclearNEWLINErgetr FSTRING_STARTrrrrOPrr ENDMARKER)0rrZrrrrrrrrrrr paren_levelmax_numcharscontstrnew_liner?rrrrendprogendmatch contstr_startcontlinerrrestfstring_end_token quote_length string_linerrrend_match_string pseudomatchr-rinitialr indent_startmrindentrrs0 ` @@r"rrks 4444444 l++2L-0K# DHGH F Q->q-A-AA$&!(--a00 se}#))!,,B;;!MM6(.%("((s33uyy{{+ e} AG7224;K^ !##M##(L#gbk11)&"dB?????|44422<@@@@@@@@@ "((s33@ q 0 0 022599;;?????????iikk !S D#;% A6%'!q8##35C<>>>>>""1%%1///]/k/')M!!!$"#KtFUF|<>$//Lt8L8LLLL 7]2%6%A%A%C%C7%j'4HHHHH!)CC(.%%-''!%."==s33",,q//C sOE%feT6BBBBBB$(M"566lG#HM))"1"I.."1"I..9&&%)%KM&{{733 8w{{587L7L 8")++eAh"7"7"566lG#H%feT6BBBBBB---$$[1DU1K%L%LMMM!-fEEEEEEDT%&&\5O%O%O!Vd566l%::!E>>$)%b)::5AAAA#q( e^^$-%b);;EBBBB&-'1,K%%c**$}$)"-?'+=>ABCC""%771<77E!)C!"eT6:::::mDjjp*g}fEEEEE   D ! ! W%5%5d%;%; H B    2)    DjG!""+33 &"gr222222 iW.? @ @@@@@@r!c# K fd}dd| t|D]\}}r@|r&|V|dd|d|d|zf A|z G|z}|r|cr!|Vd|d|d|zf |dr|VdSdS)NcDtrtntSrP)rrr)found is_illegalrr?sr" create_tokenz1_split_illegal_unicode_name..create_tokens =::uc6RRRr!r<FrrT)rr) rrr?r4ichar new_foundr2r3rs ` @@@r"rrsWSSSSSSSS EJ CU##""4  "  "" "lnn$$$" lIaL1$44  I%%'' "!9&,..(((F#A, ! q(88C!  lnnr!__main__rz3.10)rZ)K__doc__ __future__rsysrQ itertoolsr=codecsrtypingrrrrr r r r r parso.python.tokenr parso.utilsrrrrvr rrrrrrrrrrrrrdecoder r#rrr.r1rNrUr[unicode_character_namerrrrrYrrrrrrrrrrrargvpathopenrFreadrrrr r!r"rGsC   '&&&&& 0/////LLLLLLLLLL      "      &  ( ,  . !0* """""j""""(/'**BDeCHo>?DDD"+++++!!! -2:(((?%X2234%H225KK#+(+K"L"L!))!4!4iiiX K K K K KJ K K K44444%444 @@@@@@@@:&&&$%%%ROUQQQ Q %c3hQ