K id $UdZddlmZddlZddlZddlZddlmZddl m Z m Z m Z m Z mZmZmZmZddlmZddlmZmZmZdZej2Zej4Zej6Zej8Zej:Zej<Zej>Zej@Z ejBZ!ejDZ"ejFZ#ejHZ$ejJZ%Gd d e Z&ejNd Z(iZ)eee&fe*d <d ddZ+dZ,d d ddZ-dZ.dZ/dZ0e.de0zdzZ1e.de0zdzZ2e.dZ3e.dZ4dZ5Gdde Z6Gdde6Z7Gd d!Z8d"Z9d#Z:d$d%d&e;d'ed(e edd$d,d-d.e e;d'ed/ee<d(e e.different_case_versionsIsF##f%Eq!'')n%EF A''!*  %EsAA "A)brubrr6ffr)setr7 permutationsupdate)r2r3r=valid_string_prefixesresultrBr:ts r!_all_string_prefixesrJHs 2TF $K $% !UF !Q & ! u (6((0 6A MM1!4 5 66 Mr cJtj|tjSN)recompileUNICODE)exprs r!_compilerQes ::dBJJ ''r c tt|S#t$r"t|xtt|<}|cYSwxYwrL)r"tupleKeyError_create_token_collection) version_inforHs r!_get_token_collectionrWisG&u\':;;  $\ 2 3l 34v s (??z"[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*z(?:\{\{|\}\}|\\N\{z*\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+z\}|\\[^N]|[^{}\\])+z(?:\\(?:\r\n?|\n)|[^{}\r\n])+z[^{}]+c d}t|}d}dtzdz}d}d}d}d}t||||} d } td d t| z} d | z} t| | } td | dz}t|| | }t }t|}tt d}t dd}t|}d}d}d}d}t|dz|dz}tddddddd}d}gd }|d!k\r|j d"d#n|j d"d$t|}t|||}t|d%ztd&d'z|d(ztd)d'z}||g} d*}!|r| j |t|!ztd+g| }"t|d,t|"||||d,z}#i}$|D]F}%t||$|%d&z<t||$|%d)z<t||$|%dz<t||$|%dz<Ht}&t}'i}(|D]8})d-D]}*|&j|)|*zd.D]}*|'j|)|*z:|D]})|!D] }*|*|(|)|*z< d/}+t|#},t|,|&|'|$||(t|+S)0Nz[ \f\t]*z #[^\r\n]*u([A-Za-z_0-9€-z]+)z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]T)r2r1z(?:\\.|[^'\\])*'z(?:\\.|[^"\\])*"z(?:\\.|'(?!'')|[^'\\])*'''z(?:\\.|"(?!"")|[^"\\])*"""'''"""z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@`|^!=<>]=?~z[][(){}])z\.\.\.z\r\n?z\nz[;.,@])rz:=?:z'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*'z\\(?:\r\n?|\n)z"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*")r`r_rZrYz\\(?:\r\n?|\n)|\Zr#)r`r_)rZrY);importclassdeftryexceptfinallywhilewithreturncontinuebreakdelpassglobalassertnonlocal) rQ MAX_UNICODEr-r0rJinsertappendrDaddr)-rV WhitespacerCommentName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent PointfloatExpfloat Floatnumber ImagnumberNumberpossible_prefixes StringPrefixStringPrefixWithFfstring_prefixes FStringStartSingleDoubleSingle3Double3TripleOperatorBracket special_argsSpecialFunnyContStrpseudo_extra_pool all_quotes PseudoExtras PseudoTokenr_prefixrrrrIquoteALWAYS_BREAK_TOKENSpseudo_token_compileds- r!rUrU~sLJ*%JG ; . 6D*I#I$I2IiIyAI,HC-/16xAJ#h.H H-K/w1FGJ :{I 6F-.+,L3DIJ+DtT*+L!F F+G+G $u,.?%.G HF Ye+H G:LvAu%As#\"G (GW -EL#EEC!234 #EEC!2345G!&)'J  z0B!BC-B0ABL D1 lFE7D$GHK G$5!)&!1# !)&!1# #+G#4% #+G#4% 5EMEM ) )E   a%i ( )" )E   a%i ( ) )3 3E-2 E * 33S%[1 }mW'-@)A r c`eZdZUeed<eed<eeefed<eed<edeeeffdZ y)Tokentypestring start_posr:rjct|j}t|dkDr |jdt|zdz dfS|jd|jdt|jzfSNr)rrlenr)selfliness r!end_posz Token.end_posshDKK( u:>>>!$s5z1A5q8 8>>!$dnnQ&7#dkk:J&JJ Jr N) rrrr rrrintpropertyrrr r!rrsD  KS#X K KsCxKKr rceZdZdZy) PythonTokencTd|j|jjzS)Nz6TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r))r)_replacernamers r!__repr__zPythonToken.__repr__s%H 499>> 23 4r N)rrrrrr r!rrs4r rc0eZdZdZdZdZdZdZdZy) FStringNodecJ||_d|_d|_d|_d|_y)Nrr6)rparentheses_countprevious_lineslast_string_start_posformat_spec_count)rrs r!__init__zFStringNode.__init__s+ !" %)""#r c.|xjdz c_y)Nr)rr characters r!open_parentheseszFStringNode.open_parentheses s !#r c\|xjdzc_|jdk(rd|_yyrrrrs r!close_parentheseszFStringNode.close_parentheses s. !#  ! !Q &%&D " 'r c2t|jdk(S)Nr\)rrrs r!allow_multilinezFStringNode.allow_multilines4::!##r c4|j|jkDSrLrrs r! is_in_exprzFStringNode.is_in_exprs%%(>(>>>r c@|j xr |jSrL)rrrs r!is_in_format_speczFStringNode.is_in_format_specs??$$?)?)??r N) rrrrrrrrrrr r!rrs!#$' $?@r rc jt|D]\}}|j}t|t|z }|j|jsIt t |j|||zf||d|z} d}|jrJ||d=| dt|j|zfcSd|dfS)Nr:r6r) enumeratelstripr startswithrr FSTRING_ENDr) fstring_stackrline_nrcolumnadditional_prefixfstring_stack_indexnodelstripped_string len_lstriptokens r!_close_fstring_if_necessaryrs%.}%= ;!T!==?[3'7#88  & &tzz 2 &:-.( )<< E !# ** **123"c$**o :: : ; "A %%r cz|d}|j}|jr|rt}nt}n|rt}nt }|j ||}||j|fS|js ||f|_|jd} |D]K} || jj | } | $| jddt| j } M|} | t| z } | jds| jdr|xj| z c_d} | | fS|j| z} | | fS)Nr  r6) rrfstring_format_spec_multi_linefstring_format_spec_single_linefstring_string_multi_linefstring_string_single_linematchrrr-rrendswith) rrlinelnumpostosrregexrrfstring_stack_node end_matchnew_poss r!_find_fstring_stringr.sM  C))+O  2E3E -E.E KKc "E }!!3&&   %)3K! [[^F+I.445;;FC  __Q'(G#.@.F.F*G)GHFI G s6{Gt 5 f$ 7?##f, 7?r )rr)rcoderVrrjc8t|d}t|||S)z0Generate tokens from a the source code (string).T)keepends)rVr)rtokenize_lines)rrVrrs r!tokenizerVs t ,E %li PPr cfd}|S)zL A small helper function to help debug the tokenize_lines function. c?FK|i|D]}t||ywrL)print)argsr+rfuncs r!wrapperz_print_tokens..wrapperbs,4*6* E %LK s!r)rrs` r! _print_tokensr^s Nr T)indentsris_first_tokenrrc #./K./fd}t|\}}}} } } } d} dgd}d}d}d}d}d}|ddz .g}|D]m}.dz .d}t|}|rE|jtrt}|dd}t|}d|dz|z}|d}||dz }d }|rNj |}|r0|j d}t t||d|z|d}d}n ||z}|z}||ks|r{|d }|jsBt| ||.|\}}|r't t||jd d|_ X||k(r ||d}t||.||\}}}||z }|||r|} |D]s}!|!j}"| |"j ||}#|#'|#jd}$t|$t|"z |zt| ks_|d||$dt|" z} u|j | |}%n|j ||}%|%rR||%jdz}d}|%j!d \}&}.|&f/|%jd }'|'dk(r|sJ|}|'d}(n9| j ||})||)j }(|)j }&.|&f/|rS|(d vrO|(dk7s|%Hd }| dk(rA|s?|&}*|*d kDr%t t"d/dj%|*||*Ed{|%s| j ||})|r%| dk(r |s||)j Ed{|)j }d }t t&||.|f||)jdzd}|dz }|(|vs|(dk(r 'dk7r|'dk7rt t('/|n |%jd'| vrD|s| r@g|ddd} t+jd|d|&}+|+||+j Ed{|'j-rt t.|'/|nt1|'/|Ed{nt|(dvrIt3d|Dr|j5|s| dk(r|st t6'/|n|'z}d}n'|(dk(r['j9ds|'j9drJ|r.|d jrt t&|(/||&dz}n||'z}n'|vrP| |'}|j ||}|r,|j d}||&|}'t t|'/|n~/}||&d}|}|(|vs|'dd |vs|'dd|vrf|'d dvrJ.|&f}| j;|(xs*| j;|'dxs| j;|'d }||&d}|}Wt t|'/|n|'| vr2|j%t=| |'t t>|'/|n|(dk(r||&ddvr ||||&dzz }|'dvr|r|d jA|'n}| dz } nw|'dvr|r|d jC|'n\| rZ| dz} nT|'jdrC|rA|d jD|d jFz dk(r|d xjFdz c_#d}'|&dz}t tH|'/|||krp|r8t t&|||j9ds|j9drd}|r:|d }|jr)t t|j|jd .|f},ddD]&}-jKt tLd|,d(t tNd|,|y77H77Kw)a) A heavily modified Python standard library tokenizer. Additionally to the default information, yields also the prefix of each token. This idea comes from lib2to3. The prefix contains all information that is irrelevant for the parser like newlines in parentheses or comments. c3K|dkrR|dkDrttd|fd|d<yjttdd|dkrQyyw)Nrr6)r ERROR_DEDENTpopDEDENT)r,rrsposs r!dedent_if_necessaryz+tokenize_lines..dedent_if_necessaryzshgbk!wr{"!,T5M2FF#  KKMfb$3 3 gbk!s AAArN 0123456789r6Tr^Frrz #\.z...r\z [ \f\t]*$z c3>K|]}|j ywrL)r).0rBs r! z!tokenize_lines..sF11,,..Fs#rr)z\ z\ z\ z([{z)]}r^)(rWrrBOM_UTF8_STRINGrendrSTRINGrrFSTRING_STRINGrrrrr-spanINDENTrt ERRORTOKENNUMBERrM isidentifierNAME_split_illegal_unicode_nameanyclearNEWLINErgetr FSTRING_STARTrrrrOPrr ENDMARKER)0rrVrrrrrrrrrrr paren_levelmax_numcharscontstrnew_liner:rrrrendprogendmatch contstr_startcontlinerrrestfstring_end_token quote_length string_linerrrend_match_string pseudomatchr,rinitialr indent_startmrindentrrs0 ` @@r!rrks4 l+2L-0K# DHGH F Q->q-AA$&!(--a0 se}#))!,B;!M6(.%("((s3uyy{+ e}G724;K^ !#M#(L#gbk1)&"dB??|42<@@@"((s3 q 02599;???iik !S D#;% A6%'!q8#35C>""1%1//]k')M!$"#KtFU|>$/t8LLL ]2%6%A%A%C%j'4HH!)C(.%-'!%."==s3",,q/C sOE%feT6BB$(M"56lG#HM)"1I."1I.9&%)%KM&{{73 8w{{587L 8")++eAh"7"56lG#H%feT6BB--$$[1DU1K%LM!-fEEDT%&\5O%O!Vd56l%::!E>$%b)::5A#q( e^$%b);;EB&'1,K%%c*})"-??'+==>ABC""%771<7E!)C!"eT6::mDj?U;n*g}fEE   D !W%5%5d%;HB    2 2))  DjG!"+3 &"gr223 iW.? @@OA @*@PseC,]92B)]97]9D]9-]-.:]9(]0)C]90]31:]9+]6,I?]9-C]90]93]96]9c#D K fd}dd| t|D]p\}}r3|jr||dd|d|d|zf 5|z ;|z}|jr|Sr|d|d|d|zf |drr |yyw)Nc8trtntSrL)rrr)found is_illegalrr:sr! create_tokenz1_split_illegal_unicode_name..create_tokens:uc6RRr r6FrrT)rr) rrr:r1ichar new_foundr/r0rs ` @@@r!rrsS EJ CU#"4   ""n$" lIaL1$44  I%%'!&.(F#A, ! q(88C! )", n sBB __main__rz3.10)rV)J__doc__ __future__rsysrM itertoolsr7codecsrtypingrrrrr r r r parso.python.tokenr parso.utilsrrrrrr rrrrr rrrrrr rrdecoderr"rr-r0rJrQrWunicode_character_namerrrrrUrrrrrrrrrrrrargvpathopenrBreadrrrrr r!rDs ' 0LL      " "      & &  ( ( ,,  .. !00** $j$"(//'*DF/@AF"+! -2:(?%2234%225KK#++K"L!))!4iX KJ K4%4 @@:&$%RQWQ Q 1Q>CCHoQ kQ "!' ZA C=ZA$ZA#Y ZA S#X ZAkZAzB z 88A;D dqvvx$-A&-IJ e  s HH