ZL i$dZddlZddlmZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZmZmZdZdZddlZdd lmZd d l m!Z"e#e"Dcgc] }|dd k7s |c}gd zZ$["e%e&e&fZ'e%e&e(e'e'e(fZ)iejTeejVe ejXeejZeej\e ej^eej`eejbeejdeejfeejheejjeejleejneejpeejreejteejveejxeejzeej|eej~eejeeje iZBGddeCZDdejde(deejdejfdZFdde(deedee)fdZGde&de(de'de'de(ddf dZHeIdk(rleJejd kDr*eGeLejd jZNn eGejjZNeND]ZPeHeP yycc}w)aTokenization help for Python programs. generate_tokens(readline) is a generator that breaks a stream of text into Python tokens. It accepts a readline-like method which is called repeatedly to get the next line of input (or "" for EOF). It generates 5-tuples with these members: the token type (see token.py) the token (a string) the starting (row, column) indices of the token (a 2-tuple of ints) the ending (row, column) indices of the token (a 2-tuple of ints) the original line (string) It is designed to match the working of the Python tokenizer exactly, except that it produces COMMENT tokens for comments and gives type OP for all operators Older entry points tokenize_loop(readline, tokeneater) tokenize(readline, tokeneater=printtoken) are the same, except instead of generating tokens, tokeneater is a callback function to which the 5 fields described above are passed as 5 arguments, each time a new token is found.N)Iterator)Optional)Grammar)ASYNCAWAITCOMMENTDEDENT ENDMARKER FSTRING_ENDFSTRING_MIDDLE FSTRING_STARTINDENTNAMENEWLINENLNUMBEROPSTRING TSTRING_ENDTSTRING_MIDDLE TSTRING_STARTtok_namezKa-Ping Yee z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro) TokenType)token_)tokenizegenerate_tokens untokenizec eZdZy) TokenErrorN)__name__ __module__ __qualname__]/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/blib2to3/pgen2/tokenize.pyr!r!gsr&r!rsource prev_tokenreturnc |jtjk(rR|O|jtjtjfvr"||j |j }|jdrktjtj|j |j dz|j|j|j|jdzS|jds|jdrktjtj|j |j dz|j|j|j|jdzS|S)a  Black treats `\\\n` at the end of a line as a 'NL' token, while it is ignored as whitespace in the regular Python parser. But, only the first one. If there's a `\\\n` following it (as in, a \ just by itself on a line), that is not made into NL. z\ z\ z\ ) typer whitespacenlnewline start_index end_index startswithpytokensToken start_line start_col)rr(r) token_strs r'transform_whitespacer:js'  i***  " OOILL)2C2C#D D5,,u?    )>> !!!!A%    !#  ! !& )Y-A-A&-I>> !!!!A%    !#  Lr&grammarc#K|jd}|dgz }d\}}d} tj|D];}t|||}|j|j }}|j tjk(rG||j|j}|j tjk(r|dk(r|}||jdz }|j tjk(rJ|dvrF|dk(rtnt||j|j f|j|j f|fn<|j tj"k(r|dk(r|j|jk(sJ|j |j dzk(sJd }t%|j |j dzD]9} | dz} t&|j ||j| f|j| f|f;nt&j)|j } | t+d |j t&|j ||j|j f|j|j f|f|}>y#tj,$rt/d ||ftj0$r)} t/d t | j2||fd} ~ wwxYww) N )rrr)asyncawaitr?z...r,.zUnknown token type: z&Unexpected EOF in multi-line statementzFailed to parse: )splitr5rr:r7r8r.rr/r2r3r1 identifierrrend_lineend_coloprangeTOKEN_TYPE_MAPget ValueError UnexpectedEOFr! TokenizeErrorr") r(r;lineslinecolumnr)rr9 source_liner8rE token_typeexcs r'rrs LL E bTMELD&+/J:S&&v.4 E( CE ++U__&DzzY111u005??CIzzY...9?#  0 01 45KzzY111iCU6U''1Eu%%u7^^U]]3 y||+ U0B''5>>999}}!(;;;; !&u!8K!LI'!mG&uzz2!))951# ,// ; %$';EJJ>%JKK"5::.%%u7^^U]]3 Ji4 l  ! !SAD&>RR  ! !S,T#Y-?-?,@AD&>RRSs)KII76K73K*$KKKr.srow_colerow_colrNc ^|\}}|\}}t|d|d|d|dt|d| y)N,-z:  )printr) r.rrSrTrNsrowscolerowecols r' printtokenr^sFLT4LT4 TF!D64&$s8D>*:"UI FGr&__main__)N)Q__doc__syscollections.abcrtypingrblib2to3.pgen2.grammarrblib2to3.pgen2.tokenrrrr r r r r rrrrrrrrrrr __author__ __credits__r5rr>r_tokendir__all__tupleintCoordstr TokenInfoindentdedentr1r0comment semicolonlparenrparenlbracketrbracketlbracerbracecolonrFrCnumberstring fstring_startfstring_middle fstring_end tstring_starttstring_middle tstring_end endmarkerrH Exceptionr!r6r:rr^r"lenargvopenreadtoken_iteratorstdintok)xs0r'rs8 #0 $*,* P &k 1QqTS[1 15    c3h #sE5#- .  f fwLL"  w   bbbbOORLL"$ f!"f#$] n ; ] n ; 18! $ >>$#&$4