K iJdZddlmZddlmZmZddlZddlZddlm Z m Z Gdde Z Gdd e Z Gd d e ZGd d e ZGdde ZGdde ZGdde ZGddeZGddeZGddej,ZeGddZGddZeGddZd d d! d$d"Zd%d#Zy)&zTpytokens - A Fast, spec compliant Python 3.12+ tokenizer that runs on older Pythons.) annotations) dataclassfieldN)IteratorNewTypec eZdZy) TokenizeErrorN__name__ __module__ __qualname__W/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/pytokens/__init__.pyr r rr c eZdZy)IndentationErrorNr rrrrrrrrc eZdZy)InconsistentUseOfTabsAndSpacesNr rrrrrrrrc eZdZy) DedentDoesNotMatchAnyOuterIndentNr rrrrrrrrc eZdZy)UnterminatedStringNr rrrrrrrrc eZdZy) UnexpectedEOFNr rrrrrrrrc eZdZy)!UnexpectedCharacterAfterBackslashNr rrrrrrrrc eZdZy) NotAnIndentNr rrrrr rrrc eZdZy) UnderflowNr rrrr!r!#rrr!ceZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZd!dZd!dZ d"dZ!y )# TokenType c d|jS)Nz TokenType.)nameselfs r__repr__zTokenType.__repr__IsDII;''rcx|jdk(ry|jry|jjS)N identifierNAMEOP)rA is_operatorupperrBs rto_python_tokenzTokenType.to_python_tokenLs1 99 $    yy  rcZtj|cxkxrtjkScSN)r# _op_start_op_endrBs rrIzTokenType.is_operatorUs$""T=I,=,=====rNreturnstrrQbool)"r r r whitespaceindentdedentnewlinenlcommentrN semicolonlparenrparenlbracketrbracketlbracerbracecolonoprOrFnumberstring fstring_startfstring_middle fstring_end tstring_starttstring_middle tstring_end endmarker errortokenrDrKrIrrrr#r#&sJ F FG BGII F FHH F F E BGJ F FMNKMNKIJ(!>rr#c\eZdZUded<ded<ded<ded<ded<ded<ded <d d Zy ) Tokenr#typeint start_index end_index start_line start_colend_lineend_colc|jtjk(s|jtjk(r4|jt |k(r|j t |dzk(ry|jtjk(r7|jt |dzk(r|j t |dzk(ry|jtjk(ry||j|j S)Nr$) rpr#rXrYrrlenrsrWrl)rCsources r to_byte_slicezToken.to_byte_sliceesYY)++ +tyyILL/H  CK/#f+/1 II)) )  CK!O3#f+/1 99 ++ +d&&88rN)r{rRrQrR)r r r __annotations__r|rrrroroYs+ ONONM L9rroceZdZedeZedZedZedZedZ edZ edZ ddZ dd Z dd Zdd Zdd Zdd ZddZddZy) FStringStateStater$r%r&r'r(r)c<tj|_g|_yrM)r not_fstringstatestackrBs r__init__zFStringState.__init__s!-- /1 rcx|jj|jtj|_yrM)rappendrrat_fstring_middlerBs r enter_fstringzFStringState.enter_fstrings% $**%!33 rc|jtjk(sJ|jj |_yrM)rrat_fstring_endrpoprBs r leave_fstringzFStringState.leave_fstrings,zz\88888ZZ^^% rc|jtjk(r%|jj |jtj |_yrM)rrin_fstring_expr_modifierrrat_fstring_lbracerBs r!consume_fstring_middle_for_lbracez.FStringState.consume_fstring_middle_for_lbraces7 ::>> > JJ  djj )!33 rc.tj|_yrM)rrrrBs rconsume_fstring_middle_for_endz+FStringState.consume_fstring_middle_for_ends!00 rc.tj|_yrM)rin_fstring_exprrrBs rconsume_lbracezFStringState.consume_lbraces!11 rcV|jtjk(s|jtjk(sJt |j dkDr@|j dtjk(r |j j |_ytj|_y)Nr)rrrrrzrrrrBs rconsume_rbracezFStringState.consume_rbracesx JJ,66 6zz\BBB  C  Oa  2,"G"GG)DJ%77DJrcl|jtjk(sJtj|_yrM)rrrrrBs r consume_colonzFStringState.consume_colons'zz\99999!:: rNrQNone)r r r rrqrrrrrrrrrrrrrrrrrrrr}so GS !E(KaaAhO$Qx1XN24&4 12 8;rrcBeZdZUded<ded<dZded<dZded<d Zded <d Zded <dZded <dZ ded <dZ ded<dZ ded<e e Zded<dZded<e e Zded<dZded<e eZded<e e Zded<dZded<dZded <d!Zded"<d!Zded#<d!Zded$<dAd%ZdBd&ZdBd'ZdCd(ZdDd)ZdCd*ZdCd+Z d!d,dEd-Z!dFd.Z"dGd/Z#dCd0Z$dHd1Z%dHd2Z&dHd3Z'dHd4Z(dHd5Z)dHd6Z*dId7Z+dJd8Z,dHd9Z-dHd:Z.dHd;Z/dAd<Z0dAd=Z1dHd>Z2dKd?Z3dHd@Z4y)L TokenIteratorrRr{rTissue_128233_handlingrrq current_index prev_indexr$ line_numberprev_line_number byte_offsetprev_byte_offsetTall_whitespace_on_this_line bracket_level)default_factoryz list[int]bracket_level_stackNz Token | None prev_tokenz list[str] indent_stackdedent_counterr fstring_statezlist[tuple[str, str]]fstring_prefix_quote_stackz str | Nonefstring_prefix fstring_quoteF weird_op_caseweird_op_case_nlweird_whitespace_casecF|jt|jkSrMrrzr{rBs r is_in_boundszTokenIterator.is_in_boundss!!C $444rcX|jsJ|j|jSrM)rr{rrBs rpeekzTokenIterator.peeks)  """{{4--..rc|jdzt|jksJ|j|jdzSNr$rrBs r peek_nextzTokenIterator.peek_nexts>!!A%DKK(8888{{4--122rcX|xjdz c_|xjdz c_yrrrrBs radvancezTokenIterator.advances$ a ArcX|xj|z c_|xj|z c_yrMr)rCcounts r advance_byzTokenIterator.advance_bys$ e# E!rcJ|xjdz c_d|_d|_y)Nr$rT)rrrrBs r next_linezTokenIterator.next_lines# A+/(rc|j|jdk(r&|xjdz c_|jy|jy)N r$)r{rrrrBs radvance_check_newlinez#TokenIterator.advance_check_newlines= ;;t)) *d 2   ! #  NN  LLNr ignore_casec |D]}|jt|zt|jkDr2|j|j|jt|z}|r |j}|j}||k(syy)NTF)rrzr{lower)rCroptionsoptionsnippets rmatchzTokenIterator.matchs F!!CK/#dkk2BBkk$"4"4t7I7ICPVK7WXG!--/  rc |jyd|jvrk|tjk(rtj}nG|tjk(rtj }n#|tj k(rtj}|jrB|js2|tjtjfvrtjn|}|jr6|j|j|j}|dk(rd|_d|_t#||j|j|j$|j&|j(|j*}|tj,k(s|tj.k(r|j1n.|tj2k(s|tj4k(rnd|_||_|j|_|j(|_|j*|_d|_|S)Ntz F)rprrrsrtrurvrw)rr#rfrirgrjrhrkrrIrdrercr{rrrrorrrrrXrYrrUrZr)rCtok_type token_type token_strtokens r make_tokenzTokenIterator.make_tokens    *sd6I6I/I9222$22Y555$33Y222$00!!((*!1!193C3C DD LL     DOOd6H6HIIE!380!&D ((,,++%%$$  y(( (H ,D NN  -- -Y=N=N1N /4D ,,, $ 0 0 $ 0 0" rc|j?|jJ|jj|j|jf||_||_yrM)rrrr)rCprefixquotes rpush_fstring_prefix_quotez'TokenIterator.push_fstring_prefix_quote7sZ    *%%1 11  + + 2 2$$d&8&89 %"rc|j|jJtt|jdk(rdn|jj \|_|_y)Nr)NN)rrr!rzrrrBs rpop_fstring_quotezTokenIterator.pop_fstring_quoteAs`    &%%- --O4223q8 00446 0T/rc|jr,|j|jdk(r|j|j|jsB|j dkDs3|j jtjk(s |jrtjntj}|j|}d|_|S)N rF)rr{rrrrrrrrrr#rYrXr)rCrrs rrXzTokenIterator.newlineLs    4;;t/A/A#Bd#J LLN %%%%)%%++|/K/KK33 LL""  + % rc |jdk7rtt|jdkDr9|jj }|j t jS|j t jS)Nr) rrrzrrrr#rWrl)rC_s rrlzTokenIterator.endmarker^sg    "  t  !A %!!%%'A??9#3#34 4y2233rc4 d}|j|jjrd}|j|j r|j|jjs|j|jdk(re|j|j rE|j|jjrH|j|jdk(re|j r,|j|jdk(r|j|j r|j|jjsF|j|jdk(r|j|jdz jr|j|j ro|j|jjrH|j|jdk(r+|j|jdz jr|jdzt |jkr<|s+|j|jdz jr|j|jdk(s|j|jdk(r|j|jdzjs|jdzt |jkr|j|jdzd k(s|j|jdzd k(rJ|j|jdzjr |j|j|j r|j|jjsH|s*|j|jdz jr|j|jdk(r|j|j rq|j|jjrH|s*|j|jdz jr|j|jdk(r|j rt|s*|j|jdz jrH|j|jd k(s|j|jd k(r|j|j|j z dk(r|j|jdz dk(r|jdzt |jkrI|j|j|jdzd k(r |j|j|jtjS|jtjS)NFTr.r$eEr%+-jJz..) r{risdigitrrrzrrr#rcrd)rCdigit_before_decimals rdecimalzTokenIterator.decimalhs$ ;;t)) * 2 2 4#' LLN! KK** + 3 3 5{{4--.#5 LLN ! KK** + 3 3 5{{4--.#5    4;;t/A/A#Bc#I LLN! KK** + 3 3 5 D../36KK 2 2Q 67??A LLN! KK** + 3 3 5 D../36KK 2 2Q 67??A    !C $4 4 !T[[1C1Ca1G%H%P%P%R D../36;;t112c9 D..23;;=&&*S-== D$6$6$:;sB;;t'9'9A'=>#E D$6$6$:;CCE LLN LLN! KK** + 3 3 5%T5G5G!5K)L)T)T)VKK 2 23s: LLN! KK** + 3 3 5%T5G5G!5K)L)T)T)VKK 2 23s:     !T[[1C1Ca1G%H%P%P%R D../36;;t112c9 LLN    0A 5 D..23s:""Q&#dkk*::KK 2 2T5G5G!5KLPTT  ??9<<0 0y//00rc|j|j|jr|j|jdk(s8|j|jdk(s|j|jdk(rw|j|jrW|j|jdk(r=|j|jdk(rZ|j|jdk(rw|jr|j|jdk(s|j|jdk(rL|j|jr,|j|jdk(r|j|jr|j|jdk(s8|j|jdk(s|j|jdk(rw|j|jrW|j|jdk(r=|j|jdk(rZ|j|jdk(rw|j t j S)N01rrrrrrr{rrr#rdrBs rbinaryzTokenIterator.binarys  ! KK** +s 2{{4--.#5{{4--.#5 LLN ! KK** +s 2{{4--.#5{{4--.#5     KK** +s 2{{4--.#5 LLN  "t{{43E3E'F#'M ! KK** +s 2{{4--.#5{{4--.#5 LLN ! KK** +s 2{{4--.#5{{4--.#5y//00rc|j|j|jr|j|jdk\r|j|jdks|j|jdk(rv|j|jrV|j|jdk\r|j|jdkrY|j|jdk(rv|jr|j|jdk(s|j|jdk(rL|j|jr,|j|jdk(r|j|jr|j|jdk\r|j|jdks|j|jdk(rv|j|jrV|j|jdk\r|j|jdkrY|j|jdk(rv|j t j S)Nr7rrrrrrBs roctalzTokenIterator.octals  ! KK** +s 2 D../36{{4--.#5 LLN ! KK** +s 2 D../36{{4--.#5     KK** +s 2{{4--.#5 LLN  "t{{43E3E'F#'M ! KK** +s 2 D../36{{4--.#5 LLN ! KK** +s 2 D../36{{4--.#5y//00rc|j|j|jr|j|jtj vs|j|jdk(rg|j|jrG|j|jtj vrJ|j|jdk(rg|jr|j|jdk(s|j|jdk(rL|j|jr,|j|jdk(r|j|jr|j|jtj vs|j|jdk(rg|j|jrG|j|jtj vrJ|j|jdk(rg|j tjS)Nrrrr) rrr{rre hexdigitsrr#rdrBs r hexadecimalzTokenIterator.hexadecimals  ! KK** +v/?/? ?{{4--.#5 LLN ! KK** +v/?/? ?{{4--.#5     KK** +s 2{{4--.#5 LLN  "t{{43E3E'F#'M ! KK** +v/?/? ?{{4--.#5 LLN ! KK** +v/?/? ?{{4--.#5y//00rctdD]9}|j|j|z}|dk(s|dk(s*|j|zcStd)Nr&"'zQuote not found somehow)ranger{rAssertionError)rCoffsetchars rfind_opening_quotez TokenIterator.find_opening_quotesZAh 3F;;t11F:;Ds{dck))F22 3 677rcL|j}|j|j|}|j|}|dzt|jkr<|j|dz|k(r'|j|dz|k(r|j||dzn|j||dz}||fS)Nr&r$r%)rr{rrz)rC quote_indexr quote_charrs rstring_prefix_and_quotesz&TokenIterator.string_prefix_and_quotess--/ T//+>[[- a3t{{#33KK a0J>KK a0J> KK kAo 6 [;?; u}rc |jjtjtjfvr|j \}}|j ||tt|D]}|jtt|D]}|j|jj|jtjS|jjtjk(r|jJt|jdk(}|j }|j#rM|j$|j }|dk(r|rt&|dk(r|j|j(Jd|j(j+vrk|j dzt|j$krF|j-dk(r3|j/dk(r |j|j|j#r6|j-dk(s#|j-dk(s|j1)|dk(r|j/dk(r"|j|jc|jj3|j |k(r|j5S|jtj6S|jJ|j9|jrX|jj;|j |k(r|j5S|jtj6S|j1|j#rMt<|jjtj>k(ru|j|j@jC|jDd|_"|jjG|jtjHS|jjtjJk(r|jJtt|jD]}|j|jtjL}|jO|jjQ|S|jjtjRk(rN|j }|j#r+|j$|j }|jJ|dk(s|dk(rt|jdk(r}|dk(r|jj3ntj|j_|j |k(r|j5S|jtj6S|dk(r>tj|j_|jtj6S|j1|j#r+t<tUd ) Nr$r\rN{}rzUnhandled f-string state)+rrrrrrrrrzrrrr#rfrrrrr{rrrrrrrfstringrgrrrrrrrrr`rrhrrrr)rCrrris_single_quoterrrrs rr zTokenIterator.fstring!s    # #  $ $  ( ((  !99;MFE  * *65 93v;'   3u:&       , , .??9#:#:; ;    # #|'E'E E%%1 11!$"4"45:O,,K##%{{4#5#564> ..2S5EE IIK3. NN,3  ((* s*diikS.@2243;~~'3.   **LLN--<#'<<>1#y/G/GHH))555::d001&&EEG))[8#||~-??9+C+CDD**,i##%l     # #|'E'E E LLN  $ $ + +D,>,> ?!"D     - - /??9#3#34 4    # #|'B'B B%%1 113t1123   OOI$9$9:E  " " $    , , .L    # #|'L'L L,,K##%{{4#5#56))555DLDCKS9K9K5LPQ5Qs{**LLN4@3O3O**0))[8#||~-??9+C+CDDS[/;/K/KD&&,??9+C+CDD**,'##%* 788rcV|j\}}|r<|jr0|j|jtj S|D]}|dvs|j cStt|D]}|jtt|D]}|jt|dk(}|jr|j|j}|dk(r|rt|dk(r!|j|j\|j|rHtt|D]}|j|jtjS|j|jrt )N)r)fFrTr$rr)rrrrr#rcr rrzrr{rrrrrer)rCrrrrr s rrezTokenIterator.stringsj557  d(( LLN??ILL?9 9 &D++||~% &s6{# A LLN s5z" A LLN e*/!;;t112Dt|((t| **,zz% s5z*#ALLN#y'7'788  & & (%!(rc|j}d}d}|jrZ|j|j}|jr|j d}|dk(s|dk(rd}nn|jrZ|js4|j|k(rt |j tjS|r(|s&d|_ |j tjS|j}|dk(s|dk(s|jr|j tjS|j||j}t|jdk(rdn|jd }t|t|k(r>t|dk(rt ||k7rt|j tjSt|t|kDrRt|dkDr ||vrt|jj||j tj St|jdkDr|jd }t|t|krt"t|t|k(rnH|jj%} |xj&d z c_t|jdkDr|j tjS) NFT  #rrryrr$)rrr{ is_whitespacerrrr#rUrr is_newlinerzrrrrVrrr) rCrrsaw_whitespacesaw_tab_or_spacer next_char new_indentcurrent_indent top_indentrs rrVzTokenIterator.indents~((  !;;t112D!!# !%3;$$,'+$!  "!![0!!??9#7#78 8 "2)-D &??9#7#78 8IIK  yD0DOO4E??9#7#78 8[[t/A/AB "4#4#45:@Q@QRT@U z?c.1 1:!#!!^+44??9#7#78 8 _s>2 2>"Q&>+K44    $ $Z 0??9#3#34 4d''(1,!..r2 z?S_4::z?c*o5%%))+##q(#d''(1,??9#7#78 8rc|jry|j|j}|dk(xs|dk(xs|dk(xs |dk(xs|dk(S)NFrrr  )rr{r)rCrs rrzTokenIterator.is_whitespacesc ?? {{4--. CK t| t| v~ v~  rc|j|jdk(ry|j|jdk(rE|jdzt|jkr |j|jdzdk(ryy)NrTrr$F)r{rrzrBs rrzTokenIterator.is_newline sq ;;t)) *d 2 KK** +t 3""Q&T[[)99 D..23t;rc|jr/|j|jtjS|j |j d}t|D]3\}}t|dkstj|r+|dk7s1|}n t|}|j||jtjS)Nr) rrrr#rFr{r enumerateordrRisalnumrzr)rC remainingindexrlengths rrAzTokenIterator.names    LLN??9#7#78 8KK 2 2 45 $Y/ $KE44y3s{{4'8TS[ $ ^F y3344rc|SrMrrBs r__iter__zTokenIterator.__iter__(s rc|j-|jjtjk(rt|j t |jk(r|j|jS|jjtjtjtjhvr|jS|jS|j t |jkDr|jS|jjtjk7r7|jjtjk7r|j!S|j|j }|j"s|dk(r|j%|j's|jS|j|j }|dk7rAd|_|j.|jjtj*k(rd|_|dk(r|j(r/|j%|j/tj*S|j'rj|j1sZ|j"s|j|j dk(rn1|j%|j'r|j1sZ|j/tj*S|j2dkDr4|xj2dzc_|j/tjS|j1r|jS|dk(r|j%|j'st4d}d}|j'r|j7r|j%d}n|s|j1rw|j|j }|dk(r|j%|j%d}d}|j8r|j;n*|j;d|_nn|j'r|st<|j/tj>S|j@dk(s |jBrR|jDdk(rC|jjtjk(rd|_! |jG}||S|j7rp|j'rA|j7r1|j%|j'r|j7r1|j/tj>S|d vrR|j%|jKd k(r|j%|j/tjLS|d k(r|j%|jKd k(r/|j%|j/tjLS|jKd k(r|j%|jKd k(r|j%|j/tjLS|d k(ru|j%|jKd k(r|j%|jKd k(r|j%|j/tjLS|d k(ru|j%|jKd k(r|j%|jKd k(r|j%|j/tjLS|dk(ru|j%|jKdk(r|j%|jKd k(r|j%|j/tjLS|dk(r|j%|jKd k(r/|j%|j/tjLS|jKd k(r|j%|j/tjLS|dvr/|j%|j/tjLS|dk(r/|j%|j/tjLS|dk(rD|j%|xjDdz c_"|j/tjNS|dk(rZ|j%|xjDdzc_"|jDdkrd|_"|j/tjPS|dk(rD|j%|xjDdz c_"|j/tjRS|dk(rZ|j%|xjDdzc_"|jDdkrd|_"|j/tjTS|dk(rD|j%|xjDdz c_"|j/tjVS|dk(r|j%|jDdk(ra|jjtjk(r:|jjY|jZj]|_"n+|xjDdzc_"|jDdkrd|_"|j/tj^S|dk(r|j%|jDdk(re|jjtjk(r>tj`|j_ |j/tjLS|jKd k(r|j%|j/tjLS|dvr'|j dzt |jkr8|j|j |j dzdvr|jcS|j dzt |jkr8|j|j |j dzdvr|jeS|j dzt |jkr8|j|j |j dzdvr|jgS|jiS|j dzt |jkr|jkdds|j dzt |jkr|jkd d!d"d#d$d%d&d'd(d)d* sC|j d+zt |jkr.|jkd,d-d.d/d0d1d2d3d4d5d6d7d* r|jmS|joS#tH$rd}Y LwxYw)8NrrTrrr$rF) r&|^@%=!~r/<>/*r),;`()[]rr:z .0123456789r%)0b0B)0o0O)0x0Xrrzb"zb'zr"zr'zf"zf'zu"zu'zt'zt"rr&zbr"zbr'zrb"zrb'zfr"zfr'zrf"zrf'ztr'ztr"zrt'zrt")8rrpr#rl StopIterationrrzr{rXrYrWrrrrrr rrrrrZrrrrrrrrrrUrrrrVrrrcr\r]r^r_r`rrrrarrrrrrrerA)rC current_charfound_whitespace seen_newliner indent_tokens r__next__zTokenIterator.__next__+s! ?? &4??+?+?9CVCV+V    T[[!1 1&~~''##!!   ( ~~''||~%   DKK 0 0>># #    $ $ (@(@ @""((L,H,HH<<> !{{4#5#56 ))ld.B LLN$$&||~%;;t'9'9:Lt#%)"OO/,, 0A0AA,0D) 3 !! y'8'899##%doo.?22 D$6$674?  ##%doo.???9#4#45 5    "   1 $ ??9#3#34 4 ?? <<> ! 4  LLN$$&## %  L##%%%'LLN'+$%4??+<;;t'9'9:Dt| LLN'+$#'L77((;@8/##%2$77??9#7#78 8   "d&@&@""a'""((L,D,DD).D & $#{{} '##    ##%$*<*<*> ##%$*<*<*>??9#7#78 8 H H LLNyy{c! ??9<<0 0 3  LLNyy{c! y||44yy{c! yy{c! ??9<<0 0 3  LLNyy{c! yy{c! ??9<<0 0 3  LLNyy{c! yy{c! ??9<<0 0 3  LLNyy{c! yy{c! ??9<<0 0 3  LLNyy{c! y||44yy{c! ??9<<0 0 : % LLN??9<<0 0 3  LLN??9<<0 0 3  LLN   ! # ??9#3#34 4 3  LLN   ! # !!A%%&"??9#3#34 4 3  LLN   ! # ??9#5#56 6 3  LLN   ! # !!A%%&"??9#5#56 6 3  LLN   ! # ??9#3#34 4 3  LLN""a'&&,, 0L0LL""113%)%=%=%A%A%C"""a'"%%))*D&??9#3#34 4 3  LLN""a'&&,, 0L0LL+7+P+P""(y||4499;#%LLNy||44 = (!!A%T[[)99dkk""T%7%7!%;??{{}$##a'3t{{+;; ""T%7%7!%;AAzz|###a'3t{{+;; ""T%7%7!%;AA''))||~%  ! #s4;;'7 7DJJsCr^sZ"( $$I#+}*<%5;>'7=-,(M'< ;'.&%$0> 0>f  9 9  9F2;2;j ll ld "& 0  0 0 0  0 r