MpjaddlmZmZddlZddlmZmZmZmZm Z m Z m Z m Z m Z mZmZmZddlmZddlZ ddlZer ddlmZddlmZddlmZmZmZmZmZm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&dd l'm(Z(dd l)m)Z) e*eZ+GddeeZ-Gdde-Z.Gdde-Z/GddeZ0eddZ1Gdde2Z3GddZ4GddZ5GddZ6d Z7d!Z8Gd"d#Z9d$e2fd%Z:Gd&d'Z;Gd(d)Zd4d,ee0e2ffd-Z?Gd.d/e>Z@Gd0d1e@ZAGd2d3e>ZBy#e$rYwxYw#e,$rd Z+YwxYw)5)abstractmethodABCN) TypeVarTypeDictIterator CollectionCallableOptional FrozenSetAnyClassVar TYPE_CHECKINGoverload) ModuleType) LexerConf) ParserState)classifyget_regexp_width Serializelogger TextSlice TextOrSlice)UnexpectedCharactersLexErrorUnexpectedToken)TOKEN_DEFAULT_PRIORITY)suppress)copyFc eZdZUdZeed<eeed<eeed<eeed<ddedeedeeddfdZ d Z d Z d Z e defd Zee defd Zee defdZdZy)Patternz(An abstraction over regular expressions.valueflagsrawtypeNreturnc@||_t||_||_yN)r# frozensetr$r%)selfr#r$r%s X/mnt/ssd/data/Dropbox/adrian/scripts/msg_venv/lib/python3.12/site-packages/lark/lexer.py__init__zPattern.__init__)s u% c4t|jSr))repr to_regexpr+s r,__repr__zPattern.__repr__.sDNN$%%r.cXtt||j|jfSr))hashr&r#r$r2s r,__hash__zPattern.__hash__2s T$ZTZZ899r.ct|t|k(xr4|j|jk(xr|j|jk(Sr))r&r#r$r+others r,__eq__zPattern.__eq__5s<DzT%[(dTZZ5;;-Fd4::Y^YdYdKddr.ctr)NotImplementedErrorr2s r,r1zPattern.to_regexp8s !##r.ctr)r<r2s r, min_widthzPattern.min_width< "##r.ctr)r<r2s r, max_widthzPattern.max_widthAr@r.c:|jD] }d|d|d} |S)Nz(?:))r$)r+r#fs r, _get_flagszPattern._get_flagsFs$ .A#$e,E . r.)N)__name__ __module__ __qualname____doc__str__annotations__r r rr-r3r6r:rr1propertyintr?rBrGrHr.r,r"r"!s. J c? # 3-c*S/Xc]^b &:e$3$$$3$$$3$$r.r"c^eZdZUdZdZeeed<defdZe de fdZ e de fdZ y) PatternStr)r#r$r%rMr&r'c^|jtj|jSr))rGreescaper#r2s r,r1zPatternStr.to_regexpQsryy455r.c,t|jSr)lenr#r2s r,r?zPatternStr.min_widthT4::r.c,t|jSr)rWr2s r,rBzPatternStr.max_widthXrYr.N) rIrJrK__serialize_fields__r&rrMrNr1rOrPr?rBrHr.r,rRrRLsX2D(3-63633r.rRcheZdZUdZdZeeed<defdZdZ dZ e de fdZ e de fd Zy) PatternRE)r#r$r%_widthrTr&r'c8|j|jSr))rGr#r2s r,r1zPatternRE.to_regexpbstzz**r.Ncn|jt|j|_|jSr))r^rr1r2s r, _get_widthzPatternRE._get_widthfs) ;; *4>>+;A utility class for keeping track of line & column information)char_posr{r|line_start_pos newline_charcJ||_d|_d|_d|_d|_y)Nrr)rrr{r|r)r+rs r,r-zLineCounter.__init__s'(   r.ct|tstS|j|jk(xr|j|jk(Sr))rmrNotImplementedrrr8s r,r:zLineCounter.__eq__s;%-! !}}.Z43D3DHZHZ3ZZr.tokencJ|rb|j|j}|rE|xj|z c_|j|j |jzdz|_|xjt |z c_|j|j z dz|_y)zConsume a token and calculate the new line & column. As an optional optimization, set test_newline=False if token doesn't contain a newline. rN)countrr{rrindexrrXr|)r+r test_newlinenewliness r,feedzLineCounter.feed$s {{4#4#45H X% &*mmell4CTCT6U&UXY&Y# U# mmd&9&99A= r.N)T) rIrJrKrLrr-r:rrrHr.r,rrs!DNI [ >+ >r.rc eZdZddZdefdZy)UnlessCallbackc||_yr))scanner)r+rs r,r-zUnlessCallback.__init__4s  r.tcb|jj|j}|||_|Sr))r fullmatchr#r&)r+rress r,__call__zUnlessCallback.__call__7s+ll$$QWW- ?AFr.N)rScanner)rIrJrKr-rwrrHr.r,rr3s%r.rceZdZdZdZy) CallChainc.||_||_||_yr)) callback1 callback2cond)r+rrrs r,r-zCallChain.__init__?s"" r.cl|j|}|j|r|j|S|Sr))rrr)r+rt2s r,rzCallChain.__call__Ds. ^^A $(IIbMt~~a 9r9r.N)rIrJrKr-rrHr.r,rr>s  :r.rcP|j|||}|r|jdSyrc)matchgroup)re_regexpsr$ms r, _get_matchrIs* &!U#Awwqz r.c t|d}t|dksJ|jt}i}|j t gD]}g}|j t gD]} | j|jk7r| jj} | t||jj| |k(s]|j| | jj|jjks|j| |stt!||||||j"<|D cgc] } | |vs|  } } | |fScc} w)Nc,t|jSr))r&rjrs r,z _create_unless..Os4 ?r.) use_bytes)rrXkeyssetgetr]rRrkrjr#rr1appendr$addrrri) terminals g_regex_flagsrrtokens_by_type embedded_strscallbackretokunlessstrtokrr new_terminalss r,_create_unlessrNsEi)BCN ~ ! #:^%8%8%:: #EMH##Ir2 l$((R8 .F%..0$$AJsEMM$;$;$=q-PP f%>>''5==+>+>>!%%f- . #1'&-QT`i2j#kHUZZ  l!*D1Qm-CQDMD ( ""Es EEc:eZdZdZdZdefdZdedeefdZ y)rc||_||_||_||_|jDchc]}|jc}|_|j |t||_ycc}wr)) rrrrri allowed_types _build_mresrX_mres)r+rrrrrs r,r-zScanner.__init__esW"*".2nn=aff=%%iY@ >sA'cFg}|rxdjd|d|D}|jr|jd} |jj ||j }|j|||d}|rx|S#t $r|j||dzcYSwxYw)N|c3rK|]/}d|jd|jjd1yw)z(?P<>rEN)rirjr1.0rs r, z&Scanner._build_mres..us'nTU9L9L9N Ons57zlatin-1r) joinrencodercompilerAssertionErrorrr)r+rmax_sizemresrjmres r,rzScanner._build_mresosiinYbcldlYmnnG~~!..3 Bhh&&w0B0BC KK !(),I " B'' 8q=AA Bs&A??B B textc|jD]J}|j|j||j}|s-|j d|j fcSyrc)rrrendr lastgroup)r+rposrrs r,rz Scanner.matchsI:: /C $))S$((3Awwqz1;;.. /r.r'ch|jD]#}|j|}|s|jcSyr))rrr)r+rrrs r,rzScanner.fullmatchs5:: #C d#A{{" #r.N) rIrJrKr-rrrrMr rrHr.r,rrds0A$/)/chsmr.rrcFd|vxsd|vxsd|vxsd|vxs d|vxrd|vS)zExpressions that may indicate newlines in a regexp: - newlines (\n) - escaped newline (\\n) - anything but ([^...]) - any-char (.) when the flag (?s) exists - spaces (\s)  z\nz\sz[^z(?s.rH)rs r,_regexp_has_newliners> 19 Z Zeqj ZDAI Z%1*BYQTXYQYZr.cheZdZUdZdZeed<eed<ee ed<d dedeedee fdZ dZ d Z y) LexerStatezRepresents the current state of the lexer as it scans the text (Lexer objects are only instantiated per grammar, not per text) )rline_ctr last_tokenrr r Nct|tr|ftt|jtrdnd}|j dkDr0|j t|jd|j |j |jcxkr|jkstdtd||_||_ ||_ y)N rrz%LineCounter.char_pos is out of bounds) rmrrrbytesstartrrr ValueErrorr r )r+rr r s r,r-zLexerState.__init__s dI && 499e0LuRVW::>MM)DIIq$**"EFJJ("3"3?txx? !HII@ !HII   $r.ct|tstS|j|jk(xr4|j|jk(xr|j |j k(Sr))rmrrrr r r8s r,r:zLexerState.__eq__sL%,! !yyEJJ&r4==ENN+JrtbgbrbrOrrr.cvt||jt|j|jSr))r&rr r r r2s r,__copy__zLexerState.__copy__s(tDz$))T$--%8$//JJr.r) rIrJrKrLrrrNrr rwr-r:rrHr.r,rrsU1I O%Y%(;2G%\dej\k%"s Kr.rcreZdZdZdddeefdZedddeddfdZ eddd e ddfd Z d Z d Z eZy ) LexerThreadzTA thread that ties a lexer instance and a lexer state, to be used by the parser lexerLexer lexer_statec ||_||_yr))rstate)r+rrs r,r-zLexerThread.__init__s   r. text_or_slicer'cPtj|}||t|Sr))r cast_fromr)rrrrs r, from_textzLexerThread.from_texts$""=15*T*++r.rc&||t|Sr))r)rrrs r,from_custom_inputzLexerThread.from_custom_inputs5*T*++r.c||j td|jj|j|S)Nz+Cannot lex: No text assigned to lexer state)rrrlex)r+ parser_states r,r!zLexerThread.lexs1 :: IJ Jzz~~djj,77r.c`t||jt|jSr))r&rr rr2s r,rzLexerThread.__copy__s"tDz$**d4::&677r.N)rIrJrKrLr rr-rrrr rr!rrw_TokenrHr.r,rrs!g!HZ4H!,g,k,m,,,g,S,],,8 8Fr.rc@eZdZdZedededeefdZ de fdZ y) rzlLexer interface Method Signatures: lex(self, lexer_state, parser_state) -> Iterator[Token] rr"r'ctSr))r)r+rr"s r,r!z Lexer.lexsr.rc>ttj|S) Deprecated)rrr)r+rs r,make_lexer_statezLexer.make_lexer_states)--d344r.N) rIrJrKrLrrr rrwr!rMr)rHr.r,rrs@ z%5S5r.rterminal_to_regexpc|stjj|}|rdnd}|j|k\ryt |dj D]}|j |dD]\}}|j|jk(sJ|j||d|jd|jd} |j|||j} |rt|d | tjd || |j|k\stjd yy#t$rd } YfwxYw) Nrg?c|jSr))rkrs r,rz)_check_regex_collisions..s  r.T) skip_markedzCollision between Terminals z and z. zONo example could be found fast enough. However, the collision does still existsrz5%s The lexer will choose between them arbitrarily. %sz2Found 8 regex collisions, will not check for more.) interegular Comparator from_regexescount_marked_pairsrvaluescheckrkmarkriget_example_overlapformat_multilinerrrwarning) r* comparator strict_modemax_collisions_to_showmax_timerabmessageexamples r,_check_regex_collisionsr@sO  ++889KL  qSH$$&*@@,.BCJJL$$U$= DAq::+ ++ OOAq !5QVVHE!&&LG l$88AxHYY['"WI677 NNSU\^e f,,.2HHST#  lk ls>!D88 EEcpeZdZUeeefed<ed d dZed de de de fdZ de de de e fd Zy) AbstractBasicLexerterminals_by_nameNr'cyr)rH)r+confr8s r,r-zAbstractBasicLexer.__init__rr. lex_stater"cyr)rH)r+rFr"s r, next_tokenzAbstractBasicLexer.next_tokenrr.rc#nKtt5 |j||#1swYyxYwwr))rEOFErrorrH)r+rr"s r,r!zAbstractBasicLexer.lexs6 h  ;ooe\:: ; ;s5)25r)rErr'N)rIrJrKrrMrhrNrr-rr rwrHrr!rHr.r,rBrB skC,--   J c U  ;;3;8E?;r.rBceZdZUeeed<eeed<eeed<eee fed<eee fed<e ed<ddd Z de fd Z ede fd Zd Zdd ededefdZy) BasicLexerr ignore_types newline_typesuser_callbacksrrTNr'ct|j}td|DsJ||j|_|j sIi}|D]}|j j} |jj||j|j jdk(r&td|jd|j d|j jdk(s|||<t|j |Dchc]}|jc}ks.(sA!:a-AszCannot compile token z: rz,Lexer does not allow zero-width terminals. (rErTz$Ignore terminals are not defined: %szUinteregular must be installed for strict mode. Use `pip install 'lark[interegular]'`.c3|K|]4}t|jjs'|j6ywr))rrjr1rirs r,rz&BasicLexer.__init__..Ds,&o!DWXYXaXaXkXkXmDnqvv&os(<<c|j |jj t|jj |j fSr))rkrjrBrXr#ri)xs r,rz%BasicLexer.__init__..Gs9qzzkAII4G4G3G#aiiooJ^I^`a`f`f%gr.)key)listrall re_modulerTskip_validationrjr1rrerrorrrir?r&rignorehas_interegularr@strictr*rOrNsort callbacksrPrrC_scanner)r+rEr8rr*rrs r,r-zBasicLexer.__init__&s( AyAAL9LA..##!#  3,,.XGGOOFD,>,>?99&&!+"\]\b\bdedmdm#noo99>>T),2&q) 3 $(CA(CCET[[IYmv\whi]^]c]c\wIwxyy'(:J Tvww'&oy&oo%dkk2gh""nn!//!%!7!7+/ 7ww}}X"AFFAII#VWWX)D\ws/&H- I/I4 -?I,ct|j|j|j|j\}|_t |j jsJ|jjD]M\}|j vr+t|j |fd|j <?||j <Ot||j|j|jS)Nc"|jk(Sr))r&)rrs r,rz+BasicLexer._build_scanner..WsTUTZTZ^cTcr.) rrrrTrrrXr2rPitemsrr)r+rrFrs @r,_build_scannerzBasicLexer._build_scannerPs#1$..$BTBTVZV]V]_c_m_m#n 4=4=='')***++113 )HE1 %'0u1EqJc'd e$'( e$  )y$"4"4dggt~~NNr.c\|j|j|_|jSr))rarer2s r,rzBasicLexer.scanner]s& ==  //1DM}}r.c:|jj||Sr))rr)r+rrs r,rzBasicLexer.matchcs||!!$,,r.rFr"c <|j}|j|jjkr|j |j|j}|s|j j |jz }|sdh}t|jj|j|j|j||jxr |jg||j|\}}||jv}d} |r||jvr-t|||j|j|j} |j|||j v| |j| _|j| _|j| _| j(|jvr|j| j(| } |s't+| tst-d| z| |_ | S|j|jjkrt/|)Nz )allowed token_historyrrCz+Callbacks must return a token (returned %r))r rrrrrrrNrr{r|r rCrrwrrOr}r~rr&rmrrJ) r+rFr"r rrir#rignoredrs r,rHzBasicLexer.next_tokenfs%%).."4"44**Y^^X->->?C,,44t7H7HH./G*9>>+>+>@Q@QS[S`S`bjbqbq3:)J^J^Jydmdxdxcy1=QUQgQgiiLE5t000GAet}}4%(9(98==(//Z MM%$*<*!i((3 >""N(0T O O - J c U r.rMc eZdZUeeefed<eed<eZeeed<ddddeee e fde e dd fd Z d e d d de efdZy )ContextualLexerlexers root_lexerrMrErstates always_acceptr'Nc t|j}|j}t|}||_trS|j sGt jj|Dcic]}||jjc}}nd}i} i|_ |jD]%\} } t| } | | } | |j| <'|j|usJd|_|j#|||_ycc}w#t$rut| t|j zt|z} t|}| Dcgc] }||vs||ncc}wc}|_|j#||} | | | <YwxYw)NT)rWrrCr r]rZr.r/r0rjr1rordr*KeyErrorrr\rMrp)r+rErqrrrrC trad_confrr8lexer_by_tokensracceptsrVr lexer_confns r,r-zContextualLexer.__init__sp(  22J '  4#7#7$//<<`i=j[\aATATAV>V=jkJJDF $lln 'NE7G$C -',"'DKK  '""i///$( !//)Z@)>k -g,T[[)99C rs$  !6XXGG+ ;'O(i(V"$$.)0 T!ACAH>>B:: #,))V[3[#K#KL8 eWe^ $  5C 5[#5E0F@;;"h#hV3e3y  "Os#E3EEEEE