K i`tddlZ ddlZddlZddlmZejjZ dddZ ddl m Z dIdZGddeZej"d d Zej"d d Zej"d dJd Zej"d dJd Zej"d dZej"d dZej"d dZej"d dZej"d dZej"d dZej"d dZej"d dZej"d dZej"d dZej"d dZ ddddZ!dZ"dZ#dZ$dZ%d Z&d!Z'd"Z(d#Z)d$Z*d%Z+d&Z,d'Z-d(Z.d)Z/d*Z0d+Z1d,Z2d-Z3d.Z4d/Z5d0Z6d1Z7d2Z8d3Z9d4Z:d5Z;d6Zd9Z?ejjd:Z@ejjd;ZAd<ZBd=ZCejjd>ZDejjd?ZEejjd@ZFejjdAZGdBZHdCZIejjdDZJejjdEZKejjdFZMejjdGZNejjdHZOy#e$rdZY`wxYw#e$rdZ YBwxYw)KN)computesummary descriptioncDddlm}|tj|S)Nr)_get_udf_context)pyarrow._computer padefault_memory_pool) batch_lengthr s \/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/pyarrow/tests/test_udf.pymock_udf_contextr)s1 B224l CCc eZdZy)MyErrorN)__name__ __module__ __qualname__rrrr.srrsession)scopecd}d}t}tj|||dtjitj||fS)z4 Register a unary aggregate function (mean) cRtjtj|SN)r scalarnpnansum)ctxxargss rfuncz"sum_agg_func_fixture..func7syy1&&rsum_udfr ) empty_udf_docpcregister_aggregate_functionr float64r" func_namefunc_docs rsum_agg_func_fixturer+2sP 'IH""4#,#+'*BJJL$%$&::< % ?rcd}d}t}tj|||dtjitj||fS)Nctd)NOops) RuntimeErrorr rlenrr s rr"z(exception_agg_func_fixture..funcJs 6""ry=exception_len(x)r r$r%r&r int64r(s rexception_agg_func_fixturer5HsP!%IH""4#,#+'*BHHJ$%$&88: % ?rcd}d}t}tj|||dtjitj||fS)Ncdtjt|tjSr)r rr0int32r1s rr"z1wrong_output_dtype_agg_func_fixture..func^syyQ,,ry=wrong_output_dtype(x)r r3rr"r)r*s r#wrong_output_dtype_agg_func_fixturer;\sP-*IH""4#,#+'*BHHJ$%$&88: % ?rcd}d}t}tj|||dtjitj||fS)Nct|Sr)r0r1s rr"z0wrong_output_type_agg_func_fixture..funcqs 1v ry=wrong_output_type(x)r r3r:s r"wrong_output_type_agg_func_fixturer?osP)IH""4#,#+'*BHHJ$%$&88: % ?rcd}d}ddd}tj|||tjtjdtj||fS)z, Register a binary scalar function. cJtjd||g|jS)Nmultiply memory_poolr% call_functionrD)rmr s rbinary_functionz,binary_func_fixture..binary_functions$ QF,/OO= =rzy=mxzfind y from y = mxr)rGr r%register_scalar_functionr r4)rHr) binary_docs rbinary_func_fixturerLs` =I#!57J ) *&(hhj&(hhj!#!# , I %%rc d}ddd}d}tj|||tjtjtjdtj||fS)z- Register a ternary scalar function. ctjd||g|j}tjd||g|jS)NrBrCaddrE)rrGr cmxs rternary_functionz.ternary_func_fixture..ternary_functionsC   j1a&*-//;Aw,/OO= =rzy=mx+czfind y from y = mx + cr)array1array2array3rI)rR ternary_docr)s rternary_func_fixturerWsj = '":.varargs_functions? @C""53*/2@C @ rz z=ax+by+czfind z from z = ax + by + cr)rSrTrI)r^r) varargs_docs rvarargs_func_fixturer`sc  I)"?K 0 ) +.0hhj.0hhj!"!# , Y &&rctd}ddd}d}tj|||itj||fS)z- Register a nullary scalar function. ctjdg|jztj|jSN*typerDr arrayr r4rDcontexts r nullary_funcz*nullary_func_fixture..nullary_func3xxw333"((*$+$7$79 9rrandom functiongenerates a random valuertest_nullary_funcrIrkr*r)s rnullary_func_fixturerqsL 9 %1H$I ) ( " " ,  ""rcpd}ddd}d}tj|||itj|S)z Register a nullary scalar function with an ephemeral Python function. This stresses that the Python function object is properly kept alive by the registered function. ctjdg|jztj|jSrcrgris rrkz4ephemeral_nullary_func_fixture..nullary_funcrlrrmrnrtest_ephemeral_nullary_funcrIrps rephemeral_nullary_func_fixturerusG9 %1H.I ) ( " " , rc|d}d}i}tj}ddd}tj|||||||fS)zi Register a scalar function which returns something that is neither a Arrow scalar or array. cy)Nrdrrs rwrong_output_typez9wrong_output_type_func_fixture..wrong_output_typesrtest_wrong_output_typezreturn wrong output typerrr r4r%rJ)ryr)in_typesout_typedocs rwrong_output_type_func_fixturersR )IHxxzH- C 19c ((4 i ''rcd}d}dtji}tj}ddd}tj|||||||fS)zq Register a scalar function whose actual output DataType doesn't match the declared output DataType. c2tjd|dgSNrOr%rF)rrhs rwrong_output_datatypezAwrong_output_datatype_func_fixture..wrong_output_datatypesqz22rtest_wrong_output_datatyperhzreturn wrong output datatyperr)r r4int16r%rJ)rr)r|r}r~s r"wrong_output_datatype_func_fixturers[ 3,I$HxxzH1 C 5y# ((4 ) ++rc|d}d}i}tj}ddd}tj|||||||fS)z> Register a scalar function with the wrong signature. cTtjdtjS)Nrrf)r rr4rrrwrong_signaturez5wrong_signature_func_fixture..wrong_signature*syy,,rtest_wrong_signaturezUDF with wrong signaturerrr{)rr)r|r}r~s rwrong_signature_func_fixturer$sQ -'IHxxzH- CC ((4 I %%rctd}d}ddd}tj|||itj||fS)zE Register a scalar function which raises a custom exception. ctd)Nerror raised by scalar UDF)rrxs r raising_funcz*raising_func_fixture..raising_func>s233r test_raisezraising functionrrrI)rr)r~s rraising_func_fixturer9sE 4I% C i "BHHJ0  ""rcd}d}t}tj|||dtjitj||fS)z$ Register a vector function ctj|jjj dS)NT)pct)r rh to_pandascopyrankr1s rpct_rankz+unary_vector_func_fixture..pct_rankOs.xx **,11d1;<.pivot`se ^^ ' 'Aq  ' I S S U88C38?KKM~~))(3CCEEr y=pivot(x)rrv1v2)r$r%rr r4r'utf8struct)rr)r~s rstruct_vector_func_fixturer[s F I C y#hhjrzz|"'')< C$tRZZ\&:T2::<.test_reg_functionsxx~rzfunc must be a callablematchtest_none_function)DataType expected, got test_output_functionz)in_types must be a dictionary of DataTypetest_input_functionrz?Already have a function registered with name: test_reg_function)r r4pytestraises TypeErrorr%rJKeyError)r~r|r}r expected_exprs rtest_registration_errorsrs$1 C"((*%HxxzH y !. ##$5$(#x$, .. y(A B. ##D*>X$, .. @M y 6* ##$5$:C$( ** @M y 6. ##$5$93$, .. 1 3S" (*M x} 5. ##$5$7b$, ...A.. .. **....s< F FFF'F3F FF$'F03F<c|\}}d}tjt|5tj|dgdddy#1swYyxYw)Nz9VarArgs function 'z=ax\+by\+c' needs at least 2 argumentsrrd)rr ValueErrorr%rF)r`_r) error_msgs r test_varargs_function_validationrsE'LAyLI z 3* RD)***s AA cdtji}tj}ddi}d}tjtd5t j |d|||ddddd i}tjtd 5t j |d |||dddy#1swYKxYw#1swYyxYw) Nrrdescc2tjd|dgSrr)rrs r add_constz/test_function_doc_validation..add_consts{33rz#Function doc must contain a summaryrtest_no_summaryrz test summaryz'Function doc must contain a description test_no_desc)r r4rrrr%rJ)r|r}r*rs rtest_function_doc_validationrs"((*%HxxzH vH4 zB D. ##I/@$,h$, .. >H zF H. ##I~$,h$, .......s B1B=1B:=Cc"t|gddy)NFrr)r)rqs rtest_nullary_functionrs.5'(*rcd|}tj|gd}|jdgk(sJy)Nrrrd)r%rF to_pylist)rurrs rtest_ephemeral_functionrs4 )D   dBq 1F    " %% %rc|\}}tjtd5tj|gddddy#1swYyxYw)NzUnexpected output type: intrrrrrrr%rF)rrr)s rrzrzsC1LAy y: <2 Bq1222 AA c|\}}d}tjt|5tj|t j ddggdddy#1swYyxYw)NzDExpected output datatype int16, but function returned datatype int64rrr)rrrr%rFr rh)rrr)rs rrr&sW5LAy.identity; rtest_wrong_datatype_declarationrhztest output valuez test outputrz%DataType expected, got rr r4rrrr%rJrr)r|r}r~s rrr:sr2I$HH&$ C yD F= ##Hi$'8 ==== A!!A*cd}d}ddi}tj}ddd}tjtd5t j |||||dddy#1swYyxYw) Nc|Srrrs rrz3test_wrong_input_type_declaration..identityLrr!test_wrong_input_type_declarationrhztest invalid input typezinvalid input functionrrrrrs rrrKsq4IHxxzH,/ C yH J8 ##Hi$,h 8888rctjtj}|\}}tj|tj dgdztj g|}|tj dgdztj k(sJ|jdk(sJd}|jdk(sJy)NrrrCr@r)r proxy_memory_poolr r%rFrhr4bytes_allocated)r proxy_poolrr)ress rtest_scalar_udf_contextr\s%%b&<&<&>?J%LAy  9HHaS4ZbhhjAB'1 3C "((A3:BHHJ7 77 7  % % '8 33 3 C  % % '1 ,, ,rc|\}}tjtd5tj|gddddy#1swYyxYw)Nrrrr)rrrr%rF)rrr)s rtest_raising_funcr!ksA'LAy w&B C2 Bq1222r c|\}}tj|tjdg}|tjdk(sJy)Nr )r%rFr r)rrr)rs rtest_scalar_inputr$qs>,Hi  9ryy}o 6C "))B-  rc|\}}tjtj}|jdk(sJtjdgdztj |}|jdk(sJt j||g|jdk(sJd}|jdk(sJy)Nrrrrer)r rr rrhr4r%rF)rrr)rrs rtest_input_lifetimer&ws,Hi%%b&<&<&>?J  % % '1 ,, , !t"((**EA  % % '8 33 3Y$  % % '8 33 3 A  % % '1 ,, ,rc t|Dcgc]2\}}tjt|||j4}}}tj j ||Scc}}wNr)arraysschema) enumerater rhlistrfrr)r*itersirr)s r_record_batch_from_itersr/s^#E*,!QhhtAwVAY^^4,F, >> % %VF % CC,s7A*c Vt|t||dzt|dz|dzS)Nrrr#)r/range)r*ns r_record_batch_for_ranger3s1 #F$)!QV$4$)!a%R$8 ::rcfd}|S)Nc,Gfdd}|S)Nc eZdZdZfdZy),make_udt_func..udf_func..UDTcd|_yr)callerselfs r__init__z5make_udt_func..udf_func..UDT.__init__s " rcz |j|jdc|_}|j|}|jS#t$rgDcgc]#}tjg|j %ncc}w}}tj j|}Y|jSwxYwr() r9send StopIterationr rhrfrrr)r;rbatchrr) batch_genr*s r__call__z5make_udt_func..udf_func..UDT.__call__s6{{*+4S>+>+>( S KK,E ,,.. %6+13"'!hhr ;33F3NN66%f76E,,.. 6s7A B:(B?)B:9B:Nrrrr<rBrAr*srUDTr7s  # /rrEr)rrErAr*s rudf_funczmake_udt_func..udf_funcs / /u rr)r*rArFs`` r make_udt_funcrGs" Orc<tGfddfdS)zA short datasetceZdZdZfdZy)%datasource1_direct..Generatorcd|_y)Nr)r2r:s rr<z.datasource1_direct..Generator.__init__s DFrc|jdk(rtgg}|jS|xjdzc_t|j}|jS)Nrr)r2r/r3r)r;rr@r*s rrBz.datasource1_direct..Generator.__call__s[vv{0R@((* *! /?((* *rNrC)r*sr GeneratorrJs   +rrMcSrr)rrMs rz$datasource1_direct..s y{r)datasource1_schema)rMr*s@@rdatasource1_directrQs  !F + + #"rc:tfd}t|S)Nc3TKtdddD]}t|dz yw)Nrrr)r1r3rr2r*s rrAz(datasource1_generator..batch_gens/q!R 9A)&!a%8 8 9s%(rPrGrDs @rdatasource1_generatorrWs  !F9  ++rc:tfd}t|S)Nc3hKtdddD]}t|dz tdw)NrrrTrdatasource1_exception)r1r3r/rUs rrAz(datasource1_exception..batch_gens=q!R 9A)&!a%8 8 9233s/2rVrDs @rrZrZs  !F4  ++rctjdtjfdtjfgS)Nr)r r*r8rrrrPrPs, 99r288:&RXXZ(89 ::rc|ddd}i}tjdtjfdtjfg}|||||fS)Nz UDTztest {func_name} UDTrr)r rr8)r"r)r*r|r}s rdatasource1_argsr]sU'[-57HHyy2rxxz*R,<=>H Hh 88rct}|}|j}t||}tj|d}tj |D]}|dz}|t ||k(rJy)Nrr)rPrr]r%register_tabular_functioncall_tabular_functionr3) func_makerr*r"r)rr2items r_test_datasource1_udtrcst  !F ctjgdtj}tjd|g}tj d}||k(sJy)Ng$@g4@>@gD@gI@mean_udfrl)r rhr'r%rFr)unary_agg_func_fixturearrrexpecteds rtest_scalar_agg_basicrqsH ((12::< @C   j3% 0FyyH X  rctjgtj}tjtj d5t jd|gdddy#1swYyxYw)Nz empty inputsrrm)r rhr'rr ArrowInvalidr%rF)rnemptys rtest_scalar_agg_emptyrusQ HHR &E rn =. eW-...s A//A8ctjgdtj}tjtj d5t jd|gdddy#1swYyxYw)Nrrr(2output datatyperr9r rhr4rrArrowTypeErrorr%rF)r;ros r"test_scalar_agg_wrong_output_dtyper}sV ((' 4C r((0A B; 2SE:;;; A11A:ctjgdtj}tjtj d5t jd|gdddy#1swYyxYw)Nrw output typerr>r{)r?ros r!test_scalar_agg_wrong_output_typer sU ((' 4C r(( >: 1C59:::r~ctjgdtj}tjgdtj}t j d||g}tj d}||k(sJy)Nrw)?@@@g@sum_meang@@)r rhr4r'r%rFr)varargs_agg_func_fixturearr1arr2rrps rtest_scalar_agg_varargsrsf 88("((* 5D 88-rzz| >$ 8#3#3D#9 99 9rctjgtj}tjgtj}tj||gddg}|j dj dg}tjtjgtjtjgtjgddg}||k(sJy)Nrrrrr)r rhr'r8rrr)rnrrrrrps rtest_hash_agg_emptyr6s 88B %D 88B #D HHdD\$ 9E ^^D ! + +-B,C DFxx"bhhj1288 BJJL4#')9":r)r?rrrs rtest_hash_agg_wrong_output_typerLs 88("((* 5D 88ORXXZ 0D HHdD\$ 9E r(( >N t&&(K'LMNNNrctjgdtj}tjgdtj}tj||gddg}t j td5|jdjdgdddy#1swYyxYw) Nrwrrrrr.r)rr2) r rhr4r8rrrr/rr)r5rrrs rtest_hash_agg_exceptionrUs 88("((* 5D 88ORXXZ 0D HHdD\$ 9E |6 2J t&&(G'HIJJJs "B66B?c6d}d}tjtjd|tj}tjtj j ||tj}tj||gddg}|jdjdg}|jdjdgjdd g}|jd|jdk(sJy ) z2Test hash aggregate udf with randomly sampled datai@Brrrrr)rr#)rsum value_sum_udfN) r rhrrepeatr'randomchoicer8rrrrr)r+ value_num group_numrrrrrps rtest_hash_agg_randomr_sII 88BIIa+RZZ\ :D 88BII$$Y :BHHJ GD HHdD\$ 9E ^^D ! + +-A,B CF~~d#-- *ND/+BC  >>$ 8#3#3D#9 99 9rctjgdtj}tjd|g}|dd|}||k(sJy)Nrkrrr rhr'r%rFrrorrps rtest_vector_basicrrsO ((12::< @C   ou 5F+(+D#6H X  rctjdgtj}tjd|g}|dd|}||k(sJy)Nrrrrrs rtest_vector_emptyrzsP ((A3 %C   ou 5F+(+D#6H X  rc6tjgdtj}tjgdtj}tjgd}t j d|||g}|dd|||}||k(sJy)N)rrrr)rrrr)rrrrrr)r rhr4r'r%rF)rrrrPrrps rtest_vector_structrs bhhj A bjjl A   A  lQ1I 6F,)!,T1a;H X  r)r)r)Prnumpyr ImportErrorpyarrowr rr%markr pytestmarkr$pyarrow.datasetrrr/rfixturer+r5r;r?rLrWr`rqrurrrrrrrrrrrrrrrrrzrrrrrr!r$r&r/r3rGrQrWrZrPr]rcrergrirqrur}rrrrrrrrrpandasrrrrrrrsW&![[ r2  D  l i !*i !&i !$i !$i &!&(i '!'0i '!'2i #!#,i !0i (!((i ,!,(i &!&(i  #! # i  !  i !,*.'+C@ /.d*.>*& 2:2="8" -2  -D : *#$,,;9 :.15 ..; : 66::$  ONJJ::$  Y B Bs"M M. M+*M+.M98M9