K i9dZddlZddlZddlZddlZ ddlZejdZ ejZedk(Zejj!e dezZdZdZdZdZd Zejj/d gd d Zd Zd3dZejj/d gd dZejj/d gd dZdZejj/d gd dZejj/d gd dZejj/d gd dZ ejj/d gd dZ!ejj/dddgejj/d gd dZ"ejj/d gd dZ#dZ$dZ%dZ&dZ'dZ(d Z)d!Z*d"Z+d#Z,d$Z-d%Z.d&Z/eejj/d gd d'Z0d(Z1d)Z2d*Z3d+Z4ejj/d d,d-gd.Z5d/Z6d0Z7d1Z8d2Z9y#e$rejj Z YmwxYw)4z UNTESTED: read_message Nz pyarrow.cudaz linux-x86_64z'CUDA IPC not supported in platform `%s`)reasonctjd|_tjtjjdz |_y)Nr)cudaContextglobal_contextget_num_devicesglobal_context1modules ]/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/pyarrow/tests/test_cuda.py setup_moduler1s5 LLOF!\\$,,*F*F*H1*LMFc|`yN)rr s r teardown_moduler6srctjjdkDsJtjdk(sJt jtjjdz k(sJtj }|jrJdt|vsJtj}||jk(sJ|jrJ|jdk(sJ|jtjjk(sJtj t"d5tjtjjdddy#1swYyxYw)Nrrz) ==A   ??b55:: :: : z: <5 T\\1134555s 2EE"size)rrc`tj|}tj|tj}d||dzd|zdz|j }tj|tj}tj j|||j|k(sJy)Ndtyper) rnew_host_buffernp frombufferuint8copytesting assert_equalr&)r&bufarrarr_cparr2s r test_manage_allocate_free_hostr8Ps   t $C --288 ,CCa$  XXZF ==BHH -DJJD&) 88t  rctj}tjd}tj|dzk(sJ~tj|k(sJy)N)rbytes_allocated new_buffer)r;cudabufs r test_context_allocate_delr>[sM$44O'',G  ) )_s-B BB B  ) )_ << 2G;>Hct}tjtj}d}tj ||j z}tj|}|j|j|j|}tj||dzk(sJ~tj||k(sJ|j|j|j|}~|jtj ||j z}|j|j|j}~tjtj d5|jdddy#1swYyxYw)NrSrz Cuda error r)rr.r*r0rr-rdsys getrefcountforeign_bufferaddressr&rXr r!r ArrowIOError)rir*r&rkrcfbufs r test_foreign_bufferrvs3 C HHRXX E D   u~~ 5 6D  B   dllDIIt d|d zf|d z|d zd zd zf|d ffD]<\}}tjtd5|j||ddd>tj|dz}|j|tjj |d|dztj |tj|dkry|j|dtjj |dd|dzztj |tj|j|dtjj |ddtj |tjdd|j|ddtjj |ddtj |tjddd|dzfd|dz fd|d zf|dz|dzd zffD]=\}}tjtd5|j|||ddd?y#1swYxYw#1swYHxYw#1swYdxYw)NrrDr)r+rGrFrrRrrr!position argument is out-of-ranger8requested more to copy than available from device buffer)r4 )r4rG)r4rH)r4rHrGrSz,requested copy does not fit into host buffer)r4rGrH)rMrXrr.r2r3r/r0r&r r!r"rrI)r&r5rPr4rGrHs r test_copy_to_hostrs"49IC    C :::JJCs"((!CD   T1W  -C :::JJCaM2==BHH+MN   T1WT1W  =C :::JJCaaa8MM#RXX>@   T1WQ  7C ::: 88q== a ha W@6]]:!DF @   x  ? @ @@ DF dAgQ{1}-ay@6]]:#BD @   x  ? @ @@  T1W %C#JJCqM2==BHH+MN by#+JJC2dAg:.MM#RXX>@ #b)JJCHbmmCrxx&H"&MN#b15JJC$KMM#RXX>sCERL2tAv, aL47D!GAI.I6]]:#9; I   #  H I I I= @ @ @ @6 I Is$=P&P3Q&P0 3P= Q dest_ctxsameanotherct|d\}|j}|dk(r?t}jj|jk(r"t j dn j}|j|fd}||k(sJ|dkDr[||dz|d|dz|d| dzzk(sJ||k(sJ|d|d z |dd|d|d zz|||d zz dz dzk(sJ|d zd fd |dzd fdfD]3\}}t jtd5||| ddd5d |dzffD]3\}}t jtd5||| ddd5|dkry|d z|dzd zdzffD]3\}}t jtd5||| ddd5y#1swYxYw#1swYxYw#1swYXxYw)Nrr&rErznot a multi-GPU systemcjg|i|j}tj|tjj SNr))copy_from_devicerXr.r/r0tolistargskwargsrbufr4rPs r putz"test_copy_from_device..putsIc3D3F3  "}}T299;;rr+rrrRrFrrrrrrr6requested more to copy than available in device buffer) rMrr contextrr skipr<r!r") rr&r5lstrrGrHr4rPs @@r test_copy_from_devicersZ"tH=HC **,C9" ;; $ $(>(> > KK0 1;;   t $D< 5C<< axD!G$HT1W c)D5!8n(DDDDu||AdAg. Gc(47m #cDqLN*;*<&= => >>!VRL(T!VQK26]]:!DF 2 & 1 2 22 DF 26]]:#BD 2 & 1 2 22 ax q46A+a- 26]]:#@B 2 & 1 2 22 2 2 2 2 2 2s$' G% G2 G G G G) c@t|d\}|j}tj|fd}||k(sJ|dkDr[||dz|d|dz|d| dzzk(sJ||k(sJ|d|dz|dd|d|dzz|||dzz dz dzk(sJ|dzd fd |dzd fd fD]3\}}t j t d 5|||ddd5d |dzffD]3\}}t j t d5|||ddd5|dkry|dz|dzdzdzffD]3\}}t j t d5|||ddd5y#1swYxYw#1swYxYw#1swYXxYw)Nr@rcjg|i|j}tj|tjj Sr)rNrXr.r/r0rrs r rz test_copy_from_host..putsIC1$1&1  "}}T299;;rr+rrrRrFrrrrrrz6requested more to copy than available from host bufferr)rMrrr<r r!r")r&r5rrrGrHr4rPs @@r test_copy_from_hostrs!tF;HC **,C  $ $T *D< 5C<< axD!G$HT1W c)D5!8n(DDDDu||AdAg. Gc(47m #cDqLN*;*<&= => >>!VRL(T!VQK26]]:!DF 2 & 1 2 22 DF 26]]:#@B 2 & 1 2 22 ax q46A+a- 26]]:#@B 2 & 1 2 22 2 2 2 2 2 2s$ E< F& F<F F F ctjd}|jtjj k(sJt |jtjsJt |jtjsJ|jsJ|jjsJ|jtjjk(sJ|jjsJtdd\}}|jtjjk(sJt |jtjsJ|jtjjk(sJt |jtjsJ|jrJ|jjrJ|jjrJy)NrSrr)rr-rrr CUDA_HOSTrrDevicer MemoryManagerrdefault_cpu_memory_managerrMrr)r4_s r test_buffer_devicer'sr   r "C ??b55?? ?? ? cjj")) ,, , c(("*:*: ;; ; ::: ::    ::668?? ?? ?    $ $$ $ R 9FAs ??b55:: :: : cjj")) ,, , ::66== == = c(("*:*: ;; ;zz>zz   !!((( ((rcdd fd }d\}}|||||||dzd\}}|jtjdtj|j dtjdtj|jtjdtj|j tj |jtj}tjj|ddtjdtjtjj|dd tjdtjtjj|d dtjdtjy) Nc^tj|}tj|}||fSr)rr<r BufferWriter)r&rjwriters r allocatez#test_BufferWriter..allocate>s,((.""4(V|rc6 |\}}t|d\}}|dkDr||_|j}|dk(sJ|j|j ||j|k(sJ|j d|j}|dk(sJ||kr.test_writesCs_ + f%:fES ?!,F ;;=1}} SYYiY01{{} ))) A;;=1}}# J,ABN LL(>J K  &H#  yyJ&&&   "  "   "yyJ&&&}}T2 T*r)ir'dr)2K)r) rr.aranger0writeatrr/rXr2r3)rr chunk_sizerjrr5rs @r test_BufferWriterr=s5 +:+J  J' J b(89C=LD& LL3bhh/0 NN2ryy28845 LL2RXX./ LLN --))+RXX 6CJJCHbii"((&CDJJC2J "BHH(EFJJCHbii"((&CDrcd}tj|}tj|}t |d\}}|j dk(sJd|_|j dk(sJ|j |jd|jdk(sJ|j |jdd|_|j dk(sJ|jdk(sJ|j |jdd |jdk(sJ|j |jd d |jdk(sJ|j |jd d |j |jdd |j|j}|j|k(sJtj|tj}tj j#||y)Nr'r@rrrrrSi,rUi6iiir))rr<rrrMrrr\rnum_bytes_bufferedrrXr&r.r/r0r2r3)r&rjrr5r4rr7s r test_BufferWriter_edge_casesrps D  $ $T *D   t $F!tF;HC    "" "F    $$ $ LL!$% ;;=A   LL"%&F    $$ $  $ $ )) ) LL"S12  $ $ )) ) LL#c23  $ $ )) ) LL#c23 LL#c23 LLN    D 99   ==RXX .DJJC&rc.d}t|d\}}tj|}|jd|j dk(sJ|j d}t |dk(sJ|j dk(sJ|jdtjdtj}|j|}|d k(sJ|j dk(sJtjj|dd|dd |jd |j d k(sJ|j}tj|jtj}tjj||y) Nr'rrirrir)rr)rMr BufferReaderrrreadlenr.zerosr0readintor2r3 read_bufferr/rX)r&r5rjreaderdatar7nrs r test_BufferReaderrsD D"X>IC   t $F KK ;;=C   ;;s D t9?? ;;=D   KK 88Crxx (DA 7N7 ;;=D  JJCItCRy1 KKN ;;=A      D ==**,BHH =DJJC&rctdd\}}tj|}|jd|j }t |dk(sJ|j dk(sJ|j}tj|jtj}tjj||y)Nrrrr))rMrrrrrrrr.r/rXr0r2r3)r5rjrrrr7s r test_BufferReader_zero_sizers"(;IC   t $F KKN ;;=D t9>> ;;=A      D ==**,BHH =DJJC&rctjtjdtjtjdtjg}tjt j jdd|t j}tjt j jdd|t j}tj||g|}|S)Nf0f1rrA)r&r*schema) rrfieldint16rhr.rJrK record_batch)rVra0a1batchs r make_recordbatchrs YYrxxz2rxxz245F "))##Asrxx#H IB "))##Asrxx#H IB OORHV 4E LrcXtd}|j}tj|t}tj ||j }t|tjsJ|j |j k(sJ|j|jk(sJ|j|jk(sJ|j}|j|sJtjj ||j }|j|jsJ|j|jk(sJ|j|jk(sJ|jdj|jdsJ|j|sJy)NrSr)r serializerserialize_record_batchrread_record_batchrrr RecordBatch num_columnsnum_rowsrXequalsipccolumn)rrkrjcbatchr4batch2s r test_batch_serializersZ R E ?? D  & &un =D # #D%,, 7F fbnn -- - <<6== (( (    2 2 22 2 >>V__ ,, ,    C ;;s   VV % %c5<< 8F ;;v'') ** *    2 2 22 2 >>V__ ,, , <<? ! !&--"2 33 3 <<  rc tjgdtj}tjddgdggdgtjtj}tjgdtj dtj fdtjfg}tjjgd tjd d g }tjjgd | }tjjgd| }||||||g}tjt|Dcgc]\}}d||jfc}}} tj|| } tjj| g} | Scc}}w)N)rr*NtyperrrR))abT)cdeF)NNNstrsbools)rrrNfoobar)indices dictionary)rRrrRN)rRrrNfr)rrhrlist_int32structutf8bool_DictionaryArray from_arraysr enumeraterrTable from_batches) rra2a3a4a5arraysr}r5rrtables r make_tablers| " 4B Aq6A3D)0D EB Dyy62779"5#*BHHJ"7"9: ;B   ' '88UEN+ ( -B   ' ' ( B   ' ' ( B"b"b" %F YY' As QC#((F OOF6 2E HH ! !5' *E L s(G c@t}tj}tjj ||j 5}|j |dddtj|jj}tjt|}|j|t|tjjtj!|j#}||||fS#1swYxYw)N)rH)rrBufferOutputStreamr new_streamr write_tablerggetvaluerxrr<rrN open_streamrrread_all)htablesinkoutrkrPdtables r make_table_cudar's \F "D   4 / 3  << 224 5D  $ $SY /DSY/ VV   1 1$ 7 8 A A CF v %%  s DDct\}}}}|j|jk(sJ|j|jk(sJ|j|jk(sJ|j |j sJ|j t jj|j jsJyr) r'rrrrrXrrr!r")rkr#rPr&s r test_table_deserializer)s"1!2D&$ ==FMM )) ) ??foo -- -   !3!3 33 3 ;;t((* ++ + ==++ hj  rct\}}}}tjj|j|j }|j |j k(sJ|j|jk(sJ|j|jk(sJ|j|jsJ|jtjj|jjsJyr)r'rrrcolumns column_namesrrrrrXrr!r")rkr#rPr&dtable2s r %test_create_table_with_device_buffersr.s"1!2D&$hh""6>>63F3FGG ==GNN ** * ??g.. .. .   !4!4 44 4 ;;t((* ++ + ==++ hj  rc tjjd}tjjj |}|j |}|j j|j}|j|jk(s&Jt|j|jftj||j}tjj||y)Nrr))rrr IpcMemHandle from_bufferopen_ipc_bufferrrrXr&rr.r/r*r2r3) handle_buffer expected_arr other_context ipc_handleipc_bufr4r5s r other_process_for_test_IPCr8)sGGOOA&M%%11-@J++J7G OO!    C 88|(( (M$,:K:K/L*MM ( --<#5#5 6CJJC.rc"ddl}|jd}t|d\}}|j}|j }|j t ||f}|j|j|jdk(sJy)Nrspawnrr)rEr) multiprocessing get_contextrMexport_for_ipcrProcessr8startjoinexitcode)r&r;rir5rjr6r3ps r test_IPCrC4s  % %g .C"X>IC$$&J((*M 5'-  /AGGIFFH ::??rctdd\}}|j}||jfD]}tjgd}|j |}|j djrJ|j djtjjk(sJ|j dj|jk(sJ|j tj}|j|sJtjd|i}|j |}|dj d} | jrJ| jtjjk(sJ| j|jk(sJ|j tj} | j|rJy)NrSrr)rrrRrcol)rMrrrrhcopy_tobuffersrrrrrrr) rr4mm_cudadestr5arr_cuda arr_roundtripr batch_cudabuf_cudabatch_roundtrips r test_copy_torOCs R 9FAs  G'..)-hhy!;;t$##%a(////!!$00B4K4K4P4PPPP!!$++w~~=== (()F)F)HI ##C((( -]]4( e$,,.q1??""##r'>'>'C'CCCC'..000$,,R-J-J-LM%%e,,,%-rcLtjd}|j}|jd}t |j d|}|jd}t |j d|}t jt j}t jdgddgg|}td d \}} | j} |j| } | j||jdk(sJ|jt j"k(sJ|jj$dk(sJ~ t j&j)||} | j*t jt jk(sJ| jt j,} | j/|sJ~ tj0t2d 5t j&j)||ddd|j| } | j||~ t j&j)||} | j*t jt jk(sJ| jt j,} | j/|sJ~ tj0t2d 5t j&j)||dddy#1swYxYw#1swYyxYw)N pyarrow.cffistruct ArrowSchema* uintptr_tstruct ArrowDeviceArray*rrRrrrSrr!Cannot import released ArrowArrayr"Cannot import released ArrowSchema)r importorskipffinewintcastrr r rhrMrrF_export_to_c_devicerrrrrVArray_import_from_c_devicerrrr!r")cffirXc_schema ptr_schemac_array ptr_arraytypr5rr4rHcarrcarr_newarr_news r test_device_interface_arrayrh\s{   ~ .D ((Cww,-HSXXk845Jgg01GCHH['23I ((288: C ((QC!R> ,C R 9FAs  G ;;w D Y'   ! ## #    < < << < ==  1 $$ $ xx--i=H ==BHHRXXZ0 00 0r<<>?G >>#   z)L M7 &&y#67 ;;w DY 3 xx--iDH ==BHHRXXZ0 00 0r<<>?G >>#   z)M N> &&y*=>>77>>s/!L$!LLL#cXtjd}|j}|jd}t |j d|}|jd}t |j d|}t d}|j}tj|t}tj||} | j||jdk(sJ|jtjk(sJ|j j"dk(sJ~ t$j&j)||} | j|k(sJ| j+t%j,} | j/|sJ~ tj0t2d5t$j&j)||dddtj||} | j||~ t$j&j)||} | j|k(sJ| j+t%j,} | j/|sJ~ tj0t2d 5t$j&j)||dddt%j4j7|tj0t2d 5t$j&j)||dddy#1swYGxYw#1swY~xYw#1swYyxYw) NrQrRrSrTrSrRrUrrVz%ArrowSchema describes non-struct type)r rWrXrYrZr[rrrrrrr\rrrrhrVrrr^rFrrr!r"r  _export_to_c) r_rXr`rarbrcrrrjr cbatch_new batch_news r !test_device_interface_batch_arrayrms   ~ .D ((Cww,-HSXXk845Jgg01GCHH['23I R E \\F  & &un =D  # #D& 1F y)   ! ## #    < < << < ==  2 %% % 55iHJ    && &""2#@#@#BCI   E "" " z)L M@ ,,Y?@ # #D& 1F y*555iLJ    && &""2#@#@#BCI   E "" " z)M ND ,,Y CDHHJJ' zD FD ,,Y CDD)@@DD DDs$!!L6!L!L LL L)ctd}tj|t}tj ||j }|d}|d}t |t |k(sJtd}tj|t}tj ||j }|d}|d}t |t |k(sJy)NrSrrrrrrrrstr)rrjrr5res r test_print_arrayrqs R E  & &un =D  # #D%,, 7F +C $rMrbrmrvrrrrrrrrrrrrrrr'r)r.r8rCrOrhrmrqrvr~rrrrr rs$   # v>* !9 ! ! #n, ;;   4 A  CN 5,./=-..|>/|>~.4/4>6./6./4.'/'6.<I/<I~fi%89..2/:.2b.(2/(2V),0Ef!'H'4 ' 08 &  $/ . /  -22>j5Dp ! "c+!,! ;! %%M#""J#sI''JJ