K i-ddlZddlZddlmZmZddlmZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlmZmZmZmZmZmZmZm Z m!Z!m"Z"ddl#m$Z$dZ%dZ&dZ'd3d Z(d Z)d Z*d Z+Gd dZ,dZ-dZ.dZ/dZ0dZ1GddejdZ3Gdde3Z4Gdde4Z5Gdde4Z6Gdde3Z7Gdde7Z8ejrj"Gd d!e7Z:Gd"d#Z;Gd$d%e;ejxZ=Gd&d'e;ejxZ>d(Z?d)Z@d*ZAd+ZBejrjd,d-d.d/d0fd1ZDd2ZEy)4N)datedatetime)Decimal) open_csvread_csv ReadOptions ParseOptionsConvertOptionsISO8601 write_csv WriteOptions CSVWriter InvalidRow)utilc#nKtj}|Ed{|D]}|D] }||z y7wN)stringascii_lowercase)lettersfirstseconds \/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/pyarrow/tests/test_csv.pygenerate_col_namesr/sG$$G! !F&.  !!s 535c#JKtd||z|D] }||||zywNrrangearrnum_colsnum_rowsis r split_rowsr#8s4 1h)8 4"!AL!!"s!#c#BKtd|D] }||d|ywrrrs r split_columnsr%>s- 1h !+X+sc tjd}t||zDcgc]}|jdd}}t j }t tjt|}|r1|jdj||j|t|||D]A} |jdjtt| |j|C|jj!}t#|||D cgc]+} t%j&| t%j(-} } t$j*j-| |} || fScc}wcc} w)N*r,)type)randomRandomrrandintioStringIOlist itertoolsislicerwritejoinr#mapstrgetvalueencoder%paarrayint64Table from_arrays) r r!linesep write_namesrnd_rcsv col_namesrowcolumnsexpecteds rmake_random_csvrGDs= -- C).x(/B)C DA3;;q$  DC D ++-CY%%&8&:HEFI #((9%& '#x2 #((3sC=)* ' ,,.   !C'Xx@Bxx"((*-BGBxx##GY7H = EBs F-0Fctj}|jdj||jd|j j S)Nr) )r.r/r3r4r7r8) column_namesrBs rmake_empty_csvrKVsB ++-CIIchh|$%IIdO <<> ""c |}|jD]K\}}t|||dk(s Jd|z|D]%}t|||t|||k(r JdMtjt 5d|_ddd|jDcic] \}}||d }}}|di|}|jD]\}}t|||k(rJy#1swYaxYwcc}}w)zD Check setting and getting attributes of an *Options class. rzincorrect default value for zfailed setting valueTN)itemsgetattrsetattrpytestraisesAttributeErrorzzz_non_existent)cls attr_valuesoptsnamevaluesv non_defaultsvalues rcheck_options_classr_]s" 5D#))+D ftT"fQi/ 2 *T 1 2/ DA D$ "4&!+ C-C C+ DD ~ &% $%9D8I8I8KL fD&)OLLL   D#))+, etT"e+++, %%Ms?C'"C3'C0c |di|}|j|j||j}|jD]\}}t |||k(rJy)N)protocolrO)loadsdumpsHIGHEST_PROTOCOLrPrQ)rWpicklerrXrYnew_optsrZr^s rcheck_options_class_picklingrgusk   D}}W]]44;4L4L+NOH"((*0 ex&%///0rLc$eZdZdZdZdZdZy)InvalidRowHandlerc ||_g|_yr)resultrows)selfrks r__init__zInvalidRowHandler.__init__~s  rLcP|jj||jSr)rlappendrk)rmrDs r__call__zInvalidRowHandler.__call__s {{rLcXt|txr|j|jk(Sr isinstancerirkrmothers r__eq__zInvalidRowHandler.__eq__s'5"34,  + -rLcZt|t xs|j|jk7Srrsrus r__ne__zInvalidRowHandler.__ne__s*u&788,  + -rLN)__name__ __module__ __qualname__rnrqrwryrOrLrriri}s--rLricd}d}tddDcgc]}|}}tt|||}|gdgdgk(sJtt|||}|ddgddgd d gd d gdd ggk(sJycc}w)NrN )rNrr~) rrrrrrr)rr0r#r%)r r!xrrlrEs r#test_split_rows_and_columns_utilityrsHHArl #1 #C #  3(3 4D   =h9:G  A A A A B    $s A.c  t}|}t|ddgddggddggddgddgdd g t||ddddgddd |jdkDsJd |_|jd k(sJ|d }|jd k(sJ|j d}t j tj|5|}d|_|j dddd}t j tj|5|}d|_ |j dddd}t j tj|5|}d|_ |j dddd}t j tj|5|}d|_ d|_ |j dddy#1swYxYw#1swYxYw#1swYwxYw#1swYyxYw)NTFrrabcdutf8utf16) use_threads skip_rowsrJautogenerate_column_namesencodingskip_rows_after_names)rerrrJrrr90i block_sizez-ReadOptions: block_size must be at least 1: 0matchz-ReadOptions: skip_rows cannot be negative: -1z9ReadOptions: skip_rows_after_names cannot be negative: -1zTReadOptions: autogenerate_column_names cannot be true when column_names are providedab) rr_rgrvalidaterSrTr9 ArrowInvalidrrrrJ) pickle_modulerWrYrs rtest_read_optionsrs C 5D$#$a&&(4,%738$-"('!2/0"g 7!m-1+,/3Tl;@*179 ; ??Q  DO ??e ## # $ D ??d "" "MMO ;E re 4u  E re 4u  re 4u  @E re 4u  re 4u  AE re 4u  re 4u 7     sH!I -I;I#I0I<!J II #I-0I9<JJc t}|}t|ddgddgddgddggddggddgddggtdgg t||ddddddgddtdg t j t 5d |_ddd|jd kDsJd |_|jd k(sJ|jik(sJtjtjd |_ |jtjtjd k(sJddd|_ |jtjtjdk(sJtjdtjfdtj fg}||_ |jtjtj dk(sJdtj"fg|_ |jdtj"ik(sJt j t$d5ddi|_ dddt j t$5d |_ dddt'|j(t*sJd|j(vsJd|j(vsJddg|_|j(ddgk(sJt'|j,t*sJddg|_|j,ddgk(sJt'|j.t*sJddg|_|j.ddgk(sJ|j0gk(sJtg|_|j0tgk(sJ|dtjiddgddgdd gd!td"g#}|jdtjik(sJ|j(ddgk(sJ|j.dd gk(sJ|j,ddgk(sJ|jd!k(sJ|j0td"gk(sJy#1swYxYw#1swYxYw#1swYxYw)$NTF.r)defabcz%y-%m) check_utf8strings_can_be_nullquoted_strings_can_be_null decimal_pointinclude_columnsinclude_missing_columnsauto_dict_encodetimestamp_parsers) rerrrrrrrrz..r韆)rcint16null)r\wrrrrzDataType expectedrN/AxxxyyyNnnTttFffiz%Y-%m-%d) column_types null_values true_values false_valuesauto_dict_max_cardinalityr)r r_r rgrSrT ValueErrorrrrr9rfloat32rschemaint32rbinary TypeErrorrtrr0rrr)rrWrYrs rtest_convert_optionsrs( C 5D u "DM$(%=CjeU^,!& 124! ] #( %"G, . z ""!"  ) )A -- -%*D"  ) )U 22 2    "" " hhjrzz| ?FD   bhhjryy{ C CC Cryy{+,D   biik 2 22 2 y(; <( $K( y ! d&& -- - !! !! ! D$$ $$ $u~D   u~ -- - d&& -- -u~D   u~ -- - d'' .. .D    .. .  ! !R '' '%YD  ! !gY .. . S"''),;S$K $K3"):!6 8D   bggi 0 00 0   T{ ** *   d ++ +   T{ ** *  ) )S 00 0  ! !gz%: :: :k""*((s$0P'< P4'Q'P14P>Q ct}|}t|ddggdgd|jdkDsJd|_|jdk(sJ|d }|jdk(sJ|jd }t j t j| 5|}d|_|jdddy#1swYyxYw) NTF)r)r|)needednone all_valid)include_headerr quoting_stylerri&) batch_sizez.WriteOptions: batch_size must be at least 1: 0r)r r_rrrSrTr9r)rWrYrs rtest_write_optionsrfs C 5D T5M5E57 ??Q  DO ??e ## # $ D ??d "" "MMO >D,F,FFGHt$% &% &rLc fd}tddd\}}t}t|dz |_t }dt j i|_|dz}|d d }tjt j| 5j||| ddd|d z}|dd |} tjt j| 5j||| dddd} || z} |dd | ddjd} tjt j| 5j| || dddd|_ tjt j| 5j||| dddtjt j| 5j||| dddtjt j| 5j| || dddd|_ tddd\}}gd|_|dz}|d d}tjt j| 5j||| ddd|| z} |dd| ddjd} tjt j| 5j| || ddd|d z}|dd} | jt|} tjt j| 5j||| dddd|_tjt j| 5j||| dddtjt j| 5j||| dddy#1swYWxYw#1swYxYw#1swYxYw#1swYxYw#1swYJxYw#1swYxYw#1swYxYw#1swY[xYw#1swYxYw#1swYxYw#1swYyxYw)NcTjrd}nd|d}|j|g|S)NrzRow #z: )rformat) msg_formatrDargsrow_informs r format_msgz@BaseTestCSV.test_row_number_offset_in_errors..format_msg.s6"3%r?$:$$X55 5rLrr T)r?rrs1,2 z{}Expected 4 columns, got 2frrconvert_optionss a,b,c,d zDIn CSV column #0: {}CSV conversion error to int32: invalid value 'a'sthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long row,3 z#{}Expected 4 columns, got 2: {} ...r`zutf-8/Frrrde)rGrrrr r9rrrSrTrrdecoderrJrr) rmrrBrArrcsv_bad_columnsmessage_columns csv_bad_type message_valuelong_rowcsv_bad_columns_long message_longs ` r test_row_number_offset_in_errorsz,BaseTestCSV.test_row_number_offset_in_errors+sB 6!CT:Q"} "%c(Q, (*(+RXXZ'8$ *$%BCH ]]2??/ B = OOO)5,;  = = ^+ " ?  ]]2??- @ = OOL)5,;  = = <"X~!"G"*1R."7"7"@B ]]2??, ? = OO0)5,;  = = .0 * ]]2??/ B = OOO)5,;  = = ]]2??- @ = OOL)5,;  = = ]]2??, ? = OO0)5,;  = = ./ *!CU;Q$8 ! *$%BCH ]]2??/ B = OOO)5,;  = = #X~!"G"*1R."7"7"@B ]]2??, ? = OO0)5,;  = = ^+ " ?  &,,SX6 ]]2??- @ = OOL)5,;  = = "$  ]]2??/ B = OOO)5,;  = = ]]2??- @ = OOL)5,;  = = =a = = = = = = = =  = =  = = = = = = = = = =  = =s O7PP PP+P8Q$QQQ+Q77PPPP(+P58QQQQ(+Q47Rcxd}t}tjtd5j ||dddt d|_j ||}|jddgdd gd k(sJfd }td d |d dtd d|ddg}|j j|k(sJt d|_tjtd5j ||dddtd d |d dg}|j j|k(sJt d|_|j|j|}j ||}|jddgdd gd k(sJy#1swYhxYw#1swYxYw)Nsa,b c d,e f,g,h i,j zExpected 2 columns, got 1: crrrrr"ejrc$jrdS|Sr)r)rrms rrow_numz5BaseTestCSV.test_invalid_row_handler..row_nums++4 2 2rLrrNrrrzf,g,herror) r rSrTrrrirrrrlrbrc)rmrrlr rr. expected_rowss` rtest_invalid_row_handlerz$BaseTestCSV.test_invalid_row_handlers+!^ ]]46 < OOD O ; < *;6)B &J? ss%     3 q!WQZ - q!WQZ 1 --22mCCC*;7)C & ]]46 < OOD O ; <$Aq'!*c:; --22mCCC*;6)B &"(()<) =OODtO? C OOD/O B C C Cs DD&cd}t}t}gd|_d|_|j |||}t j dt jfdt jfdt jfg}|j |k(sJ|jgdgd gddk(sJddg|_ dd g|_|j |||}t j dt jfd t jfg}|j |k(sJ|jgd gd d k(sJt jt jd |_ |j |||}t j dt jfd t jfg}|j |k(sJ|jgdgd d k(sJy)NrrnTrrorrpNNNrlrrd)NNNN)rpr)scdsghsklsop)rr rrrr9rrrrrJrrrrmrlrrrrs rtest_include_missing_columnsz-BaseCSVTableRead.test_include_missing_columns|s."} (**<'26/<0? AT2779-!299;/!2779-/0||v%%% $$$%    &*4L !+/,'<0? AT299;/!2779-/0||v%%% **%    /1iik.0hhj(:$<0? AT299;/!288:.01||v%%% .*%    rLc2d}|j|}tjdtjfdtjfdtjfg}|j|k(sJ|j ddgddgd d gd k(sJy) Nsa,b,c 1,2,3 4,5,6 rrrrNrrr~rrr5)rr9rr;rrmrlrrs rtest_simple_intsz!BaseCSVTableRead.test_simple_intss'%S"((*- "((*- "((*-/0||v%%% QQQ%    rLcbd}|j|}tjdtjfdtjfdtj fdtj fg}|j|k(sJ|jddgdd gd d gd d gdk(sJy)Ns a,b,c,d 1,2,3,0 4.0,-5,foo,True rrrr?g@r3fooFTr)rr9rfloat64r;rbool_rrxs rtest_simple_variedz#BaseCSVTableRead.test_simple_varieds5%S"**,/ "((*- "))+. "((*-/0||v%%% sRu %    rLc d}|j|}tjdtjfdtjfdtj fdtj fdtjfdtjfg}|j|k(sJ|jgdgd gd gd gd gd dk(sJy)NsCa,b,c,d,e,f 1,2,,,3,N/A nan,-5,foo,,nan,TRUE 4.5,#N/A,nan,,,false rrrrr+f)r{Ng@)rr|N)rr~nanrt)3snan)NTF)rrrrr+r) rr9rrr;rrrrrrxs rtest_simple_nullsz"BaseCSVTableRead.test_simple_nullss.%S"**,/ "((*- "))+. "''), "))+. "((*- /0 ||v%%% !##($ %    rLc6td}d}|j||}tjdtjfdtj fg}|j|k(sJ|j gdgdd k(sJtd }|j||| }tjdtj fdtjfg}|j|k(sJ|j gd gdd k(sJy)N;rsa;b 1.25;2,5 NA;-3 -4;NAr*rr)g?Ng)z2,5z-3NArr))r)rr)z1.25rz-4)g@gN)r rr9rrrrr )rmrrlrrrs rtest_decimal_pointz#BaseCSVTableRead.test_decimal_points$s3 -MBS"**,/ "))+.01||v%%% #$%    )s;M0? AS"))+. "**,/12||v%%% %"%    rLc d}|j|}tjdtjfdtjdfdtjdfg}|j|k(sJ|j ddgt dd d t dd d d gt dd d d d d d t dd d d d d dgdk(sJy)Nska,b,c 1970,1970-01-01 00:00:00,1970-01-01 00:00:00.123 1989,1989-07-14 01:00:00,1989-07-14 01:00:00.123456 rrsrnsirNrrix@r5)rr9rr; timestamprrrxs rtest_simple_timestampsz'BaseCSVTableRead.test_simple_timestampssI%S"((*- ",,s"34 ",,t"4578||v%%% 4A&q"a(@A4Aq!Q74B1a8:%    rLcd}t}|j||}tjdtjfdtj dfg}|j|k(sJ|j ddgtdd d tdd d gd k(sJd g|_|j||}tjdtj dfdtjfg}|j|k(sJ|j td d d td d d gddgd k(sJd tg|_|j||}tjdtj dfdtj dfg}|j|k(sJ|j td d d td d d gtdd d tdd d gd k(sJy)Ns6a,b 1970/01/01,1980-01-01 00 1970/01/02,1980-01-02 00 rkrrrz 1970/01/01z 1970/01/02irNrrz%Y/%m/%drz 1980-01-01 00z 1980-01-02 00) r rr9rrrrrrr )rmrlrYrrs rtest_timestamp_parsersz'BaseCSVTableRead.test_timestamp_parserssKd;S"))+. ",,s"3467||v%%%  -4A&q!(<=%    #-d;S",,s"34 "))+.01||v%%% 4A&q!(<=!?3%    #-g!6d;S",,s"34 ",,s"3467||v%%% 4A&q!(<=4A&q!(<=%    rLcd}|j|}tjdtjfdtjfg}|j|k(sJ|j t dddt dddgt dddt dddgdk(sJt }tjtjd|_|j|| }tjdtjfdtjfg}|j|k(sJ|j t dddt dddgt dddt dddgdk(sJt }tjd tjd d|_|j|| }tjdtjd fdtjd fg}|j|k(sJ|j tdddtdddgtdddtdddgdk(sJy) Ns0a,b 1970-01-01,1970-01-02 1971-01-01,1971-01-02 rrrrNirrrkrms) rr9rdate32rrr date64rrr)rmrlrrrYs r test_dateszBaseCSVTableRead.test_dates*sAE%S"))+. "))+.01||v%%% tQ"Dq!$45tQ"Dq!$45%    "$))+BIIK@d;S"))+. "))+.01||v%%% tQ"Dq!$45tQ"Dq!$45%    "$,,s"3",,t:LMd;S",,s"34 ",,t"4578||v%%% 4A&q!(<=4A&q!(<=%    rLc ddlm}d}|j|}tjdtj dfdtj fg}|j|k(sJ|j|ddd |d d d gd d gdk(sJt}tjdtj dd|_ |j||}tjdtjdfdtj dfg}|j|k(sJ|j|ddd |d d d g|ddd d|d d d dgdk(sJy)Nr)times0a,b 12:34:56,12:34:56.789 23:59:59,23:59:59.999 rrr "r\r;z 12:34:56.789z 23:59:59.999rusrrki iX>) rrrr9rtime32rrr time64r)rmrrlrrrYs r test_timeszBaseCSVTableRead.test_timesNsm!E%S"))C.1 "))+.01||v%%% r2r"DR$45 .1%    "$))D/ $Hd;S"))D/2 "))D/245||v%%% r2r"DR$45r2r6*DRV,DE%    rLctd}dj}|j||}tjdtj tj tjfdtjfg}gdgdd }|j|k(sJ|j|k(sJd |_ |j||}|j|k(sJ|j|k(sJd |_ |j||}|jtjdtjfdtjfgk(sJ|j|k(sJd |_ d |_ d}|j||d }|j|k(sJ|djdj }t|d k(sJ|djdk(sJ|d jdk(sJd|_ |j||}tjdtj tj tj fdtjfg}gdgdd }|j|k(sJ|j|k(sJy)NT)rua,b ab,1 cdé,2 cdé,3 ab,4rkrr)rcdérr)rNrrrrrrN2Fsa,b ab,1 cd,2 ab,3)rrArrcd)abrrrNrr)r r8rr9r dictionaryrrr;rrrchunkras_py as_bufferr)rmrYrlrrrF dict_valuess rtest_auto_dict_encodez&BaseCSVTableRead.test_auto_dict_encodeist40779d;S"-- BIIK"HI "((*-/0. ||v%%% H,,,)*&d;||v%%% H,,,*+&d;||ryy3 *<+. *;*= >> >> H,,,*,&+d.3 5||v%%%Cj&&q)44 ;1$$$1~##%---1~'')Y666d;S"-- BIIK"HI "((*-/0+ ||v%%% H,,,rLctddg}d}|j||}tjdtjfdtj fdtj fd tj fg}|j|k(sJ|jddgdd gd d gd dgdk(sJtddgd}|j||}|jddgdd gd d gd dgdk(sJd|_|j||}|jddgdd gd d gd dgdk(sJtg}d}|j||}tjdtj fdtj fg}|j|k(sJ|jd gd gdk(sJy)NXxxZzz)rs$a,b,c,d Zzz,"Xxx",1,2 Xxx,#N/A,,Zzz rkrrrrz#N/A1rrrT)rrFs a,b #N/A, r) r rr9rrrr;rrrmrYrlrrs rtest_custom_nullsz"BaseCSVTableRead.test_custom_nullss5%.9=d;S"''), "))+. "))+. "((*-/0||v%%% rT %    5%.268d; rT %    +0'd; rT %    "-d;S"))+. "))+.01||v%%% %    rLcXtddgddg}d}|j||}tjdtjfd tj fd tjfg}|j|k(sJ|j gd gd gd dk(sJy)Nryesrno)rrs>a,b,c True,T,t False,F,f True,yes,yes False,no,no N/A,N/A,N/A rkrrr)TrueFalserrr)TFTFN)trrrrr5)r rr9rrrrrs rtest_custom_boolsz"BaseCSVTableRead.test_custom_boolss3,,/;8! d;S"))+. "((*- "))+.01||v%%% :1/%    rLc ptdddtjdddd}d }|j|| }tjd tj fd tj fd tjfdtjfdtjddfg}ddgddgddgddgtdtdgd}|j|k(sJ|j|k(sJttjd tj fd tjfdtjfdtjddfdtjfg}|j|| }|j|k(sJ|j|k(sJd}tjtj5}|j|| dddtj}d|vsJd|vsJy#1swY+xYw)Nrrbooleanrrr)rrrr+zzrs(a,b,c,d,e 1,2,3,true,1.0 4,-5,6,false,0 rkrrrrr+rNr@gr}6TF1.00z0.00)rrrrr+rs(a,b,c,d,e 1,XXX,3,true,5 4,-5,6,false,7 zIn CSV column #1: z2CSV conversion error to float: invalid value 'XXX')r r9 decimal128rrr;rrrrrrSrTrr6r^)rmrYrlrrrFexcerrs rtest_column_typesz"BaseCSVTableRead.test_column_typess191:13r11E28 ,:; >d;S"((*- "**,/ "))+. "((*- "--A"67 9: Qts&/76?3  ||v%%% H,,,S"**,$7%("))+$6%("((*$5%("--A*>$?%)288:$6 $89: d;||v%%% H,,,= ]]2?? + 8s OOD$O 7 8#))n#s***CsJJJ  8 8s .H,,H5c .dtjtjtjfdtjtjtjfdtjtjtj ddfdtjtjtj fg}tt|}d}|j|| }tj|}gd gd td td td ggdd}|j|k(sJ|j|k(sJdtjtjtjf|d<tt|}tjt 5|j|| }dddy#1swYyxYw)Nrrrrrrrs<a,b,c,d abc,123456,1.0,zz defg,123456,0.5,xx abc,N/A,1.0,xx rk)rdefgr)rrNrz0.50)rrororr)r9rrrr;r large_utf8r dictrrrrint8rSrTr)rmrrYrlrrrFs rtest_column_types_dictz'BaseCSVTableRead.test_column_types_dicts"-- BGGI6 7 "-- BHHJ7 8 "-- BMM"a,@A B "-- BMMO< = ? 4 +=>$d;<(''&/76?GFOD#   ||v%%% H,,, bggi CD Q4 +=> ]]. / @OOD$O?E @ @ @s .H  Hcdd}tddg}tdtji}|j |||}tj dtjfdtj fg}|j |k(sJ|jgdgdd k(sJy) Ns a,b c,d e,f rrrJrr)ace)rrrre)rr r9rrrrrrus r#test_column_types_with_column_namesz4BaseCSVTableRead.test_column_types_with_column_names&s""c ; (sBIIK6HI<0? AS"))+. "))+.01||v%%% # %    rLchd}|j|}|jddgddgddgdk(sJy) Na,b,c 1,2,3 4,5,6rNrrr~rrr5rVrSs rtest_no_ending_newlinez'BaseCSVTableRead.test_no_ending_newline6sH%% QQQ%    rLcXd}|j|}|jdgik(sJy)Ns, rrVrSs r test_trivialzBaseCSVTableRead.test_trivial@s.% RH,,,rLcPd}|j|}|jddgddgdk(sJtd}|j|| }|jgd gd dk(sJtd }|j||| }|jddgddgdk(sJy)Nsa,b 1,2 3,4 rNrrrrF)rr*)NrNNr)NrNr)rr7)r2)rrr r)rmrlrrrs rtest_empty_linesz!BaseCSVTableRead.test_empty_linesFs+% QQ%    %> MB ##%    #Q/ M-9 ; %    rLcd}tjtjd5|j |dddd}tjtjd5|j |ddddD]A}tjtjd5|j |dddCy#1swYxYw#1swY\xYw#1swYfxYw)Nsa,b,c 1,2 4,5,6 rfrs a,b,c 1,2,3 4zExpected 3 columns, got 1)rLrYr[rZs Empty CSV file)rSrTr9rr)rmrls rtest_invalid_csvz!BaseCSVTableRead.test_invalid_csv[s% ]]2??2M N " OOD ! "! ]]2??2M N " OOD ! "9 &Dr6FG &% & & &  " " " " & &s#C )C-C" CC"C+ cd}|j|}|jdgdgdk(sJtd}|j||}|jdgd gd k(sJy) Na;b,c de,fg;eh defg;eha;brrrr*de,fgehrb,c)rrr )rmrlrrYs rtest_options_delimiterz'BaseCSVTableRead.test_options_delimitergs#% 6%    c*D9 6%    rLctdd\}}|j|}|j|jk(sJ|j|sJ|j |j k(sJyNrrr)rGrrequalsr)rmrBrFrs rtest_small_random_csvz&BaseCSVTableRead.test_small_random_csvusf'R@ X$||x...||H%%% H$6$6$8888rLcLtdd\}}gd}||jdg}|D]x}|D]q}t|}|j||}|j|jk(sJ|j |rO|j |j k(rqJzy)Nrr)rr r or[rr)rGrstriprrrrr) rmcsv_baserF block_sizescsvsrBrrrs rtest_stress_block_sizesz(BaseCSVTableRead.test_stress_block_sizes|s,a#F(/ (//'23 EC) E *jA ,G||x666||H- ??,0B0B0DDDD  E ErLc tj}d}t|Dcgc]}d| }}t |}|}t |dddDcic]}|t jc}}|j||} ||z } | dksJ| j|k(sJ| jdk(sJ| j|k(sJy#t$rtj}YwxYwcc}wcc}w)Ni'Krrrk$@r) r thread_timerUrrKr r9rrrr!rJ) rmclockrr"rCrBt1krrdts r"test_stress_convert_options_blowupz3BaseCSVTableRead.test_stress_convert_options_blowups $$E &+K&89qW9 9Y' W(2;CaC.AQ!RYY[.AC_E Wr\Tzz  K///~~"""!!Y...# IIE :BsC C+C0 C('C(c tjjtjjk7rt j dd}d}d}d}|dkr|dkrt d|d d |zz}d} tj5} tj| }|jtj} |j|tj|gg|gd  ddd| |jn|d z}|dkr|dkrt j d|dksJ|j} t#| t$j&sJ| j(t*j,k(sJy#t$r#} | }tj| z }Yd} ~ d} ~ wwxYw#tj|gg|gd wxYw#1swYxYw#t$rYwxYw)Nz'test only works from main Python threadcttjdtjtjy)Ng?)rsleepsignal raise_signalSIGINTrOrLrsignal_from_threadz>BaseCSVTableRead.test_cancellation..signal_from_threads JJsO    .rLgirg@rzworkload size:sa,b,c s1,2,3 targetrrz0Failed to get an interruption during CSV readingr) threadingcurrent_threadident main_threadrSrprintrsignal_wakeup_fdThreadstartrrKeyboardInterruptselect __context__failrtr9ArrowCancelledsignumrr) rmr last_duration workload_sizeattempts large_csvexc_infosigfdrrr+s rtest_cancellationz"BaseCSVTableRead.test_cancellations  $ $ & , ,%%'-- . KKA B /  c!hm "M 2"Z-%??IH **,B B%,,4FG !YY[= OOI6 ugrE7DAB(#''3*A-MMc!hmP   KKJ K###  !R..///xx6==(((? 1='(H,0IIK",z=BaseCSVTableRead.test_cancellation_disabled..s4??;7rLr )r rrr4)rmrs` rtest_cancellation_disabledz+BaseCSVTableRead.test_cancellation_disableds*   7 9   rLN)(rzr{r|rrrPrTrWrargrirrrvryrrrrrrrrrrrrrrrrrrrrrrSr=r rr#rOrLrr@r@s15 8 #2 1 f=2C:+ Z    . 2 "  D" H 6/-b, \ *'KR@@  -  * &  9 E/0 [[@)@)D [[rLr@ceZdZedZy)TestSerialCSVTableReadcyNFrOrs rrz"TestSerialCSVTableRead.use_threadsrLNrzr{r|r;rrOrLrr%r% rLr%ceZdZedZy)TestThreadedCSVTableReadcyNTrOrs rrz$TestThreadedCSVTableRead.use_threadsrLNr)rOrLrr,r,s rLr,ceZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZdZdZfdZej,j.fdZxZS)BaseStreamingCSVReadcv|jdt}|j|_t|g|i|S)z Reads the CSV file into memory using pyarrow's open_csv csv The CSV bytes args Positional arguments to be forwarded to pyarrow's open_csv kwargs Keyword arguments to be forwarded to pyarrow's open_csv r)rFrrr)rmrBrrrs rrzBaseStreamingCSVRead.open_csvs;((G #'#3#3  -d-f--rLc N|jtj|fi|Sr)rr9rHrs r open_byteszBaseStreamingCSVRead.open_bytes rIrLc|j|k(sJt|}t|t|k(sJt||D]=\}}|j d|j|k(sJ|j |k(r=Jy)NTrC)rr0rziprr)rmreaderexpected_schemarMbatchesbatchexpected_batchs r check_readerz!BaseStreamingCSVRead.check_reader s}}///v,7|s=1111%(-%@ 7 !E> NNN %< ?#&d)a- LA &/"'"()!46"&"'*+ ,rLcTd}t}t|dz |_|j||}t j dt j fdt j fg}|j |k(sJ|jjdgdgdk(sJtjtd 5|jdddtjt5|jdddy#1swYs 3"} "%d)a- LA))c288:%6&)288:%6%89}}///%%'113cU8    ]]:!@B %  " " $ %]]= ) %  " " $ % %  % % % %sD8DDD'c\d}t}d|_tjtj d5|j ||}dddd}d|_|j ||}|jjdgd gd k(sJtjtj d5|jdddtjt5|jdddy#1swYxYw#1swYHxYw#1swYyxYw) Nsa,b 1,2,3 4,5 6,7 rzExpected 2 columns, got 3rrsa,b 1,2 3,4,5 6,7 rrNrr) rrrSrTr9rr4rKrrL)rmrlrr7s rrz%BaseStreamingCSVRead.test_invalid_csvRs ("} "$  ]]2??!<> 1__<%1F 1 )"# LA%%'113aSs7KKKK ]]2??!<> %  " " $ %]]= ) %  " " $ % % 1 1 % % % %s#D >D0D" DD"D+cd}|j|}tjdtjfdtjfg}|j ||dgdgdgt d}|j|| }tjd tjfd tjfg}|j ||d gd gdgy)Nrrrrrrrrr*rrrrr)r4r9rrr<r )rmrlr7r8rYs rrz+BaseStreamingCSVRead.test_options_delimiterhs#&))eRYY[%9&)299;%7%9: &/$(6"),- .c*T:))c299;%7&+RYY[%9%;< &/")$(6+, -rLcd}|j|}tjdtjfdtjfdtjfg}|j ||ddgddgd d gd gy) NrrrrrNrrr~rrr5)r4r9rr;r<rAs rrz+BaseStreamingCSVRead.test_no_ending_newlineys%&))c288:%6&)288:%6&)288:%6%89 &/"#Q"#Q"#Q)* +rLctjtd5|jddddy#1swYyxYw)NrrrL)rSrTrr4rs rtest_empty_filez$BaseStreamingCSVRead.test_empty_files2 ]]:-= > ! OOC  ! ! !s 7Acjd}t}gd|_|j||}tjdtj fdtj fdtj fg}|j ||ddgd d gd d gdgt}ddg|_|j||| }tjdtj fdtj fg}|j ||d d gd d gdgdtji|_ |j||| }tjdtj fdtjfg}|j ||ddgd d gdggd|_tjtd5|j||| }dddd|_|j||| }tjdtjfdtj fdtjfg}|j ||ddgddgd d gdgtjtj d|_ |j||| }tjdtj fdtj fdtjfg}|j ||ddgddgd d gdgy#1swYLxYw)Ns 1,2,3 4,5,6)rr+rrrr+rrNrrr~rrr)r+rr5)r3rr+z,Column 'g' in include_columns does not existrTr3)r3r+r)r+r3)rrJr4r9rr;r<r rrrrSrTrqrrr)rmrlrr7r8rs rtest_column_optionsz(BaseStreamingCSVRead.test_column_optionss4"} $3 !LA))c288:%6&)288:%6&)288:%6%89 &/"#Q"#Q"#Q)* + )*+.*'L1@!B))c288:%6&)288:%6%89 &/"#Q"#Q)* + ),RYY['9$L1@!B))c288:%6&)299;%7%9: &/"%s"#Q)* + +:' ]]DF F__T 5D%FF F 37/L1@!B))c2779%5&)288:%6&)299;%7%9: &/"&"%s"#Q)* + .0YY[rzz|'L$L1@!B))c2::<%8&)288:%6&)299;%7%9: &/"&"%s"#Q)* +/ F Fs L((L2cd}t}|j||}tjdtjfdtj fg}|j ||dgdgdgd|_|j||}tjdtjfdtjfg}|j ||dgd gdgd }d |_|j||}tjdtjfdtjfg}|j ||dgd gdgy) Nsa,b un,lphantrrrunslphantrlatin1u éléphants a,b un,lphantr)rr4r9rrrr<rrMs r test_encodingz"BaseStreamingCSVRead.test_encodings[("} LA))c299;%7&)299;%7%9: &/"&"3!467 8!) LA))c299;%7&)299;%7%9: &/"&".12 3 F ' LA))c299;%7&)299;%7%9: &/"&".12 3rLctdd\}}|j|}|j}|j|jk(sJ|j |sJ|j |j k(sJyr)rGr4r>rrr)rmrBrFr7rs rrz*BaseStreamingCSVRead.test_small_random_csvsr'R@ X%!||x...||H%%% H$6$6$8888rLctdd\}}gd}||jdg}|D]}|D]}|d|jddk\sJt|}|j ||}|j } | j |j k(sJ| j|rx| j|jk(rJy) Nrrr)rrr rr[rYrr) rGrcountrr4r>rrr) rmrrFrrrBrrr7rs rrz,BaseStreamingCSVRead.test_stress_block_sizess,a#F(/ (//'23 EC) E ;J'--e4999*jA l)4)||x666||H- ??,0B0B0DDDD E ErLctjtj}d}d}t }ddg|_d|_|j||}||dgdgd tj}||d gd gd tj|ksJ||d gd gd tj|ksJ||dgdgd tj|ksJtjt5|jdddtj|k(sJd}tj|k(sJy#1swY>xYw)NcN|j}|j|k(sJyr)rKr)r7rFr:s rcheck_one_batchzABaseStreamingCSVRead.test_batch_lifetime..check_one_batchs%**,E??$0 00rLs10,11 12,13 14,15 16,17 rrrrrrrrrrr6r) gccollectr9total_allocated_bytesrrJrr4rSrTrLrK)rm old_allocatedrbrlrr7allocated_after_first_batchs rtest_batch_lifetimez(BaseStreamingCSVRead.test_batch_lifetimesU 002  1/"} %(#J !"# LArd" 67&(&>&>&@#rd" 67'')-HHHHrd" 67'')-HHHHrd" 67'')-HHHH ]]= ) %  " " $ %'')]:::'')]:::  % %s EE"ct|d}t}d|_ddg|_|j ||}t jt5|jsJ dddy#1swYyxYw)Nrrrrr) superrrrrJr4rSrTrLrKrmrlrYr7 __class__s rrz*BaseStreamingCSVRead.test_header_skip_rowsst %'.}!4LD9 ]]= ) ,))+ ++ , , ,s A55A>ct|d}t}d|_|j ||}t j t5|jsJ dddd|_|j ||}t j t5|jsJ dddy#1swYYxYw#1swYyxYw)Nrrrr) rkrrrr4rSrTrLrKrls rrz/BaseStreamingCSVRead.test_skip_rows_after_names+s *,.}%&"D9 ]]= ) ,))+ ++ ,&+"D9 ]]= ) ,))+ ++ , ,  , ,  , ,sB;C;CC)rzr{r|rr4r<rrPrTrHrNrrrrSrVrZrrrirrSr=r>r __classcell__rms@rr1r1s~ .877D7,*%(%,-" +!?+B3<9E$;8 , [[,,rLr1ceZdZedZy)TestSerialStreamingCSVReadcyr'rOrs rrz&TestSerialStreamingCSVRead.use_threads@r(rLNr)rOrLrrrrr?r*rLrrceZdZedZy)TestThreadedStreamingCSVReadcyr.rOrs rrz(TestThreadedStreamingCSVRead.use_threadsGr/rLNr)rOrLrruruEs rLruc$eZdZdZdZdZdZy)BaseTestCompressedCSVReadc:tjd|_y)Nzarrow-csv-test-)prefix)tempfilemkdtemptmpdirrs rsetUpzBaseTestCompressedCSVRead.setUpNs&&.?@ rLcBtj|jyr)shutilrmtreer}rs rtearDownz"BaseTestCompressedCSVRead.tearDownQs dkk"rLc t|S#tj$r(}tjt |Yd}~yd}~wwxYwr)rr9ArrowNotImplementedErrorrSrr6)rmcsv_pathr+s rrz"BaseTestCompressedCSVRead.read_csvTs; H% %** KKA   s AAActdd\}}tjj|j|j }|j |||j|}|jd|j|jk(sJ|j|sJ|j|jk(sJy)Nrr rTrC) rGospathr4r} csv_filename write_filerrrrr)rmrBrFrrs rtest_random_csvz)BaseTestCompressedCSVRead.test_random_csvZs'SA X77<< T->->? #& h' D!||x...||H%%% H$6$6$8888rLN)rzr{r|r~rrrrOrLrrxrxLsA# 9rLrxceZdZdZdZdZy)TestGZipCSVReadzcompressed.csv.gzc~tj|dd5}|j|dddy#1swYyxYw)Nwbr)gzipopenr3rmrcontentsrs rrzTestGZipCSVRead.write_filehs5 YYtT1 %  GGH    s3<ctjj|j|j}t j |dd5}|jddddt j |dd5}|jdddd|j|}|jgdgddk(sJy#1swYhxYw#1swYCxYw) Nrrs ab,cd ef,gh rs ij,kl mn,op rlrmr) rrr4r}rrrr3rr)rmrrrs rtest_concatenatedz!TestGZipCSVRead.test_concatenatedls77<< T->->? YYxq ) 'Q GG% & ' YYxq ) 'Q GG% & ' h' $$%     ' ' ' 's C=CCCN)rzr{r|rrrrOrLrrres&L  rLrceZdZdZdZy)TestBZ2CSVReadzcompressed.csv.bz2c|tj|d5}|j|dddy#1swYyxYw)Nr)bz2BZ2Filer3rs rrzTestBZ2CSVRead.write_file}s3 [[s # q GGH    s2;N)rzr{r|rrrOrLrrrzs 'LrLrc`tjd}t||jrJyNr)r.rKrclosed)bufs r0test_read_csv_does_not_close_passed_file_handlesrs& **+ ,C SMzz>zrLc tjjgdgdgddg}|jdd}||fD]}t j }t ||td |jd|t|k(sJt j }t ||td |jdt|j }|t|| k(rJddtd ft|j dtd fdtd tdd ft|j tdtd dffD]\}}}t j }t||j|5}|j|ddd|jd|t|||k(sJt j }t||j|5}|jdD]}|j!| ddd|jd|t|||k(rJy#1swYxYw#1swY?  HSM!!!jjl$\?@  "? HS|<<<< =" t\67 !.. 14 U + - |c* TS 9 ; !.. 1  % Ud ; = 7:2 m]jjl sAHHM B "f   q ! "  HS|+8:: ::jjl sAHHM B *fA6 *""5) * *  HS|+8:: ::+: " "  * *s"I% )I1%I. 1I: ctjjgdgdgddg}tj}t ddft dft d d ffD]]\}}t ||j| 5}|j|ddd|j|k(sJ|jd _tjjd dggdg}tj}t ddft dtjjffD]_\}}t ||j| 5} |j| ddd|j|k(sJ|jd ay#1swYxYw#t$r!}t||sJYd}~dddyd}~wwxYw#1swYlxYw)N)rNrN)rNrrrr)rs"c1","c2" 1,a 2, ,c s"c1","c2" 1,"a" 2, ,"c" rs"c1","c2" "1","a" "2", ,"c" rrr)rrs"c1" "," """" )r9r<r=r.rKr rrrr7rlibr Exceptionrt)rrrresrr+s rtest_write_quoting_stylers l,<=d|LA **,C F +-HI 89 K 0 , .  s sAHHM B "f   q ! "||~$$$    sDk]TF3A **,C H -/CD F +RVV-@-@A  ssAHHM B f ""1% ||~$$$    " " "!S)))     s<FG F F G )G7G G  G  G c~d}tj5|}|J dddy#1swYyxYw)Ncltjd}t|}tj|Sr)r.rKrweakrefref)rrs rinnerz,test_read_csv_reference_cycle..inners)jj/0 {{5!!rL)r disabled_gc)rwrs rtest_read_csv_reference_cyclers="     Wt||s3< type_factoryc.tjddSNrNr9rrOrLrr"r"BMM"a rLc.tjddS)N&r6rrOrLrr"r"BMM"b !rLc.tjddSrr9 decimal256rOrLrr"r"rrLc.tjddS)NLrrrOrLrr"r"rrLc*|}tjdtjddgj|i}t ||dz t |dz }|j dj||j dk(sJy)NcolrNrzout.csv)r9rr:castr rcolumn)r}rr*routs rtest_write_csv_decimalrs >D HHeRXXq!f-22489 :E eVi'( 6I% &C ::e  ! !$ '5<<+> >> >rLcd}Gddtj}tdD]?}tjt j 5t||dddAy#1swYLxYw)Nsa,b,cc(eZdZfdZfdZxZS)-test_read_csv_gil_deadlock..MyBytesIOcFtjdt| |SNgMbP?)rrrkreadrmrrms rrz2test_read_csv_gil_deadlock..MyBytesIO.reads JJu 7<& &rLcFtjdt| |Sr)rrrkreadintors rrz6test_read_csv_gil_deadlock..MyBytesIO.readintos JJu 7#T* *rL)rzr{r|rrrorps@r MyBytesIOrs ' + +rLrr)r.rKrrSrTr9rr)rLrr"s rtest_read_csv_gil_deadlockrsd D+BJJ+2Y& ]]2?? + & Yt_ % & && & &s A**A3 )rrz T)Frrrrdecimalrrdrr.r1rr+rrrrr{r runittestrrSpyarrowr9 pyarrow.csvrrrr r r r r rr pyarrow.testsrrr#r%rGrKr_rgrirrrrrABCrr@r%r,r1rrr=rurxTestCaserrrrrr parametrizerrrOrLrrs$ #    444!"  $#,00--$&4n9xN;b0XK#''XKv T {T n- / @,;@,F !5 #7 992 /1B1B *.0A0A):XB  ! ! * ?  ?&rL