K iRddlZddlZddlmZddlZddlZddlZddlZddlZ ddlm Z ddl m Z ejjZ ddlmZddlZej*ddZd Zdd Zdd Zejj$ejj5d gd dZdZdZdZdZdZ dZ!dZ"ejjFdZ$dZ%dZ&dZ'dZ(y#e$rYwxYw)N)Path)fs)util)assert_frame_equalmodule)scopec |dz S)Norc) base_datadirs \/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/pyarrow/tests/test_orc.pydatadirr,s % c |D]}||}||}|dk(rs|Dcgc]}|D]}|jddhk( c}}rG||j}t|D] \}}|Dcgc] }|d|dfc}||<"|||<|dj} t | t j rt j|}nt | t jr|jj}n| tjurdgt|z} tt||D]q\}\}} t j| r|jj } d| z} tjt#| | zj%| | |<st j&| }|||<ycc}}wcc}w)z_ Fix type of expected values (as read from JSON) according to actual ORC datatype. mapkeyvaluerN )keyscopy enumerate __class__ issubclassdatetimepd to_datetimedatedtdecimalDecimallenzipisnullas_tupleexponentroundscalebSeries) actual_cols expected_colsnameexpectedactualmdcolitypconverted_decimalsvexpfactors r fix_example_valuesr71s ' &T" EM7?K!KAeW--K-K%**,C!(+ =1:;rc ddlm}|j|}|j}t |t j sJ|jtj|j}t|jt|jk(sJ|jj|js|j|j}|r t||t!||d}t#|j$D]]}|j'|} t!tj| j|||t)| z|t)| z }_||j*k(sJy)zC Check a ORC file against the expected columns dictionary. rr columns)r>r?N)pyarrowr ORCFileread isinstancepaTablevalidater DataFrame to_pydictsetrDequalsreindexr7r@rangenstripes read_striper!nrows) orc_pathr=need_fixr orc_filetabler<json_posr1batchs r check_example_filer[]s>{{8$H MMOE eRXX && & NN\\%//+ ,F {"" #s6>>': :: : >> !4!4 5!))&..)A 6;/-H 8$$ %$$Q'R\\%//*;<(#+"*SZ"7 9 CJ  x~~ %% %rfilename)zTestOrcFile.test1.orcTestOrcFile.testDate1900.orcz decimal.orcc||z }tjt|jdd}t ||dy)z Check a ORC file example against the equivalent JSON file, as given in the Apache ORC repository (the JSON file has one JSON object per line, corresponding to one row in the ORC file). z.jsn.gzT)lines)rVN)r read_jsonstr with_suffixr[)r\rpathrXs r test_example_using_jsonrds: X D LLT--i89 FEtUT2rc||dz }dt|d}tjtjd|gdy)Nr]zif 1: import os os.environ['TZDIR'] = '/tmp/non_existent' from pyarrow import orc try: orc_file = orc.ORCFile(z) orc_file.read() except Exception as e: assert "time zone database" in str(e).lower(), e else: assert False, "Should have raised exception" -cTcheck)ra subprocessrunsys executable)rrccodes r test_timezone_database_absentrnsF 3 3D$ %(I=1 DNNCNND$/t ed; T\I%%%6 3 3D"5zn- !$D }- DNNCNND$/t<% > 8<==>sB<< C'C""C'cddlm}|j|dz j}|jdk(sJt j dt jfdt jfdt jfdt jfdt jfd t jfd t jfd t jfd t jfd t j dt j"t j dt jfd t jfgfgfdt j"t j dt jfd t jfgfdt j$t jt j dt jfd t jfgfg }|j |k(sJy)NrrBzTestOrcFile.emptyFile.orcboolean1byte1short1int1long1float1double1bytes1string1middlelistr)rEr rFrGnum_rowsrIschemabool_int8int16int32int64float32float64binarystringstructlist_map_)rr rXexpected_schemas r test_orcfile_emptyrs KK"== > C C EE >>Q  ii RXXZ "'') 288:  "((* 2::< BJJL! 299; BIIK 299bhh FBHHJ/%ryy{35678    II +!299;/1 2     FBHHJ#7$-ryy{#;#=> '!O0 <>$'L << %% %>>$+L %% %% % (( (( (>>$ >:L %% %% % (( (( (rcdddlm}ddlm}|}t j gd}t j gd}t j ||d}|j|||jd|j|}|j}|j|sJy)NrrB)BytesIOrrr) rEr iorrIrrXrseekrFrGrO)r rbufrrrXrWrs r test_bytesio_readwriters )C #$A -.A HHq!, -EOOE3HHQK{{3H==?L << %% %rcddlm}tj}tjgd}tjgd}tj ||d}|j ||tj|j}|j|}|j}|j|sJ|jdk(sJ|jdk(sJ|jdk(sJ|jd k(sJtj}t!j"t$5|j ||dddtj|j}|j|}|j}|j|sJ|jdk(sJ|jdk(sJ|jdk(sJ|jd k(sJy#1swYxYw) NrrBrrr UNCOMPRESSEDz0.12i'rEr rIBufferOutputStreamrrXr BufferReadergetvaluerFrGrO compression file_versionrow_index_stridecompression_sizervwarns FutureWarningr buffer_output_streamrrrX buffer_readerrWrs r test_buffer_readwriter"s002 #$A -.A HHq!, -EOOE/0OO$8$A$A$CDM{{=)H==?L << %% %   > 11 1  F ** *  $ $ -- -  $ $ -- -002 m $5 ,e45OO$8$A$A$CDM{{=)H==?L << %% %   > 11 1  F ** *  $ $ -- -  $ $ -- -55s ;G22G;c ddlm}tj}tjgd}tjgd}tj ||d}|j ||dddd tj|j}|j|}|j}|j|sJ|jd k(sJ|jdk(sJ|jdk(sJ|jd k(sJtj}t!j"t$5|j ||d dd d dddtj|j}|j|}|j}|j|sJ|jdk(sJ|jdk(sJ|jd k(sJ|jd k(sJy#1swYxYw)NrrBrrrsnappyz0.11ir)rrrcompression_block_sizeSNAPPY uncompressedi Nrrrs r 'test_buffer_readwrite_with_writeoptionsrCs002 #$A -.A HHq!, -EOO $ OO$8$A$A$CDM{{=)H==?L << %% %   8 ++ +  F ** *  $ $ ,, ,  $ $ -- -002 m $   &"#(   OO$8$A$A$CDM{{=)H==?L << %% %   > 11 1  F ** *  $ $ -- -  $ $ -- -#  s G<<HcNddlm}tj}tjgd}tj d|i}t jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||d dddt jt5|j||d dddt jt5|j||d dddt jt5|j||d dddt jt5|j||d dddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||ddddt jt5|j||d !dddt jt5|j||dd"g!dddt jt5|j||hd#!dddt jt5|j||d$dddt jt5|j||d%$dddt jt5|j||d&$dddy#1swY-xYw#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swY`xYw#1swY7xYw#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swYjxYw#1swYAxYw#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swYtxYw#1swYKxYw#1swY"xYw#1swYxYw#1swYxYw#1swYxYw#1swY~xYw#1swYUxYw#1swY*xYw#1swYxYw#1swYxYw#1swYxYw#1swYyxYw)'NrrBrr) batch_sizeigR@gp= ף?)rz1.1) stripe_sizeipgG@)rnonezlid)ri8gR"@)compression_strategynolarge)rigGz0@cat)padding_tolerancearrow)dictionary_key_size_thresholdg333333?g r)bloom_filter_columnsgffffff?>rr)bloom_filter_fppg?g) rEr rIrrrXrvraises ValueErrorr TypeError)r rrrXs r +test_buffer_readwrite_with_bad_writeoptionsrsse002 #$A HHgq\ "E z "        z "        z "        z "        z "        z "        z "        z "        y !        z "       z "        z "    #$    z "    #'    z "    #*    y !    !"    z "    !%   z "    !(    z "        z "    !    z "    $    z "    #    z "    *1   z "    *-   z "    *.    z "    !)    z "    "#S    z "    !+    z "    $    z "         z "    !    g                                                            sh"Z"Z/Z<[ :[0[#&[0[=\ \>\$4\1*\> ] ] ]%]28]?.^ $^^&^3_<_ 2_(_' _4```"Z,/Z9<[ [[ #[-0[:=\ \\!$\.1\;>] ]]"%]/2]<?^  ^^#&^03^=_  __$'_14_>` ``$c ddlm}tjdtj}tjdtj |g}tjdtj tjdtjtjdtj tjdtj |tjd tjgtjd tjtjdtj tjd tjtjd tjgtjd tjg}dgd dggddiddgddddddggdgg}tj|tj|}t|dz }|j|||j|}|j} | j|sJ|jdd g} | j|jdd gsJ|jgd} | j|jgdsJ|jdg} tjddddiigi} | j| sJ|jdg}tjdd digi}|j|sJ|jgd}|j|jddgsJ|jdg}tjd d did diggi}|j|sJ|jddg} | j|jdd gsJ|jgd } | j|jgd!sJt!j"t$5|jd"gdddt!j"t&5|jdgdddy#1swY?xYw#1swYyxYw)#NrrBinnerrbasicritemrinner2 list-structinner1basic2rrrr)rr)rr )rrrC)rrrstruct.middle.inner struct.inner2)rrrzlist-struct.inner1r)rrrwrong)rEr rIfieldrrrrrXrrarrFrGrOselectrvrIOErrorr)tempdirr rrfieldsarrsrXrcrWresult1result2result3result4 expected4result5 expected5result6result7 expected7s r test_column_selectionr Us HHWbhhj )E XXh 5' 2 3F "((*%  BHHRXXfbhhj9:   bii(BHHJ)G HI   288BHH HHXrxxz2HHXrxxz2#%  288:&!F& q!fXGQ<1=>! $a&@ ABQC ID HHT"))F"3 4E w# $DOOE4 {{4 HmmoG >>%  mmWh$7m8G >>%,,':; << <mm$@mAG >>%,,'CD EE Emm%:$;m?@I >>) $$ $mm_$5m6G(h]O45I >>) $$ $mm@G >>%,,'9: ;; ;mm%9$:m;G-Ha=8Q-*H)IJKI >>) $$ $mmQFm+G >>%,,':; << <mmIm.G >>%,,'HI JJ J w )  wi () z "# qc "## )) ##s4S)SSScddlm}t|dz }|j|5}t j t 5|jddddddy#1swYxYw#1swYyxYw)NrrBr)rEr ra ORCWriterrvrAttributeErrortest)rr rcwriters r test_wrong_usage_orc_writerrsf w# $D t  ]]> *  KKM   s#A.A"A."A+ 'A..A7cLddlm}t|dz }tjgd}tjgd}tj ||d}t jtj5|j||dddy#1swYyxYw)NrrBrr)NNNNr) rEr rarIrrXrvrArrowNotImplementedErrorr)rr rcrrrXs r test_orc_writer_with_null_arraysrsz w# $D #$A )*A HHq!, -E r22 3% t$%%%s >BB#)NN)F))rrpathlibrrxrirkrvrErIr pyarrow.testsrmarkr pytestmarkpandas.testingrpandasr ImportErrorfixturerr7r@r[ parametrizerdrnrrrrrrrrrr rrr rr r s$   [[__  1 h   #'L? "&J& 3  3=(=>+B 0)& &.B,.,.^_ DL#^%K  s C//C76C7