K iYddlZddlmZddlmZddlZddlZddlZddlZddl Z ddl Z ddl Z ddlZddlmZmZmZmZdZddZdZdZd ZGd d ej4ZGd d eZGddeZGddee j<ZGddee j<Z Gddee j<Z!e jDjFGddee j<Z$y#e $rdZ YwxYw)N) OrderedDict)Decimal) read_json open_json ReadOptions ParseOptionsc#nKtj}|Ed{|D]}|D] }||z y7wN)stringascii_lowercase)lettersfirstseconds ]/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/pyarrow/tests/test_json.pygenerate_col_namesr%sG$$G! !F&.  !!s 535c tjjdjdd||f}t t j t|}g}|jD]Y}tt||Dcgc]\}}|t|fc}}} |jtj| [|j|j!} |D cgc]+} t#j$| t#j&-} } t"j(j+| |} | | fScc}}wcc} w)N*r)size)type)nprandom RandomStaterandintlist itertoolsislicerTrzipintappendjsondumpsjoinencodepaarrayint64Table from_arrays)num_colsnum_rowslineseparr col_nameslinesrowkvjson_objdatacolcolumnsexpecteds rmake_random_jsonr9.s ))   # + +At8X:N + OCY%%&8&:HEFI Euu+#i:MNACF NO TZZ)*+ <<  % % 'D9<=#rxx"((*-=G=xx##GY7H > O>s D> (0Ec |di|}|j|j||j}|jD]\}}t |||k(rJy)N)protocol)loadsr#HIGHEST_PROTOCOLitemsgetattr)clspickler attr_valuesoptsnew_optsnamevalues rcheck_options_class_picklingrH;sk   D}}W]]44;4L4L+NOH"((*0 ex&%///0c2t}|}|jdkDsJd|_|jdk(sJ|jdusJd|_|jdusJ|dd}|jdk(sJ|jdusJt||ddy)Nri90TFi) block_size use_threads)rBrKrL)rrKrLrH) pickle_modulerArDs rtest_read_optionsrNCs C 5D ??Q  DO ??e ## #   t ## #D   u $$ $ $E 2D ??d "" "   u $$ $ m,0-24rIct}|}|jdusJ|jJd|_|jdusJtjtj dtj g}||_|j|k(sJ|jdk(sJdD]}||_|j|k(rJtjt5d|_dddt|||ddy#1swYxYw) NFTfooinfer)ignoreerrorrQz invalid-valuerR)rBexplicit_schemanewlines_in_valuesunexpected_field_behavior) rrUrTr&schemafieldint32rVpytestraises ValueErrorrH)rMrArDrWrGs rtest_parse_optionsr]Xs C 5D  " "e ++ +    '' '"D  " "d ** * YY 34 5F!D   6 )) )  ) )W 44 4-7).&--6667 z "9)8&9!m1749;CE99s DD ceZdZejdZdZdZdZdZ dZ dZ dZ d Z d Zd Zej"j$d Zej"j$d Zej"j$dZy) BaseTestJSONc t)z :param b: bytes to be parsed :param kwargs: arguments passed on to open the json file :return: b parsed as a single Table )NotImplementedErrorselfbkwargss r read_byteszBaseTestJSON.read_bytests "!rIc|jt|k(sJ|jDcgc]}|jc}|k(sJycc}wr ) num_columnslenr7rF)rctablenamescs r check_nameszBaseTestJSON.check_names}s>  CJ... % .1.%777.sAcd}t}t}||dzfD]}dD]}||_d|_t j t d5|j|||dddtdd D]6}||_|j|||}|jd gd ik(r6Jy#1swYSxYw) Ns{"a": 1} {"a": 2} {"a": 3} FTztry to increase block sizematch read_options parse_options a)) rrrUrKrZr[r\rfrange to_pydict)rcrowsrurvr5rUrKrjs rtest_block_sizeszBaseTestJSON.test_block_sizess."} $ 4%<( AD&3 A"3E 0*+ ']]:)EGAOOD|2?$AA#(2,AJ.8L+ OOD|:G,IE ??,i0@@@@ A A AAAs B66B? chd}|j|}|jddgddgddgdk(sJy) Ns/{"a": 1,"b": 2, "c": 3} {"a": 4,"b": 5, "c": 6}rzrqr{r|ryrdrl)rfr~)rcrrjs rtest_no_newline_at_endz#BaseTestJSON.test_no_newline_at_endsHB% QQQ%    rIc2d}|j|}tjdtjfdtjfdtjfg}|j|k(sJ|j ddgddgd d gd k(sJy) Ns0{"a": 1,"b": 2, "c": 3} {"a": 4,"b": 5, "c": 6} ryrdrlrzrqr{rr|rr)rfr&rWr(r~rcrrjrWs rtest_simple_intszBaseTestJSON.test_simple_intssD%S"((*- "((*- "((*-/0||v%%% QQQ%    rIcbd}|j|}tjdtjfdtjfdtj fdtj fg}|j|k(sJ|jddgdd gd d gd d gdk(sJy)NsQ{"a": 1,"b": 2, "c": "3", "d": false} {"a": 4.0, "b": -5, "c": "foo", "d": true} ryrdrld?@r{3rPFTryrdrlr)rfr&rWfloat64r(r bool_r~rs rtest_simple_variedzBaseTestJSON.test_simple_varieds@%S"**,/ "((*- "))+. "((*-/0||v%%% sRu %    rIc d}|j|}tjdtjfdtjfdtj fdtj fdtjfg}|j|k(sJ|jgdgdgd gd gd d k(sJy) Ns{"a": 1, "b": 2, "c": null, "d": null, "e": null} {"a": null, "b": -5, "c": "foo", "d": null, "e": true} {"a": 4.5, "b": null, "c": "nan", "d": null,"e": false} ryrdrlre)rNg@)r{rN)NrPnan)NNN)NTF)ryrdrlrr) rfr&rWrr(r nullrr~rs rtest_simple_nullszBaseTestJSON.test_simple_nullssM%S"**,/ "((*- "))+. "''), "((*- /0 ||v%%% !%#$ %    rIcd}|j|}tjdtjtjfg}|j|k(sJ|j dggik(sJy)N {"a": []}ry)rfr&rWlist_rr~rs rtest_empty_listszBaseTestJSON.test_empty_listssg%S"((2779"5678||v%%% S2$K///rIcd}|j|}tjg}|j|k(sJ|jdk(sJ|jdk(sJy)Ns{} {} rr{)rfr&rWrhr,rs rtest_empty_rowszBaseTestJSON.test_empty_rowssZ%2||v%%%  A%%%~~"""rIcd}dtdtdtddgi}tjtjtjtj f}|D]f}tj d|ddfg}t|}|j|| }|j |k(sJ|j|k(rfJy) Ns'{"a": 1} {"a": 1.45} {"a": -23.456} {} ry1z1.45z-23.456rwrqrTrv) rr& decimal32 decimal64 decimal128 decimal256rWrrfr~)rcrr8 decimal_types type_factoryrWrDrjs rtest_explicit_schema_decimalz)BaseTestJSON.test_explicit_schema_decimals '#,1CTJ r||R]]BMMR ) 1LYYl1a&8 9:;F7DOODO=E<<6) ))??$0 00  1rIcd}tjdtjfg}t|}|j ||}|jtjdtjfdtj fgk(sJ|j ddgdd gd k(sJt|d }|j ||}|jtjdtjfgk(sJ|j dddgik(sJt|d }tjtjd5|j ||dddy#1swYyxYw)Ns2{"foo": "bar", "num": 0} {"foo": "baz", "num": 1} rPrrnumsbarsbazrrz)rPrrR)rTrVrS"JSON parse error: unexpected fieldrr) r&rWbinaryrrfr(r~rZr[ ArrowInvalid)rcrrWrDrjs r.test_explicit_schema_with_unexpected_behaviourz;BaseTestJSON.test_explicit_schema_with_unexpected_behaviours. BIIK   F3D9||ryy BIIK BHHJ *      F#q6%    F6>@D9||ryy BIIK *      FF#%    F6=? ]]2??!EG 6 OODO 5 6 6 6s E77Fctdd\}}|j|}|j|jk(sJ|j|sJ|j |j k(sJy)Nr{ r+r,)r9rfrWequalsr~)rcr5r8rjs rtest_small_random_jsonz#BaseTestJSON.test_small_random_jsonsf)1rBh%||x...||H%%% H$6$6$8888rIctdd\}}td}|j||}|jdk(sJ|jdk(sJy)Nr{irirKru)r9rrfr,)rcr5r8rurjs rtest_load_large_jsonz!BaseTestJSON.test_load_large_json!sT)1vFh"l; <@~~'''  F***rIc|tdd\}}t}t}||jdfD]}dD]{}||_dD]m}||_|j |||}|j|jk(sJ|j|rK|j|jk(rmJ}y)Nr{drs rp)%rt) r9rrrstriprUrKrfrWrr~) rc data_baser8rurvr5rUrKrjs rtest_stress_block_sizesz$BaseTestJSON.test_stress_block_sizes*s/CH 8"} $  0 0 9: ID&3 I"3E 0".IJ.8L+ OOD|:G,IE <<8??::: <<1$0H4F4F4HHHHI I IrIN)__name__ __module__ __qualname__abcabstractmethodrfrmrrrrrrrrrrZmarknumpyrrrr<rIrr_r_ss""8A,    " (0#1"#6J [[99 [[++ [[IIrIr_ceZdZdZdZdZy)BaseTestJSONReadc N|jtj|fi|Sr )rr& py_bufferrbs rrfzBaseTestJSONRead.read_bytes@ t~~bll1o888rIcPd}dgdgd}tj|}|j|}|j|k(sJtj|j }t jt5|j|dddy#1swYyxYw)N{"a": 1, "b": 2} rzr{ryrd) ioBytesIOrr~StringIOdecoderZr[ TypeError)rcr5 expected_databiorjsios rtest_file_objectz!BaseTestJSONRead.test_file_objectCs$ c, jjs# M111kk$++-( ]]9 % NN3    s BB%c *d}tt|}dddgfddggfddgdgggfddigfd dd did d diigffD]T\}}|j||z| }d |i}|j|k(sJ|j d j dkDrTJy)Ns"{ } rs{"a": 0}rrs{"a": []} {"a": [[1]]}rzs {"a": {}}s {"a": {}} {"a": {"b": {"c": 1}}}rdrlrry)rrirfr~column num_chunks)rc first_rowru next_rowsexpected_pylistrjr8s rtest_reconcile_across_blocksz-BaseTestJSONRead.test_reconcile_across_blocksNs: "c)n= 4) $ D": & '$aSE): ; D": & 1S$K#Qx 1 3 + 4 &IOOI $91=$?E_-H??$0 00<<$//!3 33 4rIN)rrrrfrrr<rIrrr>s9  4rIrcfeZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZy)BaseTestStreamingJSONReadcv|jdt}|j|_t|g|i|S)z Reads the JSON file into memory using pyarrow's open_json json The JSON bytes args Positional arguments to be forwarded to pyarrow's open_json kwargs Keyword arguments to be forwarded to pyarrow's open_json ru) setdefaultrrLr)rcr"argsrerus rrz#BaseTestStreamingJSONRead.open_jsoncs;((G #'#3#3  ////rIc N|jtj|fi|Sr )rr&rrbs r open_bytesz$BaseTestStreamingJSONRead.open_bytesnrrIc|j|k(sJt|}t|t|k(sJt||D]=\}}|j d|j|k(sJ|j |k(r=Jy)NTfull)rWrrirvalidater~)rcreaderexpected_schemarbatchesbatchexpected_batchs r check_readerz&BaseTestStreamingJSONRead.check_readerqs}}///v,7|s=1111%(-%@ 7 !E> NNN %<"} "$ ))c288:%6$78!1 M}}///%%'113 !8    ]] OOE  %  " " $  % ]]= ) %  " " $ % %  % %  % %-C9D9DDcd}t}d|_tjtj d5|j ||dddy#1swYyxYw)Ns{"n": } {"n": 10000}!JSON parse error: Invalid value.*rrrrrcbad_first_blockrus rtest_bad_first_parsez.BaseTestStreamingJSONRead.test_bad_first_parsesU2"} "$  ]]2??!DF H OOO,O G H H Hrcd}t}d|_tjtj d5|j ||dddy#1swYyxYw)Ns"{ }{"n": } {"n": 10000}rrrrrrrs r!test_bad_middle_parse_after_emptyz;BaseTestStreamingJSONRead.test_bad_middle_parse_after_emptysU@"} "$  ]]2??!DF H OOO,O G H H Hrc"d}t}d|_tjdtjfg}|j ||}|j|k(sJ|j jddgik(sJtjtjd5|j dddtjt5|j dddy#1swY)) "((*  "'') %  #b1 $_E ug  %'!) &/M?CrIc~d}tjdtjfdtjfg}dgdgd}t d}t d }|j ||| }|j|k(sJ|jj|k(sJtjtjd 5|jdddtjdtjfdtjfdtjfg}ddgddgddgd}t d}|j ||| }|j|k(sJ|jj|k(sJtjtjd 5|jdddtjdtjfdtjfdtjfdtjfg}gdgdgdgdd}t d}|j ||| }|j|k(sJ|jj|k(sJy#1swYxYw#1swYxYw)NsI{"a": 0, "b": "foo" } {"a": 1, "c": true } {"a": 2, "d": 4.0}ryrdrrPr rrQ)rVrtrrrrlrzTr@r)rrzr{)rPNN)NTN)NNrr`)r&rWr(rrrrrr~rZr[rrr)rcrrrrurvrs rtest_inferencez(BaseTestStreamingJSONRead.test_inferences3)) "((*  "'') %   !c0 "b1 $wG  %'!)}}///%%'113}DDD ]]2??!EG %  " " $ %)) "((*  "'')  "((* %   !!fE4=d|L "b1 L/<!>}}///%%'113}DDD ]]2??!EG %  " " $ %)) "((*  "'')  "((*  "**,  %   (.A07HJ "b1 L/<!>}}///%%'113}DDD? % % % %sJ&J3&J03J<N)rrrrrrrfrrrrr r rrrrr$r<rIrrrbsS 0977DH%(HH%*%*%*%2D$0ErIrceZdZdZy)TestSerialJSONReadc|jdt}d|_t|i|}|j d|S)NruFTrrrrLrrrcrrerurjs rrzTestSerialJSONRead.read_jsonSs@((G #(  4*6* D! rINrrrrr<rIrr&r&QrIr&ceZdZdZy)TestParallelJSONReadc|jdt}d|_t|i|}|j d|S)NruTrr(r)s rrzTestParallelJSONRead.read_json]s@((G #'  4*6* D! rINr*r<rIrr-r-[r+rIr-ceZdZdZy)TestSerialStreamingJSONReadFNrrrrLr<rIrr0r0esKrIr0ceZdZdZy)TestThreadedStreamingJSONReadTNr1r<rIrr3r3js KrIr3)r{rz )%r collectionsrdecimalrrrr"r unittestrr ImportErrorrZpyarrowr& pyarrow.jsonrrrrrr9rHrNr]ABCr_rrTestCaser&r-r0r threadingr3r<rIrr=s$ #  HH! 04*E6HI377HIV!4|!4HlE lE^)8+<+<+X->->";X=N=N $=x?P?P] BsC++C54C5