gL iQdZddlmZddlmZddlZddlZddlZddlmZddlZddl Z ddl Z ddl m Z ddl Z ddlZddlZddlmZddlZddlZddlZddlZddlmZmZdd lmZdd lmZddlmcm Z!ddl"Z#dd l"m$Z$m%Z%m&Z&m'Z'ddl(m)Z*dd l+m,Z,ddl-m cm.Z/dd l0m1Z1m2Z2dZ3ejhjkddde6de7de jpdedgde7djsdde7djsdddddejtdjwddejtdjwddejtdjwdddddddfg dZdZ?dWd Z@ejhjkd!eje>d"#eje#jd$#ejeje#je j%d&#ejeje#jd'%d(#ejeje#jd)%d*#gejhjkd+e#je>gejhjkd,e@e,d-ZDd.ZEd/ZFejd0ZHGd1d2ZIGd3d4ZJejhjkd5d6gd7e%gd8eK9fd:e#je%gd8eK9e%gd;eK9gejgd<ejgd<gfgd=ZMd>ZNejhjkd?gd@dAZOdBZPGdCdDejZRdEZSdFZTdGZUdHZVejhjkdIdJdKdIej dLdMdMdNgdOZWdPZXdQZYe!jdRZ[ejhjkdSe je jgdTZ]dUZ^dVZ_y)Xa< manage legacy pickle tests How to add pickle tests: 1. Install pandas version intended to output the pickle. 2. Execute "generate_legacy_storage_files.py" to create the pickle. $ python generate_legacy_storage_files.py pickle 3. Move the created pickle to "data/legacy_pickle/" directory. ) annotations)arrayN)partial)Path)Any) get_lzma_fileis_platform_little_endian)import_optional_dependency)flatten_buffer) DataFrameIndexSeries period_range)create_pickle_data)DayMonthEndcbt|trtj||y|j drtj ||y|dk(r/|t jur|t jus J||k(sJyyttd|dtj}|||y)Nsp_ timestampassert__equal) isinstancer tmassert_index_equal startswith assert_equalpdNaTgetattrassert_almost_equal)resultexpectedtyp comparators a/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/pandas/tests/io/test_pickle.pycompare_elementr&>s(E" h/ ~~e )  rvv RVV# ##X% %%$R73%v!68N8NO 68$datas123s123456IB)r-r,r, )r-C)orderFcPt|}t|jd}||k(sJt|tt fr||usJyt|trM|j dk(sJ|jdk(sJ|jsJ|j|jfk(sJyy)NAr+r.) r memoryviewtobytesrbytes bytearrayndimformat contiguousshapenbytes)r(r!r"s r%test_flatten_bufferr?Ts"D !F$'',H X  $ *+~~ FJ '{{a}}###    || //// (r'c tstjdttj j dD]}||}tj|}|jD]x\}}|jD]^\}}|||}|dk(r_|dk(rYtj|||jj|jjk(sJ|jjjrJtj|dkD|dkD|jj}|tdztdk(sJ|tj dz} t#| tj sJ| tj dd k(sJ|tj d z} t#| tj sJ| tj dd k(rrJ|d k(r|d k(rtj$||t#|jt&sJ|jt'k(sJ|j(dk(sJtj$|j+d|j+d|dk(r|dvrtj||1|dk(r|dvrtj,||Rt/|||a{y)Nz"known failure on non-little endianzdata/legacy_pickle/*/*.p*kl*seriestsrr+r,)hours)daysrC) nanoseconds)rDrEindexperiodM)dt_tzcatframe) dt_mixed_tzs cat_onecol cat_and_float)r pytestskipr__file__parentglobr read_pickleitemsrassert_series_equalrFfreq normalizer Timedeltarrrfreqstrshiftassert_frame_equalr&) datapath legacy_pickler(r#dvdtr!r"rWress r% test_picklesrbqsw $ & 89h..334RS*;  / ~~m,zz|% ;GC hhj$ ; F9R=(?rTz**68<!<<,,0C0CCCC%||00::::**6A:x!|D"<<,,D#a&=CF222A!66C%c2<<888",,AQ"????!!<>S000))&,,q/8>>!;LMH_/?)?**68<G^/) ))&(;#FHc:I$ ;% ; *;r'cvt|d5}tj||ddddy#1swYyxYwNwbprotocolopenpickledumpobjpathfhs r%python_picklerrqs3 dD *R Cb)***/8ct|d5}|jdtj|cdddS#1swYyxYw)Nrbr)rjseekrkload)rorps r%python_unpicklerrws9 dD R  {{2s &=Ac|jDcgc]\}}|jD]}||f c}}}Scc}}}w)zFlatten create_pickle_data)rUvalues)r(r#examplesexamples r%flattenr|sS"ZZ\   C(   g   s#= pickle_writerpython)idpandas_proto_defaultrgpandas_proto_highestr0pandas_proto_4pandas_proto_5writerz typ, expectedctj5}|||tj|}t |||t |}t |||t |d5}||||jddddt |d5}tj|}|jddddt |||dddy#1swY[xYw#1swY+xYw#1swYyxYw)Nremoderrt)r ensure_cleanrrTr&rwrjru)r#r"r}rror!handles r%test_round_trip_currentrs&  /dh%%#.!$'#.$T " f 8T " KKN $T " f^^F+F KKN  #.%//    //s<AC3&CC3'C'=C3C$ C3'C0 ,C33C<c ~tdtjdjdzt t dt t tdDcgc]}d| c}t }tj|jtj}tj||ycc}w N皙?xr0ABCDdtyperi-columnsrF)r nparangereshaper listobjectrangerround_trip_pathlib to_picklerrTr\idfr!s r%test_pickle_path_pathlibrs  biin$$W--d6l&1uRy1!r!X1@ B  " "2<< @F"f%2 B:c ~tdtjdjdzt t dt t tdDcgc]}d| c}t }tj|jtj}tj||ycc}wr)r rrrr rrrrround_trip_localpathrrrTr\rs r%test_pickle_path_localpathrs  biin$$W--d6l&1uRy1!r!X1@ B  $ $R\\2>> BF"f%2rc2dtjdS)N__z __.pickle)uuiduuid4r'r%get_random_pathrs  ~Y ''r'ceZdZejZdZdZejjdgddZ dZ dZ dZy ) TestCompressionc|tj||y|dk(rtj|d}nE|dk(rt j |d}n(|dk(r_t j|dt j5}|j|tjj|dddn|dk(rwt|d5}tj|d5}|j|tjj|}|j||ddddddnH|d k(rt!|d}n1|d k(rt#d j|d }nd |}t%||dvr@t|d5}5|j|j'ddddddyy#1swYNxYw#1swYxYw#1swYfxYw#1swY7xYw#1swYyxYw)Ngzipwbz2zip compressiontarrtrxzzstd zstandardrezUnrecognized compression type: )rr)shutilcopyfilerrjrBZ2FilezipfileZipFile ZIP_DEFLATEDwriteosrobasenametarfile gettarinfoaddfilerr ValueErrorread) selfsrc_path dest_pathrfrprtarinfomsgs r% compress_filezTestCompression.compress_files   OOHi 0  &  )S)A E ! Is+A E !CW=Q=QR >VW"''"2"28"<= > > E !h% -\\)#6-#!nnXrww7G7G7QRGKK,- - -D   3/A F "*;7< >-- - -'' ' 'sU?0G G!AG #G G1 G%/G1G  G GG"%G. *G11G:c |}|dz}|dz}tj|5}tj|5}tdtjdj dzt tdtt tdDcgc]}d | c}t } | j|| tj|| 5} t|d 5} | j| jddddddtj |d } tj"| | ddddddycc}w#1swYTxYw#1swYXxYw#1swY/xYw#1swYyxYw) N .compressed.rawrrrrrrrrrre)rrr rrrr rrrrdecompress_filerjrrrrTr\) rrrbasepath1path2p1p2rrrrpdf2s r%test_write_explicitz#TestCompression.test_write_explicitsJ}$v  __U # +r2??5+A +Rbiin,,W55d6l&9uRy9!r!X9HB LLL 5##BK@ 'A"d^'rHHQVVX&' ' ..6C  ! !"c *% + + +:'' ' ' + + + +seE;AE/ E  < < < F  E> +F 35F(F"9F>F F  F FF F""F+c ^|}|dz}|dz}tj|5}tj|5}tdtjdj dzt tdtt tdDcgc]}d | c}t } | j|d |j||| tj|| } tj| | ddddddycc}w#1swYxYw#1swYyxYw) Nrrrrrrrrrrr)rrr rrrr rrrrrrrTr\) rrrrrrrrrrrs r%test_read_explicitz"TestCompression.test_read_explicit\sv }$ __U # +r2??5+A +Rbiin,,W55d6l&9uRy9!r!X9HB LLL .   r2;  ?..=C  ! !"c * + + +:  + + + +s6D#AD DA&DD#DD D##D,c |}|dz}||z}|jj|j}tj|5}tj|5}t dt jdjdzttdtttdD cgc]} d|  c} t } | j|d |j||| tj |} tj"| | ddddddycc} w#1swYxYw#1swYyxYw) Nrrrrrrrrrr)rrrrrr rrrr rrrrrrrTr\) rrrrrrrrrrrrs r%test_read_inferzTestCompression.test_read_inferrs#v &44889N9N9PQ __U # +r2??5+A +Rbiin,,W55d6l&9uRy9!r!X9HB LLL .   r2;  ?..$C  ! !"c * + + +:  + + + +s8 E !AD>8 D9A$D>(E 9D>>E E  EN)__name__ __module__ __qualname__icomextension_to_compressionrrrrOmark parametrizerrrrrr'r%rrsN $ = ='<+2 [[],EF<G<+4+,+r'rcTeZdZejj dgddZy) TestProtocolrh)rfrr+r,c tj|5}tdtjdj dzt tdtt tdDcgc]}d| c}t}|j|| tj|}tj||dddycc}w#1swYyxYw) Nrrrrrrrrrg)rrr rrrr rrrrrrTr\)rrhrrorrrs r% test_readzTestProtocol.test_reads ___ - +biin,,W55d6l&9uRy9!r!X9HB LLL 1..&C  ! !"c * + +:  + +sAC- C9ACCC N)rrrrOrrrrr'r%rrs% [[Z7 +8 +r'r pickle_fileexcolsz test_py27.pkl)abcrztest_mi_py27.pkl)r5r.r1)rr+r,c|ddd|}tj|}tj|j|y)Nior(rk)rrTrrr)r]rrrors r%test_unicode_decode_errorrs8& D&(K 8D  B"**f-r'c Dtj5}tdtjdj dzt tdtt tdDcgc]}d| c}t}t|d 5}|j|dddt|d 5}tj|}dddtj|dddycc}w#1swYWxYw#1swY9xYw#1swYyxYw) Nrrrrrrrrrert)rrr rrrr rrrrjrrrTr\)rorrrpr!s r%test_pickle_buffer_roundtriprs   *d  "))C.((1 1$v,f5595a2aS5VD  $   LL   $  (^^B'F ( b&) * *6   ( ( * *sNAD, C98DC>(D<D D9D>D D D DDmockurl)zhttp://url.comzftp://test.comzhttp://gzip.comc d}Gddtj5fd}tdtjdj dzt tdt t td Dcgc]}d | c}t  }|||jd |tj|}tj||dddycc}w#1swYyxYw)Ncvt|d5}tj||ddddy#1swYyxYwrdrirms r%rqz3test_pickle_generalurl_read..python_picklers3 $  . KKR" - . . .rrc,eZdZddZdZdZdZdZy)5test_pickle_generalurl_read..MockReadResponsecTt|d|_d|vr ddi|_yddi|_y)NrtrzContent-Encodingr)rjfileheaders)rros r%__init__z>test_pickle_generalurl_read..MockReadResponse.__init__s/T4(DI~ 2F; 2B7 r'c|SNrrs r% __enter__z?test_pickle_generalurl_read..MockReadResponse.__enter__sKr'c$|jyr )close)rargss r%__exit__z>test_pickle_generalurl_read..MockReadResponse.__exit__s JJLr'c6|jjSr )r rrs r%rz:test_pickle_generalurl_read..MockReadResponse.reads99>># #r'c6|jjSr )r rrs r%rz;test_pickle_generalurl_read..MockReadResponse.closes99??$ $r'Nreturnr)rrrr rrrrrr'r%MockReadResponsers 8   $ %r'rcSr r)rkwargsrros r%mock_urlopen_readz6test_pickle_generalurl_read..mock_urlopen_reads#D) )r'rrrrrrrrzurllib.request.urlopen)rrr rrrr rrrsetattrrrTr\) monkeypatchrrqrrrr!rros @@r%test_pickle_generalurl_readrs.%%(   *d * "))C.((1 1$v,f5595a2aS5VD  r4 46GH( b&) * *6 * *sAC3 C. AC3.C33C<c tjdtj5d}t dt j djdzttdtttdDcgc]}d | c}t }|j|tj|}tj||dddycc}w#1swYyxYw) Nfsspeczmemory://mockfilerrrrrrrr)rO importorskiprrr rrrr rrrrrrTr\)rrrr!s r%test_pickle_fsspec_roundtripr"s !   *%  "))C.((1 1$v,f5595a2aS5VD  W( b&) * * 6 * *sAC+ C&AC+&C++C4ceZdZddZy)MyTzcyr rrs r%r z MyTz.__init__ s r'Nr)rrrr rr'r%r$r$ s r'r$ctttf}tj|}tj |d|dt |dtsJy)Nrrr+)rrr$rround_trip_picklerVr)r"r!s r%test_read_pickle_with_subclassr(sOF#TV+H  ! !( +F6!9hqk2 fQi && &r'c tdtjdjdzt t dt t tdDcgc]}d| c}t }tj5}|j|| t|j}d d d tj}|j|| |jd |j!k(s|d vsJt#j$|| }|jd tj&||y cc}w#1swYxYw) ze Read/write from binary file-objects w/wo compression. GH 26237, GH 29054, and GH 29570 rrrrrrrrrNr)rrr)r rrrr rrrrrrr read_bytesrBytesIOrugetvaluerrTr\)rrrro referencebufferread_dfs r%%test_pickle_binary_object_compressionr0s  biin$$W--d6l&1uRy1!r!X1@ B  ,d T{ 3J))+ , ZZ\FLL[L1 KKN ??  )[._test_roundtrip=s"((/  eY/r')T)/multiindex_year_month_day_dataframe_random_data multiindex_dataframe_random_dataymdrKr4s r%+test_pickle_dataframe_with_multilevel_indexr96s= :C ,E0EEGGCCEEr'ctddd}ttjj dj t ||}tj|}|jjdk(sJy)Nz1/1/2011z1/1/2012rH)rWr,) rrrrandom default_rngstandard_normallenrr'rFrZ)prngrBnew_tss r%"test_pickle_timeseries_periodindexrAGsb  JS 9D  %%a(88TCT JB  ! !" %F <<  3 && &r'namei gH@i )r+r,ctjttjdtj |}|j |k(sJy)N r)rB)rr'rrrfloat64rB)rBr3s r%test_pickle_preserve_namerGOs=$$VBIIb ,KRV%WXI >>T !! !r'cZtj|}tj||yr rr'rV)datetime_seriesunp_tss r%test_pickle_datetimesrLWs"  ! !/ 2F6?3r'cZtj|}tj||yr rI) string_series unp_seriess r%test_pickle_stringsrP\s"%%m4J:}5r'cdttdjdjdg}t j |}|j jdjdk(sJ|j jdjdk(sJt j|dg|y)Nabccategoryrr+)r+T) rrastypeilocrr'_mgrblocksr:r=rV)serras r% test_pickle_preserves_block_ndimrYas e  $ $Z 0 5 5qc :C  s #C 88??1  " "a '' ' 88??1  # #t ++ +3v;,r'rhcttd}tjt |j ||t t j|}tj||y)Ni)rhrr) r rrrrrrrTr\)rhrrr!s r%%test_pickle_big_dataframe_compressionr[nsR 5= !B  " " x[IK8F"f%r'c|ttjdddd}t|d5}t j |}dddt gg}tj|y#1swY-xYw)Nr(r^z1.2.4zempty_frame_v1_2_4-GH#42345.pklrt)rFr) rrQrRrjrkrvr rr\)r]rofdrr"s r%#test_pickle_frame_v124_unpickle_130r^yss  X)  D dD R [[_r2.H"h' s A22A;ctdgdi}d}tjt|5t j }|j |ddddy#1swYyxYw)Nrr*zpStarting with pandas version 3.0 all arguments of to_pickle except for the argument 'path' will be keyword-only.rinfer)r rassert_produces_warning FutureWarningrr+r)rrr.s r% test_pickle_pos_args_deprecationrcs[ C# $B 1 # #M =& VW%&&&s 'AA&)r(dictrzlist[tuple[str, Any]])`__doc__ __future__rrrdatetime functoolsrrrrpathlibrrkrrtypingrrrnumpyrrO pandas.compatrr pandas.compat._optionalr pandas.compat.compressorsr pandas.util._test_decoratorsutil_test_decoratorstdpandasrr r rrpandas._testing_testingr-pandas.tests.io.generate_legacy_storage_filesrpandas.io.commoncommonrpandas.tseries.offsetsrrr&rrr9r6 PickleBuffercastrrr?rbrqrwr|paramrHIGHEST_PROTOCOLrrrfixturerrrr MultiIndexrrrr"tzinfor$r(r0r9rArGrLrPskip_array_manager_invalid_testrYDEFAULT_PROTOCOLr[r^rcrr'r%rs] #     ?4)) L%, &6F# c99""3/9""3/!4 " fC0 " fC0 " fC0CaC8  0! 0/;d*   ^1 R\\&<= I  bllV5L5L M%   &Y&&r||a@EUV I  bllQ 7   BLL.#AB'2D2F*GH/IC!$/,&&((J+J+d + +H %v>?  BMM/8/8)$hbhhy&9:     .! . *& F&*&*R * 8?? ''>"' S%!2!2!24R!@& I"" 4 6 ## -$ -f&=&=v?V?V%WX&Y& ( &r'