;jkddlZddlmZddlmZddlZddlmZddlZddl m Z ddl Z ddl Z ddl Z ddlZ ddlZn #e$rdZYnwxYwddlZddlmZmZmZiaiaiadZdZd Zd Zd Zd Z d.d Z!dZ"dZ#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+d.dZ, d/dZ-dZ.d0dZ/dZ0 d1d Z1hd!Z2d"Z3d#Z4d$Z5d.d%Z6 d.d&Z7d'Z8d(Z9d)Z:d*Z;d+ZdS)2N)Sequence)futures)deepcopy) zip_longest) _pandas_api frombytesis_threading_enabledc "tstitjjdtjjdtjjdtjjdtjjdtjj dtjj dtjj dtjj d tjj d tjjd tjjd tjjd tjjdtjjdtjjdtjjdtjjdtjjdtjjditS)Nemptyboolint8int16int32int64uint8uint16uint32uint64float16float32float64datetimebytesunicode)_logical_type_mapupdatepalibType_NA Type_BOOL Type_INT8 Type_INT16 Type_INT32 Type_INT64 Type_UINT8 Type_UINT16 Type_UINT32 Type_UINT64Type_HALF_FLOAT Type_FLOAT Type_DOUBLE Type_DATE32 Type_DATE64 Type_TIME32 Type_TIME64 Type_BINARYType_FIXED_SIZE_BINARY Type_STRINGGC:\PYTHON\MyICR_Suite\python\Lib\site-packages\pyarrow/pandas_compat.pyget_logical_type_mapr7.sw    " FNG" F f"  F f"  F w " F w " F w "  F w"  F "  F "  F "  F "I"  F y"  F  "  F "  F " F !" " F #" $ F  F )7 F  )" "    , r5ct} ||jS#t$rt|tjjrYdSt|tjjrdt|j dcYSt|tjj r |j dndcYStj |rYdSYdSwxYw)N categoricalzlist[] datetimetzdatetimedecimalobject)r7idKeyError isinstancerrDictionaryTypeListTypeget_logical_type value_type TimestampTypetztypes is_decimal) arrow_typelogical_type_maps r6rDrDKs+--  ..     j"&"7 8 8  ==  BFO 4 4 E+J,ABBEEE E E E  BF$8 9 9 #-=#<<<* L L L X  , , 99xx s! )C 7C*C.CCctsttjdtjdtjdtjdtjdtjdtj dtj dtj d tj d tj d d d tjdtjditS)Nr r rrrrrrrrr datetime64[D]rstringr)_numpy_logical_type_maprnpbool_r rrrrrrrrrstr_bytes_r4r5r6get_numpy_logical_type_maprT\s " && Hf GV Hg Hg Hg Hg Ix Ix Ix J J V GX Iw(    #"r5cLt} ||jjS#t$rwt |jdrYdSt |jdrt |jcYStj|}|dkrYdS|cYSwxYw)NrGr; datetime64rNr) rTdtypetyper@hasattrstr startswithr infer_dtype)pandas_collectionnumpy_logical_type_mapresults r6get_logical_type_from_numpyr`rs799 %&7&=&BCC     $*D 1 1 <<  & ' ' 2 2< @ @ 0(.// / / /():;; X  99  s!"B#;B#B#B#"B#cz|j}t|dkrKt|d|}|Jt|j|jd}t|jj}nSt|dr2dtj |j i}d|j d}nd}t|}||fS)Ncategorycat)num_categoriesorderedrGtimezonez datetime64[r:) rWrZgetattrlen categoriesrecodesrYrrtzinfo_to_stringrGunit)columnrWcatsmetadataphysical_dtypes r6get_extension_dtype_inforqs LE 5zzZvuf--!$/22|  TZ-..   $ 7 7 A AB4uz444U 8 ##r5ct|}t|\}}|dkr|j|jd}d}|et |t rt j|sNz)Column name must be a string. Got column z of type name field_name pandas_type numpy_typero) rDrqrsrtrAfloatrPisnanrZ TypeErrorrX__name__)rmrvrJrw logical_type string_dtypeextra_metadatas r6get_column_metadatars"$J//L#;F#C#C L.y  #-%    D%(( -/Xd^^ 4%%  % % %Dzz" % %   j# & &==D,<,<(=(=== & #""   r5c D| d|D}dt||D}t|} t|} |d| | z } || | z d} g} t|||| D]/\}}}}t||||}| |0g}|dur>g}t|| D]}\\}}}|j4t |jt s||jt|t|j||}||~t|dkr tj d|dtd g}t|j d |j g}t|j d |j jg}t||D]*\}}t||}||+ngx}x}}t|d r|jni} t!j|n9#t$$r,}i}tj d|dtd Yd}~nd}~wwxYwdt!j||| |z|dt&jdt*jddiS)aReturns a dictionary containing enough metadata to reconstruct a pandas DataFrame as an Arrow Table, including index columns. Parameters ---------- columns_to_convert : list[pd.Series] df : pandas.DataFrame column_names : list[str | None] column_field_names: list[str] index_levels : List[pd.Index] index_descriptors : List[Dict] preserve_index : bool types : List[pyarrow.DataType] Returns ------- dict Nc,g|]}t|Sr4)rZ).0rvs r6 z&construct_metadata..sAAADc$iiAAAr5cFg|]\}}t|t||fSr4)rAdict)rlevel descriptors r6rz&construct_metadata..sA E:*d++  r5)rvrJrwFrz&The DataFrame has non-str index name `z@` which will be converted to string and not roundtrip correctly. stacklevellevelsnamesattrsz(Could not serialize pd.DataFrame.attrs: z!, defaulting to empty attributes.spandaspyarrow)libraryversion) index_columnscolumn_indexescolumns attributescreatorpandas_versionutf8)ziprhrappendrvrArZ_column_name_to_stringswarningswarn UserWarningrgr_get_simple_index_descriptorrYrjsondumps Exceptionr __version__rrencode)columns_to_convertdf column_names index_levelsindex_descriptorspreserve_indexrHcolumn_field_namesserialized_index_levelsnum_serialized_index_levelsntypesdf_types index_typescolumn_metadatacolrvrwrJroindex_column_metadatanon_str_index_namesrrrrrres r6construct_metadatars*!BALAAA!$\3D!E!E #&&=">">ZZF:f:::;H!<<==>KO-01C\1CX.O.O)))T:z&s2<2<>>> x((((U"" /2 #[0 0 3 3 + UJz%jS.I.I%#**5:666*,UZ88%% H " ( ( 2 2 2 2 " # #a ' ' M09L000  + + + + X |<< Gbjo->??vu-- , ,KE43E4@@H  ! !( + + + + ,FHGG1N$R119rJ' : '''   0q 0 0 0 A ' ' ' ' ' ' ' ' '' 4:.,&)>>$$>*1      6&>>  s H I)"IIct|\}}t|}d|vrtjdtd|dkr|rJddi}|||||dS) NmixedzlThe DataFrame has column names of mixed type. They will be converted to strings and not roundtrip correctly.rrrencodingUTF-8ru)rqr`rrr)rrvrrrxs r6rr2s#;E#B#B L.-e44K+  @ A ' ' ' 'i!!!!$g."""   r5ct|tr|St|tr|dSt|tr/tt t t |St|trtd|)t|trtj |r|St|S)a!Convert a column name (or level) to either a string or a recursive collection of strings. Parameters ---------- name : str or tuple Returns ------- value : str or tuple Examples -------- >>> name = 'foo' >>> _column_name_to_strings(name) 'foo' >>> name = ('foo', 'bar') >>> _column_name_to_strings(name) "('foo', 'bar')" >>> import pandas as pd >>> name = (1, pd.Timestamp('2017-02-01 00:00:00')) >>> _column_name_to_strings(name) "('1', '2017-02-01 00:00:00')" rz%Unsupported type for MultiIndex level) rArZrdecodetuplemaprrr|rzrPr{rvs r6rrFs2$  D% {{6""" D% 54d;;<<=== D( # #?@@@ *T511bhtnn t99r5cX|j|j|vrt|jSd|ddS)zReturn the name of an index level or a default name if `index.name` is None or is already a column name. Parameters ---------- index : pandas.Index i : int Returns ------- name : str N__index_level_d__)rvr)indexirs r6_index_level_namerms< z%*L"@"@&uz222''''''r5ct|||}|jjs$tdt |j|t |||Sg}g}|durt |jng}g}g}|D]} || } t| } tj | rtd| d| | | d| | | t| g} g} t|D]\} }t|| |} t!|tjjr|t'|}nA| || d| }| | | ||| z}|||| | |||fS)NzDuplicate column names found: FSparse pandas data (column ) not supported.)_resolve_columns_of_interestr is_unique ValueErrorlist$_get_columns_to_convert_given_schema_get_index_level_valuesrrr is_sparser|rrZ enumeraterrApd RangeIndex_get_range_index_descriptor)rschemarrrrrrconvert_fieldsrvrrindex_column_namesr index_leveldescr all_namess r6_get_columns_to_convertrs?*2vw??G :   ?T"*-=-= ? ?   3BOOOL.<5-H-H)))  N - -h&t,,   % % FDdDDDFF F !!#&&&d###D!!!!!#d)),,,,#L11 ( (; a>> {KN$= > > ,&/ <>) 'T'''(((HHH- 0   % % FDdDDDFF F T""!!#&&&e$$$D!!!  %  % %d + + +  $ $T * * *    $ $ $11I |\3E |-? QQs$ # B7?B7%A$$AB76B7c|}||jjvr3t|r$t|t dd}|j|S)z_ Get the index level of a DataFrame given 'name' (column name in an arrow Schema). r)rr_is_generated_index_nameintrhget_level_values)rrvkeys r6rrs^ C 28>!!&>t&D&D!$s+,,R/011 8 $ $S ) ))r5cn tj||S#t$rt|cYSwxYwN)rrr|rZrs r6 _level_namer sI 4 4yys 44cdt|jtj|dtj|dtj|ddS)Nrangestartstopstep)kindrvrrr)rrvrget_rangeindex_attribute)rs r6rrsSEJ''5eWEE4UFCC4UFCC   r5cxttdg}fdt|DS)Nrc:g|]}|Sr4)r)rrrs r6rz+_get_index_level_values..#s' 8 8 8!E " "1 % % 8 8 8r5)rhrgr)rns` r6rr!s> GE8eW - -..A 8 8 8 8uQxx 8 8 88r5cr||td||j}n|fd|D}nj}|S)NzJSchema and columns arguments are mutually exclusive, pass only one of themc&g|] }|jv |Sr4)r)rcrs r6rz0_resolve_columns_of_interest..-s 999bj1r5)rrr)rrrs` r6rr&sa g1<== =  ,  9999g999* Nr5c t|d||\}}}}}}} }g} | D]} | j} tj| rt j| dj} ntj| rZt| tj j r| dn | dd}t j|dj} nVt| | j d\} } tj| | } | t j| dj} | | t#| |||||| |}|| |fS)NT) from_pandasrr)rvaluesris_categoricalrarrayrXis_extension_array_dtyperArSeriesheadget_datetimetz_typerWr_ndarray_to_arrow_typerr)rrrrrr_rrrrHrrtype_r ros r6dataframe_to_typesr4st ""dNG D DY E     %f - - ;HQD1116EE  1& 9 9 ;!+;>("*"*5AFF1III/0!u HU555:EE/FFMFEF11&%@@E}555: U!B m=N2DH eX %%r5Tc "t||||\}}}} } } } } |Ht|t|j}}||dzkr|dkrtj}nd}t sd}fd"d}|dkr"fdt | | D}ng}tj|5}t | | D]_\}}||j r | "||5| | "||` dddn #1swxYwYt|D]6\}}t|tjr|||<7d|D}|Tg}t ||D]-\}}| tj||.tj|}t%| ||| | |||}|jrt)|jn t+}||||}d}t|dkrn | dd }|d krH| dd }| dd } | dd }!tt1|| |!}n#t2$rYnwxYw|||fS)NdrcX|d}d}n|j}|j} tj||d}nQ#tjtjtjf$r(}|xjd|jd|j fz c_|d}~wwxYw|s&|j dkrtd|d|j d|S) NT)rXrsafezConversion failed for column z with type rzField z( was non-nullable but pandas column had z null values) nullablerXrr ArrowInvalidArrowNotImplementedErrorArrowTypeErrorargsrvrW null_countr)rrfield_nullablerr_rr s r6convert_columnz+dataframe_to_arrays..convert_columnps =!NEE"^NJE Xc4dKKKFF+!#    FFPPPSYPPS SFFG    E&"3a"7"7DeDD$*$5DDDEE E s1&A?#A::A?ct|tjo/|jjo#t |jjtjSr) rArPndarrayflags contiguous issubclassrWrXinteger)arrs r6_can_definitely_zero_copyz6dataframe_to_arrays.._can_definitely_zero_copys;3 ++7 $739>2:66 8r5c.g|]\}}||Sr4r4)rrfrs r6rz'dataframe_to_arrays..s?GGGa!.A&&GGGr5cg|] }|j Sr4rX)rxs r6rz'dataframe_to_arrays..s $ $ $QV $ $ $r5rrrrrrr)rrhrr cpu_countr rrThreadPoolExecutorrrsubmitrrAFuturer_rrrrorrr with_metadatarr)#rrrnthreadsrr rrrrrrrrnrowsncolsrarraysexecutorrrr maybe_futrHfieldsrvrpandas_metadataron_rowsrrrrrs# ` @r6dataframe_to_arraysr.Ws/r6>/688Y 2wwBJu 53;  5199|~~HHH  ! !*888 1}}GGGG!"4nEEGGG  ' 1 1 IX.?? I I1,,QX66IMM..A"6"67777MM(//.!Q"G"GHHHH  I I I I I I I I I I I I I I I I&f-- / /LAy)W^44 /%,,..q $ $V $ $ $E ~y%00 1 1KD% MM"(4// 0 0 0 06""(B m=N2DO-3OGx(((H OOO$$$  ! !( + +FF 6{{a $Q'/Dw)!,W5(+F3(+F3U5$5566    D  66 !!s&9A1D77D;>D;*AK KKc|jjtjkr||fSt j|r&|$|j}|j}tj ||}n|tj |j}||fSr) rWrXrPrVr is_datetimetzrGrlr timestampfrom_numpy_dtype)rrWrrGrls r6rrsw |BM))u} ''2EM Xz T2&& #FL11 5=r5cddlmcm}|dd}|d}d|vr0tj||d|d}nd|vrtj|j \}} t||d} t j r6tj |d | d }n|}|r ||||j| } | Snnd |vrh|d }t#|dksJ||d} || } t%| dst'd| |}n|}|r|||S||fS)a Construct a pandas Block from the `item` dictionary coming from pyarrow's serialization or returned by arrow::python::ConvertTableToPandas. This function takes care of converting dictionary types to pandas categorical, Timestamp-with-timezones to the proper pandas Block, and conversion to pandas ExtensionBlock Parameters ---------- item : dict For basic types, this is a dictionary in the form of {'block': np.ndarray of values, 'placement': pandas block placement}. Additional keys are present for other types (dictionary, timezone, object). columns : Column names of the table being constructed, used for extension types extension_columns : dict Dictionary of {column_name: pandas_dtype} that includes all columns and corresponding dtypes that will be converted to a pandas ExtensionBlock. Returns ------- pandas Block rNblock placement dictionaryre)rirerfrF)rWcopy)r5klassrWpy_arrayr__from_arrow__zGThis column does not support to be converted to a pandas ExtensionArray)r5)pandas.core.internalscore internalsgetrcategorical_type from_codesrP datetime_datarWmake_datetimetz is_ge_v21rrview make_blockDatetimeTZBlockrhrYrr:)itemrextension_columns return_block_int block_arrr5rrlrrWr4rv pandas_dtypes r6_reconstruct_blockrMs8)(((((((($''I[!It*55 $|"4O6%% t  "9?33ad:&677  " " .&&w''u5'CCC  Y.2.B.3(55   t  :9~~""""y|$(. |%566 ;:;; ;))#..si888I~r5ctjrd}tj|}tj||S)NnsrG)ris_v1rrstring_to_tzinfodatetimetz_type)rlrGs r6rBrBsC   $ $B  &t 3 3 33r5Fc g}g}i}|jj}|sw|u|d}|dg}|di}|d} t||}t || ||\}} t |||||n7t j|j } t |g|||t|t|||} |j tj|||t!} t jr,ddlm} fd| D}| || | }||_|Sdd lm}dd lm}fd | D}| | g}|||}t jr|||j}n ||}||_|S) Nrrrrr)create_dataframe_from_blocksc6g|]}t|dS)F)rIrMrrGrext_columns_dtypess r6rz&table_to_dataframe..;sG    l$6U L L L   r5)rr) BlockManager) DataFramec2g|]}t|Sr4rWrXs r6rz&table_to_dataframe..Hs6    t\3E F F   r5)rr,r>_add_any_metadata_reconstruct_index_get_extension_dtypesrrrnum_rows'_check_data_column_metadata_consistency_deserialize_column_indexrrrtable_to_blocksrkeysis_ge_v3pandas.api.internalsrUrr;rZpandasr[rC _from_mgraxes)optionstableriignore_metadata types_mapper all_columnsrrr,rrrr_rUblocksrrZr[rimgrrrYs @@r6table_to_dataframerqsvKNJl2O  :%i0 (,,-=rBB$((r:: +O<!%99)%1B*5|EE u2 ; gzCC))%.992 2|Wj  ,K888'{NKKG%L V # #GUJ$();)@)@)B)B$C$CEEFEEEEEE         * )&w O O O 666666$$$$$$        l64((  " " $$S#(33BB3B r5> r r rrrrr>rrrrrrc|d}|pg}i}tj|S|r(|jD] }|j}||} | | ||j<!|jD]\}|j}|j|vrJt |t jr0 |} | ||j<L#t$rYXwxYw]|D]} | d} n#t$r | d} YnwxYw| d} | |vr| tvrtj | } t | tjrt | tj jrV|s| |vr t j|j| jrn#t$rYnwxYwt%| dr| || <tjr|s|jD]}|j|vrt j|jsHt j|js$t j|jr;|j|vr2tj t.j||j<|S)a Based on the stored column pandas metadata and the extension types in the arrow schema, infer which columns should be converted to a pandas extension dtype. The 'numpy_type' field in the column metadata stores the string representation of the original pandas dtype (and, despite its name, not the 'pandas_type' field). Based on this string representation, a pandas/numpy dtype is constructed and then we can check if this dtype supports conversion from arrow. strings_to_categoricalNrwrvryr:)na_value)rextension_dtyperrXrvrArBaseExtensionTypeto_pandas_dtypeNotImplementedErrorr@_pandas_supported_numpy_typesrLr StringDtyperH is_dictionaryrrYuses_string_dtype is_stringis_large_stringis_string_viewrPnan) rkcolumns_metadatarmrjrirs ext_columnsrtyprLcol_metarvrWs r6r_r_bs%%=>!rJK"*7\ 7 7E*C'<,,L'*6 EJ'77j :[ ( (ZR=Q-R-R ( 7"2244 +7 EJ'''     %55 $L)DD $ $ $F#DDD $& { " "u4Q'Q'Q'3E::L, (CDD 5lKN,FGG  .!1C1C 811%,2D2DT2J2J2OPP%$%#<)9::5(4K%$&&V/EV\ V VEz,,""5:..-8++EJ77-8**5:66-*J..*5.*D*Dbf*D*U*U EJ' s6;B B'&B'0B99C C;z:_check_data_column_metadata_consistency..sW  6d  0|q0JQvYd5Jr5)all)rns r6rarasF    r5c|r d|Dfd|jD}n|j}t|dkrVtjjt ttj |d|D}n1tj ||r|ddnd}t|dkrt||}|S) Nc ni|]2}|dt|d|d3Srwrv)r>rrs r6 z-_deserialize_column_index..sI    EE, 7& B B C CQvY   r5c<g|]}||Sr4r>)rrvcolumns_name_dicts r6rz-_deserialize_column_index..s7   26  ! !$ - -   r5rcg|] }|d Srr4)r col_indexs r6rz-_deserialize_column_index..sEEE9V$EEEr5rrrvr) rrhrr MultiIndex from_tuplesrrast literal_evalIndex"_reconstruct_columns_from_metadata) block_tablernrcolumns_valuesrrs @r6rbrbs# 2          :E:R   %1 >Q.+77 S%~66 7 7EEnEEE8   .&& n!V!26!:!:RV'    >Q4WnMM Nr5c:d|D}g}g}|}|D]}t|trt|||||\}} } | 1n|ddkr_|d} tj|d|d|d| } t | t |krntd |d|| || tj} t |d kr| j || } nht |d kr;|d } t| | j s| | |d  } n| |j } || fS)NcHi|]}|d|d| Srrrs r6rz&_reconstruct_index..s<  lAfI&&r5rrrvrrr)rrvzUnrecognized index kind: rrrr) rArZ_extract_index_levelrrrrhrrr from_arraysrr`) rkrrnrmfield_name_to_metadata index_arrays index_names result_tablerr index_namerrs r6r^r^s LKL"'' eS ! ! J4H|U,BL5R5R 1L+z"#6]g % %vJ%.33E'N49&M9>v9C4EEK;3u::--.HvHHII IK(((:&&&& B <1 )),k)JJ \  a  Q%** 9HHUQH88E en--  r5cP||d}t||}|j|}|dkr|ddfS||}||} d| _||j|}|| |fS)Nrv)rm) _backwards_compatible_index_namerget_field_indexrm to_pandasrv remove_column) rkrrwrrm logical_namerrrrs r6rrs)*5f=L1*lKKJ $$Z00ABwwT4'' ,,q//C--\-::KK--++J77L j 00r5c4||krt|rdS|S)a1Compute the name of an index column that is compatible with older versions of :mod:`pyarrow`. Parameters ---------- raw_name : str logical_name : str Returns ------- result : str Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` N)r)raw_namers r6rr)s($<$rZrW)rrrs r6rz6_reconstruct_columns_from_metadata..s^ E9  mS-=-=>> |T * * ,r5) fillvaluerrr;rrorfT)utcr=c6g|]}tj|Sr4)r=Decimal)rrs r6rz6_reconstruct_columns_from_metadata..s")L)L)L'/!*<*<)L)L)Lr5rZr>r)rrNrr)rWrv)rrrgroperator methodcallerrrPrSrrrrR to_datetime tz_convertreas_unitrArrWrastyperhrrrv) rrrrlabels levels_dtypes new_levelsencoderrrL numpy_dtyperWrGs r6rrps, BWh - - :'F Wgt , , 6F!, Nb! ! ! MJ#Hg66G,9'!'!(|[*<88 BI  IIg&&EE \ ) )((q!*-j9;;BNN5dN33>>rBBE#%% H b&6{&C&CA&FGG Y & &N(()L)Le)L)L)LMMEE K5 [H%<%<L((L.s6222"7C002W222r5rrwrvNonerrxr;rorfrOrPrr)r)rrhrr>rrArXrrrFrGrr1ArrayrrrvrrTabler)rkr,modified_columnsmodified_fieldsrrn_index_levels n_columnsrrridxrro metadata_tz converted tz_aware_typer$rr+s r6r]r]sO \F#O4M22M222M''NOI.//.@I!!;<<:: 8<< -- "'HI~~(Y7!$$X.. "99 &,66Cj!#(BF,@AA#J/&ll:66 :;#(+#=#= # I$&L+$F$F$FM$&H$8$8>K%9%M%MM,.8F3K4D4A,C,COC(,9$S) q  s5<(()) / /A$$$/2333 oa01111uQx((( el1o....x##GBIf4E4E#FFF r5ctj|}|jdj|}|S)zB Make a datetime64 Series timezone-aware for the given tz r)rrrRdt tz_localizer)seriesrGs r6 make_tz_awarersC   $ $Bi##E** 2 Mr5r)rNT)NNT)NFN)?rcollections.abcr concurrentrconcurrent.futures.threadr7rr= itertoolsrrrrrnumpyrP ImportErrorrr pyarrow.librrr rrOrr7rDrTr`rqrrrrrrrrrrrrrr.rrMrBrqryr_rarbr^rrrrrrr]rr4r5r6rs& $$$$$$! !!!!!!   BBBDDDDDDDDDD:"###,"$$$&,,,b+/jjjjZ($$$N(((&?Q?Q?QD;Q;Q;Q| * * *999    & & & &FIM!a"a"a"a"H   &BBBBJ444JN;;;;@!!!PPPfB2222l?C1111&0/// $$$&%%%.TUTUTUn:::Bs;AA