'VfiRldZddlmZddlmZddlmZmZddlZddlm Z ddl Z ddl m Z ddlmZmZmZmZmZdd lmZmZdd lmZdd lmZddlZdd lmZmZmZdd l m!Z!ddl"m#cm$cm%Z&ddl'm(Z(e r ddl)m*Z*m+Z+m,Z,edZ-edZ.ddZ/d dZ0GddZ1Gdde(ej2Z3dS)!a Read SAS7BDAT files Based on code written by Jared Hobbs: https://bitbucket.org/jaredhobbs/sas7bdat See also: https://github.com/BioStatMatt/sas7bdat Partial documentation of the file format: https://cran.r-project.org/package=sas7bdat/vignettes/sas7bdat.pdf Reference for binary data compression: http://collaboration.cmc.ec.gc.ca/science/rpn/biblio/ddj/Website/articles/CUJ/1992/9210/ross/ross.htm ) annotations)abc)datetime timedeltaN) TYPE_CHECKING) get_option)read_double_with_byteswapread_float_with_byteswapread_uint16_with_byteswapread_uint32_with_byteswapread_uint64_with_byteswap)Parserget_subheader_index)cast_from_unit_vectorized)EmptyDataError) DataFrame Timestampisna) get_handle) ReaderBase)CompressionOptionsFilePath ReadBufferz 1970-01-01z 1960-01-01 sas_datetimefloatunitstrct|r tjS|dkr"tdddt |zS|dkr"tdddt |zSt d)Ns)secondsd)dayszunit must be 'd' or 's')rpdNaTrr ValueError)rrs Ec:\PYTHON\DbComparer\venv\Lib\site-packages\pandas/io/sas/sas7bdat.py_parse_datetimer)Cs Lv  s{{a##i &E&E&EEE a##i\&B&B&BBB2333 sas_datetimes pd.SeriesreturncTttz d}|dkrKt|jdd}|d|z}t j||jdStj |d|z}t j|d |jd S) a Convert to Timestamp if possible, otherwise to datetime.datetime. SAS float64 lacks precision for more than ms resolution so the fit to datetime.datetime is ok. Parameters ---------- sas_datetimes : {Series, Sequence[float]} Dates or datetimes in SAS unit : {'d', 's'} "d" if the floats represent dates, "s" for datetimes Returns ------- Series Series of datetime64 dtype or datetime.datetime. rms)rout_unitzM8[ms]FindexcopyzM8[D]dtypezM8[s]r5r2r3) _sas_origin _unix_originas_unitr_valuesviewr%Seriesr2nparray)r+rtdmillisdt64msvalss r(_convert_datetimesrCQs$  $ - -c 2 2B s{{*  !d   X&&+y}':GGGGx W555:yWM4GeTTTTr*cTeZdZUded<ded<ded<ded<ded<ded <dd Zd S)_Columnintcol_id str | bytesnamelabelformatbytesctypelengthr-NonecZ||_||_||_||_||_||_dSN)rGrIrJrKrMrN)selfrGrIrJrKrMrNs r(__init__z_Column.__init__ws2      r*N)rGrFrIrHrJrHrKrHrMrLrNrFr-rO)__name__ __module__ __qualname____annotations__rSr*r(rErEosiKKKLLLKKKr*rEceZdZUdZded<ded< d=d>dZd?dZd?dZd?dZd@dZ d@dZ dAd Z dBd#Z dCd$Z dDd&ZdEd(Zd@d)ZdFd*Zd@d+Zd@d,ZdGd-ZdGd.ZdGd/ZdGd0ZdGd1ZdGd2ZdGd3ZdGd4ZdHdId6Zd7ZdAd8Zd9ZdJd<Z dS)KSAS7BDATReadera Read SAS files in SAS7BDAT format. Parameters ---------- path_or_buf : path name or buffer Name of SAS file or file-like object pointing to SAS file contents. index : column identifier, defaults to None Column to use as index. convert_dates : bool, defaults to True Attempt to convert dates to Pandas datetime values. Note that some rarely used SAS date formats may be unsupported. blank_missing : bool, defaults to True Convert empty strings to missing values (SAS uses blanks to indicate missing character variables). chunksize : int, defaults to None Return SAS7BDATReader object for iterations, returns chunks with given number of lines. encoding : str, 'infer', defaults to None String encoding acc. to Python standard encodings, encoding='infer' tries to detect the encoding from the file header, encoding=None will leave the data in binary format. convert_text : bool, defaults to True If False, text variables are left as raw bytes. convert_header_text : bool, defaults to True If False, header text, including column names, are left as raw bytes. rF _int_lengthz bytes | None _cached_pageNTinfer path_or_bufFilePath | ReadBuffer[bytes] convert_datesbool blank_missing chunksize int | Noneencoding str | None convert_textconvert_header_text compressionrr-rOc ||_||_||_||_||_||_||_d|_d|_g|_ g|_ g|_ g|_ g|_ d|_g|_g|_g|_d|_d|_d|_t)|dd| |_|jj|_|j|j|j|j|j|j|j|jdg |_ |!|"dS#tF$r|$wxYw)Nzlatin-1r*rrbF)is_textri)%r2r`rbrcrergrhdefault_encodingricolumn_names_raw column_namescolumn_formatscolumns%_current_page_data_subheader_pointersr\_column_data_lengths_column_data_offsets _column_types_current_row_in_file_index_current_row_on_page_indexrhandleshandle _path_or_buf_process_rowsize_subheader_process_columnsize_subheader_process_subheader_counts_process_columntext_subheader_process_columnname_subheader#_process_columnattributes_subheader_process_format_subheader_process_columnlist_subheader_subheader_processors_get_properties_parse_metadata Exceptionclose) rRr^r2r`rbrcrergrhris r(rSzSAS7BDATReader.__init__so **"  (#6 )-//113&( LN2 /1!/1!*,*+'*+'*+'! u+    !L/  +  .  *  .  .  4  *  .  & "   " " "  " " " " "    JJLLL  s 6(D E np.ndarraycLtj|jtjS)z5Return a numpy int64 array of the column data lengthsr4)r=asarrayrsint64rRs r(column_data_lengthsz"SAS7BDATReader.column_data_lengthsz$328DDDDr*cLtj|jtjS)z0Return a numpy int64 array of the column offsetsr4)r=rrtrrs r(column_data_offsetsz"SAS7BDATReader.column_data_offsetsrr*c\tj|jtjdS)zj Returns a numpy character array of the column types: s (string) or d (double) S1r4)r=rrur5rs r( column_typeszSAS7BDATReader.column_typess$ z$,BHTNNCCCCr*c8|jdSrQ)rxrrs r(rzSAS7BDATReader.closes r*cZ|jd|jd|_|jdt t jt jkrtd|t j t j }|t j kr1d|_ d|_ t j|_t j|_n0d|_ t j|_t j|_d|_ |t jt j}|t jkr t j}nd}|t jt j}|dkrd |_t6jd k|_nd |_t6jd k|_|t jt jd}|t j vr/t j ||_!|j"d kr |j!|_"n d|d|_!tGddd}|$t j%|zt j&}|tOj(|dz|_)|$t j*|zt j+}|tOj(|dz|_,|-t j.|zt j/|_0|j|j0dz }|xj|z c_t |j|j0krtd|-t j1|zt j2|_3dS)Nri z'magic number mismatch (not a SAS file?)TF<big>littler]zunknown (code=)r r!r)rz*The SAS7BDAT file appears to be truncated.)4rzseekreadr\lenconstmagicr' _read_bytesalign_1_offsetalign_1_lengthu64_byte_checker_valueU64r[page_bit_offset_x64_page_bit_offsetsubheader_pointer_length_x64_subheader_pointer_lengthpage_bit_offset_x86subheader_pointer_length_x86align_2_offsetalign_2_lengthalign_1_checker_value align_2_valueendianness_offsetendianness_length byte_ordersys byteorder need_byteswapencoding_offsetencoding_lengthencoding_namesinferred_encodingrer _read_floatdate_created_offsetdate_created_lengthr% to_timedelta date_createddate_modified_offsetdate_modified_length date_modified _read_uintheader_size_offsetheader_size_length header_lengthpage_size_offsetpage_size_length _page_length)rRbufalign1epochxs r(rzSAS7BDATReader._get_propertiess q!!! -22377  QU[!1!11 2ek A AFGG Gu3U5IJJ %. . .DH D $)$=D !-2-OD * *DH$)$=D !-2-OD * D u3U5IJJ %- - -(FFFu68OPP '>>!DO!$%!7D  !DO!$(!:D u4e6KLLQO %& & &%*%9#%>D "}'' $ 6 %> S  t ! !T%7 7 7IJJ J OO  "V +U-C  r*rc||jpd}|jr|t|S)Nr!)nrows)rrcemptyr StopIteration)rRdas r(__next__zSAS7BDATReader.__next__Cs> YYT^0qY 1 1 8 JJLLL  r*offsetwidthc|jJ|dkrt|j||jS|dkrt|j||jS|t d)Nrrzinvalid float width)r\r rr rr'rRrrs r(rzSAS7BDATReader._read_floatKs ,,, A::+!64+= aZZ,!64+=  JJLLL233 3r*cd|jJ|dkr||ddS|dkrt|j||jS|dkrt |j||jS|dkrt |j||jS|td)Nr!rrrzinvalid int width)r\rr rr r rr'rs r(rzSAS7BDATReader._read_uintZs ,,, A::##FA..q1 1 aZZ,!64+= aZZ,!64+= aZZ,!64+=  JJLLL011 1r*rNc|jJ||zt|jkr#|td|j|||zS)NzThe cached page is too small.)r\rrr'rRrrNs r(rzSAS7BDATReader._read_bytesns^ ,,, F?S!233 3 3 JJLLL<== = &6/!9::r*rHcz||||dS)N )_convert_header_textrrstriprs r(_read_and_convert_header_textz,SAS7BDATReader._read_and_convert_header_textus;((   VV , , 3 3H = =   r*cd}|s|j|j|_t |jdkrdSt |j|jkrt d|}|dSdS)NFrz2Failed to read a meta data page from the SAS file.)rzrrr\rr'_process_page_meta)rRdones r(rzSAS7BDATReader._parse_metadatazs - $ 1 6 6t7H I ID 4$%%**4$%%)::: !UVVV**,,D  - - - - -r*c:|tjtjtjgz}|j|vr||jtjk}|jtjk}t|p |p |j gkSrQ) _read_page_headerrpage_meta_types page_amd_type page_mix_type_current_page_type_process_page_metadatapage_data_typerarr)rRpt is_data_page is_mix_pages r(rz!SAS7BDATReader._process_page_metas     "e&95;N%O O  "b ( (  ' ' ) ) ).%2FF -1DD   @ @9R?   r*cf|j}tj|z}||tjtjz|_tj|z}||tj|_ tj |z}||tj |_ dSrQ) rrpage_type_offsetrpage_type_lengthpage_type_mask2rblock_count_offsetblock_count_length_current_page_block_countsubheader_count_offsetsubheader_count_length_current_page_subheaders_count)rR bit_offsettxs r(rz SAS7BDATReader._read_page_headers*  #j 0 OOB 6 7 7%:O O  % 2)-U=U)V)V&  )J 6.2oo ,/ / +++r*c |j}t|jD]e}tj|z}||j|zz}|||j}||jz }|||j}||jz }||d}|dz }||d}|dks|tjkr| ||j} t| } |j | } | o|tj dfv} |tj k} |jr"| r | r|j||f3|t%d| | ||gdS)Nr!rzUnknown subheader signature )rrangerrsubheader_pointers_offsetrrr[truncated_subheader_idrrrcompressed_subheader_idcompressed_subheader_typerirrappendrr')rRrir total_offsetsubheader_offsetsubheader_lengthsubheader_compressionsubheader_typesubheader_signaturesubheader_indexsubheader_processorf1f2s r(rz%SAS7BDATReader._process_page_metadatas* t:;;& H& HA4zAF!D$BQ$FFL#|T=MNN  D, ,L#|T=MNN  D, ,L$(OOL!$D$D ! A L!__\1==N!A%%(E,HHH"&"2"23CTEU"V"V 12EFFO"&"<_"M "**u/La.PP#u'FF#r>EE)+;<JJLLL$L7JLL$#$46FGGGGM& H& Hr*cr|j}|}|}|jr |dz }|dz }n |dz }|dz }||tj|zz||_||tj|zz||_||tj|zz||_ ||tj |zz||_ tj |z}|||z||_ ||d|_||d|_dS)Niiibizr)r[rrrrow_length_offset_multiplier row_lengthrow_count_offset_multiplier row_countcol_count_p1_multiplier col_count_p1col_count_p2_multiplier col_count_p2'row_count_on_mix_page_offset_multiplier_mix_page_row_count_lcs_lcp)rRrrNint_len lcs_offset lcp_offsetmxs r(r{z)SAS7BDATReader._process_rowsize_subheadersE"  8  # J # JJ # J # J// U7'A A    U6@ @   !OO U2W< O$P$P!!!G + *D Q Qr*c||j}||z }|d|zz dz dz}t|D]}|tj|dzzztjz}|tj|dzzztjz}|tj|dzzztjz}||tj} ||tj } ||tj } |j | } | | | | z} |j || dS)Nr rr!)r[rrcolumn_name_pointer_length!column_name_text_subheader_offsetcolumn_name_offset_offsetcolumn_name_length_offsetr!column_name_text_subheader_lengthcolumn_name_offset_lengthcolumn_name_length_lengthrnrorr)rRrrNrcolumn_name_pointers_countrtext_subheadercol_name_offsetcol_name_lengthidx col_offsetcol_lenname_rawcnames r(rz,SAS7BDATReader._process_columnname_subheader"sg"'&,q7{&:R&?A%E"122 G GA2a!e<=9:  2a!e<=12  2a!e<=12  // GC!@Jooou7VWWG,S1HZ*w*>>?E   $ $T%>%>u%E%E F F F F7 G Gr*cd|j}|d|zz dz |dzz}t|D]}||ztjz||dzzz}|d|zztjz||dzzz}|d|zztjz||dzzz}|||} |j| ||tj } |j | ||tj } |j | dkrdnddS)Nrr3rr!ds) r[rrcolumn_data_offset_offsetcolumn_data_length_offsetcolumn_type_offsetrrtrcolumn_data_length_lengthrscolumn_type_lengthru) rRrrNrcolumn_attributes_vectors_countrcol_data_offset col_data_len col_typesrs r(rz2SAS7BDATReader._process_columnattributes_subheaderCse"+1AK+?"+D'TU+*V'677 @ @A 5#BBQ'TU+EVV g+12w{#$ W$u'??!wQR{BSS 99A  % , ,Q / / / e.MNNA  % , ,Q / / / 5+CDDA   % %a1ffdd$ ? ? ? ?+ @ @r*cdSrQrXrs r(rz,SAS7BDATReader._process_columnlist_subheader]s r*c |j}|tjzd|zz}|tjzd|zz}|tjzd|zz}|tjzd|zz}|tjzd|zz}|tjzd|zz} ||tj } t| t|j dz } ||tj } ||tj} ||tj}t|t|j dz }||tj}|| tj}|j |}|||||z}|j | }||| | | z}t|j}t)||j||||j||j|}|j||j|dS)Nr!)r[r)column_format_text_subheader_index_offsetcolumn_format_offset_offsetcolumn_format_length_offset(column_label_text_subheader_index_offsetcolumn_label_offset_offsetcolumn_label_length_offsetr)column_format_text_subheader_index_lengthminrrncolumn_format_offset_lengthcolumn_format_length_length(column_label_text_subheader_index_lengthcolumn_label_offset_lengthcolumn_label_length_lengthrrqrErorursrpr)rRrrNrtext_subheader_formatcol_format_offsetcol_format_lentext_subheader_labelcol_label_offset col_label_lenr format_idx format_start format_len label_idx label_start label_len label_names column_label format_names column_formatcurrent_column_numbercols r(rz(SAS7BDATReader._process_format_subheaderase" UD Dq7{ R #U%FFWT%"CCa'kQ UC Ca'k Q "E$DDq7{R!AAAKO OO !5#R  C 566:;;  u@  __^U5VWW OO %"P   3t'<#=#=#ABB oo e>  OOM53STT +I6 00  kI&== >  ,Z8 11  z(AA B  !$DL 1 1 !  3 4    4 5  %&; <    ""=111 C     r*rc||j|j}n ||j}t|jdkr#|t d|dkr|j|jkrtSt||j|jz }|j d}|j d}tj ||ft|_ tj|d|zftj|_d|_t%|}|||}|j||j}|S)NrzNo columns to parse from filerErFr4r)rcrrrurrrvrrZcountr=robject _string_chunkzerosuint8 _byte_chunk_current_row_in_chunk_indexrr_chunk_to_dataframer2 set_index)rRrndnsprslts r(rzSAS7BDATReader.readsK M :NEE ]NE t! " "a ' ' JJLLL !@AA A 1998DNJJ;; E4>D,KKLL   % %d + +   % %d + +Xr5k@@@8RUO28DDD+,( 4LL u '')) : !>>$*--D r*cJg|_|j|j|_t |jdkrdSt |j|jkrE|dt |jdd|jdd}t|||j tj vr| |j tj tj tjgzvr|SdS)NrTz-failed to read complete page from file (read r#z of z bytes)F)rrrzrrr\rrr'rrrrrrr_read_next_page)rRmsgs r(rzSAS7BDATReader._read_next_pages2572 -2243DEE t ! !Q & &4 " # #t'8 8 8 JJLLLNt())MNN151BMNNN S// !     "e&; ; ;  ' ' ) ) )  "%*?   C +   '')) )ur*c6|j}|j}t||z |}i}d\}}td}t|jD]}|j|} |j|dkr|j|ddf|j dz} tj | tj |d|| <|jre|j|t jvrt%|| d|| <n2|j|t jvrt%|| d|| <|d z }|j|d krtj |j|ddf|d || <|jrJ|jC||| j|| <|r|| d || <|d z }w|t7d t9|j|t;||j|d} | S)N)rrzfuture.infer_stringrEr#r4Fr6rr!rFr1rzunknown column type )rqr2r3)ryrvrrr rorurxr;rr%r<r=float64r`rprsas_date_formatsrCsas_datetime_formatsrurgre_decode_stringrastyperr'reprr) rRnmixrjsjb infer_stringjrIcol_arrdfs r(rzz"SAS7BDATReader._chunk_to_dataframes(  ,  + 1q5!__B!"788 t()) W WA$Q'D!!$,,*2qqq5166T_s=R6SSYwbjQVWWWT %I*1-1GGG%7T C%H%HT ,Q/53MMM%7T C%H%HT a#A&$..Yt'9"aaa%'@QVWWWT $>$-*C!%!4!4T$Z^!D!DDJ#>%)$Z%6%6u%=%=T a  !UT=OPQ=R8S8S!U!UVVV tT%6bu M M M r*cD||jp|jSrQ)decoderermrRbs r(rzSAS7BDATReader._decode_stringsxx >)>???r*rrLc>|jr||S|SrQ)rhrrs r(rz#SAS7BDATReader._convert_header_texts&  # &&q)) )Hr*)NTTNNTTr])r^r_r`rarbrarcrdrerfrgrarhrarirr-rO)r-r)r-rO)r-r)rrFrrF)rrFrrFr-rF)rrFrNrF)rrFrNrFr-rH)r-ra)rrFrNrFr-rOrQ)rrdr-r)rrLr-rH)!rTrUrV__doc__rWrSrrrrrrrrrrrrrrr{r|r}r~rrrrrrrzrrrXr*r(rZrZs< "" $#!$(*1=====~EEEEEEEEDDDDD D D D L 4 4 4 42222(;;;;    ----          )H)H)H)HV3333<    +Q+Q+Q+QZGGGGB@@@@4    5!5!5!5!n>2!!!!F@@@r*rZ)rrrr)r+r,rrr-r,)4r __future__r collectionsrrrrtypingrnumpyr=pandas._configrpandas._libs.byteswapr r r r r pandas._libs.sasrrpandas._libs.tslibs.conversionr pandas.errorsrpandasr%rrrpandas.io.commonrpandas.io.sas.sas_constantsiosas sas_constantsrpandas.io.sas.sasreaderrpandas._typingrrrr8r7r)rCrEIteratorrZrXr*r(rs#""""" %%%%%%EDDDDD(((((( ('''''++++++++++++......y&& i %%  4 4 4 4UUUU<6p p p p p Zp p p p p r*