;j(<0dZddlmZmZddlmZmZmZmZm Z m Z m Z m Z m Z mZmZeZgZ ddlmZn #e$redYnwxYw ddlmZn #e$redYnwxYw ddlmZn #e$red YnwxYw dd lmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$dd l%Z%e%j&e"n #e$red YnwxYwd Z'dddZ(ddddZ) dddddZ*Gdde Z+d S)zO FileSystem abstraction to interact with various local and remote filesystems. ) _is_path_like_stringify_path) FileSelectorFileTypeFileInfo FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandler PyFileSystem _copy_files_copy_files_selector)AzureFileSystemr)HadoopFileSystemr) GcsFileSystemr) AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategy S3FileSystem S3LogLevelS3RetryStrategyensure_s3_initialized finalize_s3ensure_s3_finalized initialize_s3resolve_s3_regionNrc`|tvrtd|dtd|d)Nz8The pyarrow installation is not built with support for ''z&module 'pyarrow.fs' has no attribute ') _not_imported ImportErrorAttributeError)names >>  ?    D  BZ(8(8 B B B   sD66 EE) memory_mapct|s|td||fS|st||}t|trt |}n$t|t std||}||fSt |}t |} | |}|j tj k}n#t$rd}d}YnwxYw|sJ tj|\}}nF#t$r$}t |}d|vsd|vrn|Yd}~nd}~wwxYw||}||fS)z Return filesystem/path from path which could be an URI or a plain filesystem path or a combination of fsspec protocol and URI. Nzg'filesystem' passed but the specified path is file-like, so there is nothing to open with 'filesystem'.r&zPExpected string path; path-like objects are only allowed with a local filesystemFz empty schemezCannot parse URI)rr,r<r*r rr+r7r.r/r0rNotFoundrr-)r2r8r= file_infoexists_locallyemsgs r#_resolve_filesystem_and_pathrDs     !? 4' ZHHH j/ 2 2 "4((DDD#&& * ((..4 4 D !*555J?,,T22 $.H,==    / )2488 J   a&&C$$(:c(A(A  ((.. t s*:C%%C65C6<D ED==EiT) chunk_size use_threadsct||\}}t||\}} ||} | jtjkr't |d} t || || ||dSt|||| ||dS)a Copy files between FileSystems. This functions allows you to recursively copy directories of files from one file system to another, such as from S3 to your local machine. Parameters ---------- source : string Source file path or URI to a single file or directory. If a directory, files will be copied recursively from this path. destination : string Destination file path or URI. If `source` is a file, `destination` is also interpreted as the destination file (not directory). Directories will be created as necessary. source_filesystem : FileSystem, optional Source filesystem, needs to be specified if `source` is not a URI, otherwise inferred. destination_filesystem : FileSystem, optional Destination filesystem, needs to be specified if `destination` is not a URI, otherwise inferred. chunk_size : int, default 1MB The maximum size of block to read before flushing to the destination file. A larger chunk_size will use more memory while copying but may help accommodate high latency FileSystems. use_threads : bool, default True Whether to use multiple threads to accelerate copying. Examples -------- Inspect an S3 bucket's files: >>> s3, path = fs.FileSystem.from_uri( ... "s3://registry.opendata.aws/roda/ndjson/") >>> selector = fs.FileSelector(path) >>> s3.get_file_info(selector) [>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson", ... f"file:///{local_path}/index_copy.ndjson") >>> fs.LocalFileSystem().get_file_info(str(local_path)+ ... '/index_copy.ndjson') Copy file using a FileSystem object: >>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson", ... f"file:///{local_path}/index_copy.ndjson", ... source_filesystem=fs.S3FileSystem()) T recursiveN)rDr/r0rr1rrr) source destinationsource_filesystemdestination_filesystemrErF source_fs source_pathdestination_fsdestination_pathr@ source_sels r# copy_filesrSsp:!I{(D+(($N$'' 44I~+++!+>>> Y +-=' 6 6 6 6 6 I{"$4  - - - - -r%ceZdZdZdZdZdZdZdZe dZ dZ d Z d Z d Zd Zd ZdZdZdZdZdZdZdZdZdS)r6a Handler for fsspec-based Python filesystems. https://filesystem-spec.readthedocs.io/en/latest/index.html Parameters ---------- fs : FSSpec-compliant filesystem instance Examples -------- >>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP c||_dSN)r9)selfr9s r#__init__zFSSpecHandler.__init__s r%cZt|tr|j|jkStSrVr*r6r9NotImplementedrWothers r#__eq__zFSSpecHandler.__eq__) e] + + '7eh& &r%cZt|tr|j|jkStSrVrZr\s r#__ne__zFSSpecHandler.__ne__ r_r%c^|jj}t|tr|d}d|S)Nrzfsspec+)r9protocolr*list)rWrcs r# get_type_namezFSSpecHandler.get_type_name%s57# h % % #{H####r%c|SrVrWr2s r#r.zFSSpecHandler.normalize_path+s r%c |d}|ddkr tj}n'|ddkrtj}d}n tj}t ||||ddS)Nsizer0file directorymtime)rjrm)rFiler1Unknownrget)r2inforjftypes r#_create_file_infozFSSpecHandler._create_file_info.snF| <6 ! !MEE &\[ ( (&EDD$Ee$dhhw6M6MNNNNr%cg}|D]} |j|}||||G#t$r0|t |t jYwxYw|SrV)r9rqappendrsFileNotFoundErrorrrr?)rWpathsinfosr2rqs r#r/zFSSpecHandler.get_file_info;s A AD Aw||D)) T33D$??@@@@% @ @ @ XdH,=>>????? @ sA  7BBc6|j|jsP|j|jrt |j|jrgSt |j|jrd}nd}g}|j|j|dd}| D]c\}}| d}|j d}||kr)| | ||d|S)NT)maxdepthwithdirsdetail/) r9isdirbase_direxistsNotADirectoryErrorallow_not_foundrvrIfinditemsstriprurs) rWselectorr{rxselected_filesr2rq_pathrs r#get_file_info_selectorz$FSSpecHandler.get_file_info_selectorFs%w}}X.// ?w~~h/00 ?():;;;+?I+H,=>>>   HHH  4&  )..00 A AJD$JJsOOE(..s33H   T33D$??@@@ r%cb |j||dS#t$rYdSwxYw)N)create_parents)r9mkdirFileExistsError)rWr2rIs r# create_dirzFSSpecHandler.create_dircsG  GMM$yM 9 9 9 9 9    DD s  ..c>|j|ddSNTrH)r9rmrhs r# delete_dirzFSSpecHandler.delete_dirjs   44 (((((r%cJ |j|d}n#t$r|rYdSwxYw|D]m}|j|r|j|d9|j|r|j|ndS)NF)r}TrH)r9listdirrvrrisfile)rWr2missing_dir_oksubpathssubpaths r#_delete_dir_contentsz"FSSpecHandler._delete_dir_contentsms wtE::HH         $ $Gw}}W%% $ 7d 3333(( $ 7###  $ $s  00c|ddkrtd|d|||dS)Nr~z$delete_dir_contents called on path 'r)rr,r)rWr2rs r#delete_dir_contentsz!FSSpecHandler.delete_dir_contentszsO ::c??b 6cCC C !!$77777r%c2|dddS)Nr~F)r)rWs r#delete_root_dir_contentsz&FSSpecHandler.delete_root_dir_contentss !!#u-----r%c|j|st||j|dSrV)r9rrvrrhs r# delete_filezFSSpecHandler.delete_filesBw~~d## *#D)) )  4r%c@|j||ddSr)r9mvrWsrcdests r#movezFSSpecHandler.moves"  3 -----r%c<|j||dSrV)r9copyrs r# copy_filezFSSpecHandler.copy_files   S$r%cddlm}|j|st |||j|ddSNr PythonFilerbmoderpyarrowrr9rrvopenrWr2rs r#open_input_streamzFSSpecHandler.open_input_stream_&&&&&&w~~d## *#D)) )z$',,t$,77cBBBBr%cddlm}|j|st |||j|ddSrrrs r#open_input_filezFSSpecHandler.open_input_filerr%c\ddlm}||j|ddS)Nrrwbrwrrr9rrWr2metadatars r#open_output_streamz FSSpecHandler.open_output_stream;&&&&&&z$',,t$,77cBBBBr%c\ddlm}||j|ddS)Nrrabrrrrs r#open_append_streamz FSSpecHandler.open_append_streamrr%N)r5 __module__ __qualname____doc__rXr^rarer. staticmethodrsr/rrrrrrrrrrrrrrgr%r#r6r6 sg    $$$  O O\ O   :))) $ $ $888 ......   CCCCCCCCC CCCCCr%r6rV)NN),r pyarrow.utilrr pyarrow._fsrrrrr r r r r rr FileStatsrpyarrow._azurefsrr ru pyarrow._hdfsrpyarrow._gcsfsr pyarrow._s3fsrrrrrrrrrratexitregisterr$r<rDrSr6rgr%r#rs[$87777777                             ,0000000,,,*+++++,-.......---+,,,,,-*,,,,,,,***)))))* )LLLLLLLLLLLLLLLLLLLLLLLLMMMFO'(((()))((((()   05* * * * * Z9e99999z?CH-&DH-H-H-H-H-VcCcCcCcCcC%cCcCcCcCcCsE1A AAA65A6:BBB"CC,+C,