K iu:dZddlmZmZddlmZmZmZmZm Z m Z m Z m Z m Z mZmZeZgZ ddlmZ ddlmZ ddlmZ dd lmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$dd l%Z%e%jLe"d Z'dddZ(ddddZ) dddddZ*Gdde Z+y #e$rej+dYzwxYw#e$rej+dYwxYw#e$rej+d YwxYw#e$rej+d YwxYw)zO FileSystem abstraction to interact with various local and remote filesystems. ) _is_path_like_stringify_path) FileSelectorFileTypeFileInfo FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandler PyFileSystem _copy_files_copy_files_selector)AzureFileSystemr)HadoopFileSystemr) GcsFileSystemr) AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategy S3FileSystem S3LogLevelS3RetryStrategyensure_s3_initialized finalize_s3ensure_s3_finalized initialize_s3resolve_s3_regionNrcN|tvrtd|dtd|d)Nz8The pyarrow installation is not built with support for ''z&module 'pyarrow.fs' has no attribute ') _not_imported ImportErrorAttributeError)names P/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/pyarrow/fs.py __getattr__r$IsC } vQ    0a8 Fuse_mmapc t|tr|St|tr|r tdtj|\}}|j |}|rq|j |gd}|jtjk7r3td|jjd|jd|dt||}|S ddl }t||jr8t|jdk(r t!|St#t%|St)d t|d #t&$rY#wxYw) NzXSpecifying to use memory mapping not supported for filesystem specified as an URI stringrzWThe path component of the filesystem URI must point to a directory but it has a type: `z`. The path component is `z#` and the given filesystem URI is ``r r&zUnrecognized filesystem: zP. `filesystem` argument must be a FileSystem instance or a valid file system URI) isinstancerstr ValueErrorfrom_urinormalize_path get_file_infotyper Directoryr"pathr fsspecAbstractFileSystem__name__r r FSSpecHandlerr TypeError) filesystemr'fsr2prefix prefix_infor3s r#_ensure_filesystemr<Usd*j) J $ 8 &&z2D""4( **F84Q7K8#5#55 55@5E5E5J5J4KL%%0%5%5$67%,a) #62.B  ? *f&?&?@ #,,0AA*H==#M*$=>>'Z(8'9:B B     sD99 EE) memory_mapcht|s| td||fS|Yt||}t|tr t |}nt|t s td|j|}||fSt |}t |} |j|}|jtjk7}|s tj|\}}||fS|j|}||fS#t$rd}d}YCwxYw#t$r#}t |}d|vsd|vrn|Yd}~||fSd}~wwxYw)z Return filesystem/path from path which could be an URI or a plain filesystem path or a combination of fsspec protocol and URI. Nzg'filesystem' passed but the specified path is file-like, so there is nothing to open with 'filesystem'.r&zPExpected string path; path-like objects are only allowed with a local filesystemFz empty schemezCannot parse URI)rr,r<r*r rr+r7r.r/r0rNotFoundrr-)r2r8r= file_infoexists_locallyemsgs r#_resolve_filesystem_and_pathrDs{    !? 4' ZH j/ 2"4(DD#&* ((.4 4 D !*5J?,,T2 $..H,=,==  )2248 J t ((. t /   a&C$(:c(A t  s*C2D2 DD D1D,,D1iT) chunk_size use_threadsct||\}}t||\}} |j|} | jtjk(rt |d} t || || ||yt|||| ||y)a Copy files between FileSystems. This functions allows you to recursively copy directories of files from one file system to another, such as from S3 to your local machine. Parameters ---------- source : string Source file path or URI to a single file or directory. If a directory, files will be copied recursively from this path. destination : string Destination file path or URI. If `source` is a file, `destination` is also interpreted as the destination file (not directory). Directories will be created as necessary. source_filesystem : FileSystem, optional Source filesystem, needs to be specified if `source` is not a URI, otherwise inferred. destination_filesystem : FileSystem, optional Destination filesystem, needs to be specified if `destination` is not a URI, otherwise inferred. chunk_size : int, default 1MB The maximum size of block to read before flushing to the destination file. A larger chunk_size will use more memory while copying but may help accommodate high latency FileSystems. use_threads : bool, default True Whether to use multiple threads to accelerate copying. Examples -------- Inspect an S3 bucket's files: >>> s3, path = fs.FileSystem.from_uri( ... "s3://registry.opendata.aws/roda/ndjson/") >>> selector = fs.FileSelector(path) >>> s3.get_file_info(selector) [>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson", ... f"file:///{local_path}/index_copy.ndjson") >>> fs.LocalFileSystem().get_file_info(str(local_path)+ ... '/index_copy.ndjson') Copy file using a FileSystem object: >>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson", ... f"file:///{local_path}/index_copy.ndjson", ... source_filesystem=fs.S3FileSystem()) T recursiveN)rDr/r0rr1rrr) source destinationsource_filesystemdestination_filesystemrErF source_fs source_pathdestination_fsdestination_pathr@ source_sels r# copy_filesrSsp:!I{(D+($N$'' 4I~~+++!+> Y +-=' 6 I{"$4  -r%ceZdZdZdZdZdZdZdZe dZ dZ d Z d Z d Zd Zd ZdZdZdZdZdZdZdZdZy)r6a Handler for fsspec-based Python filesystems. https://filesystem-spec.readthedocs.io/en/latest/index.html Parameters ---------- fs : FSSpec-compliant filesystem instance Examples -------- >>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP c||_yN)r9)selfr9s r#__init__zFSSpecHandler.__init__s r%c`t|tr|j|jk(StSrVr*r6r9NotImplementedrWothers r#__eq__zFSSpecHandler.__eq__% e] +77ehh& &r%c`t|tr|j|jk7StSrVrZr\s r#__ne__zFSSpecHandler.__ne__ r_r%cb|jj}t|tr|d}d|S)Nrzfsspec+)r9protocolr*list)rWrcs r# get_type_namezFSSpecHandler.get_type_name%s177## h %{H ##r%c|SrVrWr2s r#r.zFSSpecHandler.normalize_path+s r%c |d}|ddk(rtj}n+|ddk(rtj}d}ntj}t ||||j ddS)Nsizer0file directorymtime)rjrm)rFiler1Unknownrget)r2inforjftypes r#_create_file_infozFSSpecHandler._create_file_info.scF| <6 !MME &\[ (&&ED$$Ee$dhhw6MNNr%cg}|D]?} |jj|}|j|j||A|S#t$r,|jt |t jYxwxYwrV)r9rqappendrsFileNotFoundErrorrrr?)rWpathsinfosr2rqs r#r/zFSSpecHandler.get_file_info;s~ AD Aww||D) T33D$?@  A % @ XdH,=,=>? @sA  2A>=A>cX|jj|js]|jj|jrt |j|j rgSt |j|jrd}nd}g}|jj|j|dd}|jD]X\}}|jd}|jjd}||k7s8|j|j||Z|S)NT)maxdepthwithdirsdetail/) r9isdirbase_direxistsNotADirectoryErrorallow_not_foundrvrIfinditemsstriprurs) rWselectorr{rxselected_filesr2rq_pathrs r#get_file_info_selectorz$FSSpecHandler.get_file_info_selectorFs ww}}X../ww~~h//0():):;;++I+H,=,=>>   HH   4& )..0 AJD$JJsOE((..s3H  T33D$?@  A r%c^ |jj||y#t$rYywxYw)N)create_parents)r9mkdirFileExistsError)rWr2rIs r# create_dirzFSSpecHandler.create_dircs-  GGMM$yM 9   s  ,,c>|jj|dyNTrH)r9rmrhs r# delete_dirzFSSpecHandler.delete_dirjs  44 (r%cR |jj|d}|D]r}|jj|r|jj |d<|jj |sX|jj |ty#t$r|rYywxYw)NF)r}TrH)r9listdirrvrrisfile)rWr2missing_dir_oksubpathssubpaths r#_delete_dir_contentsz"FSSpecHandler._delete_dir_contentsms wwtE:H   $Gww}}W% 7d 3( 7#  $ !   sB B&$B&cj|jddk(r td|d|j||y)Nr~z$delete_dir_contents called on path 'r)rr,r)rWr2rs r#delete_dir_contentsz!FSSpecHandler.delete_dir_contentszs8 ::c?b 6cC C !!$7r%c&|jdy)Nr~)r)rWs r#delete_root_dir_contentsz&FSSpecHandler.delete_root_dir_contentss !!#&r%c|jj|s t||jj|yrV)r9rrvrrhs r# delete_filezFSSpecHandler.delete_files/ww~~d##D) )  4r%c@|jj||dyr)r9mvrWsrcdests r#movezFSSpecHandler.moves  3 -r%c<|jj||yrV)r9copyrs r# copy_filezFSSpecHandler.copy_files  S$r%cddlm}|jj|s t |||jj |ddSNr PythonFilerbmoderpyarrowrr9rrvopenrWr2rs r#open_input_streamzFSSpecHandler.open_input_stream>&ww~~d##D) )$'',,t$,7cBBr%cddlm}|jj|s t |||jj |ddSrrrs r#open_input_filezFSSpecHandler.open_input_filerr%cXddlm}||jj|ddS)Nrrwbrwrrr9rrWr2metadatars r#open_output_streamz FSSpecHandler.open_output_stream#&$'',,t$,7cBBr%cXddlm}||jj|ddS)Nrrabrrrrs r#open_append_streamz FSSpecHandler.open_append_streamrr%N)r5 __module__ __qualname____doc__rXr^rarer. staticmethodrsr/rrrrrrrrrrrrrrgr%r#r6r6 s   $  O O :) $8 '. CCC Cr%r6rV)NN),r pyarrow.utilrr pyarrow._fsrrrrr r r r r rr FileStatsrpyarrow._azurefsrr ru pyarrow._hdfsrpyarrow._gcsfsr pyarrow._s3fsrrrrrrrrrratexitregisterr$r<rDrSr6rgr%r#rs4$8       ,0-.*, )LLLFOO'( 05* Z9e9z?CH-&DH-VcC%cC{,*+, -+,- *)*)()sDBB.C C&B+*B+.CC C#"C#&C?>C?