L i]ddlmZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z m Z mZmZmZddlmZddlmZddlmZmZdd lmZmZmZmZej:d ZGd d ZGd de Z! ddZ"dZ#dZ$ejJddZ& ddZ' d d!dZ(dZ)dZ*dZ+dZ, d"dZ-dZ.Gddej^Z0y)#) annotationsN) has_magic)Path) BaseCache BlockCache BytesCache MMapCacheReadAheadCachecaches)compr)conf) filesystemget_filesystem_class)_unstrip_protocolbuild_name_functioninfer_compressionstringify_pathfsspeccVeZdZdZ d dZdZdZdZdZe dZ d Z d Z y) OpenFileaF File-like object to be used in a context Can layer (buffered) text-mode and compression over any file-system, which are typically binary-only. These instances are safe to serialize, as the low-level file object is not created until invoked using ``with``. Parameters ---------- fs: FileSystem The file system to use for opening the file. Should be a subclass or duck-type with ``fsspec.spec.AbstractFileSystem`` path: str Location to open mode: str like 'rb', optional Mode of the opened file compression: str or None, optional Compression to apply encoding: str or None, optional The encoding to use if opened in text mode. errors: str or None, optional How to handle encoding errors if opened in text mode. newline: None or str Passed to TextIOWrapper in text mode, how to handle line endings. autoopen: bool If True, calls open() immediately. Mostly used by pickle pos: int If given and autoopen is True, seek to this location immediately Nc||_||_||_t|||_||_||_||_g|_yN) fspathmodeget_compression compressionencodingerrorsnewlinefobjects)selfrrrrrrr s Q/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/fsspec/core.py__init__zOpenFile.__init__AsD  *4=     ct|j|j|j|j|j |j |jffSr)rrrrrrrr r"s r# __reduce__zOpenFile.__reduce__TsG           r%c"d|jdS)Nz )rr's r#__repr__zOpenFile.__repr__bsTYYKr**r%c|jjddjdddz} |jj|j|}|g|_|j;t|j}|||d}|jj|d|jvrHt||j|j|j}|jj||jdS#t $r2}t |jrt d|j|d}~wwxYw) Ntb)ra%s not found. The URL contains glob characters: you maybe needed to pass expand=True in fsspec.open() or the storage_options of your library. You can also set the config value 'open_expand' before import, or fsspec.core.DEFAULT_EXPAND at runtime, to True.r)rrr )rreplaceropenrFileNotFoundErrorrr!rr appendPickleableTextIOWrapperrrr )r"rfecompresss r# __enter__zOpenFile.__enter__es$yy  b)11#r:S@  TYYT 2A    'T--.Ha)A MM  # dii 'DMM$++t||A MM  #}}R  3! #'XII     s'D E-EEc$|jyr)close)r"argss r#__exit__zOpenFile.__exit__s  r%cBt|j|jSr)rrrr's r# full_namezOpenFile.full_names DGG44r%c"|jS)aMaterialise this as a real open file without context The OpenFile object should be explicitly closed to avoid enclosed file instances persisting. You must, therefore, keep a reference to the OpenFile during the life of the file-like it generates. )r8r's r#r1z OpenFile.opens~~r%ct|jD]<}d|jvr|js|j |j >|jj y)z#Close all encapsulated file objectsrN)reversedr!rclosedflushr:clear)r"r5s r#r:zOpenFile.closesP$--( A$))#AHH GGI  r%)rbNNNN) __name__ __module__ __qualname____doc__r$r(r*r8r<propertyr>r1r:r%r#rr sQH&  +!@55 r%rcFeZdZdZdddfd ZdZdZfdZd ZxZ S) OpenFilesaList of OpenFile instances Can be used in a single context, which opens and closes all of the contained files. Normal list access to get the elements works as normal. A special case is made for caching filesystems - the files will be down/uploaded together at the start or end of the context, and this may happen concurrently, if the target filesystem supports it. rFNrrcH||_||_g|_t||yr)rrfilessuperr$)r"rrr; __class__s r#r$zOpenFiles.__init__s%   $r%c:|j td|j} t|dr"|j||_|jSt|dr|j |j}nnU|Dcgc]}|j c}Scc}w)NzContext has already been used open_manyr)r ValueErrorhasattrrUrQr8)r"rss r#r8zOpenFiles.__enter__s 77?<= = WWr;'\\$/ zz!r4 RUU%6UU(,,! ,,,s>Bc|j}|Dcgc]}|j|c}d|jvrP t|dr|j |j yt|dr|j |j}nyOycc}w)NrArUr)rr<rrW commit_manyrQ)r"r;rrXs r#r<zOpenFiles.__exit__s~ WW$()qT ) dii 2{+NN4::.2t$):B  *sB ct||}t|tr"t ||j |j S|S)NrO)rR __getitem__ isinstanceslicerNrr)r"itemoutrSs r#r\zOpenFiles.__getitem__s8g!$' dE "StyyTWW= = r%c dt|dS)Nz )lenr's r#r*zOpenFiles.__repr__s3t9+%9::r%) rGrHrIrJr$r8r<r\r* __classcell__rSs@r#rNrNs( $(D -  ;r%rNc  dt||||| || \} } }| jdk(r| | _n?d|vr;| r9|Dchc]}| j|}}|D]} | j |dt |Dcgc]}t| ||||||c}|| Scc}w#t $rYUwxYwcc}w)av Given a path or paths, return a list of ``OpenFile`` objects. For writing, a str path must contain the "*" character, which will be filled in by increasing numbers, e.g., "part*" -> "part1", "part2" if num=2. For either reading or writing, can instead provide explicit list of paths. Parameters ---------- urlpath: string or list Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` to read from alternative filesystems. To read from multiple files you can pass a globstring or a list of paths, with the caveat that they must all have the same protocol. mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode name_function: function or None if opening a set of files for writing, those files do not yet exist, so we need to generate their names by formatting the urlpath for each sequence number num: int [1] if writing mode, number of files we expect to create (passed to name+function) protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. auto_mkdir: bool (True) If in write mode, this will ensure the target directory exists before writing, by calling ``fs.mkdirs(exist_ok=True)``. expand: bool **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> files = open_files('2015-*-*.csv') # doctest: +SKIP >>> files = open_files( ... 's3://bucket/2015-*-*.csv.gz', compression='gzip' ... ) # doctest: +SKIP Returns ------- An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can be used as a single context Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations )num name_functionstorage_optionsprotocolexpandfilerAT)exist_ok)rrrrr rO)get_fs_token_pathsri auto_mkdir_parentmakedirsPermissionErrorrNr)urlpathrrrrrgrfrir rnrjkwargsrfs_tokenpathsrparentsparents r# open_filesrxs^- #B% {{f" DZ0562::d#66 F  FT 2    '!     7#   sBB6B- B*)B*c (ddlm}d|vritjd}g}|j dD]=}d|vs|j |r|j |*|j |dz?n|g}g}d}|j}t|D]}|jddxst|dxsd} t| } | j|} |j| i} ||dur| j|td i| jD cic]\} }| | vs || | k7s| |c}} | }| j!|}d|vrt#| |r|}|j || |f|}|j%|Scc}} w) Nr)ChainedFileSystemz::z .*[^a-z]+.*://rirktarget_protocolrL)fsspec.implementations.chainedrzrecompilesplitmatchr3copyrBpopsplit_protocolr_get_kwargs_from_urlsupdatedictitems_strip_protocol issubclassreverse)rrsrzxbitspr` previous_bitbitricls extra_kwargskwskvkws r# _un_chainrKs@ t| JJ} %D! 'AzQWWQZ A AI&  ' v CL [[]F~::j$/S>#3Fq3ISV"8,005 jj2& $q'> JJv   , 2 2 4T1 SQRV q!tT  !!#& B &:c;L+MC C2&'  KKM JUs F4Fc t|}hd}|jDcic] \}}||vs ||}}}t||}i}tt |D]D\}}|\} } } |t |dz k(rt di| |}+t di| ||d<| |d<| |d<F|d\} } } t| fi|}|| fScc}}w)a Turn fully-qualified and potentially chained URL into filesystem instance Parameters ---------- url : str The fsspec-compatible URL **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Returns ------- filesystem : FileSystem The new filesystem discovered from ``url`` and created with ``**kwargs``. urlpath : str The file-systems-specific URL for ``url``. >rfrrrjr rrrgtarget_optionsr|forrL)rrr enumeraterBrbrr)urlrs known_kwargsrrchaininkwargsichurlsrirrr_rs r# url_to_fsrqs(  C L &||~ Gtq!,1Fad GF G c6 "EH8E?+2h E Q -b-H-H %)%;B%;(%;!"&."#!8GXq H ) )B w;Hs B:B: open_expandFc d|tn|}td|g|||||||d|} | s t|| dS)aGiven a path or paths, return one ``OpenFile`` object. Parameters ---------- urlpath: string or list Absolute or relative filepath. Prefix with a protocol like ``s3://`` to read from alternative filesystems. Should not include glob character(s). mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. expand: bool or None Whether to regard file paths containing special glob characters as needing expansion (finding the first match) or absolute. Setting False allows using paths which do embed such characters. If None (default), this argument takes its value from the DEFAULT_EXPAND module variable, which takes its initial value from the "open_expand" config value at startup, which will be False if not set. **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> openfile = open('2015-01-01.csv') # doctest: +SKIP >>> openfile = open( ... 's3://bucket/2015-01-01.csv.gz', compression='gzip' ... ) # doctest: +SKIP >>> with openfile as f: ... df = pd.read_csv(f) # doctest: +SKIP ... Returns ------- ``OpenFile`` object. Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations )rrrrrrrir rjrrL)DEFAULT_EXPANDrxr2) rrrrrrrir rjrsr`s r#r1r1s\H &~^6F      C (( q6Mr%c Td|vr tdt|fd|i|}t|djdds td|5}|Dcgc]}|j}}dddt |t r t|rt |trdSScc}w#1swY@xYw) aKOpen file(s) which can be resolved to local For files which either are local, or get downloaded upon open (e.g., by file caching) Parameters ---------- url: str or list(str) mode: str Must be read mode storage_options: passed on to FS for or used by open_files (e.g., compression) rAz(Can only ensure local files when readingrr local_fileFzOopen_local can only be used on a filesystem which has attribute local_file=TrueN) rVrxgetattrrnamer]strrr)rrrhofrQr5rus r# open_localrs$ $CDD C 6d 6o 6B 2a588\5 1 -   (u!&'A''(3Ys^ 38MQx L(((sB BBBB'cX|dk(r t|}||tvrtd|d|S)NinferzCompression type z not supported)rr rV)rrrs r#rrs<g'0 ;e#;,[MHII r%ct|}d|vr'|jdd\}}t|dkDr||fS|jdr|jddSd|fS)zReturn protocol, path pairr{rzdata::N)rrrb startswith)rrrirs r#rr$sgW%G  ua0$ x=1 T> !'"}}S!$$ =r%cVt|\}}t|}|j|S)zCReturn only path part of full URL, according to appropriate backend)rrr)rrrirrs r#strip_protocolr1s+ )KHa x (C   w ''r%cg}t|}d|vrtd|DdkDr tdt|t |}|D]4}d|vr|j t |||$|j|6t ||kDr|d|}|S|D]?}t|r!|j |j|/|j|A|S)a(Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` in them (read mode). :param paths: list of paths mode: str Mode in which to open files. num: int If opening in writing mode, number of files we expect to create. fs: filesystem object name_function: callable If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. :return: list of paths wc3*K|] }d|vsd yw)*rNrL).0rs r# z)expand_paths_if_needed..Ls,Q3!8q,s rz;When writing data, only one filename mask can be specified.rN) listsumrVmaxrbextend _expand_pathsr3rglob)rurrfrrgexpanded_paths curr_paths r#expand_paths_if_neededr8s N KE d{ ,%, ,q 0M #s5z" 1Ii%%mI}c&RS%%i0  1 ~  $+DS1N  1I#%%bggi&89%%i0  1 r%crt|tttfr+|s t dt t t|}n t |}|xsi}|r||d<t||xsi}i} tt|D]D\} } | \} } }| t|dz k(rtd i|| } +td i|| | d<| | d<| | d<F|d\}}}t|fi| }t|tttfrg|Dcgc]}tt ||xsid }}t|Dchc]}|d c}dkDr t d||Dcgc]}|d }}n|j|}t|tttfr.|rt|||||}nut|tset|}nYd |vsd |vr|rt!|||}nAd |vr:t#|j%|Dcgc]}|j'|r|}}n|g}||j(|fScc}wcc}wcc}wcc}w) aFilesystem, deterministic token, and paths from a urlpath and options. Parameters ---------- urlpath: string or iterable Absolute or relative filepath, URL (may include protocols like ``s3://``), or globstring pointing to data. mode: str, optional Mode in which to open files. num: int, optional If opening in writing mode, number of files we expect to create. name_function: callable, optional If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. storage_options: dict, optional Additional keywords to pass to the filesystem class. protocol: str or None To override the protocol specifier in the URL expand: bool Expand string paths for writing, assuming the path is a directory zempty urlpath sequencerirrr|rrz$Protocol mismatch getting fs from %srrrrL)r]rtuplesetrVrnextiterrrrBrbrrrrrsortedrisdir _fs_token)rrrrfrgrhrirjurlpath0rrrrrnested_protocolrrurrupchainspcr5s r#rmrmgsN>'D%-.56 6!$tG}"56!'*%+O&. # h 52 6EH8E?+2$&!or E Q -b-H-H %)%;B%;(%;!"&5"#qE8Q H ) )B'D%-.LS GHInQ')>B ? B   ("1( )A -CWM M!()2A))""5)%$s+, *5$ROEE4(KE 4K3$;F!%CCjIc=#34II F5M ! NN*  L D5$- (4yCT  L  7  Js1C5c4eZdZdZ dfd ZdZxZS)r4zTextIOWrapper cannot be pickled. This solves it. Requires that ``buffer`` be pickleable, which all instances of AbstractBufferedFile are. cL||||||f|_t||jyr)r;rRr$)r"bufferrrr line_buffering write_throughrSs r#r$z PickleableTextIOWrapper.__init__s*Hfg~}T  $))$r%c&t|jfSr)r4r;r's r#r(z"PickleableTextIOWrapper.__reduce__s& 11r%)NNNFF)rGrHrIrJr$r(rcrds@r#r4r4s# %2r%r4) rFNutf8NNrNNTT)rFNrNNNN)rF)rz#str | list[str] | Path | list[Path]rrrhrreturnzstr | list[str])rFrNNNT)1 __future__riologgingrr~rrpathlibrfsspec.cachingrrrr r r fsspec.compressionr fsspec.configr fsspec.registryrr fsspec.utilsrrrr getLoggerrrrrNrxrrgetrr1rrrrrrmr TextIOWrapperr4rLr%r#rsB"  %<   8 ${{|7;7;x      pf#L/d-/      Rn ,  B (,b    K#\<2b..2r%