L i|/\ddlmZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZddlmZmZddlmZdd lmZmZdd lmZdd lmZdd lmZmZmZmZm Z m!Z!m"Z"ejFd Z$dZ%Gdde&Z'Gdde'Z(GddejRZ*dZ+y)) annotationsN)ESPIPE) has_magic)sha256)AnyClassVar)DEFAULT_CALLBACK) apply_configconf)DirCache) Transaction)_unstrip_protocolglob_translate isfilelike other_paths read_blockstringify_pathtokenizefsspecc||i|SN)clsargskwargss Q/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/fsspec/spec.py make_instancer s   c,eZdZdZfdZfdZxZS)_Cacheda Metaclass for caching file system instances. Notes ----- Instances are cached according to * The values of the class attributes listed in `_extra_tokenize_attributes` * The arguments passed to ``__init__``. This creates an additional reference to the filesystem, which prevents the filesystem from being garbage collected when all *user* references go away. A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* be made for a filesystem instance to be garbage collected. ct||i|tjdrt j |_ni|_tj|_ y)Nweakref_instance_cache) super__init__r getweakrefWeakValueDictionary_cacheosgetpid_pid)rrr __class__s rr%z_Cached.__init__5sH $)&) 88, - 446CJCJ99;rc2t|}tfdjD}jDcic]}||vs||j |}}t j tjg||i|}|j dd}tjj k7r3jjtj_|s0jr$|jvr|_j|St @|i||}||_||_||_|j(r|j*rddlm} | |jr|s|_|j|<|Scc}w)Nc38K|]}t|dywrgetattr).0attrrs r z#_Cached.__call__..Cs )-GCt $ sskip_instance_cacheFr )mirror_sync_methods)r tuple_extra_tokenize_attributes_strip_tokenize_optionspoprr, threading get_identr*r+r)clearcachable_latestr$__call__ _fs_token_ storage_argsstorage_options async_implr6asyn) rrr extra_tokenskstrip_tokenize_optionstokenskipobjr6r-s ` rr@z_Cached.__call__Asyc6* 141O1O  '*&A&A" !"Q&[Avzz!} " "  9..0 37 :F JP zz/7 99;#(( " JJ   yy{CH #**)<CK::e$ $'"DMFM6LMC"CN#C "(C ~~#"9"95#C(||D# $' 5!J7" s FF)__name__ __module__ __qualname____doc__r%r@ __classcell__)r-s@rr!r!$s   rr!czeZdZUdZdZdZdZdZdZde d<d Z dZ dZ d Z eZd Zd Zd e d <de d<dZedZedZdZdZdZdZedZdedZedZedZedZ dZ!dZ"dfdZ#dgdZ$dhd Z%d!Z&dgd"Z'd#Z(did$Z)djd%Z*dkd&Z+dfd'Z,d(Z-d)Z.d*Z/d+Z0d,Z1d-Z2d.Z3d/Z4dld0Z5 dld1Z6dmd2Z7dnd4Z8dfd5Z9 dod6Z:dpd7Z;eeZC drd?ZDdsd@ZEdsdAZFdBZGdCZHdsdDZIedEZJ dtdFZK dudGZLdgdHZMdIZNdfdJZOddKdvdLZPedwdMZQddKdxdNZRedydOZSdPZTdzdQZUedRZVdSZWdTZX d{ d|dUZYdmdVZZdWZ[dgdXZ\dhdYZ]dgdZZ^d[Z_d\Z`d]Zad}d^Zbd_Zcdsd`ZddhdaZedhdbZfd~dcZgddZhy )AbstractFileSystemz An abstract super-class for pythonic file-systems Implementations are expected to be compatible with or, better, subclass from here. TFi@/abstractzClassVar[str | tuple[str, ...]]protocolNrztuple[Any, ...]rBdict[str, Any]rCc4|jryd|_d|_d|_g|_t di||_|j ddrtjdt|j ddrtjdtd|_ y) aCreate and configure file-system instance Instances may be cachable, so if similar enough arguments are seen a new instance is not required. The token attribute exists to allow implementations to cache instances if they wish. A reasonable default should be provided if there are no arguments. Subclasses should call this method. Parameters ---------- use_listings_cache, listings_expiry_time, max_paths: passed to ``DirCache``, if the implementation supports directory listing caching. Pass use_listings_cache=False to disable such caching. skip_instance_cache: bool If this is a cachable implementation, pass True here to force creating a new instance even if a matching instance exists, and prevent storing this instance. asynchronous: bool loop: asyncio-compatible IOLoop or None NTFadd_docsz add_docs is no longer supported. add_aliaseszadd_aliases has been removed.r) _cached_intrans _transaction"_invalidated_caches_in_transactionr dircacher:warningswarn FutureWarningrA)selfrrCs rr%zAbstractFileSystem.__init__s0 <<    24/ 3?3   z4 0 MM>2&&rcbt|t|xr|j|jk(Sr) isinstancetyperjrcothers r__eq__zAbstractFileSystem.__eq__s%%d,R5??1RRrcRtt||j|jffSr)rrsrBrCrgs r __reduce__zAbstractFileSystem.__reduce__s$tDz4+<+R>RSSSrct|tr|Dcgc]}|j|c}St|}t|jt r |jfn |j}|D]N}|j |dzr|t|dzd})|j |dzs>|t|dzd}P|jd}|xs |jScc}w)zTurn path from fully-qualified to file-system-specific May require FS-specific handling, e.g., for relative paths or links. ://Nz::rS) rrlist_strip_protocolrrUstr startswithlenrstrip root_marker)rpathpprotosrUs rr~z"AbstractFileSystem._strip_protocols dD !489qC''*9 9d#$.s||S$A#,,s|| 1Hx%/0CMA-/0D1CMA-/0  1 {{3&s&:sC"ct|jtr |jfn |j}|D]}|j|ds|cS|dd|S)z6Format FS-specific path to generic, including protocolrzr)rrrUrr)rcnamerrUs runstrip_protocolz#AbstractFileSystem.unstrip_protocols_%/ s%C$--! H(3/0  )Cv&&rciS)afIf kwargs can be encoded in the paths, extract them here This should happen before instantiation of the class; incoming paths then should be amended to strip the options in methods. Examples may look like an sftp path "sftp://user@host:/my/path", where the user and host should become kwargs and later get stripped. r)rs r_get_kwargs_from_urlsz(AbstractFileSystem._get_kwargs_from_urlss  rcr|j|jvr|j|jS|S)zReturn the most recently instantiated FileSystem If no instance has been created, then create one with defaults )r?r)rs rcurrentzAbstractFileSystem.currents. ;;#** $::ckk* *u rc^|j|j||_|jS)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. )r]transaction_typergs r transactionzAbstractFileSystem.transactions/    $ $ 5 5d ;D    rcTd|_|j||_|jS)z@Begin write transaction for deferring files, non-context versionT)r\rr]rrgs rstart_transactionz$AbstractFileSystem.start_transactions(  11$7rc|jjd|_|jD]}|j ||jj y)z-Finish write transaction, non-context versionN)rcompleter]r^invalidate_cacher=rcrs rend_transactionz"AbstractFileSystem.end_transactionsQ !!# ;; (D  ! !$ ' ( //557rcT|jr|jj|yy)z Discard any cached directory information Parameters ---------- path: string or None If None, clear all listings cached else listings at or under given path. N)r\r^appendrs rrz#AbstractFileSystem.invalidate_cache s$ ==  3 3 : :4 @ rc y)a Create directory entry at path For systems that don't have true directories, may create an for this instance only and not touch the real filesystem Parameters ---------- path: str location create_parents: bool if True, this is equivalent to ``makedirs`` kwargs: may be permissions, etc. Nrrcrcreate_parentsrs rmkdirzAbstractFileSystem.mkdirs rcy)aRecursively make directories Creates directory at path and any intervening required directories. Raises exception if, for instance, the path already exists but is a file. Parameters ---------- path: str leaf directory name exist_ok: bool (False) If False, will error if the target already exists Nrrcrexist_oks rmakedirszAbstractFileSystem.makedirs/s rcy)zRemove a directory, if emptyNrrs rrmdirzAbstractFileSystem.rmdir? rc t)apList objects at path. This should include subdirectories and files at that location. The difference between a file and a directory must be clear when details are requested. The specific keys, or perhaps a FileInfo class, or similar, is TBD, but must be consistent across implementations. Must include: - full path to the entry (without protocol) - size of the entry, in bytes. If the value cannot be determined, will be ``None``. - type of entry, "file", "directory" or other Additional information may be present, appropriate to the file-system, e.g., generation, checksum, etc. May use refresh=True|False to allow use of self._ls_from_cache to check for a saved listing and avoid calling the backend. This would be common where listing may be expensive. Parameters ---------- path: str detail: bool if True, gives a list of dictionaries, where each is the same as the result of ``info(path)``. If False, gives a list of paths (str). kwargs: may have additional backend-specific options, such as version information Returns ------- List of strings if detail is False, or list of directory information dicts if detail is True. rercrdetailrs rlszAbstractFileSystem.lsCs N"!rch|j|} |j|jdS#t$rYnwxYw |j|Dcgc]+}|d|k(s|d|jdk(r |ddk(r|-ncc}w}}t |dk(r t ||S#t$rYywxYw)zCheck cache for listing Returns listing, if found (may be empty list for a directly that exists but contains nothing), None if not in cache. rSrrs directoryrN)_parentr_rKeyErrorrFileNotFoundError)rcrparentffiless r_ls_from_cachez!AbstractFileSystem._ls_from_cachels d# ==S!12 2    v.V9$fIS!11ai;6NE 5zQ'--L   s*1 ==B%0B"B%% B10B1c+K||dkr td|j|}i}i}i}|jdd} |j|fddi|} | D]S} | djd } | jd dd }| d d k(r| |k7r | ||<| ||<D| |k(r| |d <O| ||<U| st|}t|}|r|||f||dz}|dkr |s|||fy|D]$}|j||f|| |d|Ed{&|s|||fyy#tt f$r#} |dk(rt |r|| Yd} ~ yd} ~ wwxYw7Fw)aReturn all files under the given path. List all files, recursing into subdirectories; output is iterator-style, like ``os.walk()``. For a simple list of files, ``find()`` is available. When topdown is True, the caller can modify the dirnames list in-place (perhaps using del or slice assignment), and walk() will only recurse into the subdirectories whose names remain in dirnames; this can be used to prune the search, impose a specific order of visiting, or even to inform walk() about directories the caller creates or renames before it resumes walk() again. Modifying dirnames when topdown is False has no effect. (see os.walk) Note that the "files" outputted will include anything that is not a directory, such as links. Parameters ---------- path: str Root to recurse into maxdepth: int Maximum recursion depth. None means limitless, but not recommended on link-based file-systems. topdown: bool (True) Whether to walk the directory tree from the top downwards or from the bottom upwards. on_error: "omit", "raise", a callable if omit (default), path with exception will simply be empty; If raise, an underlying exception will be raised; if callable, it will be called with a single OSError instance as argument kwargs: passed to ``ls`` Nr maxdepth must be at least 1rFTraiserrSrsrrV)maxdepthrtopdown) ValueErrorr~r:rrOSErrorcallablerrsplitr}walk)rcrrron_errorr full_dirsdirsrrlistingeinfopathnamerds rrzAbstractFileSystem.walksB  HqL:; ;##D) He, dggd:4:6:G #DF|**3/H??3*2.DF|{*x4/?"* $!T T! b "d  #:DKE e# #   MH!|e++ A tyy! !      e# #]"7+ 7"!   L s;.]s-#*-s*?[rFrSrVrTrrrrsr)rrer*raltseprendswithr~r7rrminr:rexistsrrindexcountrcompilerrmatchr})rcrrrrseps ends_with_sepappend_slash_to_dirnameidx_star idx_qmark idx_bracemin_idxrrootdepthidx_double_starsdepth_double_starsallpathspatternrrrs rglobzAbstractFileSystem.glob+sR  HqL:; ;02 RWW^^,RWW[[N d+ ##D)"/# 4== -- -4 &*YYs^q%8499S>c$i&*iin&9DIIcNs4y &*iin&9DIIcNs4y h 95He,t{{4*6* 6M )$))D";F";<<II D'N "8Gn++C0G 'A+&D1'--c2Q6ED1'--c2Q6E 4<##'99T? %)*:*;%<%B%B3%G!%K" 22X=499TXEDXQWX  2!FG**W%"(.."23 4}}*tF|{/JC tG   J9  s-Kc : |j|fi|y#YyxYw)z!Is there a file at the given pathTFrrcrrs rrzAbstractFileSystem.existss'  DIId %f % sc $|j|S)zEIf there is a file at the given path (including broken links))rrs rlexistszAbstractFileSystem.lexistss{{4  rc |j|}|j|j|fddi|}|Dcgc]}|djd|k(s|}}|r|dS|j|fddi|}|jd}|Dcgc]}|djd|k(s|}}t |dk(rd|dvrd|dd<|dSt |dkDs|r|dd d St |cc}wcc}w) a`Give details of entry at path Returns a single dictionary, with exactly the same information as ``ls`` would with ``detail=True``. The default implementation calls ls and could be overridden by a shortcut. kwargs are passed on to ```ls()``. Some file systems might not be able to measure the file's size, in which case, the returned dict will include ``'size': None``. Returns ------- dict with keys: name (full path in the FS), size (in bytes), type (file, directory, or something else) and other FS-specific keys. rTrrSrr rNr)rrrs)r~rrrrr)rcrrroout1s rrzAbstractFileSystem.infos"##D)dggdll4(@@@?Q6!1!1#!6$!>q?? q6Mdggd24262{{3@a!F)"2"23"74"?@@ t9>T!W$"&Q7N Y]c ![A A#D) )@ AsC4C4C9+C9cJtt|j|dS)aUnique value for current version of file If the checksum is the same from one moment to another, the contents are guaranteed to be the same. If the checksum changes, the contents *might* have changed. This should normally be overridden; default will probably capture creation/modification timestamp (which would be good) or maybe access timestamp (which would be bad) rn)rorrrs rchecksumzAbstractFileSystem.checksums8DIIdO,b11rcD|j|jddS)zSize in bytes of filerN)rr&rs rrzAbstractFileSystem.sizesyy""6400rcJ|Dcgc]}|j|c}Scc}w)z-Size in bytes of each file in a list of paths)r)rcpathsrs rrzAbstractFileSystem.sizess&+, ! ,,,s cP |j|ddk(S#t$rYywxYw)zIs this entry directory-like?rsrF)rrrs rrzAbstractFileSystem.isdirs1 99T?6*k9 9  s  %%c@ |j|ddk(S#YyxYw)zIs this entry file-like?rsfileFrrs rrzAbstractFileSystem.isfiles) 99T?6*f4 4 sc ~|j|fd|||d|5}|jcdddS#1swYyxYw)zGet the contents of the file as a string. Parameters ---------- path: str URL of file on this filesystems encoding, errors, newline: same as `open`. rmodeencodingerrorsnewlineNopenread)rcrr r r rrs r read_textzAbstractFileSystem.read_textsSTYY       668   s3<c |j|fd|||d|5}|j|cdddS#1swYyxYw)aWrite the text to the given file. An existing file will be overwritten. Parameters ---------- path: str URL of file on this filesystems value: str Text to write. encoding, errors, newline: same as `open`. wr N)rwrite)rcrvaluer r r rrs r write_textzAbstractFileSystem.write_textsUTYY       "775> " " "s4=c z|j|dfi|5}|?|dk\r|j|n(|jtd|j|z|?|dkr|j|z}|j ||j z cdddS|j cdddS#1swYyxYw)ahGet the content of a file Parameters ---------- path: URL of file on this filesystems start, end: int Bytes limits of the read. If negative, backwards from end, like usual python slices. Either can be None for start or end of file, respectively kwargs: passed to ``open()``. rbNr)rseekmaxrrtell)rcrstartendrrs rcat_filezAbstractFileSystem.cat_filesTYYtT ,V ,  A:FF5MFF3q!&&5.127&&3,CvvcAFFHn-  668   sA9B1B11B: overwritec |dk(r|j|rt|j|dfi|5}|j|dddy#1swYyxYw)zSet the bytes of given filecreatewbN)rFileExistsErrorrr)rcrrr rrs r pipe_filezAbstractFileSystem.pipe_file.sT 8  D 1" ! TYYtT ,V ,  GGEN   s A  Ac t|tr$|j|j||fi|yt|tr<|j D](\}}|j|j||fi|*yt d)aPut value into path (counterpart to ``cat``) Parameters ---------- path: string or dict(str, bytes) If a string, a single remote location to put ``value`` bytes; if a dict, a mapping of {path: bytesvalue}. value: bytes, optional If using a single path, these are the bytes to put there. Ignored if ``path`` is a dict zpath must be str or dictN)rrrr$r~dictrr)rcrrrrGvs rpipezAbstractFileSystem.pipe7s dC DNN4//5u G G d #  E1t33A6DVD E78 8rc |tt|tstt|ts|gt |z}t|ts|gt |z}t |t |k7st |t |k7rt g}t |||D])\}} } |j|j|| | +|S#t$r"} |dk(r|j| nYd} ~ Td} ~ wwxYw)aYGet the contents of byte ranges from one or more files Parameters ---------- paths: list A list of of filepaths on this filesystems starts, ends: int or list Bytes limits of the read. If using a single int, the same value will be used to read all the specified files. Nreturn) rfrrr} TypeErrorrrziprr Exception) rcrstartsendsmax_gaprrrrsrs r cat_rangeszAbstractFileSystem.cat_rangesMs  % %%&O&$'XE *F$%6CJ&D v;#e* $D SZ(? 5&$/ GAq!  4==Aq12   x'JJqM" s("C C9C44C9c R|j|fd|i|}t|dkDs't|ts|d|j |k7r"i}|D]} |j |fi|||<|S|j |dfi|S#t $r}|dk(r|dk(r|||<Yd}~Rd}~wwxYw)aFetch (potentially multiple) paths' contents Parameters ---------- recursive: bool If True, assume the path(s) are directories, and get all the contained files on_error : "raise", "omit", "return" If raise, an underlying exception will be raised (converted to KeyError if the type is in self.missing_exceptions); if omit, keys with exception will simply not be included in the output; if "return", all keys are included in the output, but the value will be bytes or an exception instance. kwargs: passed to cat_file Returns ------- dict of {path: contents} if there are multiple paths or the path has been otherwise expanded recursiver rrr*N) expand_pathrrrr}r~rr-)rcrr4rrrrrs rcatzAbstractFileSystem.catos*!  EEfE JN$%Qx4//55C && - d =f =CI &J 4==q4V4 4!&7*8+$%D &sB B& B!!B&c ddlm}t|r|}n)|j|rt j |dy|d}|j |j |d|j|dfi|5}| t|d} |jt|d dd} | rM|j|j} |j| } | t| } |j| | rMt|s|j dddy#t|s|jwwxYw#1swYyxYw) z Copy single remote file to localr )LocalFileSystemTrN) auto_mkdirrr"r)implementations.localr8rrr*rrrset_sizer1r blocksizerrrelative_updateclose) rcrpathlpathcallbackoutfilerr8fsf1data segment_lens rget_filezAbstractFileSystem.get_files': e G ZZ  KK -  - BJJu% 5 TYYud -f - $ud+ $!!'"fd";<774>>2D")--"5K"*&)$i ,,[9 "%(MMO $ $"%(MMO) $ $s%=E A,D:ED==EE c Bt|trt|tr|}|}nddlm} m} m} t|t } |j|f||d|}| r1|r|-|D cgc]} | | r |j| r| !}} |syt|t r| |}t|dk(}t|t xr | |xs| j|}| xr,t|xr|xst| xr |xr | | }t|||| }|jt||jt||D]6\}}|j||5}|j ||fd|i|ddd8ycc} w#1swYHxYw)aTCopy file(s) to local. Copies a specific file or tree of files (if recursive=True). If lpath ends with a "/", it will be assumed to be a directory, and target files will go within. Can submit a list of paths, which may be glob-patterns and will be expanded. Calls get_file for each source. r r8make_path_posix trailing_sepr4rNrflattenrB)rrr}r;r8rKrLrr5rrrrr<wrapr,branchedrH)rcr@rAr4rBrrrpathslpathsr8rKrL source_is_strrsource_is_file dest_is_dirrchilds rr&zAbstractFileSystem.gets$ eT "z%'>FF   'uc2M%T%%!*XAGFi83G%+VLOtzzRS}!VV%%'. [A-N$UC0U#E'8'>'>u'E #5!4nV!%((T[TeAT=T !)) F #f+&$MM#ff*=> FLE5""5%0 FE eUEUEfE F F F1W2 F Fs F.FF.FF c |dk(r|j|rttjj |r|j |dyt |d5}|jdd}|j||jd|j|jtj|d|j |dfi|5}|j|kr^|j|j} |j| } | t!| } |j#| |j|kr^ddddddy#1swYxYw#1swYyxYw) zCopy single file to remoter!Tr9Nrrr|r")rr#r*rrrrrr<mkdirsrfspathrrr=rrr>) rcrAr@rBr rrErf2rFrGs rput_filezAbstractFileSystem.put_files6 8  E 2! ! 77==  MM%$M / %  :"771a=D   d # GGAJ KK RYYu%56K F5$1&1 :Rggi$&774>>2D"$((4.K"*&)$i ,,[9 ggi$& : : :  : : : :s&A>E)A1E E)E& "E))E2c t|trt|tr|}|}n/ddlm} m} m} t|t } | r| |}| } | j|f||d|}| r1|r|-|Dcgc]}| |r | j|r|!}}|syt|dk(}t|t xr| |xs|j|}t|t r|j|n|Dcgc]}|j|c}}| xr,t|xr|xst| xr |xr | | }t|||| }|jt||jt||D]6\}}|j!||5}|j"||fd|i|ddd8ycc}wcc}w#1swYMxYw)aCopy file(s) from local. Copies a specific file or tree of files (if recursive=True). If rpath ends with a "/", it will be assumed to be a directory, and target files will go within. Calls put_file for each source. r rJrMNrNrB)rrr}r;r8rKrLrr5rrr~rrr<rPr,rQr\)rcrAr@r4rBrrrRrSr8rKrLrTrDrrUrVrrWs rputzAbstractFileSystem.put s" eT "z%'>FF   'uc2M'. "B#R^^!*XAGFi83G%+TLOrxxPQ{!TT [A-N$UC0U#8tzz%'8  eS)$$U+7<=!d**1-=  #5!4nV!%((T[TeAT=T !)) F #f+&$MM#ff*=> FLE5""5%0 FE eUEUEfE F F F5U> F Fs$2 GGG;G #GG ct|j|d5}|j|cdddS#1swYyxYw)z&Get the first ``size`` bytes from filerNrrcrrrs rheadzAbstractFileSystem.headQs2 YYtT " a66$<   s.7c|j|d5}|jt| |j d|j cdddS#1swYyxYw)z%Get the last ``size`` bytes from filerr|N)rrrrrr`s rtailzAbstractFileSystem.tailVsM YYtT " a FF3uqvvg& *668   s 8AAc trrercpath1path2rs rcp_filezAbstractFileSystem.cp_file\s!!rc ||rd}n|d}t|trt|tr|}|}nddlm} t|t} |j |f||d|}| r1|r|-|D cgc]} | | r |j | r| !}} |syt|dk(} t|txr| |xs|j |} | xr,t|xr| xst| xr | xr | | }t|||| }t||D]\}} |j||fi|ycc} w#t$r |dk(rY4wxYw)a?Copy within two locations in the filesystem on_error : "raise", "ignore" If raise, any not-found exceptions will be raised; if ignore any not-found exceptions will cause the path to be skipped; defaults to raise unless recursive is true, where the default is ignore Nignorerr )rLrMrN) rrr}r;rLrr5rrrrr,rhr)rcrfrgr4rrrpaths1paths2rLrTrrUrVrp1p2s rcopyzAbstractFileSystem.copy_s   H  H eT "z%'>FF ;&uc2M%T%%!*XAGFi83G%+VLOtzzRS}!VV [A-N$UC0U#8tzz%'8 #5!4nV!%((T[TeAT=T !)) F&&) FB  R.v. )W.% w&' s$' D55D5D5D::E  E c ||dkr tdt|ttjfr|j |g||fi|}nt }|Dcgc]}|j|}}|D]}t|r`t |j|fd|i|}||z}|r:||dkr;|t |j t|f|||dz ndd|z}n|r%t |j|f|ddd|}||z}||vs|dus|j|s|j||s t|t|Scc}w) zTurn one or more globs or directories into a list of all matching paths to files or directories. kwargs are passed to ``glob`` or ``find``, which may in turn call ``ls`` Nr rrrMTFr)rrrrr*PathLiker5setr~rrr}rraddrr) rcrr4rrrrbitrecs rr5zAbstractFileSystem.expand_paths  HqL:; ; dS"++. /"$""D69hI&IC%C59:D((+:D: Q<idiiGHGGHC3JC $/HM$s,D,, $S *39A9MASW#)  ! (04QWC 3JCCGGAJ9 :#D) )c{A;sEc ||k(rtjd|y|j||||d|j||y)z)Move file(s) from one location to anotherz6%s mv: The paths are the same, so no files were moved.r)r4ronerror)r4N)loggerdebugrorm)rcrfrgr4rrs rmvzAbstractFileSystem.mvsI E> LLQSW X IIu Hg   GGEYG /rc&|j|y)z Delete a fileN)_rmrs rrm_filezAbstractFileSystem.rm_files  rct)zDelete one filerers rr}zAbstractFileSystem._rms "!rcn|j|||}t|D]}|j|y)aDelete files. Parameters ---------- path: str or list of str File(s) to delete. recursive: bool If file(s) are directories, recursively delete contents and then also remove the directory maxdepth: int or None Depth to pass to walk for finding files to delete, if recursive. If None, there will be no limit and infinite recursion may be possible. rMN)r5reversedr~)rcrr4rrs rrzzAbstractFileSystem.rms9 HM$ A LLO rc|j|}d|vr=|jdddj|j}|j|zS|jS)NrSr r)r~rlstripr)rrrs rrzAbstractFileSystem._parentsV""4( $;[[a(+223??CF??V+ +?? "rc &t|||||fd|i|S)z4Return raw bytes-mode file-like from the file-system cache_options)AbstractBufferedFile)rcrr  block_size autocommitrrs r_openzAbstractFileSystem._opens4$       (    rc ddl}|j|}d|vra|jdddz}dDcic]}||vr||j|} }|j|j ||f|||d|fi| S|jd|j } |j|f||| |d |} |'dd lm } dd l m } | ||}| |}|| |d } | s)d |vr%|jjj| | Scc}w)a  Return a file-like object from the filesystem The resultant instance must function correctly in a context ``with`` block. Parameters ---------- path: str Target file mode: str like 'rb', 'w' See builtin ``open()`` Mode "x" (exclusive write) may be implemented by the backend. Even if it is, whether it is checked up front or on commit, and whether it is atomic is implementation-dependent. block_size: int Some indication of buffering - this is a value in bytes cache_options : dict, optional Extra arguments to pass through to the cache. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding, errors, newline: passed on to TextIOWrapper for text mode rNbtrV)r r r )rr compressionr)r rrr)compr)get_compression)r r)ior~replacer: TextIOWrapperrr\rfsspec.compressionr fsspec.corerrrr)rcrr rrrrrrG text_kwargsacrrrcompresss rrzAbstractFileSystem.open s_D ##D) d?<<R(3.D;;6::a= K $2##  *"/ +      Ldmm*;>@@rc|j|d5}|j}||}| ||z|kDr||z }t||||cdddS#1swYyxYw)aRead a block of bytes from Starting at ``offset`` of the file, read ``length`` bytes. If ``delimiter`` is set then we ensure that the read starts and stops at delimiter boundaries that follow the locations ``offset`` and ``offset + length``. If ``offset`` is zero then we start at zero. The bytestring returned WILL include the end delimiter string. If offset+length is beyond the eof, reads to eof. Parameters ---------- fn: string Path to filename offset: int Byte offset to start read length: int Number of bytes to read. If None, read to end. delimiter: bytes (optional) Ensure reading starts and stops at delimiter bytestring Examples -------- >>> fs.read_block('data/file.csv', 0, 13) # doctest: +SKIP b'Alice, 100\nBo' >>> fs.read_block('data/file.csv', 0, 13, delimiter=b'\n') # doctest: +SKIP b'Alice, 100\nBob, 200\n' Use ``length=None`` to read to the end of the file. >>> fs.read_block('data/file.csv', 0, None, delimiter=b'\n') # doctest: +SKIP b'Alice, 100\nBob, 200\nCharlie, 300' See Also -------- :func:`fsspec.utils.read_block` rN)rrr)rcfnoffsetlength delimiterrrs rrzAbstractFileSystem.read_blocknseJYYr4  >"!CLL>2!+EE4=!AU1Xu//0A0AB ,,_=   rc"ddlm}|}t|}|j|}| t d|j dd|j dd||j |j ddi|j |S) a Recreate a filesystem instance from dictionary representation. See ``.to_dict()`` for the expected structure of the input. Parameters ---------- dct: Dict[str, Any] Returns ------- file system instance, not necessarily of this particular class. Warnings -------- This can import arbitrary modules (as determined by the ``cls`` key). Make sure you haven't installed any modules that may execute malicious code at import time. r rNz#Not a serialized AbstractFileSystemrrUrr)rrr&try_resolve_fs_clsrr:unmake_serializable)dctr json_decoderrs r from_dictzAbstractFileSystem.from_dicts* 0,. 3i#66s; ;BC C t  D!  - -cggfb.A B ..s3  rc|S)zW Make a version of the FS instance which will be acceptable to pyarrow rrgs r_get_pyarrow_filesystemz*AbstractFileSystem._get_pyarrow_filesystem$s  rc(ddlm}||||||S)zCreate key/value store based on this file-system Makes a MutableMapping interface to the FS at the given root path. See ``fsspec.mapping.FSMap`` for further details. r )FSMap)checkr!missing_exceptions)mappingr)rcrrr!rrs r get_mapperzAbstractFileSystem.get_mapper+s# #  1   rc8|jjy)a Clear the cache of filesystem instances. Notes ----- Unless overridden by setting the ``cachable`` class attribute to False, the filesystem class stores a reference to newly created instances. This prevents Python's normal rules around garbage collection from working, since the instances refcount will not drop to zero until ``clear_instance_cache`` is called. N)r)r=rs rclear_instance_cachez'AbstractFileSystem.clear_instance_cache;s rct)z=Return the created timestamp of a file as a datetime.datetimerers rcreatedzAbstractFileSystem.createdJ!!rct)z>Return the modified timestamp of a file as a datetime.datetimerers rmodifiedzAbstractFileSystem.modifiedNrrc 0d#d} g} |r| j||rcd|z} |j|d} | jd|"t| |kDr| d|} t| |z }n| } d}t | D]\}}|t| d z k(xr|dk(}|r d d |d z zzn d d |d z zz}|dz }||r| n dd|d z zzz}t j j|jdd}|r|jddk(r|j|jddd}td|D}td|D}|dk(r|dk(rd}n|dk(rd|d|d kDrdndd}ni|dk(rd|d|d kDrdndd}nSd|d|d kDrdndd|d|d kDrdndd }n5|r1|jddk(rd| |jddd}nd}| j|||||jddk(s|dkDs| j|j|jdd|d z ||||d||dkDr&|d }| j|d d |d z zzd|d!jd"| DS)$a Return a tree-like structure of the filesystem starting from the given path as a string. Parameters ---------- path: Root path to start traversal from recursion_limit: Maximum depth of directory traversal max_display: Maximum number of items to display per directory display_size: Whether to display file sizes prefix: Current line prefix for visual tree structure is_last: Whether current item is last in its level first: Whether this is the first call (displays root path) indent_size: Number of spaces by indent Returns ------- str: A string representing the tree structure. Example ------- >>> from fsspec import filesystem >>> fs = filesystem('ftp', host='test.rebex.net', user='demo', password='password') >>> tree = fs.tree(display_size=True, recursion_limit=3, indent_size=8, max_display=10) >>> print(tree) cLdD]\}}|d|zk\s||z dd|dcS|dS)zFormat bytes as text.))Pl )Tl)Gi@)Mi)rGg?z.2f rBr)nprefixrGs r format_bytesz-AbstractFileSystem.tree..format_bytesxsJ 4 a<!eC[&33 4S7NrrT)rcN|jddk7|jddfS)NrsrrrVr&)xs rz)AbstractFileSystem.tree..s#quuV} ;QUU62=NOr)keyNrr u└u─r|u├u│rrVrsrc3JK|]}|jddk(sdyw)rsrr Nrr2sub_items rr4z*AbstractFileSystem.tree..s&$&(,,v:NRX:X$s##c3HK|]}|jddk(rdyw)rsrr Nrrs rr4z*AbstractFileSystem.tree..s)&$#<</;>&s "z (empty folder)z (z subfolderr1)z file, rrF)rrecursion_limit max_display display_sizeris_lastfirst indent_sizez more item(s) not displayed. c3&K|] }|s| ywrr)r2rs rr4z*AbstractFileSystem.tree..s0qa0s)rror*r) rrsortr enumerater*rbasenamer&rtreejoin)rcrrrrrrrrrresultindentcontentsdisplayed_contentsremaining_countiitem is_last_itembranch new_prefixr sub_contents num_files num_foldersr more_messages rrzAbstractFileSystem.treeRseL   MM$  ;&FwwtDw1H MMO  &3x=;+F%-l{%;""%h-+"=%-""#$%787 4 !S);%88F#{27JMM !%&"!5,;a,?(3)5#-$0"'(3"  Y7 r""1!22NO  huq(ABC1\NSyy0F000rc .|j|f||d|S)z'Alias of `AbstractFileSystem.cat_file`.rr)r)rcrrrrs r read_byteszAbstractFileSystem.read_bytesst}}TBCB6BBrc ,|j||fi|y)z(Alias of `AbstractFileSystem.pipe_file`.N)r$)rcrrrs r write_byteszAbstractFileSystem.write_bytesstU-f-rc ,|j|fd|i|S)z$Alias of `AbstractFileSystem.mkdir`.r)rrs rmakedirzAbstractFileSystem.makedirstzz$H~HHHrc(|j||S)z'Alias of `AbstractFileSystem.makedirs`.r9)rrs rrYzAbstractFileSystem.mkdirss}}TH}55rc ,|j|fd|i|S)z!Alias of `AbstractFileSystem.ls`.r)rrs rlistdirzAbstractFileSystem.listdirstwwt5F5f55rc *|j||fi|S)z#Alias of `AbstractFileSystem.copy`.)rores rcpzAbstractFileSystem.cpstyy000rc *|j||fi|Sz!Alias of `AbstractFileSystem.mv`.r{res rmovezAbstractFileSystem.movetwwue.v..rc (|j|fi|S)z#Alias of `AbstractFileSystem.info`.rrs rstatzAbstractFileSystem.statstyy(((rc .|j|f||d|S)z!Alias of `AbstractFileSystem.du`.)rr)r)rcrrrrs r disk_usagezAbstractFileSystem.disk_usagestwwtF58FvFFrc *|j||fi|Sr rres rrenamezAbstractFileSystem.renamerrc*|j|||S)z!Alias of `AbstractFileSystem.rm`.rM)rz)rcrr4rs rdeletezAbstractFileSystem.deleteswwty8wDDrc .|j||fd|i|S)z"Alias of `AbstractFileSystem.put`.r4)r^)rcrAr@r4rs ruploadzAbstractFileSystem.uploadtxxuD DVDDrc .|j||fd|i|S)z"Alias of `AbstractFileSystem.get`.r4r)rcr@rAr4rs rdownloadzAbstractFileSystem.download rrc td)aCreate a signed URL representing the given path Some implementations allow temporary URLs to be generated, as a way of delegating credentials. Parameters ---------- path : str The path on the filesystem expiration : int Number of seconds to enable the URL for (if supported) Returns ------- URL : str The signed URL Raises ------ NotImplementedError : if method is not implemented for a filesystem z+Sign is not implemented for this filesystemre)rcr expirationrs rsignzAbstractFileSystem.signs,""OPPrcy)NFrrgs r _isfilestorezAbstractFileSystem._isfilestore(s r)rrr*rr)TF)NTomit)NFF)TNF)NNN)NN)r)Nr*)Fr)r)FNN)FN)rNTN)rNNN)rboolr*r)rrr*rR)rr%r*rW)rrWr*rR)rVFFN)rSr|FrVTT)rrrrorrorr%rrrr%rr%rror*r)TN)d)irLrMrNrOr>r[r=rrU__annotations__r?rDr6rrrr8r9r%propertyrhrjrlrprvrx classmethodr~r staticmethodrrrrrrrrrrrrrrrrrrrrrrrrrrr$r(r2r6r rHr&r\r^rarcrhror5r{r~r}rzrrrrrrrrrrrrrrrrrrrrYr r rrrrrrrr r"rrrrRrRdsHGI C0:H-:GJK""$ "!##'R"" 'ST''$'  !! 8A" $  '"R2\$|$7L DiV! *D 21-(@D"229.;C D&5P/?$D! >FB&6K:8! BFH  "FJ6p.` 0" &##  . L\&"A+abrr"xbzFile mode not supportedrr)corer0rrDr DEFAULT_BLOCK_SIZEr=locrrrclosedr`rarbr:rrfrdetails _fetch_rangecacherBytesIObufferrforcedlocation) rcrDrr rr cache_typerrrr0s rr%zAbstractBufferedFile.__init__<s-L !  '15F'FD # #J $   M V  MMJ  %+JJv$6M& ! / /%&?@ @ 4<   LL0 + + 1 1499@MDJ**,DKDKDK DMrc|j*|jj|j|_|jSr)_detailsrDrrrgs rr9zAbstractBufferedFile.detailss. ==  GGLL3DM}}rc&||_|d|_y)Nr)rBr)rcrs rr9zAbstractBufferedFile.detailss &M rcBt|j|jSr)rrrDrgs r full_namezAbstractBufferedFile.full_names DGG44rct|ddS)N_closedTr0rgs rr8zAbstractBufferedFile.closedstY--rc||_yr)rG)rccs rr8zAbstractBufferedFile.closeds  rcrd|jvr t|Stt|jdS)Nrrn)r idrorr9rgs rrpzAbstractBufferedFile.__hash__s- $)) d8Ox -r2 2rc||uryt|t|xr9|jdk(xr(|jdk(xrt|t|k(S)zAFiles are equal if they have the same checksum, only in read modeTr)rrrsr hashrts rrvzAbstractBufferedFile.__eq__sX 5= ud4j ) * T! * d" *T d5k)  rcy)z#Move from temp to final destinationNrrgs rcommitzAbstractBufferedFile.commitrcy)zThrow away temporary fileNrrgs rdiscardzAbstractBufferedFile.discardrPrcP|jr |jStd)z File information about this pathz Info not available while writing)readabler9rrgs rrzAbstractBufferedFile.infos! ==?<< ?@ @rc|jS)zCurrent file location)r7rgs rrzAbstractBufferedFile.tells xxrct|}|jdk(sttd|dk(r|}n9|dk(r|j|z}n$|dk(r|j |z}nt d|d|dkr t d||_|jS) zSet current file location Parameters ---------- loc: int byte location whence: {0, 1, 2} from start of file, current location or end of file, resp. rz Seek only available in read moderr r|zinvalid whence (z, should be 0, 1 or 2)zSeek before start of file)ror rrr7rr)rcr7whencenlocs rrzAbstractBufferedFile.seeks#hyyD &"DE E Q;D q[88c>D q[99s?D/x7MNO O !889 9xxrcf|js td|jr td|jr td|jj |}|xj |z c_|jj|jk\r|j|S)z Write data to buffer. Buffer only sent on flush() or if buffer is greater than or equal to blocksize. Parameters ---------- data: bytes Set of bytes to be written. zFile not in write modeI/O operation on closed file.z0This file has been force-flushed, can only close) writablerr8r>r=rr7rr=flush)rcrFrs rrzAbstractBufferedFile.writes}}56 6 ;;<= = ;;OP Pkk% C ;;    / JJL rc|jr td|r|jr td|rd|_|jry|s(|jj |j kry|jd|_ |j|j|durI|xj|jjddz c_tj|_yy#d|_xYw) a{ Write buffered data to backend store. Writes the current buffer, if it is larger than the block-size, or if the file is being closed. Parameters ---------- force: bool When closing, write the last block even if it is smaller than blocks are allowed to be. Disallows further writing to this file. zFlush on closed filez+Force flush cannot be called more than onceTNr)finalFr|) r8rr>rTr=rr=r_initiate_upload _upload_chunkrrr<)rcforces rr\zAbstractBufferedFile.flushs ;;34 4 T[[JK K DK ==? ))+dnn<  ;; DK %%'   E  *% 7 KK4;;++Aq1 1K**,DK 8  " s C77 Dcy)zWrite one part of a multi-block file upload Parameters ========== final: bool This is the last block, so should complete file, if self.autocommit is True. Nr)rcr^s rr`z"AbstractBufferedFile._upload_chunk$rPrcy)zCreate remote file/uploadNrrgs rr_z%AbstractBufferedFile._initiate_upload/rrcR|jj|j||S)z*Get the specified set of bytes from remoter)rDrr)rcrrs rr:z!AbstractBufferedFile._fetch_range3s"ww C@@rc|dn t|}|jdk7r td|dkr|j|jz }|j r td|dk(ry|j j|j|j|z}tjd||j|j|z|j j|xjt|z c_|S)z Return data from cache, or fetch pieces as necessary Parameters ---------- length: int (-1) Number of bytes to read; if <0, all remaining bytes. rrzFile not in read moderrZrz%s read: %i - %i %s) ror rrr7r8r;_fetchrxry _log_statsr)rcrrs rrzAbstractBufferedFile.read7s~3v; 99 45 5 A:YY)F ;;<= = Q;jj$((V*;< !  HH HHv  JJ ! ! #   CH rct|jd}|j|j}||dt |t |S)zxmirrors builtin file's readinto method https://docs.python.org/3/library/io.html#io.RawIOBase.readinto rN) memoryviewcastrnbytesr)rcrrrFs rreadintozAbstractBufferedFile.readintoVsD m  %yy$Kc$i4yrcxg} |j}|j|xs |j}t|dk(rni|j |}|dkDrA|j |d|t|z|j ||zt|zn|j |dj|S)aReturn data between current position and first occurrence of char char is included in the output, except if the end of the tile is encountered first. Parameters ---------- char: bytes Thing to find blocks: None or int How much to read in each go. Defaults to file blocksize - which may mean a new read on every call. rrNr)rrr=rrrrr)rccharblocksrrpartfounds r readuntilzAbstractBufferedFile.readuntil`sIIKE99V5t~~6D4yA~IIdOErz 4 3%#d)"345 %%-#d)34 JJt xx}rc$|jdS)zRead until and including the first occurrence of newline character Note that, because of character encoding, this is not necessarily a true line ending.  )rrrgs rreadlinezAbstractBufferedFile.readline|s ~~e$$rc6|j}|r|Str)ru StopIteration)rcrs r__next__zAbstractBufferedFile.__next__smmo Jrc|Srrrgs r__iter__zAbstractBufferedFile.__iter__ rc|j}|jd}|ddDcgc]}|dz }}|jdr|S||dgzScc}w)zPReturn all data, split by the newline character, including the newline characterrtNr)rsplitr)rcrFlineslrs r readlineszAbstractBufferedFile.readliness_yy{ 5!"'*-Qq5y-- == J%)$ $ .s Ac$|j|Sr)rl)rcrs r readinto1zAbstractBufferedFile.readinto1s}}Qrct|ddry|jry |jdk(rd|_n|js|j d|j c|j j|j|j j|j j|jd|_y#d|_wxYw)z=Close file Finalizes writes, discards cache _unclosableFNrT)ra) r1r8r r;r>r\rDrrrrgs rr?zAbstractBufferedFile.closes 4 .  ;;  yyD ! {{JJTJ*77&GG,,TYY7GG,,TWW__TYY-GHDK$DKs B$C Cc<d|jvxr |j S)zWhether opened for readingrr r8rgs rrTzAbstractBufferedFile.readablesdii3 O3rc"|jS)z'Whether is seekable (only in read mode))rTrgs rseekablezAbstractBufferedFile.seekables}}rc<|jdvxr |j S)zWhether opened for writing>r3r"r4rrgs rr[zAbstractBufferedFile.writablesyy..Bt{{?Brc <|jdk7r tdt|j|j|j|j |j |j|j|jr|jjnd|jf fS)Nrz*Pickling a writeable file is not supportednone) r  RuntimeErrorreopenrDrr=r7rrr;rrrgs rrxzAbstractBufferedFile.__reduce__sw 99 KL L GG II II NN HH II OO#zzDJJOOv KK   rc>|js|jyyr)r8r?rgs r__del__zAbstractBufferedFile.__del__s{{ JJLrcbdt|jjd|jdS)Nz)rsrDrLrrgs r__str__zAbstractBufferedFile.__str__s*#DM$:$:#;2dii[JJrc|Srrrgs r __enter__zAbstractBufferedFile.__enter__r{rc$|jyr)r?)rcrs r__exit__zAbstractBufferedFile.__exit__s  r)rr1T readaheadNN)rr#)r)rtN)*rLrMrNrOr6rBr%r*r9setterrEr8rprvrOrRrrrrr\r`r_r:rrlrrrurxrzrrr?rTrr[rxrr__repr__rrrrrrr0s8#H  O!b  ^^""55..  ]]3  2(A40('T  A>8% % ,4C KHrrc d|j|f|||||d|} |dkDr| j|| S)N)r rrr@rr)rr) rDrr r=r7rrr@rrs rrrsL 277        D Qw # Kr), __future__rrrloggingr*r;r`r'errnorrrhashlibrtypingrr callbacksr configr r r_r rrutilsrrrrrrr getLoggerrxrrsr!rRIOBaserrrrrrs"   '&$   8 $ =d=@I7IX6j299jZ r