L i lUddlmZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z ddl mZddlmZddlmZmZmZmZmZmZmZerddlZddlmZed Zned Zed Zej<d Zee e ge!fZ"ee#e e fge!fZ$Gd d Z%Gdde%Z&Gdde%Z'Gdde%Z(Gdde%Z)Gdde%Z*Gdde%Z+Gdde%Z,GddeeefZ-Gdde%Z.de%iZ/d e0d!<d#d$d"Z1e%e&e*e'e)e(e+e,e.f D] Z2e1e2 y)%) annotationsN) OrderedDict)FutureThreadPoolExecutor)groupby) itemgetter) TYPE_CHECKINGAnyCallableClassVarGeneric NamedTupleTypeVar) ParamSpecPTfsspeccHeZdZUdZdZded<d dZd dZd dZddZ dd Z y ) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is none ClassVar[str]namecf||_d|_||_||_d|_d|_d|_yNr) blocksizenblocksfetchersize hit_count miss_counttotal_requested_bytes)selfrrrs T/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/fsspec/caching.py__init__zBaseCache.__init__<s4"   %&"ct|d}| |j}||jk\s||k\ry|j||S)Nrr%)rrr"startstops r#_fetchzBaseCache._fetchFs@ =E <99D DII $||E4((r%c.d|_d|_d|_y)zAReset hit and miss counts for a more ganular report e.g. by file.rN)rr r!r"s r# _reset_statszBaseCache._reset_statsOs%&"r%c |jdk(r|jdk(ryd|jd|jd|jd|jd S)z2Return a formatted string of the cache statistics.rz , z: z hits, z misses, z total requested bytes)rr rr!r,s r# _log_statszBaseCache._log_statsUs^ >>Q 4??a#7TYYKr$..!18ISWSmSmRnoEF Fr%cd|jjd|jd|jd|jd|j d|j d|jdS) Nz ) __class____name__rrrrr r!r,s r#__repr__zBaseCache.__repr__\s .. ! ! "#!^^,-!\\N+!YYK(!^^,-!__-.$$($>$>#?@   r%NrintrFetcherrr6returnNoner( int | Noner)r;r8bytesr8r9)r8str) r3 __module__ __qualname____doc__r__annotations__r$r*r-r0r4r%r#rr*s- !D- ')' F  r%rcjeZdZdZdZ d d fd Zd dZd dZd dZd dZ xZ S) MMapCacheamemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: Fetcher Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is location: str Where to create the temporary file. If None, a temporary file is created using tempfile.TemporaryFile(). blocks: set[int] Set of block numbers that have already been fetched. If None, an empty set is created. multi_fetcher: MultiFetcher Function of the form f([(start, end)]) which gets bytes from remote as specified. This function is used to fetch multiple blocks at once. If not specified, the fetcher function is used instead. mmapct||||| tn||_||_||_|j |_yN)superr$setblockslocation multi_fetcher _makefilecache)r"rrrrLrKrMr2s r#r$zMMapCache.__init__sB GT2%~ce6   *^^% r%c4ddl}ddl}|jdk(r tS|j)t j j|js|j |j}t|_ nt|jd}|j|jdz |jd|jnt|jd}|j|j|jS)Nrzwb+1zr+b)rFtempfiler bytearrayrLospathexists TemporaryFilerJrKopenseekwriteflushfileno)r"rFrSfds r#rNzMMapCache._makefiles 99>;  == t}}(E}}$++-!e $--/ GGDIIM " HHTN HHJdmmU+Btyydii00r%c  tjd|d||d}| j}|jk\s||k\ry|jz}|jz}t ||dz}fd|D}xj t fd|Dz c_g}tt|d D]\}} tttd| } | djz} t| d jzjzj} xj| | z z c_ tjd | dd| d d | d| d |j| | fjj!| xj"t%| z c_|sj&||Sj(rmtjd |tj)|D]7\} } || \} } tjd| d| | j&| | 9nD|D]?\} } tjd| d| j+| | j&| | Aj&||S)NzMMap cache fetching -rr%rQc3@K|]}|jvs|ywrHrK.0ir"s r# z#MMapCache._fetch..s?a!4;;*>?c3@K|]}|jvsdyw)rQNrbrcs r#rfz#MMapCache._fetch..sIAT[[8HaIrgc|d|dz S)NrrQrC)xs r#z"MMapCache._fetch..s1!r%)keyzMMap get blocks z ()zMMap copy block (zMMap get block ()loggerdebugrrrangersumr enumeratetuplemaprminr!appendrKupdater lenrOrMr)r"r(end start_block end_block block_rangeneedranges__blockssstartsendidxrs` r#r*zMMapCache._fetchsy +E7!C59: =E ;))C DII #t~~- 4>>) KQ7 ?;? #IIII")D/7LM ,JAwC 1 w78GQZ$..0Fwr{T^^3dnnDdiiPD  & &$- 7 & LL"71:,a }BvhavQO  MM64. ) KK  w ' OOs7| +O% ,(::eC( (    LL+F84 5#D$6$6v$>? ,Q!' 0$@A*+ 6$' , !' E  /xq?@*.,,vt*D 6$' Ezz%$$r%c@|jj}|d=|S)NrO)__dict__copyr"states r# __getstate__zMMapCache.__getstate__s  ""$ 'N r%cd|jj||j|_yrH)rrxrNrOrs r# __setstate__zMMapCache.__setstate__s" U#^^% r%)NNN)rr6rr7rr6rLz str | NonerKzset[int] | NonerMzMultiFetcher | Noner8r9)r8zmmap.mmap | bytearrayr(r;rzr;r8r<r8dict[str, Any]rrr8r9) r3r?r@rArr$rNr*rr __classcell__r2s@r#rErEisx8 D $"&-1 & & & &  &  &+ &  &1,<%| &r%rEc0eZdZdZdZdfd ZddZxZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). readaheadcRt||||d|_d|_d|_y)Nr%r)rIr$rOr(rzr"rrrr2s r#r$zReadAheadCache.__init__s) GT2  r%c^|d}|||jkDr |j}||jk\s||k\ry||z }||jk\rM||jkr>|xjdz c_|j||jz ||jz S|j|cxkr|jkrOnnL|xj dz c_|j||jz d}|t |z}|j}n|xj dz c_d}t|j||jz}|xj||z z c_ |j|||_||_|jt |jz|_||jd|zSNrr%rQ) rr(rzrrOr ryrvrr!r)r"r(rzlparts r#r*zReadAheadCache._fetchse =E ;# /))C DII # %K DJJ 3$((? NNa N::edjj033CD D ZZ5 +488 + OOq O::edjj023D TNAHHE OOq OD$))S4>>12 ""cEk1"\\%-  ::DJJ/djj!n$$r%r5rr3r?r@rArr$r*rrs@r#rrs D %r%rc0eZdZdZdZdfd ZddZxZS)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstcD||kDr|}t||||d|_yrH)rIr$rOrs r#r$zFirstChunkCache.__init__.s( t I GT2#' r%cn|xsd}||jkDrtjdyt||j}||jkr&|j |xj dz c_||jkDr@|xj|z c_|jd|}|d|j|_||dS|jd|j|_|xj|jz c_|j ||}||jkDrA|xj||jz z c_||j|j|z }|xjdz c_ |S|xj dz c_|xj||z z c_|j||S)Nrz,FirstChunkCache: requested start > file sizer%rQ) rrorprvrrOr r!rr)r"r(rzdatars r#r*zFirstChunkCache._fetch5sh  499  LLG H#tyy! 4>> !zz!1$'..#5.<<3/D!%&6!7DJ<'!\\!T^^< **dnn<*::eC(DT^^#**cDNN.BB* T^^S99 NNa NK OOq O  & &#+ 5 &<<s+ +r%r5rrrs@r#rr%s D(,r%rceZdZdZdZ d d fd ZdZd dZd dZddZ dfd Z dd Z xZ S) BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. blockcachect||||tj||z |_||_t j||j|_ yrH) rIr$mathceilr maxblocks functools lru_cache _fetch_block_fetch_block_cachedr"rrrrr2s r#r$zBlockCache.__init__lsR GT2yy !12 "#A9#6#6y#A$BSBS#T r%c6|jjSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. r cache_infor,s r#rzBlockCache.cache_infot''2244r%c$|j}|d=|S)Nrrrs r#rzBlockCache.__getstate__s  ' ( r%c|jj|tj|d|j|_y)Nr)rrxrrrrrs r#rzBlockCache.__setstate__s< U##J9#6#6u[7I#J   $  r%c|d}| |j}||jk\s||k\ry||jz}||jz}t||dzD]}|j||j ||||S)Nrr%rQstart_block_numberend_block_number)rrrqr _read_cache)r"r(rzrr block_numbers r#r*zBlockCache._fetchs =E ;))C DII ##dnn4$..0""46F6JK 3L  $ $\ 2 3  1-    r%c@||jkDrtd|d|jd||jz}||jz}|xj||z z c_|xjdz c_t j d|t|!||}|S)= Fetch the block of data for `block_number`. 'block_number=(' is greater than the number of blocks (rnrQzBlockCache fetching block %d) r ValueErrorrr!r roinforIr*)r"rr(rzblock_contentsr2s r#rzBlockCache._fetch_blocks $,, & /))-a9  t~~-dnn$ ""cEk1" 1 2LAs3r%c ||jz}||jz}|xjdz c_||k(r|j|}|||S|j||dg}|jt |jt |dz||j |j|d|dj|Sz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. rQNr%rrrextendrurqrwjoin r"r(rzrr start_posend_posblockouts r#rzBlockCache._read_cachesDNN* & ! !1 1334FGE7+ +++,>? KLC JJ,,,q02BC  JJt//0@A(7K L88C= r% rr6rr7rr6rr6r8r9rrr)rr6r8r< r(r6rzr6rr6rr6r8r<) r3r?r@rArr$rrrr*rrrrs@r#rrSs, DMOUU'.U69UFIU U 5   .$&!&!"&!8;&!OR&! &!r%rcZeZdZUdZdZded< d d fd Zd dZd dZxZ S) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. r<rrc`t||||d|_d|_d|_||_y)Nr%)rIr$rOr(rztrim)r"rrrrr2s r#r$zBytesCache.__init__s2 GT2 !% # r%c|d}| |j}||jk\s||k\ry|jc||jk\rT|jH||jkr9||jz }|xjdz c_|j|||z|z S|j r$t |j||j z}n|}||k(s||jkDry|j||jkrh|j||jkDrM|xj||z z c_|xjdz c_|j|||_||_n|jJ|jJ|xjdz c_||jkr|j|j|z |j kDr8|xj||z z c_|j|||_||_n4|xj|j|z z c_|j||j}||_||jz|_n|j||jkDr|j|jkDrn||jz |j kDr7|xj||z z c_|j|||_||_nR|xj||jz z c_|j|j|}|j|z|_|jt|jz|_||jz }|j|||z|z }|jrq|j|jz |j dzz}|dkDrC|xj|j |zz c_|j|j |zd|_|Sr) rr(rzrrOrrvr!r rryr)r"r(rzoffsetbendnewrnums r#r*zBytesCache._fetchsJ =E ;))C DII # JJ "#$dhhTZZ'F NNa N::fv|e';< < >>tyy#"67DD 5=EDII- JJ %$**"4 HH dhh  & &$, 6 & OOq OeT2DJDJ::) ))88' '' OOq Otzz!88#txx#~'F..$,>.!%eT!:DJ!&DJ..$**u2DD.,,udjj9C!&DJ!$tzz!1DJ%$/88dii'488^dnn4..$,>.!%eT!:DJ!&DJ..$/A.,,txx6C!%c!1DJ::DJJ/#jj&3,"67 9988djj(dnnq.@ACQw dnns22 !ZZ(<(>?  r%c,t|jSrH)ryrOr,s r#__len__zBytesCache.__len__=s4::r%)T) rr6rr7rr6rboolr8r9r)r8r6) r3r?r@rArrBr$r*rrrs@r#rrsT "D-!IM'.69AE GRr%rcXeZdZUdZdZded< d dfd Zd dZxZS) AllBytesz!Cache entire contents of the fileallrrct|||||P|xjdz c_|xj|jz c_|j d|j}||_y)NrQr)rIr$r r!rrr)r"rrrrr2s r#r$zAllBytes.__init__FsY GT2 < OOq O  & &$)) 3 &<<499-D r%cJ|xjdz c_|j||S)NrQ)rrr's r#r*zAllBytes._fetchTs! !yyt$$r%)NNNN) rr;rzFetcher | Nonerr;rz bytes | Noner8r9r: r3r?r@rArrBr$r*rrs@r#rrAsX+D-!%"&!         %r%rc\eZdZUdZdZded< d dfd Zd fd ZxZS) KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsrrc t||||||_|rt|j }|dg}|j |dg} |ddD]m\} } |d\} } | | k(r&| | f|d<| dxx|j | | fz cc<9|j | | f| j |j | | fott|| |_ yi|_ y)NrrQrm) rIr$strictsortedkeyspoprwdictzipr)r"rrrrrr old_offsetsoffsetsrKr(r)start0stop0r2s r#r$zKnownPartsOfAFile.__init__rs GT2   -K"1~&Ghh{1~./F*12 ; t ' E>#)4.GBK2J$((E4="99JNNE4=1MM$((E4="9: ;S&12DIDIr%ct|d}| |j}d}|jjD]t\\}}}||cxkr|ksn||z }||||z|z }|jr||cxkr|kr3nn0|d||z t |z zz }|xj dz c_|cS|}n|j td||fdtjd||fdtjd|d ||xj||z z c_ |xjdz c_ |t|=||zS) Nrr%rQz&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching r`)rritemsrryrrrwarningswarnrorpr!r rIr*) r"r(r)rloc0loc1roffr2s r#r*zKnownPartsOfAFile._fetchs] =E <99D"&))//"3  LT4$u#t#dl3te!34{{dd&:d&: 7dUlSX&=>>CNNa'NJ !E' 0 << Eudm_TVWX X  4eT]OD2 3   8qGH ""dUl2" 1UW^E4000r%)NT) rr6rr7rr6rz#dict[tuple[int, int], bytes] | Nonerrrr r:rrs@r#rrYse,"D-!59   2   <+1+1r%rcPeZdZdZGddeZd d dZd dZd dZddZ ddZ y ) UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache c6eZdZUded<ded<ded<ded<y)UpdatableLRU.CacheInfor6hitsmissesmaxsizecurrsizeN)r3r?r@rBrCr%r# CacheInfors    r%rctj|_||_||_d|_d|_tj|_ yr) collectionsr_cache_func _max_size_hits_misses threadingLock_lock)r"funcmax_sizes r#r$zUpdatableLRU.__init__s<+6+B+B+D  !  ^^% r%c^|rtd|j|j5||jvrH|jj ||xj dz c_|j|cdddS ddd|j |i|}|j5||j|<|xjdz c_t|j|jkDr|jjdddd|S#1swYxYw#1swY|SxYw)Nz Got unexpected keyword argument rQFlast) TypeErrorrr r move_to_endr rr ryrpopitem)r"argskwargsresults r#__call__zUpdatableLRU.__call__s >v{{}oNO O ZZ )t{{" ''- a {{4(  ) )" ) T,V, ZZ 0 &DKK  LLA L4;;$..0 ###/  0   ) ) 0  sA D)A#D"D"D,cb|j5||jvcdddS#1swYyxYwrH)r r)r"rs r# is_key_cachedzUpdatableLRU.is_key_cacheds* ZZ '4;;& ' ' 's%.c|j5||j|<t|j|jkDr|jj ddddy#1swYyxYw)NFr)r rryrr)r"rrs r#add_keyzUpdatableLRU.add_keysV ZZ 0 &DKK 4;;$..0 ###/ 0 0 0s AA$$A-c|j5|j|jt|j|j |j cdddS#1swYyxYw)N)rrrr)r rrryrr r r,s r#rzUpdatableLRU.cache_infosP ZZ >>T[[)ZZ|| "   s AAA&N))rzCallable[P, T]rr6r8r9)rzP.argsrzP.kwargsr8r)rr r8r)rrrr r8r9r8r) r3r?r@rArrr$rrrrrCr%r#rrs, J &&'0 r%rceZdZUdZdZded< d d fd ZddZddZddZ dd Z ddfd Z dd Z xZ S)BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. backgroundrrct||||tj||z |_||_t |j||_td|_ d|_ d|_ tj|_y)NrQ max_workers)rIr$rrrrrrrr_thread_executor_fetch_future_block_number _fetch_futurer r _fetch_future_lockrs r#r$zBackgroundBlockCache.__init__ss GT2yy !12 "#/0A0A9#M 2q A6:'37"+.."2r%c6|jjSrrr,s r#rzBackgroundBlockCache.cache_info#rr%c<|j}|d=|d=|d=|d=|d=|S)Nrr'r(r)r*rrs r#rz!BackgroundBlockCache.__getstate__.s<  ' ( $ % . / / " & ' r%c|jj|t|j|d|_t d|_d|_d|_tj|_ y)NrrQr%) rrxrrrrr'r(r)r r r*rs r#rz!BackgroundBlockCache.__setstate__7sZ U##/0A0A5CU#V 2q A*.'!"+.."2r%c|d}| |j}||jk\s||k\ry||jz}||jz}d}d}|j5|j|jJ|jj rbt jd|jj|jj|jd|_d|_nKt||jcxkxr|knc}|r&|j}|j}d|_d|_ddd|?t jd|jj|j|t||dzD]}|j||dz} |j5|j]| |jkrN|jj| s3| |_|jj!|j"| d|_ddd|j%||||S#1swYxYw#1swY+xYw)Nrr%z3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.rQasyncr)rrr*r)r(donerorrrrrrqrrr'submitrr) r"r(rzrrfetch_future_block_number fetch_future must_joinrend_block_plus_1s r#r*zBackgroundBlockCache._fetch?s^ =E ;))C DII ##dnn4$..0$(!  $ $ 2!!-66BBB%%**,KK UV,,44**113T5T5T7;D3)-D&!%*::,+,!I !594S4S1'+'9'9 ;?7-1*7 2<  # KKB C  $ $ , ,##%'@  ""46F6JK 3L  $ $\ 2 3 ,a/  $ $ ""*$ 400>>?OP2B/%)%:%:%A%A%%'7&"   1-    o 2 2X  sC"H>7A*I >I IcB||jkDrtd|d|jd||jz}||jz}tj d|||xj ||z z c_|xj dz c_t|!||}|S)rrrrnz!BlockCache fetching block (%s) %drQ) rrrrorr!r rIr*)r"rlog_infor(rzrr2s r#rz!BackgroundBlockCache._fetch_blocks $,, & /))-a9  t~~-dnn$ 7<P ""cEk1" 1s3r%c ||jz}||jz}|xjdz c_||k(r|j|}|||S|j||dg}|jt |jt |dz||j |j|d|dj|Srrrs r#rz BackgroundBlockCache._read_cachesDNN* & ! !1 1,,-?@E7+ +++,>? KLC JJ,,,q02BC  JJt//0@A(7K L88C= r%rrr rr=r)sync)rr6r7r>r8r<r)r3r?r@rArrBr$rrrr*rrrrs@r#r"r"s2'D-&MO 3 3'. 369 3FI 3  3 53J X$(!(!"(!8;(!OR(! (!r%r"z!dict[str | None, type[BaseCache]]cachescr|j}|s |tvrtd|dt||t|<y)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rr:r)clsclobberrs r#register_cacher>s= 88D tv~+D83Fvd|nUVVF4Lr%)F)r<ztype[BaseCache]r=rr8r9)3 __future__rrrloggingrrUr rrconcurrent.futuresrr itertoolsroperatorrtypingr r r r r rrrFtyping_extensionsrrr getLoggerror6r<r7list MultiFetcherrrErrrrrrrr"r:rBr>crCr%r#rJs" #9+#A A CL   8 $ C:u$ %c3h(%/0 < < ~K& K&\+%Y+%\+,i+,\F!F!RbbJ%y%0b1 b1J971a4=9xK!9K!` )-) (    A1 r%