L i(ddlZddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z ddl ZddlmZ ddlZddlmZddlmZddl mZmZmZdd lmZdd lmZmZm Z ejBd Z"Gd d e#Z$dZ%dZ&dZ'GddejPjRZ*GddejPjVZ,dZ-GddejPj\Z/GddeZ0GddeZ1y#e$r e sddlZYwxYw)N) lru_cache)chain) TYPE_CHECKINGLiteral)AbstractBufferedFile)AsyncFileSystem)DEFAULT_CALLBACK) filesystemopensplit_protocol)AsyncFileSystemWrapper) isfilelikemerge_offset_ranges other_pathszfsspec.referencec$eZdZfdZdZxZS)ReferenceNotReachablec:t||||_||_yN)super__init__ referencetarget)selfrrargs __class__s f/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/fsspec/implementations/reference.pyrzReferenceNotReachable.__init__#s $" c:d|jd|jS)Nz Reference "z" failed to fetch target )rrrs r__str__zReferenceNotReachable.__str__(sT^^,,Edkk]SSr)__name__ __module__ __qualname__rr __classcell__rs@rrr"s TrrcFtt|jSr)nextitervalues)ds r_firstr+,s QXXZ !!rc|j|}t|ttfr/t|dtr|drt |ddS|dSyyNr)get isinstancelisttuplestrr )path referencesrefs r_prot_in_referencesr60sW .. C#e}%*SVS*A,/F~c!f%a(>A>+B%rct|trt|||giSi}|D]/}t||}|j|gj |1|Sr)r/r2r6 setdefaultappend)pathsr4outr3protocols r_protocol_groupsr=6s`%#E:6@@ C2&tZ8 x$++D12 JrceZdZdZy)RefsValuesViewc#K|jjjD]'}tj|j )|jj jEd{|jjD]c}|jj|}t|dk(r|j|dzA|jj|Ed{ey77 w)Nr/0) _mapping zmetadatar)jsondumpsencode_itemslistdir_get_chunk_sizeslen_generate_all_records)rvalfield chunk_sizess r__iter__zRefsValuesView.__iter__As==**113 +C**S/((* * +==''..000]]**, BE--88?K;1$mmEDL11}}::5A A A  B 1 Bs%A6D8C>9A=D6D7DDNr!r"r#rOrrr?r?@s Brr?ceZdZdZy) RefsItemsViewczt|jj|jjSr)ziprBkeysr)rs rrOzRefsItemsView.__iter__Ns)4==%%')=)=)?@@rNrPrQrrrSrSMsArrSchd}d}t|ddd|dddD]\}}|||zz }||z}|S)Nr)rU)idxsizesrLmultiss rravel_multi_indexr_RsS C DC"IuTrT{+1 q4x   JrceZdZdZedZedZ d dedfdZdZ d Z e d!d Z e d Zd"d Zd Ze ddZdZdZdZdZdZdZdZdZe dZdZd#dZd#dZdZdZdZ dZ!y)$LazyReferenceMapperaoThis interface can be used to read/write references from Parquet stores. It is not intended for other types of references. It can be used with Kerchunk's MultiZarrToZarr method to combine references into a parquet store. Examples of this use-case can be found here: https://fsspec.github.io/kerchunk/advanced.html?highlight=parquet#parquet-storagecddl}|Sr-)numpy)rnps rrdzLazyReferenceMapper.npds  rcddl}|Sr-)pandas)rpds rrgzLazyReferenceMapper.pdjs  rNengine) fastparquetpyarrowc\||_i|_||_||_||_|jdz|_|t jdn||_|jj|xs |j|_ ddl m }|jdk(r|d tdyy)a This instance will be writable, storing changes in memory until full partitions are accumulated or .flush() is called. To create an empty lazy store, use .create() Parameters ---------- root : str Root of parquet store fs : fsspec.AbstractFileSystem fsspec filesystem object, default is local filesystem. cache_size : int, default=128 Maximum size of LRU cache, where cache_size*record_size denotes the total number of references that can be loaded in memory at once. categorical_threshold : int Encode urls as pandas.Categorical to reduce memory footprint if the ratio of the number of unique urls to total number of refs for each variable is greater than or equal to this number. (default 10) engine: Literal["fastparquet","pyarrow"] Engine choice for reading parquet files. (default is "fastparquet") z/{field}/refs.{record}.parqNfiler) find_specrjz)engine choice `pyarrow` is not installed.)rootrN cat_threshrh cache_sizeurlfsspecr fsunstrip_protocolout_rootimportlib.utilrm ImportError)rrnrsrurpcategorical_thresholdrhrms rrzLazyReferenceMapper.__init__psB / $99<</1z&##F+r001FTYYG , ;;) # )(<(DIJ J)E #rc^|dvr|j|j|St|)N)rG record_sizerC)setup__dict__AttributeErrorritems r __getattr__zLazyReferenceMapper.__getattr__s. 9 9 JJL==& &T""rcPi_jjdjjdgjd<t j jd}|d_|d_tjfd}|_ y)N/ .zmetadatarzmetadata)maxsizecrjj||}tjjj |} j j|j}|jDcic]}|||j}}|Scc}w#t$rd}Y|SwxYw)zcached parquet file loader)rMrecord)rhN) rqformatioBytesIOrscat_filerg read_parquetrhcolumnsto_numpyOSError)rMrr3datadfcrefsrs r open_refsz,LazyReferenceMapper.setup..open_refss88??v?>D::dgg..t45D WW))$t{{)C57ZZ@2a5>>++@@KA K s$5B'B"B'"B'' B65B6) rGrsrjoinrnrDloadsrzrCrrpr)rmetrs` rr{zLazyReferenceMapper.setups $(GG$4$4 HHdii. /%  L!jj\23}-Z 4?? +  , #rc ni|d}|'tjj|fi|xsi\}}|j|r|j |d|j |d|j dj|dgtj|jt||fi|S)aJMake empty parquet reference set First deletes the contents of the given directory, if it exists. Parameters ---------- root: str Directory to contain the output; will be created storage_options: dict | None For making the filesystem to use for writing is fs is None fs: FileSystem | None Filesystem for writing record_size: int Number of references per parquet file kwargs: passed to __init__ Returns ------- LazyReferenceMapper instance rrzT recursiveexist_okrr) rrcore url_to_fsexistsrmmakedirspiperrDrErFra)rnstorage_optionsrsrzkwargsrs rcreatezLazyReferenceMapper.creates,k: :{{,,TMo6KMHB 99T? EE$$E ' D4 ( $ -. 30F0F0HI"46v66rc>d|jD}t|S)zList top-level directoriesc3jK|]+}|jdr|jddd-yw).zrrXrN) startswithrsplit.0ps r z.LazyReferenceMapper.listdir..s,V1<.s5dT5 c3 K|]}|ywrrQrs rrz)LazyReferenceMapper.ls..s2dT2rF directoryrrtypesizerlc |dS)NrrQ)r^s rz(LazyReferenceMapper.ls..s AfIrkeyrY)rstriprHrlstriprrCrGupdatesortedrJrDrEr_keys_in_fieldr0rKrU)rr3detailpathdashdirnamesr*rfothersrdirinfofileinforMrVrecsrecrecinfos rlszLazyReferenceMapper.lss{{3!%4#:2<<> ||H%#QXXh5G*G   !N5dnn52dkk2 <<)c(9K.K F d#f~%RVW$k1EWGW# !"4>>1 4>>$#78![[. H 'H,2EF F"nn Ld4& 0KT L $ Kt$qz/JtK L   t~~-JJt~~d34T*     ""5) U?<$t*, ,))%0!t_ c1v63r7 ;  '!!w X  MK  " s=-J -J% J8AJ3J! J!!J& ;J& AJ+-J0c||jvr|j|S||jvr0tj|j|j Sd|vs|j |r t ||jdd\}}|j|\}}}|jj||fij|d}|t |r|S|dk(ry |j||}gd} | D cgc]} | |vr|| |nd} } | d} | | S| d t d | dd ddgk(r| ddS| dd S#tttf$r} t || d} ~ wwxYwcc} w) zfGet the reference for one key Returns bytes, one-element list or three-element list. rrXFNrrr3offsetrrawrYz1This reference does not exist or has been deleted)rGrCrDrErF_is_metaKeyErrorr_key_to_recordr.r ValueError TypeErrorFileNotFoundError)rrrM_rri chunk_sizemayberexcrr selectionrs r _load_one_keyz!LazyReferenceMapper._load_one_key&s $++ ;;s# # DNN "::dnnS1299; ; ^t}}S13- ::c1%q!%!4!4S!9J 488UC =N L 1_ )>>%0D4AHIAAIT!WR[47I Im ?J Q< NO O Qq>aV #Ra= !}I'89 )3-S ( )Js'EE-E* E%%E*ic4|jdd\}}|j|}t|dk(ry|jdDcgc] }t |}}t ||}||j z}||j z} || t|fScc}w)z3Details needed to construct a reference for one keyrrXr)rrr.)rrIrJsplitintr_rz) rrrMchunkrNr chunk_idx chunk_numberrrs rrz"LazyReferenceMapper._key_to_recordNszz#q) u++E2 { q %*[[%56SV6 6(K@ !1!11 D,, ,r3{+++ 7s Bc||jvr`|j|d}t|d|dDcgc]\}}tj||z }}}|xsdg|j|<|j|Scc}}w)z6The number of chunks along each axis for a given fieldz/.zarrayshapechunksrX)rNrCrUmathceil)rrMzarrayr^r size_ratios rrIz$LazyReferenceMapper._get_chunk_sizes[s (( (^^ugX$67F-0&BR-S%)Q !a% J'1&7QCD  U #&& s"A>c|j||}tt|j}t |dk(r d|DSt |dk(r|dSd|DS)z8The references for a given parquet file of a given fieldrc32K|]}t|ywrr0rts rrz7LazyReferenceMapper._generate_record..ks(DG(rXrc3LK|]}|dst|ddn|dyw)rNrrs rrz7LazyReferenceMapper._generate_record..qs)B!qtD2AK15Bs"$)rr(rUr)rJ)rrMrrits r_generate_recordz$LazyReferenceMapper._generate_recordes`~~eV, #t{{}% & t9>(R( ( Y!^; CrB Brc#Kd}|j|D]}||z} tj||jz }t |D]}|j ||Ed{y7w)zJLoad all the references within a field by iterating over the parquet filesrXN)rIrrrzranger)rrMnrecchrs rrKz)LazyReferenceMapper._generate_all_recordssss''. B BJD yy 0 001Dk |f|j:|xstA|j6ddddd||jC|jjE||fy#t$rYwxYw)NrFr3rrrO)dtypeint64categoryrXr)copyr utf8)rr3rz/refs.z.parqTrrjwrite_statisticsri)statsobject_encoding has_nullsz not supportedrzstd)rhr compressionindex)# kerchunk.dfrcrfrGrJrzrrfullnanzerosrr/r0r2r categoriesadd_categoriesr _proc_raw DataFramer3countnuniqueroastyperursmkdirsrhNotImplementedError to_parquetgetattrclearpop)rrMrbase_urlrkerchunkrdrg partitionoriginalr:offsetsr[rawsjrrrrfndf_backend_kwargss rrzLazyReferenceMapper.writesJKK0  y>D,, , >>%8 V$Ex(GV$EE?DGGD,,bffCG@Ehht//wh?GHHT--WH=E774++RVV37?D ( 6GAt$% $ 2Qu{{'='==!00a9E7at9q=!%aGAJ#AwE!Ha aQ#++//5Q' 6*\\!     77==?bggoo/41 5 GF**:6BvJ")6:UO )DMM*!E7& F (3dmm4AeW=M ;;) #!3U ;  [[M )#2&!  & ^&DE E  ;;+9tww 148      (Q  sL** L76L7ct|jD]-}t|ts|\}}|j ||||/t|jD]H}|dk7s d|vst j |jj||j|<J|j|jd}|jjt j|j|jd<|jjdj|xs |j dg|jd|j"j%y)zOutput any modified or deleted keys Parameters ---------- base_url: str Location of the output )r4rrrrrN)r0rGr/r1rrDrr3rCrzr2rErFrsrrrur cache_clear)rr4rthingrMrkrs rflushzLazyReferenceMapper.flushs'$++& E%' % v %$3  dkk" CAL TQY$(JJt{{q/A$Bq! C >>$:J:JK $(JJsO$:$:$< L!  HHh/$--> ? KK % ""$rcd}|jD]@}|jdr|dz }|tj|j |z }B|t |j z }|td|jDz }|S)NrrrXc3BK|]}t|trdyw)rXN)r/r1)rrs rrz.LazyReferenceMapper.__len__..3sH1:a3GQHs) rHrrprodrIrJrCsumrG)rr+rMs r__len__zLazyReferenceMapper.__len__)s\\^ AE$ 4#8#8#?@@  A T^^$$ H HHH rc# Kt|j}|j|j|D]}t |t s||j D]!}|j|D] }||vs| #ywr)rrCrrGr/r2rHr)rmetasbitrMr@s rrOzLazyReferenceMapper.__iter__6sDNN# T[[! C#s#  \\^ E((/ 9G  sAB 1B; BcF |j|y#t$rYywxYw)NTF)rrr~s r __contains__z LazyReferenceMapper.__contains__Cs*    t $  s   c #K|j|}t|dk(r|dzytjd|D}|D]2}|dzdj |Dcgc] }t |c}z4ycc}ww)z|List key names in given field Produces strings like "field/x.y" appropriate from the chunking of the array rrANc32K|]}t|ywr)r)rr]s rrz5LazyReferenceMapper._keys_in_field..Ss"A58"Arrr)rIrJ itertoolsproductrr2)rrMrNindsindrs rrz"LazyReferenceMapper._keys_in_fieldJs ++E2 { q $,    "A["AB @C#+#)>Q#a&)> ?? ? @)>sABA; -B)NN ri)NNi')rTNN)"r!r"r#__doc__propertyrdrgrrrr{ staticmethodrrrHrrrrIrrKr)rrrrrrrrArFrOrKrrQrrrara[sY   4A.K01.K`##077<[ @"D&Pt_ , ,' C<$#'O(44("S)j!%F   @rraceZdZdZdZdZ d fd Zd!dZd!dZd!dZ dZ d Z e fd Z d"d Zd#d Zd$d ZdZd$dZdZdZdZd%dZd&dZdZdZdZd&dZd'fd ZdZdZdZd(dZ d(dZ!dZ"xZ#S))ReferenceFileSystemaView byte ranges of some other file as a file system Initial version: single file system target, which must support async, and must allow start and end args in _cat_file. Later versions may allow multiple arbitrary URLs for the targets. This FileSystem is read-only. It is designed to be used with async targets (for now). We do not get original file details from the target FS. Configuration is by passing a dict of references at init, or a URL to a JSON file containing the same; this dict can also contain concrete data for some set of paths. Reference dict format: {path0: bytes_data, path1: (target_url, offset, size)} https://github.com/fsspec/kerchunk/blob/main/README.md rFc Xt|di|||_| |_| |_i|_i|_i|_| |_| |_ t|trtdi|xs|xsid|i}tjj|fi|\}}d|vrL|j!ds|j#|r*t$j'd|t)||| |_nptj,|dfi|5}t$j'd|t/j0|}ddd|j3| n|j3|| t|tr|j5Dcic]A\}}|t|tr)tj6|j9d d d fi|n|Cc}}|_d|j vrt7d |j d<y|Dt|j:t<r|j:d n |j:}||j |<||j j?D]g}tA|r|}tjjC|\}}|s:||j vsIt7|fi|xsi}||j |<i||j*j?D]}tA|r|}t|tDs&|d s,tjjC|d \}}||j vs`t7|fi|xsi}||j |<n|r-||j vrt7|fi|xsi}||j |<|xs t7d |j d<|j j5D]Z\}}|jFs%tI||jJ |j |<7|jJ|jJz sQtMdy#1swYxYwcc}}w)a Parameters ---------- fo : dict or str The set of references to use for this instance, with a structure as above. If str referencing a JSON file, will use fsspec.open, in conjunction with target_options and target_protocol to open and parse JSON at this location. If a directory, then assume references are a set of parquet files to be loaded lazily. target : str For any references having target_url as None, this is the default file target to use ref_storage_args : dict If references is a str, use these kwargs for loading the JSON file. Deprecated: use target_options instead. target_protocol : str Used for loading the reference file, if it is a path. If None, protocol will be derived from the given path target_options : dict Extra FS options for loading the reference file ``fo``, if given as a path remote_protocol : str The protocol of the filesystem on which the references will be evaluated (unless fs is provided). If not given, will be derived from the first URL that has a protocol in the templates or in the references, in that order. remote_options : dict kwargs to go with remote_protocol fs : AbstractFileSystem | dict(str, (AbstractFileSystem | dict)) Directly provide a file system(s): - a single filesystem instance - a dict of protocol:filesystem, where each value is either a filesystem instance, or a dict of kwargs that can be used to create in instance for the given protocol If this is given, remote_options and remote_protocol are ignored. template_overrides : dict Swap out any templates in the references file with these - useful for testing. simple_templates: bool Whether templates can be processed with simple replace (True) or if jinja is needed (False, much slower). All reference sets produced by ``kerchunk`` are simple in this sense, but the spec allows for complex. max_gap, max_block: int For merging multiple concurrent requests to the same remote file. Neighboring byte ranges will only be merged when their inter-range gap is <= ``max_gap``. Default is 64KB. Set to 0 to only merge when it requires no extra bytes. Pass a negative number to disable merging, appropriate for local target files. Neighboring byte ranges will only be merged when the size of the aggregated range is <= ``max_block``. Default is 256MB. cache_size : int Maximum size of LRU cache, where cache_size*record_size denotes the total number of references that can be loaded in memory at once. Only used for lazily loaded references. kwargs : passed to parent class r<z.json)parqparquetrz$Open lazy reference dict from URL %s)rsrprbzRead reference from URL %sN:rXrrl) asynchronouszEReference-FS's target filesystem must have same value of asynchronousrQ)'rrrtemplate_overridessimple_templates templatesfss _dircachemax_gap max_blockr/r2dictrrrrendswithisdirloggerinforar4r rDload_process_referencesrr rr<r1r)callabler r0 async_implr r_r)rforref_storage_argstarget_protocoltarget_optionsremote_protocolremote_optionsrsr`rarerfrprdicref_fsfo2rtextr@optsr5r<rrs rrzReferenceFileSystem.__init__jsR "6" "4 0 " b# #;~;GVC!++//:c:KFCc! 45c9J BBG"5)#[[T1S1(QKK #-R[[%"@ Abkk )+DHH_ %  "~~,,. ,C=%C$kk88= ! 8#HG1E2GB)+DHHX&  ,  "--/ C=%Cc4(SV"(++"<"OF0D"FB(*DHH_ %1z&1HHNN$ DAq<<4QTEVEVW ""Q^^3 &  o((s:,P6AP&P#c  |j|}tjd| |j|}t |tr|j}t|dr|j}t |trGtjd|d|jdrtj|dd}|ddfSt|dk(r%tjd|d||d }||}}nV|\}} } tjd|d |d | d | | | z} ||d k\r| |z}n| |z}n| }||d k\r| |z}n| |z}n| }| |j }|||fS#t$r}t ||d}~wwxYw) Nzcat: rz Reference: z , type bytesbase64:rXz, whole file => rz => z , offset z, size )_strip_protocolrjdebugr4rrr/r2rFr rr rbase64 b64decoderJr) rr3startendpartrrqstart1end1start0rend0s r _cat_commonzReferenceFileSystem._cat_common s##D) uTF^$ 3??4(D dC ;;=D 4 $==?D dE " LL;tfL9 :z*''QR1t# # t9> LL;tf,>FcK|j|||\}}}t|tr|||St|\}} |j|j |||d{S7#t $r} t||| d} ~ wwxYwwN)rr)rr/r r rc _cat_file Exceptionr rr3rrr part_or_urlrrr<res rrzReferenceFileSystem._cat_file9s$($4$4TC$4$P! VT k5 )uS) )$[1 ! B(+556t6  B'k: A Bs:2B>c t|tr|rtt|tr|st d|Drtt ||j }i}|jD]\}}|j|} ggggf\} } } } |D]t} |j|\}}}t|ttfs0 | j|| j|| j|| j|vg}g}g}g}t}t| | | | D]u\}}}}t|tr|||<|!|j||j||j||j||j|wt| | | | D]S\}}}}| ||vs|j||j||j||j|Ut!t|t|t|d|j"|j$\}}}| j'|||}t| | | | D]\}}}}||vr t||||D]i\}}}}||k(r#||t|t(r|||<'|||||<0||k(s6||k\s<||ksBt|t(r|||<X|||z ||z xsd||<k|j+jD]\\} }!t|!t(s| |j vs&|| }"t-| |j | }#|"|#_|dk(r|#|dk7sX|#|| <^t1|dk(rt|trd|vr t3|S|S#t$r}|dk(r|dk7r|||<Yd}~/d}~wwxYw)Nc3$K|]}d|v yw)*NrQrs rrz*ReferenceFileSystem.cat..xs7OQq7OsraiseomitT)sortrerfrXr)r/r2r/r0anyr=r4rrcrr r9rrrUaddrrerf cat_rangesrrr __cause__rJr+)$rr3ron_errorr proto_dictr;protor:rsurlsstartsends valid_pathsrur^rerrurls2starts2ends2paths2 whole_files new_paths new_startsnew_ends bytes_outrdnsnebr@vexnew_exs$ rrzReferenceFileSystem.catus dC Y% % dD !yC7O$7O4O% %%dDOO< &,,.I BLE5%B.0"b"n +D&$  **"..q1GAq!%a%6 7KKNMM!$KKN&&q)# *(EGEF%K!$kB % 1aa'CFY OOA&LLONN1%LLOMM!$ %"$kB % 1a=Qk%9LLONN1%LLOMM!$  %/BU W U  .. / +Iz8 iXFI"$kB B 1a8%(J)%T BMBBQwBJ"*%a3%&CF%&qVCFqQ"Wb%a3%&CF%&q2vR0@D%ACF B ByI BVHHJ$$& $DAq!Y'A,@V.q$//!2DE#% w& L'#CF $ s8q=Zc2s$#;  [)%7*6)!$A %s+N O'N==Oc|jdd}||j|y|dk(r|j||ytd|)NversionrX)r`z Unknown reference spec version: )r._process_references0_process_references1r)rr4r`verss rrmz'ReferenceFileSystem._process_referencessS~~i. <  % %j 1 QY  % %jEW % X?vFG Grc t|trH|jDcic]-\}}|t|trtj|n|/}}}||_ycc}}w)z&Make reference dict for Spec Version 0N)r/rgrrDrEr4)rr4rrLs rrz(ReferenceFileSystem._process_references0s^ j$ ' !+ 0 0 2C 3(=TZZ_3FJ% s2A!cjr jrddli_j |j dit dfd}|j dijD]/\}}t|trF|jdr%tj|ddj|<|j|<]t|tr#tj|j|<jr|d}d|vrQjr=|j!dd j!d d j"dij}n||}t%|d k(r|gn ||d |d gj|<!|j|<2jj'j)|j dgy)NrrbicZj|jdijSNrQ)Templaterenderrb)rjinja2rs r _render_jinjaz?ReferenceFileSystem._process_references1.._render_jinjas&,6??1%,,>t~~> >rrzbase64:r}{{{z}}}rXrgenrQ)rarbrr4_process_templatesr.rrr/r2rrrrgrDrEreplacerrJr _process_gen)rr4r`rr@rrrs` @rrz(ReferenceFileSystem._process_references1s$$   {B ?@ 4 ?  ?NN62.446 'DAq!S!<< *)/)9)9!AB%)@DOOA&%&"At$%)ZZ]"aD19,,$AIIdC0$WT3/#V6&*nn6 *!,,/FaKaSa1qt_"%&") '* t00r1JKLrci|_|j|j|j|jD]1\}}d|vrddl|ffd |j|<#||j|<3y)NrrcFj|jdi|Sr)rr)temprrs rrz8ReferenceFileSystem._process_templates..s*>V__>&>#!>#r)rbr`rrr)rtmpr@rrs @rrz&ReferenceFileSystem._process_templatessn  " " . JJt.. /IIK &DAqqy01%#q!%&q! &rc^ i}|D]}|djDcic]H\}}|t|tr|n/t|j dd|d|j ddJc}} fdt j jD}|D]}ddl}|j|djd i||j} |j|d jd i||j} d |vrd |vr{t|j|d jd i||j} t|j|d jd i||j} | | | g|| <d |vd |vz r td | g|| <|Scc}}w)N dimensionsrrstopsteprXc3dK|]'}ttj|)ywr)rgrUrV)rr) dimensions rrz3ReferenceFileSystem._process_gen..'s,S)623s-0rrqrlengthz^Both 'offset' and 'length' are required for a reference generator entry if either is provided.rQ)rr/r0rr.rNrOr)rrrrbrr)rgensr;rr@rproductsprrrrqrrrs @rrz ReferenceFileSystem._process_gens %C -335  Aq !!T*quuWa0!F)QUU61=MNOI'//1A1A1CDH %8fooc%j188P2PP8fooc%j188P2PPO(c/ =H 6==UUdnnUF!=H 6==UUdnnUF!$VV4CH#o(c/:$K !$uCH' % %B AsA F)cdgi|_|jj}|D]H\}}t|tt fs t |dr t|}nt|dk(rd}n|\}}}d|vr|jdddnd}|}|g}|rA||jvr3|j|}|j||r||jvr3|jt||ddD]V\} } | |jvsJ| |jvsJ|j| j| dddg|j| <X|j|j|d|dKy) NrrrXrrrrrl) dircacher4rr/r r2r rJr_parentr9reverserU) rrr3rrrparpar0subdirsparentchilds r_dircache_from_itemsz(ReferenceFileSystem._dircache_from_itemsAsqR __ " " $ TJD$$ -z1J4yTa! 1d,/4K$++c1%a(RCDfG4t}}4||D)t$4t}}4 OO !$Wgabk!: * DMM111... f%,,"KC(* e$ * MM#  % %tVT&R S3 Trc |j|\}}}t|trtj|||St |\} } |&|$|j | j||f||d|St|||f||d|S)N) block_size cache_options) rr/r rrr rc_open ReferenceFile) rr3moderrrrrrr<rs rrzReferenceFileSystem._open_s$($4$4T$:! VT k5 )::k&67 7$[1 ! >dl+488H%++&+        "'     rc tjd||j|}t|jt r |jj ||S|js|j|j|}| t||r|S|Dcgc]}|d c}S#t$rYnwxYwtd|dcc}w)Nzlist %s'z' is not a known keyr) rjrr~r/r4rarrrrr_ls_from_cache)rr3rrr;os rrzReferenceFileSystem.lsys Y%##D) doo': ; ))$77}}  % % '!!$' ;#D) ) J#&'a& ''  #av-A$BC C(sB0! C0 B<;B<c J|j|xs|j|Sr)riisfilerr3rs rrzReferenceFileSystem.existsszz$44;;t#44rc|jr|jvSt|jtr|jj vSt fd|jDS)Nc3FK|]}|jdyw)rNr)rrr3s rrz,ReferenceFileSystem.isdir..s IAq||tfAJ/Is!)rr/r4rarHrrr3s `rrizReferenceFileSystem.isdirsW ==4==( ( )< =4??2244 4III Irc||jvSrrrs rrzReferenceFileSystem.isfilest&&rc2K|j||fi|Swr)r)rr3rrs r_lszReferenceFileSystem._lsstwwtV.v..rc P|rt|f|||d|Sr0|jtfd|jD}nt|j}|r?|j s|j |Dcic]}||j|dc}S|Scc}w)N)maxdepthwithdirsrc3FK|]}|js|ywrr)rr@r3s rrz+ReferenceFileSystem.find..sHQQ\\$5GqHs!!r)rfindr~rr4rrr) rr3r r rrrr@rs ` rrzReferenceFileSystem.finds 7<'(6MS  ''-DH$//HHAt'A ==))+:;GD!"hhtn11$//$2G2JKDGFOAw9s 8 C3C3c,K|j|Swr)rkrs r_infozReferenceFileSystem._infosyyscxK|jj|d|jjywr)r4r3rr2rs r_rm_filezReferenceFileSystem._rm_files/  $  s8:cK|dk(r|j|rt||j|<|jj yw)Nr)rFileExistsErrorr4rr2)rr3rrrs r _pipe_filezReferenceFileSystem._pipe_files= 8  D 1! ! $ sAA cK|dk(r|j|rtt|d5}|j|j|<ddd|j j y#1swY$xYww)Nrr])rrr readr4rr2)rrrrrrs r _put_filezReferenceFileSystem._put_filesc 8  E 2! ! %  .!%&VVXDOOE " .  . .s)A8A, #A8,A51A8c i}|jjD]0\}}t|tr |j d||<,|||<2tj|dfi|5}|jtjd|djdddy#t $r,dt j|zj ||<YwxYw#1swYyxYw)z+Write modified references into new locationasciir|rrX)rrN)r4rr/r r UnicodeDecodeErrorr b64encoderrr rrDrErF)rrqrr;r@rrs r save_jsonzReferenceFileSystem.save_jsonsOO))+ DAq!U#IXXg.CFA [[d 6o 6 F! GGDJJ1c:;BBD E F F *I(6+;+;A+>>FFHCFI F FsB&'6C&2CCC') NNNNNNNNTii@BrRrT)F)Frr)r]NN)T)NFF) overwrite)$r!r"r#rUr<cachablerrrrrrr rr.rrmrrrrrrrrrirr rrkrrrrr r$r%s@rrYrYXs HH  aF*!X BB&/? , 2`DH %MB &#JT< 4($5J'/"*  FrrYcLeZdZ dfd ZedZfdZdZxZS)rc t |||f||||||d| |jj|j\} |_|_t| \} } |jj| |_ | |_ d|_ y)N)rr autocommitr cache_typer) rrrsrr3rrr rcsrc_fssrc_path_f)rrsr3rrr%r&rrrrr<rrs rrzReferenceFile.__init__s    !!!'  -1GG,?,? ,J) TZ$[1 !ggkk(+ # rc|j|jjrY|jj|jf|j |j |jdd|j|_|jS)Nnone)rrr%r&) r)closedr'rr(r blocksizer%rrs rrzReferenceFile.f sj 77?dggnn'dkk'' YY>>??!  ++ DGwwrcl|j|jjt|Sr)r)closer)rrs rr/zReferenceFile.closes& 77  GGMMOw}rc||jz}t||jz|j}|jj ||jj ||z Sr)rminrrseekr)rrrs r _fetch_rangezReferenceFile._fetch_rangesP "# "DHH-  Evv{{3;''r)r]defaultT readaheadNN) r!r"r#rrVrr/r3r$r%s@rrrs:  :   (rr)2r collectionsrrNloggingrr functoolsrrtypingrr fsspec.corerr fsspec.specrujsonrDrw fsspec.asynrfsspec.callbacksr r r r #fsspec.implementations.asyn_wrapperr fsspec.utilsrrr getLoggerrj RuntimeErrorrr+r6r=abc ValuesViewr? ItemsViewrSr_MutableMappingrarYrrQrrrGs   ), (-88F   - .TLT"?  B[__// BAKOO--A z@+//88z@zP F/P Ff4((4(u& sD DD