L iYbddlmZmZmZddlZddlZddlmZddlmZddl m Z ddl Z ddl m Z mZmZmZddlmZdd lmZmZd d lmZmZd d lmZmZmZe j:d Ze j:dZddlm Z m!Z!m"Z"ddl#mZddl$m%Z%m&Z&m'Z'e jPdddZ)dZ*dIdZ+dZ,dZ-dZ.e j^jadddge j^jadeegdZ1ee j^jadddge j^jadd d!ge j^jadeegd"Z2ee j^jadeegd#Z3d$Z4d%Z5e j^jad&dd'gd(Z6d)Z7Gd*d+e8Z9dJd,Z:d-Z;d.Zdd0Z?d1Z@d2ZAd3ZBd4ZCd5ZDd6ZEd7ZFd8ZGd9ZHd:ZId;ZJe j^jadge j^jejd?kxrejd@k\dABdCZMdDZNe j^jadEdFdGgdHZOy)K)absolute_importdivisionprint_functionNrandom)sleep)uuid4)Paralleldelayedparallel_backendparallel_config)DaskDistributedBackend)AutoBatchingMixinThreadingBackend)np with_numpy)_recursive_backend_info_test_deadlock_with_generator8_test_parallel_unordered_generator_returns_fastest_first distributeddask)Client LocalCluster get_client)time)cleanupclusterincfunctionT)scopeautousec#4Kddlm}|jDcic]"}|tjj |$}}d|j D];\}}|!tjj|d)|tj|<=ycc}ww)Nr)ParallelBackendBase)joblib._parallel_backendsr%MAX_NUM_THREADS_VARSosenvirongetitemspop)tmp_pathr%k old_valuevs [/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/joblib/test/test_dask.pyavoid_dask_env_leaksr2!s >/B/W/WX!BJJNN1%%XIX !1 9 JJNN1d #BJJqM  YsB'BABcyN)argskwargss r1noopr83sc4t||r tdy)Nzcondition evaluated to True)r ValueError) conditiondurations r1slow_raise_value_errorr>7s (O677r9c|jd}i}|jD]3\}}tt|Dcgc] }|d|k(s |c}||<5|Scc}w)Nc|jSr4)log) dask_workers r1zcount_events..>s ;??r9r)runr+lenlist) event_nameclient worker_events event_countsweventsevents r1 count_eventsrN=skJJBCML"((* 6 $V GuaJ0FU G Q   Hs A A c t5\}\}}t|d|5}td5tdt dD}|t dDcgc] }t |c}k(sJt jt5tdt dDdddtdt dD}|t dDcgc] }t |c}k(sJ dddddddddycc}w#1swYoxYwcc}w#1swY0xYw#1swY4xYw#1swYyxYw) Naddresslooprbackendc3FK|]}tt|ywr4r r .0is r1 ztest_simple..K DQa D! c3LK|]}tt|dk(yw)N)r r>rWs r1rZztest_simple..Os'DE7 67Q?s"$c3FK|]}tt|ywr4rVrWs r1rZztest_simple..Sr[r\) rrrr ranger pytestraisesr;)rRsabrHseqrYs r1 test_simplerhGs(  :kq&1a AiLt , : 0 : hj D%) DDuRy9!s1v9999]]:.HJINr !hj D%) DDuRy9!s1v9999 : : : :: : : : : : : :soE D</D0D 0 D0!D 16D0'D+ 9D0D< ED0D( $ D00D9 5D<<E EEc @tjtjusJt5\}\}}t |d|5}t d5t 5}|j}t|tsJ|j|usJ|jdk(sJ|dttdD|jdkDsJ ddddddddddddy#1swY"xYw#1swY&xYw#1swY*xYw#1swYyxYw) NrPrQrrSrc3>K|]}tdyw)cyr4r5r5r9r1rCz?test_dask_backend_uses_autobatching...jsr9Nr rX_s r1rZz6test_dask_backend_uses_autobatching..jsN2W\24Ns@r]) rcompute_batch_sizerrrrr _backend isinstanceparallel_effective_batch_sizeraint)rRrdrerfrHrsrTs r1#test_dask_backend_uses_autobatchingrvWs11  / / 0 0 >kq&1a AiLt , > 0 >Z >8'//G%g/EFFF"++x777"88A===NeCHoNN"882=== > > >>> > > > > > >>>sTD D C<A2C0 C<DD0C9 5C<<D DD DDn_jobscontextctjdd5|d5td|ddddddy#1swYxYw#1swYyxYwNr  n_workersthreads_per_workerr)rrr)rwrys r1Atest_parallel_unordered_generator_returns_fastest_first_with_daskrnsY   aA  >OO@vNOOOOOOs A  ?A A A  A return_as generatorgenerator_unorderedctjdd5|d5td||ddddddy#1swYxYw#1swYyxYwr{)rrr)ryrrws r1%test_deadlock_with_generator_and_daskrusS   aA  >??%dIv>??????s! A AA A A  Ac tjdd5tjt dtj }t dD]C}|d5t|}dddtdk(sJtd|DrCJ|d5t}dddtdk(sJtd |DsJ dddy#1swYxxYw#1swYCxYw#1swYyxYw) Nr r|gcAdtyperdatac3,K|] \}}|dk(ywrNr5rXnamerns r1rZz4test_nested_parallelism_with_dask..s5.s 18qD, , r) rrronesruuint8rarrEall)ryrrYbackend_types_and_levelss r1!test_nested_parallelism_with_daskr~s   aA  > wws3xrxx0q A N+B+M( N/0A5 55@X   V_ A'>'@ $ A+,111 .s!G"2''"2"4!G r rrrr ra)rRrdrerfrHxys r1 test_dont_assume_function_purityrs kq&1a AiLt ,  0 !xz!GeAh!GG1Av v      s9B  B+A4B#B 4A= 9BB B  BmixedFcddlm}|s.tdDcgc]}tt|}}d}nGtdDcgc]1}|dzrtt |ntt|3}}d}t |||k(sJt5\}\}}t|d|5} td 5tdd |} dddd } |jdd}| j| } td| DsJ ddddddycc}wcc}w#1swY_xYw#1swY(xYw#1swYyxYw)Nr)Batchrbatch_of_inc_4_callsr mixed_batch_of_inc_4_callsrPrQrrSr) batch_size pre_dispatchc,t|jSr4)rFtransition_log)dask_schedulers r1fztest_dask_funcname..fsN99::r942c3*K|] }d|dv yw) batch_of_incrNr5)rXtups r1rZz%test_dask_funcname..s?C~Q/?s) joblib._daskrrar r absreprrrrr replacerun_on_schedulerr) rRrrrYtasks batch_reprrdrerfrHrnrrAs r1test_dask_funcnamersS" */(3Qa33+ HMaQ1AEa|ws|A>QQ1 e  ++ +  @kq&1a AiLt , @ 0 F>H>uE F ;$++C5J))!,C?3?? ?? @ @ @4R F F @ @ @ @sHD*6D/E 5 ED4AEE 4D= 9EE E  Ec tdDcgc]}g}}tjd  jt dd fd t dd}t |} td 5tfd |D}ddd|tdDcgc]}gc}k(sJtd |}t|jd k(sJtDcgc]}t|dk(c}sJtd 5tfd |D}dddtd |}t|jd kDsJt|Dcgc]}t|dk(c}sJ |jd|jdycc}w#1swY%xYwcc}wcc}w#1swYxYwcc}w#|jd|jdwxYw)Ndnumpyg.Ac|jj||jtj|Sr4)testingassert_array_equalappendr hex)list_rXrs r1isolated_operationzCtest_no_undesired_distributed_cache_hit..isolated_operations4   JJ ) )$ 2 UW[[! r9rr r|rrSc3@K|]}t|ywr4rl)rXrrs r1rZz:test_no_undesired_distributed_cache_hit..sSE8W%78?Ssreceive-from-scatterrc3DK|]}t|yw)rNrl)rXrrrs r1rZz:test_no_undesired_distributed_cache_hit..s)?D+*+E::s timeoutr4)rarb importorskiparangerurrrr rNsumvaluesrrEclose) rnlistsrrHrescountsrrrrs @@@r1'test_no_undesired_distributed_cache_hitrss $AR $E $   W %B #c(A Q1=G G_F" V , T(*SUSSC T U3Z00000 4f=6==?#q(((-ACFaK-... V , (*HMC 4f=6==?#a'''-ACFaK-... R   b!S % T T 1.  . R   b!se F0, G8F5G( G1:G+GGG 0;G+G G5F?:G G G'Hc"eZdZdZdZeZdZy)CountSerializedc ||_d|_y)Nr)rcount)selfrs r1__init__zCountSerialized.__init__s r9c6|jt|d|zS)Nr)rgetattr)rothers r1__add__zCountSerialized.__add__svvsE222r9cR|xjdz c_t|jffS)Nr)rrr)rs r1 __reduce__zCountSerialized.__reduce__s  a $&&++r9N)__name__ __module__ __qualname__rr__radd__rr5r9r1rrs3H,r9rc||z|z|z|zSr4r5)rerfcdes r1add5rs q519q=1 r9cdtdD\}}}}tt}tdDcgc]}||||dd}}|||||dd|||||d||||||gz }|D cgc]\}} } || i| } } }} t5\} }t | d|5} t d|||g 5t d |}|| k(sJ dddtjt5t d|d 5 dddddddddddd|j}|j|k(sJ|j|k(sJ|j}||||fD] }d |_ t5\} }t | d|5} t|||fD]!}| j|dt|<#|D cgc]h\}} } | j|gfd| Ditfd| jDdt!t#ij%j}} }} || k(sJ dddddd|j}|j|k(sJ|j|k(sJ||k(sJt'dt(j*j-dD}|dkr|j|d zk(sJy|j|k(sJycc}wcc} } }w#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swYxYwcc} } }w#1swYxYw#1swYxYw)Nc32K|]}t|ywr4)rrWs r1rZz&test_manual_scatter..s7/!$7rr])rrrPrQr)rTscatterr)r)rTrRrrT) broadcastc3TK|]}jt||!ywr4r*id)rXarg scattereds r1rZz&test_manual_scatter..:s BcimmBsGS1Bs%(c3^K|]$\}}|jt||f&ywr4r)rXkeyvaluers r1rZz&test_manual_scatter..;s1(S%immBuIu=>s*-rc32K|]}t|ywr4)ru)rXr0s r1rZz&test_manual_scatter..MsS1ASr.)ir)rar rrrrr rbrc TypeErrorrdictrrsubmitr+strr resulttupler __version__split)rRrKrrzrrnrfuncr6r7expectedrdrHresults_parallel%n_serialization_scatter_with_paralleln_serialization_with_parallelvarobjresults_nativen_serialization_scatter_nativedistributed_versionrs @r1test_manual_scatterr s8eAh7JAq!Q A+09 5aQq!Q!q ! 5E 5  !QQ! !QQ! !QQ!E AFFF*<$fd%f%FHF  fq! AiLt ,  !QC 4#98q#9%#@ '8333 4 y) $V$J   -.GG) 77; ;; ; 77; ;; ;$%GG!1a|  .fq! AiLt , .I!Qw I%+^^C4^%H "S'" I-2  )T4 BTB,2LLNEG &( N "X- --! ..*&'WW" 774 44 4 774 44 4 04R RR RS 0G0G0M0Mc0RSSY& ww7!;;;;ww7777G 6 G 4 4      :   . ...sK*:K/L*/LK6!L<L L LLL*M :L>A-L7 > L> M 6L ;LL LL LL' "L**L47L>>M M  Mc  tjd}|jtd|j |jtd|j} gdz|gdzz}t 5\}\}}t |d|5}td5td t|Ddddtd |}||d||dzd k(sJ ddddddt 5\}\}}t |d|5}td5t fd td Ddddtd |}||ddk(sJ||ddk(sJ ddddddy#1swYxYw#1swYxYw#1swYxYw#1swYexYw#1swYBxYw#1swYyxYw)Nrrorr_rPrQrrSc3TK|] \}}tt||||"yw))optNr r8)rXrYrs r1rZz$test_auto_scatter..fs04"GDM$aT::s&(rr c3PK|]}ttdd|yw)Nr_r)rXrYdata1s r1rZz$test_auto_scatter..ss$I1=74=rA6Is#&rr) rbrrrurrrrr enumeraterNra) loop_in_threadrdata2data_to_processrdrerfrHrrs @r1test_auto_scatterr[s   W %B BGGCHBHH -E BGGCHBHH -Ew{w{3O  Dkq&1a AiL~ 6 D& 0  #,_#= ""8&AF!I,'&9*>>!C CC D D -kq&1a AiL~ 6 -& 0 J IaII J""8&AF!I,'1, ,,!I,'1, ,, ---   D D D D" J J - ---sx<F% F !F *F,F%G  F=*#F1 .F=<G  F FF" F%%F.1F: 6F==G G  Gretry_noc  tjd dd} fd  fd t5\}\}}t|d|5}t d5 j d t  fd t|D}dddddddddy#1swYxYw#1swYxYw#1swYyxYw) Nrr]c(j|Sr4)r)rrYjrs r1my_sumz#test_nested_scatter..my_sumsrvvayr9ct}td5tfdtD}dddt |S#1swYt SxYw)NrrSc3JK|]}t|d|ywr4rl)rXrarrayrYrs r1rZzEtest_nested_scatter..outer_function_joblib..s+!56ab 1a0!s #)rrr rar)rrYrHresultsNUM_INNER_TASKSrs`` r1outer_function_joblibz2test_nested_scatter..outer_function_joblibsZ V ,  hj!:?:P!G 7|  7|s %AA%rPrQrrSi'c3HK|]}t|d|ywr4rl)rXrYmy_arrayr s r1rZz&test_nested_scatter..s.3G128AB<C")rbrrrrrr ra) rRrNUM_OUTER_TASKSrdrerfrnrr"rrr s @@@@@r1test_nested_scatterr%{s   W %BOO kq&1a AiLt ,  0 "2775>HJ"?3      s;B? B36B'B3B?'B0 ,B33B< 8B??Cc dt5\}\}}t|d|5}td5tdfdt d D}|D]}t t |dkrJ ddddddt|d|5}td5tdfd t d D}|D]}t t |dkrJ dddddddddy#1swYxYw#1swYxYw#1swY2xYw#1swY6xYw#1swYyxYw) Nc ttddtdD}|ttddtdDz}|S)Nr rwc3XK|]"}ttj$ywr4r r(getpidrms r1rZzOtest_nested_backend_context_manager..get_nested_pids..s%Mq&8gbii&8&:%M(*c3XK|]"}ttj$ywr4r*rms r1rZzOtest_nested_backend_context_manager..get_nested_pids..s&N'9wryy'9';&Nr,)setr ra)pidss r1get_nested_pidsz.get_nested_pidssN%81%%ME!H%MMN &HA&&NU1X&NNOO r9rPrQrrSr r(c3>K|]}tywr4rlrXrnr0s r1rZz6test_nested_backend_context_manager..!034,GO,.0r]c3>K|]}tywr4rlr2s r1rZz6test_nested_backend_context_manager..r3r4)rrrr rarEr.)rrdrerfrH pid_groups pid_groupr0s @r1#test_nested_backend_context_managerr8sM 4kq&1a AiL~ 6 4& 0 4/XQ/08=b 0 ",4Is9~.!3334  4 4AiL~ 6 4& 0 4/XQ/08=b 0 ",4Is9~.!3334  4 444 4 4 4 4 4 4 4 444swD5 DAD5D9DD5 D)&AD(D,D)4D5D DD D5D& "D))D2 .D55D>c  d fd t5\}\}}t|d|5}td5t5} |dk(sJ|jdk(sJ| fd t d D}dddD]\}}|dk(sJ|dk(rJ dddddddddy#1swY;xYw#1swY&xYw#1swY*xYw#1swYyxYw) NcB|jjjSr4)rq __class__r)ps r1 _backend_typezJtest_nested_backend_context_manager_implicit_n_jobs.._backend_typeszz##,,,r9clt5}||jfcdddS#1swYyxYwr4)r rw)r<r=s r1get_nested_implicit_n_jobszWtest_nested_backend_context_manager_implicit_n_jobs..get_nested_implicit_n_jobss0 Z .1 #QXX- . . .s*3rPrQrrSrrxc3>K|]}tywr4rl)rXrnr?s r1rZzFtest_nested_backend_context_manager_implicit_n_jobs..s"*BC; :;=*r4r )rrrr rwra) rRrdrerfrHr<all_nested_n_jobs backend_type nested_n_jobsr=r?s @@r13test_nested_backend_context_manager_implicit_n_jobsrDs -.  /kq&1a AiLt , / 0 /Z1(+/GGGG88r>)>()*GLQx*)% 4E//L-'+CCCC(B.../ / / / / / / / / / /sWC C C9B7 =CCC&C7C <CC CC CC$ctjt5}td5 dddddddt j j vsJy#1swY9xYw#1swY=xYw)NrrSzcreate a dask client)rbrcr;rrrlower)rRinfos r1 test_errorsrHsf z "d V ,    "S_%:%:%< << <  s! A-A!A-!A* &A--A6c t5\}\}}t|d|5}td5tddt dD}t |d d d t sJ dddtd5tdd t dD}t |d d d tsJ dddddddddy#1swYmxYw#1swY&xYw#1swY*xYw#1swYyxYw) NrPrQrrSr r(c3HK|]}ttdyw)Nnested_requirer outerrms r1rZz.test_correct_nested_backend..s#,<=NGEN$77, "rrc3HK|]}ttdyw) sharedmemrKNrMrms r1rZz.test_correct_nested_backend..s#,CDNGEN+>>,rO)rrrr rarrrr)rRrdrerfrHrs r1test_correct_nested_backendrRs Ekq&1a AiLt , E 0 K++,AFq,"&)A,q/3IJJJ  K!0 E++,HMa,"&)A,q/3CDDD  E EEE K K E E E EEEsQC> C2>C.C2>C&C2 C>C# C2&C/ +C22C; 7C>>DcNtddfdtdDS)Nr threads)rwpreferc3HK|]}ttywr4)r middle)rXrnrLs r1rZzouter..s!0,-'0r#rr rarKs`r1rNrNs* /81Y /016q0 r9cHtd|dtdDS)Nr )rwrequirec3DK|]}ttywr4)r innerrms r1rZzmiddle..s.RA~wu~/?.RrrrX)rZs r1rWrWs .81g ..Rq.R RRr9c*tjSr4)r rqr5r9r1r\r\s :  r9ct|dd5td5tddtd Dddddddy#1swYxYw#1swYyxYw) NFT)rR processesset_as_defaultrrSrr(c3FK|]}tt|ywr4)r rrWs r1rZz0test_secede_with_no_processes..s@!{wr{1~@r\r )rrr rarQs r1test_secede_with_no_processesrbsg TU4 @A V , A HA @uQx@ @ AAA A AAAs! A#AAA AA%c0ddlm}|jS)Nr) get_worker)rrdrP)rnrds r1_worker_addressres& <  r9c t5\}\}}t|d|5}td|d5tdt dD}||dgdzk(sJ dddtd|d5tdt dD}||dgdzk(sJ dddddddddy#1swYbxYw#1swY&xYw#1swY*xYw#1swYyxYw)NrPrQr)rTworkersc3FK|]}tt|ywr4r rerWs r1rZz-test_dask_backend_keywords.. P!9!9!!< Pr\r]c3FK|]}tt|ywr4rirWs r1rZz-test_dask_backend_keywords..rjr\r)rRrdrerfrHrgs r1test_dask_backend_keywordsrls 2kq&1a AiLt , 2 9F 2 hj PeBi PPq|nr1111 2!9F 2 hj PeBi PPq|nr1111 2  222 2 2 2 2  2 222sQC(C/C#C;/C+C3C(C CC CC% !C((C1ctd|5}td5tdtdDdddt }|j j jr>tdt |dzksJ|j j jr>|jrJ dddy#1swYxYw#1swYyxYw) NF)r_rRrrSc3FK|]}tt|ywr4rVrWs r1rZz/test_scheduler_tasks_cleanup.. :1|ws|A:r\r]g{Gz?r) rrr rarr schedulerrrfutures)rRrHstarts r1test_scheduler_tasks_cleanuprs s %d + "v V , ; HJ:b : : ;nn&&,, $K6EAI% %%nn&&,,>>!!> " " ; ; " "s( C!CA/C,CC CCcluster_strategyadaptive late_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonctddd}t|}|dk(r|jddn|dk(r|jd t d 5t d t d Dddd|j|jy#1swY+xYw#|j|jwxYw) NrFr r}r_r~ru)minimummaximumrvrrSc3FK|]}tt|ywr4rVrWs r1rZz(test_wait_for_workers..+ror\r])rradaptscalerr rar)rtrrHs r1test_wait_for_workersrs Q%ANG G_F:% a+ ^ +  a V , ; HJ:b : : ;     ; ;   s$ B/!B#9B/#B,(B//#Cctddd}t|} tdd5d}tjt | 5t d td Dddddddtdd5d }tjt| 5t d td Ddddddd|j|jy#1swYxYw#1swYxYw#1swYKxYw#1swYOxYw#|j|jwxYw)NrFr ryrg?)rTwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc3FK|]}tt|ywr4rVrWs r1rZz0test_wait_for_workers_timeout..:>q<73<?>r\r]z+DaskDistributedBackend has no active workerc3FK|]}tt|ywr4rVrWs r1rZz0test_wait_for_workers_timeout..@rr\) rrrrbrc TimeoutErrorr ra RuntimeErrorr)rrHmsgs r1test_wait_for_workers_timeoutr1sQ%ANG G_F Vc J ?KC|37 ? >E"I>> ? ? Va H ??C|37 ? >E"I>> ? ?    ? ? ? ? ? ? ? ?   sj D%D!C5'D/D%D"!D D D%5C> :DD D% D DD"D%%#ErTlokymultiprocessingctd}t|} fd}|j|}|j}t |dk(sJ|dj }t |tsJdt|vsJ |jd|jdy#|jd|jdwxYw) Nr )r}ctjd5}tddtdDddd|S#1swYSxYw)NT)recordr )rwrTc3FK|]}tt|ywr4rVrWs r1rZzftest_joblib_warning_inside_dask_daemonic_worker..func_using_joblib_parallel..Rs3WLGCLO3Wr\r])warningscatch_warningsr ra)rrTs r1func_using_joblib_parallelzStest_joblib_warning_inside_dask_daemonic_worker..func_using_joblib_parallelLsR ((5 X3733WUSUY3WW XM XMs $AArrzdistributed.worker.daemonrr) rrrrrEmessagerr UserWarningrr)rTrrHrfutrwarnings` r1/test_joblib_warning_inside_dask_daemonic_workerrFsQ'G G_F" mm676{a)##';///*c'l::: R   b!  R   b!s A&B'''C)g?)rr)P __future__rrrr(rrrruuidr rbr r r r_daskrrsrrcommonrr test_parallelrrrrrrrrrdistributed.metricsdistributed.utils_testrrr fixturer2r8r>rNrhrvmark parametrizerrrrrrrobjectrrr rrFrar%r8rDrHrRrNrWr\rbrerlrsskipifrrrrr5r9r1rs@@  CC*:" "f!!-0 v6"98$98j$/0" 8  : >.Ar7+_6F$GHOI,O  Ar7+{4I&JK_6F$GH?IL, ?  _6F$GH I  *4-0@1@24"n ,f ,P8l-@T%(^45642/2=E$ SA 2 "+j.-IJw&N;+B+Bh+N L K **V->$?@"A"r9