L i;6ddlmZddlZddlmZddlZddlmZddl m Z erddl m Z ddlZddlmZddlmZnddlmZed Z ed Zed Ze eZ dd Z dd Z ddZ ddZ ddZddd ddZdddddd ddZy) ) annotationsN) TYPE_CHECKING)*single_blas_thread_if_scipy_v1_15_or_newer) get_logger)batched_lbfgsb)BaseAcquisitionFunc) _LazyImportzscipy.optimizetorchzoptuna._gp.batched_lbfgsbc |jdk(sJtdk(r'||tjt|tfS d fd }t 5t j ||ddfz |jDcgc]}|c}fDcgc] }dd|z f c}tj|d\} } } ddd|j}  z| ddf< } | |kD dkDz}tj|dddf| |tj|| ||fScc}wcc}w#1swYsxYw) a' This function optimizes the acquisition function using preconditioning. Preconditioning equalizes the variances caused by each parameter and speeds up the convergence. In Optuna, acquisition functions use Matern 5/2 kernel, which is a function of `x / l` where `x` is `normalized_params` and `l` is the corresponding lengthscales. Then acquisition functions are a function of `x / l`, i.e. `f(x / l)`. As `l` has different values for each param, it makes the function ill-conditioned. By transforming `x / l` to `zl / l = z`, the function becomes `f(z)` and has equal variances w.r.t. `z`. So optimization w.r.t. `z` instead of `x` is the preconditioning here and speeds up the convergence. As the domain of `x` is [0, 1], that of `z` becomes [0, 1/l]. rdtypectj|}|jdk(r|jdk(sJ| z|ddf<tj|j d}j | }|jj|jjj}tj|jj}||ddf zfS)Nr T) nparrayndimr from_numpyrequires_grad_ eval_acqfsumbackwardgraddetachnumpy atleast_1d) scaled_x fixed_params next_paramsx_tensor neg_fvalsgrads neg_fvals_acqfcontinuous_indices lengthscaless \/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/optuna/_gp/optim_mixed.pynegative_acqf_with_gradz9_gradient_ascent_batched..negative_acqf_with_grad8shh|, }}!k&6&6!&;;;-5 -D A))*##K0??E^^H--    " $$&,,.]]9#3#3#5#;#;#=> 5$6!67,FFFN) func_and_grad x0_batched batched_argsboundspgtol max_iters)r np.ndarrayrzlist[np.ndarray]returntuple[np.ndarray, np.ndarray]) rlenrzerosboolrrcopymathsqrtwhere)r#initial_params_batched initial_fvalsr$r%tolr'paramsscaled_cont_xs_opt neg_fvals_opt n_iterationsxs_opt fvals_optis_updated_batchs` `` r&_gradient_ascent_batchedrFsv. " & &! ++ + !#%}bhhs=?QY]6^^^GG,<G &G" 4 5 :H:W:W1-a1C.CD|S.D.I.I.KLU5LN(451QAJ5))C. ; 7M< $ ( ( *F$6$EF1 !I!M1lQ6FG !!T'*F4JK !9m< M5   s*".E D7E D<.!E7 EE c$t|dk(r||dfS||||k7}tj|dddft|d}||dd|f<|j|}tj|}|||kDr||ddf||dfS||dfS)Nr)Fr)axisT)r4rrepeateval_acqf_no_gradargmax) r#initial_params initial_fval param_idxchoiceschoices_except_current all_paramsfvalsbest_idxs r&_exhaustive_searchrTas 7|q|U22$Wy0I%IJ>$'2C8N4OVWXJ5Jq)|  " ": .EyyH X%(A+&h== < ..r(c tdk(r||dfSd fd }||}tj||sJ|| i|jd fd d fd }d} t j |d| z |d| zfd | } || j }  |  } | |k7r| |kDr | <| d fS||dfS)Nr)Fc ttjtj|dt dz }t ||dz z t ||z kr|dz S|S)Nr))intrclip searchsortedr4abs)xigridss r&find_nearest_indexz1_discrete_line_search..find_nearest_indexsb q11c%j1nE FAa!e ,-AaL0AAq1uHqHr(cj|}||S|<tj }||<|S)N)getfloatrJ)r\ cache_valnegvalr#r]negative_fval_cachenormalized_paramsrNs r&negative_acqf_with_cachez7_discrete_line_search..negative_acqf_with_cachesY'++A.   ',Qx)$../@ABB!'A r(c 6|dks|dkDrtjSttjtj|dt dz }|dz }||}}||z ||z z }d|z }||z||zzS)Nrr)g?)rinfrWrXrYr4) r[rightleft neg_acqf_leftneg_acqf_rightw_leftw_rightr]rfs r&interpolated_negative_acqfz9_discrete_line_search..interpolated_negative_acqfs uQx<1uRy=66MBGGBOOE15q#e*q.IJqy(@(FH` I ~ ,"uU|eDk'AB, %.(@@@r(g-q=rrhbrent)bracketmethodr=T)r[rar2rW)r\rWr2ra)r[rar2ra)r4riscloser7sominimize_scalarr[)r#rLrMrNr]xtolr^current_choice_irpEPSresopt_idxfval_optrfrdres` `` @@@r&_discrete_line_searchr}ys 5zQ|U22I*.*CD ::nY/7G1H II I+l];&++-   A C  "qC'7!8%)c/J   C!'G(11H""x,'>',W~)$ (D00 < ..r(cd}|jj|}|st||krt|||||St ||||||S)N) search_spaceis_categoricalr4rTr})r#rLrMrNrOrw MAX_INT_EXHAUSTIVE_SEARCH_PARAMSrs r&_local_search_discretersZ(*$&&55i@NW)II!$ iQXYY$T><T[]abbr(c |j}|j}tjt|t}t |D]+\} } t || || |||\} } } | || <| || <| || <-|||fS)Nr )r7rr5r4r6 enumerater)r#r;r<rNrOrwbest_normalized_params_batched best_fvalsrEbatchrebest_normalized_params best_fvalupdateds r&_local_search_discrete_batchedrs&<%@%@%B"##%JxxM 2$?$-.D$E*  5K #Z%6 7D6 2 71G&u-% 5") * *:7G GGr(g-C6?d)r=max_iterc |j|jjx}}|jj}|jj }|Dcgc]=}t j t j|t jdz ?} }|j|jx} } d} t jt| | t} t jt| }t|D]}t!|| || ||||\| |<| |<}t j"|| | } t%||| D]e\}}}| | |k(x}} ||}|j&dk(r| | fccSt)|| || ||||\| |<| |<}t j"||| } g|| | k(x}}| |} |j&dk(s| | fcSt*j-d| | fScc}w)N)initialrhr rz2local_search_mixed: Local search did not converge.) length_scalesrr$discrete_indicesget_choices_of_discrete_paramsrmindiffrirJr7fullr4rWarangerangerFr:zipsizer_loggerwarning)r#xs0r=r cont_indsr%rchoices_of_discrete_paramsrOdiscrete_xtolsbest_xsr CONTINUOUSlast_changed_dimsremaining_inds_rr\rw is_convergeds r&local_search_mixed_batchedrsE%%D4E4E4X4X'XyZL((99!%!2!2!Q!Q!S 2   rwww014N ''CHHJ)>@JJG jDYYs7|,N 8_NG_ '.):n+EyR^`cH DN!;WHHWj:KL #$46PR` a H Aw 1DUYZDZ4ZL2[ \ +\M:N""a' **.'.1:n3MqRY[_ IGN #Z%? !#!5F G  H(:Kz:Y*Y,(Z[-|m<   ! #J& &+N. LM J EsAG?i )!warmstart_normalized_params_arrayn_preliminary_samplesn_local_searchr=rngc|xstjj}|+tjd|jj f}t ||dz ksJd|jj||}|j|}t|tjsJtj|}tj|||z } d| |<| | jz} ttj| dkD} t!|t |z dz | } | | k(rt"j%dtj&|g} | dkDr4|j)t || d| } tj*| | } tj,|| ddf|g}t/||| \}}tj|j1}||||fS) Nrr)zPWe must choose at least 1 best sampled point + given_initial_xs as start points.)rgzBStudy already converged, so the number of local search is reduced.F)rreplacep)r=)rrandom RandomStateemptyrdimr4sample_normalized_paramsrJ isinstancendarrayrKexprrW count_nonzerorrrrchoiceappendvstackritem)r#rrrr=r sampled_xsf_valsmax_iprobsn_non_zero_probs_improvementn_additional_warmstart chosen_idxsadditional_idxs x_warmstartsrrrSs r&optimize_acqf_mixedrs  (&&(C(0,.HHa9J9J9N9N5O,P) -..12DDZYZD"";;??!CEa!==\]((E7#K!**  O"8%5% ii _= 99ja8:[\]L4Trse" Y%)3" % &B  E !<=N X A A&AA# A  A  A/AH/ /// /  / $ /0@/ @/@/@/ @/  @/  @/$@/Fc ccc c  c  c$c(H H&HH H  H  H/H2AEVY- -$.-8=-PS-"-f<@!%(,13 13(913 13  13  13 &1313r(