`L iHddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZddlmZm Z dd l!m"Z"dd l#m$Z$dd l%m&Z&dd l'm(Z(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.ddl/m0Z0ddl1m2Z2m3Z3m4Z4eeegZ5ddZ6dZ7dZ8dZ9dZ:ejvjydddgdZ=dZ>dZ?dZ@dZAdZBdZCdZDd ZEd!ZFd"ZGd#ZHejvjyd$ejgd%gd&fejgd%dfgd'gd(fgd'dfgd)ZJejvjyd$ejgd*gd&fejgd+gd,fgd-ZKejvjyd$ejgd.dfgd/dfgd.gd&fgd/gd(fgd0ZLejvjyd1gd2d3ZMd4ZNejvjyd5d6ejgd.gd7fd6ejgd/gd8fd9ejgd*dfd:ejgd/gd;fd<ejgd.dd=gfd<ejgd/d>d?gfd@ejgd.gdAfd@ejgd/gdBfdCejgdDgd(fdCejgdBgd(fdCejgdAgd&fg ejvjydEdFdGgdHZOejvjydIdJdKdFdLfdMdKdGdLfdNdFgdOfdPdFdQdRfdSifgdTZPdUZQejvjydVe5dWZRejvjydVe5dXZSejvjdYejvjydVeegejvjydZd[d\gd]ZUejvjydVe5d^ZVejvjydddgd_ZWd`ZXejvjydddgdaZYdbZZdcZ[ddZ\deZ]dfZ^dgZ_ejvjydhgdigdjdgdgfgdigdkdgdgfgdigdldgdgfgdigdmdgdgfgdngdjgdogdpfgdngdkgdogdpfgdngdlgdogdpfgdngdmgdogdpfgdpgdjdqgdgfgdpgdkdgdgfgdpgdldgdgfgdpgdmdgdgfgdrgdjgdsgdtfgdrgdkgdsgdtfgdrgdlgdsgdtfgdrgdmgdsgdtfgdugdjgdvgdwfgdugdkgdvgdwfgdugdlgdvgdwfgdugdmgdvgdwfgdxZ`ejvjygdygdrgdjgdzgdtdfgdrgdjgd{gdpdfgdrgdkgdzgdtdfgdrgdkgd{gdpdfgdugdjgdvgdwdfgdugdjgdvgdwdfgdugdkgdvgdwdfgdugdkgdvgdwdfgd|Zaejvjydhd=dgdQdQgd=dgdd=gfdd=gdQdQgd=dgdd=gfgdigd}dQdgdd=gfgdngd}dQdgdd=gfgdpgd}dgdgfgdrgd}gdvgdpfgdugd}gdvgdjfgdvgd}gdvgdjfgd~ZbdZcejvjydgddZdejvjydgdgdgdgdgdgdZeejvjyddd=ggdjdfgdpddQgdfgdgdjdfgdgdjdfgdgddfgdZfdZgdZhdZidZjdZkdZldZm ddZnejvjydehelekeifejvjydeemfdZodZpejvjyddejvjyddejvjydeqd=dZrdZsdZtdZuejvjydgdugd}fgdugd}gfgduggd}fgdZvdZwejvjyde0dZxdZydZzdZ{dZ|dZ}dZ~dZdZejvjydddgdZdZdZdZdZejvjydgdAd=dfgdAddQfgdAddfgdZejvjydejgdâd=d=fejgdĢd=dQfejgdĢdd=fejgdŢd=d=fejgdƢd=dQfejgdƢdd=fgdDŽZejvjydejgdɢdgdAfejgdtdQgdAfejgdʢdQgdAfejgdˢdgd̢fgejvjydddgd΄ZdτZejvjydgdAd=dfgdAddQfgdAdd=fgdЄZejvjydgdAdfgdAdfgdӄZejvjydgdբgd֢gdעgdآgd٢gddfgdAgd֢gdעgdآgd٢gddfgdܢgd֢gdעgdآgd٢ggd/dfgdܢgd֢gdעgdآgd٢ggd;dfgdߢgd֢gdעgdآgd٢ggdAdfgdߢgd֢gdעgdآgd٢ggddfdd=ggdgdgddfgdZejvjyde0dZejvjydeeeegejvjydgddZdZy)N)stats)datasetssvm)make_multilabel_classification)UndefinedMetricWarning)LogisticRegression) accuracy_scoreaucaverage_precision_scorecoverage_error dcg_score det_curve%label_ranking_average_precision_scorelabel_ranking_loss ndcg_scoreprecision_recall_curve roc_auc_score roc_curvetop_k_accuracy_score)_dcg_sample_scores_ndcg_sample_scores)train_test_split)label_binarize)_sparse_random_matrix)_convert_containerassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equal)softmax)CSR_CONTAINERS) check_arraycheck_consistent_lengthcheck_random_stateFc|tj}|j}|j}|r||dk||dk}}|j\}}t j |}td}|j|||||}}t|dz }t jjd}t j||j|d|zf}tjddd} | j!|d||d|j#||d} |r | ddd f} | j%||d} ||d} | | | fS) zMake some classification predictions on a toy dataset using a SVC If binary is True restrict to a binary classification problem instead of a multiclass classification problem N%rlinearT)kernel probability random_state)r load_irisdatatargetshapenparanger$shuffleintrandom RandomStatec_randnrSVCfit predict_probapredict) datasetbinaryXy n_samples n_featuresprnghalfclfy_scorey_predy_trues h/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/sklearn/metrics/tests/test_ranking.pymake_predictionrL8sM$$& AA Qx1q51GGIz )A R CKKN Q41qA y1} D ))   "C a9cJ&6778A ''t! DCggah%4)77$%AG !Q$- [[45 "F tuXF 67 ""ctj|d}|||k(}|||k7}|jdd|jddz }tj|dkD}|t t |t |zz S)zKAlternative implementation to check for correctness of `roc_auc_score`.r-r)r2uniquereshapesumfloatlen)rJrH pos_labelposneg diff_matrix n_corrects rK_aucrZhs &!!$I &I% &C &I% &C++a$s{{2q'99K{Q'I uSXC01 11rMc`tj|d}tj||k(}tj|ddd}||}||}d}t t |D]<}|||k(s d}t d|dzD]}|||k(s |dz }||dzz}||z }>||z S)a>Alternative implementation to check for correctness of `average_precision_score`. Note that this implementation fails on some edge cases. For example, for constant predictions e.g. [0.5, 0.5, 0.5], y_true = [1, 0, 0] returns an average precision of 0.33... but y_true = [0, 0, 1] returns 1.0. r-NrOr?)r2rPrRargsortrangerT) rJrHrUn_posorderscoreiprecjs rK_average_precisionrews &!!$I FF6Y& 'E JJw " %EenG E]F E 3w<  !9 !D1a!e_ !9 )CKD  AGOD TME  5=rMct||\}}}tt|}tt|}d}tdt |D]}|||||||dz z zz }|S)aoA second alternative implementation of average precision that closely follows the Wikipedia article's definition (see References). This should give identical results as `average_precision_score` for all inputs. References ---------- .. [1] `Wikipedia entry for the Average precision `_ rr-)rlistreversedr^rT)rJrH precisionrecall thresholdaverage_precisionrbs rK_average_precision_slowrms$:&'#J IvyXi()I (6" #F 1c)n %HYq\VAYA-FGGH rMcd}||||\}}t||}d}|}d||z z||zz} ||z } dd|| z | | z z zzS)zcAlternative implementation to check for correctness of `roc_auc_score` with `max_fpr` set. c.t||\}}}|||k}tj||}|||k}tj||kD}|dz } || ||g} || ||g} tj|tj|| | }||fS)Nr-)rr2appendargmaxinterp) rJ y_predictmax_fprfprtpr_new_fprnew_tpridx_outidx_inx_interpy_interps rK _partial_rocz,_partial_roc_auc_score.._partial_rocs 2 S!cWn%))GW-cWn%))C'M*1KW.KW.))GRYYw(%KL!!rMr?r-)r ) rJrsrtr~rxry partial_aucfpr1fpr2min_areamax_areas rK_partial_roc_auc_scorerss "$FIw?GWgw'K D DdTk"dTk2Hd{H !{X-(X2EFF GGrMdropTc,td\}}}t||}t|||\}}}t||}t ||dt |t |||j|jk(sJ|j|jk(sJy)NTr?drop_intermediater&decimal)rLrZrr rrrr1) rrJrwrH expected_aucrurv thresholdsroc_aucs rKtest_roc_curvers)5FAw(L$VWMCj#smGg|Q?vw!?@ 99 !! ! 99 (( (( (rMc`tjjd}tjdgdzdgdzz}|j dd}t ||d\}}}|ddk(sJ|d dk(sJ|j |j k(sJ|j |j k(sJy) Nr2r-dsizeTrrO)r2r6r7arrayrandintrr1)rErJrIrurvthrs rKtest_roc_curve_end_pointsrs ))   "C XXqcBh!r) *F [[[ %FffEMCc q6Q;; r7a<< 99 !! ! 99 !! !rMcltd\}}}t||\}}}g}|D]I}tj||k\|z}tj|} |j d|z| z Kt ||d|j |j k(sJ|j |j k(sJy)NTrr\r&r)rLrr2rRrprr1) rJrwrHrurvr tpr_correctttprDs rKtest_roc_returns_consistencyrs)5FAw$VW5CjK ) VVW\V+ , FF6N38a<() c;: 99 !! ! 99 (( (( (rMctd\}}}tjt5t ||dddy#1swYyxYw)NFr)rLpytestraises ValueErrorr)rJrwrHs rKtest_roc_curve_multirs>(6FAw z "#&'"###s AA ctd\}}}t||dz \}}}t||}t|dd|j|jk(sJ|j|jk(sJy)NTrr?r&r)rLrr rr1)rJrwrHrurvrrs rKtest_roc_curve_confidencerso(5FAw$VWs];Cj#smGgtQ7 99 !! ! 99 (( (( (rMctd\}}}tj|j}t ||\}}}t ||}t |dd|j|jk(sJ|j|jk(sJtj|j}t ||\}}}t ||}t |dd|j|jk(sJ|j|jk(sJt ||\}}}t ||}t |dd|j|jk(sJ|j|jk(sJy)NTrrr&rg(\?)rLr2onesr1rr rzeros)rJpredrH trivial_predrurvrrs rKtest_roc_curve_hardrsE+48FD'776<<(L$V\:Cj#smGgtQ7 99 !! ! 99 (( (( (88FLL)L$V\:Cj#smGgtQ7 99 !! ! 99 (( (( (%VT2Cj#smGgtQ7 99 !! ! 99 (( (( (rMcgd}gd}d}tjt|5t||\}}}dddt t j tt j|jjk(sJ|j|jk(sJd}tjt|5t|Dcgc]}d|z  c}|\}}}dddt |t j t|t j|j|jk(sJ|j|jk(sJy#1swY,xYwcc}w#1swYxYw)N) r-r-r-r-r-r-r-r-r-r-) rr-rr-rr-rr-rr-INo negative samples in y_true, false positive value should be meaninglessmatchHNo positive samples in y_true, true positive value should be meaninglessr-) rwarnsrrrr2fullrTnanr1)rJrIexpected_messagerurvrxs rKtest_roc_curve_one_labelr!sH +F +F T ,4D E9(8S*9sBGGC ORVV<= 99 !! ! 99 (( (( ( S ,4D EJ()@A!a%)@&IS*JsBGGC ORVV<= 99 !! ! 99 (( (( (#99*AJJs)E! E3 E.  E3!E+.E33E<c ddg}ddg}t||\}}}t||}t|gdt|gdt|dddg}ddg}t||\}}}t||}t|gdt|gdt|dddg}ddg}t||\}}}t||}t|ddgt|ddgt|dddg}ddg}t||\}}}t||}t|gdt|gdt|dddg}ddg}t||\}}}t||}t|ddgt|ddgt|dddg}dd g}d }t j t | 5t||\}}}dddt|gd t|tjtjtjgd }t j t | 5t||}dddtjsJddg}dd g}d}t j t | 5t||\}}}dddt|tjtjtjgt|gd d }t j t | 5t||}dddtj|sJtjddgddgg}tjddgddgg}t j t | 5t||ddddt j t | 5t||ddddtt||ddtt||ddtjddgddgg}tjddgddgg}t j t | 5t||ddddt j t | 5t||ddddtt||ddtt||ddtjddgddgg}tjddgddgg}tt||ddtt||ddtt||ddtt||ddtjddgddgg}tjddgddgg}tt||ddtt||ddtt||ddtt||ddy#1swY)xYw#1swYxYw#1swYuxYw#1swY xYw#1swYxYw#1swYrxYw#1swYxYw#1swYxYw)Nrr-rrr-rr-r-r\r??rrrrr\OOnly one class is present in y_true. ROC AUC score is not defined in that case.rmacroaverageweightedsamplesmicro) rrrrrrrr2rmathisnanr)rJrHrvrurwrrr s rKtest_roc_curve_toydatar<sVF!fGFG,KCaFG,Gc9-c9-%VF!fGFG,KCaFG,Gc9-c9-%VF!fGFG,KCaFG,GcAq6*cAq6*%VF!fGFG,KCaFG,Gc9-c9-%VFCjGFG,KCaFG,GcAq6*cAq6*%VFTlG S ,4D E10 S!1c?3cBFFBFFBFF#;< 5 ,4D E-FG,- ::c??VFTlG T ,4D E10 S!1cBFFBFFBFF#;<c?3 5 ,4D E-FG,- ::c??XX1v1v& 'FhhAA'(G ,4D E8fgw78 ,4D E;fgz:; fgyI3O fgwGM XX1v1v& 'FhhAA'(G ,4D E8fgw78 ,4D E;fgz:; fgyI3O fgwGM XX1v1v& 'FhhAA'(G fgwGK fgzJAN fgyI1M fgwGK XX1v1v& 'Fhhc S#J/0G fgwGM fgzJCP fgyI3O fgwGM}11--11--88;;88;;s`"W W'0W4- X*XX4X(&X5W$'W14W>X XX%(X25X?c gd}gd}t||d\}}}t|tjdddggd}gd }t||d\}}}t|tjdd dd dgy) Nrrrrr-r-r皙?r333333?ffffff?r\Trr\rr rrrrrrrr-r-r-r-r-r- r皙?rrr皙?rrrrrrr\rr)rrr2inf)rJrHrvrurs rK test_roc_curve_drop_intermediaterst F,G$VWMCjj2663S*AB5FOG$VWMCjj2663S#s*KLrMcgd}gd}tjdd}t|||\}}}tj|dkj dk(sJtj|dkj dk(sJy)Nrrr-r-r-)rr333333?皙?rr sample_weightr)r2repeatrdiffrR)rJrHrrurvrws rK!test_roc_curve_fpr_tpr_increasingrszF'GIIc1%MFG=IKCa GGCL1  ! ! #q (( ( GGCL1  ! ! #q (( (rMc0ddg}ddg}tt||dddg}ddg}tt||dgd}gd}tt||dddg}ddg}tt||dgd}gd}tt||dy)Nrr-rr-rrrrrr-)rr )rrAs rKtest_aucrs AA AAc!Qi- AA AAc!Qi-AAc!Qi- AA AAc!Qi+AAc!Qi-rMctjt5tgdddgdddtjt5tdgdgdddgd}gd}dj t j |}tjttj|5t||dddy#1swYxYw#1swYxYw#1swYyxYw) Nrrrr)r&r-r)rz+x is neither increasing nor decreasing : {}r) rrrr formatr2rreescape)rrA error_messages rKtest_auc_errorsrs z ") Oc3Z() z " SEC5 AAAHHRSUM z=)A B Aq ))s#C C= C+CC(+C4zy_true, labels)rr-rr&rr-r&)abrc)rrrc vtjgdgdgdgdg}tgdgd}tgdgd}||zd z }tgd gd }tgd gd }||zd z }tddgddg} tddgddg} | | zd z } ||z| zdz } tt|||d| ||| g} gd}tj| |}tt|||dd|d}t j t|5t|||dddddy#1swYyxYw)Nrrrrrrffffff?r333333?rrrr-rr-)rrrrr-r)rrrr&r-r-r)rrrr)rrrr-rrrrrrovolabels multi_class)rrr)weightsrrrrz6average=None is not implemented for multi_class='ovo'.r)r2rrrrrrNotImplementedError)rJry_scoresscore_01score_10average_score_01score_02score_20average_score_02score_12score_21average_score_12ovo_unweighted_score pair_scores prevalenceovo_weighted_scorers rK#test_multiclass_ovo_roc_auc_toydatarsixx /+3CDEBBCCs#C"$'C."C+.C7z.ignore:Support for labels represented as bytes labels_typergrctddg|}d}tjt|5||ddgdddy#1swYyxYw)Nabz8Support for labels represented as bytes is not supportedrrr\)rrr TypeError)rOr[rr@s rK.test_binary_clf_curve_implicit_bytes_pos_labelr`lsJ  t k :F DC y ,'6C:&'''s AA cgd}gd}gd}||||}||dd|dd|dd}t||D]\}}t||y)Nr)rrrrr)r-r-r-rrrrO)rTr)rOrJrHrresult_1result_2arr_1arr_2s rK(test_binary_clf_curve_zero_sample_weightrfxsf F'G%M&'GH&"+ws|=QTRTCUVHHh/& uu%&rMctd\}}}t|||t|dd|dd|\}}}|ddk(sJ|d|ddjk(sJd|t j |dk(<|j }t|||t||gd}gd } t|| |\}}}|r+t|gd t|gd t|gd n*t|gd t|gdt|gd |j|jk(sJ|j|jdzk(sJy)NTrr-rrr\rOr-rrr-)r-r&rr)rQUU?r\r\)r\rrr)r-r&r)rrirr\r\)r\rrrr) rL_test_precision_recall_curvermeanr2wherecopyrrr) rrJrwrHrDrr y_true_copyrpredict_probass rKtest_precision_recall_curverqs<(5FAw $7%VABZPTUGAq! Q43;; Q46!":??$ $$ $%'F288FaK !++-K $7{F+ F!N$V^tTGAq! 67/09%;<45<( 66QVV   66QVVaZ  rMct|||\}}}t||}t|ddt|t||t t |||d|j |j k(sJ|j |j dzk(sJt|tj||\}}}|j |j k(sJ|j |j dzk(sJy)Nrgrh|?rr&rr-) rrmrr rrerr2 zeros_like)rJrHrrDrnrprecision_recall_aucs rKrjrjs-fgQUVAq*267C2E1=5fgF67+-A1 66QVV   66Z__q( (( (- g&$Aq* 66QVV   66Z__q( (( (rMc htjd5ddg}ddg}t|||\}}}t||}t |gdt |gdt |dddg}ddg}t|||\}}}t||}t |gd t |gd t |d ddg}ddg}t|||\}}}t||}t |d dgt |ddgt |d ddg}ddg}t|||\}}}t||}t |gdt |gdt |dddg}d d g}t|||\}}}t||}t |d dgt |dd gt |d ddg}d dg}t jtd5t|||\}}}dddt jtd5t||}dddt|gdt|gdt|dddg}d dg}t|||\}}}t t||dt |gdt |gdtjddgddgg}tjddgddgg}t jtd5tt||dd dddt jtd5tt||dddddtt||ddtt||ddtjddgddgg}tjddgddgg}t jtd5tt||dd dddt jtd5tt||dddddtt||ddtt||dd tjddgddgg}tjddgddgg}t t||dd t t||dd t t||dd t t||dd tjddgddgg}tjddgddgg}t jtd5tt||dd dddtt||dd t jtd5tt||dd dddt jtd5tt||dd dddtjddgddgg}tjddgddgg}tt||ddtt||ddtt||ddtt||ddtjddgddgg}tjd d gd d gg}t t||dd t t||dd t t||dd t t||dd dddtjd5tjddgddgg}tjddgddgg}t jtd5tt||ddddddddy#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swYgxYw#1swY8xYw#1swYxYw#1swYxYw#1swYyxYw)Nraise)allrr-r)rr-r-rr\)rrr\rrrrrz!No positive class found in y_truerr)r\r\r\)r-rrrrrrrignore) r2errstaterr rrrr UserWarningrr)rrJrHrDrnrwauc_prcs rK#test_precision_recall_curve_toydatar|s  !V Qa&(DQ1a)&':!![1!!Y/GS)Qa&(DQ1a)&':!!_5!!_5 GS)Qa&(DQ1a)&':!!c1X.!!c1X.GS)Qa&(DQ1a)&':!![1!!Y/GS)Q*(DQ1a)&':!!c1X.!!aX.GS)Q, \\+-P Q V,VWPTUGAq! V \\+-P Q ?-fg>G ?9%9%#Q,(DQ1a3FGDcJ!!_5!!]3Aq6Aq6*+((QFQF+, \\+-P Q  'I3  \\+-P Q  'Lc    #FGY G  /QSVWAq6Aq6*+((QFQF+, \\+-P Q  'I3  \\+-P Q  'Lc    #FGY G  /QSVWAq6Aq6*+((QFQF+, #FGW Es   #FGZ H#   #FGY G   #FGW Es Aq6Aq6*+((QFQF+, \\+-P Q  'I3    #FGZ H# \\+-P Q  'KS  \\+-P Q  'I3   Aq6Aq6*+((QFQF+,/QSVW #FGZ H#   #FGY G  /QSVWAq6Aq6*+((S#Jc 34 #FGW Es   #FGZ H#   #FGY G   #FGW Es iV p  "Aq6Aq6*+((QFQF+, \\+-P Q  'La   S V V ? ?         4      sV V x   s F*`^#`7 ^'C `^4*#` _&B `/_#`+_D`#_(<;`7_5#`3` D4`A`(0` `(^$ `'^1 ,`4^> 9`_ `_ `_% `(_2 -`5_? :`` ```% !`((`1cDgd}gd}t||d\}}}t|gdgd}gd}t||d\}}}t|gdgd }gd }t||d\}}}t|d d ggd }gd }t||d\}}}t|gd y)z9Check the behaviour of the `drop_intermediate` parameter.rrTr)rrr\rr)rrrrrr\rrrr)rrrrrrr-r-r-r-N)rr)rJrHrirjrs rK-test_precision_recall_curve_drop_intermediater_s F,G$:4%!IvzJ05FOG$:4%!IvzJ >?F"G$:4%!IvzJc +F"G$:4%!IvzJ 45rMctjdt}d|ddd<tjd}t ||dk(sJy)NrrKr-rr)r2rr5rr rJrHs rK&test_average_precision_constant_valuesrsE XXc %FF3Q3KggclG #67 3t ;; ;rMctjddg}tjddg}d}tjt|5t ||ddddy#1swYyxYw)Nrr-z>pos_label=2 is not a valid label. It should be one of \[0, 1\]rr&rUr2rrrrr rJrIerr_msgs rK4test_average_precision_score_binary_pos_label_errorsrsZ XXq!f F XXq!f FOG z 1=!<===s A$$A-ctjddgddgddgddgg}tjddgddgddgddgg}d}tjt|5t ||d dddy#1swYyxYw) Nr-rrrrrznParameter pos_label is fixed to 1 for multilabel-indicator y_true. Do not set pos_label or set pos_label to 1.rrrrs rK8test_average_precision_score_multilabel_pos_label_errorsrsXX1v1v1v1v6 7F XXSzC:SzC:F GF 6  z 1=!<===s A88Bc tjgd}tjgdgdgdgdgdgdg}d}tjt|5t ||d dddy#1swYyxYw) N)rr-r&rr-r&)rrr)rrr)rrr)rrrzdParameter pos_label is fixed to 1 for multiclass y_true. Do not set pos_label or set pos_label to 1.rrrrrs rK8test_average_precision_score_multiclass_pos_label_errorsrsq XX( )F XX         F 6  z 1=!<===s A44A=c\td\}}}t||}t|d|z}t|d|z}t||dz }||k(sJ||k(sJ||k(sJt||}t|d|z}t|d|z} t||dz } ||k(sJ|| k(sJ|| k(sJy)NTrrgư>rJ)rLrr ) rJrwrHrroc_auc_scaled_uproc_auc_scaled_downroc_auc_shiftedpr_aucpr_auc_scaled_uppr_auc_scaled_downpr_auc_shifteds rKtest_score_scale_invariancers )5FAwFG,G%fcGm<'w?#FGbL9O ' '' ' ) )) ) o %% % $VW 5F.vsW}E0H,VWr\BN % %% % ' '' ' ^ ## #rMz(y_true,y_score,expected_fpr,expected_fnrrr)rrr)rrr-rrrr)rrrrrr)r-r-rr)rr-r-r-rr)rrrcTt||\}}}t||t||yNrrrJrH expected_fpr expected_fnrrufnrrws rKtest_det_curve_toydatars*6FG,KCaC&C&rM)rJrHrrr)r-r-rr)r-r-rcXt|||\}}}t||t||y)Nrr)rJrHrrrrurrws rK test_det_curve_drop_intermediaters-&FG?PQKCaC&C&rMrrrcTt||\}}}t||t||yrrrs rKtest_det_curve_tie_handlingr s*FG,KCaC&C&rMcTttgdgdtgdgdy)Nrrr)rrrrr-r-)rrrMrKtest_det_curve_sanity_checkr!s )[)$&<=rMrH)rrrrr-ctgdtjd|\}}}t|ddgt|ddgt||tjgy)N)rr-rr-rr-rrr-r)rr2rrr)rHrurrks rKtest_det_curve_constant_scoresr)sT#!2771g+>CiC!Q C!Q I01rMrJ)rrrrrr-r)rrrr-r-r-)rrr-r-r-r-)rr-r-r-r-r-cZt||\}}}t|dgt|dgy)Nrrr)rJrurrws rKtest_det_curve_perfect_scoresr4s.66:KCaC!C!rMzy_true, y_pred, err_msgzinconsistent numbers of samplesrrrz#Only one class is present in y_truer-r-r-)cancerr not cancer)rrrzpos_label is not specifiedc|tjt|5t||dddy#1swYyxYw)Nr)rrrrrs rKtest_det_curve_bad_inputrEs1 z 1"&&!""" 2;cZdgdzdgdzz}tjgd}d|z }t||d\}}}t||d\}}}|dtjd k(sJ|dtjd k(sJt ||ddd t ||ddd y) Nrrrr) rrrrrrrrrrr-rrrrrO)r2rrrr'r) rJy_pred_pos_not_cancery_pred_pos_cancerfpr_pos_cancerfnr_pos_cancer th_pos_cancerfpr_pos_not_cancerfnr_pos_not_cancerth_pos_not_cancers rKtest_det_curve_pos_labelrYsZ!^|nq0 0FHH%WX114=51NNM AJA=*,=  v}}S1 11 1 Q 6==#5 55 5N$6tt$<=N$6tt$<=rMct|ddggddggdt|ddggddggdt|ddggddggdt|gdggdgdt|gdggdgdt|gd ggdgdt|gd ggdgd t|gd ggdgd t|gdggdgdt|gdggdgd t|gdggdgdt|gd ggdgdt|gd ggdgdt|gd ggdgd t|gdggdgdt|gdggdgdt|gdggdgd t|gdggdgdt|gd ggdgd t|gd ggdgdt|gd ggdgdt|gdggdgdt|gdggdgdt|ddggddggdt|ddggddggdt|ddggddggdt|gdggdgdt|gdggdgdt|gd ggdgdt|gd ggdgd t|gd ggdgdt|gdggdgdt|gdggdgdt|gdggdgdt|gdggdgdy)Nrr-rrrrrrrrgUUUUUU?rg?rg?rrrrrrrr)rrrUUUUUU?)r-r-r-r)rrrr)r lrap_scores rKcheck_lrap_toyrssp QF8tTl^ I;1A0BCSI I;1A0BCSI I;1A0BCQG I;1A0BCUKI;!1 235HI;!1 235H I;1A0BCQG I;0ABEJ L>4H3IJERrMc td}tddD]}|jd|f}tj|}tj d|f}|||dk(sJ|||dk(sJtj d|f}|||dk(sJ|||dk(rJt|dgdgdgdggdgdgdgdggdy)Nrr&rr-rr\r)r$r^uniformr2rsrrr)rr,n_labelsrH y_score_tiesrJs rK!check_zero_or_all_relevant_labelsrs%a(L!QK 7&&QM&:}}W- 1h-(&'*c111&,/3666!X'&'*c111&,/3666 7QC!qcA3'3%#u)EFrMctjt5|gdgddddtjt5|gdgdgdgdgdddtjt5|gdgdgdgdgdddtjt5|ddgddggddgdddtjt5|ddgddggddggdddtjt5|ddgddggdgdggdddtjt5|ddggddgddggdddtjt5|dgdggddgddggdddtjt5|ddgddggdgdggdddy#1swYxYw#1swYxYw#1swY_xYw#1swY9xYw#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swYyxYw) Nr)rrrr)rrr)rrr)rrrrr-)rrrrs rKcheck_lrap_error_raisedrs z "09./0 z "U90/?STU z "  //R  z "-QFQF#aV,- z "/QFQF#q!fX./ z "1QFQF#qcA3Z01 z "/QF8q!fq!f-./ z "1QC!:AA/01 z "1QFQF#qcA3Z011)00UU  --//11//1111skG G$?G15G>(H HH$H0:H<G!$G.1G;>H HH!$H-0H9<Ic tddD]v}tjd|f}td|D]N}t||z D];}tjd|f}d|d|||zf<t |||||z =Pxy)Nr&rJr-r)r^r2rrr)rrrH n_relevantrVrJs rKcheck_lrap_only_tiesrs!RL X''1h-( 8, XJX 23 X1h-045q#j 0001#Jvw$?hAVW X X XrMc tddD]}|tj|jd|fdzz }tjd|f}d|d<d|d<t |||d|z dzdz td|D]gt|z D]Ttjd|f}d|dzf<t |||t fdtDViy)Nr&rJr-)rr)rrOrc3@K|]}|dz|zdzzz yw)r-Nr).0rnrrVs rK z>check_lrap_without_tie_and_increasing_score..s/QC!GaK:#=>s)r^r2r3rQrrrR)rrrHrJrrVs @@rK+check_lrap_without_tie_and_increasing_scorers!RLbii1991h-H1LM1h-(t u Jvw7!h,:Ja9OP 8, JX 23 1h-045q#j 0001#vw/!&z!2  rMc t||t|}t|}|j\}}tj|f}t |D]}tj ||d\}}|j}||z tj |dzj} |  ||jd} | jdk(s| j|k(rd||<d||<| D]* t fd| D} ||xx| z z cc<,||xx| jzcc<|jS)z8Simple implementation of label ranking average precisionT)return_inverser-) minlengthrrc34K|]}|kywrr)rrnlabelranks rKrz_my_lrap..%s JAaDK!7 Js) r#r"r1r2emptyr^rPrbincountcumsumnonzerorRrk)rJrHrBrrarb unique_rankinv_rankn_ranks corr_rankrelevantn_ranked_aboverrs @@rK_my_lrapr sJFG,  F'"G ,,Ix HHi\ "E 9 "!# '!*T J X""!KK! <CCE !9$$&q) ==A (!:E!H a 5E! J JJN !He4 4H  5 aHMM!5"8 ::<rMrctdd|||\}}t|jd|jd|}t|dr|j }t ||}t ||}t||t|}|j||f}t ||}t ||}t||y)Nr-F)rCallow_unlabeledr, n_classesrBr) n_componentsrCr,toarrayr) rrr1hasattrrrrrr$r) rrrBr,rwrJrH score_lrap score_my_lraps rK%check_alternative_lrap_implementationr/s/! IAv$\\!_<<?!G w "//#6vwGJVW-M M2&l3L""I(>"?G6vwGJVW-M M2rMcheckfuncc||yrr)rrs rKtest_label_ranking_avprOs  $KrMc"ttyr)rrrrMrKtest_lrap_error_raisedr]s ABrMrB)r-r&rrDr)r&rrJr,c(tt|||yr)rr)rBrr,s rK$test_alternative_lrap_implementationras*-y)\rMcltjgdgdgdgt}tjgdgdgdg}tjgd}tjgd }tt ||| tj ||ztj |z y) Nrrhr~rK)rrrr)rrrrrrrr)rrr\)r\r\rr)r2rboolrrrR)rJrHsamplewise_lrapsrs rK&test_lrap_sample_weighting_zero_labelsrjs XX|\<@ MFhh 35IJGxx 01HH_-M- G=  }//0266-3HH rMc ttddggddggdttddggddggdttddggddggdttddggddggdttgdggdgdttgdggdgdttgd ggdgdttgd ggdgdttgd ggdgd ttgd ggdgd ttgdggdgd ttgdggdgd ttgdggdgdttgdggdgd ttgd ggdgdttgd ggdgd ttgd ggdgdttgd ggdgd ttgdggdgdttgdggdgd ttgdggdgdttgdggdgd ttgd ggdgdttgd ggdgd ttgd ggdgdttgd ggdgd ttgdggdgdttgdggdgd ttgd gdggdgdgdttgd gdgd ggdgdgdgdttgd gdgd ggdgdgdgdy)Nrr-rrr&rrrrrrrrrrrrrg$@rr-rr-rrJrr=g@rr-rrr rrMrKtest_coverage_errorrsAx4,@!DAx4,@!DAx4,@!DAx4,@!D {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L {5F4GH!L 9-/KL   9 - y)/T     9 - y)/T   rMcttddggddggdttddggddggdttddggddggdttddggddggdttgdggdgdttgdggdgdttgdggdgdttgd ggdgdttgd ggdgd ttgd ggdgd ttgd ggdgd ttgdggdgd y)Nrrr-r&rrrrrrrrrrrrrMrKtest_coverage_tie_handlingrs.Ax3*>BAx3*>BAx3*>BAx3*>B {5E4FGK {5E4FGK {5E4FGK {5E4FGK {5E4FGK {5E4FGK {5E4FGK {5E4FGKrMzy_true, y_scorec|tjtd5t||dddy#1swYyxYw)Nz'Expected 2D array, got 1D array insteadr)rrrr rs rKtest_coverage_1d_error_messagers2 z)S T(vw'(((rc httddggddggdttddggddggdttgdggdgdttgdggdgdttgd ggdgdttgd ggdgd ttgd ggdgdttgd ggdgd ttddggddggdttddggddggdttddggddggdttddggddggdttgdggdgdttgdggdgdttgdggdgdttgdggdgdttgdgd ggdgdgdttgdgd gd ggdgdgdgdttgdgd gd ggdgdgdgdy)Nrr-rrrrrrrrr\rrrrrrrrrr=rrrrrMrKtest_label_ranking_lossrs*QF8tTl^DaH*QF8tTl^DaH*I;9J8KLaP*I;9J8KLeT*I;9J8KLaP*I;9J8KLeT*I;9J8KLeT*I;9J8KLeT*QF8tTl^DaH*QF8tTl^DaH*QF8sCj\BAF*QF8sCj\BAF*I;9J8KLaP*I;9J8KLaP*I;9I8JKQO*I;9I8JKQOIy1OY3OP   9 - y)/T  "   9 - y)/T  " rM csr_containerc ztt|tjgdgdggdgdgdy)Nrrrrr)rrr2r)r s rKtest_label_ranking_loss_sparser s4 "((Iy#9: ;mY=W   rMc.tjt5tddgddggddgdddtjt5tddgddggddggdddtjt5tddgddggdgdggdddtjt5tddggddgddggdddtjt5tdgdggddgddggdddtjt5tddgddggdgdggdddy#1swY!xYw#1swYxYw#1swYxYw#1swYxYw#1swYsxYw#1swYyxYw)Nrr-)rrrrrrMrK$test_ranking_appropriate_input_shaper  s z "5QFQF+aV45 z "7QFQF+q!fX67 z "9QFQF+qcA3Z89 z "7QF8q!fq!f%567 z "9QC!:AA'789 z "9QFQF+qcA3Z899557799779999sGEEE'?E36E?.F EE$'E03E<?F Fcttddggddggdttddggddggdttgdggdgdttgdggdgdttgdggdgdttgdggdgdttgd ggdgdttgd ggdgdy) Nr-rrrrrrrrrrrrMrKtest_ranking_loss_ties_handlingrs*QF8sCj\BAF*QF8sCj\BAF*I;9I8JKUS*I;9I8JKUS*I;9I8JKQO*I;9I8JKQO*I;9I8JKQO*I;9I8JKQOrMctdd\}}| dz}t||tjj dj d\}}t||yNrrJ)r,rr-)r&rrJ)r_test_dcg_score_forr2r6r7 random_samplerwrJrHs rKtest_dcg_scorer&sT.ALIAvgkG(ii++A.<<\JOFG(rMc tjtj|jddz}t ||}t ||}||kj sJt ||d|kj sJ|j|jdfk(sJ|j|jdfk(sJ|t jtj|dddddf|z jdk(sJy)Nr-r&rkrrOaxis) r2log2r3r1rrwrr'sortrR)rJrHdiscountidealras rKrr.swwryya1A56H vv .E vw /E UN   !! ! vv 3u < A A CC C ;;6<<?, ,, , ;;6<<?, ,, , FMM2776?1dd7#;h#F"K"KQR"K"ST TT TrMc tjtjdg}tj|j}t ||}t ||d}dtj tjddz }|tj|j|jzgk(sJ|tj||dddddfzjgk(sJd|dd df<t ||}t ||d}|tj||dddddfzjgk(sJ|tj|ddj|dd dfjz|ddj|ddd fjzzgk(sJy) NrT ignore_tiesr-r&rrOrr) r2r3r3rr1rrrr'rRrk)rJrHdcgdcg_ignore_ties discountss rK test_dcg_tiesr%9s ZZ1 'Fhhv||$G VW -C(dKOBGGBIIaO,,I &--6;;=!@ AB BB B fmmi&DbD/.I-N-N-P,QR RR RGAqrEN VW -C(dKO fmmi&DbD/.I-N-N-P,QR RR R &-- bqM   &AB-"4"4"6 6m!F1bqb5M$6$6$88 9   rMc tjdjd}t||ddt j t||ddk(sJy)N )r&rrT)rr!)r2r3rQrrr')rs rKtest_ndcg_ignore_ties_with_kr(MsP " f%A aaT 2fmm1a1$/7  rMctjgdg}tjgdg}d}tjt|5t ||dddy#1swYyxYw)zDCheck `ndcg_score` exception when `y_true` contains negative values.)g{Gzg(\gGz޿g(\?gQ?)gQ?gףp= ?rgQ?gHzG?z7ndcg_score should not be used on negative y_true valuesrN)r2rrrrr)rJrHrs rK test_ndcg_negative_ndarray_errorr*TsZ XX89 :Fhh678GP z)9 :$67#$$$s  A$$A-ctjdjdd}|tjj dj dd|j z}t||}t||d }|tj|k(sJ|tjd k(sJ|d z }t||tjd k(sJy) NFrrJr皙ɿrrTr r\r.) r2r3rQr6r7rr1rrr')rJrHndcg ndcg_no_tiess rKtest_ndcg_invariantr0]s YYr] " "1b )Fryy,,Q/77c 7UUG fg &Dfg4@L 6==. .. . 6==% %% % tOG fg &&--*< << > c ?M [  q277299Q?33 4 55 5  ;  q277299Q?33 4 55 5 [  q277299Q?33 4 55 5 "+  q288BIIaO44 5 66 6 fg; ?6== RWWRYYq!_ % %++-D   VW+ >&-- RWWRYYq!_ % %++-C   Fbggbii1o66;;= [  )BGGAJ6 7 88 8 [  rwwqz " ## # VW+ >&--C   fg; ?6==QTCU UU UrMcd}tjt|5tdggdggdddy#1swYyxYw)zhCheck that we raise an informative error message when trying to compute NDCG with a single document.zTComputing NDCG is only meaningful when there is more than 1 document. Got 1 instead.rr-N)rrrr)rs rKtest_ndcg_error_single_documentr;sC   z 1!QC5A3% !!!s 8Actdd\}}| dz}t||tjj dj d\}}t||yr)r_test_ndcg_score_forr2r6r7rrs rKtest_ndcg_scorer>sT.ALIAvgkG)ii++A.<<\JOFG)rMct||}t||}||kjsJ|dk(jd}||tjt j |j k(sJ||tjt j|j k(sJ||tjt|||t|||z k(sJ||tjt j|j k(sJ|j|jdfk(sJ|j|jdfk(sJy)Nrr-r) rrwrr'r2rrRrrr1)rJrHrraall_zeros rKr=r=sR  /E  0E UN   !! !!   a (H ( v}}RWWxi__5F-GH HH H ?fmmBHHX\\^,DE EE E ( v}}67+XI6 VV ,hY 7 8    ?fmmBHHX\\^,DE EE E ;;6<<?, ,, , ;;6<<?, ,, ,rMc .tjgd}t||ddk(sJt||ddk(sJtjt 5t||dsJ dddtjt 5t||dsJ dddtjt 5t||dsJ dddtjgd}t||d}t||}||k(sJt||d d k(sJt d \}}}tjd ddD]%}tt|||t|||'y#1swY xYw#1swYxYw#1swYxYw)Nrr-)rtgMbP?gg?r)rrrg{Gz?rrTrg-C6?r) r2rrrrrrLlinspacerr)rJrroc_auc_with_max_fpr_oneunconstrained_roc_aucrIrwrts rKtest_partial_roc_auc_scorerEsw XXl #F  3q 88 8  71 << < z ";VVT:::; z ":VVS999: z "8VVQ7778xx+,H,VXqI)&(; #'< << < 3 73 >> >'t4FFA;;tQ*  &&' : "667 ;  ;;::88s$E2 E?=F 2E<?F Fzy_true, k, true_scorerr&rrctjgdgdgdgdg}t|||}|tj|k(sJy)Nrrrrrrrrrrrrrrr2rrrr')rJr true_scorerHras rKtest_top_k_accuracy_scorerLsJhh  G !A 6E FMM*- -- -rMzy_score, k, true_score)rOrOr-r-)rOr-rOr-)rrrr)rrrrc,gd}|jdk\r|jdkrdnd}|dk(r"||kDjtjn|}t |||}t ||}||cxk(rtj|k(sJJy)Nrrr-rr) minmaxastyper2int64rr rr')rHrrKrJrkrIra score_accs rK test_top_k_accuracy_score_binaryrSsF{{})gkkmq.@aI78Avg ! ) )"(( 36F A 6Evv.I I :z!: :: :: :rMzy_true, true_score, labels)rr-r-r&r)rr>r>r)rrrr>labels_as_ndarrayc|rtj|}tjgdgdgdgdg}t||d|}|t j |k(sJy)z,Test when labels and y_score are multiclass.rrGrHrIr&rrN)r2r3rrrr')rJrKrrTrHras rK0test_top_k_accuracy_score_multiclass_with_labelsrWs]F#hh  G !Af EE FMM*- -- -rMc tjdddd\}}t||d\}}}}td}|j ||t ||f||fD]f\}}t ddDcgc]}t||j||!}}tjtj|dkDrfJycc}w)NrJr.r)rrB n_informativer,)r,r&r) rmake_classificationrrr;rTr^rr<r2rwr) r@rAX_trainX_testy_trainy_testrGrscoress rK$test_top_k_accuracy_score_increasingr`s  ' 'BQ DAq(811'M$GVWf ! ,CGGGWWf%'89+1HMaQS  CD C$5$5a$8A >  vvbggfo)*** + s2$Cctjgdgdgdgdg}t|||tj|k(sJy)N)rrrr)r-rrr)rrrrrrrJ)rJrrKrHs rKtest_top_k_accuracy_score_tiesrb(sEhh      G 1 5z9R RR RrMz y_true, krctjgdgdgdgdg}d}tjt|5t |||}ddddk(sJy#1swYxYw) Nr)rrrr)rrrr)rrrrzu'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a perfect score and is therefore meaningless.rrr-)r2rrrrr)rJrrHrras rK!test_top_k_accuracy_score_warningrd=snhh  G 6 ,4D E;$VW:; A::;;s AA'zy_true, y_score, labels, msg)rg= ףp=?r-r&)rrr)rrrr)rrrz9y type must be 'binary' or 'multiclass', got 'continuous'zZNumber of classes in 'y_true' \(4\) not equal to the number of classes in 'y_score' \(3\).)rrrrz"Parameter 'labels' must be unique.z#Parameter 'labels' must be ordered.)rrr-r&zSNumber of given labels \(4\) not equal to the number of classes in 'y_score' \(3\).rz3'y_true' contains labels not in parameter 'labels'.)rrr)rrrz}`y_true` is binary while y_score is 2d with 3 classes. If `y_true` does not contain all the labels, `labels` must be providedctjt|5t||d|dddy#1swYyxYw)Nrr&rV)rrrr)rJrHrr@s rKtest_top_k_accuracy_score_errorrfVs:f z -BVW&ABBBs5>c|gdgdg}tjgdgdg}t||}|tjdk(sJy)Nrr)rrrr)r2rrrr')r rJrHresults rKOtest_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_inputrisI Iy1 2Fhh34G 267 CF V]]5) )) )rMmetricr0))FT)rr-)rr\)zeroonectjjd}d|d}}|j||d}|j |}||||}t |t rtj|rJy|\}} } tj|jrJtj| jrJtj| jrJy)zCheck that the metric works with different types of `pos_label`. We can expect `pos_label` to be a bool, an integer, a float, a string. No error should be raised for those types. *rJrOT)rreplacerN) r2r6r7choicerE isinstancerSrany) rjr0rErBrUrJy_probarhmetric_1metric_2rs rK#test_ranking_metric_pos_label_typesrvs ))   #Cwr{yI ZZiZ >Fhhy!G FGy 9F&% 88F####)/&(J88H%))+++88H%))+++88J'++----rMctjj|}|jddd}|j d}t ||\}}}tj |dsJy)zCheck that thresholds do not exceed 1.0 when `y_score` is a probability estimate. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/26193 rr&rJrN)r2r6r7rrErisinf)r6rErJrHrwrs rK(test_roc_curve_with_probablity_estimatesrysc ))   2 3C [[AB[ 'FhhrlG 1Aq* 88JqM "" "rM)NF)rrDr)rrnumpyr2rscipyrsklearnrrsklearn.datasetsrsklearn.exceptionsrsklearn.linear_modelrsklearn.metricsr r r r r rrrrrrrrsklearn.metrics._rankingrrsklearn.model_selectionrsklearn.preprocessingrsklearn.random_projectionrsklearn.utils._testingrrrrrsklearn.utils.extmathr sklearn.utils.fixesr!sklearn.utils.validationr"r#r$ CURVE_FUNCSrLrZrermrmark parametrizerrrrrrrrrrrrrrrr#r+r<rArHrNrQrZfilterwarningsr`rfrqrjr|rrrrrrrrrrrrrrrrrrrrrrrr^rrrrrrr r rrrr%r(r*r0r9r;r>r=rErLrSrWr`rbrdrfrirvryrrMrKrs  !;53M40;*.  )#` 2<&H8$/ )0 ) ")&#))8)6oNd M ).&" , + , & / t$ /X/Xd , + & '9: , & t$ y! / DD:( ,hbhh|.DiP / BHH) *   BHH\ "    1 BHH) *   BHH\ " F   BHH) * #J   BHH\ "    BHH) *   A BHH) *  A BHH) *  A BHH\ "  }CFN7P8OFPP'"% 8  '"% 8  F"B 7  "c 2   4R8="F 0G"F 0&*{3#4#{3E4E0LM(> 'JK(9:';LN'{3 &4 &$/ 0 >),$/`0`F6D <= ==*$2. K!qc* NQC!- NQC!- %sQC0 K : NM9= NM9= %}i@ K#, NQC!- NQC!- %sQC0 K> NNLA NNLA %~|D KM: NI}= NI}= %y-@)2'32'N K!1<G Ki> N$4lEJ NKDA KM5A KM4@ NI}eD NI}dC ''. Q#saVaV, Q#saVaV, $sAhA7 $sAhA7 $qcA3/ $i; $i= $i=  ' '$EF2G2         Q?@ QH?@ K!FG K!FG .  (  " " >48Sv.14 X6#N9:3@ 3) "G!RSTCm4j1q2325 *4n L  $% %&' &'(((V.9: 9 P)U($=u 6#V7#VL!* -  4 q$ q# q$ . . . !1a( . !1c* . !1a( & 'A. & 'C0 & 'A.   ;  ; , |4 , l3 , l3 & '/CD ,tUm<.=.&+& q$ q# q! S S q q$"     G      +  !   ! 0  !    1            A F o . W  KNQdBeQdB .9*:* & 3I9U C. .( #rM