
    -iN                    F   S SK r S SKrS SKJr  S SKJrJrJr  S SKr	S SK
r
S SKJr  S SKJr  S SKJr  S SKJrJr  S SKJr  S S	KJr  S S
KJrJrJrJrJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,J-r-J.r.  S SK/J0r0J1r1  S SK2J3r3  S SK4J5r5J6r6  S SK7J8r8  S SK9J:r:  S SK;J<r<J=r=J>r>J?r?J@r@  S SKAJBrB  S SKCJDrDJErE  S SKFJGrG  SS jrHS rIS rJ\
R                  R                  SSS S\	R                  /5      S 5       rN\
R                  R                  SS /S4S S/S4/ SQS4/5      S 5       rOS  rPS! rQ\
R                  R                  S"5      S# 5       rS\
R                  R                  S"5      S$ 5       rT\
R                  R                  S"5      S% 5       rUS& rV\
R                  R                  S'/ S(Q\	R                  " / S)Q/ S*Q/ S+Q/ S,Q/5      4/ S-Q/ S.Q4/5      S/ 5       rX\
R                  R                  S'/ S0Q\	R                  " / S,Q/ S,Q/ S1Q/ S1Q/ S2Q/5      4/ S3Q/ S4Q4/5      S5 5       rYS6 rZS7 r[S8 r\S9 r]\
R                  R                  S:\D5      \
R                  R                  S;\E5      S< 5       5       r^S= r_\
R                  R                  S>/ S?Q5      S@ 5       r`SA raSB rb\
R                  R                  SC\	R                  " / SDQ5      \	R                  " / SDQ5      SE.SF4\	R                  " / SDQ5      \	R                  " / SGQ5      SE.SH4\	R                  " / SDQ5      \	R                  " / SIQ5      SE.SJ4\	R                  " / SGQ5      \	R                  " / SDQ5      SE.SK4/5      SL 5       rc\
R                  R                  SM\	R                  " / SNQ5      \	R                  " / SOQ5      SE.SP4/5      SQ 5       rdSR re\
R                  R                  SSSS/5      ST 5       rfSU rg\
R                  R                  SVSWSX0SYSX0SZSXS[.S\S]S[.SXSXS[.S\S^S[./5      S_ 5       rh\
R                  R                  S`S\S\S[.S\4\	R                  SXS[.\	R                  4S^SXS[.S^4\	R                  \	R                  S[.\	R                  4\	R                  \	R                  4/5      Sa 5       rj\
R                  R                  S`S\S\S[.S\4\	R                  SXS[.SX4\	R                  SbS[.Sb4\	R                  \	R                  S[.\	R                  4\	R                  \	R                  4/5      Sc 5       rkSd rlSe rm\
R                  R                  SS S\	R                  /5      \
R                  R                  SfS /S /4/5      \
R                  R                  Sg\"\" \#SSh9\,\-/5      Si 5       5       5       rn\
R                  R                  SfS /S /4/5      \
R                  R                  Sg\"\" \#SSh9\,\-/5      Sj 5       5       roSk rpSl rqSm rrSn rs\
R                  R                  SoSpSq/5      Sr 5       rtSs ru\
R                  R                  St/ SuQ5      Sv 5       rvSw rwSx rxSy ry\
R                  R                  Sz/ S{4S|S}/S~4/SS/S9S 5       rz\
R                  R                  SSS S// SQ4/ SQS9S 5       r{S r|\
R                  R                  S/ SQ5      S 5       r}S r~S rS rS rS rS rS rS rS r\
R                  R                  S"5      S 5       rS rS rS rS rS rS rS r\
R                  R                  SSS/5      S 5       r\
R                  R                  S"5      S 5       r\
R                  R                  S"5      S 5       r\
R                  R                  S"5      \
R                  R                  SSSS\	R                  \	R                  4/5      S 5       5       r\
R                  R                  SS/5      \
R                  R                  St/ SQ5      \
R                  R                  SS S\	R                  /5      S 5       5       5       r\
R                  R                  St/ SQ5      S 5       r\
R                  R                  SS S\	R                  /5      S 5       rS rS r\
R                  R                  SS S\	R                  /5      S 5       r\
R                  R                  SSS S\	R                  /5      S 5       r\
R                  R                  SSS S\	R                  /5      S 5       r\
R                  R                  SSS S\	R                  /5      S 5       rS rS rS rS rS rS rS rS rS rS rS r\
R                  R                  S\	GRN                  \	GRP                  \	GRR                  /5      S 5       r\
R                  R                  S\	GRN                  \	GRP                  \	GRR                  /5      S 5       r\
R                  R                  Sf/ SQ/ SQ4/ SQSS /S S/SS //4/ SQ/ SQ/ SQ/ SQ/4/5      S 5       rS rS rS rS rS rS rS r\
R                  R                  S/ SQ/ SQ4/ SQ/ SQ4/ SQ/ SQ4/5      S 5       r\
R                  R                  Sg\&\"\" \#SbSh9\+\,\-\/5      \
R                  R                  S/ SQ5      S 5       5       r\
R                  R                  S\	R                  " S S/5      \	R                  " SS /5      SX4\	R                  " S S/5      \	R                  " S S/5      S\4\	R                  " S S/5      \	R                  " S S /5      SX4\	R                  " S S /5      \	R                  " S S /5      S\4/5      S 5       r\
R                  R                  S\(" \"\	R                  S9\(" \#S\	R                  S9\(" \,\	R                  S9\(" \-\	R                  S9/5      S 5       rS rS rS rS rg)    N)partial)chainpermutationsproduct)linalg)hamming)	bernoulli)datasetssvm)make_multilabel_classification)UndefinedMetricWarning)accuracy_scoreaverage_precision_scorebalanced_accuracy_scorebrier_score_lossclass_likelihood_ratiosclassification_reportcohen_kappa_scoreconfusion_matrixf1_scorefbeta_scorehamming_loss
hinge_lossjaccard_scorelog_lossmake_scorermatthews_corrcoefmultilabel_confusion_matrixprecision_recall_fscore_supportprecision_scorerecall_scorezero_one_loss)_check_targetsd2_log_loss_score)cross_val_score)LabelBinarizerlabel_binarize)DecisionTreeClassifier)MockDataFrame)assert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equalignore_warnings)_nanaverage)CSC_CONTAINERSCSR_CONTAINERS)check_random_stateFc                    U c  [         R                  " 5       n U R                  nU R                  nU(       a  X#S:     X3S:     p2UR                  u  pE[
        R                  " U5      n[        S5      nUR                  U5        X&   X6   p2[        US-  5      n[
        R                  R                  S5      n[
        R                  X'R                  USU-  5      4   n[        R                  " SSSS9n	U	R!                  USU USU 5      R#                  X(S 5      n
U(       a	  U
SS2S	4   n
U	R%                  X(S 5      nX8S nXU
4$ )
zMake some classification predictions on a toy dataset using a SVC

If binary is True restrict to a binary classification problem instead of a
multiclass classification problem
N   %   r      linearT)kernelprobabilityrandom_state   )r
   	load_irisdatatargetshapenparanger2   shuffleintrandomRandomStatec_randnr   SVCfitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_pred_probay_predy_trues                \/var/www/html/venv/lib/python3.13/site-packages/sklearn/metrics/tests/test_classification.pymake_predictionrZ   :   s6    $$&AAQxq51GGI
		)A
R
 CKKN4qy1}D ))


"C
a9cJ&6778A ''t!
DC771Ud8QuX.<<QuXFL $AqD)[[5"FuXF<''    c            
      6   [         R                  " 5       n [        U SS9u  pnSSSSS.SS	S
SS.SSSSS.SSSSS.SSSSSS.S.n[        UU[        R
                  " [        U R                  5      5      U R                  SS9nUR                  5       UR                  5       :X  d   eU Hy  nUS:X  a'  [        XV   [        5      (       d   eXV   XF   :X  d   eM0  XV   R                  5       XF   R                  5       :X  d   eXF    H  n[        XF   U   XV   U   5        M     M{     [        US   S   [        5      (       d   e[        US   S   [        5      (       d   e[        US   S   [        5      (       d   e[        US   S   [        5      (       d   eg ) NFrL   rM   g7Mo?gUUUUUU?ghQ?   )	precisionrecallf1-scoresupportUUUUUU?gc1Ƹ?g433333?   g)k??$I$I?   gCFQ?gc?gƢ?K   ra   r_   r`   rb   g?gD~WG?g]3p?)setosa
versicolor	virginica	macro avgaccuracyweighted avgT)labelstarget_namesoutput_dictrn   rj   r_   rm   rb   )r
   r<   rZ   r   r@   rA   lenrq   keys
isinstancefloatr+   rC   )irisrX   rW   _expected_reportreportkeymetrics           rY   ,test_classification_report_dictionary_outputr}   j   s   D'UCFA
 -)*	
 -*+	
 -)+	
 ++'	
 '++(	
5 OD #yyT../0&&F ;;=O002222*fk51111;/"6666;##%)=)B)B)DDDD).#O$8$@&+fBUV /  oh/<eDDDDok2;?GGGGoh/	:C@@@@ok29=sCCCCr[   c                  V   [        / / SS9n S[        R                  [        R                  [        R                  SS.[        R                  [        R                  [        R                  SS.S.n[        U [        5      (       d   eU R                  5       UR                  5       :X  d   eU Hy  nUS:X  a'  [        X   [        5      (       d   eX   X   :X  d   eM0  X   R                  5       X   R                  5       :X  d   eX    H  n[        X   U   X   U   5        M     M{     g )NT)rX   rW   rr           r   ri   )rn   rm   ro   rn   )r   r@   nanru   dictrt   rv   r+   )rz   ry   r{   r|   s       rY   2test_classification_report_output_dict_empty_inputr      s   ""RTJFff	
 ff	
O fd####;;=O002222*fk51111;/"6666;##%)=)B)B)DDDD).#O$8$@&+fBUV / r[   zero_divisionwarnr;   c                    / SQ/ SQp![         R                  " SS9 n[        XU SS9  U S:X  a8  [        U5      S:  d   eU H   nSnU[	        UR
                  5      ;   a  M    e   O	U(       a   eS S S 5        g ! , (       d  f       g = f)	Nabc)r   r   dTrecord)r   rr   r   r;   z7Use `zero_division` parameter to control this behavior.)warningscatch_warningsr   rs   strmessage)r   rX   rW   r   itemmsgs         rY   0test_classification_report_zero_division_warningr      s~    $oF		 	 	--T	
 F"v;?"?Oc$,,////  : 
.	-	-s   AA8!A88
Bzlabels, show_micro_avgTr   r;   r4   c                 x    SS/SS/p2[        X#U SS9nU(       a  SU;   d   eSU;  d   egSU;   d   eSU;  d   eg)a  Check the behaviour of passing `labels` as a superset or subset of the labels.
WHen a superset, we expect to show the "accuracy" in the report while it should be
the micro-averaging if this is a subset.

Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/27927
r   r;   T)rp   rr   z	micro avgrn   Nr   )rp   show_micro_avgrX   rW   rz   s        rY   1test_classification_report_labels_subset_supersetr      s\     VaVF"6&dSFf$$$'''V###&(((r[   c                  2   [         R                  " / SQ/ SQ/5      n [         R                  " / SQ/ SQ/5      n[        X5      S:X  d   e[        X 5      S:X  d   e[        X5      S:X  d   e[        U[         R                  " U5      5      S:X  d   e[        U [         R                  " U 5      5      S:X  d   e[        U [         R                  " U R
                  5      5      S:X  d   e[        U[         R                  " U R
                  5      5      S:X  d   eg )Nr   r;   r;   r;   r   r;   r   r   r;         ?r;   r   )r@   arrayr   logical_notzerosr?   y1y2s     rY   .test_multilabel_accuracy_score_subset_accuracyr      s    	9i(	)B	9i(	)B"!S((("!Q&&&"!Q&&&"bnnR01Q666"bnnR01Q666"bhhrxx01Q666"bhhrxx01Q666r[   c            	      8   [        SS9u  pn[        XS S9u  p4pV[        USS/S5        [        USS/S5        [        US	S
/S5        [        USS/5        0 SS04 H  n[        R
                  " 5          [        R                  " S5        [        X40 UD6n[        USS5        [        X40 UD6n	[        U	SS5        [        X40 UD6n
[        U
S
S5        [        [        X4SS0UD6SU-  U	-  SU-  U	-   -  S5        S S S 5        M     g ! , (       d  f       M  = f)NTrM   averageg\(\?g333333?r4   g)\(?g(\?皙?gRQ?   r   rM   errorbeta      )rZ   r   r,   r-   r   r   simplefilterr    r!   r   r+   r   )rX   rW   rx   rR   rfskwargspsrsfss              rY   %test_precision_recall_f1_score_binaryr      s'   't4FA 1NJA!a$q1a$q1a$q1q2r(#
 	8,-$$&!!'* :6:B%b$2f77B%b$2&3F3B%b$2F==f=R"$r	B7 '& .&&s   3B
D


D	z1ignore::sklearn.exceptions.UndefinedMetricWarningc            	         S[        SS/SS/5      :X  d   eS[        SS/SS/5      :X  d   eS[        SS/SS/5      :X  d   eS[        SS/SS/SS9:X  d   eS[        SS/SS/5      :X  d   eS[        SS/SS/5      :X  d   eS[        SS/SS/5      :X  d   eS[        SS/SS/[	        S5      S9:X  d   e[        SS/SS/[	        S5      S9[
        R                  " [        SS/SS/SS95      :X  d   eg )	N      ?r;   r   r   r   infg     j@)r    r!   r   r   rv   pytestapprox r[   rY   +test_precision_recall_f_binary_single_classr      s;   
 /1a&1a&1111,1v1v....(Aq6Aq6****+q!fq!f15555/2r(RH5555,Bx"b2222(B8b"X....+r2hRuU|DDDDBx"be=RHr2hS1B   r[   c                  ,   / SQn / SQn[        U [        R                  " S5      S9n[        U[        R                  " S5      S9nX4X#4/n[        U5       H  u  nu  p[	        X/ SQS S9n[        / SQU5        [	        X/ SQSS9n[        [        R                  " / SQ5      U5        S	 H1  nUS
:X  a  US:X  a  M  [        [	        X/ SQUS9[	        XS US95        M3     M     S H  n[        R                  " [        5         [	        X#[        R                  " S5      US9  S S S 5        [        R                  " [        5         [	        X#[        R                  " SS5      US9  S S S 5        M     [        R                  " / SQ/ SQ/5      n [        R                  " / SQ/ SQ/5      n[        XS
SS/S9u  pp[        [        R                  " XU
/5      [        R                  " / SQ5      5        g ! , (       d  f       N= f! , (       d  f       GM.  = f)N)r;      r   r4   )r;   r;   r   r4   r   classes)r   r;   r4   r   r   rp   r   )r   r   r   r   r   macro)microweightedsamplesr   r   )Nr   r   r      r   r   r   r;   r   r   r;   r;   r;   r   r;   r   rp   )      ?r;   竪?)r'   r@   rA   	enumerater!   r,   meanr+   r   raises
ValueErrorr   r   )rX   rW   
y_true_bin
y_pred_binr=   iactualr   rR   r   r   rx   s               rY   $test_precision_recall_f_extra_labelsr   3  s    FF		!=J		!=Jz67D(Ff_dS!";VD f_gV!"''*C"DfM 8G)#QVOWUVD'J 8  /( 7]]:&		!gV ']]:&ryyQ/? '& 7 XXy),-FXXy),-F0	1a&JA! !+RXX6G-HI '&&&s    G2!H2
H 	
H	c                     / SQn / SQn[        U [        R                  " S5      S9n[        U[        R                  " S5      S9nX4X#4/n[        U5       H  u  nu  p[	        [
        XSS/S9n[	        [
        XS S9n[        SS	/U" S S
95        [        SU" SS
95        [        SU" SS
95        [        SU" SS
95        S H  nU" US
9U" US
9:w  a  M   e   M     g )N)r;   r;   r4   r   )r;   r   r   r   r   r   r;   r   rp   r   r   r   r   r   UUUUUU?r   r   )r   r   r   )r'   r@   rA   r   r   r!   r,   r+   )	rX   rW   r   r   r=   r   	recall_13
recall_allr   s	            rY   &test_precision_recall_f_ignored_labelsr   a  s     FF		!=J		!=Jz67D(FL&!QH	\6$G
!3*i.EFOYw-GH3Yz5RSGYw%?@ 6GW-G1LLLL 6  /r[   c            	      "   [         R                  " / SQ/ SQ/ SQ/ SQ/ SQ/ SQ/5      n [         R                  " / SQ/ SQ/ SQ/ S	Q/ S
Q/ SQ/5      nSn[        R                  " [        US9   [        XSS9  SSS5        g! , (       d  f       g= f)z:Test multiclass-multiouptut for `average_precision_score`.)r4   r4   r;   r;   r4   r   r   r;   r4   r;   )r4   r   r;   ffffff?皙?皙?皙?333333?r   r   r   r   r   r   r   )r   r   r   )r   r   r   z.multiclass-multioutput format is not supportedmatchr4   	pos_labelN)r@   r   r   r   r   r   )rX   y_scoreerr_msgs      rY   -test_average_precision_score_non_binary_classr   x  sy    XX	
	F hh	
	G ?G	z	11= 
2	1	1s   ,B  
Bzy_true, y_scorer   r   r;   r4   r   r   r   r   )r   r   r   r   r;   r;   r;   r;   r;   r;   r;   )r   r   r   r   r   333333?r   re   re   r;   r;   c                 &    [        X5      S:X  d   eg)a  
Duplicate values with precision-recall require a different
processing than when computing the AUC of a ROC, because the
precision-recall curve is a decreasing curve
The following situation corresponds to a perfect
test statistic, the average_precision_score should be 1.
r;   Nr   rX   r   s     rY   -test_average_precision_score_duplicate_valuesr     s    8 #63q888r[   )r4   r4   r;   r;   r   )r   r   r   )r   r   r   r   )r   r   r   c                 &    [        X5      S:w  d   eg )Nr   r   r   s     rY   (test_average_precision_score_tied_valuesr     s    : #63s:::r[   c                      Sn [         R                  " [        U S9   [        / SQ/ SQSSS9  S S S 5        g ! , (       d  f       g = f)NzNote that pos_label \(set to 2\) is ignored when average != 'binary' \(got 'macro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   r;   r4   r4   r4   r   r   r   )r   warnsUserWarningr   r   s    rY   (test_precision_recall_f_unused_pos_labelr     s9    
	  
k	-'yAw	
 
.	-	-	   6
Ac            	          [        SS9u  pnS nU" X5        U" U  Vs/ s H  n[        U5      PM     snU Vs/ s H  n[        U5      PM     sn5        g s  snf s  snf )NTr   c                    [        X5      n[        USS/SS//5        UR                  5       u  p4pVX6-  XE-  -
  n[        R                  " X4-   X5-   -  Xd-   -  Xe-   -  5      nUS:X  a  SOXx-  n	[        X5      n
[        XSS9  [        U
SSS9  g )	N   r         r   r4   decimal=
ףp=?)r   r-   flattenr@   sqrtr   r,   )rX   rW   cmtpfpfntnnumdentrue_mccmccs              rY   test*test_confusion_matrix_binary.<locals>.test  s    f-2Q!R12gggrw27+rw727CDq1ci/!#;!#tQ7r[   rZ   r   rX   rW   rx   r  rO   s        rY   test_confusion_matrix_binaryr    sV    't4FA8 	&	!&Q#a&&	!F#;FqCFF#;<	!#;
   AA
c            	          [        SS9u  pnS nU" X5        U" U  Vs/ s H  n[        U5      PM     snU Vs/ s H  n[        U5      PM     sn5        g s  snf s  snf )NTr   c                 N    [        X5      n[        USS/SS//SS/SS///5        g )Nr  r  r   r  r   r-   )rX   rW   r  s      rY   r  5test_multilabel_confusion_matrix_binary.<locals>.test  s5    (82"a1b' 2b!Wq"g4FGHr[   r  r  s        rY   'test_multilabel_confusion_matrix_binaryr    sW    't4FAI 	&	!&Q#a&&	!F#;FqCFF#;<	!#;r  c            	          [        SS9u  pnSS jnU" X5        U" U  Vs/ s H  n[        U5      PM     snU Vs/ s H  n[        U5      PM     snSS9  g s  snf s  snf )NFr   c           	      R   [        X5      n[        USS/SS//SS/SS//S	S
/SS///5        U(       a  / SQO/ SQn[        XUS9n[        USS/SS//S	S
/SS//SS/SS///5        U(       a  / SQO/ SQn[        XUS9n[        USS/SS//S	S
/SS//SS/SS//SS/SS///5        g )N/   r   r      &   r      r      r   r4      )021r   r4   r;   r   )r(  r)  r*  3)r   r4   r;   r   rh   r   r  )rX   rW   string_typer  rp   s        rY   r  9test_multilabel_confusion_matrix_multiclass.<locals>.test  s   (82q'Ar7#r1gAw%72r(QG9LM	

 %0Y(G2q'Ar7#r2hB%8B7RG:LM	

 *5%,(Ga1b'"bAr7#a2q'"a1a&!		
r[   T)r-  )Fr  r  s        rY   +test_multilabel_confusion_matrix_multiclassr/    sX    'u5FA
6 	&	!&Q#a&&	!F#;FqCFF#;N	!#;s
   AA
csc_containercsr_containerc                    [         R                  " / SQ/ SQ/ SQ/5      n[         R                  " / SQ/ SQ/ SQ/5      nU" U5      nU" U5      nU " U5      nU " U5      n[         R                  " / SQ5      nSS	/SS//SS	/SS//S	S
/SS	///n	X$U/n
X5U/nU
 H"  nU H  n[        X5      n[        X5        M     M$     [        X#SS9n[        USS	/SS//SS/S	S//S	S/S
S	///5        [        X#S
S	/S9n[        US	S
/SS	//SS	/SS///5        [        X#S
S	/SS9n[        US	S	/SS//SS/S	S	//S	S/SS	///5        [        X#USS9n[        US
S	/S
S
//SS/S	S//S	S/SS	///5        g )Nr   r   r;   r   r;   r;   r   r   r   r   )r4   r;   r   r;   r   r4   T
samplewiser   )rp   r6  )sample_weightr6  r   r   )r@   r   r   r-   )r0  r1  rX   rW   
y_true_csr
y_pred_csr
y_true_csc
y_pred_cscr7  real_cmtruespreds
y_true_tmp
y_pred_tmpr  s                  rY   +test_multilabel_confusion_matrix_multilabelrA  &  s   
 XXy)Y78FXXy)Y78Fv&Jv&Jv&Jv&J HHY'MAA1a&1a&!1QFQF3CDG,E,E
J,ZDBr+    
%V	EBraVaV,1v1v.>!Q!Q@PQR 
%VQF	CBraVaV,1v1v.>?@ 
%VQFt	TBraVaV,1v1v.>!Q!Q@PQR 
%m
B raVaV,1v1v.>!Q!Q@PQRr[   c            	         [         R                  " / SQ/ SQ/ SQ/5      n [         R                  " / SQ/ SQ/ SQ/5      n[        R                  " [        SS9   [        XS	S
/S9  S S S 5        [        R                  " [        SS9   [        X/ SQ/ SQ/ SQ/S9  S S S 5        Sn[        R                  " [        US9   [        XS/S9  S S S 5        Sn[        R                  " [        US9   [        XS/S9  S S S 5        [        R                  " [        SS9   [        / SQ/ SQSS9  S S S 5        Sn[        R                  " [        US9   [        / SQ/ SQ// SQ/ SQ/5        S S S 5        g ! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g = f)Nr   r3  r4  r   r   r   inconsistent numbers of samplesr   r;   r4   r7  zshould be a 1d arrayr;   r4   r   )r4   r   r   )r   r   r   z%All labels must be in \[0, n labels\)r   r   r   zSamplewise metricsr   r   Tr5  z'multiclass-multioutput is not supported)r4   r;   r   r;   r   r4   )r@   r   r   r   r   r   )rX   rW   r   s      rY   'test_multilabel_confusion_matrix_errorsrG  P  sL   XXy)Y78FXXy)Y78F 
z)J	K#F1a&I 
L	z)?	@#9i*K	
 
A 7G	z	1#FB4@ 
26G	z	1#FA3? 
2 
z)=	>#IyTJ 
? 8G	z	1#Y	$:Y	<RS 
2	1+ 
L	K	@	@ 
2	1 
2	1 
?	>
 
2	1sH   E.F =F,F"F3G.
E= 
F
F"
F03
G
Gz%normalize, cm_dtype, expected_results))truer   TUU?)predr   rI  )allr   geq?)Nr   r4   c                     / SQS-  n[        [        [        / SQ5      6 5      n[        X4U S9n[	        XR5        UR
                  R                  U:X  d   eg )Nr   r   	normalize)listr   r   r   r*   dtypekind)rN  cm_dtypeexpected_resultsy_testrW   r  s         rY   test_confusion_matrix_normalizerU  n  sK     ]F%i012F	&I	>BB)88==H$$$r[   c                      / SQn / SQn[        XSS9nUR                  5       [        R                  " S5      :X  d   e[        R
                  " 5          [        R                  " S[        5        [        XSS9nS S S 5        WR                  5       [        R                  " S5      :X  d   e[        R
                  " 5          [        R                  " S[        5        [        XSS9  S S S 5        g ! , (       d  f       N|= f! , (       d  f       g = f)	N)r   r   r   r   r;   r;   r;   r;   )r   r   r   r   r   r   r   r   rH  rM         @r   rJ  r   )r   sumr   r   r   r   r   RuntimeWarning)rT  rW   cm_truecm_preds       rY   ,test_confusion_matrix_normalize_single_classr\    s    %F%Fv@G;;=FMM#.... 
	 	 	"g~6"6VD 
# ;;=FMM#....		 	 	"g~66: 
#	" 
#	" 
#	"s   &C.?&C?.
C<?
Dc                      / SQn / SQn[         R                  " [        SS9   [        X5        SSS5        g! , (       d  f       g= f)z8Test `confusion_matrix` warns when only one label found.r   r   r   r   zA single label was found inr   N)r   r   r   r   )rT  rW   s     rY   "test_confusion_matrix_single_labelr_    s0    FF	k)F	G( 
H	G	Gs	   7
Azparams, warn_msg)r;   r;   r;   r   r   r   rX   rW   z?`positive_likelihood_ratio` is ill-defined and set to `np.nan`.)r   r   r   r   r   r   zdNo samples were predicted for the positive class and `positive_likelihood_ratio` is set to `np.nan`.r   r   r   r;   r;   r;   z?`negative_likelihood_ratio` is ill-defined and set to `np.nan`.z9No samples of the positive class are present in `y_true`.c                     [         R                  " [        US9   [        S0 U D6  S S S 5        g ! , (       d  f       g = fNr   r   )r   r   r   r   )paramswarn_msgs     rY   test_likelihood_ratios_warningsrf    s*    X 
k	2)&) 
3	2	2   /
=zparams, err_msg)r   r;   r   r;   r   r;   r;   r   r   r4   zeclass_likelihood_ratios only supports binary classification problems, got targets of type: multiclassc                     [         R                  " [        US9   [        S0 U D6  S S S 5        g ! , (       d  f       g = frc  )r   r   r   r   )rd  r   s     rY   test_likelihood_ratios_errorsrj    s)    $ 
z	1)&) 
2	1	1rg  c                     [         R                  " S/S-  S/S-  -   5      n [         R                  " S/S-  S/S-  -   S/S-  -   5      n[        X5      u  p#[        US5        [        US	5        [        X 5      u  p#[	        U[         R
                  S-  5        [        U[         R                  " S5      S
S9  [         R                  " S/S-  S/S-  -   5      n[        XUS9u  p#[        US5        [        US5        g )Nr;   r   r   r  r4   
   r  g?g_B{	%?g-q=)rtolr      r   r   rD  gUUUUUU@gqq?)r@   r   r   r*   r-   r   r   )rX   rW   posnegr7  s        rY   test_likelihood_ratiosrq    s     XXqcAgb()FXXqcAgb(A3723F&v6HCC!C! 'v6HCsBFFQJ'C!51
 HHcURZ3%!)34M&v]SHCC C!r[   raise_warningc                     [         R                  " SS/5      n[         R                  " SS/5      nSn[        R                  " [        US9   [        XU S9  SSS5        g! , (       d  f       g= f)z]Test that class_likelihood_ratios raises a `FutureWarning` when `raise_warning`
param is set.r;   r   zI`raise_warning` was deprecated in version 1.7 and will be removed in 1.9.r   )rr  N)r@   r   r   r   FutureWarningr   )rr  rX   rW   r   s       rY   0test_likelihood_ratios_raise_warning_deprecationru    sQ     XXq!fFXXq!fF
UC	m3	/mL 
0	/	/   A  
A.c                  d   [         R                  " / SQ5      n [         R                  " / SQ5      n[        XSS9u  p#U[        R                  " S5      :X  d   e[         R                  " / SQ5      n [         R                  " / SQ5      n[        XSS9u  p4U[        R                  " S5      :X  d   eg)z{Test that class_likelihood_ratios returns the worst scores `1.0` for both LR+ and
LR- when `replace_undefined_by=1` is set.r4  r   r;   replace_undefined_byr   r   N)r@   r   r   r   r   )rX   rW   positive_likelihood_ratiorx   negative_likelihood_ratios        rY   1test_likelihood_ratios_replace_undefined_by_worstr|    s    
 XXi FXXi F#:Q$  %c(:::: XXi FXXi F#:Q$ A %c(::::r[   ry  LR+r   LR-g      )r}  r~  r   r   rW  c                     [         R                  " SS/5      n[         R                  " SS/5      nSn[        R                  " [        US9   [        XU S9  SSS5        g! , (       d  f       g= f)zTest that class_likelihood_ratios raises a `ValueError` if the input dict for
`replace_undefined_by` is in the wrong format or contains impossible values.r;   r   zGThe dictionary passed as `replace_undefined_by` needs to be in the formr   rx  N)r@   r   r   r   r   r   )ry  rX   rW   r   s       rY   6test_likelihood_ratios_wrong_dict_replace_undefined_byr    sU     XXq!fFXXq!fF
SC	z	-1E	
 
.	-	-rv  zreplace_undefined_by, expectedc                 &   [         R                  " / SQ5      n[         R                  " / SQ5      n[        X#U S9u  pE[         R                  " U5      (       a  [         R                  " U5      (       d   egU[        R
                  " U5      :X  d   eg)z|Test that the `replace_undefined_by` param returns the right value for the
positive_likelihood_ratio as defined by the user.r4  r   rx  Nr@   r   r   isnanr   r   )ry  expectedrX   rW   rz  rx   s         rY   0test_likelihood_ratios_replace_undefined_by_0_fpr  7  sr     XXi FXXi F#:-A$  
xxxx12222(FMM(,CCCCr[   r   c                 &   [         R                  " / SQ5      n[         R                  " / SQ5      n[        X#U S9u  pE[         R                  " U5      (       a  [         R                  " U5      (       d   egU[        R
                  " U5      :X  d   eg)z|Test that the `replace_undefined_by` param returns the right value for the
negative_likelihood_ratio as defined by the user.r   r   rx  Nr  )ry  r  rX   rW   rx   r{  s         rY   0test_likelihood_ratios_replace_undefined_by_0_tnr  S  sq     XXi FXXi F#:-A$ A 
xxxx12222(FMM(,CCCCr[   c                     [         R                  " S/S-  S/S-  -   5      n [         R                  " S/S-  S/S-  -   S/S-  -   S/S-  -   5      n[        X5      n[        USS	S
9  U[        X5      :X  d   e[         R                  " U S/S-  5      n [         R                  " US/S-  5      n[        XSS/S9U:X  d   e[        [        X 5      S5        [         R                  " S/S-  S/S-  -   S/S-  -   5      n [         R                  " S/S-  S/S-  -   S/S-  -   5      n[        [        X5      SSS
9  [         R                  " S/S-  S/S-  -   S/S-  -   5      n [         R                  " S/S-  S/S-  -   S/S-  -   5      n[        [        X5      SSS
9  [        [        XSS9SSS
9  [        [        XSS9SSS
9  g )Nr   (   r;   <   rg   rl  2   gʡE?r   r  r4   r   r   r   .   ,   4          g??g+?r7   weightsg_vO?	quadraticg#?)r@   r   r   r+   append)r   r   kappas      rY   test_cohen_kappar  o  s    
1#(aS2X%	&B	1#(aS2X%b0A38;	<Bb%Eua0%b---- 
2sQw	B	2sQw	BRQF3u<<<)"137 
1#(aS2X%b0	1B	1#(aS2X%b0	1B)"161E 
1#(aS2X%b0	1B	1#(aS2X%b0	1B)"161E)"(CVUVW"+6r[   c                      SS/n [         R                  " S/S-  S/S-  -   5      n[         R                  " S/S-  5      n[        R                  " [        SS9   [        XU S	9  S
S
S
5        g
! , (       d  f       g
= f)zLTest that correct error is raised when users pass labels that are not in y1.r;   r4   r   r   r   rl  z6At least one label in `labels` must be present in `y1`r   r   N)r@   r   r   r   r   r   )rp   r   r   s      rY   (test_cohen_kappa_score_error_wrong_labelr    sl    VF	3%!)seai'	(B	3%"*	B	R
 	"0
 
 
s   A--
A;zy_true, y_predr|   r   c                    [         R                  " 5          [         R                  " S5        U " XUS9nSSS5        [        R                  " U5      (       a  [        R                  " W5      (       d   egWU:X  d   eg! , (       d  f       NP= f)zeCheck the behaviour of `zero_division` when setting to 0, 1 or np.nan.
No warnings should be raised.
r   r   N)r   r   r   r@   r  )r|   rX   rW   r   results        rY   !test_zero_division_nan_no_warningr    sl     
	 	 	"g&mD 
# 
xxxx&&& 
#	"s   A>>
Bc                     [         R                  " [        5         U " XSS9nSSS5        WS:X  d   eg! , (       d  f       N= f)zlCheck the behaviour of `zero_division` when setting to "warn".
A `UndefinedMetricWarning` should be raised.
r   r  Nr   )r   r   r   )r|   rX   rW   r  s       rY   test_zero_division_nan_warningr    s7     
,	-f= 
.S== 
.	-s	   4
Ac                     [         R                  R                  U 5      nUR                  SSSS9nUR                  SSSS9n[	        [        X#5      [         R                  " X#5      S   S5        g )Nr   r4   rg   sizer   r;   rl  )r@   rD   rE   randintr+   r   corrcoef)global_random_seedrS   rX   rW   s       rY   -test_matthews_corrcoef_against_numpy_corrcoefr    sa    
))

 2
3C[[AB['F[[AB['F&)2;;v+Ft+Lbr[   c                    [         R                  R                  U 5      nUR                  SSSS9nUR                  SSSS9nUR	                  S5      n[        X#US9n[        U5      n[        [        U5       VVV	s/ s HD  n[        U5        H1  n[        U5        H  n	XWU4   XXU	4   -  XYU4   XWU4   -  -
  PM      M3     MF     sn	nn5      n
[        [        U5       VVVs/ s Hk  nUS S 2U4   R                  5       [         R                  " [        U5       VVs/ s H#  n[        U5        H  oU:w  d  M
  X\U4   PM     M%     snn5      -  PMm     snnn5      n[         R                  " [        U5       VVVs/ s Hj  nXWS S 24   R                  5       [         R                  " [        U5       VVs/ s H#  n[        U5        H  oU:w  d  M
  X[U4   PM     M%     snn5      -  PMl     snnn5      nU
[         R                  " X-  5      -  n[        X#US9n[        UUS5        g s  sn	nnf s  snnf s  snnnf s  snnf s  snnnf )Nr   r4   rg   r  rD  rl  )r@   rD   rE   r  randr   rs   rX  ranger  r   r+   )r  rS   rX   rW   r7  CNkmlcov_ytypr   gcov_ytytcov_ypyp
mcc_jurmanmcc_ourss                    rY   %test_matthews_corrcoef_against_jurmanr    s    ))

 2
3C[[AB['F[[AB['FHHRLM}EAAA 1X	
1X1X dGa1gQ$!qD' 11  2 2	
H  1X	
  adGKKMffuQxLx!qA!Vga1ggxLMN	
H vv 1X	
  dGKKMffuQxLx!qA!Vga1ggxLMN	
H BGGH$788J }MH*b11	
 M	
 M	
sC   <AH+$:H8H2:H2H8>9I7H?H?!I2H8?Ic           	         [         R                  R                  U 5      nUR                  SSSS9 Vs/ s H  o"S:X  a  SOSPM     nn[	        [        X35      S5        U Vs/ s H  o"S:X  a  SOSPM     nn[	        [        X45      S5        [        USS/S	9n[         R                  " USS5      n[	        [        X55      S5        [	        [        / S
Q/ S
Q5      S5        [	        [        US/[        U5      -  5      S5        / SQn/ SQn[	        [        Xg5      S5        S/S-  S/S-  -   n[        R                  " [        5         [	        [        XgUS9S5        S S S 5        g s  snf s  snf ! , (       d  f       g = f)Nr   r4   rg   r  r   r   r   r   r   r^  r   )r;   r   r;   r;   r   r;   r;   r;   r   r;   r;   r;   r;   r;   r;   r;   r   r;   r;   r;   )r;   r;   r;   r   r   r;   r;   r;   r;   r   r;   r;   r;   r   r;   r;   r;   r   r;   r;   r;   rl  rD  )r@   rD   rE   r  r+   r   r'   wherers   r   r   AssertionError)	r  rS   r   rX   
y_true_invy_true_inv2y_1y_2masks	            rY   test_matthews_corrcoefr    s\   
))

 2
3C.1kk!QRk.HI.H!Vc$.HFI )&93? 5;;Fqc#s*FJ;)&=rB #s<K((;S1K)&>C ),EsK )&3%#f+2EFL GC
FC)#3S9 38qcBhD 
~	&-cdKSQ 
'	&; J <. 
'	&s   E#!E(E--
E;c                    [         R                  R                  U 5      n[        S5      nSnUR	                  SUSS9 Vs/ s H  n[        X$-   5      PM     nn[        [        XU5      S5        / SQn/ SQn[        [        XV5      S	5        / SQn/ S
Qn[        [        XW5      S[         R                  " S5      -  5        / SQn/ SQn[        [        XX5      S5        / SQn/ SQn[        [        XX5      S5        / SQn	/ SQn
[        [        X5      S5        / SQn/ SQn/ SQn[        [        XXUS9S5        / SQn/ SQn/ SQn[        [        XXUS9S5        g s  snf )Nr   r   r   rg   r  r   )r   r   r;   r;   r4   r4   )r4   r4   r   r   r;   r;   g      )r;   r;   r   r   r   r   ii  r   )r   r   r   r   	r   r;   r4   r   r;   r4   r   r;   r4   )	r;   r;   r;   r4   r4   r4   r   r   r   )r   r   r;   r;   r4   rh  r;   r;   r;   r;   r   rD  r   r   r;   r;   r   r   )	r@   rD   rE   ordr  chrr+   r   r  )r  rS   ord_a	n_classesr   rX   
y_pred_bad
y_pred_minrW   r  r  r7  s               rY   !test_matthews_corrcoef_multiclassr    sM   
))

 2
3CHEI&)kk!YRk&HI&Hc%)n&HFI )&93?  F#J)&=tD  F#J)&=sRWWWEU?UV FF)&93? FF)&93? &C
%C)#3S9 FF#M&F FF M&F_ Js    En_pointsd   i'  c                   ^ [         R                  R                  U5      mS nU4S jn[         R                  " SS/U 5      n[	        [        XD5      S5        [         R                  " / SQU 5      n[	        [        XD5      S5        U" U 5      u  pV[	        [        XU5      S5        [	        [        XV5      U" XV5      5        g )Nc                     [        X5      nUS   nUS   nUS   n[        U 5      nX5-   U-  nX4-   U-  nX6-  Xx-  -
  n	X-  SU-
  -  SU-
  -  n
U	[        R                  " U
5      -  $ )Nr;   r;   )r;   r   r  r;   )r   rs   r@   r  )rX   rW   conf_matrixtrue_pos	false_pos	false_negr  pos_rateactivitymcc_numeratormcc_denominators              rY   mcc_safe1test_matthews_corrcoef_overflow.<locals>.mcc_safeV  s    &v6t$%	%	v;(H4(H4 +h.AA"-X>!h,Orww777r[   c                 t   > TR                  U 5      nUSTR                  U 5      S-
  -  -   nUS:  nUS:  nX44$ )Nr   r   )random_sample)r  x_truex_predrX   rW   rS   s        rY   	random_ys2test_matthews_corrcoef_overflow.<locals>.random_ysb  sL    ""8,#!2!28!<s!BCC##~r[   r   r   )r   r   rW  )r@   rD   rE   repeatr+   r   )r  r  r  r  arrrX   rW   rS   s          @rY   test_matthews_corrcoef_overflowr  Q  s     ))

 2
3C
8 ))S#J
)C)#3S9
))OX
.C)#3S9x(NF)&93?)&98F;STr[   c                     [        SS9u  pn[        XS S9u  p4pV[        U/ SQS5        [        U/ SQS5        [        U/ SQS5        [        U/ SQ5        [	        XS	S
S9n[        USS5        [        XS
S9n[        USS5        [        XS
S9n	[        U	SS5        [	        XSS9n[        USS5        [        XSS9n[        USS5        [        XSS9n	[        U	SS5        [	        XSS9n[        USS5        [        XSS9n[        USS5        [        XSS9n	[        U	SS5        [        R                  " [        5         [	        XSS9  S S S 5        [        R                  " [        5         [        XSS9  S S S 5        [        R                  " [        5         [        XSS9  S S S 5        [        R                  " [        5         [        XSSS9  S S S 5        [        X/ SQS S9u  p4pV[        U/ SQS5        [        U/ SQS5        [        U/ SQS5        [        U/ SQ5        g ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f)NFr   r   )(\?Q?gzG?r4   )HzG?g
ףp=
?re   )Q?333333?r	  )r^   rd   rg   r;   r   r   g(\?r   r   gRQ?r   gGz?r   r   r   r   r+  r   )r  g=
ףp=?r  )r  re   r   )r  r	  r  )r^   rg   rd   )rZ   r   r,   r-   r    r!   r   r   r   r   r   )
rX   rW   rx   rR   r   r   r   r   r   r   s
             rY   )test_precision_recall_f1_score_multiclassr  s  s   'u5FA 1NJA!a!3Q7a!3Q7a!3Q7q,' 
1g	FBb$*	fg	6Bb$*	&'	2Bb$*		9Bb$*	fg	6Bb$*	&'	2Bb$*		<Bb$*	fj	9Bb$*	&*	5Bb$*	z	"	: 
#	z	"VY7 
#	z	"3 
#	z	"FIC@ 
# 1y$JA! a!3Q7a!3Q7a!3Q7q,'! 
#	"	"	"	"	"	"	"s0   ?H&,H7II&
H47
I
I
I'r   )r   r   r   r   Nc                     [         R                  " / SQ/5      n[         R                  " / SQ/5      n[        X/ SQ/ U S9u  p4pV[        US5        [        US5        [        US5        U c  [        U/ SQ5        g g )Nr  r   r   r;   r;   )r   r   r;   r4   )rp   warn_forr   r   r   r;   r;   r   )r@   r   r   r-   )r   rX   rW   rR   r   r   r   s          rY   ;test_precision_refcall_f1_score_multilabel_unordered_labelsr    so     XX|n%FXX|n%F0|b'JA! q!q!q!1l+ r[   c                  "   [         R                  " / SQ5      n [         R                  " / SQ5      n[        XS S9u  p#pE[        XSS9u  pgpU[         R                  " U5      :X  d   eU[         R                  " U5      :X  d   eU[         R                  " U5      :X  d   e[        XSS9u  pgp[         R                  " U 5      n	U[         R
                  " X)S9:X  d   eU[         R
                  " X9S9:X  d   eU[         R
                  " XIS9:X  d   eg )N)r   r;   r   r   r;   r;   r   r;   r   r   r;   r   r;   r   r;   )r;   r;   r   r;   r   r;   r;   r;   r;   r   r;   r   r;   r   r;   r   r   r   r  )r@   r   r   r   bincountr   )
rX   rW   r   r   r   rx   rR   r   r   rb   s
             rY   .test_precision_recall_f1_score_binary_averagedr    s    XXCDFXXCDF 4FDQMBB0QJA!0TJA!kk&!G

2////

2////

2////r[   c                  n   [         R                  " SS9n  [         R                  " / SQ5      n[         R                  " / SQ5      n[        [	        XSS9SS5        [        [        XSS9SS5        [        [        XSS9SS5        [         R                  " S	0 U D6  g ! [         R                  " S	0 U D6  f = f)
Nraise)rK  )r   r;   r4   r   r;   r4   )r4   r   r;   r;   r4   r   r   r   r   r4   r   )r@   seterrr   r+   r    r!   r   )old_error_settingsrX   rW   s      rY   test_zero_precision_recallr    s     w/	(,-,-OFGLcSTULI3PQRHVWEsAN 			'&'		'&'s   A/B B4c                     [        SS9u  pn[        XSS/S9n[        USS/SS//5        [        XS	S/S9n[        US
S	/SS//5        [        R                  " U 5      S-   n[        XS	U/S9n[        US
S/SS//5        g )NFr   r   r;   r   r#  r   r   r4   r'  r^   )rZ   r   r-   r@   max)rX   rW   rx   r  extra_labels        rY   .test_confusion_matrix_multiclass_subset_labelsr    s    'u5FA 
&!Q	8BrRGaV,- 
&!Q	8BrRGb!W-. &&.1$K	&![1A	BBrRGaV,-r[   zlabels, err_msgz,'labels' should contains at least one label.r   r   z.At least one label specified must be in y_truez
empty listzunknown labels)idsc                     [        SS9u  p#n[        R                  " [        US9   [	        X#U S9  S S S 5        g ! , (       d  f       g = f)NFr   r   r   )rZ   r   r   r   r   )rp   r   rX   rW   rx   s        rY   test_confusion_matrix_errorr    s7     (u5FA	z	17 
2	1	1s	   :
Arp   )NonerM   
multiclassc                     U (       a  [        U 5      OSn[        R                  " X4[        S9n[	        / / U S9n[        X25        g )Nr   rP  r   )rs   r@   r   rC   r   r-   )rp   expected_n_classesr  r  s       rY   *test_confusion_matrix_on_zero_length_inputr    s<     )/VAxx+@LH	"b	0Br$r[   c            
      z   / SQn [         R                  " [        U 5      5      n[        X 5      nUR                  [         R
                  :X  d   e[         R                  [         R                  [         R                  4 H;  n[        X UR                  USS9S9nUR                  [         R
                  :X  a  M;   e   [         R                  [         R                  S [        4 H;  n[        X UR                  USS9S9nUR                  [         R                  :X  a  M;   e   [         R                  " [        U 5      S[         R                  S9n[        X US9nUS   S:X  d   eUS   S	:X  d   e[         R                  " [        U 5      S
[         R
                  S9n[        X US9nUS   S
:X  d   eUS   S:X  d   eg )Nr   F)copyrD  l    r  r   r   r  l    l    )r@   onesrs   r   rP  int64bool_int32uint64astypefloat32float64objectfulluint32)rO   weightr  rP  s       rY   test_confusion_matrix_dtyper  	  sh   AWWSV_F	!	B88rxx((BHHbii0a&--E-2RSxx288### 1 **bjj$7a&--E-2RSxx2::%%% 8
 WWSVZryy9F	!f	5Bd8z!!!d8z!!! WWSV0AF	!f	5Bd8****d8r>>r[   rP  )Int64Float64booleanc                     [         R                  " S5      n[        R                  " / SQ5      nUR	                  X S9nUR	                  / SQSS9n[        X45      n[        X$5      n[        XV5        g)zcChecks that confusion_matrix works with pandas nullable dtypes.

Non-regression test for gh-25635.
pandas)	r;   r   r   r;   r   r;   r;   r   r;   r  )	r   r   r;   r;   r   r;   r;   r;   r;   r	  N)r   importorskipr@   r   Seriesr   r-   )rP  pd	y_ndarrayrX   y_predictedoutputexpected_outputs          rY   %test_confusion_matrix_pandas_nullabler!  $  sb     
		X	&B45IYYyY.F))7w)GKf2F&y>Ov/r[   c            	          [         R                  " 5       n [        U SS9u  pnSn[        UU[        R
                  " [        U R                  5      5      U R                  S9nXT:X  d   eg )NFr]   a|                precision    recall  f1-score   support

      setosa       0.83      0.79      0.81        24
  versicolor       0.33      0.10      0.15        31
   virginica       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
rp   rq   r
   r<   rZ   r   r@   rA   rs   rq   rw   rX   rW   rx   ry   rz   s         rY   %test_classification_report_multiclassr&  6  sh    D'UCFA
O #yyT../0&&	F $$$r[   c                  :    / SQ/ SQpSn[        X5      nX2:X  d   eg )N)	r   r   r   r;   r;   r;   r4   r4   r4   r  a|                precision    recall  f1-score   support

           0       0.33      0.33      0.33         3
           1       0.33      0.33      0.33         3
           2       0.33      0.33      0.33         3

    accuracy                           0.33         9
   macro avg       0.33      0.33      0.33         9
weighted avg       0.33      0.33      0.33         9
r   )rX   rW   ry   rz   s       rY   .test_classification_report_multiclass_balancedr(  P  s)    02MF
O #62F$$$r[   c                  p    [         R                  " 5       n [        U SS9u  pnSn[        X5      nXT:X  d   eg )NFr]   a|                precision    recall  f1-score   support

           0       0.83      0.79      0.81        24
           1       0.33      0.10      0.15        31
           2       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
)r
   r<   rZ   r   r%  s         rY   :test_classification_report_multiclass_with_label_detectionr*  b  s@    D'UCFA
O #62F$$$r[   c            	          [         R                  " 5       n [        U SS9u  pnSn[        UU[        R
                  " [        U R                  5      5      U R                  SS9nXT:X  d   eg )NFr]   a|                precision    recall  f1-score   support

      setosa    0.82609   0.79167   0.80851        24
  versicolor    0.33333   0.09677   0.15000        31
   virginica    0.41860   0.90000   0.57143        20

    accuracy                        0.53333        75
   macro avg    0.52601   0.59615   0.50998        75
weighted avg    0.51375   0.53333   0.47310        75
r   )rp   rq   digitsr$  r%  s         rY   1test_classification_report_multiclass_with_digitsr-  v  sk    D'UCFA
O #yyT../0&&F $$$r[   c                      [        SS9u  pn[        R                  " / SQ5      U    n [        R                  " / SQ5      U   nSn[        X5      nXC:X  d   eSn[        X/ SQS9nXC:X  d   eg )NFr   )bluegreenreda|                precision    recall  f1-score   support

        blue       0.83      0.79      0.81        24
       green       0.33      0.10      0.15        31
         red       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
a|                precision    recall  f1-score   support

           a       0.83      0.79      0.81        24
           b       0.33      0.10      0.15        31
           c       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r   rq   rZ   r@   r   r   )rX   rW   rx   ry   rz   s        rY   7test_classification_report_multiclass_with_string_labelr4    sy    'u5FAXX./7FXX./7F
O #62F$$$
O #6PF$$$r[   c                      [        SS9u  pn[        R                  " / SQ5      nX0   n X1   nSn[        X5      nXT:X  d   eg )NFr   )u   blue¢u   green¢u   red¢u                precision    recall  f1-score   support

       blue¢       0.83      0.79      0.81        24
      green¢       0.33      0.10      0.15        31
        red¢       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r3  rX   rW   rx   rp   ry   rz   s         rY   8test_classification_report_multiclass_with_unicode_labelr7    sM    'u5FAXX:;F^F^F
O #62F$$$r[   c                      [        SS9u  pn[        R                  " / SQ5      nX0   n X1   nSn[        X5      nXT:X  d   eg )NFr   )r/  greengreengreengreengreenr1  a                             precision    recall  f1-score   support

                     blue       0.83      0.79      0.81        24
greengreengreengreengreen       0.33      0.10      0.15        31
                      red       0.42      0.90      0.57        20

                 accuracy                           0.53        75
                macro avg       0.53      0.60      0.51        75
             weighted avg       0.51      0.53      0.47        75
r3  r6  s         rY   <test_classification_report_multiclass_with_long_string_labelr:    sM    'u5FAXX23F^F^F
O #62F$$$r[   c                      / SQn / SQn/ SQnSn[         R                  " [        US9   [        XSS/US9  S S S 5        g ! , (       d  f       g = f)	Nr   r   r4   r   r   r   r4   r4   r   r   zclass 0zclass 1zclass 2z6labels size, 2, does not match size of target_names, 3r   r   r4   r#  )r   r   r   r   )rX   rW   rq   r   s       rY   =test_classification_report_labels_target_names_unequal_lengthr?    s@    FF4L
BC	k	-faV,W 
.	-	-s	   ?
Ac                      / SQn / SQn/ SQnSn[         R                  " [        US9   [        XUS9  S S S 5        g ! , (       d  f       g = f)Nr<  r=  r>  zaNumber of classes, 2, does not match size of target_names, 3. Try specifying the labels parameterr   r2  )r   r   r   r   )rX   rW   rq   r   s       rY   @test_classification_report_no_labels_target_names_unequal_lengthrA    sA    FF4L	. 
 
z	1f<H 
2	1	1s	   <
A
c                  h    Sn Sn[        SXSS9u  p#[        SXSS9u  p$Sn[        X45      nXe:X  d   eg )Nr   r  r;   r   )rQ   rP   r  r:   a                precision    recall  f1-score   support

           0       0.50      0.67      0.57        24
           1       0.51      0.74      0.61        27
           2       0.29      0.08      0.12        26
           3       0.52      0.56      0.54        27

   micro avg       0.50      0.51      0.50       104
   macro avg       0.45      0.51      0.46       104
weighted avg       0.45      0.51      0.46       104
 samples avg       0.46      0.42      0.40       104
)r   r   )r  rP   rx   rX   rW   ry   rz   s          rY   %test_multilabel_classification_reportrC    sS    II.	QIA /	QIAO #62F$$$r[   c                  2   [         R                  " / SQ/ SQ/5      n [         R                  " / SQ/ SQ/5      n[        X5      S:X  d   e[        X 5      S:X  d   e[        X5      S:X  d   e[        U[         R                  " U5      5      S:X  d   e[        U [         R                  " U 5      5      S:X  d   e[        U [         R                  " U R
                  5      5      S:X  d   e[        U[         R                  " U R
                  5      5      S:X  d   eg )Nr   r   r   r   r   r;   )r@   r   r"   r   r   r?   r   s     rY   $test_multilabel_zero_one_loss_subsetrE    s    	9i(	)B	9i(	)B C''' A%%% A%%%R^^B/0A555R^^B/0A555RXXbhh/0A555RXXbhh/0A555r[   c                      [         R                  " / SQ/ SQ/5      n [         R                  " / SQ/ SQ/5      n[         R                  " SS/5      n[        X5      S:X  d   e[        X 5      S:X  d   e[        X5      S:X  d   e[        USU-
  5      S:X  d   e[        U SU -
  5      S:X  d   e[        U [         R                  " U R                  5      5      S:X  d   e[        U[         R                  " U R                  5      5      S	:X  d   e[        XUS
9S:X  d   e[        U SU-
  US
9S:X  d   e[        U [         R
                  " U 5      US
9S:X  d   e[        U S   US   5      [        U S   US   5      :X  d   eg )Nr   r   r   r;   r   UUUUUU?r   r   r   rD  gUUUUUU?gUUUUUU?)r@   r   r   r   r?   
zeros_like
sp_hamming)r   r   ws      rY   test_multilabel_hamming_lossrK  &  se   	9i(	)B	9i(	)B
!QA5(((1$$$1$$$AF#q(((AF#q(((BHHRXX./5888BHHRXX./3666a0H<<<AF!4	AAABMM"-Q?7JJJ1r!u%BqE2a5)AAAAr[   c                     [         R                  " / SQ5      n [         R                  " / SQ5      nSn[        R                  " [        US9   [        XSSS9  S S S 5        [         R                  " / SQ/ SQ/5      n [         R                  " / S	Q/ S
Q/5      nSn[        R                  " [        US9   [        XSSS9  S S S 5        [         R                  " / SQ5      n [         R                  " / SQ5      nSn[        R                  " [        US9   [        XSS9  S S S 5        Sn[        R                  " [        US9   [        XSS9  S S S 5        Sn[        R                  " [        US9   [        XSSS9  S S S 5        g ! , (       d  f       GN2= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       Nr= f! , (       d  f       g = f)N)r   r;   r   r;   r;   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   rM   r4   r   r   r   r   r   r   Target is multilabel-indicator but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted', 'samples'\].r   )r   r;   r;   r   r4   r  Target is multiclass but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted'\].r   zJSamplewise metrics are not available outside of multilabel classification.r   zNote that pos_label \(set to 3\) is ignored when average != 'binary' \(got 'micro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   )r@   r   r   r   r   r   r   r   )rX   rW   r   msg1msg2msg3r   s          rY   test_jaccard_score_validationrS  :  sZ   XXo&FXXo&FOG	z	1fh!D 
2 XXy),-FXXy),-F	6 	
 
z	.fh"E 
/ XXo&FXXo&F	 	
 
z	.fh7 
/WD	z	.fi8 
/	  
k	-fgC 
.	-A 
2	1 
/	. 
/	. 
/	. 
.	-s<   F3FF& F7.G
F
F#&
F47
G
Gc           	         [         R                  " / SQ/ SQ/5      n[         R                  " / SQ/ SQ/5      n[        XSS9S:X  d   e[        XSS9S:X  d   e[        X"SS9S:X  d   e[        U[         R                  " U5      SS9S:X  d   e[        U[         R                  " U5      SS9S:X  d   e[        U[         R                  " UR
                  5      SS9S:X  d   e[        U[         R                  " UR
                  5      SS9S:X  d   e[         R                  " / SQ/ S	Q/5      n[         R                  " / S
Q/ SQ/5      n[        [        X4SS9S5        [        [        X4SS9S5        [        [        X4SS9S5        [        [        X4SSS/S9S5        [        [        X4SSS/S9S5        [        [        X4S S9[         R                  " / SQ5      5        [         R                  " / SQ/ SQ/5      n[         R                  " / S
Q/ SQ/5      n[        [        X4SS9S5        [        [        X4SS9S5        Sn[        R                  " [        US9   [        X4S/SS9  S S S 5        Sn[        R                  " [        US9   [        X4S/SS9  S S S 5        Sn[        R                  " [        US9   [        [         R                  " SS//5      [         R                  " SS//5      SS9S:X  d   e S S S 5        Sn[        R                  " [        US9   [        [         R                  " SS/SS//5      [         R                  " SS/SS//5      SS9S:X  d   e S S S 5        [        U 5      (       a   eg ! , (       d  f       GN = f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       NV= f)Nr   r   r   r   r   r   r;   r   r   r   r   r   r   r   g?r4   r   r   )r   r   r   r   r   g      ?z	Got 4 > 2r   r   r   z
Got -1 < 0r   zXJaccard is ill-defined and being set to 0.0 in labels with no true or predicted samples.zXJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels.)r@   r   r   r   r   r?   r+   r-   r   r   r   r   r   rO  )recwarnr   r   rX   rW   rQ  rR  r   s           rY   test_multilabel_jaccard_scorerV  b  s;   	9i(	)B	9i(	)B
 3t;;;3q8883q888R^^B/CqHHHR^^B/CqHHHRXXbhh/CqHHHRXXbhh/CqHHHXXy),-FXXy),-FfgFPfgFPfiH(SfiAG fiAG fd3RXX>U5V XXy),-FXXy),-FfgFPfjI7SD	z	.faS'B 
/D	z	.fbT7C 
/	- 
 
,C	8"((QF8,bhhAx.@'R	
 
9	, 
 
,C	81a&1a&)*1a&1a&)*!
 	
 
9 G}}}A 
/	. 
/	. 
9	8 
9	8s2   M>N8A N"AN3>
N
N"
N03
Oc           
         / SQn/ SQn/ SQn[        5       nUR                  U5        UR                  U5      nUR                  U5      n[        [        X5      n[        [        XV5      nSS/SS/SS/S/S/S/S /n	SS/SS	/S	S/S/S/S	/S /n
S
 H+  n[        X5       H  u  p[        U" XS9U" XS95        M     M-     [        R                  " SS/SS/SS//5      n[        R                  " SS/SS/SS//5      n[        5          [	        XSS9S:X  d   e S S S 5        [        U 5      (       a   eg ! , (       d  f       N!= f)N)antrX  catrY  rX  rY  birdrZ  )rY  rX  rY  rY  rX  rZ  rZ  rY  )rX  rZ  rY  rX  rZ  rY  r   r;   r4   )r   r   r   Nr   r   r   )r&   rI   	transformr   r   zipr+   r@   r   r.   rO  )rU  rX   rW   rp   lbr   r   multi_jaccard_scorebin_jaccard_scoremulti_labels_listbin_labels_listr   m_labelb_labels                 rY   test_multiclass_jaccard_scorerd    si   GFGF#F		BFF6Nf%Jf%J!-@zF						 1v1v1vsQC!dCO 8 #$5 GG#GD!'B !H 8 XX1v1v1v./FXX1v1v1v./F		VZ@AEEE 
 G}}} 
	s   D44
Ec           	         [        S/S/SS9S:X  d   eSn[        R                  " [        US9   [        SS/SS/SS9S:X  d   e S S S 5        [        S/S/SSS9S	:X  d   e[        R
                  " / S
Q5      n[        R
                  " / SQ5      n[        [        X#SS9S5        [        [        X#SSS9S5        [        U 5      (       a   eg ! , (       d  f       N= f)Nr;   r   rM   r   r   zOJaccard is ill-defined and being set to 0.0 due to no true or predicted samplesr   r   r   )r;   r   r;   r;   r   )r;   r   r;   r;   r;   r   rM  r   )r   r   r   r   r@   r   r+   rO  )rU  r   rX   rW   s       rY   !test_average_binary_jaccard_scorerf    s    !qc84;;;	'  
,C	8aVaVX>#EEE 
9 !qcQASHHHXXo&FXXo&FfhGQfh!Dg G}}} 
9	8s   C
Cc                  .   [         R                  " / SQ/ SQ/5      n [         R                  " / SQ/ SQ/5      nSn[        R                  " [        US9   [        XSSS9nU[        R                  " S5      :X  d   e S S S 5        g ! , (       d  f       g = f)	Nr   r   r   r   zJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels. Use `zero_division` parameter to control this behavior.r   r   r   r   r   r   )r@   r   r   r   r   r   r   )rX   rW   r   scores       rY   (test_jaccard_score_zero_division_warningrk    sw     XXy),-FXXy),-F	C 
 
,C	8fivVc**** 
9	8	8s   (B
Bzzero_division, expected_scorer  )r;   r   c                 V   [         R                  " / SQ/ SQ/5      n[         R                  " / SQ/ SQ/5      n[        R                  " 5          [        R                  " S[
        5        [        X#SU S9nS S S 5        W[        R                  " U5      :X  d   eg ! , (       d  f       N+= f)Nr   rh  r   r   ri  )	r@   r   r   r   r   r   r   r   r   )r   expected_scorerX   rW   rj  s        rY   *test_jaccard_score_zero_division_set_valuern    s     XXy),-FXXy),-F		 	 	"g'=>I]
 
#
 FMM.1111 
#	"s   'B
B(c            	         [         R                  " / SQ/ SQ/ SQ/5      n [         R                  " / SQ/ SQ/ SQ/5      n[        XS S9u  p#pE[        U/ SQS5        [        U/ SQS5        [        U/ S	QS5        [        U/ S
QS5        [	        XSS S9nUn[        U/ SQS5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9[         R                  " U5      5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9SU-  U-  SU-  U-   -  5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9[         R                  " XgS95        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9S5        g )Nr;   r   r   r   r   r;   r   r   r  )r;   r   r;   r   r   )r   r   r   r   r4   )r   r   r   r   )r   r   r;   r   )r;   r;   r;   r;   r   r   )r   r  r;   r   r   g      ?r   g?r   r   r   r   r  r   r@   r   r   r,   r   r+   r   r   rX   rW   rR   r   r   r   f2rb   s           rY   +test_precision_recall_f1_score_multilabel_1rv    s   
 XX|\<@AFXX|\<@AF0NJA! a!5q9a!5q9a!7;aq1	V!T	:BGb/15 1QJA!7#3+,99FG<bggbk
 1QJA!33399FG<	!a1q519% 1TJA!7#3+,99FJ?


2' 1SJA!33399FINPSTr[   c            	         [         R                  " / SQ/ SQ/ SQ/5      n [         R                  " / SQ/ SQ/ SQ/5      n[        XS S9u  p#pE[        U/ SQS5        [        U/ S	QS5        [        U/ S
QS5        [        U/ SQS5        [	        XSS S9nUn[        U/ SQS5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9SU-  U-  SU-  U-   -  5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9[         R                  " U5      5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9[         R                  " XgS95        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9SS5        g )Nrp  rq  r  r   r   r   r;   r  r   )r   r   r   r   r4   )r   r   r   r   )r   gQ?r   r   r;   r4   r;   r   rr  )r   皙?r   r   r         ?r   r   r   g      ?rG  r   r   rc   r  r   g&S?rs  rt  s           rY   +test_precision_recall_f1_score_multilabel_2r|  J  s    XX|\<@AFXX|\<@AF 1NJA!a!5q9a!5q9a!6:aq1	V!T	:BGb/150QJA!4 4 0199FG<	!a1q519%
 1QJA!4 5!6"99FG<bggbk 1TJA!5!5!=)99FJ?


2'
 1SJA! 5!5!=)99FI>r[   z%zero_division, zero_division_expected)r   r   r  c           
         [         R                  " / SQ/ SQ/ SQ/5      n[         R                  " / SQ/ SQ/ SQ/5      n[        X#S U S9u  pEpg[        XASSS/S	5        [        USS
SU/S	5        Sn[        XhSSU/S	5        [        U/ SQS	5        [	        X#S	S U S9n	Un
[        XSSU/S	5        [        X#SU S9u  pEpg[         R
                  " U5      (       a  SOUnS[         R
                  " U5      (       + -   n[        US	U-   U-  5        [        USU-   U-  5        Sn[        Xh5        Ub   e[        [	        UUS	SU S9[        U	S S95        [        X#SU S9u  pEpg[        US5        [        US
5        [        US5        Ub   e[        [	        X#S	SU S9SU-  U-  SU-  U-   -  5        [        X#SU S9u  pEpg[        XAS:X  a  SOS5        [        US
5        Sn[        USU-  5        Ub   e[        [	        X#S	SU S9[        XS95        [        X#SS9u  pEpg[        US5        [        US5        [        US5        Ub   eS n[        [	        X#S	SU S9US	5        g )!Nrq  rp  r  r^  rx  ri  r   r   r4   r   r   r   r;   ry  r   r   r   rz  r   r         ?g?r  r   rf   r   r   r   r   g@r   r   rc   gZd;O?)r@   r   r   r,   r   r  r+   r/   )r   zero_division_expectedrX   rW   rR   r   r   r   
expected_fru  rb   value_to_sumvalues_to_averageexpected_results                 rY   7test_precision_recall_f1_score_with_an_empty_predictionr    s    XX|\<@AFXX|\<@AF 1MJA! a#sC!H!La#sC1G!H!LJagq*!EqIaq1	V!T	WBGbtQ
"CQG0}JA! !7881>TL*@!AABA,0AABC,.2CCD J&99'	
 	B%	 1}JA! 5!30199G=	
 
!a1q519%	 1
-JA! a$?5SI3M->>?99Jm	
 	B(	 1SJA! 5!5!5!99OI]	
 		r[   r   )r   r   r   r   c           	      >   [         R                  " S5      n[         R                  " U5      n[        R                  " 5          [        R
                  " S5        [        UUUU US9u  pVpx[        UUU UUS9n	S S S 5        Wb   e[         R                  " U5      (       a+  WWWW	4 H   n
[         R                  " U
5      (       a  M    e   g [        U5      n[        WU5        [        WU5        [        WU5        [        W	[        U5      5        g ! , (       d  f       N= f)Nrg   r   r   r   r   r   r~  )r@   r   rH  r   r   r   r   r   r  rv   r+   )r   r   r   rX   rW   rR   r   r   r   fbetar|   s              rY   "test_precision_recall_f1_no_labelsr    s     XXgF]]6"F		 	 	"g&4'

a '
 
#" 99 
xx!Q&F88F#### '-(M=)=)=)u]34= 
#	"s   4D
Dc                    [         R                  " S5      n[         R                  " U5      n[        n[        R
                  " [        5         U" XU SS9u  pEpgS S S 5        [        WS5        [        WS5        [        WS5        Wb   e[        R
                  " [        5         [        XU SS9nS S S 5        [        WS5        g ! , (       d  f       Nr= f! , (       d  f       N,= f)Nr  r   r  r   )	r@   r   rH  r   r   r   r   r+   r   )	r   rX   rW   funcrR   r   r   r   r  s	            rY   1test_precision_recall_f1_no_labels_check_warningsr    s    XXgF]]6"F*D	,	-&'D
a 
. 11199	,	-FG#F 
. q! 
.	- 
.	-s   C$C
C
C$c           	         [         R                  " S5      n[         R                  " U5      n[        R                  " 5          [        R
                  " S5        [        UUS SU S9u  p4pV[        XSS U S9nS S S 5        [         R                  " U 5      n [        WX U /S5        [        WX U /S5        [        WX U /S5        [        W/ SQS5        [        WX U /S5        g ! , (       d  f       Np= f)Nr  r   r   r  r~  r4   rh  )
r@   r   rH  r   r   r   r   r   r  r,   )r   rX   rW   rR   r   r   r   r  s           rY   /test_precision_recall_f1_no_labels_average_noner  *  s    XXgF]]6"F 
	 	 	"g&4'

a d-
 
# JJ}-Ma-!NPQRa-!NPQRa-!NPQRaA.emM%RTUV) 
#	"s   3C
C-c                     [         R                  " S5      n [         R                  " U 5      n[        R                  " [
        5         [        XS SS9u  p#pES S S 5        [        W/ SQS5        [        W/ SQS5        [        W/ SQS5        [        W/ SQS5        [        R                  " [
        5         [        XSS S9nS S S 5        [        W/ SQS5        g ! , (       d  f       N= f! , (       d  f       N/= f)Nr  r;   r  rh  r4   rr  )	r@   r   rH  r   r   r   r   r,   r   )rX   rW   rR   r   r   r   r  s          rY   4test_precision_recall_f1_no_labels_average_none_warnr  N  s    XXgF]]6"F 
,	-4Dq

a 
.
 aA.aA.aA.aA.	,	-FDA 
. eY2 
.	- 
.	-s   C4C)
C&)
C7c            	      (   [         [        pS HY  nSn[        R                  " XS9   U " / SQ/ SQUS9  S S S 5        Sn[        R                  " XS9   U " / SQ/ SQUS9  S S S 5        M[     Sn[        R                  " XS9   U " [        R
                  " S	S
/S	S
//5      [        R
                  " S	S
/S
S
//5      SS9  S S S 5        Sn[        R                  " XS9   U " [        R
                  " S	S
/S
S
//5      [        R
                  " S	S
/S	S
//5      SS9  S S S 5        Sn[        R                  " XS9   U " [        R
                  " S	S	/S	S	//5      [        R
                  " S
S
/S
S
//5      SS9  S S S 5        Sn[        R                  " XS9   U " [        R
                  " S
S
/S
S
//5      [        R
                  " S	S	/S	S	//5      SS9  S S S 5        Sn[        R                  " XS9   U " S	S	/SS/SS9  S S S 5        Sn[        R                  " XS9   U " SS/S	S	/SS9  S S S 5        [        R                  " SS9 n[        R                  " S5        [        S
S
/S
S
/SS9  Sn[        UR                  5       R                  5      U:X  d   eSn[        UR                  5       R                  5      U:X  d   eSn[        UR                  5       R                  5      U:X  d   e S S S 5        g ! , (       d  f       GN= f! , (       d  f       GM  = f! , (       d  f       GN`= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNl= f! , (       d  f       GNS= f! , (       d  f       g = f)NNr   r   zPrecision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.r   r   r;   r;   r4   r   zRecall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.zPrecision is ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.r;   r   r   zRecall is ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.r   zRecall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.r   rM   Tr   alwaysF-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.)r   r   r   r   r@   r   r   r   r   r   popr   )r   rJ  r   r   r   s        rY   test_prf_warningsr  j  s.   *,Bq. 	 \\!'iG4 ( 	 \\!'iG4 ('! /*	  
a	#	"((QFQF#
$bhhAA/?&@)T 
$	  
a	#	"((QFQF#
$bhhAA/?&@)T 
$
	  
a	#	"((QFQF#
$bhhAA/?&@'R 
$	  
a	#	"((QFQF#
$bhhAA/?&@'R 
$
	  
a	#	1a&2r(H- 
$	  
a	#	2r(QFH- 
$ 
	 	 	-h''AAI 	
 6::<''(C/// 	 6::<''(C/// 	 6::<''(C///- 
.	-K (' (' 
$	# 
$	# 
$	# 
$	# 
$	# 
$	# 
.	-sl   K2L=L=L)7=L;=M-MM1B'N2
L	
L	
L&)
L8;
M

M
M.1
N 
Nc           	         [         R                  " 5          [         R                  " S5        S H!  n[        / SQ/ SQXS9  [        / SQ/ SQXS9  M#     [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      S	U S9  [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      S	U S9  [        SS/S
S
/SU S9  [        S
S
/SS/SU S9  S S S 5        [         R                  " SS9 n[         R                  " S5        [        SS/SS/SU S9  [        U5      S:X  d   e S S S 5        g ! , (       d  f       Nd= f! , (       d  f       g = f)Nr   r  r   r  ri  r;   r   r   r   r   rM   Tr   r  )r   r   r   r   r@   r   rs   )r   r   r   s      rY   )test_prf_no_warnings_if_zero_division_setr    s   		 	 	"g& 3G+9g ,9g 3 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(FRHhm	
 	(Hq!fhm	
a 
#h 
	 	 	-h''FQFHM	
 6{a 
.	-i 
#	"h 
.	-s   EG8G#
G #
G1c           	         [         R                  " 5          [         R                  " S5        [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  S S S 5        [         R                  " SS9 n[         R                  " S5        [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  U S	:X  a*  [        UR                  5       R                  5      S
:X  d   eO[        U5      S:X  d   e[        SS/SS/5        U S	:X  a)  [        UR                  5       R                  5      S
:X  d   eS S S 5        g ! , (       d  f       GN= f! , (       d  f       g = f)Nr   r;   r   r   ri  Tr   r  r   r  )
r   r   r   r!   r@   r   r   r  r   rs   r   r   s     rY   test_recall_warningsr  	  sc   		 	 	"g&HHq!fq!f%&HHq!fq!f%&'		
 
# 
	 	 	-h'HHq!fq!f%&HHq!fq!f%&'		
 F"FJJL(() ." "" v;!###aVaV$F"FJJL(() ." ""+ 
.	- 
#	" 
.	-s   AE)	CE;)
E8;
F	c           	         [         R                  " SS9 n[         R                  " S5        [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  U S:X  a*  [        UR                  5       R                  5      S	:X  d   eO[        U5      S:X  d   e[        SS/SS/5        U S:X  a)  [        UR                  5       R                  5      S	:X  d   eS S S 5        [         R                  " 5          [         R                  " S
5        [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)NTr   r  r;   r   r   ri  r   r  r   )
r   r   r   r    r@   r   r   r  r   rs   r  s     rY   test_precision_warningsr  8	  sc   		 	 	-h'HHq!fq!f%&HHq!fq!f%&'		
 F"FJJL(() ." "" v;!###AA'F"FJJL(() ." ""+ 
.6 
	 	 	"g&HHq!fq!f%&HHq!fq!f%&'		
 
#	"7 
.	-6 
#	"s   CE)	AE:)
E7:
Fc           
         [         R                  " SS9 n[         R                  " S5        [        [	        [
        SS94 GH   nU" [        R                  " SS/SS//5      [        R                  " SS/SS//5      SU S	9  [        U5      S:X  d   eU" [        R                  " SS/SS//5      [        R                  " SS/SS//5      SU S	9  [        U5      S:X  d   eU" [        R                  " SS/SS//5      [        R                  " SS/SS//5      SU S	9  U S
:X  a,  [        UR                  5       R                  5      S:X  d   eGM  [        U5      S:X  a  GM!   e   S S S 5        g ! , (       d  f       g = f)NTr   r  r4   r   r;   r   r   ri  r   r  )r   r   r   r   r   r   r@   r   rs   r   r  r   )r   r   rj  s      rY   test_fscore_warningsr  `	  sp   		 	 	-h'! <=E1a&1a&)*1a&1a&)*+	 v;!###1a&1a&)*1a&1a&)*+	 v;!###1a&1a&)*1a&1a&)*+	 &

,,- 2- -- 6{a'''? > 
.	-	-s   EE.!E..
E<c            	      ~   / SQn / SQnSn[         R                  " / SQ/ SQ/ SQ/5      n[         R                  " / SQ/ SQ/ SQ/5      nSnXU4X4U44 HU  u  pgn[        [        [        [        [        S	S
94 H-  n	[        R                  " [        US9   U	" Xg5        S S S 5        M/     MW     g ! , (       d  f       ME  = f)N)r;   r4   r   r   )r;   r4   r   r;   rO  r   r   r   r3  rN  r4   r   r   )
r@   r   r    r!   r   r   r   r   r   r   )
	y_true_mc	y_pred_mcmsg_mc
y_true_ind
y_pred_indmsg_indrX   rW   r   r|   s
             rY   'test_prf_average_binary_data_non_binaryr  	  s    II	1 
 9i;<J9i;<J	<  
v&	) 
 Ka(	
F z5v& 65
	  65s   	B--
B<c                  ,   Sn SnSnSnSnSnU [         R                  " / SQ/ SQ/ S	Q/5      4U [         R                  " S
S/SS
/SS//5      4U/ SQ4U/ SQ4U/ SQ4U[         R                  " S/S/S//5      4U[         R                  " S
/S/S//5      4U[         R                  " S/S/S//5      4U[         R                  " S
S/SS/SS//5      4U[         R                  " SS/SS/SS//5      4/
n0 X 4U _X4U_X"4U_X4S _X 4S _X!4U_X34S _XD4S _XU4S _X4S _X4S _X#4S _XC4S _XS4S _X4S _X4S _X$4S _XT4S X4S X4S X%4S 0En[        USS9 GH  u  u  pu  p XxU
4   nUc  [        R
                  " [        5         [        X5        S S S 5        X:w  a@  SR                  X5      n[        R
                  " [        US9   [        X5        S S S 5        M  XX4;  a@  SR                  U5      n[        R
                  " [        US9   [        X5        S S S 5        M  M  [        X5      u  pnX:X  d   eUR                  S5      (       a%  UR                  S:X  d   eUR                  S:X  d   eO@[        U[         R                  " U	5      5        [        U[         R                  " U5      5        [        R
                  " [        5         [        U	S S U5        S S S 5        GM     SS/n	S S!/nS"n[        R
                  " [        US9   [        X5        S S S 5        g ! [         a
    XzU4   n GNf = f! , (       d  f       GN= f! , (       d  f       GM	  = f! , (       d  f       GM  = f! , (       d  f       GM/  = f! , (       d  f       g = f)#Nzmultilabel-indicatorr  rM   
continuouszmulticlass-multioutputzcontinuous-multioutputr   r   r   r   r;   )r4   r   r;   )r   r  r   r4   r   r   r  r   r   rW  g?g      @)r  z@Classification metrics can't handle a mix of {0} and {1} targetsr   z{0} is not supported
multilabelcsrr   )r;   r4   )r   r4   r   )r4   )r   r4   zYou appear to be using a legacy multi-label data representation. Sequence of sequences are no longer supported; use a binary array or sparse matrix instead - the MultiLabelBinarizer transformer can convert to this format.)r@   r   r   KeyErrorr   r   r   r#   format
startswithr-   squeeze)INDMCBINCNTMMCMCNEXAMPLESEXPECTEDtype1r   type2r   r  r   merged_typey1outy2outr   s                     rY   test__check_targetsr  	  s#    !C	B
C
C
"C
"C 
bhh	9i89:	bhhAAA/01	Y	i	o	RXXsQC!o&'	bhhaS1#'(	bhhuse,-.	bhhAAA/01	bhhc
S#Jc
;<=H	
C	" 

C 
	4	
 

D 
	2 

D 

D 

D 

D 
	4 

D 

D 

D  

D!" 
	4#$ 

D%& 

D	
D		4	
D-H2 %,HQ$? [e	.u-H z*r& + ~--3VE-A  ]]:W="2* >= b.4;;EBGzA&r. BA / )7r(>%K***%%l33||u,,,||u,,,"5"**R.9"5"**R.9z*r#2w+ +*A %@H )	B
B	3  
z	-r 
.	-S  	.u-H	. +* >= BA +* 
.	-sN   L#L:MMM2N#L76L7:
M		
M	
M/	2
N	
Nc                  <    SS/n SS/n[        X5      S   S:X  d   eg )Nr   r;   r   r  )r#   r`  s     rY   Atest__check_targets_multiclass_with_both_y_true_and_y_pred_binaryr  
  s,    VFWF&)!,<<<r[   c                     [         R                  " / SQ5      n [         R                  " / SQ5      n[        X5      S:X  d   e[         R                  " / SQ5      n [         R                  " / SQ5      n[        X5      S:X  d   eg )N)r   r;   r;   r   )g      !r   r  g333333ӿr   )r   r4   r4   r   )r@   r   r   rX   pred_decisions     rY   test_hinge_loss_binaryr  
  sb    XXn%FHH34Mf,777XXl#FHH34Mf,777r[   c            
         [         R                  " / SQ/ SQ/ SQ/ SQ/ SQ/ SQ/5      n [         R                  " / SQ5      n[         R                  " SU S   S   -
  U S   S   -   SU S   S   -
  U S   S	   -   SU S	   S	   -
  U S	   S
   -   SU S
   S   -
  U S
   S	   -   SU S   S
   -
  U S   S	   -   SU S   S	   -
  U S   S
   -   /5      n[         R                  " USS US9  [         R                  " U5      n[	        X5      U:X  d   eg )N
ףp=
?(\ſ(\gGz)HzGgGz׿Q޿r  333333r  RQؿr  )r  r  r  r  gzGgHzGgHzGѿgQ?)r   r;   r4   r;   r   r4   r;   r   r4   r   r   r   outr@   r   clipr   r   )r  rX   dummy_lossesdummy_hinge_losss       rY   test_hinge_loss_multiclassr  
  sH   HH((((((	
	M XX()F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	L GGL!T|4ww|,f,0@@@@r[   c                      [         R                  " / SQ5      n [         R                  " / SQ/ SQ/ SQ/ SQ/5      nSn[        R                  " [        US9   [        X5        S S S 5        g ! , (       d  f       g = f)N)r   r;   r4   r4   )gRQ?g rh?g(\gffffffr  r  zDPlease include all labels in y_true or pass labels as third argumentr   )r@   r   r   r   r   r   )rX   r  error_messages      rY   :test_hinge_loss_multiclass_missing_labels_with_labels_noner  4
  s^    XXl#FHH((((		
M 	O  
z	76) 
8	7	7s   A++
A9c            
         [         R                  " / SQ5      n [         R                  " / SQ5      nSn[        R                  " [        [
        R                  " U5      S9   [        XS9  S S S 5        [         R                  " SS/SS/SS/SS/SS/SS/SS//5      n/ S	QnS
n[        R                  " [        [
        R                  " U5      S9   [        XUS9  S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)N)r4   r;   r   r;   r   r;   r;   )r   r;   r4   r;   r   r4   r;   zThe shape of pred_decision cannot be 1d arraywith a multiclass target. pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7,)r   r  r   r;   r4   r   zThe shape of pred_decision is not consistent with the number of classes. With a multiclass target, pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7, 2))rX   r  rp   )r@   r   r   r   r   reescaper   )rX   r  r  rp   s       rY   <test_hinge_loss_multiclass_no_consistent_pred_decision_shaper  E
  s     XX+,FHH23M	  
z=)A	B&> 
C HHq!fq!fq!fq!fq!fq!fqRSfUVMF	  
z=)A	B&fM 
C	B 
C	B 
C	Bs    
C$C5$
C25
Dc            	         [         R                  " / SQ/ SQ/ SQ/ SQ/ SQ/5      n [         R                  " / SQ5      n[         R                  " / SQ5      n[         R                  " SU S   S   -
  U S   S   -   SU S   S   -
  U S   S   -   SU S   S   -
  U S   S	   -   SU S	   S   -
  U S	   S   -   SU S
   S   -
  U S
   S	   -   /5      n[         R                  " USS US9  [         R                  " U5      n[	        XUS9U:X  d   eg )Nr  皙r  r  r  r  )r   r;   r4   r;   r4   )r   r;   r4   r   r;   r   r4   r   r   r  r   r  r  rX   rp   r  r  s        rY   .test_hinge_loss_multiclass_with_missing_labelsr  a
  s3   HH(((((	
M XXo&FXXl#F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
L GGL!T|4ww|,fF;?OOOOr[   c            	         [         R                  " / SQ/ SQ/ SQ/ SQ/ SQ/5      n [         R                  " / SQ5      n[         R                  " / SQ5      n[         R                  " SU S   S   -
  U S   S   -   SU S   S	   -
  U S   S   -   SU S	   S	   -
  U S	   S   -   SU S
   S   -
  U S
   S	   -   SU S   S	   -
  U S   S   -   /5      n[         R                  " USS US9  [         R                  " U5      n[	        [        XUS9U5        g )N)r  r  r  )g333333ÿr  r  )r  r  r  )r  g(\gzGڿ)r   r4   r4   r   r4   r   r;   r   r4   r   r   r  r   )r@   r   r  r   r+   r   r  s        rY   @test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_truer  {
  s4   
 HH!!!!!	
M XXo&FXXi F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
L GGL!T|4ww|,68:Jr[   c            
         / SQn / SQ/ SQ/ SQ/ SQ/ SQ/ SQ/n[         R                  " SUS   S   -
  US   S   -   SUS   S   -
  US   S   -   SUS   S   -
  US   S	   -   SUS	   S   -
  US	   S   -   SUS
   S	   -
  US
   S   -   SUS   S   -
  US   S	   -   /5      n[         R                  " USS US9  [         R                  " U5      n[	        X5      U:X  d   eg )N)r/  r0  r1  r0  whiter1  r  r  r  r  r;   r   r4   r   r   r   r  r  )rX   r  r  r  s       rY   +test_hinge_loss_multiclass_invariance_listsr  
  s4    ?F$$$$$$M 88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	L GGL!T|4ww|,f,0@@@@r[   c            	      &   / SQn [         R                  " SS/SS/SS/SS/SS/S	S
//5      n[        X5      n[         R                  " [        R
                  " [         R                  " U 5      S:H  US S 2S4   5      5      * n[        X#5        / SQn / SQ/ SQ/ SQ/n[        XSS9n[        US5        U S-  n US-  n[        XSS9n[        US5        / SQn SS/SS/SS//n[        R                  " [        5         [        X5        S S S 5        / SQn / SQ/ SQ/ SQ/n/ SQnS n[        R                  " [        [        R                  " U5      S!9   [        XUS"9  S S S 5        / S#Qn SS/SS/SS/SS//n[        X5      n[        US$5        SS/n S%S&/SS//n[         R                  " SS/SS//5      nS'n[        R                  " [        [        R                  " U5      S!9   [        X5        S S S 5        S%S&/SS/SS//nS(n[        R                  " [        [        R                  " U5      S!9   [        X5        S S S 5        [         R                  " [         R                  " US S 2S4   5      5      * n[        XSS/S"9n[        X5        / S)Qn / S*Q/ SQ/ S+Q/n	[        X	/ S,QS"9n[        U[         R                  " S5      * 5        g ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       N= f)-Nnor  r  yesr  r  r   r   re   {Gz?Gz?r   r{  gMbP?g+?r  r;   rF  r   r   r   r   r   r   r   r   r   TrM  g躕ʀ?r4   Fg.L`@r   r   r   r   r   )re   r   r   )r   re   r   r   r   r   )r   r   r   zPy_true contains values {'b'} not belonging to the passed labels ['a', 'c', 'd'].r   r   hamspamr  r  CT?r   r   zy_true contains only one label (2). Please provide the list of all expected class labels explicitly through the labels argument.zBFound input variables with inconsistent numbers of samples: [3, 2]r   )r   r   r   r   r   r   rE  )r@   r   r   r   r	   logpmfr*   r   r   r   r  r  log)
rX   rW   loss	loss_truerp   	error_strr   true_log_losscalculated_log_lossy_score2s
             rY   test_log_lossr  
  s   4FXX
sc3Z$SzD$<%QVXF F#D))"((6*:e*CVAqD\RSSID$ F@FFd3DD)$ aKF
aKFFe4DD-( FCj3*sCj1F	z	"  
# F@FF	"  
z9)=	>/ 
? ,FCj3*sCj3*=FF#DD)$ VFCj3*%Fhhc
S#J/0G	H  
z9)=	>  
? Cj3*sCj1FTI	z9)=	>  
?
 WWRVVGAqDM233M"6Aq6B'7 F/BHFY7DD266#;,'_ 
#	" 
?	>$ 
?	>
 
?	>s0   ;KK$K02L
K
K-0
K?
Lc                     [         R                  " SS/U S9n[         R                  " SS/U S9n[        X5      n[         R                  " U5      (       d   eg)zCheck the behaviour internal eps that changes depending on the input dtype.

Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/24315
r   r;   r  N)r@   r   r   isfinite)rP  rX   rW   r  s       rY   test_log_loss_epsr    sK     XXq!fE*FXXq!fE*FF#D;;tr[   c                     [         R                  " / SQ5      n[         R                  " SS/SS/SS/SS//U S9n[        R                  " [        S	S
9   [        X5        SSS5        g! , (       d  f       g= f)zGCheck that log_loss raises a warning when y_pred values don't sum to 1.r  r   r   r   r   r   r   r  z$The y_prob values do not sum to one.r   N)r@   r   r   r   r   r   )rP  rX   rW   s      rY   'test_log_loss_not_probabilities_warningr    s_     XXl#FXXSzC:SzC:FeTF	k)O	P  
Q	P	Ps   A((
A6r3  r   r   c                 N    [        X5      [        R                  " S5      :X  d   eg)z6Check that log_loss returns 0 for perfect predictions.r   N)r   r   r   r`  s     rY   !test_log_loss_perfect_predictionsr	    s      F#v}}Q'7777r[   c                  J   [         R                  " / SQ5      n [         R                  " SS/SS/SS/SS//5      n[        [        4/n SSKJnJn  UR                  XC45        U H+  u  pVU" U 5      U" U5      p[        Xx5      n	[        U	S5        M-     g ! [         a     N>f = f)	Nr  r   r   r   r   r   )	DataFramer  r  )
r@   r   r)   r  r  r  r  ImportErrorr   r*   )
y_try_prtypesr  r  TrueInputTypePredInputTyperX   rW   r  s
             rY   test_log_loss_pandas_inputr  )  s    8823D88c3Z#sc3Z#sDED]+,E,f() ).$&t,mD.A'i(	 ).  s   	B 
B"!B"c                      [         R                  " S5      n [        R                  " [        U S9   [        / SQ/ SQ/ SQ/ SQ// SQS9  S S S 5        g ! , (       d  f       g = f	NzLabels passed were ['spam', 'eggs', 'ham']. But this function assumes labels are ordered lexicographically. Pass the ordered labels=['eggs', 'ham', 'spam'] and ensure that the columns of y_prob correspond to this ordering.r   eggsr  r  r   r3  r   )r  r  r  r   )r  r  r   r   r   r   expected_messages    rY   test_log_loss_warningsr  ;  sK    yy	= 
k)9	:#	9-*	
 
;	:	:   A
A c                     [         R                  " / SQ5      n [         R                  " / SQ5      n[        R                  " X-
  5      S-  [	        U 5      -  n[        [        X 5      S5        [        [        X5      U5        [        [        SU -   U5      U5        [        [        SU -  S-
  U5      U5        [         R                  " SU-
  U45      n[         R                  " SU -
  U 45      n[        [        X5      U5        [        [        XC5      U5        [        [        XSS9U5        [        [        XS	S9U5        [        [        XS
S9SU-  5        [        [        S/S/5      S5        [        [        S/S/5      S5        [        [        S/S/5      S5        [        [        S/S/SS9S5        [        [        S/S/SS9S5        g )Nr   r;   r;   r   r;   r;   r   r   re   r   r   gffffff?r4   r   r   r;   auto)scale_by_halfTFr   r   g|Gz?r   r  foobarr   )r@   r   r   normrs   r+   r   column_stack)rX   y_prob
true_scorey_prob_reshapedy_true_reshapeds        rY   test_brier_score_loss_binaryr(  J  s   XX()FXX56FV_-2S[@J(8#>(8*E(vv>
K(Va@*M ooq6z6&:;Oooq6z6&:;O(A:N(JJW v>
 t<j u=q:~
 ("u5v>(!se4f=(!se4nE(%3%5I6R%3%59r[   c            	         [        [        / SQ/ SQ/ SQ/ SQ// SQS9S5        [        [        / SQ/ SQ/ S	Q/ S
Q/5      S5        [        [        / SQ/ SQ/ SQ/ SQ/5      S5        [        [        / SQ/ SQ/ SQ/ SQ/5      S5        g )Nr  rp  rq  )r  r  r  yamsr   r   rF  r  r  r  gt?r   )r   r   r   )r   r   r   )r   r   r   r   r4   )r+   r   r   r[   rY    test_brier_score_loss_multiclassr+  q  s    #<62	

 	 /J	
 		 /J	
 	
	 /J	
 	
	r[   c                  z   [         R                  " / SQ5      n [         R                  " / SQ5      n[        R                  " [        5         [        XSS  5        S S S 5        [        R                  " [        5         [        XS-   5        S S S 5        [        R                  " [        5         [        XS-
  5        S S S 5        [         R                  " / SQ5      n [         R                  " / SQ/ SQ/ SQ/5      n[        R                  " [        5         [        XSS  5        S S S 5        [        R                  " [        5         [        XS-   5        S S S 5        [        R                  " [        5         [        XS-
  5        S S S 5        [         R                  " / S	Q5      n [         R                  " / S
Q5      n[        R                  " S5      n[        R                  " [        US9   [        X5        S S S 5        / SQn SS/SS/SS//nSn[        R                  " [        [        R                  " U5      S9   [        X5        S S S 5        / SQn / SQ/ SQ/ SQ/n/ SQnSn[        R                  " [        [        R                  " U5      S9   [        XUS9  S S S 5        S/n SS//nSn[        R                  " [        [        R                  " U5      S9   [        X5        S S S 5        [        [        XSS/S9S5        g ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNy= f! , (       d  f       GN#= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GND= f! , (       d  f       GN= f! , (       d  f       N= f)Nr  r  r;   r   rF  r  r  r  )r   r;   r4   r   )r   r   r   r   zpThe type of the target inferred from y_true is multiclass but should be binary according to the shape of y_prob.r   r   r   zy_true and y_prob contain different number of classes: 3 vs 2. Please provide the true labels explicitly through the labels argument. Classes found in y_true: [0 1 2]r  r   r3  )r  r  r  r*  zwThe number of classes in labels is different from that in y_prob. Classes found in labels: ['eggs' 'ham' 'spam' 'yams']r   r  re   r   zy_true contains only one label (eggs). Please provide the list of all expected class labels explicitly through the labels argument.r  r  )	r@   r   r   r   r   r   r  r  r+   )rX   r$  r  rp   s       rY   $test_brier_score_loss_invalid_inputsr-    s   XX()FXX56F	z	", 
# 
z	"#. 
# 
z	"#. 
#
 XXi FXXIJF	z	", 
# 
z	"#. 
# 
z	"#. 
#
 XXl#FXX*+FII	AM 
z	7( 
8 F!fq!fq!f%F	  
z=)A	B( 
C %FI.F,F	/ 
 
z=)A	B7 
C XFCj\F	 
 
z=)A	B( 
C (PRVWK 
#	" 
#	" 
#	" 
#	" 
#	" 
#	" 
8	7 
C	B 
C	B 
C	Bsx   L
<L-L.M M7M$-M6?NNN,

L
L+.
L= 
M
M!$
M36
N
N
N),
N:c                      [         R                  " S5      n [        R                  " [        U S9   [        / SQ/ SQ/ SQ/ SQ// SQS9  S S S 5        g ! , (       d  f       g = fr  )r  r  r   r   r   r   r  s    rY   test_brier_score_loss_warningsr/    sQ    yy	= 
k)9	:#
 +	
 
;	:	:r  c                      Sn [         R                  " [        U S9   [        / SQ/ SQ5        S S S 5        g ! , (       d  f       g = f)Nz%y_pred contains classes not in y_truer   rh  r   )r   r   r   r   r   s    rY   #test_balanced_accuracy_score_unseenr1    s+    
1C	k	-	95 
.	-	-r  zy_true,y_pred)r   r   r   r   )r   r   r   r   )r   r   r   r   c                 R   [        XS[        R                  " U 5      S9n[        5          [	        X5      nS S S 5        W[
        R                  " U5      :X  d   e[	        XSS9n[	        U [        R                  " X S   5      5      nXCU-
  SU-
  -  :X  d   eg ! , (       d  f       Nh= f)Nr   r   T)adjustedr   r;   )r!   r@   uniquer.   r   r   r   	full_like)rX   rW   macro_recallbalancedr3  chances         rY   test_balanced_accuracy_scorer9    s      		&0AL 
	*6: 
 v}}\2222&vEH$VR\\&)-LMF6)a&j9999 
	s   B
B&r   ))FTr  )r   r   )zeroonec                 0   [         R                  R                  S5      nSUS   pCUR                  XSS9nU [        L a  UR                  US9nOUR                  5       nU " XVUS9n[         R                  " [         R                  " U5      5      (       a   eg)	zCheck that the metric works with different types of `pos_label`.

We can expect `pos_label` to be a bool, an integer, a float, a string.
No error should be raised for those types.
*   rl  r   T)r  replacer  r   N)	r@   rD   rE   choicer   uniformr  anyr  )r|   r   rS   rP   r   rX   rW   r  s           rY   *test_classification_metric_pos_label_typesrB    s    * ))


#Cwr{yZZZ>F!!),Fi8Fvvbhhv&'''''r[   zy_true, y_pred, expected_scorec                 L    [        XSS9[        R                  " U5      :X  d   eg)zCheck the behaviour of `zero_division` for f1-score.

Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/26965
r   r  N)r   r   r   )rX   rW   rm  s      rY   2test_f1_for_small_binary_inputs_with_zero_divisionrD  /  s"     F#6&--:WWWWr[   scoringr  r4   )r   r   c           	      z    [         R                  " SS9u  p[        SSS9R                  X5      n[	        X1X SSS9  g)	aB  Check that we validate `np.nan` properly for classification metrics.

With `n_jobs=2` in cross-validation, the `np.nan` used for the singleton will be
different in the sub-process and we should not use the `is` operator but
`math.isnan`.

Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/27563
r   )r:   r   )	max_depthr:   r4   r  )rE  n_jobserror_scoreN)r
   make_classificationr(   rI   r%   )rE  rN   rO   
classifiers       rY   9test_classification_metric_division_by_zero_nan_validatonrL  A  s;    & ''Q7DA'!!DHHNJJ1aWUr[   c            	         / SQn / SQn[         R                  " SS/SS/SS/SS/SS	/S
S//5      n[         R                  " SS/SS/SS/SS/SS/SS//5      n[        XS9n[        XSS9n[        XSS9nSXV-  -
  nU[        R
                  " U5      :X  d   e[         R                  " / SQ5      nUS S R                  5       UR                  5       -  US S 2S4'   USS  R                  5       UR                  5       -  US S 2S4'   [        XUS9n[        U UUSS9n[        U UUSS9nSXV-  -
  nU[        R
                  " U5      :X  d   e[         R                  " SS/SS/SS/SS/SS/SS//5      n[        X5      nSUs=:  a  S:  d   e   e[        X5      n	U	[        R
                  " U5      :X  d   e[         R                  " SS/SS/SS/SS/SS/SS//5      n[        X5      nUS:  d   e[        X5      n	U	[        R
                  " U5      :X  d   e/ SQn [         R                  " SS/SS/SS/SS/SS/SS//5      n[        X5      nUS:X  d   e[        X5      n	U	S:X  d   e/ SQn / SQn[         R                  " SS/SS/SS/SS//5      n[        X5      nUS:X  d   e[        X5      n	U	S:X  d   e/ SQn[        XUS9n
U
S:X  d   e/ SQn / SQn[         R                  " / S Q/ S Q/ S!Q/ S"Q/5      n[        X5      nSUs=:  a  S:  d   e   e[        XUS9nSUs=:  a  S:  d   e   e[         R                  " / S#Q/ S$Q/ S"Q/ S%Q/5      n[        X5      nUS:  d   e[        XUS9nUS:  d   eg )&Nra  r  r   re   r   r   r   gffffff?g?r  r  r`  F)rX   rW   rN  r;   )r4   r;   r   r   r   r;   r   r   )rX   rW   r7  )rX   rW   r7  rN  r   r   r   r   r{  )r   r;   r;   r;   )r  r  r  r  )r4   r4   r4   r4   rD  )highrN  lowneutral)ffffff?r   r   r   )r   r   r   r   r  )r   r   r   r  r  )r@   r   r$   r   r   r   rX  )rX   y_true_stringrW   y_pred_nulld2_scorelog_likelihoodlog_likelihood_nulld2_score_truer7  d2_score_stringd2_score_with_sample_weights              rY   test_d2_log_loss_scorerZ  Y  s\   F;MXX#J#J#J#J4L4L	
	F ((#J#J#J#J#J#J	
	K !>HVeLN"&PUV<<Mv}}]3333 HH/0M%bq)--/-2C2C2EEK1%ab)--/-2C2C2EEK1 MH #	N ##	 <<Mv}}]3333 XX#J#J#J#J#J#J	
	F !0HC'>OfmmH5555 XX#J#J#J#J4L#J	
	F !0Ha<<'>OfmmH5555  FXX#J#J#J#J#J#J	
	F !0Hq=='>Oa F/MXXd|dD\D$<$NOF 0Hq=='>Oa M"3m# '!+++ 0F(MXX		
F !0HC }MHCXX		
F !0Ha<< }MHa<<r[   c                      / SQn / SQn/ SQn[         R                  " / SQS5      n[        XX!S9n[         R                  " / SQS5      n[        XX!S9nSXF-  -
  n[        XX!S9n[	        X5        g	)
zCheck that d2_log_loss_score works when not all labels are present in y_true

non-regression test for https://github.com/scikit-learn/scikit-learn/issues/30713
r4   r   r4   r   r   )rQ  r   r   r   r   r   r;   )r7  rp   )r   r   r   r;   N)r@   tiler   r$   r*   )	rX   rp   r7  rW   log_loss_obsrS  log_loss_nullexpected_d2_scorerT  s	            rY   %test_d2_log_loss_score_missing_labelsrb    sy    
 FF(MWWY'FF-WL ''-0K=M L88 mH H0r[   c                      / SQn [         R                  " / SQS5      n[        X/ SQS9n[        X/ SQS9n[        X#5        g)zGCheck that d2_log_loss_score doesn't depend on the order of the labels.r\  r   r]  r   r   r+  N)r@   r^  r$   r*   )rX   rW   rT  d2_score_others       rY   "test_d2_log_loss_score_label_orderre    s7    FWWY'F 	BH&viHNH-r[   c                     / SQn SS/SS/SS//nSn[         R                  " [        US9   [        X5        S	S	S	5        / SQn SS/SS/SS//n/ SQnS
n[         R                  " [        US9   [        XUS9  S	S	S	5        / SQn / SQ/ SQ/nSn[         R                  " [        US9   [        X5        S	S	S	5        S/n SS//nSn[         R                  " [
        US9   [        X5        S	S	S	5        / SQn SS/SS/SS//nSn[         R                  " [        US9   [        X5        S	S	S	5        / SQn S/nSS/SS/SS//nSn[         R                  " [        US9   [        XUS9  S	S	S	5        g	! , (       d  f       GNA= f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g	= f)zLTest that d2_log_loss_score raises the appropriate errors on
invalid inputs.r   r   r   r   r   r   z#contain different number of classesr   Nz(number of classes in labels is differentr   )r   r   r   )r   r   r   rC  r;   zscore is not well-definedr   zy_true contains only one labelz.The labels array needs to contain at least two)r   r   r   r$   r   r   )rX   rW   errrp   s       rY   test_d2_log_loss_score_raisesrh    s    FCj3*sCj1F
/C	z	-&) 
.
 FCj3*sCj1FF
4C	z	-&8 
. F/F
+C	z	-&) 
. SFCj\F
%C	,C	8&) 
9 FCj3*sCj1F
*C	z	-&) 
.
 FSFCj3*sCj1F
:C	z	-&8 
.	-O 
.	- 
.	- 
.	- 
9	8 
.	- 
.	-sG   E1-F'FF&F7G1
F 
F
F#&
F47
G
G)NF)r  r   	functoolsr   	itertoolsr   r   r   numpyr@   r   scipyr   scipy.spatial.distancer   rI  scipy.statsr	   sklearnr
   r   sklearn.datasetsr   sklearn.exceptionsr   sklearn.metricsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   sklearn.metrics._classificationr#   r$   sklearn.model_selectionr%   sklearn.preprocessingr&   r'   sklearn.treer(   sklearn.utils._mockingr)   sklearn.utils._testingr*   r+   r,   r-   r.   sklearn.utils.extmathr/   sklearn.utils.fixesr0   r1   sklearn.utils.validationr2   rZ   r}   r   markparametrizer   r   r   r   r   filterwarningsr   r   r   r   r   r   r   r   r  r  r/  rA  rG  rU  r\  r_  rf  rj  rq  ru  r|  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r!  r&  r(  r*  r-  r4  r7  r:  r?  rA  rC  rE  rK  rS  rV  rd  rf  rk  rn  rv  r|  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  float16r  r  r	  r  r  r(  r+  r-  r/  r1  r9  rB  rD  rL  rZ  rb  re  rh  r   r[   rY   <module>r     s   	   2 2    8 ! ! ; 5     . N 3 @ / 0  . > 7)(`>DBW< 61a*@A B T{aVUOi=OP))(7D PQ R$ PQ*J R*JZ PQM RM,>6  HH####	
	
 .=	
(9)(9  HH#####	
 	
*;+*;
 =*	= OF .9.9%S : :%SPT< +%%;&) 
 ((#56((#56 N	
 ((#56((#56
B		
 ((#56((#56 N	
 ((#56((#56 H	
;$'P*Q'P*  ((?3((?3
<		
**"4 4-8M 9M;0 		S!E"C C 





 $S	!3'	$bff-S	!3'	'0		D	D$ $S	!3'	$c*	$c*	'0		D	D$>1 1a.9)aS1#J<8!$	' 9 :' )aS1#J<8!$	 9$2N RF5p c5\2U 3UB7(t $ST, U,0"(".& 	;<
QAB 
'(  88 taVY'-M  %%6 "AB0 C0"%4%$%(%6 %F%,%.XI PQ% R%<6B(%DPJZ"J.+ 868:LM	2 N	2 PQ?U R?UD PQ= R=@ PQ+&&266266"23[	 R
[| !%$MN1a.9"5 : O &"5J $MN" O"& 1a.9 W : WF38e0P 1a.9:  :: z 61a*@A$ B$N 61a*@A$
 B$
N 61a*@A#( B#(L'>_D=8A6*"N8P4@A8I(X 2::rzz2::"FG
 H
 2::rzz2::"FG! H! 	I	aVaVaV,-	Y	95688)$
$N FIXX
&6 	34	34	34
:
: #&' C(($ $	1a&	288QF+S1	1a&	288QF+S1	1a&	288QF+S1	1a&	288QF+S1	XX HBFF3Karvv>O266:L7	VVSl1:..9r[   