========================================================================================================================
SUITE4 EXPERIMENT 2 — COUNCIL OF SPECIALISTS + COMPETENCE MODEL
V2-COMPATIBLE COMPLETE OPERATIONAL EVALUATION ENABLED
========================================================================================================================

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=1
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0]
  block=54 n=680 AP q/global/recent/rel=0.4814/0.5208/0.4518/0.2064
  block=56 n=997 AP q/global/recent/rel=0.4039/0.3303/0.4485/0.3486
  block=57 n=1,084 AP q/global/recent/rel=0.5275/0.4676/0.4675/0.1190
  block=59 n=1,460 AP q/global/recent/rel=0.1041/0.1940/0.1466/0.1647
  block=60 n=1,355 AP q/global/recent/rel=0.4148/0.2471/0.2782/0.3387
[POLICY] alpha=0.50 temperature=2.00
committee                    AP/AUC=0.3476/0.8074 F2@20%=0.4132 P/R=0.1887/0.5882
competence_council           AP/AUC=0.3703/0.8324 F2@20%=0.4270 P/R=0.1950/0.6078 ΔAP=+0.0226
[COMPETENCE] target top1 accuracy=0.304 | NON_DEPLOYABLE best-expert oracle AP=0.9471
  history0     n=647 AP 0.2053->0.2288 weights=[0.255 0.254 0.255 0.237]
  history1_2   n=327 AP 0.5097->0.5187 weights=[0.256 0.251 0.247 0.246]
  history3plus n=614 AP 0.5508->0.5484 weights=[0.253 0.25  0.245 0.252]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=2
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0]
  block=54 n=680 AP q/global/recent/rel=0.4770/0.3655/0.5041/0.1279
  block=56 n=997 AP q/global/recent/rel=0.3961/0.3711/0.4476/0.0904
  block=57 n=1,084 AP q/global/recent/rel=0.5193/0.4396/0.3612/0.1071
  block=59 n=1,460 AP q/global/recent/rel=0.1251/0.0945/0.0899/0.1484
  block=60 n=1,355 AP q/global/recent/rel=0.4056/0.3625/0.3654/0.3673
  block=62 n=1,588 AP q/global/recent/rel=0.3455/0.3286/0.2769/0.3120
[POLICY] alpha=0.75 temperature=2.00
committee                    AP/AUC=0.7595/0.9786 F2@20%=0.6357 P/R=0.2620/0.9880
competence_council           AP/AUC=0.8192/0.9791 F2@20%=0.6279 P/R=0.2588/0.9759 ΔAP=+0.0597
[COMPETENCE] target top1 accuracy=0.390 | NON_DEPLOYABLE best-expert oracle AP=0.9833
  history0     n=4 AP nan->nan weights=[0.291 0.279 0.224 0.205]
  history1_2   n=765 AP 0.7113->0.8145 weights=[0.258 0.252 0.244 0.246]
  history3plus n=798 AP 0.8756->0.8460 weights=[0.256 0.253 0.248 0.243]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=3
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0]
  block=54 n=680 AP q/global/recent/rel=0.4736/0.4097/0.3739/0.2138
  block=56 n=997 AP q/global/recent/rel=0.3915/0.3257/0.3195/0.4110
  block=57 n=1,084 AP q/global/recent/rel=0.5211/0.4343/0.4515/0.1115
  block=59 n=1,460 AP q/global/recent/rel=0.0963/0.0605/0.1043/0.1022
  block=60 n=1,355 AP q/global/recent/rel=0.4154/0.2992/0.2796/0.3319
  block=62 n=1,588 AP q/global/recent/rel=0.3323/0.3047/0.1405/0.2876
  block=63 n=1,567 AP q/global/recent/rel=0.7643/0.8183/0.5067/0.4862
[POLICY] alpha=0.50 temperature=2.00
committee                    AP/AUC=0.4220/0.9314 F2@20%=0.5150 P/R=0.1980/0.8587
competence_council           AP/AUC=0.4460/0.9429 F2@20%=0.5476 P/R=0.2105/0.9130 ΔAP=+0.0240
[COMPETENCE] target top1 accuracy=0.315 | NON_DEPLOYABLE best-expert oracle AP=0.9618
  history0     n=466 AP 0.6189->0.6281 weights=[0.261 0.254 0.243 0.241]
  history1_2   n=1,054 AP 0.3300->0.3531 weights=[0.257 0.256 0.248 0.239]
  history3plus n=476 AP 0.5177->0.5165 weights=[0.255 0.26  0.248 0.236]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=4
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0]
  block=54 n=680 AP q/global/recent/rel=0.4921/0.3882/0.4419/0.0355
  block=56 n=997 AP q/global/recent/rel=0.4366/0.3491/0.3569/0.3270
  block=57 n=1,084 AP q/global/recent/rel=0.4999/0.3879/0.3345/0.0931
  block=59 n=1,460 AP q/global/recent/rel=0.1217/0.0825/0.1491/0.1790
  block=60 n=1,355 AP q/global/recent/rel=0.4131/0.2636/0.3090/0.3867
  block=62 n=1,588 AP q/global/recent/rel=0.3215/0.3412/0.1792/0.2872
  block=63 n=1,567 AP q/global/recent/rel=0.7625/0.8631/0.6555/0.5176
  block=65 n=1,996 AP q/global/recent/rel=0.4046/0.4136/0.3579/0.3601
[POLICY] alpha=1.00 temperature=2.00
committee                    AP/AUC=0.6177/0.9271 F2@20%=0.6028 P/R=0.2861/0.8333
competence_council           AP/AUC=0.6256/0.9268 F2@20%=0.6028 P/R=0.2861/0.8333 ΔAP=+0.0079
[COMPETENCE] target top1 accuracy=0.387 | NON_DEPLOYABLE best-expert oracle AP=0.9485
  history0     n=3 AP 1.0000->1.0000 weights=[0.245 0.252 0.261 0.242]
  history1_2   n=584 AP 0.6241->0.6474 weights=[0.263 0.262 0.258 0.217]
  history3plus n=1,072 AP 0.5968->0.5933 weights=[0.263 0.259 0.259 0.219]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=5
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0]
  block=54 n=680 AP q/global/recent/rel=0.4854/0.3706/0.4753/0.1794
  block=56 n=997 AP q/global/recent/rel=0.3954/0.3907/0.2783/0.4298
  block=57 n=1,084 AP q/global/recent/rel=0.5169/0.4969/0.3883/0.1142
  block=59 n=1,460 AP q/global/recent/rel=0.1145/0.1003/0.0863/0.0076
  block=60 n=1,355 AP q/global/recent/rel=0.4329/0.3212/0.3266/0.1853
  block=62 n=1,588 AP q/global/recent/rel=0.3321/0.3398/0.2563/0.3020
  block=63 n=1,567 AP q/global/recent/rel=0.7619/0.8111/0.6472/0.5027
  block=65 n=1,996 AP q/global/recent/rel=0.4030/0.3807/0.2785/0.4232
  block=66 n=1,659 AP q/global/recent/rel=0.6400/0.6144/0.5797/0.5145
[POLICY] alpha=0.75 temperature=1.00
committee                    AP/AUC=0.4660/0.9261 F2@20%=0.6296 P/R=0.2910/0.8881
competence_council           AP/AUC=0.4806/0.9259 F2@20%=0.6085 P/R=0.2812/0.8582 ΔAP=+0.0146
[COMPETENCE] target top1 accuracy=0.369 | NON_DEPLOYABLE best-expert oracle AP=0.9217
  history0     n=577 AP 0.4884->0.5059 weights=[0.275 0.256 0.248 0.22 ]
  history1_2   n=506 AP 0.5992->0.6272 weights=[0.279 0.258 0.25  0.213]
  history3plus n=964 AP 0.3659->0.3313 weights=[0.275 0.265 0.26  0.2  ]

========================================================================================================================
DEVELOPMENT-SELECTED OPERATIONAL POLICIES [beta=2]
Thresholds and warning ratios below are frozen before holdout evaluation.
========================================================================================================================
  competence_council  : AP=0.4947 | AUC=0.8954 | F2-opt tau=0.306967 | P=0.4220 R=0.6800 F1=0.5208 F2=0.6059 | flag=9.55% | ratio=1.611
  committee           : AP=0.4985 | AUC=0.8966 | F2-opt tau=0.143236 | P=0.3901 R=0.7238 F1=0.5070 F2=0.6181 | flag=11.00% | ratio=1.855

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=6
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0]
  block=54 n=680 AP q/global/recent/rel=0.4723/0.4087/0.5099/0.2237
  block=56 n=997 AP q/global/recent/rel=0.4110/0.3952/0.2978/0.3222
  block=57 n=1,084 AP q/global/recent/rel=0.5031/0.4554/0.5216/0.1008
  block=59 n=1,460 AP q/global/recent/rel=0.1080/0.0675/0.0853/0.1757
  block=60 n=1,355 AP q/global/recent/rel=0.4195/0.3523/0.3262/0.2890
  block=62 n=1,588 AP q/global/recent/rel=0.3421/0.2994/0.2231/0.3159
  block=63 n=1,567 AP q/global/recent/rel=0.7496/0.7607/0.6789/0.5393
  block=65 n=1,996 AP q/global/recent/rel=0.4216/0.3902/0.3982/0.3956
  block=66 n=1,659 AP q/global/recent/rel=0.6342/0.6068/0.5326/0.3767
  block=68 n=2,047 AP q/global/recent/rel=0.4663/0.4399/0.4332/0.5388
[POLICY] alpha=0.75 temperature=2.00
committee                    AP/AUC=0.4801/0.7788 F2@20%=0.4924 P/R=0.3179/0.5707
competence_council           AP/AUC=0.4951/0.7899 F2@20%=0.5051 P/R=0.3261/0.5854 ΔAP=+0.0150
[COMPETENCE] target top1 accuracy=0.518 | NON_DEPLOYABLE best-expert oracle AP=0.7803
  history0     n=5 AP 0.2000->0.3333 weights=[0.262 0.21  0.249 0.279]
  history1_2   n=620 AP 0.6554->0.6651 weights=[0.257 0.242 0.247 0.255]
  history3plus n=1,217 AP 0.3576->0.3651 weights=[0.256 0.25  0.253 0.241]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=7
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0, 69.0]
  block=54 n=680 AP q/global/recent/rel=0.4888/0.4890/0.4612/0.1859
  block=56 n=997 AP q/global/recent/rel=0.4055/0.3945/0.3076/0.4101
  block=57 n=1,084 AP q/global/recent/rel=0.5136/0.3726/0.2853/0.1150
  block=59 n=1,460 AP q/global/recent/rel=0.1031/0.0788/0.0610/0.1657
  block=60 n=1,355 AP q/global/recent/rel=0.4051/0.3330/0.4316/0.3505
  block=62 n=1,588 AP q/global/recent/rel=0.3328/0.3059/0.1913/0.2981
  block=63 n=1,567 AP q/global/recent/rel=0.7700/0.8054/0.5850/0.5231
  block=65 n=1,996 AP q/global/recent/rel=0.4233/0.4205/0.3838/0.4075
  block=66 n=1,659 AP q/global/recent/rel=0.6374/0.5810/0.5981/0.4103
  block=68 n=2,047 AP q/global/recent/rel=0.4641/0.4561/0.4324/0.4709
  block=69 n=1,842 AP q/global/recent/rel=0.4785/0.4806/0.5071/0.4501
[POLICY] alpha=1.00 temperature=2.00
committee                    AP/AUC=0.2739/0.7723 F2@20%=0.4446 P/R=0.2312/0.5779
competence_council           AP/AUC=0.2841/0.7829 F2@20%=0.4645 P/R=0.2416/0.6039 ΔAP=+0.0102
[COMPETENCE] target top1 accuracy=0.436 | NON_DEPLOYABLE best-expert oracle AP=0.7025
  history0     n=674 AP 0.3682->0.3821 weights=[0.257 0.255 0.249 0.238]
  history1_2   n=543 AP 0.2135->0.2217 weights=[0.255 0.253 0.251 0.241]
  history3plus n=707 AP 0.2169->0.2120 weights=[0.252 0.255 0.255 0.239]

========================================================================================================================
SUITE4 ORIGINAL RANKING / EQUAL-BUDGET SUMMARY
========================================================================================================================
development NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.9194 mean-fold AP=0.9525 F2@20=0.6756
development committee                          pooled AP=0.4985 mean-fold AP=0.5226 F2@20=0.5593
development competence_council                 pooled AP=0.4947 mean-fold AP=0.5483 F2@20=0.5627
holdout     NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.7455 mean-fold AP=0.7414 F2@20=0.7112
holdout     committee                          pooled AP=0.3828 mean-fold AP=0.3770 F2@20=0.4685
holdout     competence_council                 pooled AP=0.3908 mean-fold AP=0.3896 F2@20=0.4848

### DEVELOPMENT [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    1588   6.42% |  0.3703  0.8324 |  0.5833  0.2059  0.3043  0.2365    2.27% |  0.3158  0.4706  0.3780  0.4286    9.57%
     2    1567   5.30% |  0.8192  0.9791 |  0.6063  0.9277  0.7333  0.8388    8.10% |  0.4877  0.9518  0.6449  0.7996   10.34%
     3    1996   4.61% |  0.4460  0.9429 |  0.4145  0.6848  0.5164  0.6058    7.62% |  0.3824  0.7065  0.4962  0.6041    8.52%
     4    1659   6.87% |  0.6256  0.9268 |  0.4410  0.7544  0.5566  0.6605   11.75% |  0.5854  0.6316  0.6076  0.6218    7.41%
     5    2047   6.55% |  0.4806  0.9259 |  0.3274  0.8209  0.4681  0.6307   16.41% |  0.3921  0.6642  0.4931  0.5832   11.09%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    8857   5.93% |  0.4947  0.8954 |  0.4220  0.6800  0.5208  0.6059    9.55% |  0.4233  0.6724  0.5195  0.6016    9.42%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    1842  11.13% |  0.4951  0.7899 |  0.4163  0.5220  0.4632  0.4968   13.95% |  0.5155  0.4878  0.5013  0.4931   10.53%

  Confusion matrix — fold 6 — fixed development threshold 0.306967
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,487             150
  Failure <5                            98             107
  ------------------------------------------------------------------
  Normal correctly rejected: 90.84%
  Failures successfully detected: 52.20%

  Confusion matrix — fold 6 — drift-aware rate 10.549%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,543              94
  Failure <5                           105             100
  ------------------------------------------------------------------
  Normal correctly rejected: 94.26%
  Failures successfully detected: 48.78%
     7    1924   8.00% |  0.2841  0.7829 |  0.2804  0.4935  0.3576  0.4284   14.09% |  0.2406  0.5390  0.3327  0.4318   17.93%

  Confusion matrix — fold 7 — fixed development threshold 0.306967
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,575             195
  Failure <5                            78              76
  ------------------------------------------------------------------
  Normal correctly rejected: 88.98%
  Failures successfully detected: 49.35%

  Confusion matrix — fold 7 — drift-aware rate 17.934%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,508             262
  Failure <5                            71              83
  ------------------------------------------------------------------
  Normal correctly rejected: 85.20%
  Failures successfully detected: 53.90%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    3766   9.53% |  0.3908  0.7719 |  0.3466  0.5097  0.4126  0.4659   14.02% |  0.3395  0.5097  0.4076  0.4633   14.31%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,062             345
  Failure <5                           176             183
  ------------------------------------------------------------------
  Normal correctly rejected: 89.87%
  Failures successfully detected: 50.97%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,051             356
  Failure <5                           176             183
  ------------------------------------------------------------------
  Normal correctly rejected: 89.55%
  Failures successfully detected: 50.97%

### DEVELOPMENT [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    1588   6.42% |  0.3476  0.8074 |  0.4648  0.3235  0.3815  0.3445    4.47% |  0.2914  0.5000  0.3682  0.4374   11.02%
     2    1567   5.30% |  0.7595  0.9786 |  0.5455  0.9398  0.6903  0.8211    9.13% |  0.4332  0.9759  0.6000  0.7803   11.93%
     3    1996   4.61% |  0.4220  0.9314 |  0.3318  0.7717  0.4641  0.6100   10.72% |  0.3469  0.7391  0.4722  0.6028    9.82%
     4    1659   6.87% |  0.6177  0.9271 |  0.4492  0.7368  0.5581  0.6532   11.27% |  0.5493  0.6842  0.6094  0.6522    8.56%
     5    2047   6.55% |  0.4660  0.9261 |  0.3175  0.8507  0.4625  0.6369   17.54% |  0.3640  0.7090  0.4810  0.5960   12.75%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    8857   5.93% |  0.4985  0.8966 |  0.3901  0.7238  0.5070  0.6181   11.00% |  0.3881  0.7105  0.5020  0.6093   10.85%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    1842  11.13% |  0.4801  0.7788 |  0.3459  0.5366  0.4207  0.4833   17.26% |  0.4420  0.4829  0.4615  0.4741   12.16%

  Confusion matrix — fold 6 — fixed development threshold 0.143236
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,429             208
  Failure <5                            95             110
  ------------------------------------------------------------------
  Normal correctly rejected: 87.29%
  Failures successfully detected: 53.66%

  Confusion matrix — fold 6 — drift-aware rate 12.145%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,512             125
  Failure <5                           106              99
  ------------------------------------------------------------------
  Normal correctly rejected: 92.36%
  Failures successfully detected: 48.29%
     7    1924   8.00% |  0.2739  0.7723 |  0.2276  0.6104  0.3316  0.4568   21.47% |  0.2317  0.5974  0.3339  0.4541   20.63%

  Confusion matrix — fold 7 — fixed development threshold 0.143236
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,451             319
  Failure <5                            60              94
  ------------------------------------------------------------------
  Normal correctly rejected: 81.98%
  Failures successfully detected: 61.04%

  Confusion matrix — fold 7 — drift-aware rate 20.647%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,465             305
  Failure <5                            62              92
  ------------------------------------------------------------------
  Normal correctly rejected: 82.77%
  Failures successfully detected: 59.74%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    3766   9.53% |  0.3828  0.7474 |  0.2791  0.5682  0.3743  0.4707   19.41% |  0.3076  0.5320  0.3898  0.4643   16.49%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,880             527
  Failure <5                           155             204
  ------------------------------------------------------------------
  Normal correctly rejected: 84.53%
  Failures successfully detected: 56.82%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,977             430
  Failure <5                           168             191
  ------------------------------------------------------------------
  Normal correctly rejected: 87.38%
  Failures successfully detected: 53.20%
