========================================================================================================================
SUITE4 EXPERIMENT 2 — COUNCIL OF SPECIALISTS + COMPETENCE MODEL
V2-COMPATIBLE COMPLETE OPERATIONAL EVALUATION ENABLED
========================================================================================================================

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=1
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0]
  block=54 n=1,617 AP q/global/recent/rel=0.5750/0.4645/0.4629/0.1669
  block=56 n=3,115 AP q/global/recent/rel=0.1315/0.2024/0.0593/0.2531
  block=57 n=2,815 AP q/global/recent/rel=0.4715/0.5130/0.4211/0.1726
  block=59 n=5,277 AP q/global/recent/rel=0.0693/0.1238/0.1186/0.1012
  block=60 n=3,992 AP q/global/recent/rel=0.3551/0.2970/0.2633/0.1163
[POLICY] alpha=0.25 temperature=2.00
committee                    AP/AUC=0.2032/0.7519 F2@20%=0.3472 P/R=0.1197/0.6614
competence_council           AP/AUC=0.2606/0.8010 F2@20%=0.3556 P/R=0.1226/0.6772 ΔAP=+0.0574
[COMPETENCE] target top1 accuracy=0.462 | NON_DEPLOYABLE best-expert oracle AP=0.9148
  history0     n=1,961 AP 0.0405->0.0560 weights=[0.258 0.258 0.255 0.229]
  history1_2   n=1,959 AP 0.2991->0.3702 weights=[0.253 0.253 0.25  0.244]
  history3plus n=1,302 AP 0.4366->0.5355 weights=[0.255 0.253 0.248 0.244]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=2
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0]
  block=54 n=1,617 AP q/global/recent/rel=0.6076/0.5216/0.4920/0.2623
  block=56 n=3,115 AP q/global/recent/rel=0.1118/0.1774/0.1379/0.0696
  block=57 n=2,815 AP q/global/recent/rel=0.4777/0.4356/0.4619/0.2624
  block=59 n=5,277 AP q/global/recent/rel=0.0767/0.0959/0.1198/0.1279
  block=60 n=3,992 AP q/global/recent/rel=0.3539/0.3107/0.2710/0.2827
  block=62 n=5,222 AP q/global/recent/rel=0.2223/0.2704/0.1320/0.2148
[POLICY] alpha=0.50 temperature=2.00
committee                    AP/AUC=0.4729/0.9206 F2@20%=0.4254 P/R=0.1413/0.8553
competence_council           AP/AUC=0.4767/0.9152 F2@20%=0.4221 P/R=0.1402/0.8487 ΔAP=+0.0038
[COMPETENCE] target top1 accuracy=0.314 | NON_DEPLOYABLE best-expert oracle AP=0.8793
  history0     n=4 AP nan->nan weights=[0.304 0.251 0.254 0.191]
  history1_2   n=1,855 AP 0.2965->0.2698 weights=[0.257 0.257 0.25  0.235]
  history3plus n=2,739 AP 0.6240->0.6533 weights=[0.251 0.251 0.245 0.254]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=3
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0]
  block=54 n=1,617 AP q/global/recent/rel=0.6079/0.5093/0.4944/0.2000
  block=56 n=3,115 AP q/global/recent/rel=0.1323/0.1338/0.0974/0.1917
  block=57 n=2,815 AP q/global/recent/rel=0.4682/0.4978/0.5008/0.1788
  block=59 n=5,277 AP q/global/recent/rel=0.0856/0.0990/0.1159/0.0989
  block=60 n=3,992 AP q/global/recent/rel=0.3607/0.3174/0.2366/0.1899
  block=62 n=5,222 AP q/global/recent/rel=0.2212/0.2912/0.1953/0.1636
  block=63 n=4,598 AP q/global/recent/rel=0.4608/0.4366/0.3998/0.2569
[POLICY] alpha=1.00 temperature=1.00
committee                    AP/AUC=0.2738/0.8506 F2@20%=0.3785 P/R=0.1283/0.7385
competence_council           AP/AUC=0.3497/0.8692 F2@20%=0.3667 P/R=0.1243/0.7156 ΔAP=+0.0759
[COMPETENCE] target top1 accuracy=0.502 | NON_DEPLOYABLE best-expert oracle AP=0.8306
  history0     n=1,952 AP 0.1570->0.1063 weights=[0.26  0.26  0.25  0.229]
  history1_2   n=2,218 AP 0.3780->0.4925 weights=[0.257 0.252 0.236 0.255]
  history3plus n=2,107 AP 0.3613->0.4941 weights=[0.258 0.254 0.24  0.248]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=4
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0]
  block=54 n=1,617 AP q/global/recent/rel=0.6079/0.5185/0.4949/0.2149
  block=56 n=3,115 AP q/global/recent/rel=0.1293/0.1400/0.1467/0.1175
  block=57 n=2,815 AP q/global/recent/rel=0.4734/0.4800/0.5367/0.2152
  block=59 n=5,277 AP q/global/recent/rel=0.0704/0.0816/0.1527/0.0876
  block=60 n=3,992 AP q/global/recent/rel=0.3578/0.3309/0.2704/0.2981
  block=62 n=5,222 AP q/global/recent/rel=0.2215/0.2828/0.1913/0.1416
  block=63 n=4,598 AP q/global/recent/rel=0.4783/0.4305/0.3942/0.2288
  block=65 n=6,277 AP q/global/recent/rel=0.2675/0.3146/0.2657/0.3538
[POLICY] alpha=0.50 temperature=1.00
committee                    AP/AUC=0.3788/0.8880 F2@20%=0.4824 P/R=0.1893/0.7869
competence_council           AP/AUC=0.4144/0.9037 F2@20%=0.5050 P/R=0.1982/0.8238 ΔAP=+0.0356
[COMPETENCE] target top1 accuracy=0.494 | NON_DEPLOYABLE best-expert oracle AP=0.8333
  history0     n=11 AP 0.4042->0.4306 weights=[0.29  0.231 0.224 0.255]
  history1_2   n=1,995 AP 0.3650->0.4224 weights=[0.259 0.25  0.241 0.25 ]
  history3plus n=3,066 AP 0.4161->0.4376 weights=[0.255 0.249 0.243 0.252]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=5
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0]
  block=54 n=1,617 AP q/global/recent/rel=0.6087/0.5400/0.4610/0.2257
  block=56 n=3,115 AP q/global/recent/rel=0.1305/0.0674/0.1555/0.2015
  block=57 n=2,815 AP q/global/recent/rel=0.4697/0.4675/0.4260/0.1646
  block=59 n=5,277 AP q/global/recent/rel=0.0879/0.1027/0.1343/0.1239
  block=60 n=3,992 AP q/global/recent/rel=0.3574/0.3044/0.2164/0.1709
  block=62 n=5,222 AP q/global/recent/rel=0.2118/0.2820/0.2355/0.1656
  block=63 n=4,598 AP q/global/recent/rel=0.4701/0.4579/0.3652/0.1981
  block=65 n=6,277 AP q/global/recent/rel=0.2771/0.3305/0.2330/0.3411
  block=66 n=5,072 AP q/global/recent/rel=0.3906/0.3844/0.3922/0.4168
[POLICY] alpha=1.00 temperature=0.50
committee                    AP/AUC=0.2751/0.8292 F2@20%=0.4359 P/R=0.1828/0.6667
competence_council           AP/AUC=0.2918/0.8347 F2@20%=0.4545 P/R=0.1906/0.6952 ΔAP=+0.0167
[COMPETENCE] target top1 accuracy=0.556 | NON_DEPLOYABLE best-expert oracle AP=0.9107
  history0     n=1,906 AP 0.2591->0.2585 weights=[0.291 0.281 0.276 0.151]
  history1_2   n=2,223 AP 0.2526->0.3007 weights=[0.289 0.283 0.273 0.156]
  history3plus n=2,269 AP 0.3863->0.3842 weights=[0.289 0.292 0.275 0.144]

========================================================================================================================
DEVELOPMENT-SELECTED OPERATIONAL POLICIES [beta=2]
Thresholds and warning ratios below are frozen before holdout evaluation.
========================================================================================================================
  competence_council  : AP=0.3202 | AUC=0.8527 | F2-opt tau=0.199906 | P=0.2234 R=0.5572 F1=0.3189 F2=0.4290 | flag=10.44% | ratio=2.494
  committee           : AP=0.3020 | AUC=0.8457 | F2-opt tau=0.114276 | P=0.2228 R=0.5555 F1=0.3180 F2=0.4277 | flag=10.44% | ratio=2.493

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=6
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0]
  block=54 n=1,617 AP q/global/recent/rel=0.6110/0.4861/0.4469/0.1654
  block=56 n=3,115 AP q/global/recent/rel=0.1351/0.1404/0.0958/0.1686
  block=57 n=2,815 AP q/global/recent/rel=0.4574/0.4790/0.4767/0.3284
  block=59 n=5,277 AP q/global/recent/rel=0.0754/0.0752/0.1255/0.0679
  block=60 n=3,992 AP q/global/recent/rel=0.3584/0.3171/0.2526/0.1567
  block=62 n=5,222 AP q/global/recent/rel=0.2210/0.2696/0.1762/0.2245
  block=63 n=4,598 AP q/global/recent/rel=0.4809/0.4655/0.3932/0.1643
  block=65 n=6,277 AP q/global/recent/rel=0.2682/0.3225/0.2415/0.3185
  block=66 n=5,072 AP q/global/recent/rel=0.3823/0.3827/0.3760/0.3933
  block=68 n=6,398 AP q/global/recent/rel=0.2690/0.3145/0.2729/0.2658
[POLICY] alpha=1.00 temperature=0.50
committee                    AP/AUC=0.3724/0.8361 F2@20%=0.5337 P/R=0.3195/0.6412
competence_council           AP/AUC=0.4203/0.8460 F2@20%=0.5514 P/R=0.3301/0.6625 ΔAP=+0.0479
[COMPETENCE] target top1 accuracy=0.575 | NON_DEPLOYABLE best-expert oracle AP=0.8234
  history1_2   n=2,037 AP 0.3341->0.3770 weights=[0.277 0.245 0.219 0.259]
  history3plus n=3,614 AP 0.4092->0.4623 weights=[0.265 0.247 0.241 0.247]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=7
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0, 69.0, 70.0]
  block=54 n=1,617 AP q/global/recent/rel=0.5979/0.4400/0.4135/0.1822
  block=56 n=3,115 AP q/global/recent/rel=0.1389/0.0988/0.1310/0.0883
  block=57 n=2,815 AP q/global/recent/rel=0.4646/0.5284/0.4838/0.2974
  block=59 n=5,277 AP q/global/recent/rel=0.0717/0.0925/0.1097/0.1248
  block=60 n=3,992 AP q/global/recent/rel=0.3539/0.2748/0.2998/0.1610
  block=62 n=5,222 AP q/global/recent/rel=0.2329/0.2820/0.1830/0.2292
  block=63 n=4,598 AP q/global/recent/rel=0.4734/0.4239/0.3809/0.2610
  block=65 n=6,277 AP q/global/recent/rel=0.2657/0.3172/0.2370/0.3448
  block=66 n=5,072 AP q/global/recent/rel=0.3854/0.3740/0.3930/0.4049
  block=68 n=6,398 AP q/global/recent/rel=0.2732/0.2933/0.2767/0.2922
  block=69 n=5,651 AP q/global/recent/rel=0.3750/0.3862/0.3853/0.3794
  block=70 n=374 AP q/global/recent/rel=0.3436/0.2879/0.3079/0.3516
[POLICY] alpha=1.00 temperature=2.00
committee                    AP/AUC=0.4527/0.8086 F2@20%=0.5236 P/R=0.4217/0.5573
competence_council           AP/AUC=0.4696/0.8201 F2@20%=0.5370 P/R=0.4325/0.5716 ΔAP=+0.0168
[COMPETENCE] target top1 accuracy=0.367 | NON_DEPLOYABLE best-expert oracle AP=0.8728
  history0     n=2,145 AP 0.5482->0.5540 weights=[0.255 0.256 0.252 0.238]
  history1_2   n=1,772 AP 0.4357->0.4571 weights=[0.254 0.252 0.248 0.247]
  history3plus n=3,009 AP 0.3912->0.4112 weights=[0.253 0.251 0.249 0.247]

========================================================================================================================
SUITE4 ORIGINAL RANKING / EQUAL-BUDGET SUMMARY
========================================================================================================================
development NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.8647 mean-fold AP=0.8737 F2@20=0.5549
development committee                          pooled AP=0.3020 mean-fold AP=0.3207 F2@20=0.4139
development competence_council                 pooled AP=0.3202 mean-fold AP=0.3586 F2@20=0.4208
holdout     NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.8512 mean-fold AP=0.8481 F2@20=0.7756
holdout     committee                          pooled AP=0.4140 mean-fold AP=0.4126 F2@20=0.5286
holdout     competence_council                 pooled AP=0.4469 mean-fold AP=0.4449 F2@20=0.5442

### DEVELOPMENT [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    5222   3.62% |  0.2606  0.8010 |  0.5227  0.2434  0.3321  0.2725    1.69% |  0.1927  0.5556  0.2861  0.4035   10.44%
     2    4598   3.31% |  0.4767  0.9152 |  0.3307  0.5592  0.4156  0.4913    5.59% |  0.2506  0.6842  0.3668  0.5083    9.03%
     3    6277   3.47% |  0.3497  0.8692 |  0.1414  0.6376  0.2315  0.3747   15.66% |  0.2317  0.5505  0.3261  0.4317    8.25%
     4    5072   4.81% |  0.4144  0.9037 |  0.3403  0.5984  0.4339  0.5196    8.46% |  0.3394  0.6107  0.4363  0.5265    8.66%
     5    6398   5.49% |  0.2918  0.8347 |  0.2025  0.6467  0.3084  0.4495   17.52% |  0.2370  0.5185  0.3253  0.4190   12.00%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   27567   4.19% |  0.3202  0.8527 |  0.2234  0.5572  0.3189  0.4290   10.44% |  0.2458  0.5719  0.3438  0.4520    9.74%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    5651   9.96% |  0.4203  0.8460 |  0.3244  0.6874  0.4408  0.5617   21.11% |  0.3907  0.5364  0.4521  0.4992   13.68%

  Confusion matrix — fold 6 — fixed development threshold 0.199906
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,282             806
  Failure <5                           176             387
  ------------------------------------------------------------------
  Normal correctly rejected: 84.16%
  Failures successfully detected: 68.74%

  Confusion matrix — fold 6 — drift-aware rate 13.682%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,617             471
  Failure <5                           261             302
  ------------------------------------------------------------------
  Normal correctly rejected: 90.74%
  Failures successfully detected: 53.64%
     7    6926  15.13% |  0.4696  0.8201 |  0.3922  0.6508  0.4894  0.5749   25.11% |  0.3934  0.6460  0.4890  0.5725   24.85%

  Confusion matrix — fold 7 — fixed development threshold 0.199906
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,821           1,057
  Failure <5                           366             682
  ------------------------------------------------------------------
  Normal correctly rejected: 82.02%
  Failures successfully detected: 65.08%

  Confusion matrix — fold 7 — drift-aware rate 24.847%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,834           1,044
  Failure <5                           371             677
  ------------------------------------------------------------------
  Normal correctly rejected: 82.24%
  Failures successfully detected: 64.60%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   12577  12.81% |  0.4469  0.8329 |  0.3646  0.6636  0.4706  0.5701   23.31% |  0.3925  0.6077  0.4770  0.5477   19.83%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         9,103           1,863
  Failure <5                           542           1,069
  ------------------------------------------------------------------
  Normal correctly rejected: 83.01%
  Failures successfully detected: 66.36%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         9,451           1,515
  Failure <5                           632             979
  ------------------------------------------------------------------
  Normal correctly rejected: 86.18%
  Failures successfully detected: 60.77%

### DEVELOPMENT [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    5222   3.62% |  0.2032  0.7519 |  0.2834  0.2804  0.2819  0.2810    3.58% |  0.1505  0.4339  0.2234  0.3151   10.44%
     2    4598   3.31% |  0.4729  0.9206 |  0.3824  0.5987  0.4667  0.5378    5.18% |  0.2554  0.6974  0.3739  0.5181    9.03%
     3    6277   3.47% |  0.2738  0.8506 |  0.1500  0.6055  0.2404  0.3767   14.02% |  0.2089  0.4954  0.2939  0.3888    8.24%
     4    5072   4.81% |  0.3788  0.8880 |  0.2759  0.6434  0.3862  0.5081   11.22% |  0.3075  0.5533  0.3953  0.4770    8.66%
     5    6398   5.49% |  0.2751  0.8292 |  0.2074  0.5926  0.3072  0.4321   15.68% |  0.2282  0.4986  0.3131  0.4030   11.99%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   27567   4.19% |  0.3020  0.8457 |  0.2228  0.5555  0.3180  0.4277   10.44% |  0.2259  0.5251  0.3159  0.4151    9.73%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    5651   9.96% |  0.3724  0.8361 |  0.3070  0.6483  0.4167  0.5304   21.04% |  0.3648  0.5009  0.4222  0.4661   13.68%

  Confusion matrix — fold 6 — fixed development threshold 0.114276
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,264             824
  Failure <5                           198             365
  ------------------------------------------------------------------
  Normal correctly rejected: 83.81%
  Failures successfully detected: 64.83%

  Confusion matrix — fold 6 — drift-aware rate 13.677%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,597             491
  Failure <5                           281             282
  ------------------------------------------------------------------
  Normal correctly rejected: 90.35%
  Failures successfully detected: 50.09%
     7    6926  15.13% |  0.4527  0.8086 |  0.3688  0.6574  0.4726  0.5685   26.97% |  0.3837  0.6298  0.4769  0.5582   24.83%

  Confusion matrix — fold 7 — fixed development threshold 0.114276
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,699           1,179
  Failure <5                           359             689
  ------------------------------------------------------------------
  Normal correctly rejected: 79.94%
  Failures successfully detected: 65.74%

  Confusion matrix — fold 7 — drift-aware rate 24.838%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,818           1,060
  Failure <5                           388             660
  ------------------------------------------------------------------
  Normal correctly rejected: 81.97%
  Failures successfully detected: 62.98%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   12577  12.81% |  0.4140  0.8221 |  0.3448  0.6543  0.4516  0.5547   24.31% |  0.3779  0.5847  0.4591  0.5270   19.82%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         8,963           2,003
  Failure <5                           557           1,054
  ------------------------------------------------------------------
  Normal correctly rejected: 81.73%
  Failures successfully detected: 65.43%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         9,415           1,551
  Failure <5                           669             942
  ------------------------------------------------------------------
  Normal correctly rejected: 85.86%
  Failures successfully detected: 58.47%
