========================================================================================================================
SUITE4 EXPERIMENT 2 — COUNCIL OF SPECIALISTS + COMPETENCE MODEL
V2-COMPATIBLE COMPLETE OPERATIONAL EVALUATION ENABLED
========================================================================================================================

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=1
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0]
  block=54 n=1,216 AP q/global/recent/rel=0.1422/0.1468/0.1157/0.0752
  block=56 n=1,871 AP q/global/recent/rel=0.0868/0.0319/0.0267/0.1623
  block=57 n=1,836 AP q/global/recent/rel=0.0660/0.0607/0.0426/0.0191
  block=59 n=2,282 AP q/global/recent/rel=0.3720/0.0484/0.0420/0.1295
  block=60 n=1,984 AP q/global/recent/rel=0.3807/0.2781/0.1421/0.2010
[POLICY] alpha=0.25 temperature=2.00
committee                    AP/AUC=0.2622/0.8663 F2@20%=0.3103 P/R=0.0921/0.7609
competence_council           AP/AUC=0.2610/0.8724 F2@20%=0.3103 P/R=0.0921/0.7609 ΔAP=-0.0012
[COMPETENCE] target top1 accuracy=0.833 | NON_DEPLOYABLE best-expert oracle AP=0.9987
  history0     n=664 AP 0.2452->0.2091 weights=[0.265 0.259 0.257 0.218]
  history1_2   n=696 AP 0.5509->0.5689 weights=[0.265 0.259 0.259 0.218]
  history3plus n=540 AP 0.1752->0.1802 weights=[0.264 0.262 0.262 0.213]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=2
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0]
  block=54 n=1,216 AP q/global/recent/rel=0.1506/0.1466/0.1281/0.0959
  block=56 n=1,871 AP q/global/recent/rel=0.0928/0.0831/0.0777/0.1909
  block=57 n=1,836 AP q/global/recent/rel=0.0702/0.0536/0.0446/0.0241
  block=59 n=2,282 AP q/global/recent/rel=0.3720/0.0311/0.3256/0.2093
  block=60 n=1,984 AP q/global/recent/rel=0.3759/0.3065/0.3199/0.2272
  block=62 n=1,900 AP q/global/recent/rel=0.2579/0.2086/0.1877/0.1605
[POLICY] alpha=0.25 temperature=2.00
committee                    AP/AUC=0.3930/0.8337 F2@20%=0.2876 P/R=0.0897/0.6415
competence_council           AP/AUC=0.4145/0.8228 F2@20%=0.2961 P/R=0.0923/0.6604 ΔAP=+0.0214
[COMPETENCE] target top1 accuracy=0.459 | NON_DEPLOYABLE best-expert oracle AP=0.8707
  history0     n=10 AP nan->nan weights=[0.265 0.26  0.26  0.215]
  history1_2   n=643 AP 0.4582->0.4821 weights=[0.259 0.259 0.255 0.227]
  history3plus n=1,240 AP 0.2801->0.2910 weights=[0.258 0.259 0.258 0.226]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=3
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0]
  block=54 n=1,216 AP q/global/recent/rel=0.1548/0.1471/0.1295/0.0941
  block=56 n=1,871 AP q/global/recent/rel=0.0794/0.0468/0.0438/0.1617
  block=57 n=1,836 AP q/global/recent/rel=0.0933/0.0440/0.0539/0.0585
  block=59 n=2,282 AP q/global/recent/rel=0.3720/0.1957/0.3667/0.1337
  block=60 n=1,984 AP q/global/recent/rel=0.3703/0.1796/0.3850/0.2662
  block=62 n=1,900 AP q/global/recent/rel=0.2624/0.1716/0.1737/0.1853
  block=63 n=1,893 AP q/global/recent/rel=0.3566/0.3360/0.3326/0.2251
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.2476/0.8529 F2@20%=0.3158 P/R=0.0942/0.7660
competence_council           AP/AUC=0.2476/0.8529 F2@20%=0.3158 P/R=0.0942/0.7660 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.290 | NON_DEPLOYABLE best-expert oracle AP=0.9148
  history0     n=443 AP 0.0695->0.0695 weights=[0.257 0.282 0.278 0.183]
  history1_2   n=794 AP 0.3224->0.3224 weights=[0.272 0.242 0.26  0.226]
  history3plus n=672 AP 0.4601->0.4601 weights=[0.255 0.249 0.266 0.23 ]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=4
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0]
  block=54 n=1,216 AP q/global/recent/rel=0.1496/0.1133/0.1758/0.0885
  block=56 n=1,871 AP q/global/recent/rel=0.0894/0.0708/0.0587/0.0875
  block=57 n=1,836 AP q/global/recent/rel=0.0683/0.0107/0.0196/0.0150
  block=59 n=2,282 AP q/global/recent/rel=0.3658/0.0378/0.0958/0.1041
  block=60 n=1,984 AP q/global/recent/rel=0.3883/0.3507/0.3498/0.2365
  block=62 n=1,900 AP q/global/recent/rel=0.2788/0.1576/0.3003/0.1683
  block=63 n=1,893 AP q/global/recent/rel=0.4308/0.4091/0.3452/0.2633
  block=65 n=1,909 AP q/global/recent/rel=0.3302/0.2660/0.1837/0.1929
[POLICY] alpha=0.50 temperature=2.00
committee                    AP/AUC=0.4463/0.9197 F2@20%=0.3714 P/R=0.1117/0.8864
competence_council           AP/AUC=0.4506/0.9394 F2@20%=0.3905 P/R=0.1175/0.9318 ΔAP=+0.0043
[COMPETENCE] target top1 accuracy=0.786 | NON_DEPLOYABLE best-expert oracle AP=0.9695
  history0     n=8 AP nan->nan weights=[0.266 0.264 0.268 0.202]
  history1_2   n=454 AP 0.3903->0.3800 weights=[0.27  0.259 0.269 0.201]
  history3plus n=1,284 AP 0.4579->0.4838 weights=[0.273 0.263 0.268 0.196]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=5
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0]
  block=54 n=1,216 AP q/global/recent/rel=0.1435/0.1635/0.1126/0.0436
  block=56 n=1,871 AP q/global/recent/rel=0.0922/0.0636/0.0731/0.1908
  block=57 n=1,836 AP q/global/recent/rel=0.0864/0.2292/0.0425/0.0144
  block=59 n=2,282 AP q/global/recent/rel=0.3628/0.0551/0.2458/0.1314
  block=60 n=1,984 AP q/global/recent/rel=0.3909/0.3228/0.3600/0.2165
  block=62 n=1,900 AP q/global/recent/rel=0.2690/0.1644/0.1899/0.2093
  block=63 n=1,893 AP q/global/recent/rel=0.3762/0.4067/0.3150/0.2579
  block=65 n=1,909 AP q/global/recent/rel=0.3109/0.1768/0.2764/0.2725
  block=66 n=1,746 AP q/global/recent/rel=0.4967/0.4354/0.5164/0.2034
[POLICY] alpha=0.50 temperature=2.00
committee                    AP/AUC=0.1602/0.8684 F2@20%=0.3487 P/R=0.1114/0.7458
competence_council           AP/AUC=0.1676/0.8853 F2@20%=0.3962 P/R=0.1266/0.8475 ΔAP=+0.0074
[COMPETENCE] target top1 accuracy=0.565 | NON_DEPLOYABLE best-expert oracle AP=0.9953
  history0     n=687 AP 0.3318->0.3672 weights=[0.27  0.262 0.265 0.204]
  history1_2   n=610 AP 0.1597->0.1543 weights=[0.271 0.268 0.262 0.199]
  history3plus n=678 AP 0.0733->0.0700 weights=[0.269 0.264 0.265 0.202]

========================================================================================================================
DEVELOPMENT-SELECTED OPERATIONAL POLICIES [beta=2]
Thresholds and warning ratios below are frozen before holdout evaluation.
========================================================================================================================
  competence_council  : AP=0.2857 | AUC=0.8302 | F2-opt tau=0.184379 | P=0.2255 R=0.4980 F1=0.3104 F2=0.4010 | flag=5.84% | ratio=2.209
  committee           : AP=0.2845 | AUC=0.8455 | F2-opt tau=0.121594 | P=0.2251 R=0.5181 F1=0.3139 F2=0.4111 | flag=6.08% | ratio=2.301

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=6
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0]
  block=54 n=1,216 AP q/global/recent/rel=0.1499/0.1259/0.1717/0.1350
  block=56 n=1,871 AP q/global/recent/rel=0.1013/0.0777/0.0619/0.0707
  block=57 n=1,836 AP q/global/recent/rel=0.0757/0.0113/0.0130/0.0103
  block=59 n=2,282 AP q/global/recent/rel=0.3756/0.0455/0.0970/0.1752
  block=60 n=1,984 AP q/global/recent/rel=0.3905/0.2750/0.3478/0.2831
  block=62 n=1,900 AP q/global/recent/rel=0.2572/0.2111/0.1666/0.1852
  block=63 n=1,893 AP q/global/recent/rel=0.3916/0.3780/0.2394/0.1163
  block=65 n=1,909 AP q/global/recent/rel=0.2672/0.2584/0.1837/0.2104
  block=66 n=1,746 AP q/global/recent/rel=0.4711/0.4009/0.4791/0.2037
  block=68 n=1,975 AP q/global/recent/rel=0.1564/0.1409/0.0894/0.1936
[POLICY] alpha=1.00 temperature=1.00
committee                    AP/AUC=0.4558/0.9230 F2@20%=0.5185 P/R=0.1984/0.8690
competence_council           AP/AUC=0.4457/0.9053 F2@20%=0.5185 P/R=0.1984/0.8690 ΔAP=-0.0102
[COMPETENCE] target top1 accuracy=0.806 | NON_DEPLOYABLE best-expert oracle AP=0.9004
  history0     n=14 AP 0.1667->0.1667 weights=[0.331 0.253 0.221 0.194]
  history1_2   n=590 AP 0.6056->0.5473 weights=[0.278 0.253 0.267 0.202]
  history3plus n=1,235 AP 0.3629->0.4445 weights=[0.268 0.255 0.253 0.224]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=7
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0, 69.0]
  block=54 n=1,216 AP q/global/recent/rel=0.1497/0.0980/0.1066/0.1729
  block=56 n=1,871 AP q/global/recent/rel=0.0853/0.0412/0.0677/0.1129
  block=57 n=1,836 AP q/global/recent/rel=0.0787/0.0607/0.1435/0.0361
  block=59 n=2,282 AP q/global/recent/rel=0.3694/0.0503/0.0476/0.1286
  block=60 n=1,984 AP q/global/recent/rel=0.3794/0.3628/0.3582/0.1379
  block=62 n=1,900 AP q/global/recent/rel=0.2594/0.2021/0.1660/0.1666
  block=63 n=1,893 AP q/global/recent/rel=0.3792/0.4191/0.3181/0.3094
  block=65 n=1,909 AP q/global/recent/rel=0.3085/0.1785/0.2201/0.2033
  block=66 n=1,746 AP q/global/recent/rel=0.4512/0.3862/0.4317/0.2191
  block=68 n=1,975 AP q/global/recent/rel=0.1523/0.1447/0.0915/0.2103
  block=69 n=1,839 AP q/global/recent/rel=0.4518/0.3964/0.3253/0.4017
[POLICY] alpha=0.25 temperature=2.00
committee                    AP/AUC=0.2956/0.8739 F2@20%=0.4717 P/R=0.1860/0.7658
competence_council           AP/AUC=0.3134/0.8815 F2@20%=0.4828 P/R=0.1904/0.7838 ΔAP=+0.0178
[COMPETENCE] target top1 accuracy=0.656 | NON_DEPLOYABLE best-expert oracle AP=0.8942
  history0     n=656 AP 0.2207->0.2198 weights=[0.261 0.257 0.255 0.227]
  history1_2   n=832 AP 0.3631->0.3854 weights=[0.259 0.253 0.249 0.238]
  history3plus n=796 AP 0.3188->0.3499 weights=[0.26  0.252 0.243 0.244]

========================================================================================================================
SUITE4 ORIGINAL RANKING / EQUAL-BUDGET SUMMARY
========================================================================================================================
development NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.9200 mean-fold AP=0.9498 F2@20=0.4310
development committee                          pooled AP=0.2845 mean-fold AP=0.3019 F2@20=0.3268
development competence_council                 pooled AP=0.2857 mean-fold AP=0.3082 F2@20=0.3418
holdout     NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.8811 mean-fold AP=0.8973 F2@20=0.6035
holdout     committee                          pooled AP=0.3638 mean-fold AP=0.3757 F2@20=0.4951
holdout     competence_council                 pooled AP=0.3526 mean-fold AP=0.3795 F2@20=0.5006

### DEVELOPMENT [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    1900   2.42% |  0.2610  0.8724 |  0.3889  0.3043  0.3415  0.3182    1.89% |  0.2252  0.5435  0.3185  0.4237    5.84%
     2    1893   2.80% |  0.4145  0.8228 |  0.3968  0.4717  0.4310  0.4545    3.33% |  0.2673  0.5094  0.3506  0.4313    5.34%
     3    1909   2.46% |  0.2476  0.8529 |  0.2330  0.5106  0.3200  0.4124    5.40% |  0.2119  0.5319  0.3030  0.4085    6.18%
     4    1746   2.52% |  0.4506  0.9394 |  0.1916  0.7273  0.3033  0.4665    9.56% |  0.2947  0.6364  0.4029  0.5166    5.44%
     5    1975   2.99% |  0.1676  0.8853 |  0.1602  0.4915  0.2417  0.3477    9.16% |  0.1909  0.3559  0.2485  0.3035    5.57%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    9423   2.64% |  0.2857  0.8302 |  0.2255  0.4980  0.3104  0.4010    5.84% |  0.2355  0.5060  0.3214  0.4115    5.68%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    1839   4.57% |  0.4457  0.9053 |  0.2491  0.8333  0.3836  0.5673   15.28% |  0.3884  0.5595  0.4585  0.5142    6.58%

  Confusion matrix — fold 6 — fixed development threshold 0.184379
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,544             211
  Failure <5                            14              70
  ------------------------------------------------------------------
  Normal correctly rejected: 87.98%
  Failures successfully detected: 83.33%

  Confusion matrix — fold 6 — drift-aware rate 6.599%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,681              74
  Failure <5                            37              47
  ------------------------------------------------------------------
  Normal correctly rejected: 95.78%
  Failures successfully detected: 55.95%
     7    2284   4.86% |  0.3134  0.8815 |  0.2941  0.4505  0.3559  0.4072    7.44% |  0.2652  0.5495  0.3578  0.4525   10.07%

  Confusion matrix — fold 7 — fixed development threshold 0.184379
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,053             120
  Failure <5                            61              50
  ------------------------------------------------------------------
  Normal correctly rejected: 94.48%
  Failures successfully detected: 45.05%

  Confusion matrix — fold 7 — drift-aware rate 10.089%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,004             169
  Failure <5                            50              61
  ------------------------------------------------------------------
  Normal correctly rejected: 92.22%
  Failures successfully detected: 54.95%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    4123   4.73% |  0.3526  0.8814 |  0.2661  0.6154  0.3715  0.4874   10.94% |  0.3077  0.5538  0.3956  0.4775    8.51%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,597             331
  Failure <5                            75             120
  ------------------------------------------------------------------
  Normal correctly rejected: 91.57%
  Failures successfully detected: 61.54%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,685             243
  Failure <5                            87             108
  ------------------------------------------------------------------
  Normal correctly rejected: 93.81%
  Failures successfully detected: 55.38%

### DEVELOPMENT [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    1900   2.42% |  0.2622  0.8663 |  0.3333  0.3261  0.3297  0.3275    2.37% |  0.2328  0.5870  0.3333  0.4500    6.11%
     2    1893   2.80% |  0.3930  0.8337 |  0.3968  0.4717  0.4310  0.4545    3.33% |  0.2667  0.5283  0.3544  0.4416    5.55%
     3    1909   2.46% |  0.2476  0.8529 |  0.1782  0.6596  0.2805  0.4282    9.11% |  0.2276  0.5957  0.3294  0.4502    6.44%
     4    1746   2.52% |  0.4463  0.9197 |  0.2177  0.7273  0.3351  0.4954    8.42% |  0.2828  0.6364  0.3916  0.5091    5.67%
     5    1975   2.99% |  0.1602  0.8684 |  0.1806  0.4407  0.2562  0.3421    7.29% |  0.1913  0.3729  0.2529  0.3134    5.82%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    9423   2.64% |  0.2845  0.8455 |  0.2251  0.5181  0.3139  0.4111    6.08% |  0.2384  0.5341  0.3296  0.4279    5.92%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    1839   4.57% |  0.4558  0.9230 |  0.3107  0.6548  0.4215  0.5361    9.62% |  0.3810  0.5714  0.4571  0.5195    6.85%

  Confusion matrix — fold 6 — fixed development threshold 0.121594
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,633             122
  Failure <5                            29              55
  ------------------------------------------------------------------
  Normal correctly rejected: 93.05%
  Failures successfully detected: 65.48%

  Confusion matrix — fold 6 — drift-aware rate 6.874%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,677              78
  Failure <5                            36              48
  ------------------------------------------------------------------
  Normal correctly rejected: 95.56%
  Failures successfully detected: 57.14%
     7    2284   4.86% |  0.2956  0.8739 |  0.2400  0.5405  0.3324  0.4323   10.95% |  0.2458  0.5315  0.3362  0.4313   10.51%

  Confusion matrix — fold 7 — fixed development threshold 0.121594
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,983             190
  Failure <5                            51              60
  ------------------------------------------------------------------
  Normal correctly rejected: 91.26%
  Failures successfully detected: 54.05%

  Confusion matrix — fold 7 — drift-aware rate 10.511%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,992             181
  Failure <5                            52              59
  ------------------------------------------------------------------
  Normal correctly rejected: 91.67%
  Failures successfully detected: 53.15%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    4123   4.73% |  0.3638  0.8946 |  0.2693  0.5897  0.3698  0.4764   10.36% |  0.2923  0.5487  0.3815  0.4668    8.88%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,616             312
  Failure <5                            80             115
  ------------------------------------------------------------------
  Normal correctly rejected: 92.06%
  Failures successfully detected: 58.97%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,669             259
  Failure <5                            88             107
  ------------------------------------------------------------------
  Normal correctly rejected: 93.41%
  Failures successfully detected: 54.87%
