========================================================================================================================
SUITE4 EXPERIMENT 2 — COUNCIL OF SPECIALISTS + COMPETENCE MODEL
V2-COMPATIBLE COMPLETE OPERATIONAL EVALUATION ENABLED
========================================================================================================================

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=1
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0]
  block=54 n=1,108 AP q/global/recent/rel=0.7354/0.5910/0.6610/0.2779
  block=56 n=1,927 AP q/global/recent/rel=0.2164/0.2261/0.2078/0.2690
  block=57 n=1,680 AP q/global/recent/rel=0.6668/0.5027/0.5508/0.2490
  block=59 n=2,287 AP q/global/recent/rel=0.0765/0.0459/0.0387/0.0888
  block=60 n=2,017 AP q/global/recent/rel=0.3149/0.2332/0.1698/0.2865
[POLICY] alpha=0.25 temperature=2.00
committee                    AP/AUC=0.2598/0.7668 F2@20%=0.3421 P/R=0.1386/0.5405
competence_council           AP/AUC=0.2830/0.7746 F2@20%=0.3535 P/R=0.1432/0.5586 ΔAP=+0.0232
[COMPETENCE] target top1 accuracy=0.602 | NON_DEPLOYABLE best-expert oracle AP=1.0000
  history0     n=516 AP 0.3227->0.3294 weights=[0.267 0.266 0.262 0.205]
  history1_2   n=985 AP 0.1468->0.1762 weights=[0.264 0.266 0.263 0.207]
  history3plus n=665 AP 0.5339->0.5564 weights=[0.267 0.266 0.264 0.203]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=2
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0]
  block=54 n=1,108 AP q/global/recent/rel=0.7143/0.6089/0.5952/0.1779
  block=56 n=1,927 AP q/global/recent/rel=0.2387/0.1984/0.2188/0.3207
  block=57 n=1,680 AP q/global/recent/rel=0.6729/0.5145/0.5883/0.2715
  block=59 n=2,287 AP q/global/recent/rel=0.0946/0.0764/0.0608/0.0118
  block=60 n=2,017 AP q/global/recent/rel=0.3323/0.3297/0.1710/0.2630
  block=62 n=2,166 AP q/global/recent/rel=0.2868/0.1972/0.1551/0.0435
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.6839/0.9550 F2@20%=0.4717 P/R=0.1605/0.9155
competence_council           AP/AUC=0.6839/0.9550 F2@20%=0.4717 P/R=0.1605/0.9155 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.218 | NON_DEPLOYABLE best-expert oracle AP=0.9406
  history0     n=20 AP nan->nan weights=[0.296 0.267 0.238 0.199]
  history1_2   n=647 AP 0.8513->0.8513 weights=[0.267 0.253 0.247 0.233]
  history3plus n=1,357 AP 0.5083->0.5083 weights=[0.266 0.256 0.253 0.225]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=3
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0]
  block=54 n=1,108 AP q/global/recent/rel=0.7249/0.6719/0.6188/0.1638
  block=56 n=1,927 AP q/global/recent/rel=0.2359/0.3012/0.2271/0.1633
  block=57 n=1,680 AP q/global/recent/rel=0.6750/0.5864/0.5666/0.2740
  block=59 n=2,287 AP q/global/recent/rel=0.0860/0.0725/0.0562/0.1280
  block=60 n=2,017 AP q/global/recent/rel=0.3200/0.1847/0.1824/0.2094
  block=62 n=2,166 AP q/global/recent/rel=0.2695/0.2029/0.0928/0.1979
  block=63 n=2,024 AP q/global/recent/rel=0.6528/0.5456/0.3893/0.4252
[POLICY] alpha=0.25 temperature=0.50
committee                    AP/AUC=0.2073/0.8333 F2@20%=0.3067 P/R=0.1002/0.6324
competence_council           AP/AUC=0.2294/0.8595 F2@20%=0.2924 P/R=0.0956/0.6029 ΔAP=+0.0221
[COMPETENCE] target top1 accuracy=0.380 | NON_DEPLOYABLE best-expert oracle AP=0.9816
  history0     n=397 AP 0.3235->0.3868 weights=[0.318 0.28  0.266 0.135]
  history1_2   n=950 AP 0.2244->0.2434 weights=[0.281 0.251 0.23  0.238]
  history3plus n=798 AP 0.4861->0.5657 weights=[0.265 0.258 0.25  0.227]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=4
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0]
  block=54 n=1,108 AP q/global/recent/rel=0.7083/0.6562/0.6208/0.0240
  block=56 n=1,927 AP q/global/recent/rel=0.2443/0.2405/0.3117/0.2642
  block=57 n=1,680 AP q/global/recent/rel=0.6205/0.5239/0.5137/0.2555
  block=59 n=2,287 AP q/global/recent/rel=0.0852/0.0446/0.0226/0.1848
  block=60 n=2,017 AP q/global/recent/rel=0.2994/0.2518/0.2713/0.2399
  block=62 n=2,166 AP q/global/recent/rel=0.2680/0.2371/0.0841/0.2095
  block=63 n=2,024 AP q/global/recent/rel=0.6540/0.4870/0.4436/0.4216
  block=65 n=2,145 AP q/global/recent/rel=0.2001/0.1566/0.1517/0.1222
[POLICY] alpha=0.75 temperature=0.50
committee                    AP/AUC=0.3979/0.9106 F2@20%=0.3030 P/R=0.0836/0.8824
competence_council           AP/AUC=0.3846/0.9043 F2@20%=0.3030 P/R=0.0836/0.8824 ΔAP=-0.0133
[COMPETENCE] target top1 accuracy=0.506 | NON_DEPLOYABLE best-expert oracle AP=0.9146
  history1_2   n=494 AP 0.4542->0.4842 weights=[0.301 0.272 0.299 0.127]
  history3plus n=1,301 AP 0.4176->0.3681 weights=[0.271 0.262 0.268 0.199]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=5
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0]
  block=54 n=1,108 AP q/global/recent/rel=0.7220/0.6195/0.6318/0.2735
  block=56 n=1,927 AP q/global/recent/rel=0.2370/0.3155/0.2697/0.0210
  block=57 n=1,680 AP q/global/recent/rel=0.6882/0.5285/0.5458/0.2836
  block=59 n=2,287 AP q/global/recent/rel=0.0788/0.0625/0.0455/0.1037
  block=60 n=2,017 AP q/global/recent/rel=0.3284/0.2229/0.2434/0.2547
  block=62 n=2,166 AP q/global/recent/rel=0.2642/0.1678/0.1172/0.1959
  block=63 n=2,024 AP q/global/recent/rel=0.6484/0.4867/0.4384/0.4462
  block=65 n=2,145 AP q/global/recent/rel=0.2297/0.1879/0.1240/0.1694
  block=66 n=1,795 AP q/global/recent/rel=0.3904/0.3976/0.3808/0.2959
[POLICY] alpha=0.50 temperature=0.50
committee                    AP/AUC=0.2155/0.7906 F2@20%=0.3496 P/R=0.1244/0.6386
competence_council           AP/AUC=0.2174/0.7958 F2@20%=0.3232 P/R=0.1150/0.5904 ΔAP=+0.0019
[COMPETENCE] target top1 accuracy=0.619 | NON_DEPLOYABLE best-expert oracle AP=0.9175
  history0     n=501 AP 0.1226->0.1276 weights=[0.297 0.274 0.281 0.148]
  history1_2   n=728 AP 0.3912->0.3815 weights=[0.289 0.277 0.282 0.151]
  history3plus n=900 AP 0.0975->0.1010 weights=[0.3   0.285 0.279 0.136]

========================================================================================================================
DEVELOPMENT-SELECTED OPERATIONAL POLICIES [beta=2]
Thresholds and warning ratios below are frozen before holdout evaluation.
========================================================================================================================
  competence_council  : AP=0.3073 | AUC=0.8440 | F2-opt tau=0.142377 | P=0.2091 R=0.5014 F1=0.2951 F2=0.3918 | flag=8.58% | ratio=2.398
  committee           : AP=0.3093 | AUC=0.8328 | F2-opt tau=0.116308 | P=0.2092 R=0.4850 F1=0.2923 F2=0.3838 | flag=8.30% | ratio=2.319

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=6
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0]
  block=54 n=1,108 AP q/global/recent/rel=0.6927/0.6472/0.6504/0.1840
  block=56 n=1,927 AP q/global/recent/rel=0.2483/0.3232/0.2683/0.1283
  block=57 n=1,680 AP q/global/recent/rel=0.6708/0.5929/0.5732/0.2559
  block=59 n=2,287 AP q/global/recent/rel=0.0787/0.0406/0.0698/0.0815
  block=60 n=2,017 AP q/global/recent/rel=0.3200/0.2893/0.1556/0.2365
  block=62 n=2,166 AP q/global/recent/rel=0.2768/0.1240/0.1358/0.1577
  block=63 n=2,024 AP q/global/recent/rel=0.6693/0.6046/0.3986/0.5034
  block=65 n=2,145 AP q/global/recent/rel=0.1868/0.1321/0.1403/0.1599
  block=66 n=1,795 AP q/global/recent/rel=0.4178/0.3656/0.3978/0.2476
  block=68 n=2,129 AP q/global/recent/rel=0.2160/0.1942/0.1711/0.0957
[POLICY] alpha=0.25 temperature=1.00
committee                    AP/AUC=0.2286/0.7987 F2@20%=0.3841 P/R=0.1481/0.6383
competence_council           AP/AUC=0.2294/0.8132 F2@20%=0.3905 P/R=0.1506/0.6489 ΔAP=+0.0008
[COMPETENCE] target top1 accuracy=0.605 | NON_DEPLOYABLE best-expert oracle AP=0.6971
  history0     n=10 AP 0.3333->0.5000 weights=[0.276 0.232 0.242 0.25 ]
  history1_2   n=525 AP 0.2339->0.2348 weights=[0.274 0.244 0.242 0.24 ]
  history3plus n=1,491 AP 0.2375->0.2416 weights=[0.266 0.253 0.246 0.234]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=7
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0, 69.0]
  block=54 n=1,108 AP q/global/recent/rel=0.7291/0.5785/0.6603/0.3103
  block=56 n=1,927 AP q/global/recent/rel=0.2611/0.3178/0.2925/0.2170
  block=57 n=1,680 AP q/global/recent/rel=0.6752/0.5969/0.5556/0.2980
  block=59 n=2,287 AP q/global/recent/rel=0.0887/0.0442/0.0373/0.0948
  block=60 n=2,017 AP q/global/recent/rel=0.3365/0.2622/0.1808/0.2056
  block=62 n=2,166 AP q/global/recent/rel=0.2753/0.1711/0.1678/0.1464
  block=63 n=2,024 AP q/global/recent/rel=0.6787/0.4716/0.4397/0.4345
  block=65 n=2,145 AP q/global/recent/rel=0.2170/0.1229/0.1450/0.1746
  block=66 n=1,795 AP q/global/recent/rel=0.4082/0.3669/0.3214/0.2674
  block=68 n=2,129 AP q/global/recent/rel=0.2156/0.1849/0.1883/0.1011
  block=69 n=2,026 AP q/global/recent/rel=0.2324/0.2767/0.1974/0.2017
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.3767/0.8338 F2@20%=0.5043 P/R=0.3096/0.5984
competence_council           AP/AUC=0.3767/0.8338 F2@20%=0.5043 P/R=0.3096/0.5984 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.627 | NON_DEPLOYABLE best-expert oracle AP=0.9264
  history0     n=530 AP 0.3670->0.3670 weights=[0.319 0.272 0.216 0.193]
  history1_2   n=960 AP 0.4304->0.4304 weights=[0.293 0.261 0.227 0.218]
  history3plus n=963 AP 0.3032->0.3032 weights=[0.283 0.273 0.235 0.209]

========================================================================================================================
SUITE4 ORIGINAL RANKING / EQUAL-BUDGET SUMMARY
========================================================================================================================
development NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.9531 mean-fold AP=0.9509 F2@20=0.5048
development committee                          pooled AP=0.3093 mean-fold AP=0.3529 F2@20=0.3546
development competence_council                 pooled AP=0.3073 mean-fold AP=0.3597 F2@20=0.3488
holdout     NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.8287 mean-fold AP=0.8117 F2@20=0.6926
holdout     committee                          pooled AP=0.2573 mean-fold AP=0.3026 F2@20=0.4442
holdout     competence_council                 pooled AP=0.2532 mean-fold AP=0.3030 F2@20=0.4474

### DEVELOPMENT [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    2166   5.12% |  0.2830  0.7746 |  0.4561  0.2342  0.3095  0.2595    2.63% |  0.2527  0.4234  0.3165  0.3730    8.59%
     2    2024   3.51% |  0.6839  0.9550 |  0.4041  0.8310  0.5438  0.6860    7.21% |  0.2530  0.8873  0.3937  0.5910   12.30%
     3    2145   3.17% |  0.2294  0.8595 |  0.1319  0.5588  0.2135  0.3393   13.43% |  0.1611  0.4265  0.2339  0.3208    8.39%
     4    1795   1.89% |  0.3846  0.9043 |  0.1515  0.7353  0.2513  0.4153    9.19% |  0.1765  0.7059  0.2824  0.4412    7.58%
     5    2129   3.90% |  0.2174  0.7958 |  0.1607  0.4337  0.2345  0.3237   10.52% |  0.2165  0.2530  0.2333  0.2448    4.56%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   10259   3.58% |  0.3073  0.8440 |  0.2091  0.5014  0.2951  0.3918    8.58% |  0.2170  0.5014  0.3029  0.3972    8.27%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    2026   4.64% |  0.2294  0.8132 |  0.1711  0.6170  0.2679  0.4056   16.73% |  0.2381  0.4787  0.3180  0.3982    9.33%

  Confusion matrix — fold 6 — fixed development threshold 0.142377
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,651             281
  Failure <5                            36              58
  ------------------------------------------------------------------
  Normal correctly rejected: 85.46%
  Failures successfully detected: 61.70%

  Confusion matrix — fold 6 — drift-aware rate 9.348%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,788             144
  Failure <5                            49              45
  ------------------------------------------------------------------
  Normal correctly rejected: 92.55%
  Failures successfully detected: 47.87%
     7    2453  10.35% |  0.3767  0.8338 |  0.3887  0.4606  0.4216  0.4442   12.27% |  0.4103  0.4409  0.4250  0.4344   11.13%

  Confusion matrix — fold 7 — fixed development threshold 0.142377
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,015             184
  Failure <5                           137             117
  ------------------------------------------------------------------
  Normal correctly rejected: 91.63%
  Failures successfully detected: 46.06%

  Confusion matrix — fold 7 — drift-aware rate 11.125%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,038             161
  Failure <5                           142             112
  ------------------------------------------------------------------
  Normal correctly rejected: 92.68%
  Failures successfully detected: 44.09%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    4479   7.77% |  0.2532  0.8229 |  0.2734  0.5029  0.3543  0.4306   14.29% |  0.3398  0.4511  0.3877  0.4234   10.31%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,666             465
  Failure <5                           173             175
  ------------------------------------------------------------------
  Normal correctly rejected: 88.74%
  Failures successfully detected: 50.29%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,826             305
  Failure <5                           191             157
  ------------------------------------------------------------------
  Normal correctly rejected: 92.62%
  Failures successfully detected: 45.11%

### DEVELOPMENT [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    2166   5.12% |  0.2598  0.7668 |  0.4333  0.2342  0.3041  0.2579    2.77% |  0.2500  0.4054  0.3093  0.3606    8.31%
     2    2024   3.51% |  0.6839  0.9550 |  0.3856  0.8310  0.5268  0.6751    7.56% |  0.2614  0.8873  0.4038  0.6000   11.91%
     3    2145   3.17% |  0.2073  0.8333 |  0.1061  0.5588  0.1784  0.3016   16.69% |  0.1494  0.3824  0.2149  0.2915    8.11%
     4    1795   1.89% |  0.3979  0.9106 |  0.1967  0.7059  0.3077  0.4651    6.80% |  0.1818  0.7059  0.2892  0.4478    7.35%
     5    2129   3.90% |  0.2155  0.7906 |  0.1962  0.3735  0.2573  0.3163    7.42% |  0.2021  0.2289  0.2147  0.2230    4.42%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   10259   3.58% |  0.3093  0.8328 |  0.2092  0.4850  0.2923  0.3838    8.30% |  0.2156  0.4823  0.2980  0.3866    8.00%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    2026   4.64% |  0.2286  0.7987 |  0.1713  0.5957  0.2660  0.3983   16.14% |  0.2350  0.4574  0.3105  0.3846    9.03%

  Confusion matrix — fold 6 — fixed development threshold 0.116308
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,661             271
  Failure <5                            38              56
  ------------------------------------------------------------------
  Normal correctly rejected: 85.97%
  Failures successfully detected: 59.57%

  Confusion matrix — fold 6 — drift-aware rate 9.040%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,792             140
  Failure <5                            51              43
  ------------------------------------------------------------------
  Normal correctly rejected: 92.75%
  Failures successfully detected: 45.74%
     7    2453  10.35% |  0.3767  0.8338 |  0.3213  0.5591  0.4080  0.4870   18.02% |  0.4167  0.4331  0.4247  0.4297   10.76%

  Confusion matrix — fold 7 — fixed development threshold 0.116308
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,899             300
  Failure <5                           112             142
  ------------------------------------------------------------------
  Normal correctly rejected: 86.36%
  Failures successfully detected: 55.91%

  Confusion matrix — fold 7 — drift-aware rate 10.759%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,045             154
  Failure <5                           144             110
  ------------------------------------------------------------------
  Normal correctly rejected: 93.00%
  Failures successfully detected: 43.31%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    4479   7.77% |  0.2573  0.8202 |  0.2575  0.5690  0.3545  0.4581   17.17% |  0.3423  0.4397  0.3849  0.4160    9.98%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,560             571
  Failure <5                           150             198
  ------------------------------------------------------------------
  Normal correctly rejected: 86.18%
  Failures successfully detected: 56.90%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,837             294
  Failure <5                           195             153
  ------------------------------------------------------------------
  Normal correctly rejected: 92.88%
  Failures successfully detected: 43.97%
