========================================================================================================================
SUITE4 EXPERIMENT 2 — COUNCIL OF SPECIALISTS + COMPETENCE MODEL
V2-COMPATIBLE COMPLETE OPERATIONAL EVALUATION ENABLED
========================================================================================================================

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=1
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0]
  block=54 n=1,355 AP q/global/recent/rel=0.6845/0.6771/0.6481/0.0840
  block=56 n=1,841 AP q/global/recent/rel=0.0220/0.0680/0.0029/0.0323
  block=57 n=1,641 AP q/global/recent/rel=0.6693/0.6220/0.7389/0.1794
  block=59 n=2,231 AP q/global/recent/rel=0.1075/0.0168/0.3087/0.0089
  block=60 n=1,690 AP q/global/recent/rel=0.3264/0.2459/0.2868/0.0432
[POLICY] alpha=1.00 temperature=2.00
committee                    AP/AUC=0.0704/0.8048 F2@20%=0.1843 P/R=0.0465/0.7097
competence_council           AP/AUC=0.0486/0.7822 F2@20%=0.1759 P/R=0.0444/0.6774 ΔAP=-0.0218
[COMPETENCE] target top1 accuracy=0.670 | NON_DEPLOYABLE best-expert oracle AP=1.0000
  history0     n=982 AP 0.0404->0.0310 weights=[0.264 0.264 0.261 0.21 ]
  history1_2   n=917 AP 0.0983->0.0713 weights=[0.269 0.27  0.263 0.198]
  history3plus n=464 AP 0.1862->0.1925 weights=[0.271 0.272 0.264 0.193]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=2
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0]
  block=54 n=1,355 AP q/global/recent/rel=0.6850/0.5258/0.6663/0.0796
  block=56 n=1,841 AP q/global/recent/rel=0.0239/0.0139/0.0193/0.0266
  block=57 n=1,641 AP q/global/recent/rel=0.6482/0.6270/0.3768/0.1589
  block=59 n=2,231 AP q/global/recent/rel=0.1101/0.0250/0.2150/0.1078
  block=60 n=1,690 AP q/global/recent/rel=0.3233/0.3333/0.3161/0.2341
  block=62 n=2,363 AP q/global/recent/rel=0.0778/0.0540/0.0464/0.0635
[POLICY] alpha=0.25 temperature=2.00
committee                    AP/AUC=0.4660/0.9595 F2@20%=0.1818 P/R=0.0430/0.9474
competence_council           AP/AUC=0.4315/0.9589 F2@20%=0.1818 P/R=0.0430/0.9474 ΔAP=-0.0346
[COMPETENCE] target top1 accuracy=0.698 | NON_DEPLOYABLE best-expert oracle AP=0.8887
  history0     n=5 AP nan->nan weights=[0.264 0.283 0.264 0.188]
  history1_2   n=961 AP 0.3911->0.3424 weights=[0.259 0.258 0.246 0.237]
  history3plus n=1,129 AP 0.6491->0.6494 weights=[0.258 0.256 0.25  0.236]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=3
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0]
  block=54 n=1,355 AP q/global/recent/rel=0.7021/0.2658/0.0469/0.2953
  block=56 n=1,841 AP q/global/recent/rel=0.0226/0.0097/0.0047/0.0219
  block=57 n=1,641 AP q/global/recent/rel=0.6447/0.5909/0.5540/0.4021
  block=59 n=2,231 AP q/global/recent/rel=0.1430/0.0383/0.1597/0.1061
  block=60 n=1,690 AP q/global/recent/rel=0.3307/0.2711/0.2861/0.1795
  block=62 n=2,363 AP q/global/recent/rel=0.0631/0.0317/0.0423/0.0836
  block=63 n=2,095 AP q/global/recent/rel=0.4879/0.1517/0.2335/0.0968
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.1991/0.8143 F2@20%=0.1256 P/R=0.0300/0.6154
competence_council           AP/AUC=0.1991/0.8143 F2@20%=0.1256 P/R=0.0300/0.6154 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.700 | NON_DEPLOYABLE best-expert oracle AP=0.9940
  history0     n=942 AP 0.0172->0.0172 weights=[0.301 0.284 0.281 0.134]
  history1_2   n=1,102 AP 0.3655->0.3655 weights=[0.296 0.282 0.278 0.144]
  history3plus n=621 AP 0.4925->0.4925 weights=[0.299 0.283 0.276 0.142]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=4
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0]
  block=54 n=1,355 AP q/global/recent/rel=0.6847/0.0052/0.4855/0.0927
  block=56 n=1,841 AP q/global/recent/rel=0.0199/0.0187/0.0293/0.0038
  block=57 n=1,641 AP q/global/recent/rel=0.6479/0.5431/0.6312/0.1569
  block=59 n=2,231 AP q/global/recent/rel=0.1236/0.0765/0.2230/0.1629
  block=60 n=1,690 AP q/global/recent/rel=0.3310/0.3584/0.2708/0.2315
  block=62 n=2,363 AP q/global/recent/rel=0.0704/0.0303/0.0316/0.1180
  block=63 n=2,095 AP q/global/recent/rel=0.4505/0.3633/0.3256/0.0677
  block=65 n=2,665 AP q/global/recent/rel=0.1468/0.1876/0.3101/0.1268
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.0971/0.8807 F2@20%=0.2500 P/R=0.0659/0.8286
competence_council           AP/AUC=0.0971/0.8807 F2@20%=0.2500 P/R=0.0659/0.8286 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.752 | NON_DEPLOYABLE best-expert oracle AP=0.8306
  history0     n=16 AP 0.7636->0.7636 weights=[0.399 0.2   0.281 0.119]
  history1_2   n=849 AP 0.0815->0.0815 weights=[0.308 0.297 0.222 0.173]
  history3plus n=1,337 AP 0.1418->0.1418 weights=[0.301 0.278 0.228 0.194]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=5
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0]
  block=54 n=1,355 AP q/global/recent/rel=0.6854/0.6189/0.4884/0.2178
  block=56 n=1,841 AP q/global/recent/rel=0.0226/0.0265/0.0233/0.0078
  block=57 n=1,641 AP q/global/recent/rel=0.6708/0.6784/0.6887/0.1408
  block=59 n=2,231 AP q/global/recent/rel=0.1514/0.0186/0.2695/0.1448
  block=60 n=1,690 AP q/global/recent/rel=0.3304/0.1045/0.3015/0.1993
  block=62 n=2,363 AP q/global/recent/rel=0.0708/0.0374/0.0448/0.0967
  block=63 n=2,095 AP q/global/recent/rel=0.4778/0.2889/0.4050/0.1297
  block=65 n=2,665 AP q/global/recent/rel=0.1598/0.1814/0.1408/0.1093
  block=66 n=2,202 AP q/global/recent/rel=0.0979/0.0749/0.0681/0.0954
[POLICY] alpha=1.00 temperature=2.00
committee                    AP/AUC=0.1635/0.8346 F2@20%=0.2412 P/R=0.0692/0.6379
competence_council           AP/AUC=0.1768/0.8253 F2@20%=0.2347 P/R=0.0673/0.6207 ΔAP=+0.0133
[COMPETENCE] target top1 accuracy=0.828 | NON_DEPLOYABLE best-expert oracle AP=0.9370
  history0     n=1,016 AP 0.2011->0.2181 weights=[0.265 0.256 0.254 0.225]
  history1_2   n=905 AP 0.2126->0.2417 weights=[0.27  0.255 0.255 0.221]
  history3plus n=753 AP 0.2816->0.3399 weights=[0.266 0.253 0.251 0.23 ]

========================================================================================================================
DEVELOPMENT-SELECTED OPERATIONAL POLICIES [beta=2]
Thresholds and warning ratios below are frozen before holdout evaluation.
========================================================================================================================
  competence_council  : AP=0.1358 | AUC=0.7972 | F2-opt tau=0.330850 | P=0.1654 R=0.2485 F1=0.1986 F2=0.2258 | flag=2.12% | ratio=1.503
  committee           : AP=0.1547 | AUC=0.8498 | F2-opt tau=0.091636 | P=0.1540 R=0.3846 F1=0.2200 F2=0.2960 | flag=3.52% | ratio=2.497

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=6
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0]
  block=54 n=1,355 AP q/global/recent/rel=0.6859/0.6568/0.3065/0.0806
  block=56 n=1,841 AP q/global/recent/rel=0.0266/0.0260/0.0731/0.0567
  block=57 n=1,641 AP q/global/recent/rel=0.6598/0.5731/0.6533/0.0486
  block=59 n=2,231 AP q/global/recent/rel=0.1194/0.0357/0.2541/0.0700
  block=60 n=1,690 AP q/global/recent/rel=0.3447/0.2795/0.3051/0.1940
  block=62 n=2,363 AP q/global/recent/rel=0.0660/0.0118/0.0370/0.1053
  block=63 n=2,095 AP q/global/recent/rel=0.4693/0.2532/0.2773/0.1356
  block=65 n=2,665 AP q/global/recent/rel=0.1579/0.1443/0.2070/0.1796
  block=66 n=2,202 AP q/global/recent/rel=0.0916/0.0882/0.0738/0.1065
  block=68 n=2,674 AP q/global/recent/rel=0.1756/0.1249/0.0959/0.2344
[POLICY] alpha=0.75 temperature=0.50
committee                    AP/AUC=0.2778/0.8239 F2@20%=0.2729 P/R=0.0811/0.6667
competence_council           AP/AUC=0.2698/0.8408 F2@20%=0.2729 P/R=0.0811/0.6667 ΔAP=-0.0081
[COMPETENCE] target top1 accuracy=0.789 | NON_DEPLOYABLE best-expert oracle AP=0.8395
  history1_2   n=902 AP 0.2082->0.2075 weights=[0.31  0.235 0.234 0.222]
  history3plus n=1,564 AP 0.3415->0.3288 weights=[0.28  0.244 0.247 0.228]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=7
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0, 69.0]
  block=54 n=1,355 AP q/global/recent/rel=0.6859/0.3841/0.6529/0.1502
  block=56 n=1,841 AP q/global/recent/rel=0.0219/0.0807/0.0183/0.0245
  block=57 n=1,641 AP q/global/recent/rel=0.6667/0.5526/0.5074/0.0652
  block=59 n=2,231 AP q/global/recent/rel=0.1437/0.0937/0.1924/0.0210
  block=60 n=1,690 AP q/global/recent/rel=0.3038/0.2962/0.2458/0.2045
  block=62 n=2,363 AP q/global/recent/rel=0.0623/0.0139/0.0588/0.0435
  block=63 n=2,095 AP q/global/recent/rel=0.4186/0.3334/0.2841/0.3173
  block=65 n=2,665 AP q/global/recent/rel=0.1524/0.2121/0.3005/0.1319
  block=66 n=2,202 AP q/global/recent/rel=0.0996/0.1138/0.0520/0.0745
  block=68 n=2,674 AP q/global/recent/rel=0.1891/0.1477/0.1115/0.2196
  block=69 n=2,466 AP q/global/recent/rel=0.2767/0.1848/0.1357/0.1339
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.2446/0.8356 F2@20%=0.3857 P/R=0.1488/0.6406
competence_council           AP/AUC=0.2446/0.8356 F2@20%=0.3857 P/R=0.1488/0.6406 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.777 | NON_DEPLOYABLE best-expert oracle AP=0.9267
  history0     n=799 AP 0.2573->0.2573 weights=[0.299 0.257 0.243 0.201]
  history1_2   n=1,037 AP 0.2382->0.2382 weights=[0.275 0.25  0.237 0.239]
  history3plus n=919 AP 0.2736->0.2736 weights=[0.283 0.254 0.242 0.222]

========================================================================================================================
SUITE4 ORIGINAL RANKING / EQUAL-BUDGET SUMMARY
========================================================================================================================
development NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.9254 mean-fold AP=0.9301 F2@20=0.2671
development committee                          pooled AP=0.1547 mean-fold AP=0.1992 F2@20=0.1966
development competence_council                 pooled AP=0.1358 mean-fold AP=0.1906 F2@20=0.1936
holdout     NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.8503 mean-fold AP=0.8831 F2@20=0.5023
holdout     committee                          pooled AP=0.2024 mean-fold AP=0.2612 F2@20=0.3293
holdout     competence_council                 pooled AP=0.1590 mean-fold AP=0.2572 F2@20=0.3293

### DEVELOPMENT [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    2363   1.31% |  0.0486  0.7822 |  0.0303  0.0323  0.0312  0.0318    1.40% |  0.0200  0.0323  0.0247  0.0287    2.12%
     2    2095   0.91% |  0.4315  0.9589 |  0.3913  0.4737  0.4286  0.4545    1.10% |  0.2683  0.5789  0.3667  0.4701    1.96%
     3    2665   0.98% |  0.1991  0.8143 |  0.4444  0.1538  0.2286  0.1770    0.34% |  0.1944  0.2692  0.2258  0.2500    1.35%
     4    2202   1.59% |  0.0971  0.8807 |  0.1429  0.0857  0.1071  0.0932    0.95% |  0.1250  0.1143  0.1194  0.1163    1.45%
     5    2674   2.17% |  0.1768  0.8253 |  0.1488  0.4310  0.2212  0.3125    6.28% |  0.2656  0.2931  0.2787  0.2872    2.39%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   11999   1.41% |  0.1358  0.7972 |  0.1654  0.2485  0.1986  0.2258    2.12% |  0.1794  0.2367  0.2041  0.2225    1.86%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    2466   2.43% |  0.2698  0.8408 |  0.2466  0.3000  0.2707  0.2875    2.96% |  0.2375  0.3167  0.2714  0.2969    3.24%

  Confusion matrix — fold 6 — fixed development threshold 0.330850
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,351              55
  Failure <5                            42              18
  ------------------------------------------------------------------
  Normal correctly rejected: 97.71%
  Failures successfully detected: 30.00%

  Confusion matrix — fold 6 — drift-aware rate 3.260%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,345              61
  Failure <5                            41              19
  ------------------------------------------------------------------
  Normal correctly rejected: 97.46%
  Failures successfully detected: 31.67%
     7    2755   4.65% |  0.2446  0.8356 |  0.5000  0.0078  0.0154  0.0097    0.07% |  0.3465  0.2734  0.3057  0.2855    3.67%

  Confusion matrix — fold 7 — fixed development threshold 0.330850
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,626               1
  Failure <5                           127               1
  ------------------------------------------------------------------
  Normal correctly rejected: 99.96%
  Failures successfully detected: 0.78%

  Confusion matrix — fold 7 — drift-aware rate 3.657%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,561              66
  Failure <5                            93              35
  ------------------------------------------------------------------
  Normal correctly rejected: 97.49%
  Failures successfully detected: 27.34%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    5221   3.60% |  0.1590  0.7753 |  0.2533  0.1011  0.1445  0.1149    1.44% |  0.2983  0.2872  0.2927  0.2894    3.47%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,977              56
  Failure <5                           169              19
  ------------------------------------------------------------------
  Normal correctly rejected: 98.89%
  Failures successfully detected: 10.11%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,906             127
  Failure <5                           134              54
  ------------------------------------------------------------------
  Normal correctly rejected: 97.48%
  Failures successfully detected: 28.72%

### DEVELOPMENT [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    2363   1.31% |  0.0704  0.8048 |  0.2000  0.0968  0.1304  0.1079    0.63% |  0.0602  0.1613  0.0877  0.1208    3.51%
     2    2095   0.91% |  0.4660  0.9595 |  0.2917  0.7368  0.4179  0.5645    2.29% |  0.2319  0.8421  0.3636  0.5517    3.29%
     3    2665   0.98% |  0.1991  0.8143 |  0.1449  0.3846  0.2105  0.2890    2.59% |  0.1500  0.3462  0.2093  0.2744    2.25%
     4    2202   1.59% |  0.0971  0.8807 |  0.1308  0.4000  0.1972  0.2834    4.86% |  0.1296  0.2000  0.1573  0.1804    2.45%
     5    2674   2.17% |  0.1635  0.8346 |  0.1311  0.4138  0.1992  0.2892    6.84% |  0.1604  0.2931  0.2073  0.2515    3.96%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   11999   1.41% |  0.1547  0.8498 |  0.1540  0.3846  0.2200  0.2960    3.52% |  0.1452  0.3195  0.1996  0.2576    3.10%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    2466   2.43% |  0.2778  0.8239 |  0.1129  0.4667  0.1818  0.2869   10.06% |  0.1642  0.3667  0.2268  0.2941    5.43%

  Confusion matrix — fold 6 — fixed development threshold 0.091636
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,186             220
  Failure <5                            32              28
  ------------------------------------------------------------------
  Normal correctly rejected: 90.86%
  Failures successfully detected: 46.67%

  Confusion matrix — fold 6 — drift-aware rate 5.416%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,294             112
  Failure <5                            38              22
  ------------------------------------------------------------------
  Normal correctly rejected: 95.34%
  Failures successfully detected: 36.67%
     7    2755   4.65% |  0.2446  0.8356 |  0.3535  0.2734  0.3084  0.2864    3.59% |  0.2874  0.3750  0.3254  0.3535    6.06%

  Confusion matrix — fold 7 — fixed development threshold 0.091636
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,563              64
  Failure <5                            93              35
  ------------------------------------------------------------------
  Normal correctly rejected: 97.56%
  Failures successfully detected: 27.34%

  Confusion matrix — fold 7 — drift-aware rate 6.076%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,508             119
  Failure <5                            80              48
  ------------------------------------------------------------------
  Normal correctly rejected: 95.47%
  Failures successfully detected: 37.50%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    5221   3.60% |  0.2024  0.8262 |  0.1816  0.3351  0.2355  0.2866    6.65% |  0.2326  0.3723  0.2863  0.3324    5.77%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,749             284
  Failure <5                           125              63
  ------------------------------------------------------------------
  Normal correctly rejected: 94.36%
  Failures successfully detected: 33.51%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,802             231
  Failure <5                           118              70
  ------------------------------------------------------------------
  Normal correctly rejected: 95.41%
  Failures successfully detected: 37.23%
