========================================================================================================================
SUITE4 EXPERIMENT 2 — COUNCIL OF SPECIALISTS + COMPETENCE MODEL
V2-COMPATIBLE COMPLETE OPERATIONAL EVALUATION ENABLED
========================================================================================================================

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=1
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0]
  block=56 n=1,192 AP q/global/recent/rel=0.3031/0.2376/0.2558/0.5490
  block=57 n=866 AP q/global/recent/rel=0.7015/0.5908/0.6154/0.3576
  block=59 n=993 AP q/global/recent/rel=0.4599/0.4476/0.3809/0.4684
  block=60 n=818 AP q/global/recent/rel=0.5537/0.4578/0.4384/0.3887
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.1373/0.7654 F2@20%=0.2827 P/R=0.0982/0.5333
competence_council           AP/AUC=0.1373/0.7654 F2@20%=0.2827 P/R=0.0982/0.5333 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.428 | NON_DEPLOYABLE best-expert oracle AP=0.7377
  history0     n=367 AP 0.1082->0.1082 weights=[0.275 0.272 0.259 0.194]
  history1_2   n=243 AP 0.0556->0.0556 weights=[0.283 0.246 0.263 0.208]
  history3plus n=203 AP 0.5177->0.5177 weights=[0.279 0.249 0.257 0.215]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=2
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0]
  block=56 n=1,192 AP q/global/recent/rel=0.2928/0.1613/0.2354/0.4512
  block=57 n=866 AP q/global/recent/rel=0.6903/0.6901/0.6336/0.4696
  block=59 n=993 AP q/global/recent/rel=0.4519/0.4266/0.4710/0.5813
  block=60 n=818 AP q/global/recent/rel=0.5594/0.4245/0.3851/0.3358
  block=62 n=813 AP q/global/recent/rel=0.1506/0.1682/0.2185/0.1711
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.4830/0.8092 F2@20%=0.4869 P/R=0.2342/0.6667
competence_council           AP/AUC=0.4830/0.8092 F2@20%=0.4869 P/R=0.2342/0.6667 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.655 | NON_DEPLOYABLE best-expert oracle AP=1.0000
  history1_2   n=280 AP 0.5034->0.5034 weights=[0.304 0.271 0.265 0.16 ]
  history3plus n=276 AP 0.4262->0.4262 weights=[0.322 0.289 0.293 0.095]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=3
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0]
  block=56 n=1,192 AP q/global/recent/rel=0.2901/0.1901/0.2341/0.5434
  block=57 n=866 AP q/global/recent/rel=0.7393/0.6089/0.6910/0.3995
  block=59 n=993 AP q/global/recent/rel=0.4653/0.4524/0.3925/0.5374
  block=60 n=818 AP q/global/recent/rel=0.5639/0.4700/0.4427/0.3588
  block=62 n=813 AP q/global/recent/rel=0.1807/0.2054/0.1833/0.1094
  block=63 n=556 AP q/global/recent/rel=0.4918/0.5399/0.5025/0.3089
[POLICY] alpha=0.25 temperature=2.00
committee                    AP/AUC=0.4887/0.9212 F2@20%=0.4878 P/R=0.1806/0.8485
competence_council           AP/AUC=0.4977/0.9217 F2@20%=0.4878 P/R=0.1806/0.8485 ΔAP=+0.0090
[COMPETENCE] target top1 accuracy=0.252 | NON_DEPLOYABLE best-expert oracle AP=0.9822
  history0     n=390 AP 0.4939->0.4770 weights=[0.26  0.254 0.25  0.236]
  history1_2   n=192 AP 0.2582->0.2873 weights=[0.254 0.249 0.249 0.248]
  history3plus n=191 AP 0.8260->0.8260 weights=[0.252 0.251 0.245 0.251]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=4
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0]
  block=56 n=1,192 AP q/global/recent/rel=0.3073/0.2148/0.2117/0.5025
  block=57 n=866 AP q/global/recent/rel=0.6919/0.6949/0.7096/0.3514
  block=59 n=993 AP q/global/recent/rel=0.4484/0.5060/0.4309/0.5144
  block=60 n=818 AP q/global/recent/rel=0.5471/0.3857/0.4795/0.4083
  block=62 n=813 AP q/global/recent/rel=0.1150/0.2319/0.1668/0.1261
  block=63 n=556 AP q/global/recent/rel=0.4942/0.5233/0.4089/0.3174
  block=65 n=773 AP q/global/recent/rel=0.5089/0.5212/0.2325/0.2131
[POLICY] alpha=0.75 temperature=2.00
committee                    AP/AUC=0.5192/0.9606 F2@20%=0.4913 P/R=0.1683/0.9444
competence_council           AP/AUC=0.5327/0.9638 F2@20%=0.4913 P/R=0.1683/0.9444 ΔAP=+0.0135
[COMPETENCE] target top1 accuracy=0.315 | NON_DEPLOYABLE best-expert oracle AP=0.9189
  history1_2   n=299 AP 0.5508->0.5417 weights=[0.269 0.255 0.255 0.222]
  history3plus n=206 AP 0.2262->0.2262 weights=[0.258 0.255 0.256 0.232]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=5
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0]
  block=56 n=1,192 AP q/global/recent/rel=0.2904/0.2330/0.2250/0.4877
  block=57 n=866 AP q/global/recent/rel=0.6864/0.5809/0.7255/0.3420
  block=59 n=993 AP q/global/recent/rel=0.4380/0.4504/0.3744/0.5004
  block=60 n=818 AP q/global/recent/rel=0.5450/0.4616/0.5407/0.3744
  block=62 n=813 AP q/global/recent/rel=0.1262/0.1394/0.2128/0.2400
  block=63 n=556 AP q/global/recent/rel=0.4993/0.4060/0.4670/0.3057
  block=65 n=773 AP q/global/recent/rel=0.5068/0.4222/0.3228/0.4614
  block=66 n=505 AP q/global/recent/rel=0.5329/0.4525/0.5196/0.3085
[POLICY] alpha=0.75 temperature=2.00
committee                    AP/AUC=0.5245/0.9075 F2@20%=0.6586 P/R=0.3828/0.8033
competence_council           AP/AUC=0.5554/0.9182 F2@20%=0.6720 P/R=0.3906/0.8197 ΔAP=+0.0309
[COMPETENCE] target top1 accuracy=0.411 | NON_DEPLOYABLE best-expert oracle AP=0.9499
  history0     n=346 AP 0.5170->0.5469 weights=[0.266 0.255 0.238 0.241]
  history1_2   n=172 AP 0.7289->0.6602 weights=[0.254 0.25  0.253 0.244]
  history3plus n=120 AP 0.3870->0.4214 weights=[0.254 0.25  0.251 0.245]

========================================================================================================================
DEVELOPMENT-SELECTED OPERATIONAL POLICIES [beta=2]
Thresholds and warning ratios below are frozen before holdout evaluation.
========================================================================================================================
  competence_council  : AP=0.4151 | AUC=0.8504 | F2-opt tau=0.238593 | P=0.3528 R=0.6685 F1=0.4618 F2=0.5670 | flag=10.44% | ratio=1.895
  committee           : AP=0.4098 | AUC=0.8612 | F2-opt tau=0.185057 | P=0.3370 R=0.6796 F1=0.4505 F2=0.5647 | flag=11.11% | ratio=2.017

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=6
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0]
  block=56 n=1,192 AP q/global/recent/rel=0.2857/0.2334/0.2863/0.4619
  block=57 n=866 AP q/global/recent/rel=0.6899/0.6374/0.6585/0.3618
  block=59 n=993 AP q/global/recent/rel=0.4673/0.4195/0.4052/0.6084
  block=60 n=818 AP q/global/recent/rel=0.5738/0.4402/0.4074/0.3191
  block=62 n=813 AP q/global/recent/rel=0.1269/0.2321/0.2271/0.0780
  block=63 n=556 AP q/global/recent/rel=0.4897/0.4224/0.4255/0.3098
  block=65 n=773 AP q/global/recent/rel=0.4759/0.5335/0.2571/0.3239
  block=66 n=505 AP q/global/recent/rel=0.5305/0.5065/0.5548/0.2096
  block=68 n=638 AP q/global/recent/rel=0.5109/0.5220/0.4250/0.6239
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.6504/0.9018 F2@20%=0.7294 P/R=0.5702/0.7841
competence_council           AP/AUC=0.6504/0.9018 F2@20%=0.7294 P/R=0.5702/0.7841 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.432 | NON_DEPLOYABLE best-expert oracle AP=0.9168
  history0     n=9 AP 1.0000->1.0000 weights=[0.3   0.251 0.251 0.199]
  history1_2   n=401 AP 0.6435->0.6435 weights=[0.308 0.234 0.269 0.189]
  history3plus n=194 AP 0.6604->0.6604 weights=[0.285 0.275 0.271 0.169]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=7
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0, 69.0]
  block=56 n=1,192 AP q/global/recent/rel=0.3124/0.2909/0.3266/0.4881
  block=57 n=866 AP q/global/recent/rel=0.6983/0.6893/0.6684/0.4214
  block=59 n=993 AP q/global/recent/rel=0.4454/0.4950/0.4280/0.5097
  block=60 n=818 AP q/global/recent/rel=0.5381/0.3659/0.4903/0.3982
  block=62 n=813 AP q/global/recent/rel=0.1134/0.2310/0.2211/0.2330
  block=63 n=556 AP q/global/recent/rel=0.4813/0.4062/0.4061/0.3621
  block=65 n=773 AP q/global/recent/rel=0.5054/0.2949/0.2961/0.4663
  block=66 n=505 AP q/global/recent/rel=0.5529/0.5053/0.5902/0.3321
  block=68 n=638 AP q/global/recent/rel=0.5309/0.5093/0.4093/0.4594
  block=69 n=604 AP q/global/recent/rel=0.6513/0.5802/0.5298/0.6014
[POLICY] alpha=1.00 temperature=2.00
committee                    AP/AUC=0.6040/0.8547 F2@20%=0.5882 P/R=0.5306/0.6047
competence_council           AP/AUC=0.5844/0.8584 F2@20%=0.5882 P/R=0.5306/0.6047 ΔAP=-0.0196
[COMPETENCE] target top1 accuracy=0.519 | NON_DEPLOYABLE best-expert oracle AP=0.9045
  history0     n=315 AP 0.7748->0.7702 weights=[0.265 0.253 0.251 0.231]
  history1_2   n=555 AP 0.5098->0.4842 weights=[0.262 0.255 0.245 0.238]
  history3plus n=108 AP 0.2955->0.2833 weights=[0.259 0.256 0.255 0.23 ]

========================================================================================================================
SUITE4 ORIGINAL RANKING / EQUAL-BUDGET SUMMARY
========================================================================================================================
development NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.8990 mean-fold AP=0.9177 F2@20=0.6351
development committee                          pooled AP=0.4098 mean-fold AP=0.4305 F2@20=0.4815
development competence_council                 pooled AP=0.4151 mean-fold AP=0.4412 F2@20=0.4842
holdout     NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.9087 mean-fold AP=0.9107 F2@20=0.8678
holdout     committee                          pooled AP=0.6180 mean-fold AP=0.6272 F2@20=0.6588
holdout     competence_council                 pooled AP=0.5956 mean-fold AP=0.6174 F2@20=0.6588

### DEVELOPMENT [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1     813   3.69% |  0.1373  0.7654 |  0.0968  0.2000  0.1304  0.1648    7.63% |  0.0941  0.2667  0.1391  0.1951   10.46%
     2     556   7.01% |  0.4830  0.8092 |  0.7059  0.6154  0.6575  0.6316    6.12% |  0.6410  0.6410  0.6410  0.6410    7.01%
     3     773   4.27% |  0.4977  0.9217 |  0.3125  0.7576  0.4425  0.5896   10.35% |  0.2718  0.8485  0.4118  0.5957   13.32%
     4     505   3.56% |  0.5327  0.9638 |  0.4167  0.8333  0.5556  0.6944    7.13% |  0.3659  0.8333  0.5085  0.6637    8.12%
     5     638   9.56% |  0.5554  0.9182 |  0.3893  0.8361  0.5312  0.6800   20.53% |  0.6047  0.4262  0.5000  0.4530    6.74%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    3285   5.51% |  0.4151  0.8504 |  0.3528  0.6685  0.4618  0.5670   10.44% |  0.3280  0.5635  0.4146  0.4928    9.47%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6     604  14.57% |  0.6504  0.9018 |  0.5433  0.7841  0.6419  0.7203   21.03% |  0.5780  0.7159  0.6396  0.6833   18.05%

  Confusion matrix — fold 6 — fixed development threshold 0.238593
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                           458              58
  Failure <5                            19              69
  ------------------------------------------------------------------
  Normal correctly rejected: 88.76%
  Failures successfully detected: 78.41%

  Confusion matrix — fold 6 — drift-aware rate 18.119%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                           470              46
  Failure <5                            25              63
  ------------------------------------------------------------------
  Normal correctly rejected: 91.09%
  Failures successfully detected: 71.59%
     7     978  17.59% |  0.5844  0.8584 |  0.3446  0.8895  0.4968  0.6758   45.40% |  0.4889  0.7674  0.5973  0.6889   27.61%

  Confusion matrix — fold 7 — fixed development threshold 0.238593
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                           515             291
  Failure <5                            19             153
  ------------------------------------------------------------------
  Normal correctly rejected: 63.90%
  Failures successfully detected: 88.95%

  Confusion matrix — fold 7 — drift-aware rate 27.610%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                           668             138
  Failure <5                            40             132
  ------------------------------------------------------------------
  Normal correctly rejected: 82.88%
  Failures successfully detected: 76.74%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    1582  16.43% |  0.5956  0.8649 |  0.3888  0.8538  0.5343  0.6890   36.09% |  0.5145  0.7500  0.6103  0.6871   23.96%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                           973             349
  Failure <5                            38             222
  ------------------------------------------------------------------
  Normal correctly rejected: 73.60%
  Failures successfully detected: 85.38%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,138             184
  Failure <5                            65             195
  ------------------------------------------------------------------
  Normal correctly rejected: 86.08%
  Failures successfully detected: 75.00%

### DEVELOPMENT [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1     813   3.69% |  0.1373  0.7654 |  0.1236  0.3667  0.1849  0.2632   10.95% |  0.1222  0.3667  0.1833  0.2619   11.07%
     2     556   7.01% |  0.4830  0.8092 |  0.6486  0.6154  0.6316  0.6218    6.65% |  0.6098  0.6410  0.6250  0.6345    7.37%
     3     773   4.27% |  0.4887  0.9212 |  0.2947  0.8485  0.4375  0.6167   12.29% |  0.2569  0.8485  0.3944  0.5809   14.10%
     4     505   3.56% |  0.5192  0.9606 |  0.3846  0.8333  0.5263  0.6757    7.72% |  0.3488  0.8333  0.4918  0.6522    8.51%
     5     638   9.56% |  0.5245  0.9075 |  0.4286  0.7377  0.5422  0.6447   16.46% |  0.6522  0.4918  0.5607  0.5172    7.21%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    3285   5.51% |  0.4098  0.8612 |  0.3370  0.6796  0.4505  0.5647   11.11% |  0.3313  0.6022  0.4275  0.5176   10.02%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6     604  14.57% |  0.6504  0.9018 |  0.4964  0.7841  0.6079  0.7026   23.01% |  0.5776  0.7614  0.6569  0.7158   19.21%

  Confusion matrix — fold 6 — fixed development threshold 0.185057
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                           446              70
  Failure <5                            19              69
  ------------------------------------------------------------------
  Normal correctly rejected: 86.43%
  Failures successfully detected: 78.41%

  Confusion matrix — fold 6 — drift-aware rate 19.281%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                           467              49
  Failure <5                            21              67
  ------------------------------------------------------------------
  Normal correctly rejected: 90.50%
  Failures successfully detected: 76.14%
     7     978  17.59% |  0.6040  0.8547 |  0.3691  0.8605  0.5166  0.6795   41.00% |  0.4669  0.7791  0.5839  0.6872   29.35%

  Confusion matrix — fold 7 — fixed development threshold 0.185057
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                           553             253
  Failure <5                            24             148
  ------------------------------------------------------------------
  Normal correctly rejected: 68.61%
  Failures successfully detected: 86.05%

  Confusion matrix — fold 7 — drift-aware rate 29.381%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                           653             153
  Failure <5                            38             134
  ------------------------------------------------------------------
  Normal correctly rejected: 81.02%
  Failures successfully detected: 77.91%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    1582  16.43% |  0.6180  0.8711 |  0.4019  0.8346  0.5425  0.6867   34.13% |  0.4988  0.7731  0.6063  0.6965   25.47%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                           999             323
  Failure <5                            43             217
  ------------------------------------------------------------------
  Normal correctly rejected: 75.57%
  Failures successfully detected: 83.46%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         1,120             202
  Failure <5                            59             201
  ------------------------------------------------------------------
  Normal correctly rejected: 84.72%
  Failures successfully detected: 77.31%
