========================================================================================================================
SUITE4 EXPERIMENT 2 — COUNCIL OF SPECIALISTS + COMPETENCE MODEL
V2-COMPATIBLE COMPLETE OPERATIONAL EVALUATION ENABLED
========================================================================================================================

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=1
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0]
  block=54 n=1,925 AP q/global/recent/rel=0.9000/0.9000/0.8860/0.8636
  block=56 n=3,087 AP q/global/recent/rel=0.0309/0.0067/0.0423/0.0182
  block=57 n=2,789 AP q/global/recent/rel=0.0573/0.0539/0.0459/0.0751
  block=59 n=3,094 AP q/global/recent/rel=0.0188/0.0150/0.0119/0.0341
  block=60 n=3,265 AP q/global/recent/rel=0.0528/0.0218/0.0775/0.0340
[POLICY] alpha=1.00 temperature=2.00
committee                    AP/AUC=0.0544/0.8479 F2@20%=0.1210 P/R=0.0275/0.8000
competence_council           AP/AUC=0.0330/0.8751 F2@20%=0.1286 P/R=0.0293/0.8500 ΔAP=-0.0214
[COMPETENCE] target top1 accuracy=0.963 | NON_DEPLOYABLE best-expert oracle AP=1.0000
  history0     n=937 AP 0.2028->0.1175 weights=[0.271 0.266 0.252 0.211]
  history1_2   n=1,003 AP 0.0313->0.0256 weights=[0.273 0.267 0.254 0.206]
  history3plus n=964 AP 0.0810->0.0443 weights=[0.277 0.269 0.259 0.195]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=2
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0]
  block=54 n=1,925 AP q/global/recent/rel=0.9000/0.8704/0.8667/0.3433
  block=56 n=3,087 AP q/global/recent/rel=0.0316/0.0864/0.0274/0.0276
  block=57 n=2,789 AP q/global/recent/rel=0.0619/0.0560/0.0317/0.0011
  block=59 n=3,094 AP q/global/recent/rel=0.0150/0.0042/0.0079/0.0364
  block=60 n=3,265 AP q/global/recent/rel=0.0519/0.0948/0.0292/0.0169
  block=62 n=2,904 AP q/global/recent/rel=0.0450/0.0109/0.0166/0.0317
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.3792/0.9529 F2@20%=0.1660 P/R=0.0390/0.8966
competence_council           AP/AUC=0.3792/0.9529 F2@20%=0.1660 P/R=0.0390/0.8966 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.955 | NON_DEPLOYABLE best-expert oracle AP=0.9736
  history0     n=32 AP nan->nan weights=[0.307 0.295 0.249 0.149]
  history1_2   n=1,121 AP 0.5130->0.5130 weights=[0.31  0.276 0.249 0.165]
  history3plus n=2,184 AP 0.1537->0.1537 weights=[0.312 0.279 0.252 0.157]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=3
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0]
  block=54 n=1,925 AP q/global/recent/rel=0.8922/0.8636/0.8690/0.2528
  block=56 n=3,087 AP q/global/recent/rel=0.0358/0.0179/0.0416/0.0157
  block=57 n=2,789 AP q/global/recent/rel=0.0544/0.0238/0.0121/0.1069
  block=59 n=3,094 AP q/global/recent/rel=0.0168/0.0044/0.0140/0.0186
  block=60 n=3,265 AP q/global/recent/rel=0.0562/0.1359/0.0370/0.0268
  block=62 n=2,904 AP q/global/recent/rel=0.0531/0.0223/0.0200/0.0143
  block=63 n=3,337 AP q/global/recent/rel=0.3206/0.1809/0.2037/0.0882
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.2147/0.8411 F2@20%=0.1718 P/R=0.0433/0.6667
competence_council           AP/AUC=0.2147/0.8411 F2@20%=0.1718 P/R=0.0433/0.6667 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.733 | NON_DEPLOYABLE best-expert oracle AP=1.0000
  history0     n=901 AP 0.1110->0.1110 weights=[0.295 0.377 0.199 0.129]
  history1_2   n=1,226 AP 0.4908->0.4908 weights=[0.327 0.284 0.254 0.134]
  history3plus n=1,108 AP 0.2673->0.2673 weights=[0.326 0.311 0.245 0.119]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=4
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0]
  block=54 n=1,925 AP q/global/recent/rel=0.9167/0.8922/0.8922/0.2177
  block=56 n=3,087 AP q/global/recent/rel=0.0314/0.0268/0.0467/0.0719
  block=57 n=2,789 AP q/global/recent/rel=0.0587/0.0132/0.0851/0.0304
  block=59 n=3,094 AP q/global/recent/rel=0.0183/0.0049/0.0052/0.0109
  block=60 n=3,265 AP q/global/recent/rel=0.0525/0.0398/0.0198/0.0323
  block=62 n=2,904 AP q/global/recent/rel=0.0482/0.0172/0.0171/0.0271
  block=63 n=3,337 AP q/global/recent/rel=0.3377/0.1865/0.2641/0.1460
  block=65 n=3,235 AP q/global/recent/rel=0.1973/0.2543/0.0366/0.0538
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.4396/0.9574 F2@20%=0.2129 P/R=0.0522/0.9211
competence_council           AP/AUC=0.4396/0.9574 F2@20%=0.2129 P/R=0.0522/0.9211 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.741 | NON_DEPLOYABLE best-expert oracle AP=0.9407
  history0     n=69 AP nan->nan weights=[0.259 0.262 0.28  0.198]
  history1_2   n=1,074 AP 0.6222->0.6222 weights=[0.299 0.289 0.276 0.137]
  history3plus n=2,208 AP 0.4210->0.4210 weights=[0.313 0.283 0.254 0.15 ]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=5
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0]
  block=54 n=1,925 AP q/global/recent/rel=0.9333/0.8889/0.8533/0.2168
  block=56 n=3,087 AP q/global/recent/rel=0.0292/0.0477/0.0544/0.1427
  block=57 n=2,789 AP q/global/recent/rel=0.0569/0.0916/0.0394/0.1011
  block=59 n=3,094 AP q/global/recent/rel=0.0164/0.0051/0.0045/0.0358
  block=60 n=3,265 AP q/global/recent/rel=0.0555/0.0129/0.0478/0.0548
  block=62 n=2,904 AP q/global/recent/rel=0.0476/0.0696/0.0180/0.0299
  block=63 n=3,337 AP q/global/recent/rel=0.3188/0.3231/0.1107/0.2658
  block=65 n=3,235 AP q/global/recent/rel=0.1993/0.0933/0.1037/0.1052
  block=66 n=3,351 AP q/global/recent/rel=0.4286/0.2030/0.2345/0.0901
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.1911/0.8916 F2@20%=0.2848 P/R=0.0783/0.8358
competence_council           AP/AUC=0.1911/0.8916 F2@20%=0.2848 P/R=0.0783/0.8358 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.904 | NON_DEPLOYABLE best-expert oracle AP=1.0000
  history0     n=970 AP 0.1191->0.1191 weights=[0.337 0.31  0.28  0.072]
  history1_2   n=1,632 AP 0.2172->0.2172 weights=[0.334 0.301 0.288 0.077]
  history3plus n=971 AP 0.3505->0.3505 weights=[0.315 0.289 0.283 0.114]

========================================================================================================================
DEVELOPMENT-SELECTED OPERATIONAL POLICIES [beta=2]
Thresholds and warning ratios below are frozen before holdout evaluation.
========================================================================================================================
  competence_council  : AP=0.1832 | AUC=0.8372 | F2-opt tau=0.167283 | P=0.1616 R=0.4031 F1=0.2307 F2=0.3103 | flag=2.98% | ratio=2.495
  committee           : AP=0.2340 | AUC=0.8903 | F2-opt tau=0.086544 | P=0.1913 R=0.4694 F1=0.2718 F2=0.3636 | flag=2.93% | ratio=2.454

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=6
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0]
  block=54 n=1,925 AP q/global/recent/rel=0.9333/0.2549/0.8750/0.3079
  block=56 n=3,087 AP q/global/recent/rel=0.0358/0.0789/0.0370/0.0071
  block=57 n=2,789 AP q/global/recent/rel=0.0661/0.0502/0.0641/0.0925
  block=59 n=3,094 AP q/global/recent/rel=0.0125/0.0052/0.0059/0.0339
  block=60 n=3,265 AP q/global/recent/rel=0.0499/0.0893/0.0225/0.0338
  block=62 n=2,904 AP q/global/recent/rel=0.0474/0.0427/0.0219/0.0408
  block=63 n=3,337 AP q/global/recent/rel=0.3672/0.1369/0.3003/0.2095
  block=65 n=3,235 AP q/global/recent/rel=0.1809/0.1110/0.0986/0.0966
  block=66 n=3,351 AP q/global/recent/rel=0.4272/0.2332/0.3088/0.1386
  block=68 n=3,573 AP q/global/recent/rel=0.2057/0.1318/0.1130/0.1591
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.2531/0.8587 F2@20%=0.4098 P/R=0.1435/0.7642
competence_council           AP/AUC=0.2531/0.8587 F2@20%=0.4098 P/R=0.1435/0.7642 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.906 | NON_DEPLOYABLE best-expert oracle AP=0.9076
  history0     n=5 AP nan->nan weights=[0.289 0.285 0.267 0.158]
  history1_2   n=938 AP 0.2445->0.2445 weights=[0.3   0.277 0.266 0.157]
  history3plus n=2,334 AP 0.2607->0.2607 weights=[0.306 0.274 0.259 0.162]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=7
[FORWARD EXPERTS] blocks=[53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0, 69.0]
  block=54 n=1,925 AP q/global/recent/rel=0.8860/0.2659/0.3529/0.2985
  block=56 n=3,087 AP q/global/recent/rel=0.0416/0.0071/0.0415/0.0139
  block=57 n=2,789 AP q/global/recent/rel=0.0591/0.0456/0.0445/0.2408
  block=59 n=3,094 AP q/global/recent/rel=0.0151/0.0039/0.0088/0.0241
  block=60 n=3,265 AP q/global/recent/rel=0.0514/0.0202/0.0255/0.0522
  block=62 n=2,904 AP q/global/recent/rel=0.0462/0.0162/0.0183/0.0180
  block=63 n=3,337 AP q/global/recent/rel=0.3561/0.3289/0.2566/0.0532
  block=65 n=3,235 AP q/global/recent/rel=0.1817/0.1625/0.0997/0.1875
  block=66 n=3,351 AP q/global/recent/rel=0.4068/0.2205/0.2885/0.1260
  block=68 n=3,573 AP q/global/recent/rel=0.2076/0.1321/0.1019/0.1291
  block=69 n=3,277 AP q/global/recent/rel=0.2564/0.2453/0.2287/0.2898
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.3786/0.8930 F2@20%=0.4212 P/R=0.1456/0.8000
competence_council           AP/AUC=0.3786/0.8930 F2@20%=0.4212 P/R=0.1456/0.8000 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.506 | NON_DEPLOYABLE best-expert oracle AP=0.9606
  history0     n=821 AP 0.3080->0.3080 weights=[0.297 0.272 0.261 0.171]
  history1_2   n=1,383 AP 0.3452->0.3452 weights=[0.289 0.257 0.252 0.202]
  history3plus n=1,230 AP 0.5287->0.5287 weights=[0.267 0.26  0.282 0.191]

========================================================================================================================
SUITE4 ORIGINAL RANKING / EQUAL-BUDGET SUMMARY
========================================================================================================================
development NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.9863 mean-fold AP=0.9829 F2@20=0.2332
development committee                          pooled AP=0.2340 mean-fold AP=0.2558 F2@20=0.1913
development competence_council                 pooled AP=0.1832 mean-fold AP=0.2515 F2@20=0.1928
holdout     NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.9382 mean-fold AP=0.9341 F2@20=0.5314
holdout     committee                          pooled AP=0.2925 mean-fold AP=0.3159 F2@20=0.4155
holdout     competence_council                 pooled AP=0.2925 mean-fold AP=0.3159 F2@20=0.4155

### DEVELOPMENT [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    2904   0.69% |  0.0330  0.8751 |  0.0350  0.4500  0.0650  0.1335    8.85% |  0.0345  0.1500  0.0561  0.0898    3.00%
     2    3337   0.87% |  0.3792  0.9529 |  0.2361  0.5862  0.3366  0.4521    2.16% |  0.2807  0.5517  0.3721  0.4624    1.71%
     3    3235   1.30% |  0.2147  0.8411 |  0.5000  0.3095  0.3824  0.3351    0.80% |  0.2571  0.4286  0.3214  0.3782    2.16%
     4    3351   1.13% |  0.4396  0.9574 |  0.2976  0.6579  0.4098  0.5297    2.51% |  0.2294  0.6579  0.3401  0.4789    3.25%
     5    3573   1.88% |  0.1911  0.8916 |  0.3000  0.2239  0.2564  0.2358    1.40% |  0.2178  0.3284  0.2619  0.2981    2.83%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   16400   1.20% |  0.1832  0.8372 |  0.1616  0.4031  0.2307  0.3103    2.98% |  0.1981  0.4286  0.2710  0.3477    2.59%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    3277   3.75% |  0.2531  0.8587 |  0.2486  0.3659  0.2961  0.3343    5.52% |  0.2484  0.3089  0.2754  0.2946    4.67%

  Confusion matrix — fold 6 — fixed development threshold 0.167283
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,018             136
  Failure <5                            78              45
  ------------------------------------------------------------------
  Normal correctly rejected: 95.69%
  Failures successfully detected: 36.59%

  Confusion matrix — fold 6 — drift-aware rate 4.678%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,039             115
  Failure <5                            85              38
  ------------------------------------------------------------------
  Normal correctly rejected: 96.35%
  Failures successfully detected: 30.89%
     7    3434   3.64% |  0.3786  0.8930 |  0.3696  0.5440  0.4401  0.4971    5.36% |  0.2609  0.6720  0.3758  0.5109    9.38%

  Confusion matrix — fold 7 — fixed development threshold 0.167283
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,193             116
  Failure <5                            57              68
  ------------------------------------------------------------------
  Normal correctly rejected: 96.49%
  Failures successfully detected: 54.40%

  Confusion matrix — fold 7 — drift-aware rate 9.364%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,071             238
  Failure <5                            41              84
  ------------------------------------------------------------------
  Normal correctly rejected: 92.81%
  Failures successfully detected: 67.20%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    6711   3.70% |  0.2925  0.8506 |  0.3096  0.4556  0.3687  0.4164    5.44% |  0.2568  0.4919  0.3375  0.4158    7.08%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         6,211             252
  Failure <5                           135             113
  ------------------------------------------------------------------
  Normal correctly rejected: 96.10%
  Failures successfully detected: 45.56%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         6,110             353
  Failure <5                           126             122
  ------------------------------------------------------------------
  Normal correctly rejected: 94.54%
  Failures successfully detected: 49.19%

### DEVELOPMENT [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    2904   0.69% |  0.0544  0.8479 |  0.0000  0.0000  0.0000  0.0000    0.31% |  0.0471  0.2000  0.0762  0.1212    2.93%
     2    3337   0.87% |  0.3792  0.9529 |  0.2179  0.5862  0.3178  0.4381    2.34% |  0.2857  0.5517  0.3765  0.4651    1.68%
     3    3235   1.30% |  0.2147  0.8411 |  0.1869  0.4762  0.2685  0.3636    3.31% |  0.2609  0.4286  0.3243  0.3797    2.13%
     4    3351   1.13% |  0.4396  0.9574 |  0.2119  0.6579  0.3205  0.4630    3.52% |  0.2336  0.6579  0.3448  0.4826    3.19%
     5    3573   1.88% |  0.1911  0.8916 |  0.1775  0.4478  0.2542  0.3432    4.73% |  0.2222  0.3284  0.2651  0.2997    2.77%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   16400   1.20% |  0.2340  0.8903 |  0.1913  0.4694  0.2718  0.3636    2.93% |  0.2043  0.4337  0.2778  0.3542    2.54%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    3277   3.75% |  0.2531  0.8587 |  0.2302  0.5203  0.3192  0.4156    8.48% |  0.2450  0.3008  0.2701  0.2877    4.61%

  Confusion matrix — fold 6 — fixed development threshold 0.086544
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,940             214
  Failure <5                            59              64
  ------------------------------------------------------------------
  Normal correctly rejected: 93.21%
  Failures successfully detected: 52.03%

  Confusion matrix — fold 6 — drift-aware rate 4.602%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,040             114
  Failure <5                            86              37
  ------------------------------------------------------------------
  Normal correctly rejected: 96.39%
  Failures successfully detected: 30.08%
     7    3434   3.64% |  0.3786  0.8930 |  0.2160  0.7120  0.3315  0.4879   12.00% |  0.2627  0.6640  0.3764  0.5086    9.20%

  Confusion matrix — fold 7 — fixed development threshold 0.086544
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,986             323
  Failure <5                            36              89
  ------------------------------------------------------------------
  Normal correctly rejected: 90.24%
  Failures successfully detected: 71.20%

  Confusion matrix — fold 7 — drift-aware rate 9.211%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         3,076             233
  Failure <5                            42              83
  ------------------------------------------------------------------
  Normal correctly rejected: 92.96%
  Failures successfully detected: 66.40%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    6711   3.70% |  0.2925  0.8506 |  0.2217  0.6169  0.3262  0.4548   10.28% |  0.2570  0.4839  0.3357  0.4112    6.96%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         5,926             537
  Failure <5                            95             153
  ------------------------------------------------------------------
  Normal correctly rejected: 91.69%
  Failures successfully detected: 61.69%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         6,116             347
  Failure <5                           128             120
  ------------------------------------------------------------------
  Normal correctly rejected: 94.63%
  Failures successfully detected: 48.39%
