========================================================================================================================
SUITE4 EXPERIMENT 2 — COUNCIL OF SPECIALISTS + COMPETENCE MODEL
V2-COMPATIBLE COMPLETE OPERATIONAL EVALUATION ENABLED
========================================================================================================================

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=1
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0]
  block=53 n=2,100 AP q/global/recent/rel=0.0818/0.0458/0.0345/0.0230
  block=54 n=1,854 AP q/global/recent/rel=0.1974/0.4254/0.4311/0.0092
  block=56 n=2,549 AP q/global/recent/rel=0.1177/0.0345/0.0940/0.0103
  block=57 n=2,428 AP q/global/recent/rel=0.4976/0.4394/0.2765/0.0330
  block=59 n=2,758 AP q/global/recent/rel=0.0705/0.0772/0.0479/0.1983
  block=60 n=2,471 AP q/global/recent/rel=0.4441/0.3289/0.2183/0.2784
[POLICY] alpha=0.25 temperature=0.50
committee                    AP/AUC=0.2279/0.8058 F2@20%=0.1742 P/R=0.0461/0.5714
competence_council           AP/AUC=0.2281/0.8115 F2@20%=0.2032 P/R=0.0537/0.6667 ΔAP=+0.0002
[COMPETENCE] target top1 accuracy=0.367 | NON_DEPLOYABLE best-expert oracle AP=1.0000
  history0     n=802 AP 0.1194->0.1162 weights=[0.288 0.282 0.29  0.14 ]
  history1_2   n=1,038 AP 0.2743->0.2778 weights=[0.291 0.288 0.292 0.128]
  history3plus n=766 AP 0.6521->0.6400 weights=[0.288 0.283 0.287 0.141]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=2
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0]
  block=53 n=2,100 AP q/global/recent/rel=0.0584/0.0515/0.0335/0.0211
  block=54 n=1,854 AP q/global/recent/rel=0.2148/0.2049/0.1951/0.0079
  block=56 n=2,549 AP q/global/recent/rel=0.1565/0.1379/0.0454/0.0529
  block=57 n=2,428 AP q/global/recent/rel=0.5204/0.4681/0.1819/0.1292
  block=59 n=2,758 AP q/global/recent/rel=0.0774/0.0354/0.0545/0.2604
  block=60 n=2,471 AP q/global/recent/rel=0.4453/0.3980/0.2893/0.2833
  block=62 n=2,606 AP q/global/recent/rel=0.2101/0.1594/0.1585/0.1764
[POLICY] alpha=0.50 temperature=2.00
committee                    AP/AUC=0.5846/0.9370 F2@20%=0.2995 P/R=0.0816/0.9000
competence_council           AP/AUC=0.5540/0.9239 F2@20%=0.2995 P/R=0.0816/0.9000 ΔAP=-0.0307
[COMPETENCE] target top1 accuracy=0.839 | NON_DEPLOYABLE best-expert oracle AP=0.9291
  history0     n=6 AP nan->nan weights=[0.268 0.266 0.264 0.202]
  history1_2   n=918 AP 0.8551->0.8928 weights=[0.264 0.26  0.259 0.217]
  history3plus n=1,280 AP 0.3772->0.3656 weights=[0.265 0.263 0.263 0.209]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=3
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0]
  block=53 n=2,100 AP q/global/recent/rel=0.0566/0.0309/0.0342/0.0511
  block=54 n=1,854 AP q/global/recent/rel=0.2545/0.2324/0.2234/0.0047
  block=56 n=2,549 AP q/global/recent/rel=0.1194/0.1168/0.1434/0.1569
  block=57 n=2,428 AP q/global/recent/rel=0.5126/0.4142/0.3191/0.0206
  block=59 n=2,758 AP q/global/recent/rel=0.0915/0.0740/0.0639/0.3907
  block=60 n=2,471 AP q/global/recent/rel=0.4891/0.2935/0.2616/0.3109
  block=62 n=2,606 AP q/global/recent/rel=0.1980/0.2337/0.1081/0.2465
  block=63 n=2,204 AP q/global/recent/rel=0.6120/0.4085/0.5856/0.2088
[POLICY] alpha=0.25 temperature=2.00
committee                    AP/AUC=0.1018/0.6932 F2@20%=0.1125 P/R=0.0297/0.3721
competence_council           AP/AUC=0.1115/0.7029 F2@20%=0.1195 P/R=0.0315/0.3953 ΔAP=+0.0096
[COMPETENCE] target top1 accuracy=0.768 | NON_DEPLOYABLE best-expert oracle AP=0.9359
  history0     n=717 AP 0.2474->0.2467 weights=[0.263 0.253 0.255 0.229]
  history1_2   n=1,071 AP 0.1092->0.1077 weights=[0.264 0.254 0.259 0.224]
  history3plus n=905 AP 0.1750->0.1566 weights=[0.262 0.263 0.259 0.216]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=4
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0]
  block=53 n=2,100 AP q/global/recent/rel=0.0554/0.0306/0.0478/0.0408
  block=54 n=1,854 AP q/global/recent/rel=0.2154/0.1390/0.0332/0.0037
  block=56 n=2,549 AP q/global/recent/rel=0.1279/0.0657/0.0655/0.0583
  block=57 n=2,428 AP q/global/recent/rel=0.4994/0.4378/0.4808/0.2219
  block=59 n=2,758 AP q/global/recent/rel=0.0974/0.0518/0.0319/0.3095
  block=60 n=2,471 AP q/global/recent/rel=0.4621/0.3368/0.3394/0.3594
  block=62 n=2,606 AP q/global/recent/rel=0.2159/0.2027/0.2132/0.2765
  block=63 n=2,204 AP q/global/recent/rel=0.5823/0.3035/0.2755/0.2469
  block=65 n=2,693 AP q/global/recent/rel=0.0891/0.0433/0.0625/0.1786
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.2944/0.8394 F2@20%=0.1318 P/R=0.0308/0.7368
competence_council           AP/AUC=0.2944/0.8394 F2@20%=0.1318 P/R=0.0308/0.7368 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.938 | NON_DEPLOYABLE best-expert oracle AP=0.8070
  history1_2   n=837 AP 0.5583->0.5583 weights=[0.317 0.288 0.256 0.14 ]
  history3plus n=1,438 AP 0.0616->0.0616 weights=[0.317 0.289 0.253 0.142]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=5
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0]
  block=53 n=2,100 AP q/global/recent/rel=0.0577/0.0388/0.0585/0.0234
  block=54 n=1,854 AP q/global/recent/rel=0.2453/0.0506/0.2264/0.0088
  block=56 n=2,549 AP q/global/recent/rel=0.1256/0.0620/0.0452/0.0405
  block=57 n=2,428 AP q/global/recent/rel=0.5140/0.1788/0.2706/0.0387
  block=59 n=2,758 AP q/global/recent/rel=0.1002/0.0461/0.0587/0.2506
  block=60 n=2,471 AP q/global/recent/rel=0.4759/0.2693/0.2357/0.2826
  block=62 n=2,606 AP q/global/recent/rel=0.1866/0.2275/0.1809/0.3127
  block=63 n=2,204 AP q/global/recent/rel=0.6137/0.2881/0.5584/0.2282
  block=65 n=2,693 AP q/global/recent/rel=0.0877/0.0868/0.0558/0.0661
  block=66 n=2,275 AP q/global/recent/rel=0.3123/0.2321/0.1166/0.0731
[POLICY] alpha=0.00 temperature=0.50
committee                    AP/AUC=0.2549/0.8676 F2@20%=0.2727 P/R=0.0754/0.7895
competence_council           AP/AUC=0.2549/0.8676 F2@20%=0.2727 P/R=0.0754/0.7895 ΔAP=+0.0000
[COMPETENCE] target top1 accuracy=0.856 | NON_DEPLOYABLE best-expert oracle AP=0.9591
  history0     n=868 AP 0.3116->0.3116 weights=[0.323 0.289 0.267 0.121]
  history1_2   n=1,180 AP 0.1560->0.1560 weights=[0.33  0.304 0.269 0.097]
  history3plus n=935 AP 0.2025->0.2025 weights=[0.319 0.301 0.264 0.116]

========================================================================================================================
DEVELOPMENT-SELECTED OPERATIONAL POLICIES [beta=2]
Thresholds and warning ratios below are frozen before holdout evaluation.
========================================================================================================================
  competence_council  : AP=0.2454 | AUC=0.8170 | F2-opt tau=0.227462 | P=0.3439 R=0.3781 F1=0.3602 F2=0.3707 | flag=1.73% | ratio=1.100
  committee           : AP=0.2615 | AUC=0.8460 | F2-opt tau=0.114172 | P=0.2348 R=0.4627 F1=0.3116 F2=0.3875 | flag=3.10% | ratio=1.970

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=6
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0]
  block=53 n=2,100 AP q/global/recent/rel=0.0553/0.0345/0.0369/0.0222
  block=54 n=1,854 AP q/global/recent/rel=0.2149/0.4411/0.1967/0.0055
  block=56 n=2,549 AP q/global/recent/rel=0.1169/0.0654/0.0182/0.1417
  block=57 n=2,428 AP q/global/recent/rel=0.5151/0.4576/0.1904/0.1316
  block=59 n=2,758 AP q/global/recent/rel=0.0740/0.0710/0.0495/0.3078
  block=60 n=2,471 AP q/global/recent/rel=0.4443/0.2615/0.1868/0.3249
  block=62 n=2,606 AP q/global/recent/rel=0.2325/0.1553/0.1319/0.2469
  block=63 n=2,204 AP q/global/recent/rel=0.6092/0.3115/0.5840/0.2599
  block=65 n=2,693 AP q/global/recent/rel=0.0998/0.0574/0.0440/0.0791
  block=66 n=2,275 AP q/global/recent/rel=0.2827/0.1172/0.0648/0.1003
  block=68 n=2,983 AP q/global/recent/rel=0.2380/0.2124/0.1659/0.1539
[POLICY] alpha=0.25 temperature=2.00
committee                    AP/AUC=0.2791/0.9146 F2@20%=0.3292 P/R=0.0949/0.8600
competence_council           AP/AUC=0.2728/0.9205 F2@20%=0.3292 P/R=0.0949/0.8600 ΔAP=-0.0063
[COMPETENCE] target top1 accuracy=0.925 | NON_DEPLOYABLE best-expert oracle AP=0.8893
  history0     n=8 AP 0.3333->0.3333 weights=[0.292 0.232 0.18  0.296]
  history1_2   n=805 AP 0.3261->0.3178 weights=[0.274 0.262 0.262 0.201]
  history3plus n=1,450 AP 0.2314->0.2363 weights=[0.272 0.265 0.264 0.199]

========================================================================================================================
EXP2 COMPETENCE COUNCIL | fold=7
[FORWARD EXPERTS] blocks=[51.0, 53.0, 54.0, 56.0, 57.0, 59.0, 60.0, 62.0, 63.0, 65.0, 66.0, 68.0, 69.0]
  block=53 n=2,100 AP q/global/recent/rel=0.0572/0.0254/0.0274/0.0216
  block=54 n=1,854 AP q/global/recent/rel=0.2149/0.2686/0.1514/0.0089
  block=56 n=2,549 AP q/global/recent/rel=0.1208/0.0867/0.0949/0.4608
  block=57 n=2,428 AP q/global/recent/rel=0.4985/0.3369/0.0816/0.1613
  block=59 n=2,758 AP q/global/recent/rel=0.0820/0.0801/0.0270/0.3511
  block=60 n=2,471 AP q/global/recent/rel=0.4969/0.4007/0.2581/0.3271
  block=62 n=2,606 AP q/global/recent/rel=0.2136/0.1450/0.2505/0.2846
  block=63 n=2,204 AP q/global/recent/rel=0.5998/0.5046/0.5203/0.1794
  block=65 n=2,693 AP q/global/recent/rel=0.1040/0.0364/0.0660/0.1005
  block=66 n=2,275 AP q/global/recent/rel=0.3565/0.1921/0.2520/0.1852
  block=68 n=2,983 AP q/global/recent/rel=0.2296/0.2150/0.2250/0.0463
  block=69 n=2,263 AP q/global/recent/rel=0.2718/0.3501/0.2778/0.2540
[POLICY] alpha=0.25 temperature=1.00
committee                    AP/AUC=0.2748/0.8276 F2@20%=0.3399 P/R=0.1235/0.6050
competence_council           AP/AUC=0.2830/0.8468 F2@20%=0.3541 P/R=0.1286/0.6303 ΔAP=+0.0082
[COMPETENCE] target top1 accuracy=0.874 | NON_DEPLOYABLE best-expert oracle AP=0.8577
  history0     n=820 AP 0.2189->0.2265 weights=[0.28  0.257 0.259 0.204]
  history1_2   n=1,076 AP 0.4921->0.4841 weights=[0.274 0.263 0.253 0.211]
  history3plus n=1,019 AP 0.3437->0.3858 weights=[0.268 0.264 0.25  0.219]

========================================================================================================================
SUITE4 ORIGINAL RANKING / EQUAL-BUDGET SUMMARY
========================================================================================================================
development NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.9154 mean-fold AP=0.9262 F2@20=0.2929
development committee                          pooled AP=0.2615 mean-fold AP=0.2927 F2@20=0.1981
development competence_council                 pooled AP=0.2454 mean-fold AP=0.2886 F2@20=0.2054
holdout     NON_DEPLOYABLE_best_expert_oracle  pooled AP=0.8192 mean-fold AP=0.8735 F2@20=0.4723
holdout     committee                          pooled AP=0.2543 mean-fold AP=0.2769 F2@20=0.3346
holdout     competence_council                 pooled AP=0.2549 mean-fold AP=0.2779 F2@20=0.3417

### DEVELOPMENT [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    2606   1.61% |  0.2281  0.8115 |  0.3846  0.1190  0.1818  0.1381    0.50% |  0.3556  0.3810  0.3678  0.3756    1.73%
     2    2204   1.81% |  0.5540  0.9239 |  0.5172  0.7500  0.6122  0.6881    2.63% |  0.6667  0.6500  0.6582  0.6533    1.77%
     3    2693   1.60% |  0.1115  0.7029 |  0.2340  0.2558  0.2444  0.2511    1.75% |  0.2037  0.2558  0.2268  0.2434    2.01%
     4    2275   0.84% |  0.2944  0.8394 |  0.1964  0.5789  0.2933  0.4167    2.46% |  0.2750  0.5789  0.3729  0.4741    1.76%
     5    2983   1.91% |  0.2549  0.8676 |  0.4043  0.3333  0.3654  0.3455    1.58% |  0.4815  0.2281  0.3095  0.2549    0.91%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   12761   1.58% |  0.2454  0.8170 |  0.3439  0.3781  0.3602  0.3707    1.73% |  0.3756  0.3831  0.3793  0.3816    1.61%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=competence_council, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    2263   2.21% |  0.2728  0.9205 |  0.2719  0.6200  0.3780  0.4936    5.04% |  0.4375  0.4200  0.4286  0.4234    2.12%

  Confusion matrix — fold 6 — fixed development threshold 0.227462
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,130              83
  Failure <5                            19              31
  ------------------------------------------------------------------
  Normal correctly rejected: 96.25%
  Failures successfully detected: 62.00%

  Confusion matrix — fold 6 — drift-aware rate 2.101%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,186              27
  Failure <5                            29              21
  ------------------------------------------------------------------
  Normal correctly rejected: 98.78%
  Failures successfully detected: 42.00%
     7    2915   4.08% |  0.2830  0.8468 |  0.3462  0.3025  0.3229  0.3103    3.57% |  0.4225  0.2521  0.3158  0.2742    2.44%

  Confusion matrix — fold 7 — fixed development threshold 0.227462
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,728              68
  Failure <5                            83              36
  ------------------------------------------------------------------
  Normal correctly rejected: 97.57%
  Failures successfully detected: 30.25%

  Confusion matrix — fold 7 — drift-aware rate 2.429%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,755              41
  Failure <5                            89              30
  ------------------------------------------------------------------
  Normal correctly rejected: 98.53%
  Failures successfully detected: 25.21%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    5178   3.26% |  0.2549  0.8629 |  0.3073  0.3964  0.3463  0.3747    4.21% |  0.4286  0.3018  0.3542  0.3208    2.30%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,858             151
  Failure <5                           102              67
  ------------------------------------------------------------------
  Normal correctly rejected: 96.99%
  Failures successfully detected: 39.64%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,941              68
  Failure <5                           118              51
  ------------------------------------------------------------------
  Normal correctly rejected: 98.64%
  Failures successfully detected: 30.18%

### DEVELOPMENT [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     1    2606   1.61% |  0.2279  0.8058 |  0.4815  0.3095  0.3768  0.3333    1.04% |  0.1975  0.3810  0.2602  0.3213    3.11%
     2    2204   1.81% |  0.5846  0.9370 |  0.5263  0.7500  0.6186  0.6912    2.59% |  0.4429  0.7750  0.5636  0.6739    3.18%
     3    2693   1.60% |  0.1018  0.6932 |  0.0909  0.2558  0.1341  0.1877    4.49% |  0.1146  0.2558  0.1583  0.2052    3.56%
     4    2275   0.84% |  0.2944  0.8394 |  0.1341  0.5789  0.2178  0.3481    3.60% |  0.1528  0.5789  0.2418  0.3716    3.16%
     5    2983   1.91% |  0.2549  0.8676 |  0.2569  0.4912  0.3373  0.4154    3.65% |  0.3878  0.3333  0.3585  0.3430    1.64%
------------------------------------------------------------------------------------------------------------------------------------
POOLED   12761   1.58% |  0.2615  0.8460 |  0.2348  0.4627  0.3116  0.3875    3.10% |  0.2391  0.4378  0.3093  0.3754    2.88%

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
HOLDOUT: pooled development threshold and warning ratio are frozen. No holdout label is used for threshold tuning.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

### HOLDOUT (test) [model=committee, beta=2]
  fold    test   base% |      AP     AUC |     P@t     R@t    F1@t    Fβ@t   rate@t |     P@r     R@r    F1@r    Fβ@r   rate@r
     6    2263   2.21% |  0.2791  0.9146 |  0.2238  0.6400  0.3316  0.4665    6.32% |  0.2824  0.4800  0.3556  0.4211    3.76%

  Confusion matrix — fold 6 — fixed development threshold 0.114172
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,102             111
  Failure <5                            18              32
  ------------------------------------------------------------------
  Normal correctly rejected: 94.98%
  Failures successfully detected: 64.00%

  Confusion matrix — fold 6 — drift-aware rate 3.765%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,152              61
  Failure <5                            26              24
  ------------------------------------------------------------------
  Normal correctly rejected: 97.24%
  Failures successfully detected: 48.00%
     7    2915   4.08% |  0.2748  0.8276 |  0.2908  0.3445  0.3154  0.3323    4.84% |  0.3071  0.3277  0.3171  0.3234    4.36%

  Confusion matrix — fold 7 — fixed development threshold 0.114172
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,696             100
  Failure <5                            78              41
  ------------------------------------------------------------------
  Normal correctly rejected: 96.42%
  Failures successfully detected: 34.45%

  Confusion matrix — fold 7 — drift-aware rate 4.353%
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         2,708              88
  Failure <5                            80              39
  ------------------------------------------------------------------
  Normal correctly rejected: 96.85%
  Failures successfully detected: 32.77%
------------------------------------------------------------------------------------------------------------------------------------
POOLED    5178   3.26% |  0.2543  0.8685 |  0.2570  0.4320  0.3223  0.3802    5.48% |  0.2972  0.3728  0.3307  0.3547    4.09%

  Confusion matrix — POOLED — fixed development threshold
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,798             211
  Failure <5                            96              73
  ------------------------------------------------------------------
  Normal correctly rejected: 95.79%
  Failures successfully detected: 43.20%

  Confusion matrix — POOLED — drift-aware warning policy
  ------------------------------------------------------------------
  Actual / Predicted            Normal >=5      Failure <5
  ------------------------------------------------------------------
  Normal >=5                         4,860             149
  Failure <5                           106              63
  ------------------------------------------------------------------
  Normal correctly rejected: 97.03%
  Failures successfully detected: 37.28%
