{
  "rows": 96,
  "models": [
    "committee",
    "suite26_seed_forest",
    "suite26_tree_only",
    "suite26_memory",
    "suite26_mint"
  ],
  "folds": [
    1,
    2
  ],
  "dev_folds": [
    1
  ],
  "holdout_folds": [
    2
  ],
  "macro": [
    {
      "split": "all",
      "model": "committee",
      "cells": 4,
      "valid_ap_cells": 4,
      "valid_auc_cells": 4,
      "total_rows": 96,
      "rows": 96,
      "missing_scores": 0,
      "positives": 54,
      "negatives": 42,
      "coverage": 1.0,
      "prevalence": 0.5624999999999999,
      "ap": 0.6950314737207438,
      "auc": 0.6128032124125874,
      "brier": 0.23375430051557888,
      "ece10": 0.15849057398736477,
      "ece": 0.15849057398736477,
      "mean_score": 0.49001228995621204,
      "calibration_gap": -0.07248771004378794,
      "tp_at_10": 7,
      "fp_at_10": 1,
      "fn_at_10": 47,
      "tn_at_10": 41,
      "precision_at_10": 0.875,
      "recall_at_10": 0.13060897435897434,
      "f1_at_10": 0.22698412698412698,
      "f2_at_10": 0.1573232323232323,
      "flagged_at_10": 8,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.5,
      "fbeta_at_10": 0.1573232323232323,
      "tp_at_15": 11,
      "fp_at_15": 5,
      "fn_at_15": 43,
      "tn_at_15": 37,
      "precision_at_15": 0.6875,
      "recall_at_15": 0.20552884615384615,
      "f1_at_15": 0.3158088235294118,
      "f2_at_15": 0.2388695862960569,
      "flagged_at_15": 16,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.5,
      "fbeta_at_15": 0.2388695862960569,
      "tp_at_20": 13,
      "fp_at_20": 7,
      "fn_at_20": 41,
      "tn_at_20": 35,
      "precision_at_20": 0.65,
      "recall_at_20": 0.24399038461538464,
      "f1_at_20": 0.3541083099906629,
      "f2_at_20": 0.27861131499791314,
      "flagged_at_20": 20,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.5,
      "fbeta_at_20": 0.27861131499791314,
      "tp_at_25": 15,
      "fp_at_25": 9,
      "fn_at_25": 39,
      "tn_at_25": 33,
      "precision_at_25": 0.625,
      "recall_at_25": 0.27884615384615385,
      "f1_at_25": 0.384768740031898,
      "f2_at_25": 0.3132868089764641,
      "flagged_at_25": 24,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.5,
      "fbeta_at_25": 0.3132868089764641
    },
    {
      "split": "all",
      "model": "suite26_seed_forest",
      "cells": 4,
      "valid_ap_cells": 4,
      "valid_auc_cells": 4,
      "total_rows": 96,
      "rows": 96,
      "missing_scores": 0,
      "positives": 54,
      "negatives": 42,
      "coverage": 1.0,
      "prevalence": 0.5624999999999999,
      "ap": 0.5559535858877963,
      "auc": 0.4678576631701632,
      "brier": 0.25384106162983244,
      "ece10": 0.08521077409386636,
      "ece": 0.08521077409386636,
      "mean_score": 0.5935048324366411,
      "calibration_gap": 0.031004832436641122,
      "tp_at_10": 4,
      "fp_at_10": 4,
      "fn_at_10": 50,
      "tn_at_10": 38,
      "precision_at_10": 0.5,
      "recall_at_10": 0.0765224358974359,
      "f1_at_10": 0.1325396825396825,
      "f2_at_10": 0.09208754208754209,
      "flagged_at_10": 8,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.4375,
      "fbeta_at_10": 0.09208754208754209,
      "tp_at_15": 9,
      "fp_at_15": 7,
      "fn_at_15": 45,
      "tn_at_15": 35,
      "precision_at_15": 0.5625,
      "recall_at_15": 0.17067307692307693,
      "f1_at_15": 0.2613970588235294,
      "f2_at_15": 0.19816580478345186,
      "flagged_at_15": 16,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.4583333333333333,
      "fbeta_at_15": 0.19816580478345186,
      "tp_at_20": 10,
      "fp_at_20": 10,
      "fn_at_20": 44,
      "tn_at_20": 32,
      "precision_at_20": 0.5,
      "recall_at_20": 0.18990384615384615,
      "f1_at_20": 0.2747432306255836,
      "f2_at_20": 0.21663572384612062,
      "flagged_at_20": 20,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.4375,
      "fbeta_at_20": 0.21663572384612062,
      "tp_at_25": 13,
      "fp_at_25": 11,
      "fn_at_25": 41,
      "tn_at_25": 31,
      "precision_at_25": 0.5416666666666666,
      "recall_at_25": 0.24399038461538464,
      "f1_at_25": 0.3357256778309409,
      "f2_at_25": 0.27387794198139026,
      "flagged_at_25": 24,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.45833333333333326,
      "fbeta_at_25": 0.27387794198139026
    },
    {
      "split": "all",
      "model": "suite26_tree_only",
      "cells": 4,
      "valid_ap_cells": 4,
      "valid_auc_cells": 4,
      "total_rows": 96,
      "rows": 96,
      "missing_scores": 0,
      "positives": 54,
      "negatives": 42,
      "coverage": 1.0,
      "prevalence": 0.5624999999999999,
      "ap": 0.6262186863370941,
      "auc": 0.5176873907342657,
      "brier": 0.2514407571962456,
      "ece10": 0.08607422374188899,
      "ece": 0.08607422374188899,
      "mean_score": 0.5959216871609291,
      "calibration_gap": 0.03342168716092908,
      "tp_at_10": 5,
      "fp_at_10": 3,
      "fn_at_10": 49,
      "tn_at_10": 39,
      "precision_at_10": 0.625,
      "recall_at_10": 0.09575320512820512,
      "f1_at_10": 0.16587301587301584,
      "f2_at_10": 0.11523569023569023,
      "flagged_at_10": 8,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.4583333333333333,
      "fbeta_at_10": 0.11523569023569023,
      "tp_at_15": 11,
      "fp_at_15": 5,
      "fn_at_15": 43,
      "tn_at_15": 37,
      "precision_at_15": 0.6875,
      "recall_at_15": 0.2107371794871795,
      "f1_at_15": 0.3220588235294118,
      "f2_at_15": 0.24452569489334197,
      "flagged_at_15": 16,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.49999999999999994,
      "fbeta_at_15": 0.24452569489334197,
      "tp_at_20": 13,
      "fp_at_20": 7,
      "fn_at_20": 41,
      "tn_at_20": 35,
      "precision_at_20": 0.65,
      "recall_at_20": 0.24559294871794873,
      "f1_at_20": 0.35574229691876746,
      "f2_at_20": 0.280266396096887,
      "flagged_at_20": 20,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.5,
      "fbeta_at_20": 0.280266396096887,
      "tp_at_25": 15,
      "fp_at_25": 9,
      "fn_at_25": 39,
      "tn_at_25": 33,
      "precision_at_25": 0.625,
      "recall_at_25": 0.28205128205128205,
      "f1_at_25": 0.3876927166400851,
      "f2_at_25": 0.3164796569968984,
      "flagged_at_25": 24,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.5,
      "fbeta_at_25": 0.3164796569968984
    },
    {
      "split": "all",
      "model": "suite26_memory",
      "cells": 4,
      "valid_ap_cells": 4,
      "valid_auc_cells": 4,
      "total_rows": 96,
      "rows": 96,
      "missing_scores": 0,
      "positives": 54,
      "negatives": 42,
      "coverage": 1.0,
      "prevalence": 0.5624999999999999,
      "ap": 0.6360014412103354,
      "auc": 0.5414147484459985,
      "brier": 0.25057021483093045,
      "ece10": 0.09496284089982511,
      "ece": 0.09496284089982511,
      "mean_score": 0.5948701010396082,
      "calibration_gap": 0.03237010103960833,
      "tp_at_10": 5,
      "fp_at_10": 3,
      "fn_at_10": 49,
      "tn_at_10": 39,
      "precision_at_10": 0.625,
      "recall_at_10": 0.09575320512820512,
      "f1_at_10": 0.16587301587301584,
      "f2_at_10": 0.11523569023569023,
      "flagged_at_10": 8,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.4583333333333333,
      "fbeta_at_10": 0.11523569023569023,
      "tp_at_15": 11,
      "fp_at_15": 5,
      "fn_at_15": 43,
      "tn_at_15": 37,
      "precision_at_15": 0.6875,
      "recall_at_15": 0.2107371794871795,
      "f1_at_15": 0.3220588235294118,
      "f2_at_15": 0.24452569489334197,
      "flagged_at_15": 16,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.49999999999999994,
      "fbeta_at_15": 0.24452569489334197,
      "tp_at_20": 13,
      "fp_at_20": 7,
      "fn_at_20": 41,
      "tn_at_20": 35,
      "precision_at_20": 0.65,
      "recall_at_20": 0.24559294871794873,
      "f1_at_20": 0.35574229691876746,
      "f2_at_20": 0.280266396096887,
      "flagged_at_20": 20,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.5,
      "fbeta_at_20": 0.280266396096887,
      "tp_at_25": 16,
      "fp_at_25": 8,
      "fn_at_25": 38,
      "tn_at_25": 34,
      "precision_at_25": 0.6666666666666666,
      "recall_at_25": 0.3012820512820513,
      "f1_at_25": 0.41400850611376927,
      "f2_at_25": 0.33803138113482945,
      "flagged_at_25": 24,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.5208333333333333,
      "fbeta_at_25": 0.33803138113482945
    },
    {
      "split": "all",
      "model": "suite26_mint",
      "cells": 4,
      "valid_ap_cells": 4,
      "valid_auc_cells": 4,
      "total_rows": 96,
      "rows": 96,
      "missing_scores": 0,
      "positives": 54,
      "negatives": 42,
      "coverage": 1.0,
      "prevalence": 0.5624999999999999,
      "ap": 0.6489803026266968,
      "auc": 0.5468643769425019,
      "brier": 0.2506089490450475,
      "ece10": 0.09520786690215269,
      "ece": 0.09520786690215269,
      "mean_score": 0.5947610642760992,
      "calibration_gap": 0.03226106427609923,
      "tp_at_10": 5,
      "fp_at_10": 3,
      "fn_at_10": 49,
      "tn_at_10": 39,
      "precision_at_10": 0.625,
      "recall_at_10": 0.09575320512820512,
      "f1_at_10": 0.16587301587301584,
      "f2_at_10": 0.11523569023569023,
      "flagged_at_10": 8,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.4583333333333333,
      "fbeta_at_10": 0.11523569023569023,
      "tp_at_15": 11,
      "fp_at_15": 5,
      "fn_at_15": 43,
      "tn_at_15": 37,
      "precision_at_15": 0.6875,
      "recall_at_15": 0.2107371794871795,
      "f1_at_15": 0.3220588235294118,
      "f2_at_15": 0.24452569489334197,
      "flagged_at_15": 16,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.49999999999999994,
      "fbeta_at_15": 0.24452569489334197,
      "tp_at_20": 13,
      "fp_at_20": 7,
      "fn_at_20": 41,
      "tn_at_20": 35,
      "precision_at_20": 0.65,
      "recall_at_20": 0.24559294871794873,
      "f1_at_20": 0.35574229691876746,
      "f2_at_20": 0.280266396096887,
      "flagged_at_20": 20,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.5,
      "fbeta_at_20": 0.280266396096887,
      "tp_at_25": 16,
      "fp_at_25": 8,
      "fn_at_25": 38,
      "tn_at_25": 34,
      "precision_at_25": 0.6666666666666666,
      "recall_at_25": 0.3012820512820513,
      "f1_at_25": 0.41400850611376927,
      "f2_at_25": 0.33803138113482945,
      "flagged_at_25": 24,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.5208333333333333,
      "fbeta_at_25": 0.33803138113482945
    },
    {
      "split": "development",
      "model": "committee",
      "cells": 2,
      "valid_ap_cells": 2,
      "valid_auc_cells": 2,
      "total_rows": 48,
      "rows": 48,
      "missing_scores": 0,
      "positives": 25,
      "negatives": 23,
      "coverage": 1.0,
      "prevalence": 0.5208333333333333,
      "ap": 0.7116829934388236,
      "auc": 0.6448863636363636,
      "brier": 0.2269242431321784,
      "ece10": 0.15063556904594103,
      "ece": 0.15063556904594103,
      "mean_score": 0.47503101204832393,
      "calibration_gap": -0.045802321285009384,
      "tp_at_10": 4,
      "fp_at_10": 0,
      "fn_at_10": 21,
      "tn_at_10": 23,
      "precision_at_10": 1.0,
      "recall_at_10": 0.16025641025641024,
      "f1_at_10": 0.2761904761904762,
      "f2_at_10": 0.1925925925925926,
      "flagged_at_10": 4,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.5625,
      "fbeta_at_10": 0.1925925925925926,
      "tp_at_15": 6,
      "fp_at_15": 2,
      "fn_at_15": 19,
      "tn_at_15": 21,
      "precision_at_15": 0.75,
      "recall_at_15": 0.2403846153846154,
      "f1_at_15": 0.36397058823529416,
      "f2_at_15": 0.27815934065934067,
      "flagged_at_15": 8,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.5625,
      "fbeta_at_15": 0.27815934065934067,
      "tp_at_20": 7,
      "fp_at_20": 3,
      "fn_at_20": 18,
      "tn_at_20": 20,
      "precision_at_20": 0.7,
      "recall_at_20": 0.27884615384615385,
      "f1_at_20": 0.39869281045751637,
      "f2_at_20": 0.3169480304534922,
      "flagged_at_20": 10,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.5625,
      "fbeta_at_20": 0.3169480304534922,
      "tp_at_25": 8,
      "fp_at_25": 4,
      "fn_at_25": 17,
      "tn_at_25": 19,
      "precision_at_25": 0.6666666666666667,
      "recall_at_25": 0.3173076923076923,
      "f1_at_25": 0.4298245614035088,
      "f2_at_25": 0.3544061302681992,
      "flagged_at_25": 12,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.5625,
      "fbeta_at_25": 0.3544061302681992
    },
    {
      "split": "development",
      "model": "suite26_seed_forest",
      "cells": 2,
      "valid_ap_cells": 2,
      "valid_auc_cells": 2,
      "total_rows": 48,
      "rows": 48,
      "missing_scores": 0,
      "positives": 25,
      "negatives": 23,
      "coverage": 1.0,
      "prevalence": 0.5208333333333333,
      "ap": 0.5261470985155194,
      "auc": 0.5085955710955712,
      "brier": 0.25801281464443526,
      "ece10": 0.07957709208130842,
      "ece": 0.07957709208130842,
      "mean_score": 0.6004104254146417,
      "calibration_gap": 0.07957709208130842,
      "tp_at_10": 3,
      "fp_at_10": 1,
      "fn_at_10": 22,
      "tn_at_10": 22,
      "precision_at_10": 0.75,
      "recall_at_10": 0.12179487179487179,
      "f1_at_10": 0.2095238095238095,
      "f2_at_10": 0.1462962962962963,
      "flagged_at_10": 4,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.5208333333333334,
      "fbeta_at_10": 0.1462962962962963,
      "tp_at_15": 5,
      "fp_at_15": 3,
      "fn_at_15": 20,
      "tn_at_15": 20,
      "precision_at_15": 0.625,
      "recall_at_15": 0.20192307692307693,
      "f1_at_15": 0.30514705882352944,
      "f2_at_15": 0.23351648351648352,
      "flagged_at_15": 8,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.5208333333333334,
      "fbeta_at_15": 0.23351648351648352,
      "tp_at_20": 6,
      "fp_at_20": 4,
      "fn_at_20": 19,
      "tn_at_20": 19,
      "precision_at_20": 0.6,
      "recall_at_20": 0.2403846153846154,
      "f1_at_20": 0.3431372549019608,
      "f2_at_20": 0.2730883813306852,
      "flagged_at_20": 10,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.5208333333333333,
      "fbeta_at_20": 0.2730883813306852,
      "tp_at_25": 7,
      "fp_at_25": 5,
      "fn_at_25": 18,
      "tn_at_25": 18,
      "precision_at_25": 0.5833333333333333,
      "recall_at_25": 0.27884615384615385,
      "f1_at_25": 0.3771929824561403,
      "f2_at_25": 0.3113026819923372,
      "flagged_at_25": 12,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.5208333333333333,
      "fbeta_at_25": 0.3113026819923372
    },
    {
      "split": "development",
      "model": "suite26_tree_only",
      "cells": 2,
      "valid_ap_cells": 2,
      "valid_auc_cells": 2,
      "total_rows": 48,
      "rows": 48,
      "missing_scores": 0,
      "positives": 25,
      "negatives": 23,
      "coverage": 1.0,
      "prevalence": 0.5208333333333333,
      "ap": 0.6487141736214367,
      "auc": 0.5537587412587412,
      "brier": 0.25797697789953694,
      "ece10": 0.086236039797465,
      "ece": 0.086236039797465,
      "mean_score": 0.6070693731307983,
      "calibration_gap": 0.08623603979746503,
      "tp_at_10": 3,
      "fp_at_10": 1,
      "fn_at_10": 22,
      "tn_at_10": 22,
      "precision_at_10": 0.75,
      "recall_at_10": 0.12179487179487179,
      "f1_at_10": 0.2095238095238095,
      "f2_at_10": 0.1462962962962963,
      "flagged_at_10": 4,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.5208333333333334,
      "fbeta_at_10": 0.1462962962962963,
      "tp_at_15": 7,
      "fp_at_15": 1,
      "fn_at_15": 18,
      "tn_at_15": 22,
      "precision_at_15": 0.875,
      "recall_at_15": 0.28205128205128205,
      "f1_at_15": 0.42647058823529416,
      "f2_at_15": 0.32623626373626374,
      "flagged_at_15": 8,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.6041666666666666,
      "fbeta_at_15": 0.32623626373626374,
      "tp_at_20": 7,
      "fp_at_20": 3,
      "fn_at_20": 18,
      "tn_at_20": 20,
      "precision_at_20": 0.7,
      "recall_at_20": 0.28205128205128205,
      "f1_at_20": 0.4019607843137255,
      "f2_at_20": 0.32025819265143995,
      "flagged_at_20": 10,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.5625,
      "fbeta_at_20": 0.32025819265143995,
      "tp_at_25": 8,
      "fp_at_25": 4,
      "fn_at_25": 17,
      "tn_at_25": 19,
      "precision_at_25": 0.6666666666666667,
      "recall_at_25": 0.32371794871794873,
      "f1_at_25": 0.43567251461988304,
      "f2_at_25": 0.3607918263090677,
      "flagged_at_25": 12,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.5625,
      "fbeta_at_25": 0.3607918263090677
    },
    {
      "split": "development",
      "model": "suite26_memory",
      "cells": 2,
      "valid_ap_cells": 2,
      "valid_auc_cells": 2,
      "total_rows": 48,
      "rows": 48,
      "missing_scores": 0,
      "positives": 25,
      "negatives": 23,
      "coverage": 1.0,
      "prevalence": 0.5208333333333333,
      "ap": 0.6597263460653553,
      "auc": 0.5851787101787101,
      "brier": 0.25650510047545716,
      "ece10": 0.10674304887652399,
      "ece": 0.10674304887652399,
      "mean_score": 0.603540938347578,
      "calibration_gap": 0.08270760501424473,
      "tp_at_10": 3,
      "fp_at_10": 1,
      "fn_at_10": 22,
      "tn_at_10": 22,
      "precision_at_10": 0.75,
      "recall_at_10": 0.12179487179487179,
      "f1_at_10": 0.2095238095238095,
      "f2_at_10": 0.1462962962962963,
      "flagged_at_10": 4,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.5208333333333334,
      "fbeta_at_10": 0.1462962962962963,
      "tp_at_15": 7,
      "fp_at_15": 1,
      "fn_at_15": 18,
      "tn_at_15": 22,
      "precision_at_15": 0.875,
      "recall_at_15": 0.28205128205128205,
      "f1_at_15": 0.42647058823529416,
      "f2_at_15": 0.32623626373626374,
      "flagged_at_15": 8,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.6041666666666666,
      "fbeta_at_15": 0.32623626373626374,
      "tp_at_20": 7,
      "fp_at_20": 3,
      "fn_at_20": 18,
      "tn_at_20": 20,
      "precision_at_20": 0.7,
      "recall_at_20": 0.28205128205128205,
      "f1_at_20": 0.4019607843137255,
      "f2_at_20": 0.32025819265143995,
      "flagged_at_20": 10,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.5625,
      "fbeta_at_20": 0.32025819265143995,
      "tp_at_25": 8,
      "fp_at_25": 4,
      "fn_at_25": 17,
      "tn_at_25": 19,
      "precision_at_25": 0.6666666666666667,
      "recall_at_25": 0.32371794871794873,
      "f1_at_25": 0.43567251461988304,
      "f2_at_25": 0.3607918263090677,
      "flagged_at_25": 12,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.5625,
      "fbeta_at_25": 0.3607918263090677
    },
    {
      "split": "development",
      "model": "suite26_mint",
      "cells": 2,
      "valid_ap_cells": 2,
      "valid_auc_cells": 2,
      "total_rows": 48,
      "rows": 48,
      "missing_scores": 0,
      "positives": 25,
      "negatives": 23,
      "coverage": 1.0,
      "prevalence": 0.5208333333333333,
      "ap": 0.6789571152961246,
      "auc": 0.5886752136752136,
      "brier": 0.2563922969490187,
      "ece10": 0.10685616110761961,
      "ece": 0.10685616110761961,
      "mean_score": 0.6037377826869488,
      "calibration_gap": 0.08290444935361546,
      "tp_at_10": 3,
      "fp_at_10": 1,
      "fn_at_10": 22,
      "tn_at_10": 22,
      "precision_at_10": 0.75,
      "recall_at_10": 0.12179487179487179,
      "f1_at_10": 0.2095238095238095,
      "f2_at_10": 0.1462962962962963,
      "flagged_at_10": 4,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.5208333333333334,
      "fbeta_at_10": 0.1462962962962963,
      "tp_at_15": 7,
      "fp_at_15": 1,
      "fn_at_15": 18,
      "tn_at_15": 22,
      "precision_at_15": 0.875,
      "recall_at_15": 0.28205128205128205,
      "f1_at_15": 0.42647058823529416,
      "f2_at_15": 0.32623626373626374,
      "flagged_at_15": 8,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.6041666666666666,
      "fbeta_at_15": 0.32623626373626374,
      "tp_at_20": 7,
      "fp_at_20": 3,
      "fn_at_20": 18,
      "tn_at_20": 20,
      "precision_at_20": 0.7,
      "recall_at_20": 0.28205128205128205,
      "f1_at_20": 0.4019607843137255,
      "f2_at_20": 0.32025819265143995,
      "flagged_at_20": 10,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.5625,
      "fbeta_at_20": 0.32025819265143995,
      "tp_at_25": 8,
      "fp_at_25": 4,
      "fn_at_25": 17,
      "tn_at_25": 19,
      "precision_at_25": 0.6666666666666667,
      "recall_at_25": 0.32371794871794873,
      "f1_at_25": 0.43567251461988304,
      "f2_at_25": 0.3607918263090677,
      "flagged_at_25": 12,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.5625,
      "fbeta_at_25": 0.3607918263090677
    },
    {
      "split": "holdout",
      "model": "committee",
      "cells": 2,
      "valid_ap_cells": 2,
      "valid_auc_cells": 2,
      "total_rows": 48,
      "rows": 48,
      "missing_scores": 0,
      "positives": 29,
      "negatives": 19,
      "coverage": 1.0,
      "prevalence": 0.6041666666666666,
      "ap": 0.6783799540026638,
      "auc": 0.5807200611888113,
      "brier": 0.24058435789897936,
      "ece10": 0.1663455789287885,
      "ece": 0.1663455789287885,
      "mean_score": 0.5049935678641001,
      "calibration_gap": -0.0991730988025665,
      "tp_at_10": 3,
      "fp_at_10": 1,
      "fn_at_10": 26,
      "tn_at_10": 18,
      "precision_at_10": 0.75,
      "recall_at_10": 0.10096153846153846,
      "f1_at_10": 0.17777777777777776,
      "f2_at_10": 0.12205387205387205,
      "flagged_at_10": 4,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.4375,
      "fbeta_at_10": 0.12205387205387205,
      "tp_at_15": 5,
      "fp_at_15": 3,
      "fn_at_15": 24,
      "tn_at_15": 16,
      "precision_at_15": 0.625,
      "recall_at_15": 0.17067307692307693,
      "f1_at_15": 0.2676470588235294,
      "f2_at_15": 0.19957983193277312,
      "flagged_at_15": 8,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.4375,
      "fbeta_at_15": 0.19957983193277312,
      "tp_at_20": 6,
      "fp_at_20": 4,
      "fn_at_20": 23,
      "tn_at_20": 15,
      "precision_at_20": 0.6,
      "recall_at_20": 0.2091346153846154,
      "f1_at_20": 0.30952380952380953,
      "f2_at_20": 0.2402745995423341,
      "flagged_at_20": 10,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.4375,
      "fbeta_at_20": 0.2402745995423341,
      "tp_at_25": 7,
      "fp_at_25": 5,
      "fn_at_25": 22,
      "tn_at_25": 14,
      "precision_at_25": 0.5833333333333333,
      "recall_at_25": 0.2403846153846154,
      "f1_at_25": 0.3397129186602871,
      "f2_at_25": 0.2721674876847291,
      "flagged_at_25": 12,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.4375,
      "fbeta_at_25": 0.2721674876847291
    },
    {
      "split": "holdout",
      "model": "suite26_seed_forest",
      "cells": 2,
      "valid_ap_cells": 2,
      "valid_auc_cells": 2,
      "total_rows": 48,
      "rows": 48,
      "missing_scores": 0,
      "positives": 29,
      "negatives": 19,
      "coverage": 1.0,
      "prevalence": 0.6041666666666666,
      "ap": 0.5857600732600732,
      "auc": 0.4271197552447552,
      "brier": 0.24966930861522968,
      "ece10": 0.0908444561064243,
      "ece": 0.0908444561064243,
      "mean_score": 0.5865992394586405,
      "calibration_gap": -0.017567427208026176,
      "tp_at_10": 1,
      "fp_at_10": 3,
      "fn_at_10": 28,
      "tn_at_10": 16,
      "precision_at_10": 0.25,
      "recall_at_10": 0.03125,
      "f1_at_10": 0.05555555555555555,
      "f2_at_10": 0.03787878787878788,
      "flagged_at_10": 4,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.35416666666666663,
      "fbeta_at_10": 0.03787878787878788,
      "tp_at_15": 4,
      "fp_at_15": 4,
      "fn_at_15": 25,
      "tn_at_15": 15,
      "precision_at_15": 0.5,
      "recall_at_15": 0.13942307692307693,
      "f1_at_15": 0.21764705882352942,
      "f2_at_15": 0.16281512605042017,
      "flagged_at_15": 8,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.3958333333333333,
      "fbeta_at_15": 0.16281512605042017,
      "tp_at_20": 4,
      "fp_at_20": 6,
      "fn_at_20": 25,
      "tn_at_20": 13,
      "precision_at_20": 0.4,
      "recall_at_20": 0.13942307692307693,
      "f1_at_20": 0.20634920634920634,
      "f2_at_20": 0.16018306636155605,
      "flagged_at_20": 10,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.3541666666666667,
      "fbeta_at_20": 0.16018306636155605,
      "tp_at_25": 6,
      "fp_at_25": 6,
      "fn_at_25": 23,
      "tn_at_25": 13,
      "precision_at_25": 0.5,
      "recall_at_25": 0.2091346153846154,
      "f1_at_25": 0.2942583732057416,
      "f2_at_25": 0.23645320197044334,
      "flagged_at_25": 12,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.3958333333333333,
      "fbeta_at_25": 0.23645320197044334
    },
    {
      "split": "holdout",
      "model": "suite26_tree_only",
      "cells": 2,
      "valid_ap_cells": 2,
      "valid_auc_cells": 2,
      "total_rows": 48,
      "rows": 48,
      "missing_scores": 0,
      "positives": 29,
      "negatives": 19,
      "coverage": 1.0,
      "prevalence": 0.6041666666666666,
      "ap": 0.6037231990527514,
      "auc": 0.4816160402097902,
      "brier": 0.2449045364929543,
      "ece10": 0.08591240768631297,
      "ece": 0.08591240768631297,
      "mean_score": 0.5847740011910598,
      "calibration_gap": -0.019392665475606863,
      "tp_at_10": 2,
      "fp_at_10": 2,
      "fn_at_10": 27,
      "tn_at_10": 17,
      "precision_at_10": 0.5,
      "recall_at_10": 0.06971153846153846,
      "f1_at_10": 0.12222222222222222,
      "f2_at_10": 0.08417508417508418,
      "flagged_at_10": 4,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.3958333333333333,
      "fbeta_at_10": 0.08417508417508418,
      "tp_at_15": 4,
      "fp_at_15": 4,
      "fn_at_15": 25,
      "tn_at_15": 15,
      "precision_at_15": 0.5,
      "recall_at_15": 0.13942307692307693,
      "f1_at_15": 0.21764705882352942,
      "f2_at_15": 0.16281512605042017,
      "flagged_at_15": 8,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.3958333333333333,
      "fbeta_at_15": 0.16281512605042017,
      "tp_at_20": 6,
      "fp_at_20": 4,
      "fn_at_20": 23,
      "tn_at_20": 15,
      "precision_at_20": 0.6,
      "recall_at_20": 0.2091346153846154,
      "f1_at_20": 0.30952380952380953,
      "f2_at_20": 0.2402745995423341,
      "flagged_at_20": 10,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.4375,
      "fbeta_at_20": 0.2402745995423341,
      "tp_at_25": 7,
      "fp_at_25": 5,
      "fn_at_25": 22,
      "tn_at_25": 14,
      "precision_at_25": 0.5833333333333333,
      "recall_at_25": 0.2403846153846154,
      "f1_at_25": 0.3397129186602871,
      "f2_at_25": 0.2721674876847291,
      "flagged_at_25": 12,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.4375,
      "fbeta_at_25": 0.2721674876847291
    },
    {
      "split": "holdout",
      "model": "suite26_memory",
      "cells": 2,
      "valid_ap_cells": 2,
      "valid_auc_cells": 2,
      "total_rows": 48,
      "rows": 48,
      "missing_scores": 0,
      "positives": 29,
      "negatives": 19,
      "coverage": 1.0,
      "prevalence": 0.6041666666666666,
      "ap": 0.6122765363553154,
      "auc": 0.4976507867132867,
      "brier": 0.2446353291864038,
      "ece10": 0.08318263292312622,
      "ece": 0.08318263292312622,
      "mean_score": 0.5861992637316386,
      "calibration_gap": -0.017967402935028076,
      "tp_at_10": 2,
      "fp_at_10": 2,
      "fn_at_10": 27,
      "tn_at_10": 17,
      "precision_at_10": 0.5,
      "recall_at_10": 0.06971153846153846,
      "f1_at_10": 0.12222222222222222,
      "f2_at_10": 0.08417508417508418,
      "flagged_at_10": 4,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.3958333333333333,
      "fbeta_at_10": 0.08417508417508418,
      "tp_at_15": 4,
      "fp_at_15": 4,
      "fn_at_15": 25,
      "tn_at_15": 15,
      "precision_at_15": 0.5,
      "recall_at_15": 0.13942307692307693,
      "f1_at_15": 0.21764705882352942,
      "f2_at_15": 0.16281512605042017,
      "flagged_at_15": 8,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.3958333333333333,
      "fbeta_at_15": 0.16281512605042017,
      "tp_at_20": 6,
      "fp_at_20": 4,
      "fn_at_20": 23,
      "tn_at_20": 15,
      "precision_at_20": 0.6,
      "recall_at_20": 0.2091346153846154,
      "f1_at_20": 0.30952380952380953,
      "f2_at_20": 0.2402745995423341,
      "flagged_at_20": 10,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.4375,
      "fbeta_at_20": 0.2402745995423341,
      "tp_at_25": 8,
      "fp_at_25": 4,
      "fn_at_25": 21,
      "tn_at_25": 15,
      "precision_at_25": 0.6666666666666666,
      "recall_at_25": 0.27884615384615385,
      "f1_at_25": 0.3923444976076555,
      "f2_at_25": 0.31527093596059114,
      "flagged_at_25": 12,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.47916666666666663,
      "fbeta_at_25": 0.31527093596059114
    },
    {
      "split": "holdout",
      "model": "suite26_mint",
      "cells": 2,
      "valid_ap_cells": 2,
      "valid_auc_cells": 2,
      "total_rows": 48,
      "rows": 48,
      "missing_scores": 0,
      "positives": 29,
      "negatives": 19,
      "coverage": 1.0,
      "prevalence": 0.6041666666666666,
      "ap": 0.6190034899572692,
      "auc": 0.5050535402097902,
      "brier": 0.24482560114107624,
      "ece10": 0.08355957269668576,
      "ece": 0.08355957269668576,
      "mean_score": 0.5857843458652496,
      "calibration_gap": -0.018382320801416996,
      "tp_at_10": 2,
      "fp_at_10": 2,
      "fn_at_10": 27,
      "tn_at_10": 17,
      "precision_at_10": 0.5,
      "recall_at_10": 0.06971153846153846,
      "f1_at_10": 0.12222222222222222,
      "f2_at_10": 0.08417508417508418,
      "flagged_at_10": 4,
      "flag_rate_at_10": 0.08333333333333333,
      "accuracy_at_10": 0.3958333333333333,
      "fbeta_at_10": 0.08417508417508418,
      "tp_at_15": 4,
      "fp_at_15": 4,
      "fn_at_15": 25,
      "tn_at_15": 15,
      "precision_at_15": 0.5,
      "recall_at_15": 0.13942307692307693,
      "f1_at_15": 0.21764705882352942,
      "f2_at_15": 0.16281512605042017,
      "flagged_at_15": 8,
      "flag_rate_at_15": 0.16666666666666666,
      "accuracy_at_15": 0.3958333333333333,
      "fbeta_at_15": 0.16281512605042017,
      "tp_at_20": 6,
      "fp_at_20": 4,
      "fn_at_20": 23,
      "tn_at_20": 15,
      "precision_at_20": 0.6,
      "recall_at_20": 0.2091346153846154,
      "f1_at_20": 0.30952380952380953,
      "f2_at_20": 0.2402745995423341,
      "flagged_at_20": 10,
      "flag_rate_at_20": 0.20833333333333334,
      "accuracy_at_20": 0.4375,
      "fbeta_at_20": 0.2402745995423341,
      "tp_at_25": 8,
      "fp_at_25": 4,
      "fn_at_25": 21,
      "tn_at_25": 15,
      "precision_at_25": 0.6666666666666666,
      "recall_at_25": 0.27884615384615385,
      "f1_at_25": 0.3923444976076555,
      "f2_at_25": 0.31527093596059114,
      "flagged_at_25": 12,
      "flag_rate_at_25": 0.25,
      "accuracy_at_25": 0.47916666666666663,
      "fbeta_at_25": 0.31527093596059114
    }
  ],
  "paired_summary": [
    {
      "split": "all",
      "model_a": "suite26_mint",
      "model_b": "committee",
      "cells": 4,
      "valid_ap_cells": 4,
      "rows": 96,
      "total_rows": 96,
      "wins": 1,
      "ties": 0,
      "mean_delta_ap": -0.04605117109404691,
      "mean_delta_auc": -0.0659388354700855,
      "mean_delta_f2_at_20": 0.0016550810989738596,
      "mean_delta_brier": 0.016854648529468587,
      "mean_delta_ece10": -0.06328270708521208
    },
    {
      "split": "all",
      "model_a": "suite26_mint",
      "model_b": "suite26_seed_forest",
      "cells": 4,
      "valid_ap_cells": 4,
      "rows": 96,
      "total_rows": 96,
      "wins": 4,
      "ties": 0,
      "mean_delta_ap": 0.09302671673890048,
      "mean_delta_auc": 0.07900671377233875,
      "mean_delta_f2_at_20": 0.06363067225076638,
      "mean_delta_brier": -0.0032321125847850005,
      "mean_delta_ece10": 0.009997092808286333
    },
    {
      "split": "all",
      "model_a": "suite26_mint",
      "model_b": "suite26_tree_only",
      "cells": 4,
      "valid_ap_cells": 4,
      "rows": 96,
      "total_rows": 96,
      "wins": 4,
      "ties": 0,
      "mean_delta_ap": 0.02276161628960277,
      "mean_delta_auc": 0.02917698620823618,
      "mean_delta_f2_at_20": 0.0,
      "mean_delta_brier": -0.0008318081511981476,
      "mean_delta_ece10": 0.00913364316026371
    },
    {
      "split": "all",
      "model_a": "suite26_mint",
      "model_b": "suite26_memory",
      "cells": 4,
      "valid_ap_cells": 4,
      "rows": 96,
      "total_rows": 96,
      "wins": 3,
      "ties": 1,
      "mean_delta_ap": 0.012978861416361481,
      "mean_delta_auc": 0.005449628496503489,
      "mean_delta_f2_at_20": 0.0,
      "mean_delta_brier": 3.8734214117021326e-05,
      "mean_delta_ece10": 0.0002450260023275884
    },
    {
      "split": "all",
      "model_a": "suite26_tree_only",
      "model_b": "suite26_seed_forest",
      "cells": 4,
      "valid_ap_cells": 4,
      "rows": 96,
      "total_rows": 96,
      "wins": 4,
      "ties": 0,
      "mean_delta_ap": 0.07026510044929771,
      "mean_delta_auc": 0.049829727564102574,
      "mean_delta_f2_at_20": 0.06363067225076638,
      "mean_delta_brier": -0.002400304433586853,
      "mean_delta_ece10": 0.0008634496480226222
    },
    {
      "split": "all",
      "model_a": "suite26_memory",
      "model_b": "suite26_tree_only",
      "cells": 4,
      "valid_ap_cells": 4,
      "rows": 96,
      "total_rows": 96,
      "wins": 4,
      "ties": 0,
      "mean_delta_ap": 0.00978275487324129,
      "mean_delta_auc": 0.02372735771173269,
      "mean_delta_f2_at_20": 0.0,
      "mean_delta_brier": -0.0008705423653151689,
      "mean_delta_ece10": 0.008888617157936122
    },
    {
      "split": "development",
      "model_a": "suite26_mint",
      "model_b": "committee",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 1,
      "ties": 0,
      "mean_delta_ap": -0.03272587814269906,
      "mean_delta_auc": -0.05621114996115001,
      "mean_delta_f2_at_20": 0.003310162197947719,
      "mean_delta_brier": 0.029468053816840298,
      "mean_delta_ece10": -0.04377940793832141
    },
    {
      "split": "development",
      "model_a": "suite26_mint",
      "model_b": "suite26_seed_forest",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 2,
      "ties": 0,
      "mean_delta_ap": 0.15281001678060507,
      "mean_delta_auc": 0.08007964257964251,
      "mean_delta_f2_at_20": 0.047169811320754734,
      "mean_delta_brier": -0.0016205176954165734,
      "mean_delta_ece10": 0.0272790690263112
    },
    {
      "split": "development",
      "model_a": "suite26_mint",
      "model_b": "suite26_tree_only",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 2,
      "ties": 0,
      "mean_delta_ap": 0.030242941674687884,
      "mean_delta_auc": 0.03491647241647236,
      "mean_delta_f2_at_20": 0.0,
      "mean_delta_brier": -0.0015846809505182247,
      "mean_delta_ece10": 0.020620121310154626
    },
    {
      "split": "development",
      "model_a": "suite26_mint",
      "model_b": "suite26_memory",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 1,
      "ties": 1,
      "mean_delta_ap": 0.019230769230769273,
      "mean_delta_auc": 0.003496503496503489,
      "mean_delta_f2_at_20": 0.0,
      "mean_delta_brier": -0.00011280352643841529,
      "mean_delta_ece10": 0.00011311223109563423
    },
    {
      "split": "development",
      "model_a": "suite26_tree_only",
      "model_b": "suite26_seed_forest",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 2,
      "ties": 0,
      "mean_delta_ap": 0.12256707510591719,
      "mean_delta_auc": 0.04516317016317015,
      "mean_delta_f2_at_20": 0.047169811320754734,
      "mean_delta_brier": -3.583674489834876e-05,
      "mean_delta_ece10": 0.006658947716156573
    },
    {
      "split": "development",
      "model_a": "suite26_memory",
      "model_b": "suite26_tree_only",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 2,
      "ties": 0,
      "mean_delta_ap": 0.01101217244391861,
      "mean_delta_auc": 0.03141996891996887,
      "mean_delta_f2_at_20": 0.0,
      "mean_delta_brier": -0.0014718774240798094,
      "mean_delta_ece10": 0.02050700907905899
    },
    {
      "split": "holdout",
      "model_a": "suite26_mint",
      "model_b": "committee",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 0,
      "ties": 0,
      "mean_delta_ap": -0.05937646404539476,
      "mean_delta_auc": -0.07566652097902099,
      "mean_delta_f2_at_20": 0.0,
      "mean_delta_brier": 0.004241243242096876,
      "mean_delta_ece10": -0.08278600623210275
    },
    {
      "split": "holdout",
      "model_a": "suite26_mint",
      "model_b": "suite26_seed_forest",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 2,
      "ties": 0,
      "mean_delta_ap": 0.03324341669719588,
      "mean_delta_auc": 0.077933784965035,
      "mean_delta_f2_at_20": 0.08009153318077802,
      "mean_delta_brier": -0.0048437074741534275,
      "mean_delta_ece10": -0.007284883409738534
    },
    {
      "split": "holdout",
      "model_a": "suite26_mint",
      "model_b": "suite26_tree_only",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 2,
      "ties": 0,
      "mean_delta_ap": 0.015280290904517657,
      "mean_delta_auc": 0.0234375,
      "mean_delta_f2_at_20": 0.0,
      "mean_delta_brier": -7.893535187807055e-05,
      "mean_delta_ece10": -0.0023528349896272047
    },
    {
      "split": "holdout",
      "model_a": "suite26_mint",
      "model_b": "suite26_memory",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 2,
      "ties": 0,
      "mean_delta_ap": 0.006726953601953689,
      "mean_delta_auc": 0.007402753496503489,
      "mean_delta_f2_at_20": 0.0,
      "mean_delta_brier": 0.00019027195467245794,
      "mean_delta_ece10": 0.00037693977355954256
    },
    {
      "split": "holdout",
      "model_a": "suite26_tree_only",
      "model_b": "suite26_seed_forest",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 2,
      "ties": 0,
      "mean_delta_ap": 0.017963125792678225,
      "mean_delta_auc": 0.054496284965035,
      "mean_delta_f2_at_20": 0.08009153318077802,
      "mean_delta_brier": -0.004764772122275357,
      "mean_delta_ece10": -0.004932048420111329
    },
    {
      "split": "holdout",
      "model_a": "suite26_memory",
      "model_b": "suite26_tree_only",
      "cells": 2,
      "valid_ap_cells": 2,
      "rows": 48,
      "total_rows": 48,
      "wins": 2,
      "ties": 0,
      "mean_delta_ap": 0.008553337302563968,
      "mean_delta_auc": 0.01603474650349651,
      "mean_delta_f2_at_20": 0.0,
      "mean_delta_brier": -0.0002692073065505285,
      "mean_delta_ece10": -0.0027297747631867472
    }
  ],
  "thresholds": [
    {
      "faculty": "BUS",
      "model": "committee",
      "status": "selected",
      "threshold": 0.26477789878845215,
      "development_rows": 24,
      "development_total_rows": 24,
      "development_positives": 12,
      "development_f2": 0.8450704225352113,
      "development_flag_rate": 0.9583333333333334,
      "development_folds": "1"
    },
    {
      "faculty": "BUS",
      "model": "suite26_seed_forest",
      "status": "selected",
      "threshold": 0.6023946404457092,
      "development_rows": 24,
      "development_total_rows": 24,
      "development_positives": 12,
      "development_f2": 0.8333333333333334,
      "development_flag_rate": 1.0,
      "development_folds": "1"
    },
    {
      "faculty": "BUS",
      "model": "suite26_tree_only",
      "status": "selected",
      "threshold": 0.6016896963119507,
      "development_rows": 24,
      "development_total_rows": 24,
      "development_positives": 12,
      "development_f2": 0.8571428571428571,
      "development_flag_rate": 0.9166666666666666,
      "development_folds": "1"
    },
    {
      "faculty": "BUS",
      "model": "suite26_memory",
      "status": "selected",
      "threshold": 0.6007558703422546,
      "development_rows": 24,
      "development_total_rows": 24,
      "development_positives": 12,
      "development_f2": 0.8571428571428571,
      "development_flag_rate": 0.9166666666666666,
      "development_folds": "1"
    },
    {
      "faculty": "BUS",
      "model": "suite26_mint",
      "status": "selected",
      "threshold": 0.6007840633392334,
      "development_rows": 24,
      "development_total_rows": 24,
      "development_positives": 12,
      "development_f2": 0.8571428571428571,
      "development_flag_rate": 0.9166666666666666,
      "development_folds": "1"
    },
    {
      "faculty": "IT",
      "model": "committee",
      "status": "selected",
      "threshold": 0.3632957637310028,
      "development_rows": 24,
      "development_total_rows": 24,
      "development_positives": 13,
      "development_f2": 0.8783783783783784,
      "development_flag_rate": 0.9166666666666666,
      "development_folds": "1"
    },
    {
      "faculty": "IT",
      "model": "suite26_seed_forest",
      "status": "selected",
      "threshold": 0.5257606506347656,
      "development_rows": 24,
      "development_total_rows": 24,
      "development_positives": 13,
      "development_f2": 0.8552631578947368,
      "development_flag_rate": 1.0,
      "development_folds": "1"
    },
    {
      "faculty": "IT",
      "model": "suite26_tree_only",
      "status": "selected",
      "threshold": 0.5612519979476929,
      "development_rows": 24,
      "development_total_rows": 24,
      "development_positives": 13,
      "development_f2": 0.8552631578947368,
      "development_flag_rate": 1.0,
      "development_folds": "1"
    },
    {
      "faculty": "IT",
      "model": "suite26_memory",
      "status": "selected",
      "threshold": 0.5595220327377319,
      "development_rows": 24,
      "development_total_rows": 24,
      "development_positives": 13,
      "development_f2": 0.8783783783783784,
      "development_flag_rate": 0.9166666666666666,
      "development_folds": "1"
    },
    {
      "faculty": "IT",
      "model": "suite26_mint",
      "status": "selected",
      "threshold": 0.5599199533462524,
      "development_rows": 24,
      "development_total_rows": 24,
      "development_positives": 13,
      "development_f2": 0.8783783783783784,
      "development_flag_rate": 0.9166666666666666,
      "development_folds": "1"
    }
  ],
  "bootstrap": [
    {
      "model_a": "suite26_mint",
      "model_b": "committee",
      "split": "holdout",
      "statistic": "macro_faculty_fold_ap_delta",
      "delta_ap": -0.05937646404539476,
      "ci_low": -0.20192891998088225,
      "ci_high": 0.061854936615144636,
      "cells": 2,
      "excluded_cells": 0,
      "rows": 48,
      "students": 48,
      "requested_replicates": 20,
      "valid_replicates": 20,
      "invalid_replicates": 0,
      "seed": 20260930,
      "status": "ok"
    },
    {
      "model_a": "suite26_mint",
      "model_b": "suite26_seed_forest",
      "split": "holdout",
      "statistic": "macro_faculty_fold_ap_delta",
      "delta_ap": 0.03324341669719588,
      "ci_low": -0.017592266368231427,
      "ci_high": 0.19432209554869795,
      "cells": 2,
      "excluded_cells": 0,
      "rows": 48,
      "students": 48,
      "requested_replicates": 20,
      "valid_replicates": 20,
      "invalid_replicates": 0,
      "seed": 20260930,
      "status": "ok"
    },
    {
      "model_a": "suite26_mint",
      "model_b": "suite26_tree_only",
      "split": "holdout",
      "statistic": "macro_faculty_fold_ap_delta",
      "delta_ap": 0.015280290904517657,
      "ci_low": -0.015427447168829789,
      "ci_high": 0.05081135689581161,
      "cells": 2,
      "excluded_cells": 0,
      "rows": 48,
      "students": 48,
      "requested_replicates": 20,
      "valid_replicates": 20,
      "invalid_replicates": 0,
      "seed": 20260930,
      "status": "ok"
    },
    {
      "model_a": "suite26_mint",
      "model_b": "suite26_memory",
      "split": "holdout",
      "statistic": "macro_faculty_fold_ap_delta",
      "delta_ap": 0.006726953601953689,
      "ci_low": -0.00518003676391828,
      "ci_high": 0.030197646482708874,
      "cells": 2,
      "excluded_cells": 0,
      "rows": 48,
      "students": 48,
      "requested_replicates": 20,
      "valid_replicates": 20,
      "invalid_replicates": 0,
      "seed": 20260930,
      "status": "ok"
    },
    {
      "model_a": "suite26_tree_only",
      "model_b": "suite26_seed_forest",
      "split": "holdout",
      "statistic": "macro_faculty_fold_ap_delta",
      "delta_ap": 0.017963125792678225,
      "ci_low": -0.02907867490785157,
      "ci_high": 0.16746562326768308,
      "cells": 2,
      "excluded_cells": 0,
      "rows": 48,
      "students": 48,
      "requested_replicates": 20,
      "valid_replicates": 20,
      "invalid_replicates": 0,
      "seed": 20260930,
      "status": "ok"
    },
    {
      "model_a": "suite26_memory",
      "model_b": "suite26_tree_only",
      "split": "holdout",
      "statistic": "macro_faculty_fold_ap_delta",
      "delta_ap": 0.008553337302563968,
      "ci_low": -0.027904598995399092,
      "ci_high": 0.0401761585823678,
      "cells": 2,
      "excluded_cells": 0,
      "rows": 48,
      "students": 48,
      "requested_replicates": 20,
      "valid_replicates": 20,
      "invalid_replicates": 0,
      "seed": 20260930,
      "status": "ok"
    }
  ],
  "reference_alignment": [],
  "warnings": [],
  "paths": {
    "predictions": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/predictions.csv",
    "per_cell_metrics": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/per_cell_metrics.csv",
    "macro_metrics": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/macro_metrics.csv",
    "per_fold_metrics": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/per_fold_metrics.csv",
    "per_faculty_metrics": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/per_faculty_metrics.csv",
    "paired_metrics": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/paired_metrics.csv",
    "paired_summary": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/paired_summary.csv",
    "component_uplifts": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/component_uplifts.csv",
    "aligned_cells": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/aligned_cells.csv",
    "thresholds": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/thresholds.csv",
    "operational_metrics": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/operational_metrics.csv",
    "it_confusion": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/it_confusion.csv",
    "bootstrap_ap": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/bootstrap_ap.csv",
    "bootstrap_replicates": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/bootstrap_replicates.csv",
    "reference_alignment_audit": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/reference_alignment_audit.csv",
    "report": "/workspace/scratch/2ea77c60b542/suite26_final_validation_run/results/report.json"
  },
  "conventions": {
    "primary": "suite26_mint (prespecified; no architecture selection by this report)",
    "macro": "Unweighted mean over eligible faculty-fold cells; one-class AP/AUC undefined",
    "paired": "Each model pair is evaluated on exactly its shared scored rows within each cell",
    "budget": "Stable descending score order; round(rate*N), minimum one when N>0",
    "threshold": "Per-faculty/model F2 maximization using development folds only; ties prefer fewer warnings",
    "bootstrap": "Paired student-cluster resampling, reusing each student's multiplicity across all folds/faculties; fixed eligible cells",
    "bootstrap_interval": "2.5/97.5 percentile of valid replicates; class-degenerate replicates explicitly excluded",
    "historical_holdouts": "F6/F7 are historical comparative benchmarks with acknowledged prior project-level reuse",
    "missing_scores": "Never imputed; coverage, matched rows and eligible cells are saved"
  }
}