{
  "frozen_decision": {
    "selected": "S4",
    "promoted": true,
    "frozen_from": "F1-F5_ONLY",
    "reasons": [],
    "dev_macro": [
      {
        "arm": "S4",
        "cells": 70,
        "faculties": 14,
        "ap": 0.33683655891597325,
        "auc": 0.880315570387176,
        "brier": 0.024634606578384274,
        "ece10": 0.016229473721171665,
        "f2_at_20": 0.33981840754549325
      },
      {
        "arm": "P1",
        "cells": 70,
        "faculties": 14,
        "ap": 0.33664211289537793,
        "auc": 0.880224274443391,
        "brier": 0.024566497926721144,
        "ece10": 0.016443816970855205,
        "f2_at_20": 0.3382736420408926
      },
      {
        "arm": "S0",
        "cells": 70,
        "faculties": 14,
        "ap": 0.32994675731824347,
        "auc": 0.874562339183872,
        "brier": 0.024769274370079895,
        "ece10": 0.016411964909653857,
        "f2_at_20": 0.33287868810622906
      },
      {
        "arm": "S1",
        "cells": 70,
        "faculties": 14,
        "ap": 0.3249719456180986,
        "auc": 0.8710557629014212,
        "brier": 0.024704495738360987,
        "ece10": 0.01597848311568243,
        "f2_at_20": 0.3297728944368709
      },
      {
        "arm": "S3",
        "cells": 70,
        "faculties": 14,
        "ap": 0.3197112217242169,
        "auc": 0.8703478607852542,
        "brier": 0.02497753216524837,
        "ece10": 0.0163067823627997,
        "f2_at_20": 0.330170022408481
      },
      {
        "arm": "S2",
        "cells": 70,
        "faculties": 14,
        "ap": 0.31959165292151703,
        "auc": 0.8716614390513888,
        "brier": 0.02495274598766289,
        "ece10": 0.016709293650768762,
        "f2_at_20": 0.3316588629479458
      }
    ],
    "deltas": {
      "S4_minus_S1": {
        "candidate": "S4",
        "reference": "S1",
        "cells": 70,
        "mean_delta_ap": 0.01186461329787467,
        "median_delta_ap": 0.013734467723916857,
        "wins": 44,
        "losses": 26,
        "ties": 0,
        "worst": -0.13932436862731337,
        "best": 0.10519911758414074
      },
      "S3_minus_S2": {
        "candidate": "S3",
        "reference": "S2",
        "cells": 70,
        "mean_delta_ap": 0.00011956880269985327,
        "median_delta_ap": 0.000833532149920907,
        "wins": 36,
        "losses": 34,
        "ties": 0,
        "worst": -0.05879885501627968,
        "best": 0.0715307795207385
      },
      "S4_minus_P1": {
        "candidate": "S4",
        "reference": "P1",
        "cells": 70,
        "mean_delta_ap": 0.00019444602059534696,
        "median_delta_ap": -0.004179929622246531,
        "wins": 32,
        "losses": 38,
        "ties": 0,
        "worst": -0.1319308199751066,
        "best": 0.1696551896587239
      }
    },
    "positive_folds": 5,
    "positive_faculties": 9,
    "placebo_matchable_fraction": 0.8185747704646991,
    "qualified_placebo_cells": 70,
    "qualified_placebo_delta": 0.001004688404617149,
    "scientific_status": "EXPLORATORY_DEVELOPMENT_DIAGNOSTIC_NOT_DEPLOYABLE",
    "promoted_for_deployment": false
  },
  "retrospective": [
    {
      "arm": "P1",
      "cells": 28,
      "faculties": 14,
      "ap": 0.3542304339545757,
      "auc": 0.8522849710144519,
      "brier": 0.054598807199818966,
      "ece10": 0.03493909778655987,
      "f2_at_20": 0.45525800317021253
    },
    {
      "arm": "S0",
      "cells": 28,
      "faculties": 14,
      "ap": 0.346168400967462,
      "auc": 0.8474242317514314,
      "brier": 0.05446635364344565,
      "ece10": 0.03377602645247134,
      "f2_at_20": 0.4525447785562446
    },
    {
      "arm": "S2",
      "cells": 28,
      "faculties": 14,
      "ap": 0.3443816826546898,
      "auc": 0.8483897144822732,
      "brier": 0.05474602781108538,
      "ece10": 0.03416643912253926,
      "f2_at_20": 0.4507501877740395
    },
    {
      "arm": "S1",
      "cells": 28,
      "faculties": 14,
      "ap": 0.34407184022059506,
      "auc": 0.8478533915254348,
      "brier": 0.05461005179131995,
      "ece10": 0.03345531200474754,
      "f2_at_20": 0.45473289266987094
    },
    {
      "arm": "S4",
      "cells": 28,
      "faculties": 14,
      "ap": 0.34351616608287644,
      "auc": 0.8505707469887707,
      "brier": 0.05470065258331801,
      "ece10": 0.03472982453684127,
      "f2_at_20": 0.4560170182958259
    },
    {
      "arm": "S3",
      "cells": 28,
      "faculties": 14,
      "ap": 0.34281258608988624,
      "auc": 0.8472548576191006,
      "brier": 0.05501018499955519,
      "ece10": 0.03477354277538875,
      "f2_at_20": 0.4504616386396942
    }
  ],
  "archived_reference": {
    "committee": 0.350152,
    "suite25_committee_blend": 0.369526,
    "suite22": 0.369159,
    "suite31_full_history_hgb": 0.353807,
    "suite32_local_common": 0.346168,
    "council_13_faculty_archive": 0.364204,
    "council_IT_only": 0.444927
  },
  "holdout_status": "REPEATEDLY_INSPECTED_RETROSPECTIVE_NOT_PRISTINE"
}