{
  "reviewed_on": "2026-10-18",
  "scope": "Summary of inspected research panels and a frozen synthetic panel for the two served Bloobability v2.1.0 tiers, with their served calibration maps. Not an untouched release audit, a customer-traffic measurement or a latency guarantee.",
  "model_versions": {
    "instant": "bloobability-v2-instant-720537bfb0685920",
    "full": "bloobability-v2-full-040b3c1d4f9151a4",
    "note": "Each model_version is bound to its tier's pinned weights, calibration maps, tokenizer, wrapper and runtime; any change to them yields a new one. Instant's weights and calibration are unchanged from v2.0.0; its version changed with the serving package. The base revisions below are provenance only."
  },
  "release_gates": {
    "all_research_gates_passed": false,
    "reserved_tests_opened": false,
    "instant_scope": "short explicit decisions with complete relevant facts; answerability unqualified on incomplete inputs"
  },
  "tiers": {
    "instant": {
      "decision_map": {
        "family": "positive_slope_affine",
        "slope": 0.3758054193754821,
        "intercept": 0.2677287085682785,
        "offsets": {},
        "penalty": 10.0
      },
      "answerability_map": {
        "family": "temperature",
        "slope": 0.2855929608810894,
        "intercept": 0.0,
        "offsets": {},
        "penalty": 10.0
      },
      "numerical_path": {
        "torch": "2.8.0+cu128",
        "cuda": "12.8",
        "transformers": "5.17.0",
        "parameter_dtypes": [
          "torch.bfloat16",
          "torch.float32"
        ],
        "tf32": false
      },
      "base": "Qwen/Qwen3.5-0.8B-Base@dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68",
      "trained_parameters": {
        "lora": 10822656,
        "decision_pointer": 524800,
        "answerability_head": 264705,
        "total": 11612161
      },
      "frozen_base_parameters": 752393024,
      "calibration_sha256": "0b0213143c0093702c28f7e18a21baa9c705169b1bbae87d4fef86e04fc31e63",
      "reads": "conversation once; each policy continues from a private copy of the model state"
    },
    "full": {
      "decision_map": {
        "family": "positive_slope_affine",
        "slope": 1.100958206210643,
        "intercept": 0.23635422976800013,
        "offsets": {},
        "fit_nll": 0.15152274553663053
      },
      "answerability_map": {
        "family": "positive_slope_affine",
        "slope": 0.35499641877876714,
        "intercept": 0.967295005763001,
        "offsets": {},
        "fit_nll": 0.06746448712362119
      },
      "numerical_path": {
        "torch": "2.8.0+cu128",
        "cuda": "12.8",
        "transformers": "5.17.0",
        "parameter_dtypes": [
          "torch.bfloat16",
          "torch.float32"
        ],
        "tf32": false
      },
      "base": "Qwen/Qwen3.5-4B-Base@1001bb4d826a52d1f399e183466143f4da7b741b",
      "trained_parameters": {
        "lora": 125878272,
        "decision_pointer": 1311232,
        "answerability_head": 660993,
        "policy_reader": 2376449,
        "total": 130226946
      },
      "frozen_base_parameters": 4205751296,
      "calibration_sha256": "59bbb24c48414bdf36550b06736a61974a733420f86b54b1cb39d05377258a00",
      "reads": "each policy first, then the whole conversation, in a separate pass per policy"
    }
  },
  "latency": {
    "hardware": "RTX 4090",
    "scope": "Served: the package's gateway workloads sent to the serving gateway over loopback HTTP on the GPU host, both tiers resident behind one GPU gate. Includes authentication, validation, inference, calibration and serialization; excludes the internet and the public API. Warm, uncached, concurrency one, 5 warm-ups per cell, then 120 observations per cell over 3 blocks with the tier order alternated.",
    "messages": {
      "short": 2,
      "medium": 40,
      "long": 125
    },
    "peak_allocated_gib": {
      "instant": 3.88,
      "full": 14.17
    },
    "instant": {
      "short/k1": {
        "n": 120,
        "p50_ms": 7.78,
        "p95_ms": 7.95
      },
      "short/k6": {
        "n": 120,
        "p50_ms": 26.01,
        "p95_ms": 26.12
      },
      "medium/k1": {
        "n": 120,
        "p50_ms": 30.87,
        "p95_ms": 31.39
      },
      "medium/k6": {
        "n": 120,
        "p50_ms": 53.77,
        "p95_ms": 54.19
      },
      "long/k1": {
        "n": 120,
        "p50_ms": 82.15,
        "p95_ms": 82.76
      }
    },
    "full": {
      "short/k1": {
        "n": 120,
        "p50_ms": 25.49,
        "p95_ms": 25.62
      },
      "short/k6": {
        "n": 120,
        "p50_ms": 152.78,
        "p95_ms": 153.02
      },
      "medium/k1": {
        "n": 120,
        "p50_ms": 105.96,
        "p95_ms": 106.47
      },
      "medium/k6": {
        "n": 120,
        "p50_ms": 629.04,
        "p95_ms": 630.63
      },
      "long/k1": {
        "n": 120,
        "p50_ms": 286.39,
        "p95_ms": 287.27
      }
    }
  },
  "fresh_panel": {
    "scope": "268 synthetic conversations (234 known, 34 missing a needed fact) written and checked by independent AI models, references frozen before scoring. Not customer or human-labeled data. changed_definitions judges the same 117 complex conversations against different definitions.",
    "instant": {
      "all": {
        "n": 234,
        "correct": 213,
        "accuracy": 0.9102564102564102,
        "nll": 0.25247050754526884,
        "brier": 0.07260297784622716,
        "ece_diagnostic": 0.06000903756568846,
        "auroc": 0.9639830508474577
      },
      "simple": {
        "n": 79,
        "correct": 76
      },
      "complex": {
        "n": 117,
        "correct": 103
      },
      "changed_definitions_complex": {
        "n": 117,
        "correct": 82
      },
      "answerability": {
        "unknown": 34,
        "unknown_detected": 0,
        "known": 234,
        "false_deferrals": 0
      }
    },
    "full": {
      "all": {
        "n": 234,
        "correct": 227,
        "accuracy": 0.9700854700854701,
        "nll": 0.09491524769365349,
        "brier": 0.023706571376035223,
        "ece_diagnostic": 0.025094831357387732,
        "auroc": 0.9967124488603156
      },
      "simple": {
        "n": 79,
        "correct": 79
      },
      "complex": {
        "n": 117,
        "correct": 111
      },
      "changed_definitions_complex": {
        "n": 117,
        "correct": 106
      },
      "answerability": {
        "unknown": 34,
        "unknown_detected": 31,
        "known": 234,
        "false_deferrals": 4
      }
    }
  },
  "narrow_screen": {
    "scope": "240 short AI-labeled rows: 180 known explicit-attribute answers and 60 missing-value unknowns from one repeated template. Intentionally easy.",
    "instant": {
      "n": 180,
      "correct": 180,
      "accuracy": 1.0,
      "nll": 0.0699326348448808,
      "brier": 0.006450661507691848,
      "ece_diagnostic": 0.06616077955583666,
      "auroc": 1.0,
      "unknown": 60,
      "unknown_detected": 0,
      "known": 180,
      "false_deferrals": 0
    },
    "full": {
      "n": 180,
      "correct": 178,
      "accuracy": 0.9888888888888889,
      "nll": 0.050838780473508156,
      "brier": 0.010264022875485137,
      "ece_diagnostic": 0.014078631669226833,
      "auroc": 1.0,
      "unknown": 60,
      "unknown_detected": 60,
      "known": 180,
      "false_deferrals": 0
    }
  },
  "public_900": {
    "scope": "The same 900 rows as v2.0.0: 300 each from HANS train, BoolQ train and SNLI validation. Inspected during development; familiar public tasks, not unseen policy families.",
    "instant": {
      "n": 900,
      "correct": 701,
      "accuracy": 0.7788888888888889,
      "nll": 0.574187541063297,
      "brier": 0.1764159289945752,
      "ece_diagnostic": 0.13033397939737484,
      "auroc": 0.8135061728395063,
      "auprc": 0.7463920615888683,
      "confidence_ge90_count": 388,
      "confidence_ge90_wrong": 88,
      "reliability_all": [
        {
          "bin": 0,
          "count": 65,
          "probability": 0.059357657944827426,
          "frequency": 0.07692307692307693
        },
        {
          "bin": 1,
          "count": 108,
          "probability": 0.14855076031645081,
          "frequency": 0.06481481481481481
        },
        {
          "bin": 2,
          "count": 65,
          "probability": 0.24697255883428873,
          "frequency": 0.12307692307692308
        },
        {
          "bin": 3,
          "count": 48,
          "probability": 0.34847243222630436,
          "frequency": 0.125
        },
        {
          "bin": 4,
          "count": 35,
          "probability": 0.4457431026699728,
          "frequency": 0.2571428571428571
        },
        {
          "bin": 5,
          "count": 44,
          "probability": 0.5444042982991016,
          "frequency": 0.38636363636363635
        },
        {
          "bin": 6,
          "count": 33,
          "probability": 0.6469369308359557,
          "frequency": 0.6060606060606061
        },
        {
          "bin": 7,
          "count": 50,
          "probability": 0.7552700235335169,
          "frequency": 0.62
        },
        {
          "bin": 8,
          "count": 129,
          "probability": 0.8581498168099062,
          "frequency": 0.8294573643410853
        },
        {
          "bin": 9,
          "count": 323,
          "probability": 0.9379780497134157,
          "frequency": 0.7430340557275542
        }
      ]
    },
    "full": {
      "n": 900,
      "correct": 815,
      "accuracy": 0.9055555555555556,
      "nll": 0.26340219283397776,
      "brier": 0.0731951102490448,
      "ece_diagnostic": 0.04010750528009278,
      "auroc": 0.9550123456790124,
      "auprc": 0.9432435611747481,
      "confidence_ge90_count": 710,
      "confidence_ge90_wrong": 32,
      "reliability_all": [
        {
          "bin": 0,
          "count": 317,
          "probability": 0.06513535598270098,
          "frequency": 0.01892744479495268
        },
        {
          "bin": 1,
          "count": 82,
          "probability": 0.1334558866166263,
          "frequency": 0.15853658536585366
        },
        {
          "bin": 2,
          "count": 16,
          "probability": 0.256877304648903,
          "frequency": 0.375
        },
        {
          "bin": 3,
          "count": 12,
          "probability": 0.3406293615713574,
          "frequency": 0.4166666666666667
        },
        {
          "bin": 4,
          "count": 8,
          "probability": 0.4496688543993279,
          "frequency": 0.625
        },
        {
          "bin": 5,
          "count": 10,
          "probability": 0.5500086434036748,
          "frequency": 0.4
        },
        {
          "bin": 6,
          "count": 8,
          "probability": 0.6641049978660678,
          "frequency": 0.375
        },
        {
          "bin": 7,
          "count": 15,
          "probability": 0.7504175987192843,
          "frequency": 0.7333333333333333
        },
        {
          "bin": 8,
          "count": 39,
          "probability": 0.8597492363021584,
          "frequency": 0.7692307692307693
        },
        {
          "bin": 9,
          "count": 393,
          "probability": 0.95314951410439,
          "frequency": 0.9338422391857506
        }
      ]
    }
  },
  "public_sources": {
    "scope": "Ten public datasets adapted to yes-or-no questions, inspected during development. Not official leaderboard entries; base-model pretraining overlap unknown. mean_nll weights each source equally.",
    "instant": {
      "anli": {
        "n": 780,
        "correct": 473,
        "accuracy": 0.6064102564102564,
        "nll": 0.7945007351656386,
        "brier": 0.2700607558597355,
        "ece_diagnostic": 0.15648363730955347,
        "auroc": 0.6301058201058201
      },
      "boolq": {
        "n": 1320,
        "correct": 1054,
        "accuracy": 0.7984848484848485,
        "nll": 0.4707817656406466,
        "brier": 0.1490313965504255,
        "ece_diagnostic": 0.05191312199719342,
        "auroc": 0.8594849285288245
      },
      "cb": {
        "n": 100,
        "correct": 90,
        "accuracy": 0.9,
        "nll": 0.30517398074401764,
        "brier": 0.08694181657317497,
        "ece_diagnostic": 0.06900913447975184,
        "auroc": 0.9299999999999999
      },
      "hans": {
        "n": 1740,
        "correct": 1119,
        "accuracy": 0.6431034482758621,
        "nll": 0.9258662958086455,
        "brier": 0.29096752097337764,
        "ece_diagnostic": 0.2702643798943803,
        "auroc": 0.6795058792442858
      },
      "mnli": {
        "n": 254,
        "correct": 207,
        "accuracy": 0.8149606299212598,
        "nll": 0.4043875953189992,
        "brier": 0.12683904886120667,
        "ece_diagnostic": 0.06083877357394137,
        "auroc": 0.9015438030876062
      },
      "paws": {
        "n": 646,
        "correct": 537,
        "accuracy": 0.8312693498452013,
        "nll": 0.4032615607671098,
        "brier": 0.12450458204447493,
        "ece_diagnostic": 0.07321674450951779,
        "auroc": 0.9107362552864283
      },
      "qnli": {
        "n": 80,
        "correct": 70,
        "accuracy": 0.875,
        "nll": 0.3714372020751527,
        "brier": 0.11194140811680967,
        "ece_diagnostic": 0.14247744481376018,
        "auroc": 0.9468749999999999
      },
      "rte": {
        "n": 100,
        "correct": 78,
        "accuracy": 0.78,
        "nll": 0.5524614605888417,
        "brier": 0.17936683728103917,
        "ece_diagnostic": 0.14625785104906985,
        "auroc": 0.852
      },
      "snli": {
        "n": 1192,
        "correct": 1055,
        "accuracy": 0.8850671140939598,
        "nll": 0.3007078259052294,
        "brier": 0.08796281112606522,
        "ece_diagnostic": 0.08984013223572296,
        "auroc": 0.9574101052536478
      },
      "tweet_offensive": {
        "n": 80,
        "correct": 40,
        "accuracy": 0.5,
        "nll": 0.7458007038615022,
        "brier": 0.2716868871765791,
        "ece_diagnostic": 0.38569876442189166,
        "auroc": 0.8605128205128205
      },
      "mean_nll": 0.5274379125875783
    },
    "full": {
      "anli": {
        "n": 780,
        "correct": 601,
        "accuracy": 0.7705128205128206,
        "nll": 0.544514102957208,
        "brier": 0.17504537332819844,
        "ece_diagnostic": 0.10258050649200787,
        "auroc": 0.8387301587301588
      },
      "boolq": {
        "n": 1320,
        "correct": 1189,
        "accuracy": 0.9007575757575758,
        "nll": 0.2797800279543687,
        "brier": 0.07984504216309943,
        "ece_diagnostic": 0.03836599992408691,
        "auroc": 0.9540866497325157
      },
      "cb": {
        "n": 100,
        "correct": 90,
        "accuracy": 0.9,
        "nll": 0.24076216134192827,
        "brier": 0.07327674763896784,
        "ece_diagnostic": 0.10648824043833226,
        "auroc": 0.9868
      },
      "hans": {
        "n": 1740,
        "correct": 1536,
        "accuracy": 0.8827586206896552,
        "nll": 0.33101609122433034,
        "brier": 0.09507038150711289,
        "ece_diagnostic": 0.08697138352875303,
        "auroc": 0.9493354472189193
      },
      "mnli": {
        "n": 254,
        "correct": 229,
        "accuracy": 0.9015748031496063,
        "nll": 0.25227512149869913,
        "brier": 0.07250279446832185,
        "ece_diagnostic": 0.07441120630829764,
        "auroc": 0.9733399466798934
      },
      "paws": {
        "n": 646,
        "correct": 590,
        "accuracy": 0.913312693498452,
        "nll": 0.2394165347285514,
        "brier": 0.0679575663984853,
        "ece_diagnostic": 0.050489543714791606,
        "auroc": 0.9751345636293733
      },
      "qnli": {
        "n": 80,
        "correct": 75,
        "accuracy": 0.9375,
        "nll": 0.18132643856530195,
        "brier": 0.04911785926953267,
        "ece_diagnostic": 0.043182298997837695,
        "auroc": 0.9837499999999999
      },
      "rte": {
        "n": 100,
        "correct": 87,
        "accuracy": 0.87,
        "nll": 0.3305519751009339,
        "brier": 0.10182082814769995,
        "ece_diagnostic": 0.10237572749197252,
        "auroc": 0.964
      },
      "snli": {
        "n": 1192,
        "correct": 1121,
        "accuracy": 0.9404362416107382,
        "nll": 0.18403980308205578,
        "brier": 0.04789950412039185,
        "ece_diagnostic": 0.046608015572414334,
        "auroc": 0.9841611329448435
      },
      "tweet_offensive": {
        "n": 80,
        "correct": 68,
        "accuracy": 0.85,
        "nll": 0.395295028284704,
        "brier": 0.12141805270819243,
        "ece_diagnostic": 0.11829618765888354,
        "auroc": 0.8728205128205129
      },
      "mean_nll": 0.29789772847380813
    }
  },
  "capability_panels": {
    "scope": "Synthetic research references inspected during development, not untouched final tests. original_reversals32 is all negatives.",
    "instant": {
      "targeted_rules": {
        "n": 872,
        "correct": 421,
        "nll": 1.0957397360212588,
        "answerability": {
          "unknown": 264,
          "unknown_detected": 71,
          "known": 872,
          "false_deferrals": 0
        }
      },
      "entity_scheduling": {
        "n": 320,
        "correct": 202,
        "nll": 0.6469282454593035
      },
      "source_authority": {
        "n": 220,
        "correct": 203,
        "nll": 0.26444167660982193
      },
      "policy_scope": {
        "n": 160,
        "correct": 156,
        "nll": 0.06319891707250269
      },
      "difficult_wording": {
        "n": 144,
        "correct": 126,
        "nll": 0.29265766639654495
      },
      "original_reversals32": {
        "n": 32,
        "correct": 12,
        "nll": 1.295557451855
      },
      "short_implicit_reversals36": {
        "n": 36,
        "correct": 35,
        "nll": 0.1791483140144361
      },
      "independent_custom": {
        "n": 279,
        "correct": 182,
        "nll": 0.6585849063360892,
        "answerability": {
          "unknown": 70,
          "unknown_detected": 1,
          "known": 279,
          "false_deferrals": 0
        }
      },
      "independent_public": {
        "n": 200,
        "correct": 168,
        "nll": 0.42881772066642965
      }
    },
    "full": {
      "targeted_rules": {
        "n": 872,
        "correct": 826,
        "nll": 0.14168702676397146,
        "answerability": {
          "unknown": 264,
          "unknown_detected": 264,
          "known": 872,
          "false_deferrals": 0
        }
      },
      "entity_scheduling": {
        "n": 320,
        "correct": 304,
        "nll": 0.1195599335571905
      },
      "source_authority": {
        "n": 220,
        "correct": 220,
        "nll": 0.016150565124889237
      },
      "policy_scope": {
        "n": 160,
        "correct": 159,
        "nll": 0.03636350307667528
      },
      "difficult_wording": {
        "n": 144,
        "correct": 136,
        "nll": 0.16795415860457272
      },
      "original_reversals32": {
        "n": 32,
        "correct": 32,
        "nll": 0.04180360835410911
      },
      "short_implicit_reversals36": {
        "n": 36,
        "correct": 30,
        "nll": 0.6561510331547075
      },
      "independent_custom": {
        "n": 279,
        "correct": 244,
        "nll": 0.30221695970723006,
        "answerability": {
          "unknown": 70,
          "unknown_detected": 48,
          "known": 279,
          "false_deferrals": 0
        }
      },
      "independent_public": {
        "n": 200,
        "correct": 177,
        "nll": 0.2856570682214311
      }
    }
  },
  "sensitivity": {
    "conjnli_punctuation_pairs": {
      "pairs": 520,
      "instant": {
        "crossings_of_0_5": 22,
        "max_gap_points": 59.99
      },
      "full": {
        "crossings_of_0_5": 12,
        "max_gap_points": 80.65
      }
    },
    "conjnli_623": {
      "instant": {
        "correct": 395,
        "nll": 0.9229
      },
      "full": {
        "correct": 513,
        "nll": 0.5019
      }
    }
  },
  "source_sha256": {
    "MODEL_IDENTITIES.json": "dea1bfbe4136e44ff3958c3968d076f2aedb1eb5c811c3f7e451d173e0b87cd7",
    "BENCHMARKS.md": "4ba8f14b161f37aad24bd5719cb64526083711a36516b4e81d8dcb861b42e44a",
    "LIMITATIONS.md": "082a8ffc066c9e811549d9a0de1194ddaa0c39b77c53f07dd90962e96146ef88",
    "benchmarks/ATLAS.json": "bdad2cbb081876aa5ab072f632815e3373c29ad5d5892e186395321791dbdbf4",
    "benchmarks/FRESH_QUALITY.json": "3ac08466fbd215292364a4d0c969684aa4d39d2ef9ae44f1cfc1655927b032af",
    "serving/latency.json": "ef26a034514b6853fc8411aaefaa1b88f59d1c1802ef5ed864fe635e0d578af6",
    "serving/narrow-screen.json": "3b837739d2d099b0c812dd6b2bcad252a4c868b8cd0f60a4c82176aa5f4b76b4"
  }
}