{
  "reviewed_on": "2026-10-01",
  "scope": "Retrospective summary of inspected synthetic development data; not an untouched release audit.",
  "model_versions": {
    "full": "bloobability-v2-candidate-HE-4ad40b8b5dffeeba6001",
    "compact": "bloobability-v2-candidate-P-8210e74fcf18341e4340"
  },
  "development": {
    "compact": {
      "records": 1639,
      "lineages": 216,
      "families": 60,
      "correct_at_half": 1346,
      "known": 1542,
      "unknown": 97,
      "probability": {
        "count": 1542,
        "nll": 0.29369306003107654,
        "brier": 0.09141200370747359,
        "ece_diagnostic": 0.019078156867346378,
        "auroc": 0.943319906576854,
        "auprc": 0.9626409932472062
      },
      "raw_probability": {
        "count": 1542,
        "nll": 0.5229168587734022,
        "brier": 0.10694932793549572,
        "ece_diagnostic": 0.09059405126909256,
        "auroc": 0.943319906576854,
        "auprc": 0.9626409932472062
      },
      "answerability_at_half": {
        "threshold": 0.5,
        "unknown": 97,
        "known": 1542,
        "detected_unknown": 70,
        "false_deferrals": 6,
        "unknown_precision": 0.9210526315789473,
        "unknown_recall": 0.7216494845360825,
        "false_deferral": 0.0038910505836575876
      },
      "mean_probability": 0.6141302791981003,
      "observed_positive_rate": 0.6180285343709468,
      "reliability": [
        {
          "bin": 0,
          "count": 324,
          "probability": 0.0385620567895689,
          "frequency": 0.024691358024691357
        },
        {
          "bin": 1,
          "count": 82,
          "probability": 0.14194714971208958,
          "frequency": 0.24390243902439024
        },
        {
          "bin": 2,
          "count": 57,
          "probability": 0.25226924929822725,
          "frequency": 0.3333333333333333
        },
        {
          "bin": 3,
          "count": 60,
          "probability": 0.3549047893703448,
          "frequency": 0.38333333333333336
        },
        {
          "bin": 4,
          "count": 66,
          "probability": 0.4519934487166279,
          "frequency": 0.42424242424242425
        },
        {
          "bin": 5,
          "count": 53,
          "probability": 0.5555126866665723,
          "frequency": 0.6037735849056604
        },
        {
          "bin": 6,
          "count": 56,
          "probability": 0.6523163049119994,
          "frequency": 0.6607142857142857
        },
        {
          "bin": 7,
          "count": 73,
          "probability": 0.7535467678033942,
          "frequency": 0.7534246575342466
        },
        {
          "bin": 8,
          "count": 123,
          "probability": 0.8531424065859595,
          "frequency": 0.8455284552845529
        },
        {
          "bin": 9,
          "count": 648,
          "probability": 0.9744330248944766,
          "frequency": 0.9675925925925926
        }
      ]
    },
    "full": {
      "records": 1639,
      "lineages": 216,
      "families": 60,
      "correct_at_half": 1435,
      "known": 1542,
      "unknown": 97,
      "probability": {
        "count": 1542,
        "nll": 0.19315083865694685,
        "brier": 0.05342148487444378,
        "ece_diagnostic": 0.0175691242550951,
        "auroc": 0.9754701888594146,
        "auprc": 0.9832566052550138
      },
      "raw_probability": {
        "count": 1542,
        "nll": 0.3633197234711969,
        "brier": 0.06113421198447874,
        "ece_diagnostic": 0.05756948125073427,
        "auroc": 0.9754701888594146,
        "auprc": 0.9832566052550138
      },
      "answerability_at_half": {
        "threshold": 0.5,
        "unknown": 97,
        "known": 1542,
        "detected_unknown": 82,
        "false_deferrals": 25,
        "unknown_precision": 0.7663551401869159,
        "unknown_recall": 0.845360824742268,
        "false_deferral": 0.01621271076523995
      },
      "mean_probability": 0.6249508750136072,
      "observed_positive_rate": 0.6180285343709468,
      "reliability": [
        {
          "bin": 0,
          "count": 444,
          "probability": 0.04713086911220519,
          "frequency": 0.02252252252252252
        },
        {
          "bin": 1,
          "count": 58,
          "probability": 0.13838428746248668,
          "frequency": 0.13793103448275862
        },
        {
          "bin": 2,
          "count": 27,
          "probability": 0.24545403284166825,
          "frequency": 0.18518518518518517
        },
        {
          "bin": 3,
          "count": 24,
          "probability": 0.3480122809597041,
          "frequency": 0.5
        },
        {
          "bin": 4,
          "count": 19,
          "probability": 0.4432413031156748,
          "frequency": 0.5263157894736842
        },
        {
          "bin": 5,
          "count": 22,
          "probability": 0.5607842026115162,
          "frequency": 0.6363636363636364
        },
        {
          "bin": 6,
          "count": 24,
          "probability": 0.6547692268148527,
          "frequency": 0.5833333333333334
        },
        {
          "bin": 7,
          "count": 34,
          "probability": 0.7567426317500969,
          "frequency": 0.7058823529411765
        },
        {
          "bin": 8,
          "count": 51,
          "probability": 0.8599912758380666,
          "frequency": 0.803921568627451
        },
        {
          "bin": 9,
          "count": 839,
          "probability": 0.9698214515794146,
          "frequency": 0.9713945172824792
        }
      ]
    }
  },
  "entity_scope": {
    "compact": {
      "probability": {
        "count": 320,
        "nll": 0.6036310941076785,
        "brier": 0.2077425209731863,
        "ece_diagnostic": 0.07222340854194183,
        "auroc": 0.7168375651041666,
        "auprc": 0.7717013636304222
      },
      "correct_at_half": 216,
      "answerability_at_half": {
        "threshold": 0.5,
        "unknown": 64,
        "known": 320,
        "detected_unknown": 44,
        "false_deferrals": 0,
        "unknown_precision": 1.0,
        "unknown_recall": 0.6875,
        "false_deferral": 0.0
      }
    },
    "full": {
      "probability": {
        "count": 320,
        "nll": 0.4383892591521926,
        "brier": 0.13905293402983862,
        "ece_diagnostic": 0.11328072412852704,
        "auroc": 0.8935953776041666,
        "auprc": 0.9262788762327441
      },
      "correct_at_half": 265,
      "answerability_at_half": {
        "threshold": 0.5,
        "unknown": 64,
        "known": 320,
        "detected_unknown": 64,
        "false_deferrals": 11,
        "unknown_precision": 0.8533333333333334,
        "unknown_recall": 1.0,
        "false_deferral": 0.034375
      }
    },
    "v1": {
      "probability": {
        "count": 320,
        "nll": 0.5492491719787308,
        "brier": 0.1883268207833792,
        "ece_diagnostic": 0.049366047866713746,
        "auroc": 0.727783203125,
        "auprc": 0.8465307467033879
      },
      "correct_at_half": 215,
      "returned_verdict_correct": 217,
      "known": 320,
      "source": "48 archived live API responses; no new calls; fewer metadata fields than v2"
    }
  },
  "evidence": {
    "support": {
      "exhaustive_records": 254,
      "observed_bits": 2148,
      "positive_bits": 540,
      "selected": {
        "tp": 483,
        "fp": 69,
        "fn": 57,
        "precision": 0.875,
        "recall": 0.8944444444444445,
        "f1": 0.8846153846153846
      },
      "threshold": 0.54,
      "pooled_average_precision": 0.9387055325146613
    },
    "contradiction": {
      "exhaustive_records": 254,
      "observed_bits": 2148,
      "positive_bits": 96,
      "selected": {
        "tp": 52,
        "fp": 33,
        "fn": 44,
        "precision": 0.611764705882353,
        "recall": 0.5416666666666666,
        "f1": 0.574585635359116
      },
      "threshold": 0.372,
      "pooled_average_precision": 0.5330166683603068
    }
  },
  "latency": {
    "hardware": "RTX 4090",
    "runtime": "PyTorch 2.11.0+cu128",
    "common_scope": "Warm, uncached inputs, concurrency one; local request including JSON; no network or queue.",
    "matched_between_modes": false,
    "production_load_or_v1_speedup_qualified": false,
    "compact": {
      "arm": "unchanged P control, not C or R",
      "evidence": false,
      "blocks": 3,
      "warmups_per_block_cell": 3,
      "models_resident": 3,
      "joint_token_lengths": {
        "short1": {
          "min": 344,
          "max": 344
        },
        "long1": {
          "min": 874,
          "max": 874
        },
        "varied10": {
          "min": 604,
          "max": 874
        },
        "short10": {
          "min": 321,
          "max": 597
        },
        "varied79": {
          "min": 592,
          "max": 923
        }
      },
      "cells": {
        "short1": {
          "n": 30,
          "p50_ms": 13.442083029076457,
          "p95_ms": 13.842577405739576
        },
        "long1": {
          "n": 30,
          "p50_ms": 29.533948050811887,
          "p95_ms": 29.710745229385793
        },
        "varied10": {
          "n": 30,
          "p50_ms": 193.94623348489404,
          "p95_ms": 194.86262616119348
        },
        "short10": {
          "n": 30,
          "p50_ms": 130.05370693281293,
          "p95_ms": 131.95534712285735
        },
        "varied79": {
          "n": 30,
          "p50_ms": 1589.161553478334,
          "p95_ms": 1595.3494424524251
        }
      }
    },
    "full": {
      "arm": "corrected HE",
      "evidence": true,
      "blocks": 4,
      "warmups_per_block_cell": 2,
      "models_resident": 1,
      "cells": {
        "counterexample_k1": {
          "n": 40,
          "p50_ms": 71.62006650469266,
          "p95_ms": 72.10153219493804
        },
        "counterexample_k128": {
          "n": 40,
          "p50_ms": 4030.945905498811,
          "p95_ms": 4033.7427462014602
        },
        "counterexample_k79": {
          "n": 40,
          "p50_ms": 2507.638903502084,
          "p95_ms": 2509.9402080479194
        },
        "longer_varied_k10": {
          "n": 40,
          "p50_ms": 472.9785629970138,
          "p95_ms": 474.88675479617086
        }
      },
      "token_lengths": {
        "counterexample_k1_k79_k128": {
          "conversation": 403,
          "each_policy": 352,
          "repeated_definition": true
        },
        "longer_varied_k10": {
          "conversation": 630,
          "policy_min": 23,
          "policy_max": 262
        }
      }
    }
  },
  "source_sha256": {
    "final-v2-candidate/benchmarks/QUALITY.json": "830d97189d67e8d261627e3324f528b9fd9d46073b06aef032108256d27efc2f",
    "reports/he-runtime-integration-20261001/VERSIONS.json": "0f62c24b966e5c05571d828715a93cf580a442f5d9270d580f832f960b8955f1",
    "reports/g1-entity-scope-diagnostic-20260921/V4_COMPARISON.json": "de04029b2d3420306006c0ec3e36e2966a7e89be0403c7eba187813919de1d68",
    "_local/research/g1-entity-scope-v1/dataset/records.jsonl": "46358ddd275781a298e5827a94eebca9b5b4e4ef4ec0b59cda8168ec6c8f5589",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/000.json": "dfd13527ddc06e97b65f949c8bb29235128e92f04705be9a98d961917ab688a2",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/001.json": "391d15b16fae4b8f3e92e1b729dfcae1877b551db5f01355d4b5a3fac31cefa2",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/002.json": "0712c0b3c1e09d62e0fb3f334e7f52efcd8514d06f85930d19bce5805e59a11e",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/003.json": "e78ea00dc3b5c56eded22dd1c7eea1120136fb02f7724d0019e2bcc8c6e57195",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/004.json": "5582096173f6886dd53d5ec055d77a0f580c23d3954b4a03c015a0baf38b0203",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/005.json": "857684ea19579f4fe98e3c07a34c4319ecc13191d0846a5f7611afc3dec83ab9",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/006.json": "a0e14202ea4bc9f2f1cfdda8bb72de1c0d4ae1b67577b3a12781df8333c00675",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/007.json": "8e97a40efe2ba7fd56166df470a7b45943c5266dd58c42b7f4dc47208893b848",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/008.json": "d22316a201d3131d8d211f4410fe5ca311ef5b465c62935f4a47926aa7b5b99a",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/009.json": "c2a9ce76b49a7b2656475730edc4746eab38e141edd61f1a927304a201e532ee",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/010.json": "1087c5a7a66f19a8d90f8cf4b33505fab7e4f0c7b7f3a92257f89441a5a9dc6b",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/011.json": "aa5fb226b71d31a023d8613dddd433f2a68ac8b4968fcab823f3aa9e0ad58404",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/012.json": "50d75cc04b6a7f70f96004d2cd31d2d5afb7b03ce63135d441d76286c503b667",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/013.json": "9ad217132c1fb206b511b8573b58e6f0e3bd0f32e21bbee240cb9d3fd281f98b",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/014.json": "5065e052bc537bf4a5777c29c7bd92188e1040dbd766c9984f6df11a6f943ee1",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/015.json": "341168878ac328d956adc5153dc66583be5193ca3aecb52836916364d7f2d7c4",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/016.json": "d53219d72fcec027ad486a91045fce0d2671462f4b06ae59342141abb39825d7",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/017.json": "674725fd0e7d7999d3a3f0a0bd664ea5fc7d147df22f242686868c2fbd919524",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/018.json": "bea90ef575c9f1d67477399fa4dcf524edb837eed4b5a44f64cb1d7541c4d4c5",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/019.json": "725f32a86bd90dc6b51db90e0c631815ed6bfb5f0d46879b6e80756b4852c1d9",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/020.json": "ef57e6c118198ce1617100f6312e3ae85d62763c6443e8c8747f8acc62cfde5c",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/021.json": "7a69bf597e27b2931b0606a8e31cbce5144dae3b73928a3919e448cdc45dedda",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/022.json": "5f8de409009ceb067e686a27cf603e337a000df2d3e4b64b79c980a7d3ab6ae1",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/023.json": "3cf349a2f5aea91149ee2539a90cceeb2b3bcc6e5a3ec962381f3e938eb64a78",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/024.json": "8b3a451aad6389216c62ef945f26f5bb60ba3c996d48a35cb67a3c76b5a1548d",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/025.json": "ba6d1ee0e9b9995c50301d3e51061ed14f340d172c8617631cc46537e698d17c",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/026.json": "1e3ba8a9eedf075dd5dc35a1b16945474ad6bd35213986d623d98c3d15409e7a",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/027.json": "fd4daca41c1248d3b4f811bb9d11d08f5466c4c675d44bd61c05f6864b554770",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/028.json": "8ad05857e1ff8466ea87d36720ff959bb843c102b6e69cccae0a21d9cc606caf",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/029.json": "8c56ff1050cb95a4912de7fb7cb28212c142c4141d0fc9c7172de8150a2f8dbd",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/030.json": "ff5c0d42b2dd6795216d10ab8720c19d649d5fd307ad857170af3be57afe4655",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/031.json": "08a8b1751ed1733c63d1a6ad773473c5272964d118e32f12228caab17af32eac",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/032.json": "d808d5aa2ed66a93dfe426b1f116f16c514ff4567042c26cfd37d772738bba2d",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/033.json": "afec36b33e54ed7060e08a59cc6d3b553a9477836ed2ba7a054fe3a7fe4f830b",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/034.json": "20b415cc71763bfa2759dd99f6faa123c39c225f597f9bb730f34b1de1a94006",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/035.json": "a73a4ac4b6976d1ce1121bde9fe0f6b204f6978ef41f446005bb8f79ee6e4fd2",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/036.json": "49d04d7a65ea530d45ad3637de7584068ec96eaa1071dc1c97a0d693b6bfa10a",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/037.json": "9f5e0ff8d771011a39a8f43eee4d5b38d052a64743b57076fb32f1e55b1bf66c",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/038.json": "8179b0bb214793611a4c238660c20caa029c69686aaec5f99f78b7fdc5744e99",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/039.json": "c5fcf3be019683efaf3bc7eef3acacc195ce9691710e881b395c59dd064c635d",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/040.json": "40c0404d9bee36e370fc10d4e300b886aa2cff5d72d64641ce69e134ef08face",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/041.json": "aae614693b704320e78bb8f34c71fcc47bf08d142430a857a934e18fb297e199",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/042.json": "3fd5dfbf15c5f92822fd3a230acc866c65624676059b7cd7b27c364f443dc636",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/043.json": "c63e318b791b22fb68599db26685e2252f16d37f942c2ce48cb5d5fd6f5ea392",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/044.json": "e76685d8f50e54a949821c71cbcce53181cbde32d71978b9e1cf7baca784b995",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/045.json": "3e9f669942749bd0e071aa0a17976375c1e701e0005d2af54b6ff0ce7acc79fe",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/046.json": "8e8961c0dff6758204448207973c87dcf2b90b2f28a7b49f0d64b175024e14f2",
    "reports/g1-entity-scope-diagnostic-20260921/v4-reference-001/responses/047.json": "750d1c7175202144a6854e41f68950c22baacf80fadced89f85d99d88c74b3da",
    "_local/v3-compact-position-cache-20261001/run-002/TIMING.json": "fe7294ae87c1fe0b40c6024ca92f37d4d6d181f06645f0c43aa9bcc2ac04b495",
    "final-v2-candidate/provenance/he-l2-correction/timing-001/timing.jsonl": "365eec8b827a1ad6e043e1ac099e1099ce8e650364ad40ac3e36e08dbfc353a8"
  }
}