{
  "development_count": 500,
  "checkpoints": [
    {
      "count": 500,
      "correct": 329,
      "accuracy": 0.658,
      "macro_f1": 0.6429597300672101,
      "multiclass_brier": 0.5157039541936063,
      "log_loss": 1.6165883713911995,
      "ece_10_bins": 0.09552006841056127,
      "confident_cases_at_90_percent": 134,
      "confident_wrong_at_90_percent": 25,
      "latency_median_seconds": 0.13130959749969406,
      "latency_p95_seconds": 0.1361333881997325,
      "predictions_sha256": "abec0b72f1c986d7dad46406003765ec178fde780dc467b96cd82c21b4a68135",
      "checkpoint_completion_sha256": "b902b442d88f699fa6de53f51ef2ddafbf6d242e79bc20ca2857643f6c5a2cf8",
      "name": "4b-full-1024-epoch1",
      "model": "4b",
      "training_count": 1024,
      "epoch": 1,
      "training_exposures": 1024,
      "mode": "full"
    },
    {
      "count": 500,
      "correct": 389,
      "accuracy": 0.778,
      "macro_f1": 0.72882101287091,
      "multiclass_brier": 0.36826298220508774,
      "log_loss": 0.9998892839252047,
      "ece_10_bins": 0.11482434688950899,
      "confident_cases_at_90_percent": 369,
      "confident_wrong_at_90_percent": 37,
      "latency_median_seconds": 0.17915957699983664,
      "latency_p95_seconds": 0.18935665374983726,
      "predictions_sha256": "4075774630dff1edde0512286ac7b93a6f4d150a042be1af99429ce652eb75a4",
      "checkpoint_completion_sha256": "59cf0c5384eb8d5a01aebde840b88917856bda7d2415fa6e491523cf56d3a259",
      "name": "4b-lora-1024-epoch1",
      "model": "4b",
      "training_count": 1024,
      "epoch": 1,
      "training_exposures": 1024,
      "mode": "lora"
    },
    {
      "count": 500,
      "correct": 391,
      "accuracy": 0.782,
      "macro_f1": 0.7291612072575956,
      "multiclass_brier": 0.3807139943264316,
      "log_loss": 1.0859781518021792,
      "ece_10_bins": 0.13902254282166449,
      "confident_cases_at_90_percent": 439,
      "confident_wrong_at_90_percent": 69,
      "latency_median_seconds": 0.18229001399981826,
      "latency_p95_seconds": 0.19021608215034574,
      "predictions_sha256": "256fe712dc7683e672483a1f488f61086dad54745715fb03cbefaf0d0b7c79ff",
      "checkpoint_completion_sha256": "0068d54186484b9dc79e8373150c55473ea9942835aba5bdf9721e652ebf2b89",
      "name": "4b-lora-1024-epoch2",
      "model": "4b",
      "training_count": 1024,
      "epoch": 2,
      "training_exposures": 2048,
      "mode": "lora"
    },
    {
      "count": 500,
      "correct": 380,
      "accuracy": 0.76,
      "macro_f1": 0.6923376997288417,
      "multiclass_brier": 0.36718465177186943,
      "log_loss": 0.9465669754276925,
      "ece_10_bins": 0.09308712155366348,
      "confident_cases_at_90_percent": 156,
      "confident_wrong_at_90_percent": 7,
      "latency_median_seconds": 0.19599763100018208,
      "latency_p95_seconds": 0.2277528414497283,
      "predictions_sha256": "cb0b7a772cd5922c8037d526a00b66fea3800a347921c99aa0496c1f3c874dd6",
      "checkpoint_completion_sha256": "98daad4315e70de7053d796d90bfd1fd6e9933a66efffcf3eca1abd7ad60d027",
      "name": "9b-lora-1024-epoch1",
      "model": "9b",
      "training_count": 1024,
      "epoch": 1,
      "training_exposures": 1024,
      "mode": "lora"
    },
    {
      "count": 500,
      "correct": 396,
      "accuracy": 0.792,
      "macro_f1": 0.739273325126829,
      "multiclass_brier": 0.33799052087913184,
      "log_loss": 0.922927808809139,
      "ece_10_bins": 0.0934250288100772,
      "confident_cases_at_90_percent": 153,
      "confident_wrong_at_90_percent": 6,
      "latency_median_seconds": 0.19604570100000274,
      "latency_p95_seconds": 0.22840786285028117,
      "predictions_sha256": "9d530ab0e2cd5b0a07334193e8d5c4370a59010136bdd70e7ba05078201108b7",
      "checkpoint_completion_sha256": "1caffb2d154a4263997fa394ded113d6ddede26fb05513c0115815fb2bd13081",
      "name": "9b-lora-1024-epoch2",
      "model": "9b",
      "training_count": 1024,
      "epoch": 2,
      "training_exposures": 2048,
      "mode": "lora"
    },
    {
      "count": 500,
      "correct": 413,
      "accuracy": 0.826,
      "macro_f1": 0.7767857340411757,
      "multiclass_brier": 0.29570533145611694,
      "log_loss": 0.7906679991179867,
      "ece_10_bins": 0.09815093241692077,
      "confident_cases_at_90_percent": 424,
      "confident_wrong_at_90_percent": 43,
      "latency_median_seconds": 0.17900002599981235,
      "latency_p95_seconds": 0.18928736599982585,
      "predictions_sha256": "2ae946c20354458d748bd48f70f294305085a729d69f789459a87d93fe4a7ba0",
      "checkpoint_completion_sha256": "6f654be4bd1b49382df9cf34757be897b0a8479e0c81325f34f35e5ea875ebe7",
      "name": "4b-lora-4096-epoch1",
      "model": "4b",
      "training_count": 4096,
      "epoch": 1,
      "training_exposures": 4096,
      "mode": "lora"
    },
    {
      "count": 500,
      "correct": 415,
      "accuracy": 0.83,
      "macro_f1": 0.7935619096011766,
      "multiclass_brier": 0.2921094578532696,
      "log_loss": 0.7930063991676939,
      "ece_10_bins": 0.09504490742037733,
      "confident_cases_at_90_percent": 451,
      "confident_wrong_at_90_percent": 56,
      "latency_median_seconds": 0.17716964999999618,
      "latency_p95_seconds": 0.18878398850010852,
      "predictions_sha256": "0807417db636b4e0a766b3a582daacba103b1f28f05f6f66f9d45ccf0c2b3e69",
      "checkpoint_completion_sha256": "e6a33c4f8b44b19bfb4015260abb580a39034dd0126955d5e143aeec9921cb7c",
      "name": "4b-lora-4096-epoch2",
      "model": "4b",
      "training_count": 4096,
      "epoch": 2,
      "training_exposures": 8192,
      "mode": "lora"
    },
    {
      "count": 500,
      "correct": 421,
      "accuracy": 0.842,
      "macro_f1": 0.8056528774465388,
      "multiclass_brier": 0.26583025312197983,
      "log_loss": 0.7355673779345749,
      "ece_10_bins": 0.06130473068518262,
      "confident_cases_at_90_percent": 334,
      "confident_wrong_at_90_percent": 22,
      "latency_median_seconds": 0.19584478100023262,
      "latency_p95_seconds": 0.22733898815076828,
      "predictions_sha256": "94152b212a1b5b405234b63fc0d91f93a0355259c3d69c2190c5da90cab99837",
      "checkpoint_completion_sha256": "673b5a4dae98738a30036ca83d086448b8d65aebd248a3d3e8e899986fe186c7",
      "name": "9b-lora-4096-epoch1",
      "model": "9b",
      "training_count": 4096,
      "epoch": 1,
      "training_exposures": 4096,
      "mode": "lora"
    },
    {
      "count": 500,
      "correct": 421,
      "accuracy": 0.842,
      "macro_f1": 0.7899354446397956,
      "multiclass_brier": 0.26736485749964894,
      "log_loss": 0.7510739512351298,
      "ece_10_bins": 0.07734492452110014,
      "confident_cases_at_90_percent": 386,
      "confident_wrong_at_90_percent": 26,
      "latency_median_seconds": 0.1957448100001784,
      "latency_p95_seconds": 0.2278874622497824,
      "predictions_sha256": "e73a7e3bc2e7afa710258e37212ba3c58f3409db43160f3a0f7a337b42859091",
      "checkpoint_completion_sha256": "f10af8c186e727c96ef1f4cbe010c68d8c30b6e344dc397ca5d7aac4cc76c6cc",
      "name": "9b-lora-4096-epoch2",
      "model": "9b",
      "training_count": 4096,
      "epoch": 2,
      "training_exposures": 8192,
      "mode": "lora"
    }
  ],
  "selection_rule": "Highest development accuracy, then lower multiclass Brier, fewer training exposures, and smaller training dataset.",
  "test_labels_used_for_selection": false,
  "dev_inputs_sha256": "5613ef23f6175467d23b3ab90f6e120e47681da2e82edbae108f5097fe4d6813",
  "dev_labels_sha256": "4fba9dc1a8313c08caaf36f6167e596c1f2d830e63c1d571442c3a507ea17aa8",
  "bases": {
    "4b": {
      "count": 500,
      "correct": 268,
      "accuracy": 0.536,
      "macro_f1": 0.4420890901516878,
      "multiclass_brier": 0.6561339003668186,
      "log_loss": 1.5366070366736617,
      "ece_10_bins": 0.13803523207087895,
      "confident_cases_at_90_percent": 66,
      "confident_wrong_at_90_percent": 20,
      "latency_median_seconds": 0.12980040099978396,
      "latency_p95_seconds": 0.13462674484999299,
      "predictions_sha256": "10cb96cd9e38439982fa0f3e22ee2c49d732aa0c8de5bf0bcf5ef2a20b6367b2"
    },
    "9b": {
      "count": 500,
      "correct": 274,
      "accuracy": 0.548,
      "macro_f1": 0.4653380384526951,
      "multiclass_brier": 0.639323859703227,
      "log_loss": 1.5544340306847382,
      "ece_10_bins": 0.0909277337667119,
      "confident_cases_at_90_percent": 50,
      "confident_wrong_at_90_percent": 20,
      "latency_median_seconds": 0.1369005034998736,
      "latency_p95_seconds": 0.15854899214996293,
      "predictions_sha256": "2f496137629d1eb9f8bb9b6d6aba451560fa188b2db562688a48da4a851053fa"
    }
  },
  "best_lora_by_model": {
    "4b": "4b-lora-4096-epoch2",
    "9b": "9b-lora-4096-epoch1"
  },
  "numerical_controls": {
    "4b-full-1024": {
      "original_fp32_gate_passed": true
    },
    "4b-lora-1024": {
      "original_fp32_gate_passed": true
    },
    "4b-lora-4096": {
      "original_fp32_gate_passed": true
    },
    "9b-lora-1024": {
      "original_fp32_gate_passed": true
    },
    "9b-lora-4096": {
      "original_fp32_gate_passed": false,
      "numerical_exception_sha256": "9cbcedc67ef3831f41c306ebf06737e6140c4d678d12cb5c28c378ee547ba932"
    }
  }
}