{
  "models": {
    "candidate": {
      "name": "Trained Zils H2O",
      "episodes": 100,
      "outcomes": {
        "timeout": 2,
        "success": 97,
        "death": 1
      },
      "success_rate": 0.97,
      "success_wilson_95_interval": [
        0.9154806357094724,
        0.9897454759759611
      ],
      "mean_return": 2212.7676718139646,
      "mean_episode_game_seconds": 12.055428571428571,
      "mean_success_game_seconds": 11.177319587628865,
      "decisions": 23762
    },
    "base": {
      "name": "Untrained H2O",
      "episodes": 100,
      "outcomes": {
        "death": 100
      },
      "success_rate": 0.0,
      "success_wilson_95_interval": [
        3.469446951953614e-18,
        0.03699349820698568
      ],
      "mean_return": -5.706725311279297,
      "mean_episode_game_seconds": 1.949142857142857,
      "mean_success_game_seconds": null,
      "decisions": 1777
    },
    "jev": {
      "name": "JEV 1.13.0",
      "episodes": 100,
      "outcomes": {
        "death": 99,
        "technical_failure": 1
      },
      "success_rate": 0.0,
      "success_wilson_95_interval": [
        3.469446951953614e-18,
        0.03699349820698568
      ],
      "mean_return": -26.16599319458008,
      "mean_episode_game_seconds": 2.094,
      "mean_success_game_seconds": null,
      "decisions": 1867
    },
    "reference": {
      "name": "Scripted reference",
      "episodes": 100,
      "outcomes": {
        "success": 100
      },
      "success_rate": 1.0,
      "success_wilson_95_interval": [
        0.9630065017930143,
        1
      ],
      "mean_return": 2278.1630140686034,
      "mean_episode_game_seconds": 9.103428571428571,
      "mean_success_game_seconds": 9.103428571428571,
      "decisions": 15692
    }
  },
  "paired_comparisons": {
    "jev": {
      "success_difference": 0.97,
      "paired_seed_bootstrap_95_interval": [
        0.93,
        1.0
      ],
      "bootstrap_boundary_warning": false,
      "trained_only_successes": 97,
      "comparator_only_successes": 0,
      "exact_mcnemar_two_sided_p": 1.262177448353619e-29,
      "bootstrap_samples": 10000,
      "completed_pair_count": 99,
      "completed_pairs_only": {
        "success_difference": 0.9696969696969697,
        "paired_seed_bootstrap_95_interval": [
          0.9292929292929293,
          1.0
        ],
        "bootstrap_boundary_warning": false,
        "trained_only_successes": 96,
        "comparator_only_successes": 0,
        "exact_mcnemar_two_sided_p": 2.524354896707238e-29,
        "bootstrap_samples": 10000
      }
    },
    "base": {
      "success_difference": 0.97,
      "paired_seed_bootstrap_95_interval": [
        0.93,
        1.0
      ],
      "bootstrap_boundary_warning": false,
      "trained_only_successes": 97,
      "comparator_only_successes": 0,
      "exact_mcnemar_two_sided_p": 1.262177448353619e-29,
      "bootstrap_samples": 10000,
      "completed_pair_count": 100,
      "completed_pairs_only": {
        "success_difference": 0.97,
        "paired_seed_bootstrap_95_interval": [
          0.93,
          1.0
        ],
        "bootstrap_boundary_warning": false,
        "trained_only_successes": 97,
        "comparator_only_successes": 0,
        "exact_mcnemar_two_sided_p": 1.262177448353619e-29,
        "bootstrap_samples": 10000
      }
    },
    "reference": {
      "success_difference": -0.03,
      "paired_seed_bootstrap_95_interval": [
        -0.07,
        0.0
      ],
      "bootstrap_boundary_warning": false,
      "trained_only_successes": 0,
      "comparator_only_successes": 3,
      "exact_mcnemar_two_sided_p": 0.25,
      "bootstrap_samples": 10000,
      "completed_pair_count": 100,
      "completed_pairs_only": {
        "success_difference": -0.03,
        "paired_seed_bootstrap_95_interval": [
          -0.07,
          0.0
        ],
        "bootstrap_boundary_warning": false,
        "trained_only_successes": 0,
        "comparator_only_successes": 3,
        "exact_mcnemar_two_sided_p": 0.25,
        "bootstrap_samples": 10000
      }
    }
  },
  "selected_seed": 100001,
  "eligible_highlights": [
    100001,
    100002,
    100003,
    100004,
    100005,
    100006,
    100007,
    100008,
    100009,
    100010,
    100011,
    100012,
    100014,
    100015,
    100016,
    100017,
    100018,
    100019,
    100020,
    100021,
    100022,
    100023,
    100024,
    100025,
    100026,
    100027,
    100028,
    100029,
    100030,
    100031,
    100032,
    100033,
    100034,
    100035,
    100036,
    100037,
    100038,
    100039,
    100040,
    100041,
    100042,
    100043,
    100044,
    100045,
    100046,
    100047,
    100048,
    100049,
    100050,
    100051,
    100052,
    100053,
    100054,
    100055,
    100056,
    100057,
    100058,
    100059,
    100060,
    100061,
    100062,
    100063,
    100064,
    100065,
    100066,
    100067,
    100068,
    100069,
    100070,
    100071,
    100072,
    100073,
    100074,
    100075,
    100076,
    100077,
    100078,
    100079,
    100080,
    100081,
    100082,
    100083,
    100084,
    100086,
    100087,
    100088,
    100089,
    100090,
    100092,
    100093,
    100094,
    100095,
    100096,
    100097,
    100098,
    100099
  ],
  "reverse_highlights": [],
  "jev_api": {
    "attempts": 1867,
    "retries": 0,
    "errors": {},
    "reported_usage": {
      "input_tokens": 1776354,
      "output_tokens": 181099
    },
    "bill_verified": false,
    "operator_interrupted_episode_seed": 100085,
    "environment_start_failures_before_model_calls": 14
  },
  "training_seconds": 718.773829547,
  "audit": {
    "passed": true,
    "episodes": 400,
    "replays_verified": 399,
    "matched_initial_conditions": 100,
    "outcomes": {
      "candidate": {
        "timeout": 2,
        "success": 97,
        "death": 1
      },
      "base": {
        "death": 100
      },
      "jev": {
        "death": 99,
        "technical_failure": 1
      },
      "reference": {
        "success": 100
      }
    }
  },
  "protocol_sha256": "4ba4a0ffb62eb21ba01988373ce71e5d01a6bed5dfaa9e7daabcd5e7e4b02bed"
}