{
  "version": 3,
  "summary": {
    "controlled_and_web_pairs": 96,
    "terminal_pairs": 6,
    "frozen_lake": {"success": "100% -> 100%", "llm_calls_change": "-64.4%"},
    "webshop": {"success": "67% -> 100%", "llm_calls_change": "-21.4%", "pairs": 3},
    "terminal_bench": {"success": "1/6 -> 3/6", "llm_calls_change": "-9.0%"}
  },
  "frozen_lake": {
    "seed": 3000,
    "selection": "predeclared first paired seed",
    "subgoal": "Reach the goal by moving Right, Right, Right, Up while checking every new state",
    "action_menu": ["Left", "Down", "Right", "Up"],
    "steps": [
      {
        "player": [1, 0], "action": "Right", "confidence": 0.99,
        "comparison": {"Left": "wall", "Down": "away", "Right": "route", "Up": "alternate"}
      },
      {
        "player": [1, 1], "action": "Right", "confidence": 0.99,
        "comparison": {"Left": "backtrack", "Down": "away", "Right": "route", "Up": "alternate"}
      },
      {
        "player": [1, 2], "action": "Right", "confidence": 0.82,
        "comparison": {"Left": "backtrack", "Down": "away", "Right": "route", "Up": "alternate"}
      },
      {
        "player": [1, 3], "action": "Up", "confidence": 1.0,
        "comparison": {"Left": "backtrack", "Down": "away", "Right": "wall", "Up": "goal"}
      }
    ],
    "pair": {
      "baseline": {"reward": 1, "steps": 4, "step_label": "moves", "llm_calls": 4, "tokens": 2338, "wall_time_seconds": 19.7},
      "jev_harness": {"reward": 1, "steps": 4, "step_label": "moves", "llm_calls": 1, "tokens": 663, "wall_time_seconds": 16.7}
    },
    "ten_pair_result": {
      "baseline_success": 1.0,
      "jev_harness_success": 1.0,
      "baseline_mean_llm_calls": 4.5,
      "jev_harness_mean_llm_calls": 1.6,
      "baseline_mean_tokens": 3057.6,
      "jev_harness_mean_tokens": 1128.0,
      "baseline_mean_wall_time_seconds": 14.8,
      "jev_harness_mean_wall_time_seconds": 9.2
    },
    "source": "runs/agent-harness/holdout-gpt-fl-27b.json",
    "source_sha256": "15980db9bf914c4bfc86927a0f2c7e12de461ad1385cf8e7e13a21ee427aaf2f"
  },
  "sqlite": {
    "benchmark": "Terminal-Bench 2",
    "task": "sqlite-db-truncate",
    "options": [
      {
        "name": "Inspect raw page",
        "command": "od -A x -t x1z -v /app/trunc.db | head -80",
        "display_command": "od -A x -t x1z -v trunc.db | head",
        "expected": "hex dump"
      },
      {
        "name": "Read SQLite schema",
        "command": "sqlite3 /app/trunc.db '.schema' 2>&1; echo '---'; sqlite3 /app/trunc.db 'SELECT * FROM sqlite_master;' 2>&1",
        "display_command": "sqlite3 trunc.db '.schema'; SELECT sqlite_master",
        "expected": "schema"
      },
      {
        "name": "Check available tools",
        "command": "which sqlite3 python3",
        "display_command": "which sqlite3 python3",
        "expected": "paths"
      }
    ],
    "jev_choice": "Inspect raw page",
    "confidence": 0.81,
    "observation": "SQLite leaf-table page with 10 cell pointers",
    "result": "10 rows recovered and verifier passed",
    "baseline": {
      "reward": 1,
      "steps": 13,
      "step_label": "commands",
      "llm_calls": 15,
      "tokens": 202050,
      "wall_time_seconds": 187.9,
      "source": "runs/terminal-bench/frontier-v4/sqlite-db-truncate/d0/jobs/jev-tb2-sqlite-db-truncate-baseline/sqlite-db-truncate__JEG29Mk/result.json",
      "sha256": "009be2b0f3b3302240186e30981e869408a5d8434eb3b065ef565b91e5b83546"
    },
    "jev_harness": {
      "reward": 1,
      "steps": 7,
      "step_label": "commands",
      "llm_calls": 8,
      "tokens": 121293,
      "wall_time_seconds": 144.7,
      "source": "runs/terminal-bench/frontier-v4/sqlite-db-truncate/d100/jobs/jev-tb2-sqlite-db-truncate-optional/sqlite-db-truncate__yq4Z9qx/result.json",
      "sha256": "d64fc17bbc5fcc6119f6f6d90ba600452c79bcc781ca1a172f46431057d10ee4"
    }
  },
  "webshop": {
    "seed": 3107,
    "selection": "positive pair from the LLM-authored candidate supplement",
    "goal": "Women's long-sleeve blazer, z-dark green, small, under $80",
    "candidate_groups": [
      {
        "subgoal": "Select z-dark green color",
        "actions": ["click[z-dark green]", "click[z-army green]", "click[z-khaki]"],
        "selected": "click[z-dark green]",
        "confidence": 1.0,
        "hidden_state": {"color": "z-dark green"}
      },
      {
        "subgoal": "Select small size",
        "actions": ["click[x-small]", "click[small]", "click[medium]"],
        "selected": "click[small]",
        "confidence": 1.0,
        "hidden_state": {"color": "z-dark green", "size": "small"}
      }
    ],
    "baseline": {
      "reward": 1,
      "steps": 9,
      "step_label": "actions",
      "llm_calls": 9,
      "tokens": 38852,
      "wall_time_seconds": 18.54,
      "path": [
        "search",
        "open product",
        "select color",
        "select size",
        "inspect features",
        "repeat color and size",
        "buy now"
      ]
    },
    "jev_harness": {
      "reward": 1,
      "steps": 5,
      "step_label": "actions",
      "llm_calls": 4,
      "tokens": 14256,
      "wall_time_seconds": 7.83,
      "path": [
        {"controller": "llm", "action": "search"},
        {"controller": "llm", "action": "open matching product"},
        {"controller": "jev", "action": "click[z-dark green]", "confidence": 1.0, "action_is_valid": true},
        {"controller": "jev", "action": "click[small]", "confidence": 1.0, "action_is_valid": true},
        {"controller": "llm", "action": "click[buy now]"}
      ]
    },
    "three_pair_result": {
      "baseline_success": 0.6667,
      "jev_harness_success": 1.0,
      "llm_calls_change": -0.2143,
      "tokens_change": -0.1430,
      "wall_time_change": -0.1501
    },
    "source": "runs/agent-harness/webshop-llm-candidates-supplement-v2.json",
    "source_sha256": "0363fb8d6fcc241fbe7e1d3eed5b0d4d9fc571a76fee204dfe24b9b3833bbd8e"
  }
}