{
 "name": "efficiency-a100-v1",
 "measured": "2026-09-30",
 "hardware": "NVIDIA A100-SXM4-40GB (one node per run); multi-GPU rows use 2 or 3 GPUs of one node",
 "request_shape": "batch size 1, serial requests, every request has one question",
 "suites": {
  "transfer": "Transfer-v9 development split from jaredpalmer/kev@5920c5fe: 1,264 requests, 1,046 scored (clean, knowable)",
  "jevbench": "JevBench public (231 questions, fstandhartinger/jevbench@d06ee959), converted with scripts/build_external_eval.py; development.jsonl sha256 63f4b0c5..., not the official jevbench-public-v1.4.2.2 file"
 },
 "latency": {
  "jevany": "ModelPredictor latency_ms: backbone forward, readout and probability transfer to CPU; excludes tokenization",
  "third_party": "latency reported around each project's own inference call, including its tokenization",
  "warmup": "re-runs drop the first five requests; Default rows of the 27B, 30B and third-party models come from the original evaluation runs over all requests"
 },
 "configurations": {
  "Default": "JevAny evaluation path (LocalPredictor: math SDPA kernel, TF32 off) without flash-linear-attention or causal-conv1d; third-party models with each project's official inference code in an environment without those packages",
  "+ kernels": "flash-linear-attention 0.5.2 and causal-conv1d 1.7.0 installed (Transformers uses them for Gated DeltaNet layers)",
  "+ fused SDPA": "PyTorch's fused SDPA kernels, as jevany serve uses",
  "+ CUDA graphs": "--cuda-graphs with lengths captured up to 16,384 tokens (the current default caps capture at --cuda-graph-max-tokens 2048)",
  "+ own CUDA graphs": "decider's own CUDA-graph engine (compiled per shape during the run)",
  "multi-GPU": "--device-map auto with --max-memory-gib 22 (3 GPUs) or 31 (2 GPUs)"
 },
 "notes": [
  "JevAny-Qwen3.8-27B is the step 22,160 release; the current step 44,319 release has the same architecture",
  "Gemma 4, Jev-Omni (Gemma 4) and Laya (ModernBERT) have no Gated DeltaNet layers, so the kernels do not apply",
  "decider means after acceleration include one-time per-shape compilation and are not representative; use median and p90"
 ],
 "rows": [
  {
   "model": "JevAny-Qwen3.5-4B",
   "configuration": "Default",
   "gpus": 1,
   "transfer": {
    "accuracy": 78.68,
    "mean_ms": 105.4,
    "median_ms": 104.6,
    "p90_ms": 112.7
   },
   "jevbench": {
    "accuracy": 77.92,
    "mean_ms": 184.9,
    "median_ms": 108.8,
    "p90_ms": 474.5
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Qwen3.5-4B",
   "configuration": "+ kernels",
   "gpus": 1,
   "transfer": {
    "accuracy": 78.59,
    "mean_ms": 94.9,
    "median_ms": 93.6,
    "p90_ms": 98.7
   },
   "jevbench": {
    "accuracy": 78.35,
    "mean_ms": 174.0,
    "median_ms": 98.5,
    "p90_ms": 282.2
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Qwen3.5-4B",
   "configuration": "+ kernels + fused SDPA",
   "gpus": 1,
   "transfer": {
    "accuracy": 78.78,
    "mean_ms": 93.6,
    "median_ms": 92.1,
    "p90_ms": 96.7
   },
   "jevbench": {
    "accuracy": 78.35,
    "mean_ms": 117.6,
    "median_ms": 92.7,
    "p90_ms": 216.4
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Qwen3.5-4B",
   "configuration": "+ kernels + fused SDPA + CUDA graphs",
   "gpus": 1,
   "transfer": {
    "accuracy": 78.87,
    "mean_ms": 27.4,
    "median_ms": 25.3,
    "p90_ms": 33.0
   },
   "jevbench": {
    "accuracy": 78.35,
    "mean_ms": 73.7,
    "median_ms": 25.4,
    "p90_ms": 229.1
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Qwen3.5-4B-Direct-Token",
   "configuration": "Default",
   "gpus": 1,
   "transfer": {
    "accuracy": 78.11,
    "mean_ms": 106.6,
    "median_ms": 106.4,
    "p90_ms": 113.7
   },
   "jevbench": {
    "accuracy": 80.52,
    "mean_ms": 186.7,
    "median_ms": 108.5,
    "p90_ms": 480.7
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Qwen3.5-4B-Direct-Token",
   "configuration": "+ kernels",
   "gpus": 1,
   "transfer": {
    "accuracy": 78.01,
    "mean_ms": 97.3,
    "median_ms": 96.3,
    "p90_ms": 99.6
   },
   "jevbench": {
    "accuracy": 80.52,
    "mean_ms": 134.5,
    "median_ms": 96.0,
    "p90_ms": 283.0
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Qwen3.5-4B-Direct-Token",
   "configuration": "+ kernels + fused SDPA",
   "gpus": 1,
   "transfer": {
    "accuracy": 78.11,
    "mean_ms": 95.7,
    "median_ms": 94.6,
    "p90_ms": 98.0
   },
   "jevbench": {
    "accuracy": 80.52,
    "mean_ms": 119.9,
    "median_ms": 96.6,
    "p90_ms": 217.8
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Qwen3.5-4B-Direct-Token",
   "configuration": "+ kernels + fused SDPA + CUDA graphs",
   "gpus": 1,
   "transfer": {
    "accuracy": 78.39,
    "mean_ms": 28.4,
    "median_ms": 25.9,
    "p90_ms": 33.7
   },
   "jevbench": {
    "accuracy": 80.52,
    "mean_ms": 74.5,
    "median_ms": 25.9,
    "p90_ms": 229.4
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Gemma-4B",
   "configuration": "Default",
   "gpus": 1,
   "transfer": {
    "accuracy": 70.84,
    "mean_ms": 107.1,
    "median_ms": 106.3,
    "p90_ms": 109.3
   },
   "jevbench": {
    "accuracy": 75.76,
    "mean_ms": 179.2,
    "median_ms": 107.2,
    "p90_ms": 453.7
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Gemma-4B",
   "configuration": "+ kernels + fused SDPA",
   "gpus": 1,
   "transfer": {
    "accuracy": 71.03,
    "mean_ms": 98.8,
    "median_ms": 97.9,
    "p90_ms": 101.4
   },
   "jevbench": {
    "accuracy": 75.76,
    "mean_ms": 138.3,
    "median_ms": 103.4,
    "p90_ms": 281.2
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Gemma-4B",
   "configuration": "+ kernels + fused SDPA + CUDA graphs",
   "gpus": 1,
   "transfer": {
    "accuracy": 70.84,
    "mean_ms": 34.5,
    "median_ms": 31.9,
    "p90_ms": 39.9
   },
   "jevbench": {
    "accuracy": 75.32,
    "mean_ms": 94.0,
    "median_ms": 32.1,
    "p90_ms": 288.0
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Qwen3.8-27B",
   "configuration": "Default",
   "gpus": 3,
   "transfer": {
    "accuracy": 85.66,
    "mean_ms": 247.0,
    "median_ms": 240.2,
    "p90_ms": 261.1
   },
   "jevbench": {
    "accuracy": 88.31,
    "mean_ms": 533.8,
    "median_ms": 251.6,
    "p90_ms": 1591.0
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Qwen3.8-27B",
   "configuration": "+ kernels + fused SDPA",
   "gpus": 3,
   "transfer": {
    "accuracy": 85.66,
    "mean_ms": 222.1,
    "median_ms": 220.1,
    "p90_ms": 225.7
   },
   "jevbench": {
    "accuracy": 88.31,
    "mean_ms": 366.1,
    "median_ms": 220.9,
    "p90_ms": 931.8
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Qwen3.8-27B",
   "configuration": "+ kernels + fused SDPA",
   "gpus": 2,
   "transfer": {
    "accuracy": 85.66,
    "mean_ms": 214.7,
    "median_ms": 212.5,
    "p90_ms": 220.0
   },
   "jevbench": {
    "accuracy": 88.31,
    "mean_ms": 363.2,
    "median_ms": 218.8,
    "p90_ms": 932.6
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Muse-Glimmer-30B",
   "configuration": "Default",
   "gpus": 3,
   "transfer": {
    "accuracy": 83.37,
    "mean_ms": 173.8,
    "median_ms": 171.2,
    "p90_ms": 178.6
   },
   "jevbench": {
    "accuracy": 85.71,
    "mean_ms": 416.9,
    "median_ms": 170.8,
    "p90_ms": 1362.7
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Muse-Glimmer-30B",
   "configuration": "+ kernels + fused SDPA",
   "gpus": 3,
   "transfer": {
    "accuracy": 83.37,
    "mean_ms": 156.5,
    "median_ms": 154.2,
    "p90_ms": 158.9
   },
   "jevbench": {
    "accuracy": 85.71,
    "mean_ms": 304.4,
    "median_ms": 154.1,
    "p90_ms": 884.1
   },
   "mean_includes_compilation": false
  },
  {
   "model": "JevAny-Muse-Glimmer-30B",
   "configuration": "+ kernels + fused SDPA",
   "gpus": 2,
   "transfer": {
    "accuracy": 83.37,
    "mean_ms": 164.0,
    "median_ms": 156.8,
    "p90_ms": 181.1
   },
   "jevbench": {
    "accuracy": 85.71,
    "mean_ms": 320.6,
    "median_ms": 181.1,
    "p90_ms": 880.7
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Open-Jev-27B-v1.1",
   "configuration": "Default",
   "gpus": 3,
   "transfer": {
    "accuracy": 76.39,
    "mean_ms": 1709.6,
    "median_ms": 1348.6,
    "p90_ms": 4336.8
   },
   "jevbench": {
    "accuracy": 85.71,
    "mean_ms": 2201.9,
    "median_ms": 1785.3,
    "p90_ms": 5477.1
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Open-Jev-27B-v1.1",
   "configuration": "+ kernels",
   "gpus": 3,
   "transfer": {
    "accuracy": 76.29,
    "mean_ms": 594.2,
    "median_ms": 447.6,
    "p90_ms": 1486.8
   },
   "jevbench": {
    "accuracy": 85.28,
    "mean_ms": 939.9,
    "median_ms": 671.3,
    "p90_ms": 2744.4
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Open-Jev-9B",
   "configuration": "Default",
   "gpus": 1,
   "transfer": {
    "accuracy": 72.37,
    "mean_ms": 746.4,
    "median_ms": 572.6,
    "p90_ms": 1906.3
   },
   "jevbench": {
    "accuracy": 77.06,
    "mean_ms": 902.0,
    "median_ms": 761.0,
    "p90_ms": 2043.0
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Open-Jev-9B",
   "configuration": "+ kernels",
   "gpus": 1,
   "transfer": {
    "accuracy": 72.47,
    "mean_ms": 268.1,
    "median_ms": 208.2,
    "p90_ms": 687.4
   },
   "jevbench": {
    "accuracy": 77.06,
    "mean_ms": 354.5,
    "median_ms": 272.5,
    "p90_ms": 880.7
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Bespoke-Nimble-9B",
   "configuration": "Default",
   "gpus": 1,
   "transfer": {
    "accuracy": 76.58,
    "mean_ms": 123.8,
    "median_ms": 119.7,
    "p90_ms": 141.1
   },
   "jevbench": {
    "accuracy": 79.22,
    "mean_ms": 198.2,
    "median_ms": 126.8,
    "p90_ms": 480.4
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Bespoke-Nimble-9B",
   "configuration": "+ kernels",
   "gpus": 1,
   "transfer": {
    "accuracy": 76.39,
    "mean_ms": 97.3,
    "median_ms": 96.3,
    "p90_ms": 101.7
   },
   "jevbench": {
    "accuracy": 79.22,
    "mean_ms": 141.5,
    "median_ms": 97.0,
    "p90_ms": 312.5
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Intern-Decision-4B",
   "configuration": "Default",
   "gpus": 1,
   "transfer": {
    "accuracy": 70.46,
    "mean_ms": 203.5,
    "median_ms": 201.1,
    "p90_ms": 214.1
   },
   "jevbench": {
    "accuracy": 87.01,
    "mean_ms": 260.6,
    "median_ms": 200.2,
    "p90_ms": 480.6
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Intern-Decision-4B",
   "configuration": "+ kernels",
   "gpus": 1,
   "transfer": {
    "accuracy": 70.36,
    "mean_ms": 53.8,
    "median_ms": 53.6,
    "p90_ms": 54.4
   },
   "jevbench": {
    "accuracy": 86.58,
    "mean_ms": 120.9,
    "median_ms": 54.2,
    "p90_ms": 147.5
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Intern-Decision-2B",
   "configuration": "Default",
   "gpus": 1,
   "transfer": {
    "accuracy": 60.61,
    "mean_ms": 152.9,
    "median_ms": 152.0,
    "p90_ms": 162.7
   },
   "jevbench": {
    "accuracy": 78.35,
    "mean_ms": 192.0,
    "median_ms": 150.4,
    "p90_ms": 351.6
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Intern-Decision-2B",
   "configuration": "+ kernels",
   "gpus": 1,
   "transfer": {
    "accuracy": 60.33,
    "mean_ms": 40.9,
    "median_ms": 40.8,
    "p90_ms": 41.4
   },
   "jevbench": {
    "accuracy": 78.35,
    "mean_ms": 46.4,
    "median_ms": 41.2,
    "p90_ms": 67.0
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Intern-Decision-0.8B",
   "configuration": "Default",
   "gpus": 1,
   "transfer": {
    "accuracy": 55.35,
    "mean_ms": 151.9,
    "median_ms": 150.7,
    "p90_ms": 162.0
   },
   "jevbench": {
    "accuracy": 71.0,
    "mean_ms": 193.0,
    "median_ms": 153.1,
    "p90_ms": 347.1
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Intern-Decision-0.8B",
   "configuration": "+ kernels",
   "gpus": 1,
   "transfer": {
    "accuracy": 55.45,
    "mean_ms": 40.3,
    "median_ms": 40.2,
    "p90_ms": 40.7
   },
   "jevbench": {
    "accuracy": 71.0,
    "mean_ms": 42.2,
    "median_ms": 40.5,
    "p90_ms": 47.9
   },
   "mean_includes_compilation": false
  },
  {
   "model": "decider-4b",
   "configuration": "Default",
   "gpus": 1,
   "transfer": {
    "accuracy": 74.76,
    "mean_ms": 70.0,
    "median_ms": 68.9,
    "p90_ms": 78.7
   },
   "jevbench": {
    "accuracy": 82.68,
    "mean_ms": 143.6,
    "median_ms": 73.9,
    "p90_ms": 348.5
   },
   "mean_includes_compilation": false
  },
  {
   "model": "decider-4b",
   "configuration": "+ kernels + own CUDA graphs",
   "gpus": 1,
   "transfer": {
    "accuracy": 74.67,
    "mean_ms": 393.7,
    "median_ms": 26.7,
    "p90_ms": 47.9
   },
   "jevbench": {
    "accuracy": 82.68,
    "mean_ms": 1962.0,
    "median_ms": 35.5,
    "p90_ms": 259.8
   },
   "mean_includes_compilation": true
  },
  {
   "model": "decider-2b",
   "configuration": "Default",
   "gpus": 1,
   "transfer": {
    "accuracy": 67.21,
    "mean_ms": 53.1,
    "median_ms": 50.9,
    "p90_ms": 64.2
   },
   "jevbench": {
    "accuracy": 75.76,
    "mean_ms": 89.2,
    "median_ms": 54.5,
    "p90_ms": 189.1
   },
   "mean_includes_compilation": false
  },
  {
   "model": "decider-2b",
   "configuration": "+ kernels + own CUDA graphs",
   "gpus": 1,
   "transfer": {
    "accuracy": 67.59,
    "mean_ms": 289.4,
    "median_ms": 16.2,
    "p90_ms": 22.5
   },
   "jevbench": {
    "accuracy": 76.19,
    "mean_ms": 13993.8,
    "median_ms": 21.4,
    "p90_ms": 127.0
   },
   "mean_includes_compilation": true
  },
  {
   "model": "Jev-Omni",
   "configuration": "Default (not re-run)",
   "gpus": 1,
   "transfer": {
    "accuracy": 75.62,
    "mean_ms": 69.3,
    "median_ms": 68.4,
    "p90_ms": 69.7
   },
   "jevbench": {
    "accuracy": 85.71,
    "mean_ms": 146.4,
    "median_ms": 68.8,
    "p90_ms": 436.7
   },
   "mean_includes_compilation": false
  },
  {
   "model": "Laya",
   "configuration": "Default (not re-run)",
   "gpus": 1,
   "transfer": {
    "accuracy": 52.2,
    "mean_ms": 22.9,
    "median_ms": 22.0,
    "p90_ms": 25.0
   },
   "jevbench": {
    "accuracy": 58.87,
    "mean_ms": 22.3,
    "median_ms": 21.8,
    "p90_ms": 24.7
   },
   "mean_includes_compilation": false
  }
 ]
}