{
 "benchmark": "LarkBench v2.3",
 "environment": "generator 2.3.0, settlement v2.1 (going concern)",
 "frozen_record": "6c51b158b3ea957e",
 "secret_set_commitment_sha256": "11f510d9816457339d5088ca28a6eb4dab6e7e72e308f29f887273de2182cd44",
 "books": 160,
 "draws_per_book": 3,
 "book_draws": 480,
 "hard_mix": {
  "n_hard": 40,
  "cut_pp": 17.196544314939928,
  "hard_share": 0.5,
  "from": "holdout.json",
  "definition": "top quartile of internally recorded hardness (privileged planner minus careful rule, on calibration draws that never overlap the scoring draws) carries half of the book weight, not half of the weighted dollar denominator; no independently timestamped pre-run commitment is supplied"
 },
 "intervals": "95% percentile bootstrap over books, each resampled with its three draws (2,000 resamples, the same resamples for every row); differences paired on the same book-draws; conditional on the observed run, so they do not cover run-to-run variation",
 "primary_results": "each model's trial 0: one full run on all 160 books x 3 draws, as played (ai_runs with trial 0 and setup AI alone)",
 "score": "sum of net profit / sum of demand margin (field possible_profit), pooled over book-draws (book weights: natural 1; hard 2 for the top-quartile books, 2/3 otherwise); losing seasons count in full. Demand margin is a common yardstick, not a proven upper bound: one privileged-planner season reaches 101.8%",
 "free_players_all_160_books": {
  "ERP auto reorder point": {
   "hard": {
    "pct": 37.708290458618585,
    "ci_pct": [
     32.127037105341486,
     43.08934128380995
    ]
   },
   "natural": {
    "pct": 43.343126841206306,
    "ci_pct": [
     39.209127826899895,
     46.95887107820202
    ]
   },
   "vs_careful": {
    "hard": {
     "pp": -18.594066744114286,
     "pp_ci": [
      -21.91694971110735,
      -15.560142277687069
     ]
    },
    "natural": {
     "pp": -19.10162281835966,
     "pp_ci": [
      -21.37066226119895,
      -16.920094161632314
     ]
    }
   }
  },
  "Careful rule (hand-written, tuned on practice books)": {
   "hard": {
    "pct": 56.30235720273287,
    "ci_pct": [
     51.698008004369946,
     60.77193361102081
    ]
   },
   "natural": {
    "pct": 62.444749659565964,
    "ci_pct": [
     59.25615831014308,
     65.34836745243466
    ]
   }
  },
  "Screen-only planner (no look-ups or notes; demand and lead times estimated from the screen)": {
   "hard": {
    "pct": 59.54122041085309,
    "ci_pct": [
     54.731604669669345,
     63.93144503927148
    ]
   },
   "natural": {
    "pct": 65.72890501660703,
    "ci_pct": [
     62.52194411478242,
     68.60043149556256
    ]
   },
   "vs_careful": {
    "hard": {
     "pp": 3.2388632081202173,
     "pp_ci": [
      1.947477100287726,
      4.508712327430723
     ]
    },
    "natural": {
     "pp": 3.284155357041057,
     "pp_ci": [
      2.2779723994348733,
      4.308591766329293
     ]
    }
   }
  },
  "Privileged planner (true demand and lead-time laws, true stock position)": {
   "hard": {
    "pct": 70.03830538542668,
    "ci_pct": [
     65.95353111775685,
     73.38049615981106
    ]
   },
   "natural": {
    "pct": 73.59574086218089,
    "ci_pct": [
     71.09674202714645,
     75.73990053564566
    ]
   },
   "vs_careful": {
    "hard": {
     "pp": 13.735948182693797,
     "pp_ci": [
      11.82502781869386,
      15.736585190976172
     ]
    },
    "natural": {
     "pp": 11.150991202614929,
     "pp_ci": [
      9.755812086519313,
      12.610679086592846
     ]
    }
   }
  },
  "Privileged base-stock reference (true laws and position; lots and service level from costs)": {
   "hard": {
    "pct": 66.50343288058397,
    "ci_pct": [
     62.99976652798559,
     69.64139823556398
    ]
   },
   "natural": {
    "pct": 70.0219729558243,
    "ci_pct": [
     67.61704053000746,
     72.25767793160182
    ]
   },
   "vs_careful": {
    "hard": {
     "pp": 10.201075677851083,
     "pp_ci": [
      8.160599442678992,
      12.377217866163488
     ]
    },
    "natural": {
     "pp": 7.577223296258325,
     "pp_ci": [
      6.109925166097186,
      9.090902207679239
     ]
    }
   }
  },
  "Careful rule + hidden item specs (upper bound for reading descriptions)": {
   "hard": {
    "pct": 56.917267733442515,
    "ci_pct": [
     52.31043238200228,
     61.34399739196418
    ]
   },
   "natural": {
    "pct": 62.75485968008884,
    "ci_pct": [
     59.55769232853556,
     65.61864379303147
    ]
   },
   "vs_careful": {
    "hard": {
     "pp": 0.6149105307096389,
     "pp_ci": [
      0.07039649770456288,
      1.397003437350691
     ]
    },
    "natural": {
     "pp": 0.310110020522878,
     "pp_ci": [
      0.051905321051900044,
      0.6932871115778614
     ]
    }
   }
  },
  "Textbook formula": {
   "hard": {
    "pct": 42.17576190124803,
    "ci_pct": [
     37.089565171643486,
     47.24703844346679
    ]
   },
   "natural": {
    "pct": 49.760646783441985,
    "ci_pct": [
     46.01522173622335,
     53.23763855307633
    ]
   },
   "vs_careful": {
    "hard": {
     "pp": -14.126595301484848,
     "pp_ci": [
      -17.491248184572584,
      -11.375644805155986
     ]
    },
    "natural": {
     "pp": -12.684102876123985,
     "pp_ci": [
      -14.866871549449032,
      -10.778602092183686
     ]
    }
   }
  }
 },
 "ai_runs": [
  {
   "model": "gpt-6.1-sol",
   "model_label": "GPT-6.1 Sol",
   "setup": "AI alone",
   "player": "tools_alone",
   "trial": 0,
   "effort": "high",
   "draws_per_book": 3,
   "books": 160,
   "book_draws": 480,
   "invalid_week_share": 0.0,
   "invalid_by_reason": {},
   "spend_usd": 1906.97,
   "decisions": 12480,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 160,
    "hard_books": 40,
    "score": {
     "hard": {
      "pct": 63.55065805596593,
      "ci_pct": [
       59.3797700819914,
       67.23616623964145
      ]
     },
     "natural": {
      "pct": 67.90778811920161,
      "ci_pct": [
       65.14242778511978,
       70.38355701555453
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 7.248300853233053,
       "pp_ci": [
        5.597120030046899,
        8.986712278840551
       ]
      },
      "natural": {
       "pp": 5.463038459635645,
       "pp_ci": [
        4.2145044410000425,
        6.773639370823732
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 56.30235720273287,
     "natural": 62.444749659565964
    }
   },
   "common_20_books": {
    "hard": 66.06477739357597,
    "natural": 69.74492688123304
   }
  },
  {
   "model": "gpt-6.1-sol",
   "model_label": "GPT-6.1 Sol",
   "setup": "AI alone",
   "player": "tools_alone",
   "trial": 1,
   "effort": "high",
   "draws_per_book": 3,
   "books": 20,
   "book_draws": 60,
   "invalid_week_share": 0.0,
   "invalid_by_reason": {},
   "spend_usd": 268.72,
   "decisions": 1560,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 20,
    "hard_books": 5,
    "score": {
     "hard": {
      "pct": 65.75634007359024,
      "ci_pct": [
       56.60830067713266,
       72.84729436333352
      ]
     },
     "natural": {
      "pct": 69.77253070747823,
      "ci_pct": [
       63.580661843305144,
       74.56248126205413
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 4.878772520812371,
       "pp_ci": [
        0.18702179803952212,
        8.357030846833931
       ]
      },
      "natural": {
       "pp": 4.31690437287971,
       "pp_ci": [
        1.4820983857688308,
        6.730902024160507
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 60.87756755277786,
     "natural": 65.4556263345985
    }
   },
   "common_20_books": {
    "hard": 65.75634007359024,
    "natural": 69.77253070747823
   }
  },
  {
   "model": "gpt-6.1-sol",
   "model_label": "GPT-6.1 Sol",
   "setup": "AI alone",
   "player": "tools_alone",
   "trial": 2,
   "effort": "high",
   "draws_per_book": 3,
   "books": 20,
   "book_draws": 60,
   "invalid_week_share": 0.0,
   "invalid_by_reason": {},
   "spend_usd": 241.51,
   "decisions": 1560,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 20,
    "hard_books": 5,
    "score": {
     "hard": {
      "pct": 65.1133518219319,
      "ci_pct": [
       56.19618330397359,
       72.58096654475331
      ]
     },
     "natural": {
      "pct": 69.45281570532154,
      "ci_pct": [
       63.13078793381338,
       74.29396626607033
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 4.235784269154043,
       "pp_ci": [
        -0.2600686811076528,
        7.476094749600755
       ]
      },
      "natural": {
       "pp": 3.997189370723019,
       "pp_ci": [
        0.8799727359359913,
        6.752645466062386
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 60.87756755277786,
     "natural": 65.4556263345985
    }
   },
   "common_20_books": {
    "hard": 65.1133518219319,
    "natural": 69.45281570532154
   }
  },
  {
   "model": "gpt-6.1-sol",
   "model_label": "GPT-6.1 Sol",
   "setup": "formula \u2192 AI",
   "player": "tools_formula_ai",
   "trial": 0,
   "effort": "high",
   "draws_per_book": 3,
   "books": 50,
   "book_draws": 150,
   "invalid_week_share": 0.0,
   "invalid_by_reason": {},
   "spend_usd": 594.75,
   "decisions": 3900,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 50,
    "hard_books": 13,
    "score": {
     "hard": {
      "pct": 63.31435827122656,
      "ci_pct": [
       53.57297401977695,
       70.71505518695541
      ]
     },
     "natural": {
      "pct": 68.50177773277531,
      "ci_pct": [
       62.61617526019309,
       72.7197815224464
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 5.846977509118923,
       "pp_ci": [
        3.687062849154674,
        7.741099979676188
       ]
      },
      "natural": {
       "pp": 5.393359318917036,
       "pp_ci": [
        3.660382434260047,
        7.048036788453683
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 57.467380762107645,
     "natural": 63.10841841385825
    }
   },
   "common_20_books": {
    "hard": 66.63287730369947,
    "natural": 70.8093609862697
   }
  },
  {
   "model": "claude-opus-5-5",
   "model_label": "Claude Opus 5.5",
   "setup": "AI alone",
   "player": "tools_alone",
   "trial": 0,
   "effort": "high",
   "draws_per_book": 3,
   "books": 160,
   "book_draws": 480,
   "invalid_week_share": 0.004166666666666667,
   "invalid_by_reason": {
    "max_tokens": 0.0034455128205128204,
    "parse": 8.012820512820513e-05,
    "bad_unit": 0.0005608974358974359,
    "refusal": 8.012820512820513e-05
   },
   "spend_usd": 1691.74,
   "decisions": 12480,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 160,
    "hard_books": 40,
    "score": {
     "hard": {
      "pct": 57.60983351749953,
      "ci_pct": [
       53.41954072278242,
       61.51412693251963
      ]
     },
     "natural": {
      "pct": 62.81507956322956,
      "ci_pct": [
       59.862730836163344,
       65.55180655765169
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 1.3074763147666464,
       "pp_ci": [
        -0.4662479471668902,
        3.0802172045057774
       ]
      },
      "natural": {
       "pp": 0.37032990366358925,
       "pp_ci": [
        -0.9673442591954589,
        1.6953397942502706
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 56.30235720273287,
     "natural": 62.444749659565964
    }
   },
   "common_20_books": {
    "hard": 59.53076818435499,
    "natural": 63.236244034592765
   }
  },
  {
   "model": "claude-opus-5-5",
   "model_label": "Claude Opus 5.5",
   "setup": "AI alone",
   "player": "tools_alone",
   "trial": 1,
   "effort": "high",
   "draws_per_book": 3,
   "books": 20,
   "book_draws": 60,
   "invalid_week_share": 0.0019230769230769232,
   "invalid_by_reason": {
    "max_tokens": 0.0019230769230769232
   },
   "spend_usd": 235.16,
   "decisions": 1560,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 20,
    "hard_books": 5,
    "score": {
     "hard": {
      "pct": 62.80770244574383,
      "ci_pct": [
       55.40881932409616,
       69.06046590634828
      ]
     },
     "natural": {
      "pct": 65.48107952005006,
      "ci_pct": [
       60.13274891205099,
       70.01317600163371
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 1.9301348929659692,
       "pp_ci": [
        -2.195556859023483,
        5.389957135059953
       ]
      },
      "natural": {
       "pp": 0.025453185451549454,
       "pp_ci": [
        -2.5812601582273245,
        2.657654169944893
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 60.87756755277786,
     "natural": 65.4556263345985
    }
   },
   "common_20_books": {
    "hard": 62.80770244574383,
    "natural": 65.48107952005006
   }
  },
  {
   "model": "claude-opus-5-5",
   "model_label": "Claude Opus 5.5",
   "setup": "AI alone",
   "player": "tools_alone",
   "trial": 2,
   "effort": "high",
   "draws_per_book": 3,
   "books": 20,
   "book_draws": 60,
   "invalid_week_share": 0.005128205128205128,
   "invalid_by_reason": {
    "max_tokens": 0.004487179487179487,
    "bad_unit": 0.000641025641025641
   },
   "spend_usd": 217.58,
   "decisions": 1560,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 20,
    "hard_books": 5,
    "score": {
     "hard": {
      "pct": 61.39775310142096,
      "ci_pct": [
       53.7868016128076,
       68.42615469212626
      ]
     },
     "natural": {
      "pct": 65.59491845259126,
      "ci_pct": [
       59.84352671980333,
       70.48620441360413
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 0.5201855486430811,
       "pp_ci": [
        -2.997634117928285,
        4.16662287531036
       ]
      },
      "natural": {
       "pp": 0.1392921179927444,
       "pp_ci": [
        -2.4166129261690394,
        2.451082725596104
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 60.87756755277786,
     "natural": 65.4556263345985
    }
   },
   "common_20_books": {
    "hard": 61.39775310142093,
    "natural": 65.59491845259126
   }
  },
  {
   "model": "claude-opus-5-5",
   "model_label": "Claude Opus 5.5",
   "setup": "formula \u2192 AI",
   "player": "tools_formula_ai",
   "trial": 0,
   "effort": "high",
   "draws_per_book": 3,
   "books": 50,
   "book_draws": 150,
   "invalid_week_share": 0.004871794871794872,
   "invalid_by_reason": {
    "bad_unit": 0.0002564102564102564,
    "max_tokens": 0.0038461538461538464,
    "refusal": 0.0005128205128205128,
    "parse": 0.0002564102564102564
   },
   "spend_usd": 451.23,
   "decisions": 3900,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 50,
    "hard_books": 13,
    "score": {
     "hard": {
      "pct": 59.5015509305231,
      "ci_pct": [
       49.77058611910388,
       66.79090294856607
      ]
     },
     "natural": {
      "pct": 64.595633246231,
      "ci_pct": [
       58.69556098487182,
       68.7992595794926
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 2.0341701684154474,
       "pp_ci": [
        0.32138294656739247,
        3.44774543889605
       ]
      },
      "natural": {
       "pp": 1.4872148323727457,
       "pp_ci": [
        -0.007770079092980532,
        2.8603839802309095
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 57.467380762107645,
     "natural": 63.10841841385825
    }
   },
   "common_20_books": {
    "hard": 63.00298894006943,
    "natural": 66.57067819949665
   }
  },
  {
   "model": "gemini-4-argon",
   "model_label": "Gemini 4 Argon",
   "setup": "AI alone",
   "player": "tools_alone",
   "trial": 0,
   "effort": "high",
   "draws_per_book": 3,
   "books": 160,
   "book_draws": 480,
   "invalid_week_share": 0.01746794871794872,
   "invalid_by_reason": {
    "tool_call": 0.015865384615384615,
    "bad_unit": 0.0011217948717948717,
    "po_not_list": 0.00040064102564102563,
    "refusal": 8.012820512820513e-05
   },
   "spend_usd": 3177.24,
   "decisions": 12480,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 160,
    "hard_books": 40,
    "score": {
     "hard": {
      "pct": 58.27691251206669,
      "ci_pct": [
       53.6922877579758,
       62.65028173038707
      ]
     },
     "natural": {
      "pct": 63.34789587272636,
      "ci_pct": [
       59.90128061610718,
       66.42315632583788
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 1.9745553093338188,
       "pp_ci": [
        -0.463208452637434,
        4.175057060325722
       ]
      },
      "natural": {
       "pp": 0.9031462131603921,
       "pp_ci": [
        -0.8761057489284199,
        2.6203543085194894
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 56.30235720273287,
     "natural": 62.444749659565964
    }
   },
   "common_20_books": {
    "hard": 60.17512824159501,
    "natural": 64.61550983715527
   }
  },
  {
   "model": "gemini-4-argon",
   "model_label": "Gemini 4 Argon",
   "setup": "AI alone",
   "player": "tools_alone",
   "trial": 1,
   "effort": "high",
   "draws_per_book": 3,
   "books": 20,
   "book_draws": 60,
   "invalid_week_share": 0.016025641025641024,
   "invalid_by_reason": {
    "tool_call": 0.014743589743589743,
    "bad_unit": 0.001282051282051282
   },
   "spend_usd": 420.98,
   "decisions": 1560,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 20,
    "hard_books": 5,
    "score": {
     "hard": {
      "pct": 60.634126291754534,
      "ci_pct": [
       49.34048823800713,
       68.77467811396276
      ]
     },
     "natural": {
      "pct": 64.29652182331785,
      "ci_pct": [
       56.27196112789563,
       70.54157325772732
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": -0.24344126102333044,
       "pp_ci": [
        -7.924537959593481,
        5.472074427286257
       ]
      },
      "natural": {
       "pp": -1.1591045112806557,
       "pp_ci": [
        -7.208798573595435,
        3.4452344667183508
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 60.87756755277786,
     "natural": 65.4556263345985
    }
   },
   "common_20_books": {
    "hard": 60.634126291754534,
    "natural": 64.29652182331785
   }
  },
  {
   "model": "gemini-4-argon",
   "model_label": "Gemini 4 Argon",
   "setup": "AI alone",
   "player": "tools_alone",
   "trial": 2,
   "effort": "high",
   "draws_per_book": 3,
   "books": 20,
   "book_draws": 60,
   "invalid_week_share": 0.019230769230769232,
   "invalid_by_reason": {
    "po_not_list": 0.000641025641025641,
    "tool_call": 0.017948717948717947,
    "max_tokens": 0.000641025641025641
   },
   "spend_usd": 387.97,
   "decisions": 1560,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 20,
    "hard_books": 5,
    "score": {
     "hard": {
      "pct": 60.63915703797053,
      "ci_pct": [
       46.42112025764998,
       70.50722140208144
      ]
     },
     "natural": {
      "pct": 65.1123871837221,
      "ci_pct": [
       56.03921580459854,
       71.94792804810535
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": -0.23841051480733735,
       "pp_ci": [
        -10.215172214977898,
        7.2068402325274645
       ]
      },
      "natural": {
       "pp": -0.34323915087639517,
       "pp_ci": [
        -6.985126788127964,
        4.761297149191275
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 60.87756755277786,
     "natural": 65.4556263345985
    }
   },
   "common_20_books": {
    "hard": 60.63915703797053,
    "natural": 65.1123871837221
   }
  },
  {
   "model": "gemini-4-argon",
   "model_label": "Gemini 4 Argon",
   "setup": "formula \u2192 AI",
   "player": "tools_formula_ai",
   "trial": 0,
   "effort": "high",
   "draws_per_book": 3,
   "books": 50,
   "book_draws": 150,
   "invalid_week_share": 0.017435897435897435,
   "invalid_by_reason": {
    "tool_call": 0.016666666666666666,
    "bad_unit": 0.0005128205128205128,
    "line_not_object": 0.0002564102564102564
   },
   "spend_usd": 954.88,
   "decisions": 3900,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 50,
    "hard_books": 13,
    "score": {
     "hard": {
      "pct": 60.99694975273956,
      "ci_pct": [
       52.181671597796985,
       68.20420825291932
      ]
     },
     "natural": {
      "pct": 65.30936914356516,
      "ci_pct": [
       59.396606021902,
       70.04991989092207
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 3.529568990631923,
       "pp_ci": [
        0.2124062078962834,
        6.302223613051666
       ]
      },
      "natural": {
       "pp": 2.2009507297068995,
       "pp_ci": [
        -1.020496707491395,
        4.933222299735988
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 57.467380762107645,
     "natural": 63.10841841385825
    }
   },
   "common_20_books": {
    "hard": 64.41495689577836,
    "natural": 68.01249314207442
   }
  },
  {
   "model": "gpt-6.1-sol",
   "model_label": "GPT-6.1 Sol",
   "setup": "AI alone, one screen",
   "player": "ai_alone",
   "trial": 0,
   "effort": "high",
   "draws_per_book": 3,
   "books": 50,
   "book_draws": 150,
   "invalid_week_share": 0.0,
   "invalid_by_reason": {},
   "spend_usd": 347.64,
   "decisions": 3900,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 50,
    "hard_books": 13,
    "score": {
     "hard": {
      "pct": 62.5601951183604,
      "ci_pct": [
       52.61840200225144,
       70.37020557859488
      ]
     },
     "natural": {
      "pct": 67.950327828326,
      "ci_pct": [
       62.078548775255335,
       72.26541577707529
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 5.092814356252751,
       "pp_ci": [
        2.991907872435798,
        6.9903332213774885
       ]
      },
      "natural": {
       "pp": 4.84190941446774,
       "pp_ci": [
        3.176515929512717,
        6.54293329481649
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 57.467380762107645,
     "natural": 63.10841841385825
    }
   },
   "common_20_books": {
    "hard": 65.39575877295354,
    "natural": 69.7436291225928
   }
  },
  {
   "model": "claude-opus-5-5",
   "model_label": "Claude Opus 5.5",
   "setup": "AI alone, one screen",
   "player": "ai_alone",
   "trial": 0,
   "effort": "high",
   "draws_per_book": 3,
   "books": 50,
   "book_draws": 150,
   "invalid_week_share": 0.0,
   "invalid_by_reason": {},
   "spend_usd": 679.09,
   "decisions": 3900,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 50,
    "hard_books": 13,
    "score": {
     "hard": {
      "pct": 60.207024403765836,
      "ci_pct": [
       51.2006136069625,
       67.00371125031297
      ]
     },
     "natural": {
      "pct": 64.81030906836347,
      "ci_pct": [
       59.339303821010425,
       68.81039807172337
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 2.739643641658193,
       "pp_ci": [
        0.6452200507458885,
        4.513068089538996
       ]
      },
      "natural": {
       "pp": 1.7018906545051964,
       "pp_ci": [
        0.2000005333521326,
        3.05579801730953
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 57.467380762107645,
     "natural": 63.10841841385825
    }
   },
   "common_20_books": {
    "hard": 64.0570763885269,
    "natural": 67.237313475397
   }
  },
  {
   "model": "gemini-4-argon",
   "model_label": "Gemini 4 Argon",
   "setup": "AI alone, one screen",
   "player": "ai_alone",
   "trial": 0,
   "effort": "high",
   "draws_per_book": 3,
   "books": 50,
   "book_draws": 150,
   "invalid_week_share": 0.0033333333333333335,
   "invalid_by_reason": {
    "bad_unit": 0.003076923076923077,
    "parse": 0.0002564102564102564
   },
   "spend_usd": 680.9,
   "decisions": 3900,
   "spend_definition": "Conservative estimated budget charge, including failed, orphaned, and usage-unknown calls at reservation bounds; not verified invoiced API cost",
   "own_set": {
    "books": 50,
    "hard_books": 13,
    "score": {
     "hard": {
      "pct": 58.67723879855062,
      "ci_pct": [
       47.97296705875888,
       68.15159063492597
      ]
     },
     "natural": {
      "pct": 65.51655793394312,
      "ci_pct": [
       59.227563603361,
       70.53169896546385
      ]
     },
     "vs_careful": {
      "hard": {
       "pp": 1.2098580364429738,
       "pp_ci": [
        -5.033507600205364,
        6.048446597817643
       ]
      },
      "natural": {
       "pp": 2.4081395200848688,
       "pp_ci": [
        -1.0807435752875358,
        5.352518164445987
       ]
      }
     }
    },
    "careful_same_set": {
     "hard": 57.467380762107645,
     "natural": 63.10841841385825
    }
   },
   "common_20_books": {
    "hard": 62.0154992628473,
    "natural": 67.66939057861232
   }
  }
 ],
 "supplementary_worst_run_sensitivity": {
  "GPT-6.1 Sol": {
   "hard": {
    "trial0_160_books": {
     "pct": 63.55065805596593,
     "ci_pct": [
      59.3797700819914,
      67.23616623964145
     ]
    },
    "trials_on_20_books": {
     "0": 66.06477739357597,
     "1": 65.75634007359024,
     "2": 65.1133518219319
    },
    "adjustment_pp": -0.9514255716440658,
    "reported": 62.59923248432187
   },
   "natural": {
    "trial0_160_books": {
     "pct": 67.90778811920161,
     "ci_pct": [
      65.14242778511978,
      70.38355701555453
     ]
    },
    "trials_on_20_books": {
     "0": 69.74492688123304,
     "1": 69.77253070747823,
     "2": 69.45281570532154
    },
    "adjustment_pp": -0.2921111759115007,
    "reported": 67.61567694329011
   }
  },
  "Claude Opus 5.5": {
   "hard": {
    "trial0_160_books": {
     "pct": 57.60983351749953,
     "ci_pct": [
      53.41954072278242,
      61.51412693251963
     ]
    },
    "trials_on_20_books": {
     "0": 59.53076818435499,
     "1": 62.80770244574383,
     "2": 61.39775310142093
    },
    "adjustment_pp": 0.0,
    "reported": 57.60983351749953
   },
   "natural": {
    "trial0_160_books": {
     "pct": 62.81507956322956,
     "ci_pct": [
      59.862730836163344,
      65.55180655765169
     ]
    },
    "trials_on_20_books": {
     "0": 63.236244034592765,
     "1": 65.48107952005006,
     "2": 65.59491845259126
    },
    "adjustment_pp": 0.0,
    "reported": 62.81507956322956
   }
  },
  "Gemini 4 Argon": {
   "hard": {
    "trial0_160_books": {
     "pct": 58.27691251206669,
     "ci_pct": [
      53.6922877579758,
      62.65028173038707
     ]
    },
    "trials_on_20_books": {
     "0": 60.17512824159501,
     "1": 60.634126291754534,
     "2": 60.63915703797053
    },
    "adjustment_pp": 0.0,
    "reported": 58.27691251206669
   },
   "natural": {
    "trial0_160_books": {
     "pct": 63.34789587272636,
     "ci_pct": [
      59.90128061610718,
      66.42315632583788
     ]
    },
    "trials_on_20_books": {
     "0": 64.61550983715527,
     "1": 64.29652182331785,
     "2": 65.1123871837221
    },
    "adjustment_pp": -0.31898801383742637,
    "reported": 63.028907858888935
   }
  }
 },
 "total_spend_usd": 12256.36,
 "spend_definition": "Conservative estimated budget charges from the run ledger, averaged over the same book-draws as the scores. Includes settled calls plus failed, orphaned, and usage-unknown calls charged at reservation bounds; not verified invoiced API costs. Final-decision recorded usage is reported separately.",
 "figure2_matched_50": {
  "books": 50,
  "book_worlds": 150,
  "hard_books": 13,
  "sample_definition": "Actual intersection of all nine trial-0 model/setup runs; tools-alone reuses a subset of the primary full run. The 20-book repeats overlap this subset and are not independent replications.",
  "weight_definition": "Fixed full-holdout weights: hard books 2, other books 2/3; natural books 1. Hardness is not recomputed on the subset.",
  "interval_definition": "95% percentile bootstrap over sorted books, retaining all three shared draws within each sampled book; 2000 resamples, numpy default_rng seed 1. Paired comparisons use the same sampled books on both sides. Conditional on trial 0 and the frozen draws; excludes run-to-run and calibration uncertainty.",
  "cost_definition": "Conservative estimated budget charges from the run ledger, averaged over the same book-draws as the scores. Includes settled calls plus failed, orphaned, and usage-unknown calls charged at reservation bounds; not verified invoiced API costs. Final-decision recorded usage is reported separately.",
  "rows": [
   {
    "model": "sol",
    "setup": "tools",
    "score": 62.6,
    "score_natural": 67.4,
    "usd": 3.98,
    "recorded_decision_usd": 3.85,
    "out_tok": 129014,
    "tool_calls": 563.4,
    "seasons": 150,
    "score_estimates": {
     "hard": {
      "pct": 62.55341040609338,
      "ci_pct": [
       52.78219851808051,
       69.73149765051853
      ]
     },
     "natural": {
      "pct": 67.44331628233823,
      "ci_pct": [
       61.57672981313911,
       71.55231025867786
      ]
     }
    },
    "cost": {
     "budget_charge_usd": 597.3611802999992,
     "budget_charge_usd_per_season": 3.9824078686666615,
     "recorded_decision_usd": 577.7110878000011,
     "recorded_decision_usd_per_season": 3.8514072520000076,
     "failed_charge_usd": 14.222724999999999,
     "orphaned_charge_usd": 5.4273675,
     "usage_unknown_settled_charge_usd": 0.0
    }
   },
   {
    "model": "sol",
    "setup": "formula_ai",
    "score": 63.3,
    "score_natural": 68.5,
    "usd": 3.97,
    "recorded_decision_usd": 3.81,
    "out_tok": 129971,
    "tool_calls": 554.6,
    "seasons": 150,
    "score_estimates": {
     "hard": {
      "pct": 63.31435827155708,
      "ci_pct": [
       53.282917275785884,
       70.65526637235615
      ]
     },
     "natural": {
      "pct": 68.50177773301675,
      "ci_pct": [
       62.57543374005265,
       72.68554420920029
      ]
     }
    },
    "cost": {
     "budget_charge_usd": 594.7507773999996,
     "budget_charge_usd_per_season": 3.965005182666664,
     "recorded_decision_usd": 571.1369548999999,
     "recorded_decision_usd_per_season": 3.8075796993333326,
     "failed_charge_usd": 5.930610000000001,
     "orphaned_charge_usd": 17.6832125,
     "usage_unknown_settled_charge_usd": 0.0
    }
   },
   {
    "model": "sol",
    "setup": "alone",
    "score": 62.6,
    "score_natural": 68.0,
    "usd": 2.32,
    "recorded_decision_usd": 2.16,
    "out_tok": 96418,
    "tool_calls": 0.0,
    "seasons": 150,
    "score_estimates": {
     "hard": {
      "pct": 62.56019511868698,
      "ci_pct": [
       52.19466966849608,
       70.14532757610728
      ]
     },
     "natural": {
      "pct": 67.9503278285655,
      "ci_pct": [
       61.86925499681888,
       72.24975462723458
      ]
     }
    },
    "cost": {
     "budget_charge_usd": 347.64199360000066,
     "budget_charge_usd_per_season": 2.317613290666671,
     "recorded_decision_usd": 324.2159036000005,
     "recorded_decision_usd_per_season": 2.161439357333337,
     "failed_charge_usd": 4.0346,
     "orphaned_charge_usd": 19.39149,
     "usage_unknown_settled_charge_usd": 0.0
    }
   },
   {
    "model": "opus",
    "setup": "tools",
    "score": 57.7,
    "score_natural": 62.5,
    "usd": 3.45,
    "recorded_decision_usd": 3.43,
    "out_tok": 108093,
    "tool_calls": 179.1,
    "seasons": 150,
    "score_estimates": {
     "hard": {
      "pct": 57.675411927847605,
      "ci_pct": [
       48.69498483547737,
       64.8701104599389
      ]
     },
     "natural": {
      "pct": 62.52966399332108,
      "ci_pct": [
       56.75115267047524,
       66.97537517774701
      ]
     }
    },
    "cost": {
     "budget_charge_usd": 518.2062633999999,
     "budget_charge_usd_per_season": 3.454708422666666,
     "recorded_decision_usd": 513.8973333999999,
     "recorded_decision_usd_per_season": 3.425982222666666,
     "failed_charge_usd": 1.42273,
     "orphaned_charge_usd": 2.8862,
     "usage_unknown_settled_charge_usd": 0.0
    }
   },
   {
    "model": "opus",
    "setup": "formula_ai",
    "score": 59.5,
    "score_natural": 64.6,
    "usd": 3.01,
    "recorded_decision_usd": 2.86,
    "out_tok": 90014,
    "tool_calls": 137.7,
    "seasons": 150,
    "score_estimates": {
     "hard": {
      "pct": 59.50155093083371,
      "ci_pct": [
       49.87414878462539,
       66.65387095053208
      ]
     },
     "natural": {
      "pct": 64.59563324645869,
      "ci_pct": [
       58.75675698744796,
       68.79160925169342
      ]
     }
    },
    "cost": {
     "budget_charge_usd": 451.2307168000009,
     "budget_charge_usd_per_season": 3.0082047786666726,
     "recorded_decision_usd": 429.1673317999995,
     "recorded_decision_usd_per_season": 2.86111554533333,
     "failed_charge_usd": 1.434765,
     "orphaned_charge_usd": 20.628620000000005,
     "usage_unknown_settled_charge_usd": 0.0
    }
   },
   {
    "model": "opus",
    "setup": "alone",
    "score": 60.2,
    "score_natural": 64.8,
    "usd": 4.53,
    "recorded_decision_usd": 4.38,
    "out_tok": 82337,
    "tool_calls": 0.0,
    "seasons": 150,
    "score_estimates": {
     "hard": {
      "pct": 60.20702440408013,
      "ci_pct": [
       51.10949764968748,
       66.79745648065571
      ]
     },
     "natural": {
      "pct": 64.81030906859189,
      "ci_pct": [
       59.31008823248489,
       68.7178012941693
      ]
     }
    },
    "cost": {
     "budget_charge_usd": 679.0853991999986,
     "budget_charge_usd_per_season": 4.527235994666658,
     "recorded_decision_usd": 657.1967791999987,
     "recorded_decision_usd_per_season": 4.381311861333325,
     "failed_charge_usd": 0.0,
     "orphaned_charge_usd": 21.88862,
     "usage_unknown_settled_charge_usd": 0.0
    }
   },
   {
    "model": "gemini",
    "setup": "tools",
    "score": 59.0,
    "score_natural": 63.3,
    "usd": 6.55,
    "recorded_decision_usd": 6.37,
    "out_tok": 297957,
    "tool_calls": 589.2,
    "seasons": 150,
    "score_estimates": {
     "hard": {
      "pct": 58.976387740103526,
      "ci_pct": [
       49.905322916180744,
       66.47667477149757
      ]
     },
     "natural": {
      "pct": 63.27604104696215,
      "ci_pct": [
       57.22879369890433,
       68.28057828651039
      ]
     }
    },
    "cost": {
     "budget_charge_usd": 983.0036981000023,
     "budget_charge_usd_per_season": 6.5533579873333485,
     "recorded_decision_usd": 955.6308680999986,
     "recorded_decision_usd_per_season": 6.370872453999991,
     "failed_charge_usd": 2.70139,
     "orphaned_charge_usd": 16.673792000000002,
     "usage_unknown_settled_charge_usd": 7.997648
    }
   },
   {
    "model": "gemini",
    "setup": "formula_ai",
    "score": 61.0,
    "score_natural": 65.3,
    "usd": 6.37,
    "recorded_decision_usd": 6.06,
    "out_tok": 277609,
    "tool_calls": 558.1,
    "seasons": 150,
    "score_estimates": {
     "hard": {
      "pct": 60.99694975305798,
      "ci_pct": [
       52.032796332005766,
       67.94478766601028
      ]
     },
     "natural": {
      "pct": 65.30936914379535,
      "ci_pct": [
       59.44992872839039,
       70.00513707414014
      ]
     }
    },
    "cost": {
     "budget_charge_usd": 954.8825394999972,
     "budget_charge_usd_per_season": 6.365883596666648,
     "recorded_decision_usd": 909.5677274999996,
     "recorded_decision_usd_per_season": 6.063784849999998,
     "failed_charge_usd": 2.753396,
     "orphaned_charge_usd": 33.220304000000006,
     "usage_unknown_settled_charge_usd": 9.341111999999999
    }
   },
   {
    "model": "gemini",
    "setup": "alone",
    "score": 58.7,
    "score_natural": 65.5,
    "usd": 4.54,
    "recorded_decision_usd": 4.34,
    "out_tok": 264566,
    "tool_calls": 0.0,
    "seasons": 150,
    "score_estimates": {
     "hard": {
      "pct": 58.67723879885693,
      "ci_pct": [
       47.39548892328293,
       67.85444422860316
      ]
     },
     "natural": {
      "pct": 65.51655793417405,
      "ci_pct": [
       58.67896800292585,
       70.45676784662174
      ]
     }
    },
    "cost": {
     "budget_charge_usd": 680.8992566000012,
     "budget_charge_usd_per_season": 4.539328377333342,
     "recorded_decision_usd": 650.2668446000005,
     "recorded_decision_usd_per_season": 4.335112297333336,
     "failed_charge_usd": 7.006493999999999,
     "orphaned_charge_usd": 23.625918000000002,
     "usage_unknown_settled_charge_usd": 0.0
    }
   }
  ],
  "baselines": {
   "careful": {
    "hard": 57.5,
    "natural": 63.1
   },
   "erp_auto_rop": {
    "hard": 36.5,
    "natural": 42.9
   },
   "reference_rollout": {
    "hard": 68.3,
    "natural": 72.5
   }
  },
  "paired_setup_comparisons": {
   "sol_formula_ai_minus_tools": {
    "hard": {
     "pp": 0.7609478654637059,
     "ci_pp": [
      -0.001091101460131727,
      1.7118025119104578
     ]
    },
    "natural": {
     "pp": 1.0584614506784973,
     "ci_pp": [
      0.46363140445199036,
      1.6867821559068072
     ]
    }
   },
   "sol_alone_minus_tools": {
    "hard": {
     "pp": 0.0067847125935959485,
     "ci_pp": [
      -0.8167981077250022,
      0.887994320631098
     ]
    },
    "natural": {
     "pp": 0.507011546227258,
     "ci_pp": [
      -0.2699152900840748,
      1.3040109456616735
     ]
    }
   },
   "sol_alone_minus_formula_ai": {
    "hard": {
     "pp": -0.7541631528701096,
     "ci_pp": [
      -1.789338264360324,
      0.2288391864951634
     ]
    },
    "natural": {
     "pp": -0.5514499044512393,
     "ci_pp": [
      -1.3719449293752224,
      0.34674836589434876
     ]
    }
   },
   "opus_formula_ai_minus_tools": {
    "hard": {
     "pp": 1.826139002986107,
     "ci_pp": [
      -0.12156394280806762,
      4.307515465564833
     ]
    },
    "natural": {
     "pp": 2.065969253137598,
     "ci_pp": [
      0.3060039216819034,
      4.460489436572263
     ]
    }
   },
   "opus_alone_minus_tools": {
    "hard": {
     "pp": 2.5316124762325374,
     "ci_pp": [
      0.9407970061256263,
      4.631086637114237
     ]
    },
    "natural": {
     "pp": 2.2806450752708063,
     "ci_pp": [
      0.6504407273020477,
      4.588338142709219
     ]
    }
   },
   "opus_alone_minus_formula_ai": {
    "hard": {
     "pp": 0.7054734732464306,
     "ci_pp": [
      -0.4627801851121231,
      1.8364929727155606
     ]
    },
    "natural": {
     "pp": 0.21467582213320893,
     "ci_pp": [
      -0.6937135930025922,
      1.143363594762419
     ]
    }
   },
   "gemini_formula_ai_minus_tools": {
    "hard": {
     "pp": 2.0205620129544632,
     "ci_pp": [
      0.031141495636671946,
      4.526464701010491
     ]
    },
    "natural": {
     "pp": 2.033328096833204,
     "ci_pp": [
      0.39447250812766677,
      3.9723084661166568
     ]
    }
   },
   "gemini_alone_minus_tools": {
    "hard": {
     "pp": -0.2991489412465971,
     "ci_pp": [
      -4.595047711075416,
      3.7444620768429893
     ]
    },
    "natural": {
     "pp": 2.2405168872119043,
     "ci_pp": [
      -1.1378405013814565,
      5.792237702057543
     ]
    }
   },
   "gemini_alone_minus_formula_ai": {
    "hard": {
     "pp": -2.319710954201059,
     "ci_pp": [
      -7.537684133806834,
      1.9346469364648324
     ]
    },
    "natural": {
     "pp": 0.2071887903786999,
     "ci_pp": [
      -3.252929276426843,
      3.3840930540017906
     ]
    }
   }
  },
  "paired_vs_careful": {
   "sol_tools": {
    "hard": {
     "pp": 5.086029643685739,
     "ci_pp": [
      2.934999245346658,
      6.93993899673533
     ]
    },
    "natural": {
     "pp": 4.334897868257549,
     "ci_pp": [
      2.4784553876134514,
      6.015022993848352
     ]
    }
   },
   "sol_formula_ai": {
    "hard": {
     "pp": 5.846977509149445,
     "ci_pp": [
      3.6870628491901902,
      7.741099979709031
     ]
    },
    "natural": {
     "pp": 5.393359318936046,
     "ci_pp": [
      3.6603824342927456,
      7.04803678845634
     ]
    }
   },
   "sol_alone": {
    "hard": {
     "pp": 5.092814356279335,
     "ci_pp": [
      2.9919078724531976,
      6.990333221429655
     ]
    },
    "natural": {
     "pp": 4.841909414484807,
     "ci_pp": [
      3.176515929536749,
      6.542933294867176
     ]
    }
   },
   "opus_tools": {
    "hard": {
     "pp": 0.20803116543995717,
     "ci_pp": [
      -2.8771580089677866,
      2.7700687667357164
     ]
    },
    "natural": {
     "pp": -0.5787544207596115,
     "ci_pp": [
      -3.4855472159665335,
      1.5003027838589538
     ]
    }
   },
   "opus_formula_ai": {
    "hard": {
     "pp": 2.0341701684260634,
     "ci_pp": [
      0.32138294656913663,
      3.447745438907019
     ]
    },
    "natural": {
     "pp": 1.4872148323779861,
     "ci_pp": [
      -0.007770079092981017,
      2.8603839802388515
     ]
    }
   },
   "opus_alone": {
    "hard": {
     "pp": 2.7396436416724947,
     "ci_pp": [
      0.6452200507492336,
      4.513068089563427
     ]
    },
    "natural": {
     "pp": 1.701890654511195,
     "ci_pp": [
      0.20000053335207013,
      3.055798017325751
     ]
    }
   },
   "gemini_tools": {
    "hard": {
     "pp": 1.5090069776958837,
     "ci_pp": [
      -3.1729220862140273,
      5.087856553212994
     ]
    },
    "natural": {
     "pp": 0.16762263288145163,
     "ci_pp": [
      -3.5519724120996807,
      3.306812717103084
     ]
    }
   },
   "gemini_formula_ai": {
    "hard": {
     "pp": 3.529568990650347,
     "ci_pp": [
      0.2124062078977797,
      6.302223613047267
     ]
    },
    "natural": {
     "pp": 2.200950729714656,
     "ci_pp": [
      -1.020496707494093,
      4.933222299744275
     ]
    }
   },
   "gemini_alone": {
    "hard": {
     "pp": 1.209858036449288,
     "ci_pp": [
      -5.033507600270872,
      6.048446597823866
     ]
    },
    "natural": {
     "pp": 2.4081395200933557,
     "ci_pp": [
      -1.0807435752925216,
      5.352518164460168
     ]
    }
   }
  }
 },
 "primary_model_comparisons": {
  "books": 160,
  "book_draws": 480,
  "setup": "tools",
  "trial": 0,
  "pairs": {
   "sol_minus_gemini": {
    "hard": {
     "pp": 5.273745543927181,
     "ci_pp": [
      3.272576686984285,
      7.594446555750819
     ]
    },
    "natural": {
     "pp": 4.559892246484555,
     "ci_pp": [
      2.9428157259958843,
      6.33394627791525
     ]
    }
   },
   "sol_minus_opus": {
    "hard": {
     "pp": 5.940824538497887,
     "ci_pp": [
      4.814489689422486,
      7.131175692067254
     ]
    },
    "natural": {
     "pp": 5.092708555982445,
     "ci_pp": [
      4.187863098583448,
      6.060196330175457
     ]
    }
   }
  },
  "interval_definition": "95% percentile bootstrap over sorted books, retaining all three shared draws within each sampled book; 2000 resamples, numpy default_rng seed 1. Paired comparisons use the same sampled books on both sides. Conditional on trial 0 and the frozen draws; excludes run-to-run and calibration uncertainty."
 },
 "legacy_settlement_sensitivity": {
  "schema_version": "larkbench-legacy-settlement-audit-1",
  "status": "new_offline_calculation_from_existing_trial0_records",
  "books": 160,
  "worlds_per_book": 3,
  "seasons": 480,
  "input_settlement_field": "net_profit_v20",
  "denominator": "same contemporaneous demand-margin reference as main v2.1 scoring",
  "comparison_limits": [
   "This is a bundled alternative settlement, not a controlled test of the terminal rule alone.",
   "Legacy v2.0 omits explicit waiting cost and continuation value, and substitutes its own end-inventory carrying charge.",
   "All policies/actions remain the recorded trial-0 actions; no model was rerun under legacy incentives.",
   "No score sensitivity to alternative writeoff percentages, continuation duration or unprivileged continuation buyer has been calculated here."
  ],
  "identity": "net_profit_v20 = net_profit_current - terminal_value + cost_backorder - cost_writedown_v20",
  "max_identity_residual_usd": 2.0000006770715117e-06,
  "bootstrap": {
   "resamples": 2000,
   "unit": "book, with all three worlds kept together",
   "level_seed": 0,
   "paired_seed": 1,
   "interval": "percentile 2.5th to97.5th",
   "hard_weights": "2 on top-quartile40 books;2/3 on other120 books; chosen with separate calibration worlds"
  },
  "scores": {
   "natural": {
    "sol": {
     "score_pct": 73.29286331046252,
     "ci95_pct": [
      71.28157413651408,
      75.19941959807994
     ]
    },
    "opus": {
     "score_pct": 69.58167152880465,
     "ci95_pct": [
      67.4288900833537,
      71.62127626342551
     ]
    },
    "gemini": {
     "score_pct": 69.50116180463857,
     "ci95_pct": [
      66.57067839566916,
      72.14829205797359
     ]
    },
    "careful": {
     "score_pct": 67.82522779007303,
     "ci95_pct": [
      65.21276170763755,
      70.17650409695729
     ]
    },
    "erp_auto_rop": {
     "score_pct": 54.528672383903164,
     "ci95_pct": [
      52.1366715293956,
      56.70218478845752
     ]
    },
    "best_at_time": {
     "score_pct": 71.82726685363602,
     "ci95_pct": [
      69.32604136625768,
      74.11649673938294
     ]
    },
    "reference_rollout": {
     "score_pct": 75.6382216410789,
     "ci95_pct": [
      73.76287520378453,
      77.41095105588667
     ]
    }
   },
   "hard": {
    "sol": {
     "score_pct": 70.25577219756995,
     "ci95_pct": [
      67.6345786887503,
      72.88748633435117
     ]
    },
    "opus": {
     "score_pct": 66.30692832025967,
     "ci95_pct": [
      63.621467107373775,
      69.0005178123371
     ]
    },
    "gemini": {
     "score_pct": 66.03036260959958,
     "ci95_pct": [
      62.4902444399376,
      69.50305154682377
     ]
    },
    "careful": {
     "score_pct": 63.39854115604634,
     "ci95_pct": [
      59.99740184095203,
      66.72349852480407
     ]
    },
    "erp_auto_rop": {
     "score_pct": 50.90208265490473,
     "ci95_pct": [
      47.813130262748224,
      53.8694922467148
     ]
    },
    "best_at_time": {
     "score_pct": 67.71072263005932,
     "ci95_pct": [
      64.17484530134556,
      71.03156581236041
     ]
    },
    "reference_rollout": {
     "score_pct": 72.67111691347331,
     "ci95_pct": [
      70.00623269475282,
      75.27512544596279
     ]
    }
   }
  },
  "paired_differences": {
   "natural": {
    "sol__minus__careful": {
     "difference_pp": 5.467635520389489,
     "ci95_pp": [
      4.430704147103005,
      6.557172822631096
     ]
    },
    "opus__minus__careful": {
     "difference_pp": 1.7564437387316372,
     "ci95_pp": [
      0.7046960739601149,
      2.850773222597149
     ]
    },
    "gemini__minus__careful": {
     "difference_pp": 1.6759340145655455,
     "ci95_pp": [
      0.15884575320468095,
      3.1156636014332744
     ]
    },
    "sol__minus__opus": {
     "difference_pp": 3.7111917816578535,
     "ci95_pp": [
      2.8246359979482762,
      4.596806885575449
     ]
    },
    "sol__minus__gemini": {
     "difference_pp": 3.7917015058239447,
     "ci95_pp": [
      2.3615315860891384,
      5.287111604290812
     ]
    },
    "gemini__minus__opus": {
     "difference_pp": -0.08050972416609187,
     "ci95_pp": [
      -1.8048755642372125,
      1.5323836955273225
     ]
    }
   },
   "hard": {
    "sol__minus__careful": {
     "difference_pp": 6.857231041523636,
     "ci95_pp": [
      5.406346178495118,
      8.297795216302315
     ]
    },
    "opus__minus__careful": {
     "difference_pp": 2.908387164213333,
     "ci95_pp": [
      1.5788891425321183,
      4.259442971166431
     ]
    },
    "gemini__minus__careful": {
     "difference_pp": 2.63182145355325,
     "ci95_pp": [
      0.7246831258464236,
      4.381162068295553
     ]
    },
    "sol__minus__opus": {
     "difference_pp": 3.9488438773103023,
     "ci95_pp": [
      2.882807956305237,
      4.9900132096683265
     ]
    },
    "sol__minus__gemini": {
     "difference_pp": 4.225409587970385,
     "ci95_pp": [
      2.5196190158484977,
      6.126657274330425
     ]
    },
    "gemini__minus__opus": {
     "difference_pp": -0.2765657106600821,
     "ci95_pp": [
      -2.4225135184578477,
      1.576760477819075
     ]
    }
   }
  },
  "source_code_revision": "6cccaf7b4aaa0d2f628f48854b6c071ba7263cd8",
  "privacy": "aggregate-only; no book IDs, world IDs, secret seed, per-book outcomes, prompts or transcripts",
  "change_components": {
   "sol": {
    "natural": {
     "current_score_pct": 67.90778811934014,
     "legacy_score_pct": 73.29286331061202,
     "change_pp": 5.385075191271881,
     "waiting_cost_added_back_pp": 2.950932629276423,
     "current_terminal_value_removed_pp": 3.6689077384105477,
     "legacy_inventory_charge_applied_pp": -1.2347651764150847
    },
    "hard": {
     "current_score_pct": 63.550658056302694,
     "legacy_score_pct": 70.25577219794225,
     "change_pp": 6.705114141639555,
     "waiting_cost_added_back_pp": 3.5253802902984446,
     "current_terminal_value_removed_pp": 4.609386333592406,
     "legacy_inventory_charge_applied_pp": -1.4296524822512813
    }
   },
   "opus": {
    "natural": {
     "current_score_pct": 62.8150795633577,
     "legacy_score_pct": 69.58167152894659,
     "change_pp": 6.766591965588887,
     "waiting_cost_added_back_pp": 2.6693084310568533,
     "current_terminal_value_removed_pp": 6.208717003531909,
     "legacy_inventory_charge_applied_pp": -2.1114334689998553
    },
    "hard": {
     "current_score_pct": 57.60983351780481,
     "legacy_score_pct": 66.30692832061104,
     "change_pp": 8.697094802806227,
     "waiting_cost_added_back_pp": 3.0893779133327093,
     "current_terminal_value_removed_pp": 8.037975346725247,
     "legacy_inventory_charge_applied_pp": -2.430258457251727
    }
   },
   "gemini": {
    "natural": {
     "current_score_pct": 63.34789587285558,
     "legacy_score_pct": 69.50116180478035,
     "change_pp": 6.153265931924771,
     "waiting_cost_added_back_pp": 2.526535461328308,
     "current_terminal_value_removed_pp": 8.422708948679814,
     "legacy_inventory_charge_applied_pp": -4.79597847808336
    },
    "hard": {
     "current_score_pct": 58.2769125123755,
     "legacy_score_pct": 66.03036260994948,
     "change_pp": 7.753450097573975,
     "waiting_cost_added_back_pp": 2.9808266558027587,
     "current_terminal_value_removed_pp": 10.275735107183856,
     "legacy_inventory_charge_applied_pp": -5.503111665412638
    }
   },
   "careful": {
    "natural": {
     "current_score_pct": 62.444749659565964,
     "legacy_score_pct": 67.82522779007303,
     "change_pp": 5.380478130507065,
     "waiting_cost_added_back_pp": 2.5910756840507876,
     "current_terminal_value_removed_pp": 6.05042179438201,
     "legacy_inventory_charge_applied_pp": -3.2610193478866996
    },
    "hard": {
     "current_score_pct": 56.302357202732885,
     "legacy_score_pct": 63.39854115604634,
     "change_pp": 7.0961839533134565,
     "waiting_cost_added_back_pp": 3.1670808411857885,
     "current_terminal_value_removed_pp": 8.008880993773454,
     "legacy_inventory_charge_applied_pp": -4.079777881388289
    }
   }
  }
 },
 "loss_breakdown_trial0": {
  "sol-tools": {
   "hard": {
    "kept": 63.550658056302694,
    "missed": 11.576781595922462,
    "holding": 9.971471921335064,
    "ordering": 6.766321802548925,
    "waiting": 3.5253802902984446,
    "leftovers": 4.609386333592405
   },
   "natural": {
    "kept": 67.90778811934014,
    "missed": 10.243062187739165,
    "holding": 9.493186845716842,
    "ordering": 5.736122479516883,
    "waiting": 2.9509326292764233,
    "leftovers": 3.6689077384105477
   },
   "seasons": 480,
   "loss_seasons": 10,
   "worst_pct": -80.09052178534841,
   "usd_per_season": 3.972851463749968
  },
  "opus-tools": {
   "hard": {
    "kept": 57.60983351780481,
    "missed": 10.27715649507594,
    "holding": 12.855761508646435,
    "ordering": 8.12989521841486,
    "waiting": 3.0893779133327093,
    "leftovers": 8.037975346725247
   },
   "natural": {
    "kept": 62.8150795633577,
    "missed": 9.033212874995417,
    "holding": 12.101971506269274,
    "ordering": 7.171710620788851,
    "waiting": 2.6693084310568524,
    "leftovers": 6.208717003531909
   },
   "seasons": 480,
   "loss_seasons": 15,
   "worst_pct": -87.65496686670409,
   "usd_per_season": 3.5244604416666823
  },
  "gemini-tools": {
   "hard": {
    "kept": 58.27691251237551,
    "missed": 10.317987396160504,
    "holding": 13.463075618265098,
    "ordering": 4.685462710212266,
    "waiting": 2.9808266558027587,
    "leftovers": 10.275735107183856
   },
   "natural": {
    "kept": 63.347895872855595,
    "missed": 9.03428541708012,
    "holding": 12.798097720055832,
    "ordering": 3.8704765800003402,
    "waiting": 2.5265354613283084,
    "leftovers": 8.422708948679816
   },
   "seasons": 480,
   "loss_seasons": 15,
   "worst_pct": -265.7952389107475,
   "usd_per_season": 6.6192396362499615
  }
 },
 "end_of_season_adjustment_components_trial0": {
  "note": "accounting differences between the continuation buyer's 26-week tail from the run's end position and from a reference shelf, in points of demand margin, plus the residual the tail records do not itemize; not a causal split into overbuying and underbuying",
  "by_run": {
   "sol-tools": {
    "hard": {
     "leftovers": 4.609386333592405,
     "season write-down": 0.0,
     "tail: excess carrying + sell-down": 0.8952336739051604,
     "tail: lost sales + waiting": 2.301850804577557,
     "tail: ordering + freight + fees": 1.0885877978360299,
     "tail: not itemized": 0.3237140572736581
    },
    "natural": {
     "leftovers": 3.6689077384105477,
     "season write-down": 0.0,
     "tail: excess carrying + sell-down": 0.7388891113337336,
     "tail: lost sales + waiting": 1.728417944982224,
     "tail: ordering + freight + fees": 0.9945147773720925,
     "tail: not itemized": 0.20708590472249727
    }
   },
   "opus-tools": {
    "hard": {
     "leftovers": 8.037975346725247,
     "season write-down": 0.0,
     "tail: excess carrying + sell-down": 4.295384226972143,
     "tail: lost sales + waiting": 1.9676228298278686,
     "tail: ordering + freight + fees": 1.2699921843665074,
     "tail: not itemized": 0.5049761055587267
    },
    "natural": {
     "leftovers": 6.208717003531909,
     "season write-down": 0.0,
     "tail: excess carrying + sell-down": 3.3601921925508083,
     "tail: lost sales + waiting": 1.5793939762140194,
     "tail: ordering + freight + fees": 1.0914145971638918,
     "tail: not itemized": 0.17771623760318936
    }
   },
   "gemini-tools": {
    "hard": {
     "leftovers": 10.275735107183856,
     "season write-down": 0.0,
     "tail: excess carrying + sell-down": 9.530329198134025,
     "tail: lost sales + waiting": 1.150914372737105,
     "tail: ordering + freight + fees": 0.5705207330395304,
     "tail: not itemized": -0.9760291967268057
    },
    "natural": {
     "leftovers": 8.422708948679816,
     "season write-down": 0.0,
     "tail: excess carrying + sell-down": 7.891614232591947,
     "tail: lost sales + waiting": 0.9485598820929932,
     "tail: ordering + freight + fees": 0.5182922100040296,
     "tail: not itemized": -0.9357573760091554
    }
   }
  }
 }
}