win-speed bonus: record + reward how fast a model wins

Per user: a faster win is worth a bonus and must be recorded/compared.
- scoring.py: _win_budget (tightest within_ticks in the win tree, else
max_ticks); ScoreCard gains win_tick/win_turns/win_budget/speed/
composite_base (in score.json + manifest). On a WIN, composite +=
SPEED_BONUS*speed (0.05 cap, speed=1-win_tick/budget) — orders fast
vs slow wins, never lifts a loss above a win or overrides correctness.
- run_eval._agg: win_speed_mean / win_turns_mean (over wins only).
- leaderboard: records win_speed/win_turns; composite/win_rate ties
now broken by faster wins.
- SCENARIO_QUALITY: #2 closer-look note + win-speed section.
+5 scoring/budget tests; suite 362 passed.

SCENARIO_QUALITY.md

@@ -430,3 +430,51 @@ failure attributes to a single capability):
 default `exact`) is a reusable knob: any pack can opt into
 coordinate-blind objectives. The win predicate always evaluates
 against the real engine coordinates regardless of disclosure.
+
+## Per-scenario closer look — #2 action-sequenced-execution
+
+Original defect: the pack claimed to test ordered, non-stalling route
+execution but the win condition referenced only the *final* region +
+a tick band — a beeline-to-final that skipped every waypoint and
+idled to the gate won (verified: model arrived tick 1353, idled ~900
+ticks, won). Not model compensation: the scenario didn't enforce what
+it advertised.
+
+Fix — new reusable **stateful** predicate `waypoint_sequence`: latches
+ordered region-visit progress (monotonic) on a per-episode
+`signals.seq_progress` scratch. Wk+1 only counts after Wk; skip /
+out-of-order / idle ⇒ never satisfied. Coords-aware phrase
+(`exact|relative`). Difficulty axis (one new controlled variable per
+tier):
+
+- **easy** — ONE ordered 3-waypoint route, clear lanes, generous
+  budget. Run: WIN, W1→W2→W3 in order, tick 1893 < 2400.
+- **medium** — TWO ordered routes that *cross*, BOTH required ⇒ must
+  split into two parallel columns (serial overruns); attrition cap.
+  Run: LOSS (valid) — model split correctly, finished route S in
+  order, lost column discipline on N and timed out (the exact
+  re-deliberation failure the pack targets).
+- **hard** — TWO 4-waypoint serpentine routes on the larger
+  `scout-arena` (176×80, `scripts/build_scout_arena_map.py`), given
+  ONLY by relative *search bands* (no coords); fogged markers revealed
+  by the `enemy_building_spotted` interrupt; seed-varied 2 spawn
+  points. Run: LOSS (valid) — machinery all verified working
+  (interrupts fired on the big map); model drove to literal corners
+  instead of scanning the bands, ignored the revealed positions,
+  attrition-bust. Solvable in principle; not compensated.
+
+All tick budgets aligned to ~90 ticks/turn so timeout / attrition /
+wipe are reachable **losses** within `max_turns` (no draw degeneracy).
+Footgun recorded: a `base_map` placed at the Level level is silently
+ignored — it must go inside `overrides:` (guard test added).
+
+## Win-speed bonus (cross-cutting scoring)
+
+Every episode records `win_tick` / `win_turns` / `win_budget` /
+`speed` / `composite_base` (score.json + manifest). On a **win** the
+composite gets `+SPEED_BONUS·speed` (`SPEED_BONUS=0.05`,
+`speed = 1 − win_tick/budget`, budget = tightest `within_ticks` in the
+win tree else `max_ticks`). Bounded so a fast win ranks above a slow
+win but never lifts a loss above a win or overrides correctness.
+Leaderboard carries `win_speed`/`win_turns` and breaks composite/
+win-rate ties by faster wins.

eval_stats.json

@@ -1,25 +1,25 @@
 {
-  "run_id": "20260519-155206",
+  "run_id": "20260519-162229",
   "model": "qwen/qwen3.6-flash",
   "truncated": false,
   "resumed": 0,
   "cost": {
-    "calls": 34,
-    "prompt_tokens": 224195,
-    "completion_tokens": 34787,
+    "calls": 41,
+    "prompt_tokens": 332060,
+    "completion_tokens": 43651,
     "usd": 0.0,
     "max_usd": 0.0
   },
   "summary": {
-    "action-sequenced-execution:medium": {
+    "action-sequenced-execution:hard": {
       "n": 1,
       "win_rate": 0.0,
-      "composite_mean": 0.3749,
+      "composite_mean": 0.1773,
       "composite_std": 0.0,
-      "perception_mean": 0.868,
-      "reasoning_mean": 0.8663,
+      "perception_mean": 0.6844,
+      "reasoning_mean": 0.6737,
       "action_mean": 1.0,
-      "objective_mean": 0.7449,
+      "objective_mean": 0.375,
       "weakest_link_hist": {
         "reasoning": 1
       }
@@ -28,44 +28,44 @@
   "overall": {
     "n": 1,
     "win_rate": 0.0,
-    "composite_mean": 0.3749,
+    "composite_mean": 0.1773,
     "composite_std": 0.0,
-    "perception_mean": 0.868,
-    "reasoning_mean": 0.8663,
+    "perception_mean": 0.6844,
+    "reasoning_mean": 0.6737,
     "action_mean": 1.0,
-    "objective_mean": 0.7449,
+    "objective_mean": 0.375,
     "weakest_link_hist": {
       "reasoning": 1
     }
   },
   "reward_vector_mean": {
     "economy": 0.5,
-    "military": 0.4,
-    "territory": 0.8114,
-    "scouting": 0.7,
-    "objective": 0.7449
+    "military": 0.0,
+    "territory": 0.5491,
+    "scouting": 0.6,
+    "objective": 0.375
   },
   "episodes": [
     {
-      "cell": "action-sequenced-execution:medium",
+      "cell": "action-sequenced-execution:hard",
       "capability": "action",
       "split": "public",
       "seed": 1,
       "outcome": "loss",
-      "composite": 0.3749,
-      "perception": 0.868,
-      "reasoning": 0.8663,
+      "composite": 0.1773,
+      "perception": 0.6844,
+      "reasoning": 0.6737,
       "action": 1.0,
       "weakest_link": "reasoning",
-      "objective_progress": 0.7449,
+      "objective_progress": 0.375,
       "reward_vector": {
         "economy": 0.5,
-        "military": 0.4,
-        "territory": 0.8114,
-        "scouting": 0.7,
-        "objective": 0.7449
+        "military": 0.0,
+        "territory": 0.5491,
+        "scouting": 0.6,
+        "objective": 0.375
       },
-      "turns": 34,
+      "turns": 41,
       "notes": [
         "objective not met (loss); weakest link: reasoning"
       ]

openra_bench/leaderboard.py

@@ -68,6 +68,9 @@ def ingest_run(
         # Continuous goal achievement (partial credit) + the cumulative
         # reward-vector signature, comparable across models/runs.
         "objective": overall.get("objective_mean", 0.0),
+        # Win speed (mean over wins): how decisively the model wins.
+        "win_speed": overall.get("win_speed_mean", 0.0),
+        "win_turns": overall.get("win_turns_mean", 0.0),
         "reward_vector": stats.get("reward_vector_mean", {}),
         # Adversarial 1v1 spotlight: mean ladder rating (0–3) + per-pack.
         "adversarial_rating": stats.get("adversarial", {}).get(
@@ -125,7 +128,11 @@ def build_table(
             best[m] = r
     rows = sorted(
         best.values(),
-        key=lambda r: (-r["composite"], -r["win_rate"], r["model"]),
+        key=lambda r: (
+            -r["composite"], -r["win_rate"],
+            -r.get("win_speed", 0.0),  # faster wins break ties
+            r["model"],
+        ),
     )
     for i, r in enumerate(rows, 1):
         r["rank"] = i

openra_bench/run_eval.py

@@ -85,6 +85,16 @@ def _agg(scores: list) -> dict:
         "objective_mean": round(
             statistics.fmean(s.dimensions.get("objective", 0.0) for s in scores), 4
         ),
+        # Win-speed: averaged over WINS only (0 when there are none) so
+        # it compares how decisively a model wins, not diluted by losses.
+        "win_speed_mean": round(
+            statistics.fmean([s.speed for s in scores if s.outcome == "win"]), 4
+        ) if any(s.outcome == "win" for s in scores) else 0.0,
+        "win_turns_mean": round(
+            statistics.fmean(
+                [s.win_turns for s in scores if s.outcome == "win"]
+            ), 2
+        ) if any(s.outcome == "win" for s in scores) else 0.0,
         "weakest_link_hist": dict(Counter(s.weakest_link for s in scores)),
     }
 

openra_bench/scoring.py

@@ -43,7 +43,7 @@ def _clamp(x: float, lo: float = 0.0, hi: float = 1.0) -> float:
 
 @dataclass
 class ScoreCard:
-    composite: float            # weighted scalar in [0,1]
+    composite: float            # weighted scalar in [0,1] (incl. speed bonus)
     outcome: str                # win | draw | loss
     perception: float           # P/R/A link sub-scores, each in [0,1]
     reasoning: float
@@ -52,11 +52,52 @@ class ScoreCard:
     weights: dict               # weights actually used (scenario or default)
     weakest_link: str           # "perception" | "reasoning" | "action"
     notes: list
+    # Win-speed bonus (recorded for every episode; only non-zero on a
+    # win). speed ∈ [0,1] = 1 − win_tick/budget (faster ⇒ higher); the
+    # composite gets at most SPEED_BONUS·speed added — enough to rank
+    # fast wins above slow wins, never enough to lift a loss above a
+    # win or override correctness.
+    win_tick: int = 0           # game tick the win fired (0 if not won)
+    win_turns: int = 0          # decision turns to the win (0 if not won)
+    win_budget: int = 0         # the tick budget judged against
+    speed: float = 0.0          # [0,1], 0 unless won
+    composite_base: float = 0.0  # composite before the speed bonus
 
     def to_dict(self) -> dict:
         return asdict(self)
 
 
+# Max additive speed bonus on the composite (wins only). Small by
+# design: orders fast vs slow wins without dominating correctness.
+SPEED_BONUS = 0.05
+
+
+def _win_budget(compiled: CompiledLevel) -> int:
+    """Tick budget a win is judged 'fast' against: the tightest
+    `within_ticks` in the win tree, else the scenario max_ticks."""
+    best: list[int] = []
+
+    def walk(node):
+        if node is None:
+            return
+        d = node if isinstance(node, dict) else dict(
+            getattr(node, "__pydantic_extra__", {}) or {}
+        )
+        for k, v in d.items():
+            if k in ("all_of", "any_of"):
+                for c in v:
+                    walk(c)
+            elif k == "not":
+                walk(v)
+            elif k == "within_ticks":
+                best.append(int(v))
+
+    walk(compiled.win_condition)
+    if best:
+        return max(1, min(best))
+    return max(1, compiled.scenario.termination.max_ticks)
+
+
 def _dimension_values(compiled: CompiledLevel, res: EpisodeResult) -> dict:
     """Map adapter signals -> Training reward dimensions, each in [0,1].
 
@@ -172,11 +213,23 @@ def _pra_diagnostics(compiled: CompiledLevel, res: EpisodeResult, dims: dict) ->
 def score_episode(compiled: CompiledLevel, res: EpisodeResult) -> ScoreCard:
     dims = _dimension_values(compiled, res)
     weights = _weights(compiled)
-    composite = _composite(dims, weights)
+    composite_base = _composite(dims, weights)
     pra = _pra_diagnostics(compiled, res, dims)
 
+    won = res.outcome == "win"
+    win_tick = int(res.signals.game_tick) if won else 0
+    win_turns = int(res.turns) if won else 0
+    budget = _win_budget(compiled)
+    speed = _clamp(1.0 - win_tick / budget) if won and win_tick > 0 else 0.0
+    composite = _clamp(composite_base + SPEED_BONUS * speed) if won else composite_base
+
     weakest = min(pra, key=pra.get)
     notes: list[str] = []
+    if won:
+        notes.append(
+            f"won in {win_turns} turns / tick {win_tick} of {budget} "
+            f"(speed {speed:.2f}, +{SPEED_BONUS * speed:.3f} bonus)"
+        )
     if res.outcome != "win":
         notes.append(f"objective not met ({res.outcome}); weakest link: {weakest}")
     if res.actions_issued and res.actions_warned / res.actions_issued > 0.25:
@@ -197,4 +250,9 @@ def score_episode(compiled: CompiledLevel, res: EpisodeResult) -> ScoreCard:
         weights=weights,
         weakest_link=weakest,
         notes=notes,
+        win_tick=win_tick,
+        win_turns=win_turns,
+        win_budget=budget,
+        speed=round(speed, 4),
+        composite_base=round(composite_base, 4),
     )

playback/run1/20260519-162229__qwen_qwen3.6-flash/_journal.jsonl

@@ -0,0 +1 @@
+{"cell": "action-sequenced-execution:hard", "capability": "action", "split": "public", "seed": 1, "outcome": "loss", "composite": 0.1773, "perception": 0.6844, "reasoning": 0.6737, "action": 1.0, "weakest_link": "reasoning", "objective_progress": 0.375, "reward_vector": {"economy": 0.5, "military": 0.0, "territory": 0.5491, "scouting": 0.6, "objective": 0.375}, "turns": 41, "notes": ["objective not met (loss); weakest link: reasoning"], "_key": "action-sequenced-execution|hard|public|1"}

playback/run1/20260519-162229__qwen_qwen3.6-flash/action-sequenced-execution:hard:public/seed1/manifest.json

@@ -0,0 +1,33 @@
+{
+  "scenario": "action-sequenced-execution:hard",
+  "pack_id": "action-sequenced-execution",
+  "level": "hard",
+  "capability": "action",
+  "run_id": "20260519-162229",
+  "model": "qwen/qwen3.6-flash",
+  "seed": 1,
+  "outcome": "loss",
+  "turns": 41,
+  "max_turns": 70,
+  "actions_issued": 78,
+  "actions_warned": 0,
+  "agent_stats": {
+    "turns": 41,
+    "tool_calls": 78,
+    "empty_replies": 0
+  },
+  "objective_progress": 0.375,
+  "reward_vector": {
+    "economy": 0.5,
+    "military": 0.0,
+    "territory": 0.5491,
+    "scouting": 0.6,
+    "objective": 0.375
+  },
+  "signals": {
+    "economy_value": 5000,
+    "explored_percent": 54.91,
+    "units_killed": 0,
+    "units_lost": 2
+  }
+}