fix(codex): store command output whole in result_summary (+ code-review fixes)

The Codex agent truncated command output with `output[:100]` when building
CommandTelemetry.result_summary, so result_tokens (derived from its length)
under-reported every Bash result and skewed the cost model. Store the output
whole and add lint rule CE043 to forbid re-introducing output truncation in
agents/.

Code-review fixes applied on top:
- streaming/renderers.py: cap the Rich live-feed ToolEnd preview at
  _MAX_RESULT_LEN so a now-whole result_summary can't flood the console; the
  reported char count still shows the true full length (+ regression test).
- models/telemetry.py: narrow the result_tokens docstring — the untruncated
  contract covers captured command stdout/stderr, not the intentionally-brief
  one-line summaries of non-command tool items (_summarize_tool_item).
- experiments/default.yaml: clarify the max_turns:100 comment as a hard safety
  ceiling above the typical 3-18 range (task_timeout/turn_timeout are the
  practical guards).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: joeysbase

experiments/default.yaml

@@ -18,9 +18,11 @@ defaults:
   # Per-row dataset multiplication: a 100-row task with max_usd: 0.10 permits up to
   # $10 cumulative spend.
   run_limits:
-    # Maximum agent inner-loop turns per iteration (null = SDK default).
-    # Typical range in tasks: 3-18 depending on complexity.
-    max_turns: 20
+    # Hard safety ceiling on agent inner-loop turns per iteration (null = SDK
+    # default). Set well above the typical 3-18 turns a task needs so it only
+    # trips on runaway loops; task_timeout / turn_timeout below are the practical
+    # guards. Override per-task when a task legitimately needs more.
+    max_turns: 100
     # Soft target: when cumulative SDK turns across all iterations of a task
     # exceed this, the orchestrator logs a one-shot warning and the report
     # surfaces a badge. Does NOT abort the run (max_turns is the hard cap).

src/coder_eval/agents/codex_agent.py

@@ -2178,10 +2178,15 @@ def _extract_command_telemetry(self, command_item: Any, sequence: int) -> Comman
             # Determine result status from exit code
             result_status = "success" if exit_code == 0 else "error" if exit_code is not None else "unknown"
 
-            # Build result summary with output if available
+            # Build result summary with output if available. Store the output WHOLE:
+            # CommandTelemetry.result_summary is the untruncated tool-result body (its
+            # length drives CommandTelemetry.result_tokens), so truncating here would
+            # under-report tool-output size for every command (see CE043). The output is
+            # already bounded by the Codex harness's own exec-output truncation; any
+            # further trimming for display belongs in the renderers/reports, not capture.
             summary_parts = [f"Exit code: {exit_code}" if exit_code is not None else "Command executed"]
             if output and len(output.strip()) > 0:
-                summary_parts.append(f"Output: {output[:100]}")
+                summary_parts.append(f"Output: {output}")
             result_summary = " | ".join(summary_parts)
 
             # Try to parse output as JSON

src/coder_eval/models/telemetry.py

@@ -410,6 +410,16 @@ def result_tokens(self) -> int:
         result size from prompt-cache growth (which is unavailable when caching is
         disabled). Approximate, not the API's exact tokenizer count, but
         deterministic and always present. 0 when the tool returned no content.
+
+        This measure is only meaningful while ``result_summary`` stays whole: an
+        agent that truncates a command's output before recording it (as the Codex
+        agent once did with ``output[:100]``) silently under-reports that command's
+        result. Lint rule CE043 forbids truncating captured command output
+        (stdout/stderr) in the agents, so the "untruncated" contract holds for
+        command results across agents. (One-line summaries of non-command tool
+        items — e.g. collab/MCP status lines built by ``_summarize_tool_item`` —
+        are intentionally brief and out of scope.) Trim for DISPLAY in the
+        renderers/reports instead.
         """
         if not self.result_summary:
             return 0

src/coder_eval/streaming/renderers.py

@@ -104,14 +104,18 @@ def _format_event(self, event: StreamEvent) -> str | None:
             return f"[cyan]>>> TOOL: {escape(event.tool.tool_name)}[/cyan] | {params_str}"
 
         if isinstance(event, ToolEndEvent):
-            preview = escape(event.tool.result_summary or "")
+            # result_summary is stored WHOLE (untruncated) at capture; cap it here
+            # for the terse live feed so a large command output doesn't flood the
+            # console. The reported char count reflects the true (full) length.
+            full = event.tool.result_summary or ""
+            preview = escape(_truncate(full, _MAX_RESULT_LEN))
             if event.status == ToolEndStatus.OK:
                 tag = "[green]<<< OK[/green]"
             elif event.status == ToolEndStatus.UNRESOLVED:
                 tag = "[yellow]<<< UNRESOLVED:[/yellow]"
             else:
                 tag = f"[red]<<< {escape(event.status.value.upper())}:[/red]"
-            return f"{tag} ({len(preview)} chars) {preview}"
+            return f"{tag} ({len(full)} chars) {preview}"
 
         if isinstance(event, TextChunkEvent):
             return f"[dim]{escape(event.text)}[/dim]"

tests/lint/rules/ce043_no_command_output_truncation.py

@@ -0,0 +1,75 @@
+"""CE043: Agents must not truncate a command's output when recording it.
+
+``CommandTelemetry.result_summary`` is contractually the *untruncated* tool-result
+body, and its length drives the ``result_tokens`` computed field (the cost
+simulator's cache-independent measure of tool-output size). An agent that clips a
+command's captured output before storing it silently under-reports every tool
+result — exactly the bug the Codex agent shipped with (``f"Output: {output[:100]}"``),
+which pinned ~77% of its Bash results at ~31 tokens and skewed the cost model.
+
+This rule flags, inside ``src/coder_eval/agents/``, a constant-upper-bound slice
+(``x[:N]``) applied to a value that denotes captured command output:
+
+  * a name whose id is/ends with ``output`` / ``stdout`` / ``stderr``
+    (e.g. ``output[:100]``, ``aggregated_output[:512]``, ``proc_stdout[:80]``)
+  * an attribute access ``.aggregated_output`` / ``.output`` / ``.stdout`` / ``.stderr``
+    (e.g. ``command_item.aggregated_output[:100]``)
+
+Store the output whole (it is already bounded by the harness's own exec-output
+truncation) and trim for DISPLAY in the renderers/reports instead.
+
+Add ``# noqa: CE043`` on the offending line for a genuinely non-recorded use
+(e.g. slicing stdout only to build a short crash/log message that never becomes a
+``result_summary``), with a comment explaining why.
+"""
+
+import ast
+
+from tests.lint.rules.base import BaseRule
+
+
+_OUTPUT_NAMES = {"output", "stdout", "stderr", "aggregated_output"}
+
+
+def _denotes_output(value: ast.AST) -> str | None:
+    """Return the output-ish identifier being sliced, or None."""
+    if isinstance(value, ast.Name):
+        low = value.id.lower()
+        if low in _OUTPUT_NAMES or low.endswith("_output") or low.endswith("_stdout") or low.endswith("_stderr"):
+            return value.id
+    elif isinstance(value, ast.Attribute):
+        if value.attr.lower() in _OUTPUT_NAMES:
+            return value.attr
+    return None
+
+
+def _is_const_upper_slice(sl: ast.AST) -> bool:
+    """True for ``[:N]`` / ``[:N:...]`` with a constant int upper bound."""
+    return (
+        isinstance(sl, ast.Slice)
+        and sl.lower is None
+        and isinstance(sl.upper, ast.Constant)
+        and isinstance(sl.upper.value, int)
+    )
+
+
+class NoCommandOutputTruncation(BaseRule):
+    id = "CE043"
+
+    def check(self, tree: ast.AST) -> list:
+        # Scope to the agent implementations — only they record CommandTelemetry.
+        if "/agents/" not in self.filepath.replace("\\", "/"):
+            return []
+        self.visit(tree)
+        return self.violations
+
+    def visit_Subscript(self, node: ast.Subscript) -> None:
+        name = _denotes_output(node.value)
+        if name is not None and _is_const_upper_slice(node.slice):
+            self.violation(
+                node,
+                f"Do not truncate command output ({name}[:...]); CommandTelemetry.result_summary "
+                "must stay whole (it drives result_tokens). Store the full output and trim for "
+                "display in the renderers/reports instead.",
+            )
+        self.generic_visit(node)

tests/lint/runner.py

@@ -21,6 +21,7 @@
 from tests.lint.rules.ce023_no_proxy_shim_import import NoProxyShimImports
 from tests.lint.rules.ce024_discriminated_unions import DiscriminatedUnions
 from tests.lint.rules.ce032_criteria_path_seam import CriteriaPathSeam
+from tests.lint.rules.ce043_no_command_output_truncation import NoCommandOutputTruncation
 from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess
 from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync
 from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore
@@ -65,6 +66,7 @@
     NoProxyShimImports,
     DiscriminatedUnions,
     CriteriaPathSeam,
+    NoCommandOutputTruncation,
 ]
 
 # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule

tests/test_codex_agent_unit.py

@@ -104,3 +104,28 @@ def test_command_dispatch_mutates_lists_in_place(self):
         # A tool_use block was recorded into the open buffer (cut at the next
         # tokenUsage flush, not here), joinable to the command by tool_id.
         assert any(b.block_type == "tool_use" and b.tool_use_id == "c1" for b in state.open_blocks)
+
+    def test_command_output_recorded_whole_not_truncated(self):
+        # Regression for the Codex `output[:100]` bug (CE043): result_summary must
+        # carry the FULL command output so result_tokens reflects real tool-output
+        # size instead of being pinned at a ~31-token, 100-char cap.
+        agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, model="gpt-5-codex"))
+        state, commands, _messages = self._state(agent)
+
+        big_output = "X" * 4000  # far beyond the old 100-char clip
+        cmd_root = SimpleNamespace(
+            type="commandExecution",
+            id="c2",
+            command="cat big.txt",
+            exit_code=0,
+            aggregated_output=big_output,
+            duration_ms=5,
+        )
+        state.on_item_started(_item_notification("item/started", cmd_root))
+        state.on_item_completed(_item_notification("item/completed", cmd_root))
+
+        cmd = commands[0]
+        assert big_output in (cmd.result_summary or ""), "full output must be recorded, not truncated"
+        # result_tokens (ceil(len/4)) must scale with the real output, not ~31.
+        assert cmd.result_tokens >= len(big_output) // 4
+        assert cmd.result_tokens > 100

tests/test_custom_lint.py

@@ -60,6 +60,48 @@ def test_ignores_input_tokens_on_other_models(self):
         assert not self._run("ReconciliationMessage(input_tokens=-5)")
 
 
+@pytest.mark.lint
+class TestCE043NoCommandOutputTruncation:
+    """CE043 flags truncation of captured command output inside agents/, only."""
+
+    @staticmethod
+    def _run(src: str, *, in_agents: bool = True):
+        import ast
+
+        from tests.lint.rules.ce043_no_command_output_truncation import NoCommandOutputTruncation
+
+        path = "src/coder_eval/agents/codex_agent.py" if in_agents else "src/coder_eval/reports_html.py"
+        return NoCommandOutputTruncation(path).check(ast.parse(src))
+
+    @pytest.mark.parametrize(
+        "expr",
+        [
+            "output[:100]",
+            "aggregated_output[:512]",
+            "command_item.aggregated_output[:100]",
+            "proc_stdout[:80]",
+            "result.stderr[:200]",
+            'f"Output: {output[:100]}"',
+        ],
+    )
+    def test_flags_output_truncation_in_agents(self, expr: str):
+        assert self._run(f"x = {expr}"), f"expected CE043 to flag {expr!r}"
+
+    def test_allows_untruncated_output(self):
+        assert not self._run('summary = f"Output: {output}"')
+        assert not self._run("summary = aggregated_output")
+
+    def test_ignores_non_output_slices(self):
+        # Legit error/orchestration summaries that are NOT captured command output.
+        assert not self._run("detail = summary.result[:200]")
+        assert not self._run("msg = content_str[:200]")
+        assert not self._run("s = '; '.join(messages)[:200]")
+
+    def test_scoped_to_agents_only(self):
+        # The same pattern outside agents/ is not this rule's concern (display code).
+        assert not self._run("preview = output[:100]", in_agents=False)
+
+
 @pytest.mark.lint
 class TestCE017ModelsLazyAgentImports:
     """CE017 flags only module-level agents/plugins imports inside models/."""

tests/test_streaming_renderers.py

@@ -78,6 +78,28 @@ def test_tool_end_renders_success():
     assert "OK" in output or "ok" in output.lower()
 
 
+def test_tool_end_caps_long_result_summary_but_reports_full_length():
+    """A large (untruncated-at-capture) result_summary is capped for the terse
+    live feed, while the reported char count still reflects the full length."""
+    from coder_eval.streaming.renderers import _MAX_RESULT_LEN
+
+    renderer, buf = _make_renderer()
+    big = "Z" * 5000  # far beyond the display cap
+    renderer.on_event(
+        ToolEndEvent(
+            task_id="t1",
+            tool=_tool(tool_name="Bash", result_summary=big),
+            status=ToolEndStatus.OK,
+        )
+    )
+    output = buf.getvalue()
+    # true length is reported...
+    assert "(5000 chars)" in output
+    # ...but the console body is capped (not the full 5000 Z's dumped).
+    assert output.count("Z") <= _MAX_RESULT_LEN
+    assert "..." in output
+
+
 def test_tool_end_renders_error():
     """ToolEndEvent renders ERROR for failed results."""
     renderer, buf = _make_renderer()