From c98929eade1908423dcc696d5c3d76bf34d20676 Mon Sep 17 00:00:00 2001
From: Akshaya Shanbhogue <akshaya.shanbhogue@uipath.com>
Date: Thu, 17 Sep 2026 07:30:02 -0700
Subject: [PATCH 1/3] feat(harbor): unblock trajectory-dependent criteria and
 always allow credentials at export

- Flatten the run_dir layout for --workspace-dir (Harbor single-task) runs so
  task.json/task.html/task.log/artifacts live directly under run_dir, next to
  the trajectory.json emit_trajectories_for_run already writes as task.json's
  sibling -- matching CoderEvalAgent's own run-directory shape.
- Unblock NEEDS_TRAJECTORY criteria (command_executed/commands_efficiency/
  skill_triggered) at export time: the verifier's tests/test.sh now grades via
  `coder-eval evaluate /tests/task.yaml /logs/agent --run-dir /logs/verifier`,
  hydrating the agent phase's native task.json trajectory directly instead of
  an ATIF round-trip, and without touching the untrusted-recorded-config gate.
- Remove --allow-credentials entirely: NEEDS_CREDENTIALS criteria (llm_judge/
  agent_judge/uipath_eval) now always export, on the same assumption the flag
  encoded -- the operator provisions model access in the verifier themselves.
- Extend the Harbor E2E CI script with a trajectory_criteria scenario that
  checks the criterion's own score, not just aggregate reward, so a silently
  ungraded trajectory can't hide behind an unrelated criterion passing.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/scripts/harbor_e2e.py                 | 35 ++++++++++--
 src/coder_eval/cli/export_command.py          | 12 +---
 src/coder_eval/harbor/experiment_packager.py  |  3 -
 src/coder_eval/harbor/packager.py             | 56 ++++++++++---------
 src/coder_eval/harbor/portability.py          | 39 +++++++------
 src/coder_eval/orchestration/batch.py         | 13 ++++-
 .../expected/tests/test.sh                    | 23 +++++---
 .../fixtures/trajectory_criteria.yaml         | 32 +++++++++++
 tests/test_harbor_packager.py                 | 48 +++++++++++-----
 tests/test_harbor_portability.py              | 31 +++++-----
 10 files changed, 192 insertions(+), 100 deletions(-)
 create mode 100644 tests/harbor_e2e/fixtures/trajectory_criteria.yaml

diff --git a/.github/scripts/harbor_e2e.py b/.github/scripts/harbor_e2e.py
index 2dd436a2..908606ae 100644
--- a/.github/scripts/harbor_e2e.py
+++ b/.github/scripts/harbor_e2e.py
@@ -36,7 +36,6 @@ class Scenario:
 
     name: str
     task_file: Path
-    allow_credentials: bool = False
 
 
 SCENARIOS: list[Scenario] = [
@@ -45,15 +44,27 @@ class Scenario:
     # export -> CoderEvalAgent -> --workspace-dir -> verifier round trip.
     Scenario("baseline", REPO_ROOT / "tests/harbor_e2e/fixtures/docker_baseline.yaml"),
     # llm_judge: real model call inside the VERIFIER phase, not just the agent.
-    Scenario("llm_judge", REPO_ROOT / "tests/harbor_e2e/fixtures/llm_judge.yaml", allow_credentials=True),
+    Scenario("llm_judge", REPO_ROOT / "tests/harbor_e2e/fixtures/llm_judge.yaml"),
     # Custom (BYOD) Docker image via dockerfile_path -- reuses the in-tree
     # byod_smoke_test task/image rather than duplicating it.
     Scenario("docker_custom_image", REPO_ROOT / "tasks/byod_smoke_test.yaml"),
     # template_sources: TemplateDirSource copy-in + rewritten path, plus
     # sandbox.python.env_packages surviving the agent-phase task.yaml merge.
     Scenario("template_sources", REPO_ROOT / "tests/harbor_e2e/fixtures/template_sources.yaml"),
+    # command_executed: catches a regression where the verifier phase grades
+    # against a directory with no trajectory data -- reward could still land on
+    # 1.0 "by luck" from unrelated criteria while this one silently scores 0.0,
+    # so `assert_scenario_artifacts` checks its own criterion score directly
+    # rather than trusting the aggregate reward alone.
+    Scenario("trajectory_criteria", REPO_ROOT / "tests/harbor_e2e/fixtures/trajectory_criteria.yaml"),
 ]
 
+# Criteria types that can only score correctly if the verifier phase actually
+# hydrated the agent phase's trajectory (see portability.py's NEEDS_TRAJECTORY
+# class). Checked by name per scenario below rather than globally, since only
+# `trajectory_criteria` declares one.
+_TRAJECTORY_CRITERION_TYPES = frozenset({"command_executed", "commands_efficiency", "skill_triggered"})
+
 
 def _run(cmd: list[str]) -> subprocess.CompletedProcess[str]:
     print(f"+ {' '.join(cmd)}", flush=True)
@@ -64,8 +75,6 @@ def export_task(scenario: Scenario, out_dir: Path) -> None:
     if out_dir.exists():
         shutil.rmtree(out_dir)
     cmd = ["coder-eval", "export", str(scenario.task_file), "-o", str(out_dir)]
-    if scenario.allow_credentials:
-        cmd.append("--allow-credentials")
     result = _run(cmd)
     print(result.stdout)
     print(result.stderr, file=sys.stderr)
@@ -128,9 +137,27 @@ def assert_scenario_artifacts(scenario: Scenario, trial_dir: Path) -> None:
     if not verifier_task_json.is_file():
         raise RuntimeError(f"[{scenario.name}] missing {verifier_task_json}")
 
+    # An overall reward of 1.0 does not prove a trajectory criterion was
+    # actually graded -- it could pass "by luck" from unrelated criteria while
+    # this one silently scored 0.0 against an ungraded/empty trajectory. Check
+    # each trajectory-dependent criterion's OWN score directly.
+    verifier_result = json.loads(verifier_task_json.read_text(encoding="utf-8"))
+    trajectory_results = [
+        r
+        for r in verifier_result.get("success_criteria_results", [])
+        if r.get("criterion_type") in _TRAJECTORY_CRITERION_TYPES
+    ]
+    for r in trajectory_results:
+        if r.get("score") != 1.0:
+            raise RuntimeError(
+                f"[{scenario.name}] {r.get('criterion_type')} criterion did not score 1.0 "
+                + f"(got {r.get('score')!r}); trajectory hydration likely broken: {r.get('details')!r}"
+            )
+
     print(
         f"[{scenario.name}] OK: reward=1.0, trajectory.json present, "
         + f"{len(agent_task_jsons)} agent task.json + verifier/task.json present"
+        + (f", {len(trajectory_results)} trajectory criterion/criteria verified" if trajectory_results else "")
     )
 
 
diff --git a/src/coder_eval/cli/export_command.py b/src/coder_eval/cli/export_command.py
index 5afd69de..c913aede 100644
--- a/src/coder_eval/cli/export_command.py
+++ b/src/coder_eval/cli/export_command.py
@@ -60,15 +60,6 @@ def export_command(
         "--format",
         help=f"Target format. Supported: {', '.join(_SUPPORTED_FORMATS)}.",
     ),
-    allow_credentials: bool = typer.Option(
-        False,
-        "--allow-credentials",
-        help=(
-            "Export criteria that need model credentials/network inside the verifier "
-            "(llm_judge / agent_judge / uipath_eval) anyway. Only pass this if you have "
-            "already provisioned that access yourself — the export does not do it for you."
-        ),
-    ),
 ) -> None:
     """Export a coder-eval task (or task x experiment.yaml variants) to another framework's directory format.
 
@@ -85,7 +76,7 @@ def export_command(
             console.print("[red]✗[/] Without --experiment, pass exactly one task YAML.")
             raise typer.Exit(1)
         try:
-            result = export_task(task_files[0], output_dir, allow_credentials=allow_credentials)
+            result = export_task(task_files[0], output_dir)
         except (TaskNotExportableError, CriteriaNotExportableError) as e:
             console.print(f"[red]✗[/] {e}")
             raise typer.Exit(1) from e
@@ -99,7 +90,6 @@ def export_command(
         all_task_files,
         experiment,
         output_dir,
-        allow_credentials=allow_credentials,
     )
 
     for exported in exp_result.exported:
diff --git a/src/coder_eval/harbor/experiment_packager.py b/src/coder_eval/harbor/experiment_packager.py
index 264d75ae..92aed5f3 100644
--- a/src/coder_eval/harbor/experiment_packager.py
+++ b/src/coder_eval/harbor/experiment_packager.py
@@ -119,8 +119,6 @@ def export_experiment(
     task_files: list[Path],
     experiment_file: Path,
     out_dir: Path,
-    *,
-    allow_credentials: bool = False,
 ) -> ExperimentExportResult:
     """Export every (task x variant x replicate[ x dataset row]) combination to Harbor directories.
 
@@ -183,7 +181,6 @@ def export_experiment(
                 resolved.task,
                 resolved.task_file,
                 dest,
-                allow_credentials=allow_credentials,
             )
         except (TaskNotExportableError, CriteriaNotExportableError) as e:
             skipped.append(
diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py
index b5ca72e2..d76a0002 100644
--- a/src/coder_eval/harbor/packager.py
+++ b/src/coder_eval/harbor/packager.py
@@ -61,13 +61,22 @@
 # always reach it.
 set -u
 
-# `$(pwd)` -- not a baked-in path -- so this script is agnostic of whatever
-# WORKDIR the agent's container actually used (task.toml's `environment.workdir`
-# when the task set one explicitly, or the image's own built-in WORKDIR
-# otherwise; see _write_environment). `docker exec` (or `-w`, when set) always
-# lands this shell's cwd there, so `pwd` is authoritative at run time -- no
-# export-time guess needed, and nothing to drift if the image changes later.
-coder-eval evaluate /tests/task.yaml "$(pwd)" --in-place --run-dir /logs/verifier || true
+# `/tests/task.yaml /logs/agent` -- an explicit task file over a RUN DIRECTORY,
+# not a plain workdir. CoderEvalAgent's `coder-eval execute --run-dir /logs/agent
+# ...` always finishes with `/logs/agent/task.json` (this task's own recorded
+# trajectory) and `/logs/agent/artifacts/<task_id>/` (the workspace it produced),
+# so `coder-eval evaluate` recognizes /logs/agent as a run directory and grades
+# against it directly -- no `$(pwd)` guess of the agent's WORKDIR needed (the
+# workspace is located from task.json's own recorded sandbox_path instead), and
+# no ATIF trajectory.json round-trip either (task.json already carries the same
+# trajectory natively). Passing the task file explicitly (rather than the bare
+# run directory alone) makes coder-eval grade with THIS file -- the exported
+# contract -- instead of rebuilding the task from the run's own recorded config,
+# which is also what keeps this off the untrusted-recorded-config path: that
+# path exists for a shared run directory whose config is not to be trusted
+# without --allow-recorded-commands, and does not apply once an explicit,
+# operator-supplied task file is in hand.
+coder-eval evaluate /tests/task.yaml /logs/agent --in-place --run-dir /logs/verifier || true
 coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json
 """
 
@@ -84,12 +93,7 @@ def __init__(self, issues: list[PortabilityIssue]) -> None:
         lines = "\n".join(
             f"  - {i.criterion_description!r} ({i.criterion_type}): {i.portability.value}" for i in issues
         )
-        super().__init__(
-            f"{len(issues)} criterion/criteria cannot export to Harbor v1:\n{lines}\n"
-            + "Remove them, or pass allow_credentials=True (--allow-credentials on the CLI) for the "
-            + "NEEDS_CREDENTIALS ones if you have already provisioned model access inside the verifier "
-            + "container yourself."
-        )
+        super().__init__(f"{len(issues)} criterion/criteria cannot export to Harbor v1:\n{lines}\nRemove them.")
 
 
 @dataclass(frozen=True)
@@ -104,8 +108,6 @@ class ExportResult:
 def export_task(
     task_file: Path,
     out_dir: Path,
-    *,
-    allow_credentials: bool = False,
 ) -> ExportResult:
     """Emit a Harbor task directory at ``out_dir`` from the coder-eval task at ``task_file``.
 
@@ -118,15 +120,13 @@ def export_task(
             portability audit.
     """
     task, _raw_yaml = load_task(task_file)
-    return export_resolved_task(task, task_file, out_dir, allow_credentials=allow_credentials)
+    return export_resolved_task(task, task_file, out_dir)
 
 
 def export_resolved_task(
     task: TaskDefinition,
     task_file: Path,
     out_dir: Path,
-    *,
-    allow_credentials: bool = False,
 ) -> ExportResult:
     """Emit a Harbor task directory from an already-loaded/resolved ``TaskDefinition``.
 
@@ -142,7 +142,7 @@ def export_resolved_task(
 
     Raises the same two errors as ``export_task``, for the same reasons.
     """
-    issues = audit_criteria(task.success_criteria, allow_credentials=allow_credentials)
+    issues = audit_criteria(task.success_criteria)
     if issues:
         raise CriteriaNotExportableError(issues)
 
@@ -245,8 +245,12 @@ def _write_environment(
 
     ``workdir`` is an EXPLICIT override only — ``sandbox.docker.working_dir`` or a
     Dockerfile's own ``WORKDIR`` line. ``None`` otherwise, so Harbor's ``docker exec``
-    gets no ``-w`` and lands wherever the image's OWN ``WORKDIR`` already puts it;
-    ``tests/test.sh`` resolves the real cwd itself at run time via ``$(pwd)``.
+    gets no ``-w`` and lands wherever the image's OWN ``WORKDIR`` already puts it. This
+    still matters for the AGENT phase: ``CoderEvalAgent.run()`` shells out with
+    ``--workspace-dir "$(pwd)"``, so wherever ``docker exec`` actually lands decides
+    what that captures. The VERIFIER phase no longer depends on it at all —
+    ``tests/test.sh`` grades against ``/logs/agent`` as a run directory (its own
+    recorded ``sandbox_path``), not a live cwd.
 
     ``docker_image`` is set only when no ``environment/Dockerfile`` was written, so
     ``task.toml``'s ``[environment].docker_image`` points at the pre-built image;
@@ -274,8 +278,8 @@ def _write_environment(
         shutil.copy2(source_dockerfile, dest_dockerfile)
         # No fabricated WORKDIR appended when the Dockerfile declares none --
         # the built image just inherits its base image's own default, and
-        # tests/test.sh finds it at run time via `$(pwd)` either way (see
-        # _TEST_SH_TEMPLATE and this function's docstring).
+        # CoderEvalAgent's own `--workspace-dir "$(pwd)"` (agent.py) finds it
+        # at run time either way (see this function's docstring).
         workdir = docker_cfg.working_dir or _find_workdir(dest_dockerfile)
         if not _from_line_mentions_coder_eval_agent(dest_dockerfile):
             warnings.append(_MISSING_CODER_EVAL_WARNING)
@@ -567,9 +571,9 @@ def _write_agent_phase_task_yaml(
 
 
 def _write_test_sh(out_dir: Path) -> None:
-    # No task-controlled value is interpolated into the template anymore --
-    # `$(pwd)` is a fixed literal (see _TEST_SH_TEMPLATE) -- so there is no
-    # longer an injection surface here to shlex.quote against.
+    # No task-controlled value is interpolated into the template anymore -- it
+    # is a fixed literal (see _TEST_SH_TEMPLATE) -- so there is no longer an
+    # injection surface here to shlex.quote against.
     path = out_dir / "tests" / "test.sh"
     path.write_text(_TEST_SH_TEMPLATE, encoding="utf-8")
     path.chmod(0o755)
diff --git a/src/coder_eval/harbor/portability.py b/src/coder_eval/harbor/portability.py
index db5d489d..5f93afd1 100644
--- a/src/coder_eval/harbor/portability.py
+++ b/src/coder_eval/harbor/portability.py
@@ -10,11 +10,15 @@
 - ``NEEDS_REFERENCE`` — ``reference_comparison``; never actually blocking, since the
   export always emits ``tests/reference/`` when the task declares one.
 - ``NEEDS_TRAJECTORY`` — ``command_executed``, ``commands_efficiency``,
-  ``skill_triggered``. Hard-error until ATIF ingestion lands.
+  ``skill_triggered``; never actually blocking, since the packager's ``CoderEvalAgent``
+  always runs ``coder-eval execute --format harbor`` and the generated ``tests/test.sh``
+  always grades against the resulting ``/logs/agent/trajectory.json`` (ATIF).
 - ``NEEDS_CLI_RECORDER`` — ``cli_called``. Hard-error until the export bakes the
   recorder shim in.
-- ``NEEDS_CREDENTIALS`` — ``llm_judge``, ``agent_judge``, ``uipath_eval``.
-  Hard-error, with an opt-in escape hatch for an operator who has provisioned it.
+- ``NEEDS_CREDENTIALS`` — ``llm_judge``, ``agent_judge``, ``uipath_eval``. Never
+  actually blocking — exporting one of these always assumes the operator will
+  provision model credentials/network access inside the verifier container
+  themselves; C2 does not do it for them.
 
 Rationale: .claude/notes/reporting.md § Not every criterion can grade inside someone else's container
 """
@@ -62,12 +66,13 @@ class CriterionPortability(enum.Enum):
 
 # Which non-PORTABLE classes v1 refuses to export outright (vs. tolerating
 # with a caveat, like NEEDS_REFERENCE — C2 always emits tests/reference/ when
-# task.reference is set, so that class is never actually blocking).
+# task.reference is set — NEEDS_TRAJECTORY, which C2's CoderEvalAgent +
+# generated test.sh always wire up via /logs/agent/trajectory.json, and
+# NEEDS_CREDENTIALS, which the export always assumes the operator has
+# provisioned themselves — none of these three classes is actually blocking).
 _BLOCKING_IN_V1 = frozenset(
     {
-        CriterionPortability.NEEDS_TRAJECTORY,
         CriterionPortability.NEEDS_CLI_RECORDER,
-        CriterionPortability.NEEDS_CREDENTIALS,
     }
 )
 
@@ -97,28 +102,22 @@ def classify(criterion_type: str) -> CriterionPortability:
         ) from None
 
 
-def audit_criteria(
-    criteria: list[SuccessCriterion],
-    *,
-    allow_credentials: bool = False,
-) -> list[PortabilityIssue]:
+def audit_criteria(criteria: list[SuccessCriterion]) -> list[PortabilityIssue]:
     """Return every criterion this v1 export would refuse, or ``[]`` if the task exports cleanly.
 
-    ``allow_credentials`` is the escape hatch for ``NEEDS_CREDENTIALS`` criteria
-    (``llm_judge`` / ``agent_judge`` / ``uipath_eval``) — an operator who has
-    already provisioned model credentials and network access inside the
-    verifier container may pass it to export anyway. It does not affect
-    ``NEEDS_TRAJECTORY`` or ``NEEDS_CLI_RECORDER``, which are missing
-    functionality (C1.3, a recorder-baking step in C2), not a missing
-    permission — no flag can supply what does not exist yet.
+    Only ``NEEDS_CLI_RECORDER`` (``cli_called``) actually blocks — it is missing
+    functionality (a recorder-baking step in C2 that does not exist yet), so no
+    flag can supply it. Every other non-``PORTABLE`` class exports unconditionally:
+    ``NEEDS_REFERENCE``/``NEEDS_TRAJECTORY`` because C2 always wires up the
+    supporting artifact itself, and ``NEEDS_CREDENTIALS`` because the export
+    always assumes the operator provisions model credentials/network access
+    inside the verifier container themselves.
     """
     issues: list[PortabilityIssue] = []
     for c in criteria:
         portability = classify(c.type)
         if portability not in _BLOCKING_IN_V1:
             continue
-        if portability is CriterionPortability.NEEDS_CREDENTIALS and allow_credentials:
-            continue
         issues.append(
             PortabilityIssue(criterion_description=c.description, criterion_type=c.type, portability=portability)
         )
diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py
index d00a63f2..4d49ee1a 100644
--- a/src/coder_eval/orchestration/batch.py
+++ b/src/coder_eval/orchestration/batch.py
@@ -148,7 +148,16 @@ async def run_single(rt: ResolvedTask) -> TaskResult:
         task_callback = stream_callback_factory(stream_label) if stream_callback_factory else None
         async with semaphore:
             try:
-                rt.run_dir.mkdir(parents=True, exist_ok=True)  # noqa: CE002 — mkdir on local FS is nanoseconds
+                # --workspace-dir mode (Harbor CoderEvalAgent, single task): write
+                # task.json/task.html/task.log/artifacts flat at the top-level run_dir
+                # instead of the usual <variant>/<task_id>/<NN> nesting. The guard above
+                # already guarantees exactly one resolved task here, so that nesting only
+                # exists to disambiguate sibling tasks that can never occur in this mode —
+                # and a flat run_dir means trajectory.json (written by
+                # emit_trajectories_for_run as task.json's sibling) lands at a fixed,
+                # predictable path instead of requiring a recursive glob to find it.
+                effective_run_dir = config.run_dir if config.workspace_dir is not None else rt.run_dir
+                effective_run_dir.mkdir(parents=True, exist_ok=True)  # noqa: CE002 — mkdir on local FS is nanoseconds
                 sandbox_cfg = rt.task.sandbox
                 # HERE, where the original driver is still visible: the
                 # in-container orchestrator sees it forced to tempdir.
@@ -187,7 +196,7 @@ async def run_single(rt: ResolvedTask) -> TaskResult:
                 else:
                     orchestrator = Orchestrator(
                         task=rt.task,
-                        run_dir=rt.run_dir,
+                        run_dir=effective_run_dir,
                         preservation_mode=preservation_mode,
                         task_file=rt.task_file,
                         stream_callback=task_callback,
diff --git a/tests/_fixtures/harbor_export_golden/expected/tests/test.sh b/tests/_fixtures/harbor_export_golden/expected/tests/test.sh
index 967e3159..6168226d 100755
--- a/tests/_fixtures/harbor_export_golden/expected/tests/test.sh
+++ b/tests/_fixtures/harbor_export_golden/expected/tests/test.sh
@@ -13,11 +13,20 @@
 # always reach it.
 set -u
 
-# `$(pwd)` -- not a baked-in path -- so this script is agnostic of whatever
-# WORKDIR the agent's container actually used (task.toml's `environment.workdir`
-# when the task set one explicitly, or the image's own built-in WORKDIR
-# otherwise; see _write_environment). `docker exec` (or `-w`, when set) always
-# lands this shell's cwd there, so `pwd` is authoritative at run time -- no
-# export-time guess needed, and nothing to drift if the image changes later.
-coder-eval evaluate /tests/task.yaml "$(pwd)" --in-place --run-dir /logs/verifier || true
+# `/tests/task.yaml /logs/agent` -- an explicit task file over a RUN DIRECTORY,
+# not a plain workdir. CoderEvalAgent's `coder-eval execute --run-dir /logs/agent
+# ...` always finishes with `/logs/agent/task.json` (this task's own recorded
+# trajectory) and `/logs/agent/artifacts/<task_id>/` (the workspace it produced),
+# so `coder-eval evaluate` recognizes /logs/agent as a run directory and grades
+# against it directly -- no `$(pwd)` guess of the agent's WORKDIR needed (the
+# workspace is located from task.json's own recorded sandbox_path instead), and
+# no ATIF trajectory.json round-trip either (task.json already carries the same
+# trajectory natively). Passing the task file explicitly (rather than the bare
+# run directory alone) makes coder-eval grade with THIS file -- the exported
+# contract -- instead of rebuilding the task from the run's own recorded config,
+# which is also what keeps this off the untrusted-recorded-config path: that
+# path exists for a shared run directory whose config is not to be trusted
+# without --allow-recorded-commands, and does not apply once an explicit,
+# operator-supplied task file is in hand.
+coder-eval evaluate /tests/task.yaml /logs/agent --in-place --run-dir /logs/verifier || true
 coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json
diff --git a/tests/harbor_e2e/fixtures/trajectory_criteria.yaml b/tests/harbor_e2e/fixtures/trajectory_criteria.yaml
new file mode 100644
index 00000000..eafff21f
--- /dev/null
+++ b/tests/harbor_e2e/fixtures/trajectory_criteria.yaml
@@ -0,0 +1,32 @@
+task_id: "harbor_e2e_trajectory_criteria"
+description: >
+  Harbor E2E trajectory criteria: exercises command_executed (and, by the same
+  mechanism, commands_efficiency/skill_triggered would work identically) --
+  criteria that can only score correctly if the verifier phase actually reads
+  the agent phase's recorded trajectory. Catches a regression where the
+  verifier grades against a directory with no trajectory data at all (which
+  would silently score 0.0 here rather than fail loudly).
+
+agent:
+  type: "claude-code"
+  permission_mode: "acceptEdits"
+  allowed_tools: ["Read", "Write", "Bash"]
+
+sandbox:
+  driver: docker
+  docker:
+    image: coder-eval-agent:latest
+
+initial_prompt: >
+  Create a file named done.txt containing the single word "done", then run
+  `touch done.txt` again to confirm it exists.
+
+success_criteria:
+  - type: "file_exists"
+    path: "done.txt"
+    description: "The file done.txt must be created."
+  - type: "command_executed"
+    tool_name: "Bash"
+    command_pattern: "touch\\s+done\\.txt"
+    min_count: 1
+    description: "The agent must have run touch done.txt (requires trajectory hydration to grade)."
diff --git a/tests/test_harbor_packager.py b/tests/test_harbor_packager.py
index 974170ff..e4f9d50d 100644
--- a/tests/test_harbor_packager.py
+++ b/tests/test_harbor_packager.py
@@ -60,7 +60,7 @@ def test_unsupported_criteria_are_refused_before_any_file_is_written(self, tmp_p
             tmp_path,
             {
                 "success_criteria": [
-                    {"type": "skill_triggered", "expected_skill": "s", "skill_name": "s", "description": "d"},
+                    {"type": "cli_called", "verb": "v", "description": "d"},
                 ]
             },
         )
@@ -69,12 +69,27 @@ def test_unsupported_criteria_are_refused_before_any_file_is_written(self, tmp_p
             export_task(task_file, out_dir)
         assert not out_dir.exists(), "a refused export must not leave a partial directory behind"
 
-    def test_credentials_criteria_can_be_allowed_explicitly(self, tmp_path: Path) -> None:
+    def test_trajectory_criteria_now_export_cleanly(self, tmp_path: Path) -> None:
+        """NEEDS_TRAJECTORY is no longer blocking -- test.sh always wires /logs/agent/trajectory.json."""
+        task_file = _write_task(
+            tmp_path,
+            {
+                "success_criteria": [
+                    {"type": "skill_triggered", "expected_skill": "s", "skill_name": "s", "description": "d"},
+                ]
+            },
+        )
+        result = export_task(task_file, tmp_path / "out")
+        assert result.out_dir.exists()
+
+    def test_credentials_criteria_export_cleanly(self, tmp_path: Path) -> None:
+        """NEEDS_CREDENTIALS never blocks -- the operator is assumed to provision
+        model access inside the verifier container themselves."""
         task_file = _write_task(
             tmp_path,
             {"success_criteria": [{"type": "llm_judge", "prompt": "grade it", "description": "d"}]},
         )
-        result = export_task(task_file, tmp_path / "out", allow_credentials=True)
+        result = export_task(task_file, tmp_path / "out")
         assert result.out_dir.exists()
 
 
@@ -105,10 +120,12 @@ def test_instruction_md_is_a_fixed_placeholder_not_the_real_prompt(self, tmp_pat
         emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8"))
         assert emitted["initial_prompt"] == "Write 'hello' to greeting.txt."  # the real prompt lives here instead
 
-    def test_test_sh_is_executable_and_resolves_workdir_dynamically(self, tmp_path: Path) -> None:
-        """test.sh must be workdir-agnostic: it uses `$(pwd)`, not a value baked in at
-        export time, so it works regardless of whether the task pinned a workdir or
-        left it to the image's own default (see packager.py's `_write_environment`)."""
+    def test_test_sh_is_executable_and_grades_the_run_directory(self, tmp_path: Path) -> None:
+        """test.sh must be workdir-agnostic: it grades `/logs/agent` as a run
+        directory (its own recorded sandbox_path locates the workspace), not a
+        `$(pwd)` guess baked in at export time -- so it works regardless of
+        whether the task pinned a workdir or left it to the image's own default
+        (see packager.py's `_write_environment`)."""
         task_file = _write_task(tmp_path)
         out_dir = tmp_path / "out"
 
@@ -118,7 +135,7 @@ def test_test_sh_is_executable_and_resolves_workdir_dynamically(self, tmp_path:
         if os.name != "nt":  # NTFS has no chmod executable bit
             assert test_sh.stat().st_mode & 0o111, "test.sh must be executable"
         content = test_sh.read_text(encoding="utf-8")
-        assert 'coder-eval evaluate /tests/task.yaml "$(pwd)"' in content
+        assert "coder-eval evaluate /tests/task.yaml /logs/agent --in-place --run-dir /logs/verifier" in content
         assert "coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json" in content
 
     def test_task_toml_parses_and_carries_the_mapped_fields(self, tmp_path: Path) -> None:
@@ -283,8 +300,9 @@ def test_prebuilt_image_with_no_dockerfile_path_gets_no_dockerfile_at_all(self,
 class TestDockerfileWorkdirResolution:
     def test_dockerfile_with_no_workdir_is_left_unset_and_untouched(self, tmp_path: Path) -> None:
         """No WORKDIR line is fabricated and appended anymore: the built image simply
-        inherits its base image's own default, and tests/test.sh finds the real cwd
-        at run time via `$(pwd)` regardless (see packager.py's `_write_environment`)."""
+        inherits its base image's own default. CoderEvalAgent's own `--workspace-dir
+        "$(pwd)"` finds the real cwd at run time regardless (see packager.py's
+        `_write_environment`); the verifier phase doesn't depend on it at all."""
         original = "FROM ubuntu:24.04\nRUN apt-get update\n"
         env_dir = tmp_path / "environment"
         env_dir.mkdir()
@@ -327,10 +345,12 @@ def test_dockerfile_with_an_existing_workdir_is_respected_and_not_touched(self,
         volumes = compose["services"]["main"]["volumes"]
         assert any(v.endswith(":/opt/coder-eval-task/task.yaml:ro") for v in volumes)
         assert not any("declared no WORKDIR" in w for w in result.warnings)
-        # test.sh no longer needs to agree on a literal value -- it resolves the real
-        # cwd itself via `$(pwd)` -- but task.toml still surfaces the Dockerfile's own
-        # explicit WORKDIR so Harbor's `docker exec -w` pins the same path deliberately.
-        assert 'coder-eval evaluate /tests/task.yaml "$(pwd)"' in (out_dir / "tests" / "test.sh").read_text(
+        # test.sh no longer needs to agree on a literal value -- it grades
+        # /logs/agent as a run directory -- but task.toml still surfaces the
+        # Dockerfile's own explicit WORKDIR so the AGENT phase's `docker exec
+        # -w` (and CoderEvalAgent's own `--workspace-dir "$(pwd)"`) pin the
+        # same path deliberately.
+        assert "coder-eval evaluate /tests/task.yaml /logs/agent" in (out_dir / "tests" / "test.sh").read_text(
             encoding="utf-8"
         )
         doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8"))
diff --git a/tests/test_harbor_portability.py b/tests/test_harbor_portability.py
index 98a6dd08..1da59fd6 100644
--- a/tests/test_harbor_portability.py
+++ b/tests/test_harbor_portability.py
@@ -90,14 +90,20 @@ def test_reference_comparison_does_not_block_export() -> None:
         CommandExecutedCriterion(description="d", command_pattern="."),
         CommandsEfficiencyCriterion(description="d", expected_commands=3),
         SkillTriggeredCriterion(description="d", expected_skill="s", skill_name="s"),
-        CliCalledCriterion(description="d", verb="v"),
     ],
 )
-def test_missing_functionality_criteria_always_block_export(criterion: object) -> None:
-    """No flag can supply what does not exist yet (C1.3 / a recorder-baking step)."""
-    issues = audit_criteria([criterion], allow_credentials=True)  # type: ignore[list-item]
+def test_trajectory_criteria_do_not_block_export(criterion: object) -> None:
+    """NEEDS_TRAJECTORY is not in _BLOCKING_IN_V1 — CoderEvalAgent + test.sh always wire
+    /logs/agent/trajectory.json into the verifier's `coder-eval evaluate --format harbor` call."""
+    assert audit_criteria([criterion]) == []  # type: ignore[list-item]
+
+
+def test_cli_called_still_blocks_export() -> None:
+    """No flag can supply what does not exist yet (a recorder-baking step in C2)."""
+    criterion = CliCalledCriterion(description="d", verb="v")
+    issues = audit_criteria([criterion])
     assert len(issues) == 1
-    assert issues[0].criterion_type == criterion.type  # type: ignore[attr-defined]
+    assert issues[0].criterion_type == criterion.type
 
 
 @pytest.mark.parametrize(
@@ -108,9 +114,10 @@ def test_missing_functionality_criteria_always_block_export(criterion: object) -
         UiPathEvalCriterion(description="d", agent_name="a", eval_set="p", thresholds={}),
     ],
 )
-def test_credentials_criteria_block_by_default_but_have_an_escape_hatch(criterion: object) -> None:
-    assert len(audit_criteria([criterion])) == 1  # type: ignore[list-item]
-    assert audit_criteria([criterion], allow_credentials=True) == []  # type: ignore[list-item]
+def test_credentials_criteria_never_block_export(criterion: object) -> None:
+    """NEEDS_CREDENTIALS always exports -- the operator is assumed to provision
+    model access inside the verifier container themselves."""
+    assert audit_criteria([criterion]) == []  # type: ignore[list-item]
 
 
 def test_a_clean_multi_criterion_task_reports_no_issues() -> None:
@@ -125,11 +132,9 @@ def test_a_mixed_task_reports_only_the_blocking_criteria() -> None:
     criteria = [
         FileExistsCriterion(description="portable", path="p1"),
         SkillTriggeredCriterion(description="needs trajectory", expected_skill="s", skill_name="s"),
+        CliCalledCriterion(description="needs cli recorder", verb="v"),
         LLMJudgeCriterion(description="needs credentials", prompt="p"),
     ]
     issues = audit_criteria(criteria)
-    assert {i.criterion_description for i in issues} == {"needs trajectory", "needs credentials"}
-    assert {i.portability for i in issues} == {
-        CriterionPortability.NEEDS_TRAJECTORY,
-        CriterionPortability.NEEDS_CREDENTIALS,
-    }
+    assert {i.criterion_description for i in issues} == {"needs cli recorder"}
+    assert {i.portability for i in issues} == {CriterionPortability.NEEDS_CLI_RECORDER}

From 203f9f5797f43244946c37cea6d545eac32fe0ce Mon Sep 17 00:00:00 2001
From: Akshaya Shanbhogue <akshaya.shanbhogue@uipath.com>
Date: Thu, 17 Sep 2026 08:11:03 -0700
Subject: [PATCH 2/3] docs(harbor): trim two docstrings over the prose budget

packager.py's _write_environment and portability.py's module docstring
exceeded the 150-word prose cap the CI quality gate enforces. Both already
had a Rationale: pointer into .claude/notes/reporting.md, so trim to the
essential contract and let the notes carry the narrative.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/coder_eval/harbor/packager.py    | 22 ++++++----------------
 src/coder_eval/harbor/portability.py | 13 +++++--------
 2 files changed, 11 insertions(+), 24 deletions(-)

diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py
index d76a0002..c4e20220 100644
--- a/src/coder_eval/harbor/packager.py
+++ b/src/coder_eval/harbor/packager.py
@@ -244,22 +244,12 @@ def _write_environment(
     """Derive ``environment/`` and return ``(workdir, docker_image)``.
 
     ``workdir`` is an EXPLICIT override only — ``sandbox.docker.working_dir`` or a
-    Dockerfile's own ``WORKDIR`` line. ``None`` otherwise, so Harbor's ``docker exec``
-    gets no ``-w`` and lands wherever the image's OWN ``WORKDIR`` already puts it. This
-    still matters for the AGENT phase: ``CoderEvalAgent.run()`` shells out with
-    ``--workspace-dir "$(pwd)"``, so wherever ``docker exec`` actually lands decides
-    what that captures. The VERIFIER phase no longer depends on it at all —
-    ``tests/test.sh`` grades against ``/logs/agent`` as a run directory (its own
-    recorded ``sandbox_path``), not a live cwd.
-
-    ``docker_image`` is set only when no ``environment/Dockerfile`` was written, so
-    ``task.toml``'s ``[environment].docker_image`` points at the pre-built image;
-    ``None`` when a Dockerfile was written and Harbor must build from it.
-
-    A ``dockerfile_path`` is the only shape that writes a Dockerfile, copied in
-    UNCHANGED — no ``WORKDIR`` appended even when it declares none.
-    ``environment/task.yaml`` is bind-mounted at :data:`AGENT_TASK_YAML_PATH`, never
-    ``COPY``'d.
+    Dockerfile's own ``WORKDIR`` line — ``None`` otherwise. ``docker_image`` is set only
+    when no ``environment/Dockerfile`` was written, so ``task.toml``'s
+    ``[environment].docker_image`` points at the pre-built image; ``None`` when a
+    Dockerfile was written and Harbor must build from it. A ``dockerfile_path`` is the
+    only shape that writes a Dockerfile, copied in UNCHANGED. ``environment/task.yaml``
+    is bind-mounted at :data:`AGENT_TASK_YAML_PATH`, never ``COPY``'d.
 
     Rationale: .claude/notes/reporting.md § What the export carries, and what it refuses to carry
     """
diff --git a/src/coder_eval/harbor/portability.py b/src/coder_eval/harbor/portability.py
index 5f93afd1..7625a112 100644
--- a/src/coder_eval/harbor/portability.py
+++ b/src/coder_eval/harbor/portability.py
@@ -7,18 +7,15 @@
 Classification, v1:
 
 - ``PORTABLE`` — filesystem/exit-code checks.
-- ``NEEDS_REFERENCE`` — ``reference_comparison``; never actually blocking, since the
-  export always emits ``tests/reference/`` when the task declares one.
+- ``NEEDS_REFERENCE`` — ``reference_comparison``; never blocking, since the export
+  always emits ``tests/reference/`` when the task declares one.
 - ``NEEDS_TRAJECTORY`` — ``command_executed``, ``commands_efficiency``,
-  ``skill_triggered``; never actually blocking, since the packager's ``CoderEvalAgent``
-  always runs ``coder-eval execute --format harbor`` and the generated ``tests/test.sh``
-  always grades against the resulting ``/logs/agent/trajectory.json`` (ATIF).
+  ``skill_triggered``; never blocking, since the generated ``tests/test.sh`` always
+  grades against ``/logs/agent/trajectory.json`` (ATIF).
 - ``NEEDS_CLI_RECORDER`` — ``cli_called``. Hard-error until the export bakes the
   recorder shim in.
 - ``NEEDS_CREDENTIALS`` — ``llm_judge``, ``agent_judge``, ``uipath_eval``. Never
-  actually blocking — exporting one of these always assumes the operator will
-  provision model credentials/network access inside the verifier container
-  themselves; C2 does not do it for them.
+  blocking — assumes the operator provisions credentials themselves.
 
 Rationale: .claude/notes/reporting.md § Not every criterion can grade inside someone else's container
 """

From cc51552d6a7a2436fc16826e801f44cfbca06cf2 Mon Sep 17 00:00:00 2001
From: Akshaya Shanbhogue <akshaya.shanbhogue@uipath.com>
Date: Thu, 17 Sep 2026 08:11:13 -0700
Subject: [PATCH 3/3] fix(harbor): reject --resume with --workspace-dir, sync
 design notes

--workspace-dir mode writes the finalized task.json flat at config.run_dir,
but --resume's bookkeeping (clear_rerun_artifacts, _load_completed_result)
still reads/clears the nested per-task run_dir. Nothing stopped the two
flags from being passed together, so a resumed workspace-dir run would
never recognize its own prior result. Add a guard next to the existing
docker-driver check, and a batch-level test asserting workspace_dir mode
constructs Orchestrator with the flat run_dir.

Also bring .claude/notes/reporting.md's two harbor sections in line with
the grading-contract change from the prior commits (the /logs/agent
run-directory approach, and the now-unconditional export of
NEEDS_TRAJECTORY/NEEDS_CREDENTIALS criteria), and drop a stale
--allow-credentials mention from the llm_judge e2e fixture.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .claude/notes/reporting.md               | 36 ++++++++++------
 src/coder_eval/cli/run_command.py        |  6 +++
 tests/harbor_e2e/fixtures/llm_judge.yaml |  8 ++--
 tests/test_parallel.py                   | 54 ++++++++++++++++++++++++
 4 files changed, 87 insertions(+), 17 deletions(-)

diff --git a/.claude/notes/reporting.md b/.claude/notes/reporting.md
index 29c7b674..58c8e060 100644
--- a/.claude/notes/reporting.md
+++ b/.claude/notes/reporting.md
@@ -422,13 +422,17 @@ where it is an unexplained low reward with no obvious cause.
 Filesystem and exit-code checks are portable — nothing about the verifier container changes
 what they need. `reference_comparison` needs the reference tree, which the export always
 places verifier-side when the task declares one, so that class never actually blocks.
-Trajectory-reading criteria cannot work in the export direction at all: the verifier is a
-separate process from the agent phase, and the agent may not even BE coder-eval, so there is
-no iterations list to read without ATIF ingestion. `cli_called` reads a log written by a
-recorder shim coder-eval's own sandbox installs, which the exported Dockerfile does not
-provision. Credential-needing criteria need a model reachable from inside the verifier
-container and a judge that does not follow the agent's route; they are refused with an
-explicit opt-in escape hatch for an operator who has provisioned that themselves.
+Trajectory-reading criteria once could not work in the export direction at all, but no
+longer block: the packager's `CoderEvalAgent` always runs `coder-eval execute --format
+harbor`, which writes `/logs/agent/trajectory.json` (ATIF) alongside `task.json`, and the
+generated `tests/test.sh` grades `/logs/agent` as a run directory — so the trajectory is
+always there by the time the verifier runs. `cli_called` reads a log written by a recorder
+shim coder-eval's own sandbox installs, which the exported Dockerfile does not provision —
+this is the one class that still hard-blocks. Credential-needing criteria need a model
+reachable from inside the verifier container and a judge that does not follow the agent's
+route; the export no longer gates them behind a flag — it exports them unconditionally on
+the assumption that an operator exporting one has already provisioned that access
+themselves (C2 does not do it for them).
 
 Coverage is registry-derived, so a new criterion type added to the union without a
 classification fails CLOSED rather than silently exporting as if it were portable.
@@ -508,12 +512,18 @@ letting a bare `OSError` escape, because the CLI catches only the export errors
 unreadable tree would otherwise abort a whole experiment export the docstring promises it
 will not abort.
 
-The generated shell script no longer interpolates a workdir value at all — it resolves its
-own cwd via `$(pwd)` at run time, a fixed literal in `_TEST_SH_TEMPLATE`. `docker exec`
-(with `-w` when the task set an explicit override, or none when it did not — see above)
-always lands the shell there, whether or not the same container's agent phase used an
-explicit override too, so `pwd` is authoritative and there is no longer an injection
-surface to `shlex.quote` against.
+The generated shell script no longer interpolates a workdir value, or guesses a cwd, at
+all — it is a fixed literal in `_TEST_SH_TEMPLATE` that calls `coder-eval evaluate
+/tests/task.yaml /logs/agent --in-place --run-dir /logs/verifier`, passing `/logs/agent`
+as an explicit RUN DIRECTORY rather than a workdir guess. `coder-eval execute --run-dir
+/logs/agent ...` (the agent phase) always finishes with `/logs/agent/task.json` and
+`/logs/agent/artifacts/<task_id>/`, so `coder-eval evaluate` locates the workspace from
+that `task.json`'s own recorded `sandbox_path` instead of a live `$(pwd)`. Passing the
+task file explicitly also keeps this off the untrusted-recorded-config path (which exists
+for a shared run directory whose config is not to be trusted without
+`--allow-recorded-commands`) — an explicit, operator-supplied task file always overrides
+the run's recorded config. Nothing task-controlled is interpolated into the template, so
+there is no injection surface to `shlex.quote` against.
 
 ## The ATIF trajectory bridge
 
diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py
index cad6cff0..275f267d 100644
--- a/src/coder_eval/cli/run_command.py
+++ b/src/coder_eval/cli/run_command.py
@@ -468,6 +468,12 @@ def run_pipeline(
     # --resume needs an explicit run dir to resume into (auto-generated dirs are always fresh).
     if resume and run_dir is None:
         raise typer.BadParameter("--resume requires --run-dir pointing at the run to continue.")
+    # --workspace-dir writes the finalized task.json flat at run_dir (batch.py's
+    # effective_run_dir); --resume's bookkeeping (clear_rerun_artifacts,
+    # _load_completed_result) reads/clears the nested per-task run_dir instead, so a
+    # resumed workspace-dir run would never recognize its own prior result.
+    if resume and workspace_dir is not None:
+        raise typer.BadParameter("--resume is not supported together with --workspace-dir.")
     # Without --resume this flag parsed, was accepted, and did nothing at all. Its
     # sibling mode-scoped flag (`evaluate --workspace`) hard-errors on exactly this.
     if allow_host_grading and not resume:
diff --git a/tests/harbor_e2e/fixtures/llm_judge.yaml b/tests/harbor_e2e/fixtures/llm_judge.yaml
index d8f1cbb7..9209049c 100644
--- a/tests/harbor_e2e/fixtures/llm_judge.yaml
+++ b/tests/harbor_e2e/fixtures/llm_judge.yaml
@@ -1,10 +1,10 @@
 task_id: "harbor_e2e_llm_judge"
 description: >
   Harbor E2E llm_judge case: a real claude-code agent turn, graded by a real
-  llm_judge call inside the Harbor verifier phase (--allow-credentials).
-  Catches regressions in judge-prompt construction / route resolution / JSON
-  verdict parsing when run through a Harbor-exported task, not just through
-  coder-eval's own `run`/`evaluate`.
+  llm_judge call inside the Harbor verifier phase (credentials are always
+  allowed at export). Catches regressions in judge-prompt construction / route
+  resolution / JSON verdict parsing when run through a Harbor-exported task,
+  not just through coder-eval's own `run`/`evaluate`.
 
 run_limits:
   max_turns: 2
diff --git a/tests/test_parallel.py b/tests/test_parallel.py
index 63178f31..54f5786b 100644
--- a/tests/test_parallel.py
+++ b/tests/test_parallel.py
@@ -79,6 +79,60 @@ async def test_sequential_mode(tmp_path):
     assert summary_file.exists(), "Run summary should be created"
 
 
+@pytest.mark.asyncio
+async def test_workspace_dir_constructs_orchestrator_with_flat_run_dir(tmp_path):
+    """--workspace-dir mode must pass config.run_dir (flat) to Orchestrator, not the
+    nested <variant>/<task_id>/<NN> path a ResolvedTask normally carries.
+
+    Exercises the effective_run_dir branch in run_batch's run_single -- the exact
+    code path the real Harbor CoderEvalAgent drives via `coder-eval execute
+    --format harbor --workspace-dir "$(pwd)"`. Patches Orchestrator itself (rather
+    than driving a full run) so the assertion is directly on the value this branch
+    computes, independent of what a real run happens to persist to disk.
+    """
+    task = TaskDefinition(
+        task_id="test_workspace_dir",
+        description="Test workspace_dir flat run_dir",
+        initial_prompt="Test prompt",
+        agent={"type": "claude-code"},
+        sandbox={"driver": "tempdir"},
+        success_criteria=[{"type": "file_exists", "path": "test.txt", "description": "Check for test.txt"}],
+    )
+    task_file = tmp_path / "test_task.yaml"
+    task_file.write_text("task_id: test_workspace_dir\n")
+
+    run_dir = tmp_path / "run"
+    workspace_dir = tmp_path / "workspace"
+    workspace_dir.mkdir()
+    config = BatchRunConfig(
+        run_dir=run_dir,
+        max_parallel=1,
+        preservation_mode=PreservationMode.NONE,
+        workspace_dir=workspace_dir,
+    )
+
+    nested_run_dir = run_dir / "default" / "test_workspace_dir" / "default"
+    resolved_task = ResolvedTask(
+        task=task,
+        task_file=task_file,
+        run_dir=nested_run_dir,
+        variant_id="default",
+        original_task_id="test_workspace_dir",
+    )
+
+    mock_orchestrator = MagicMock()
+    mock_orchestrator.run = AsyncMock(return_value=MagicMock(duration_seconds=0.0))
+    mock_orchestrator_cls = MagicMock(return_value=mock_orchestrator)
+
+    with patch("coder_eval.orchestrator.Orchestrator", mock_orchestrator_cls):
+        await run_batch([resolved_task], config)
+
+    assert mock_orchestrator_cls.call_count == 1
+    assert mock_orchestrator_cls.call_args.kwargs["run_dir"] == run_dir, (
+        "workspace_dir mode must construct Orchestrator with the flat config.run_dir, not the nested per-task run_dir"
+    )
+
+
 @pytest.mark.asyncio
 async def test_semaphore_limits_concurrency(tmp_path):
     """Test that semaphore actually limits concurrent tasks."""