From 334faffcfdebd3f7b45c7154f0f8ca1b33846e7c Mon Sep 17 00:00:00 2001 From: "my.nguyen" Date: Thu, 10 Sep 2026 16:27:00 +0700 Subject: [PATCH 1/3] feat(gooddata-eval): make the K verdict a gate, pass@K or pass^K --- packages/gooddata-eval/README.md | 15 +- .../src/gooddata_eval/cli/agentic_runner.py | 15 +- .../src/gooddata_eval/cli/main.py | 22 ++- .../src/gooddata_eval/core/agentic/_gate.py | 64 +++++++ .../gooddata_eval/core/agentic/alert_skill.py | 16 +- .../core/agentic/general_question.py | 16 +- .../gooddata_eval/core/agentic/guardrail.py | 18 +- .../gooddata_eval/core/agentic/kda_skill.py | 16 +- .../core/agentic/metric_skill.py | 16 +- .../gooddata_eval/core/agentic/search_tool.py | 16 +- .../core/agentic/visualization.py | 17 +- .../src/gooddata_eval/core/config.py | 21 +++ .../core/reporting/json_report.py | 1 + .../src/gooddata_eval/core/runner.py | 2 + .../gooddata-eval/tests/test_agentic_gate.py | 171 ++++++++++++++++++ .../tests/test_agentic_kda_skill.py | 12 +- .../tests/test_langfuse_e2e_fake_server.py | 19 +- 17 files changed, 427 insertions(+), 30 deletions(-) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py create mode 100644 packages/gooddata-eval/tests/test_agentic_gate.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index 71a8cb911..ca1e6d371 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -126,7 +126,8 @@ gd-eval run \ | Flag | Default | Description | |---|---|---| -| `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. | +| `--runs K` | `2` | Independent runs per item. | +| `--gate` | `any` | Which verdict decides an item: `any` = pass@K (a run passing is enough), `power` = pass^K (every run must pass, so the verdict measures stability). Identical at `--runs 1`. Agentic kinds only. | | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests — see *Concurrency and workspace safety* below. | | `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail` and `dashboard_summary`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | | `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. | @@ -244,6 +245,11 @@ stays quiet for a unanimous one, and its summary line reads `3/4 passed, 1 on ev `runs` is what the item actually ran, which is not always the requested `--runs`: `agentic_conversation` takes no K and drives its fixture exactly once. +Which of the two decides pass/fail is `--gate`: `any` (default) gates on `pass_at_k`, `power` gates on +`pass_power_k`. The run records it as a top-level `gate`, and a failure under `power` says so — +`Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure` — because the message body describes +the best run, which under pass^K can be a run that passed. `agentic_conversation` has no K gate. + Each item additionally carries a per-phase breakdown: ```json @@ -410,8 +416,13 @@ the item's own root span. On the agentic path each score is mirrored onto the ag | Score | Description | |---|---| -| `pass_at_k` | 1 if any of the K runs passed strict checks, else 0. | +| `pass_at_k` | 1 if **any** of the K runs passed strict checks, else 0. | +| `pass_power_k` | 1 only if **every** one of the K runs passed. Agentic kinds only. | +| `gate_passed` | The verdict that decided the item: `pass_at_k` under `--gate any`, `pass_power_k` under `--gate power`. Agentic kinds only. | | `quality_score` | Fraction of strict check flags that are `True` (0.0–1.0). Shown in CLI as a percentage. | | `value_score` | Weighted blend: 0.6 × quality + 0.2 × speed (speed = max(0, 1 − latency/60s)). | | `latency_s` | Average per-run latency in seconds. | | `provider_type` | Model vendor + gateway label (e.g. `ANTHROPIC`, `BEDROCK/ANTHROPIC`, `AZURE/OPENAI`). Stored in Langfuse trace metadata and tags. | + +Score names carry no K; K and the gate are on the dataset-run metadata as `eval_k` and `eval_gate`. +`agentic_visualization` also still writes its historic `pass_at_{K}` / `pass_power_{K}` pair. diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 0bd6f5cf8..4ce3f6f09 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -8,6 +8,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, TypedDict +from gooddata_eval.core.agentic._gate import DEFAULT_GATE, EvalGate, normalize_gate from gooddata_eval.core.agentic._langfuse import make_langfuse_client from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill @@ -128,9 +129,12 @@ def _dispatch_agentic( reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, + gate: EvalGate = DEFAULT_GATE, ) -> AgenticEvalOutcome: """Call the appropriate evaluate_agentic_* function for the item's test_kind. + `gate` reaches every kind except agentic_conversation, which has no K to gate over. + Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps, conversation_id, response_id, detail) on success and attaches the same four attributes to its raised *AssertionError on failure -- no kind is exempt. @@ -155,6 +159,7 @@ def _dispatch_agentic( question=item.question, expected_outputs=_parse_visualization_expected(eo), k=k, + gate=gate, agent_id=agent_id, **lf_kw, ) @@ -166,6 +171,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, (dict, list)) else {}, k=k, + gate=gate, agent_id=agent_id, **lf_kw, ) @@ -177,6 +183,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, dict) else {}, k=k, + gate=gate, agent_id=agent_id, **lf_kw, ) @@ -191,6 +198,7 @@ def _dispatch_agentic( question=item.question, expected_tool_call=expected_args, k=k, + gate=gate, agent_id=agent_id, **lf_kw, ) @@ -202,6 +210,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + gate=gate, agent_id=agent_id, user_context=item.user_context, **lf_kw, @@ -214,6 +223,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + gate=gate, agent_id=agent_id, **lf_kw, ) @@ -225,6 +235,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, dict) else {}, k=k, + gate=gate, agent_id=agent_id, **lf_kw, ) @@ -291,6 +302,7 @@ def run_agentic_items( on_item_done: Any = None, agent_id: str | None = None, concurrency: int = 1, + gate: EvalGate = DEFAULT_GATE, ) -> EvalReport: """Run agentic items through evaluate_agentic_* and return an EvalReport. @@ -303,7 +315,7 @@ def run_agentic_items( """ langfuse = make_langfuse_client() if use_langfuse else None - report = EvalReport(model=model_version) + report = EvalReport(model=model_version, gate=normalize_gate(gate)) total = len(items) # Trace linking runs here rather than inside each evaluate_agentic_*, so an item's # Langfuse poll overlaps the NEXT item's agent call instead of extending its own @@ -339,6 +351,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: reasoning_effort, agent_id, submit_trace_link=linker.submit, + gate=gate, ) if isinstance(outcome, AgenticEvalOutcome): reasoning_steps = outcome.reasoning_steps diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 77dbc9dde..5359c245a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -16,7 +16,15 @@ from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items from gooddata_eval.core.chat.sse_client import ChatClient -from gooddata_eval.core.config import DEFAULT_JUDGE_MODEL, JUDGE_MODEL_ENV_VAR, ReasoningEffort, RunConfig +from gooddata_eval.core.config import ( + DEFAULT_GATE, + DEFAULT_JUDGE_MODEL, + JUDGE_MODEL_ENV_VAR, + EvalGate, + ReasoningEffort, + RunConfig, + normalize_gate, +) from gooddata_eval.core.connection import ConnectionError_, resolve_connection from gooddata_eval.core.dataset.local import load_local_dataset from gooddata_eval.core.langfuse.sink import LangfuseSink @@ -91,7 +99,15 @@ def _build_parser() -> argparse.ArgumentParser: "Default: workspace's current active model." ), ) - run.add_argument("--runs", type=int, default=2, help="Independent runs per item (pass@K). Default 2.") + run.add_argument("--runs", type=int, default=2, help="Independent runs per item. Default 2.") + run.add_argument( + "--gate", + choices=get_args(EvalGate), + default=DEFAULT_GATE, + help="Which verdict decides an item: 'any' = pass@K (a run passing is enough, the " + "default and historic behaviour), 'power' = pass^K (every run must pass, so the verdict " + "measures stability). Identical at --runs 1. Agentic kinds only.", + ) run.add_argument( "--concurrency", type=int, @@ -416,6 +432,7 @@ def on_langfuse_item_done( token=config.token, workspace_id=config.workspace_id, k=config.runs, + gate=config.gate, model_version=resolved.model_id, reasoning_effort=config.reasoning_effort, use_langfuse=config.log_to_langfuse, @@ -529,6 +546,7 @@ def main(argv: list[str] | None = None) -> int: kind=args.kind, preserve_failed=args.preserve_failed, reasoning_effort=args.reasoning_effort, + gate=normalize_gate(args.gate), agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"), ) return _run(config) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py new file mode 100644 index 000000000..09271174a --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py @@ -0,0 +1,64 @@ +# (C) 2026 GoodData Corporation +"""Which of pass@K / pass^K decides an item, and how both reach Langfuse.""" + +from __future__ import annotations + +from typing import Any + +from gooddata_eval.core.config import DEFAULT_GATE, EvalGate, normalize_gate + +__all__ = [ + "DEFAULT_GATE", + "EvalGate", + "gate_failure_note", + "gate_label", + "gate_passed", + "log_gate_scores", + "normalize_gate", + "stamp_gate_metadata", +] + + +def gate_passed(gate: str | None, *, pass_at_k: bool, pass_power_k: bool) -> bool: + """Whether the item passes under ``gate``.""" + return pass_power_k if normalize_gate(gate) == "power" else pass_at_k + + +def gate_label(gate: str | None, k: int) -> str: + """Short name for the gate, e.g. ``pass^3`` or ``pass@2``.""" + return f"pass^{k}" if normalize_gate(gate) == "power" else f"pass@{k}" + + +def gate_failure_note(gate: str | None, runs_passed: int, runs_total: int) -> str: + """Gate and how many runs met it, for the assertion message. + + Needed because the message body describes the BEST run, which under pass^K can be a run + that passed — so the reported detail on its own looks like a pass. + """ + label = gate_label(gate, runs_total) + if normalize_gate(gate) == "power" and runs_passed: + return f"Gate {label} failed: {runs_passed}/{runs_total} runs passed — unstable, not a clean failure." + return f"Gate {label} failed: {runs_passed}/{runs_total} runs passed." + + +def log_gate_scores(ctx: Any, trace_id: Any, *, gate: str | None, pass_at_k: bool, pass_power_k: bool) -> None: + """Log both candidate verdicts and the one that decided. + + The names carry no K on purpose: `pass_at_2` becomes `pass_at_3` the moment K changes, + splitting every Langfuse view built on the old name. + """ + ctx.score(trace_id, name="pass_at_k", value=pass_at_k, data_type="BOOLEAN") + ctx.score(trace_id, name="pass_power_k", value=pass_power_k, data_type="BOOLEAN") + ctx.score( + trace_id, + name="gate_passed", + value=gate_passed(gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k), + data_type="BOOLEAN", + ) + + +def stamp_gate_metadata(metadata: dict, *, k: int, gate: str | None) -> dict: + """Record K and the gate on the dataset-run metadata (mutates and returns it).""" + metadata["eval_k"] = k + metadata["eval_gate"] = normalize_gate(gate) + return metadata diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 330dea31e..13fd8fbc8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -12,6 +12,14 @@ from gooddata_sdk import GoodDataSdk from gooddata_eval.core.agentic._catalog import AnomalyDetectionGranularity, CatalogMetricAlert +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -783,6 +791,7 @@ def evaluate_agentic_alert_skill( question: str, expected_output: dict, k: int = _DEFAULT_K, + gate: EvalGate = DEFAULT_GATE, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, agent_id: str | None = None, @@ -823,6 +832,7 @@ def evaluate_agentic_alert_skill( window_end = utc_now() def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) for run_idx, run in enumerate(summary.run_results): pt = ctx.trace(run.conversation_id) @@ -841,6 +851,7 @@ def _write_scores(ctx: RunTraceContext) -> None: with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, strict_checks=strict_checks, @@ -890,9 +901,10 @@ def _write_scores(ctx: RunTraceContext) -> None: "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), } - if not summary.pass_at_k: + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, runs_effective) exc = AlertSkillAssertionError( - f"Alert skill assertion failed. strict_pass={ev.strict_pass}. " + f"Alert skill assertion failed. {gate_note} strict_pass={ev.strict_pass}. " f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, " f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, " f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, " diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index fefb6c888..fd1671dc9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -6,6 +6,14 @@ import time from dataclasses import dataclass, field +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -213,6 +221,7 @@ def evaluate_agentic_general_question( question: str, expected_output: str, k: int = _DEFAULT_K, + gate: EvalGate = DEFAULT_GATE, initial_conversation_id: str | None = None, agent_id: str | None = None, langfuse: object | None = None, @@ -250,6 +259,7 @@ def evaluate_agentic_general_question( window_end = utc_now() def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) for run_idx, run in enumerate(summary.run_results): if run.judge_error is not None: @@ -262,6 +272,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) as tid: ctx.score(tid, name="general_question_pass", value=float(run.passed), data_type="BOOLEAN") ctx.score(tid, name="llm_judge_score", value=run.llm_judge_score, data_type="NUMERIC") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, strict_checks={"general_question_pass": run.passed}, @@ -324,9 +335,10 @@ def _write_scores(ctx: RunTraceContext) -> None: **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), } - if not summary.pass_at_k: + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, runs_effective) exc = GeneralQuestionAssertionError( - f"General question assertion failed. passed={best.passed}. Reasoning: {best.reasoning}" + f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}" ) exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index ffb5c8d56..5282dab8d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -5,6 +5,14 @@ from dataclasses import dataclass, field +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -186,6 +194,7 @@ def evaluate_agentic_guardrail( question: str, expected_output: str, k: int = _DEFAULT_K, + gate: EvalGate = DEFAULT_GATE, initial_conversation_id: str | None = None, agent_id: str | None = None, langfuse: object | None = None, @@ -222,6 +231,7 @@ def evaluate_agentic_guardrail( window_end = utc_now() def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) for run_idx, run in enumerate(summary.run_results): if run.judge_error is not None: @@ -234,6 +244,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) as tid: ctx.score(tid, name="guardrail_pass", value=float(run.passed), data_type="BOOLEAN") ctx.score(tid, name="llm_judge_score", value=run.llm_judge_score, data_type="NUMERIC") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, strict_checks={"guardrail_pass": run.passed}, @@ -290,8 +301,11 @@ def _write_scores(ctx: RunTraceContext) -> None: **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), } - if not summary.pass_at_k: - exc = GuardrailAssertionError(f"Guardrail assertion failed. passed={best.passed}. Reasoning: {best.reasoning}") + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, runs_effective) + exc = GuardrailAssertionError( + f"Guardrail assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}" + ) exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id exc.response_id = best.response_id diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 0b5122d23..efdfe5759 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -7,6 +7,14 @@ import os from dataclasses import dataclass, field +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -372,6 +380,7 @@ def evaluate_agentic_kda_skill( question: str, expected_output: dict, k: int = _DEFAULT_K, + gate: EvalGate = DEFAULT_GATE, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, agent_id: str | None = None, @@ -409,6 +418,7 @@ def evaluate_agentic_kda_skill( window_end = utc_now() def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) for run_idx, run in enumerate(summary.run_results): # No custom selector -- same default (max-latency) as every other skill; harmless @@ -440,6 +450,7 @@ def _write_scores(ctx: RunTraceContext) -> None: value=turn_wall_clock_sec, data_type="NUMERIC", ) + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, strict_checks=strict_checks, @@ -486,9 +497,10 @@ def _write_scores(ctx: RunTraceContext) -> None: "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), } - if not summary.pass_at_k: + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, runs_effective) message = ( - f"KDA skill assertion failed. strict_pass={ev.strict_pass} " + f"KDA skill assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, " f"success={ev.success}, turn_completed={ev.turn_completed}). " f"Actual create args: {best.actual_create_args}. " diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 6bd960873..64d181ce0 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -11,6 +11,14 @@ from gooddata_sdk import GoodDataSdk +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -405,6 +413,7 @@ def evaluate_agentic_metric_skill( question: str, expected_output: dict | list, k: int = _DEFAULT_K, + gate: EvalGate = DEFAULT_GATE, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, agent_id: str | None = None, @@ -445,6 +454,7 @@ def evaluate_agentic_metric_skill( window_end = utc_now() def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) for run_idx, run in enumerate(summary.run_results): pt = ctx.trace(run.conversation_id) @@ -456,6 +466,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) as tid: ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN") ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, strict_checks={"metric_created": run.metric_created, "maql_correct": run.maql_correct}, @@ -501,10 +512,11 @@ def _write_scores(ctx: RunTraceContext) -> None: "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), } - if not summary.pass_at_k: + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, runs_effective) candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) exc = MetricSkillAssertionError( - f"Metric skill assertion failed. " + f"Metric skill assertion failed. {gate_note} " f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. " f"Expected MAQL (candidates): {candidates_str}. " f"Actual MAQL: {best.actual_maql}." diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index cf406f5dd..17fd699cc 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -5,6 +5,14 @@ from dataclasses import dataclass, field +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -167,6 +175,7 @@ def evaluate_agentic_search_tool( question: str, expected_tool_call: dict, k: int = _DEFAULT_K, + gate: EvalGate = DEFAULT_GATE, initial_conversation_id: str | None = None, agent_id: str | None = None, langfuse: object | None = None, @@ -202,6 +211,7 @@ def evaluate_agentic_search_tool( window_end = utc_now() def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) for run_idx, run in enumerate(summary.run_results): pt = ctx.trace(run.conversation_id) @@ -210,6 +220,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) as tid: ctx.score(tid, name="tool_selection", value=float(run.tool_selected), data_type="BOOLEAN") ctx.score(tid, name="tool_correctness", value=float(run.tool_correct), data_type="BOOLEAN") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, strict_checks={"tool_selection": run.tool_selected}, @@ -251,9 +262,10 @@ def _write_scores(ctx: RunTraceContext) -> None: "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), } - if not summary.pass_at_k: + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, runs_effective) exc = SearchToolAssertionError( - f"Search tool assertion failed. " + f"Search tool assertion failed. {gate_note} " f"tool_selected={best.tool_selected}, tool_correct={best.tool_correct}. " f"Tool calls made: {best.tool_call_names}" ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 159dea564..4c8601fe2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -10,6 +10,14 @@ import os from dataclasses import dataclass, field +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -314,6 +322,7 @@ def evaluate_agentic_visualization( question: str, expected_outputs: list[CreatedVisualization], k: int = _DEFAULT_K, + gate: EvalGate = DEFAULT_GATE, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, agent_id: str | None = None, @@ -354,6 +363,7 @@ def evaluate_agentic_visualization( window_end = utc_now() def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) K = len(summary.run_results) for run_idx, run in enumerate(summary.run_results): @@ -373,10 +383,13 @@ def _write_scores(ctx: RunTraceContext) -> None: ctx.score(tid, name="assertion-vis-filters", value=ev.filters_correct, data_type="BOOLEAN") ctx.score(tid, name="assertion-vis-type", value=ev.viz_type_hard, data_type="BOOLEAN") ctx.score(tid, name="skill_selection", value=ev.skill_activated, data_type="BOOLEAN") + # Superseded by log_gate_scores' K-stable names, kept until the readers + # migrate: gdc-nas combo_report.py matches on the literal "pass_at_2". ctx.score(tid, name=f"pass_at_{K}", value=summary.pass_at_k, data_type="BOOLEAN") ctx.score(tid, name=f"pass_power_{K}", value=summary.pass_power_k, data_type="BOOLEAN") ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, strict_checks=strict_checks, @@ -427,7 +440,8 @@ def _write_scores(ctx: RunTraceContext) -> None: "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), } - if not summary.pass_at_k: + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, runs_effective) n = len(expected_outputs) candidate_note = f" (closest of {n} candidates)" if n > 1 else "" cross_ref_detail = (" → " + "; ".join(ev.cross_ref_errors)) if ev.cross_ref_errors else "" @@ -436,6 +450,7 @@ def _write_scores(ctx: RunTraceContext) -> None: exc = VisualizationAssertionError( "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" "Agentic Visualization Assertion Failed! (Critical Mode)\n" + f"{gate_note}\n" "------------------------------------------\n" f"Question:\n{question}\n" "------------------------------------------\n" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index 06c836c97..9a3f7a253 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -33,6 +33,26 @@ def judge_model() -> str: ReasoningEffort = Literal["LOW", "MEDIUM", "HIGH"] """Effort values the AI chat endpoint accepts, uppercase as the server enum requires.""" +EvalGate = Literal["any", "power"] +"""`any` = pass@K, `power` = pass^K. Behaviour is in core/agentic/_gate.py; the type lives here +because putting it there would make config import the agentic package, whose __init__ imports +back through chat.sse_client into config.""" + +DEFAULT_GATE: EvalGate = "any" +"""Historic behaviour — changing it makes every caller that passes no gate stricter.""" + + +def normalize_gate(value: str | None) -> EvalGate: + """Canonical gate name; ``None``/blank means the default.""" + if value is None: + return DEFAULT_GATE + candidate = value.strip().lower() + if not candidate: + return DEFAULT_GATE + if candidate not in get_args(EvalGate): + raise ValueError(f"Invalid eval gate {value!r}; expected one of {', '.join(get_args(EvalGate))}.") + return cast("EvalGate", candidate) + def normalize_reasoning_effort(value: str | None) -> ReasoningEffort | None: """Canonical effort, or None when unset. @@ -70,3 +90,4 @@ class RunConfig: preserve_failed: bool = False reasoning_effort: ReasoningEffort | None = None agent_id: str | None = None + gate: EvalGate = DEFAULT_GATE diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py index 81091e141..98c3dc0e7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py @@ -13,6 +13,7 @@ def _build_run_dict(report: EvalReport) -> dict: return { "model": report.model, "workspace_id": report.workspace_id, + "gate": report.gate, "summary": { "total": report.total, "passed": report.passed, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index 16373975e..9fa15bf5f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -8,6 +8,7 @@ from functools import partial from typing import Callable, Protocol +from gooddata_eval.core.config import DEFAULT_GATE, EvalGate from gooddata_eval.core.evaluators import get_evaluator, supported_test_kinds from gooddata_eval.core.evaluators.base import ItemEvaluation from gooddata_eval.core.models import ChatResult, DatasetItem @@ -105,6 +106,7 @@ class EvalReport: provider_name: str = "" provider_type: str = "" workspace_id: str = "" + gate: EvalGate = DEFAULT_GATE items: list[ItemReport] = field(default_factory=list) wall_clock_s: float = 0.0 # actual elapsed time; differs from latency_s under concurrency diff --git a/packages/gooddata-eval/tests/test_agentic_gate.py b/packages/gooddata-eval/tests/test_agentic_gate.py new file mode 100644 index 000000000..e1fa11b1f --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_gate.py @@ -0,0 +1,171 @@ +# (C) 2026 GoodData Corporation +"""The K gate: which of pass@K / pass^K decides an item, and what reaches Langfuse.""" + +import contextlib +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + gate_failure_note, + gate_label, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic.general_question import ( + GeneralQuestionAssertionError, + evaluate_agentic_general_question, +) +from gooddata_eval.core.config import normalize_gate +from gooddata_eval.core.models import ChatResult + + +def test_the_default_gate_is_the_historic_pass_at_k(): + assert DEFAULT_GATE == "any" + assert gate_passed(None, pass_at_k=True, pass_power_k=False) is True + + +@pytest.mark.parametrize("value", ["power", "POWER", " Power "]) +def test_gate_is_normalized_from_any_casing(value): + assert normalize_gate(value) == "power" + + +@pytest.mark.parametrize("value", [None, "", " "]) +def test_absent_gate_means_the_default(value): + assert normalize_gate(value) == DEFAULT_GATE + + +@pytest.mark.parametrize("value", ["all", "majority", "pass^k", "1"]) +def test_an_unknown_gate_is_rejected_rather_than_silently_lax(value): + """Falling through to pass@K would pick the laxer gate and look deliberate.""" + with pytest.raises(ValueError, match="Invalid eval gate"): + normalize_gate(value) + + +@pytest.mark.parametrize( + "gate,pass_at_k,pass_power_k,expected", + [ + ("any", True, False, True), + ("power", True, False, False), + ("any", True, True, True), + ("power", True, True, True), + ("any", False, False, False), + ("power", False, False, False), + ], +) +def test_gate_selects_the_matching_verdict(gate, pass_at_k, pass_power_k, expected): + assert gate_passed(gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k) is expected + + +def test_the_two_gates_agree_at_k_1(): + """What makes it safe to set the gate globally and raise K only where it is wanted.""" + for verdict in (True, False): + assert gate_passed("any", pass_at_k=verdict, pass_power_k=verdict) is gate_passed( + "power", pass_at_k=verdict, pass_power_k=verdict + ) + + +def test_gate_label_names_the_gate_and_k(): + assert gate_label("power", 3) == "pass^3" + assert gate_label("any", 2) == "pass@2" + + +def test_failure_note_calls_out_an_unstable_item_rather_than_a_broken_one(): + """The message body describes the BEST run, which under pass^K can be one that passed.""" + note = gate_failure_note("power", 2, 3) + assert "pass^3" in note and "2/3" in note and "unstable" in note + + +def test_failure_note_does_not_call_a_clean_failure_unstable(): + assert "unstable" not in gate_failure_note("power", 0, 3) + assert "0/3" in gate_failure_note("power", 0, 3) + assert "0/1" in gate_failure_note("any", 0, 1) + + +def test_gate_scores_use_K_stable_names(): + """`pass_at_2` becomes `pass_at_3` the moment K changes, splitting every view on it.""" + ctx = MagicMock() + log_gate_scores(ctx, "trace-1", gate="power", pass_at_k=True, pass_power_k=False) + + logged = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list} + assert logged == {"pass_at_k": True, "pass_power_k": False, "gate_passed": False} + assert not [name for name in logged if name[-1].isdigit()] + + +def test_gate_passed_is_logged_not_left_to_be_derived(): + """A reader cannot recompute it without the gate, and a report that disagrees with what + the run gated on is worse than no report.""" + ctx = MagicMock() + log_gate_scores(ctx, "trace-1", gate="any", pass_at_k=True, pass_power_k=False) + + logged = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list} + assert logged["gate_passed"] is True # same inputs as above, opposite verdict + + +def test_k_and_gate_ride_the_run_metadata_not_extra_scores(): + metadata = {"github_run_id": "1", "model_version": "gpt-5.6-luna"} + assert stamp_gate_metadata(metadata, k=3, gate="POWER") is metadata + assert metadata == { + "github_run_id": "1", + "model_version": "gpt-5.6-luna", + "eval_k": 3, + "eval_gate": "power", + } + + +# --------------------------------------------------------------------------- # +# end to end through an evaluator: general_question is the smallest one +# --------------------------------------------------------------------------- # +@contextlib.contextmanager +def _judged(*verdicts: bool): + judge = MagicMock() + judge.model = "gpt-4o" + judge.score.side_effect = [(v, f"reason-{i}") for i, v in enumerate(verdicts)] + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, len(verdicts) + 1)] + client.send_message.return_value = ChatResult.model_validate({"textResponse": "answer"}) + with ( + patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=client), + patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=judge), + ): + yield + + +def _evaluate(gate, verdicts): + with _judged(*verdicts): + evaluate_agentic_general_question( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Q", + expected_output="rubric", + k=len(verdicts), + gate=gate, + initial_conversation_id="conv-0", + langfuse=None, + ) + + +def test_a_flaky_item_passes_under_any_and_fails_under_power(): + """2 of 3 runs passing is green under pass@K and red under pass^K.""" + _evaluate("any", (True, False, True)) # does not raise + + with pytest.raises(GeneralQuestionAssertionError) as exc: + _evaluate("power", (True, False, True)) + assert "pass^3" in str(exc.value) + assert "2/3" in str(exc.value) + assert exc.value.runs_passed == 2 + assert exc.value.runs_effective == 3 + + +def test_a_clean_item_passes_under_both_gates(): + _evaluate("any", (True, True, True)) + _evaluate("power", (True, True, True)) + + +def test_a_hard_failure_fails_under_both_gates(): + """No gate choice rescues an item that fails every run.""" + for gate in ("any", "power"): + with pytest.raises(GeneralQuestionAssertionError): + _evaluate(gate, (False, False, False)) diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index f6eecf8f9..6e493f4db 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -1046,13 +1046,9 @@ def test_evaluate_agentic_kda_skill_reports_trace_latency_when_kda_triggered(): assert wall_clock_calls[0].kwargs["value"] == 76.0 -def test_evaluate_agentic_kda_skill_does_not_log_pass_at_k_or_pass_power_k(): - # Matches metric_skill/alert_skill/guardrail/search_tool/general_question, which all - # compute pass_at_k/pass_power_k but never log them to Langfuse at their default k=1 -- - # nothing reads a kda_pass_at_1 score, and the score name shifts if k ever changes, - # silently splitting any Langfuse view built on the old name. Only visualization.py - # logs this pair, with a real consumer at k=2 (combo_report.py's viz_flaky) that - # justifies it. +def test_evaluate_agentic_kda_skill_does_not_log_k_suffixed_score_names(): + # `pass_at_2` becomes `pass_at_3` the moment K changes, splitting every Langfuse view built + # on the old name. Only visualization.py still writes that historic pair. mock_client = _client() mock_client.send_message.return_value = _kda_chat_result(success=True) @@ -1077,6 +1073,8 @@ def test_evaluate_agentic_kda_skill_does_not_log_pass_at_k_or_pass_power_k(): assert "kda_pass_power_2" not in logged assert "pass_at_2" not in logged assert "pass_power_2" not in logged + assert not [name for name in logged if name[-1].isdigit()] + assert {"pass_at_k", "pass_power_k", "gate_passed"} <= logged def test_run_agentic_kda_skill_accumulates_reasoning_steps_across_iterations(): diff --git a/packages/gooddata-eval/tests/test_langfuse_e2e_fake_server.py b/packages/gooddata-eval/tests/test_langfuse_e2e_fake_server.py index 09a18ae45..76feccb9e 100644 --- a/packages/gooddata-eval/tests/test_langfuse_e2e_fake_server.py +++ b/packages/gooddata-eval/tests/test_langfuse_e2e_fake_server.py @@ -154,9 +154,15 @@ def test_agentic_inline_path_polls_looks_up_exports_and_scores(fake_langfuse: Fa on_span = [b for b in bodies if b["traceId"] == span["traceId"]] assert len(bodies) == len(on_gen_ai) + len(on_span) assert {b["name"] for b in on_gen_ai} == {b["name"] for b in on_span} - assert {"general_question_pass", "llm_judge_score", "quality_score", "value_score"} == { - b["name"] for b in on_gen_ai - } + assert { + "general_question_pass", + "llm_judge_score", + "pass_at_k", + "pass_power_k", + "gate_passed", + "quality_score", + "value_score", + } == {b["name"] for b in on_gen_ai} assert all("observationId" not in b for b in on_gen_ai) assert all(b["observationId"] == span["spanId"] for b in on_span) @@ -227,6 +233,9 @@ def test_local_dataset_item_keeps_gen_ai_scores_and_warns_once(fake_langfuse: Fa assert {b["name"] for b in bodies} == { "general_question_pass", "llm_judge_score", + "pass_at_k", + "pass_power_k", + "gate_passed", "quality_score", "value_score", } @@ -341,9 +350,9 @@ def test_a_rate_limited_score_is_retried_and_lands(fake_langfuse: FakeLangfuse) _run_general_question(dataset_item_id="item-1", dataset_name="throttled") bodies = _score_bodies(fake_langfuse) - # Eight writes -- four scores on each of the gen-ai trace and the experiment span -- + # Fourteen writes -- seven scores on each of the gen-ai trace and the experiment span -- # plus the one refused attempt the client repeated. - assert len(bodies) == 9 + assert len(bodies) == 15 posted_twice = [b for b in bodies if bodies.count(b) == 2] assert len(posted_twice) == 2, "exactly one score body was posted twice" From 76470ba925c61da660da6213e0f87f7400a65cf6 Mon Sep 17 00:00:00 2001 From: "my.nguyen" Date: Thu, 10 Sep 2026 16:52:43 +0700 Subject: [PATCH 2/3] fix(gooddata-eval): keep gate out of the positional signature and in the merged report --- packages/gooddata-eval/README.md | 2 +- .../src/gooddata_eval/cli/main.py | 19 +++++ .../gooddata_eval/core/agentic/alert_skill.py | 2 +- .../core/agentic/general_question.py | 2 +- .../gooddata_eval/core/agentic/guardrail.py | 2 +- .../gooddata_eval/core/agentic/kda_skill.py | 2 +- .../core/agentic/metric_skill.py | 2 +- .../gooddata_eval/core/agentic/search_tool.py | 2 +- .../core/agentic/visualization.py | 2 +- .../gooddata-eval/tests/test_agentic_gate.py | 82 ++++++++++++++++++- 10 files changed, 107 insertions(+), 10 deletions(-) diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index ca1e6d371..03b3a3876 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -127,7 +127,7 @@ gd-eval run \ | Flag | Default | Description | |---|---|---| | `--runs K` | `2` | Independent runs per item. | -| `--gate` | `any` | Which verdict decides an item: `any` = pass@K (a run passing is enough), `power` = pass^K (every run must pass, so the verdict measures stability). Identical at `--runs 1`. Agentic kinds only. | +| `--gate` | `any` | Which verdict decides an item: `any` = pass@K (a run passing is enough), `power` = pass^K (every run must pass, so the verdict measures stability). Identical at `--runs 1`. Agentic kinds only — `power` is refused when the dataset also has non-agentic items, which are always decided on pass@K. | | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests — see *Concurrency and workspace safety* below. | | `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail` and `dashboard_summary`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | | `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. | diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 5359c245a..f60f1f1ac 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -196,6 +196,23 @@ def _apply_timer_flag(enabled: bool) -> None: os.environ[TIMERS_ENV_VAR] = "1" +def _reject_power_gate_on_non_agentic_items(config: RunConfig, non_agentic_items: list) -> None: + """Refuse a pass^K request the run cannot honour for every item. + + `run_items` has no gate: the non-agentic path always decides on pass@K. Running a mixed + dataset anyway would decide half the items under each rule and label the whole report + `power`. test_kind is resolved per item, so a dataset does not have to be homogeneous. + """ + if normalize_gate(config.gate) != "power" or not non_agentic_items: + return + kinds = sorted({i.test_kind for i in non_agentic_items}) + raise ValueError( + f"--gate power applies to agentic kinds only, but this dataset has {len(non_agentic_items)} " + f"item(s) of kind {kinds}, which are always decided on pass@K. Run them separately, or " + f"use --gate any." + ) + + def _warn_if_local_dataset_cannot_link(config: RunConfig, agentic_items: list) -> None: """Say up front that experiment assembly will fail, rather than after the run. @@ -358,6 +375,7 @@ def _run(config: RunConfig) -> int: items = _load_dataset(config) agentic_items = [i for i in items if i.test_kind in AGENTIC_TEST_KINDS] non_agentic_items = [i for i in items if i.test_kind not in AGENTIC_TEST_KINDS] + _reject_power_gate_on_non_agentic_items(config, non_agentic_items) _warn_if_local_dataset_cannot_link(config, agentic_items) models = config.models or [] run_ts = datetime.now(timezone.utc).strftime("%Y-%m-%d-%H-%M") @@ -482,6 +500,7 @@ def on_langfuse_item_done( provider_name=resolved.provider_name or resolved.provider_id, provider_type=resolved.provider_type, workspace_id=config.workspace_id, + gate=config.gate, ) if agentic_report is not None: report.items.extend(agentic_report.items) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 13fd8fbc8..1b5b18bb2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -791,7 +791,6 @@ def evaluate_agentic_alert_skill( question: str, expected_output: dict, k: int = _DEFAULT_K, - gate: EvalGate = DEFAULT_GATE, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, agent_id: str | None = None, @@ -803,6 +802,7 @@ def evaluate_agentic_alert_skill( run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, + gate: EvalGate = DEFAULT_GATE, ) -> AgenticEvalOutcome: """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index fd1671dc9..9265f6baa 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -221,7 +221,6 @@ def evaluate_agentic_general_question( question: str, expected_output: str, k: int = _DEFAULT_K, - gate: EvalGate = DEFAULT_GATE, initial_conversation_id: str | None = None, agent_id: str | None = None, langfuse: object | None = None, @@ -233,6 +232,7 @@ def evaluate_agentic_general_question( reasoning_effort: ReasoningEffort | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, user_context: dict | None = None, + gate: EvalGate = DEFAULT_GATE, ) -> AgenticEvalOutcome: """Run general-question evaluation, log to Langfuse, and raise GeneralQuestionAssertionError on failure. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index 5282dab8d..817f6538b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -194,7 +194,6 @@ def evaluate_agentic_guardrail( question: str, expected_output: str, k: int = _DEFAULT_K, - gate: EvalGate = DEFAULT_GATE, initial_conversation_id: str | None = None, agent_id: str | None = None, langfuse: object | None = None, @@ -205,6 +204,7 @@ def evaluate_agentic_guardrail( run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, + gate: EvalGate = DEFAULT_GATE, ) -> AgenticEvalOutcome: """Run guardrail evaluation, log to Langfuse, and raise GuardrailAssertionError on failure. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index efdfe5759..3898e80d9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -380,7 +380,6 @@ def evaluate_agentic_kda_skill( question: str, expected_output: dict, k: int = _DEFAULT_K, - gate: EvalGate = DEFAULT_GATE, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, agent_id: str | None = None, @@ -392,6 +391,7 @@ def evaluate_agentic_kda_skill( run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, + gate: EvalGate = DEFAULT_GATE, ) -> AgenticEvalOutcome: """Run KDA-skill evaluation, log to Langfuse, and raise KdaSkillAssertionError on failure. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 64d181ce0..6ad68cd59 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -413,7 +413,6 @@ def evaluate_agentic_metric_skill( question: str, expected_output: dict | list, k: int = _DEFAULT_K, - gate: EvalGate = DEFAULT_GATE, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, agent_id: str | None = None, @@ -425,6 +424,7 @@ def evaluate_agentic_metric_skill( run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, + gate: EvalGate = DEFAULT_GATE, ) -> AgenticEvalOutcome: """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index 17fd699cc..ac70deeb7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -175,7 +175,6 @@ def evaluate_agentic_search_tool( question: str, expected_tool_call: dict, k: int = _DEFAULT_K, - gate: EvalGate = DEFAULT_GATE, initial_conversation_id: str | None = None, agent_id: str | None = None, langfuse: object | None = None, @@ -186,6 +185,7 @@ def evaluate_agentic_search_tool( run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, + gate: EvalGate = DEFAULT_GATE, ) -> AgenticEvalOutcome: """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 4c8601fe2..3b680debc 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -322,7 +322,6 @@ def evaluate_agentic_visualization( question: str, expected_outputs: list[CreatedVisualization], k: int = _DEFAULT_K, - gate: EvalGate = DEFAULT_GATE, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, agent_id: str | None = None, @@ -335,6 +334,7 @@ def evaluate_agentic_visualization( record_output_path: str | None = None, reasoning_effort: ReasoningEffort | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, + gate: EvalGate = DEFAULT_GATE, ) -> AgenticEvalOutcome: """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure. diff --git a/packages/gooddata-eval/tests/test_agentic_gate.py b/packages/gooddata-eval/tests/test_agentic_gate.py index e1fa11b1f..9fc65a85d 100644 --- a/packages/gooddata-eval/tests/test_agentic_gate.py +++ b/packages/gooddata-eval/tests/test_agentic_gate.py @@ -2,9 +2,12 @@ """The K gate: which of pass@K / pass^K decides an item, and what reaches Langfuse.""" import contextlib +import importlib +import inspect from unittest.mock import MagicMock, patch import pytest +from gooddata_eval.cli.main import _reject_power_gate_on_non_agentic_items from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, gate_failure_note, @@ -17,8 +20,8 @@ GeneralQuestionAssertionError, evaluate_agentic_general_question, ) -from gooddata_eval.core.config import normalize_gate -from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.config import RunConfig, normalize_gate +from gooddata_eval.core.models import ChatResult, DatasetItem def test_the_default_gate_is_the_historic_pass_at_k(): @@ -169,3 +172,78 @@ def test_a_hard_failure_fails_under_both_gates(): for gate in ("any", "power"): with pytest.raises(GeneralQuestionAssertionError): _evaluate(gate, (False, False, False)) + + +# --------------------------------------------------------------------------- # +# signature compatibility — `gate` must not shift an existing positional argument +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + "module_name", + [ + "visualization", + "metric_skill", + "alert_skill", + "kda_skill", + "general_question", + "guardrail", + "search_tool", + ], +) +def test_gate_is_the_last_parameter_of_every_evaluator(module_name): + """Inserted anywhere earlier, a positional caller binds max_iterations or + initial_conversation_id to `gate`, which then reaches normalize_gate and raises.""" + module = importlib.import_module(f"gooddata_eval.core.agentic.{module_name}") + fn = next(v for k, v in vars(module).items() if k.startswith("evaluate_agentic_")) + names = [p.name for p in inspect.signature(fn).parameters.values()] + + assert names[-1] == "gate" + assert names[5] == "k" + assert names[6] in ("max_iterations", "initial_conversation_id") + + +def test_a_positional_seventh_argument_still_binds_where_it_used_to(): + """The regression the parameter order protects: 7 positional args, no keywords.""" + with _judged(True): + evaluate_agentic_general_question( + "https://example.com", # host + "tok", # token + "ws", # workspace_id + "Q", # question + "rubric", # expected_output + 1, # k + "conv-0", # initial_conversation_id -- NOT gate + ) + + +# --------------------------------------------------------------------------- # +# mixed datasets — run_items has no gate, so pass^K cannot be honoured for it +# --------------------------------------------------------------------------- # +def _item(test_kind, item_id="i1"): + return DatasetItem(id=item_id, dataset_name="d", test_kind=test_kind, question="q", expected_output="e") + + +def _config(gate): + return RunConfig(host="https://h", token="t", workspace_id="w", gate=gate) + + +def test_power_gate_is_refused_when_the_dataset_has_non_agentic_items(): + """test_kind is resolved per item, so a dataset can mix the two paths. Running anyway + would decide half the items on pass@K and still label the report `power`.""" + with pytest.raises(ValueError, match="applies to agentic kinds only"): + _reject_power_gate_on_non_agentic_items(_config("power"), [_item("visualization")]) + + +def test_the_refusal_names_the_kinds_to_split_out(): + with pytest.raises(ValueError) as exc: + _reject_power_gate_on_non_agentic_items(_config("power"), [_item("visualization", "i1"), _item("search", "i2")]) + assert "2 item(s)" in str(exc.value) + assert "['search', 'visualization']" in str(exc.value) + + +def test_a_purely_agentic_dataset_is_accepted_under_the_power_gate(): + _reject_power_gate_on_non_agentic_items(_config("power"), []) + + +def test_the_default_gate_accepts_a_mixed_dataset(): + """pass@K is what the non-agentic path already does, so nothing is misrepresented.""" + _reject_power_gate_on_non_agentic_items(_config("any"), [_item("visualization")]) From 454286c9265e3f57c9d929e759c363fe40cf3e90 Mon Sep 17 00:00:00 2001 From: "my.nguyen" Date: Thu, 10 Sep 2026 19:40:42 +0700 Subject: [PATCH 3/3] fix(gooddata-eval): report the gate verdict without redefining pass@K jira: QA-29251 risk: low --- packages/gooddata-eval/README.md | 13 +- .../src/gooddata_eval/cli/agentic_runner.py | 21 ++- .../src/gooddata_eval/cli/main.py | 33 ++-- .../src/gooddata_eval/core/agentic/_gate.py | 12 +- .../core/agentic/general_question.py | 2 +- .../gooddata_eval/core/agentic/guardrail.py | 2 +- .../gooddata_eval/core/reporting/console.py | 14 +- .../core/reporting/json_report.py | 7 +- .../src/gooddata_eval/core/runner.py | 21 ++- .../gooddata-eval/tests/test_agentic_gate.py | 148 +++++++++++++++++- 10 files changed, 239 insertions(+), 34 deletions(-) diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index 03b3a3876..2aa2d67bf 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -127,7 +127,7 @@ gd-eval run \ | Flag | Default | Description | |---|---|---| | `--runs K` | `2` | Independent runs per item. | -| `--gate` | `any` | Which verdict decides an item: `any` = pass@K (a run passing is enough), `power` = pass^K (every run must pass, so the verdict measures stability). Identical at `--runs 1`. Agentic kinds only — `power` is refused when the dataset also has non-agentic items, which are always decided on pass@K. | +| `--gate` | `any` | Which verdict decides an item: `any` = pass@K (a run passing is enough), `power` = pass^K (every run must pass, so the verdict measures stability). Identical at `--runs 1`. Kinds that repeat K runs only — `power` is refused when the dataset also has non-agentic items or `agentic_conversation`, which are always decided on pass@K. | | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests — see *Concurrency and workspace safety* below. | | `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail` and `dashboard_summary`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | | `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. | @@ -232,10 +232,12 @@ Winner is selected by **pass rate → quality score → latency** (lower latency Each item reports **how many of its runs passed**, not only whether one did: ```json -"runs": 5, "runs_passed": 4, "pass_at_k": true, "pass_power_k": false +"runs": 5, "runs_passed": 4, "pass_at_k": true, "pass_power_k": false, "gate_passed": true ``` -`pass_at_k` is "did any run pass" and is what `passed` counts. `runs_passed` is the fact that separates a +`pass_at_k` is "did any run pass" — always literal, whatever the gate. `gate_passed` is the verdict the item +was decided on and is what `passed` counts; the two differ only under `--gate power`, where an item that +passed 4 of 5 runs is `"pass_at_k": true, "gate_passed": false`. `runs_passed` is the fact that separates a reliable item from a coin-flip — without it a 5/5 item and a 1/5 item are identical in every field, because `quality_score` is derived from the best run alone. `pass_power_k` is true only when every run passed, and the run summary carries `passed_all_runs` beside `passed`; a large gap between the two means the model is @@ -248,7 +250,10 @@ no K and drives its fixture exactly once. Which of the two decides pass/fail is `--gate`: `any` (default) gates on `pass_at_k`, `power` gates on `pass_power_k`. The run records it as a top-level `gate`, and a failure under `power` says so — `Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure` — because the message body describes -the best run, which under pass^K can be a run that passed. `agentic_conversation` has no K gate. +the best run, which under pass^K can be a run that passed, and the console repeats the count in `Notes` for +the same reason. When some runs were ungraded the note says so instead of calling the remainder unstable. +`agentic_conversation` has no K gate, so `--gate power` is refused for a dataset containing one rather than +labelling a report `power` that only part of the dataset was decided under. Each item additionally carries a per-phase breakdown: diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 4ce3f6f09..129f4c5cf 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -49,6 +49,13 @@ class _LfKw(TypedDict, total=False): ) +# Agentic kinds that no gate applies to: they drive their fixture exactly once, so there is +# no K to take pass@K or pass^K over. Named here rather than inline in _dispatch_agentic so +# the CLI can refuse --gate power for a dataset containing one instead of labelling the whole +# report `power` when part of it was never gated. +UNGATED_AGENTIC_TEST_KINDS = frozenset({"agentic_conversation"}) + + # Kinds cleared to run several at a time. An EXPLICIT allowlist, not a subtraction: nothing # in this package can prove a kind is read-only, because the mutation happens server-side in # whichever tools the agent decides to call. So each entry here is a reviewed judgement, and @@ -133,7 +140,8 @@ def _dispatch_agentic( ) -> AgenticEvalOutcome: """Call the appropriate evaluate_agentic_* function for the item's test_kind. - `gate` reaches every kind except agentic_conversation, which has no K to gate over. + `gate` reaches every kind except those in UNGATED_AGENTIC_TEST_KINDS, which have no K + to gate over; the CLI refuses --gate power for a dataset containing one. Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps, conversation_id, response_id, detail) on success and attaches the same four attributes @@ -337,6 +345,9 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: test_kind=item.test_kind, question=item.question, ) + # None, not False, for the kinds _dispatch_agentic passes no gate to: ItemReport.passed + # then falls back to pass_at_k and gate_passed keeps meaning "a gate ran". + gated = item.test_kind not in UNGATED_AGENTIC_TEST_KINDS t0 = time.perf_counter() try: outcome = _dispatch_agentic( @@ -360,6 +371,8 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: detail = outcome.detail else: reasoning_steps, conversation_id, response_id, detail = outcome, None, None, {} + item_report.gate_passed = True if gated else None + # Whichever gate decided the item, clearing it means at least one run passed. item_report.pass_at_k = True item_report.runs = k item_report.reasoning_steps = reasoning_steps or [] @@ -369,7 +382,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: _apply_timings(item_report, getattr(outcome, "timings", None)) _apply_run_counts(item_report, outcome) except AssertionError as exc: - item_report.pass_at_k = False + item_report.gate_passed = False if gated else None item_report.runs = k item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or [] item_report.conversation_id = getattr(exc, "conversation_id", None) @@ -377,6 +390,10 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: item_report.best_detail = getattr(exc, "detail", None) or {} _apply_timings(item_report, getattr(exc, "timings", None)) _apply_run_counts(item_report, exc) + # Read off the counts, not off the gate: pass^K fails items where runs did pass, + # and reporting those as pass_at_k False would contradict the Langfuse score of + # the same name. Kinds that report no count read as 0, i.e. a clean failure. + item_report.pass_at_k = item_report.runs_passed > 0 print(f"[agentic] {item.id} FAIL: {exc}", flush=True) except Exception as exc: item_report.error = f"{type(exc).__name__}: {exc}" diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index f60f1f1ac..1966f482e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -14,7 +14,7 @@ from rich.console import Console from rich.table import Table -from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items +from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, UNGATED_AGENTIC_TEST_KINDS, run_agentic_items from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ( DEFAULT_GATE, @@ -27,6 +27,7 @@ ) from gooddata_eval.core.connection import ConnectionError_, resolve_connection from gooddata_eval.core.dataset.local import load_local_dataset +from gooddata_eval.core.evaluators import supported_test_kinds from gooddata_eval.core.langfuse.sink import LangfuseSink from gooddata_eval.core.models import ChatResult, DatasetItem from gooddata_eval.core.reporting.console import render_comparison, render_console @@ -196,18 +197,32 @@ def _apply_timer_flag(enabled: bool) -> None: os.environ[TIMERS_ENV_VAR] = "1" -def _reject_power_gate_on_non_agentic_items(config: RunConfig, non_agentic_items: list) -> None: +def _reject_power_gate_on_ungated_items(config: RunConfig, items: list) -> None: """Refuse a pass^K request the run cannot honour for every item. - `run_items` has no gate: the non-agentic path always decides on pass@K. Running a mixed - dataset anyway would decide half the items under each rule and label the whole report + Two kinds of item are never gated: everything on the non-agentic path, because + `run_items` has no gate and always decides on pass@K, and agentic_conversation, which + drives its fixture once whatever --runs says and so has no K to gate over. Running a + mixed dataset anyway would decide part of it under each rule and label the whole report `power`. test_kind is resolved per item, so a dataset does not have to be homogeneous. + + Kinds no evaluator supports are not counted: those items are skipped rather than + decided, so refusing on them would make --gate power fail where --gate any runs. """ - if normalize_gate(config.gate) != "power" or not non_agentic_items: + if normalize_gate(config.gate) != "power": + return + supported = supported_test_kinds() + ungated = [ + i + for i in items + if i.test_kind in UNGATED_AGENTIC_TEST_KINDS + or (i.test_kind not in AGENTIC_TEST_KINDS and i.test_kind in supported) + ] + if not ungated: return - kinds = sorted({i.test_kind for i in non_agentic_items}) + kinds = sorted({i.test_kind for i in ungated}) raise ValueError( - f"--gate power applies to agentic kinds only, but this dataset has {len(non_agentic_items)} " + f"--gate power applies to kinds that repeat K runs, but this dataset has {len(ungated)} " f"item(s) of kind {kinds}, which are always decided on pass@K. Run them separately, or " f"use --gate any." ) @@ -285,7 +300,7 @@ def on_item_done(index: int, total: int, report: ItemReport) -> None: tag = "[yellow]SKIP[/yellow]" elif report.error: tag = "[red]ERR [/red]" - elif report.pass_at_k: + elif report.passed: tag = "[green]PASS[/green]" else: tag = "[red]FAIL[/red]" @@ -375,7 +390,7 @@ def _run(config: RunConfig) -> int: items = _load_dataset(config) agentic_items = [i for i in items if i.test_kind in AGENTIC_TEST_KINDS] non_agentic_items = [i for i in items if i.test_kind not in AGENTIC_TEST_KINDS] - _reject_power_gate_on_non_agentic_items(config, non_agentic_items) + _reject_power_gate_on_ungated_items(config, items) _warn_if_local_dataset_cannot_link(config, agentic_items) models = config.models or [] run_ts = datetime.now(timezone.utc).strftime("%Y-%m-%d-%H-%M") diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py index 09271174a..63980bb07 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py @@ -29,16 +29,22 @@ def gate_label(gate: str | None, k: int) -> str: return f"pass^{k}" if normalize_gate(gate) == "power" else f"pass@{k}" -def gate_failure_note(gate: str | None, runs_passed: int, runs_total: int) -> str: +def gate_failure_note(gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0) -> str: """Gate and how many runs met it, for the assertion message. Needed because the message body describes the BEST run, which under pass^K can be a run that passed — so the reported detail on its own looks like a pass. + + An ungraded run counts in runs_total but can never count in runs_passed, so the remainder + is not evidence of instability — saying so would blame the agent for a judge outage. """ label = gate_label(gate, runs_total) + note = f"Gate {label} failed: {runs_passed}/{runs_total} runs passed" + if runs_ungraded: + return f"{note}, {runs_ungraded} ungraded." if normalize_gate(gate) == "power" and runs_passed: - return f"Gate {label} failed: {runs_passed}/{runs_total} runs passed — unstable, not a clean failure." - return f"Gate {label} failed: {runs_passed}/{runs_total} runs passed." + return f"{note} — unstable, not a clean failure." + return f"{note}." def log_gate_scores(ctx: Any, trace_id: Any, *, gate: str | None, pass_at_k: bool, pass_power_k: bool) -> None: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 9265f6baa..8c8fb3727 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -336,7 +336,7 @@ def _write_scores(ctx: RunTraceContext) -> None: } if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) exc = GeneralQuestionAssertionError( f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}" ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index 817f6538b..255a57299 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -302,7 +302,7 @@ def _write_scores(ctx: RunTraceContext) -> None: } if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) exc = GuardrailAssertionError( f"Guardrail assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}" ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py index 3acf69100..31759f7ff 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py @@ -4,6 +4,7 @@ from rich.console import Console from rich.table import Table +from gooddata_eval.core.agentic._gate import gate_label from gooddata_eval.core.runner import EvalReport, ItemReport @@ -38,7 +39,7 @@ def render_console(report: EvalReport, *, console: Console | None = None) -> str result, notes = "SKIPPED", f"test_kind '{item.test_kind}' not supported in this phase" elif item.error: result, notes = "ERROR", item.error - elif item.pass_at_k: + elif item.passed: # A pass@K that was not unanimous is a materially weaker result than one that # was, and every other column looks identical for the two: quality_score reads # best_detail, which describes the winning run alone. So say it here. @@ -49,7 +50,16 @@ def render_console(report: EvalReport, *, console: Console | None = None) -> str # (visualization uses metrics_correct/…; dashboard_summary uses # include_*/exclude_*/rubric_*). Falls back to a generic message. failing = [k for k, v in item.best_detail.items() if v is False] - notes = "failed: " + ", ".join(failing) if failing else "did not pass strict checks" + if item.runs_passed: + # Only pass^K fails an item whose runs passed, and best_detail then + # describes one of those -- so no check reads False and Quality prints + # 100%. The count is the whole reason the row is a FAIL. + label = gate_label(report.gate, item.runs_total) + notes = f"{label} failed: {item.runs_passed}/{item.runs_total} runs passed" + elif failing: + notes = "failed: " + ", ".join(failing) + else: + notes = "did not pass strict checks" result = "FAIL" if result in ("PASS", "FAIL") and (ungraded := _ungraded_note(item)): # Said on both verdicts: a PASS over fewer runs is weaker evidence, and a FAIL diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py index 98c3dc0e7..72b3dc349 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py @@ -23,7 +23,7 @@ def _build_run_dict(report: EvalReport) -> dict: # Counted explicitly rather than by subtraction: an errored item has # pass_at_k False and skipped False, so subtraction would count it as both a # failure and an error. A judge fault is an error, not K failures. - "failed": sum(1 for i in report.items if not i.pass_at_k and not i.skipped and i.error is None), + "failed": sum(1 for i in report.items if not i.passed and not i.skipped and i.error is None), "skipped": report.skipped, "errored": report.errored, "latency_s": round(report.latency_s, 3), @@ -35,7 +35,12 @@ def _build_run_dict(report: EvalReport) -> dict: "dataset_name": item.dataset_name, "test_kind": item.test_kind, "question": item.question, + # Both, because under --gate power they differ: pass_at_k stays literal + # (any run passed) and matches the Langfuse score of that name, while + # gate_passed is the verdict the run was decided on and drives the summary + # counts and the exit code. "pass_at_k": item.pass_at_k, + "gate_passed": item.passed, "skipped": item.skipped, "error": item.error, # What actually ran, not the requested K: agentic_conversation runs once. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index 9fa15bf5f..039361d2a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -27,6 +27,11 @@ class ItemReport: test_kind: str question: str pass_at_k: bool = False + # An explicit gate verdict, where a gate ran: pass^K under --gate power. None means no + # gate ran -- run_items has none -- and `passed` falls back to pass_at_k. Kept apart + # from pass_at_k so that field stays literal and the JSON report never contradicts the + # Langfuse score of the same name. + gate_passed: bool | None = None skipped: bool = False error: str | None = None runs: int = 0 @@ -63,6 +68,15 @@ class ItemReport: # trusting K there reports four runs that never happened. runs_effective: int | None = None + @property + def passed(self) -> bool: + """Whether the item counts as a pass: the gate's verdict where one ran, else pass@K. + + pass_at_k stays literal beside it -- under --gate power an item that passed 2 of 3 + runs has pass_at_k True and this False. + """ + return self.pass_at_k if self.gate_passed is None else self.gate_passed + @property def runs_total(self) -> int: """What the item actually ran: the kind's own count when it has one, else K.""" @@ -91,13 +105,13 @@ def avg_latency_s(self) -> float: def quality_score(self) -> float: """Fraction of bool-valued strict checks in best_detail that are True. - Falls back to 1.0 if pass_at_k else 0.0 when no bool checks exist + Falls back to 1.0 if the gate passed else 0.0 when no bool checks exist (e.g. text evaluators where best_detail has no bool flags). """ checks = {k: v for k, v in self.best_detail.items() if isinstance(v, bool)} if checks: return sum(1 for v in checks.values() if v) / len(checks) - return 1.0 if self.pass_at_k else 0.0 + return 1.0 if self.passed else 0.0 @dataclass @@ -116,7 +130,8 @@ def total(self) -> int: @property def passed(self) -> int: - return sum(1 for i in self.items if i.pass_at_k) + """Items that counted as a pass -- pass@K, or pass^K under --gate power.""" + return sum(1 for i in self.items if i.passed) @property def skipped(self) -> int: diff --git a/packages/gooddata-eval/tests/test_agentic_gate.py b/packages/gooddata-eval/tests/test_agentic_gate.py index 9fc65a85d..e79af9093 100644 --- a/packages/gooddata-eval/tests/test_agentic_gate.py +++ b/packages/gooddata-eval/tests/test_agentic_gate.py @@ -7,7 +7,8 @@ from unittest.mock import MagicMock, patch import pytest -from gooddata_eval.cli.main import _reject_power_gate_on_non_agentic_items +from gooddata_eval.cli.agentic_runner import run_agentic_items +from gooddata_eval.cli.main import _reject_power_gate_on_ungated_items from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, gate_failure_note, @@ -16,12 +17,16 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError from gooddata_eval.core.agentic.general_question import ( GeneralQuestionAssertionError, evaluate_agentic_general_question, ) from gooddata_eval.core.config import RunConfig, normalize_gate -from gooddata_eval.core.models import ChatResult, DatasetItem +from gooddata_eval.core.models import AgenticEvalOutcome, ChatResult, DatasetItem +from gooddata_eval.core.reporting.console import render_console +from gooddata_eval.core.reporting.json_report import build_json_report +from gooddata_eval.core.runner import EvalReport, ItemReport def test_the_default_gate_is_the_historic_pass_at_k(): @@ -86,6 +91,14 @@ def test_failure_note_does_not_call_a_clean_failure_unstable(): assert "0/1" in gate_failure_note("any", 0, 1) +def test_failure_note_blames_the_judge_not_the_agent_for_an_ungraded_run(): + """An ungraded run is in runs_total but can never be in runs_passed, so calling the + remainder instability reports a judge outage as a flaky agent.""" + note = gate_failure_note("power", 1, 2, 1) + assert "1 ungraded" in note + assert "unstable" not in note + + def test_gate_scores_use_K_stable_names(): """`pass_at_2` becomes `pass_at_3` the moment K changes, splitting every view on it.""" ctx = MagicMock() @@ -229,21 +242,140 @@ def _config(gate): def test_power_gate_is_refused_when_the_dataset_has_non_agentic_items(): """test_kind is resolved per item, so a dataset can mix the two paths. Running anyway would decide half the items on pass@K and still label the report `power`.""" - with pytest.raises(ValueError, match="applies to agentic kinds only"): - _reject_power_gate_on_non_agentic_items(_config("power"), [_item("visualization")]) + with pytest.raises(ValueError, match="applies to kinds that repeat K runs"): + _reject_power_gate_on_ungated_items(_config("power"), [_item("visualization")]) + + +def test_power_gate_is_refused_for_an_agentic_kind_that_takes_no_k(): + """agentic_conversation is in AGENTIC_TEST_KINDS but _dispatch_agentic passes it no + gate, so accepting it labels a report `power` that it was not decided under.""" + with pytest.raises(ValueError, match="agentic_conversation"): + _reject_power_gate_on_ungated_items( + _config("power"), [_item("agentic_visualization", "i1"), _item("agentic_conversation", "i2")] + ) + + +def test_an_unsupported_kind_is_skipped_rather_than_refused(): + """run_items skips it with a warning under --gate any; refusing here would make one + misspelled test_kind abort a dataset the default gate runs in full.""" + _reject_power_gate_on_ungated_items( + _config("power"), [_item("agentic_visualization", "i1"), _item("nonsense_kind", "i2")] + ) def test_the_refusal_names_the_kinds_to_split_out(): with pytest.raises(ValueError) as exc: - _reject_power_gate_on_non_agentic_items(_config("power"), [_item("visualization", "i1"), _item("search", "i2")]) + _reject_power_gate_on_ungated_items( + _config("power"), [_item("visualization", "i1"), _item("search_tool", "i2")] + ) assert "2 item(s)" in str(exc.value) - assert "['search', 'visualization']" in str(exc.value) + assert "['search_tool', 'visualization']" in str(exc.value) def test_a_purely_agentic_dataset_is_accepted_under_the_power_gate(): - _reject_power_gate_on_non_agentic_items(_config("power"), []) + _reject_power_gate_on_ungated_items(_config("power"), [_item("agentic_visualization")]) + _reject_power_gate_on_ungated_items(_config("power"), []) def test_the_default_gate_accepts_a_mixed_dataset(): """pass@K is what the non-agentic path already does, so nothing is misrepresented.""" - _reject_power_gate_on_non_agentic_items(_config("any"), [_item("visualization")]) + _reject_power_gate_on_ungated_items(_config("any"), [_item("visualization")]) + + +# --------------------------------------------------------------------------- # +# reporting — the gate verdict and pass@K are different facts and both are reported +# --------------------------------------------------------------------------- # +def _flaky_item(): + """2 of 3 runs passed: pass@K true, pass^K false, and the best run is a passing one.""" + return ItemReport( + id="i1", + dataset_name="d", + test_kind="agentic_general_question", + question="q", + pass_at_k=True, + gate_passed=False, + runs=3, + runs_passed=2, + best_detail={"general_question_pass": True}, + ) + + +def test_pass_at_k_stays_literal_when_the_power_gate_fails_the_item(): + """The Langfuse score named pass_at_k is summary.pass_at_k, so a JSON field of the same + name reporting the gate instead makes two outputs of one run disagree.""" + data = build_json_report(EvalReport(model="m", gate="power", items=[_flaky_item()])) + + assert data["items"]["i1"]["pass_at_k"] is True + assert data["items"]["i1"]["gate_passed"] is False + assert data["summary"]["passed"] == 0 + assert data["summary"]["failed"] == 1 + + +def test_an_ungated_item_reports_pass_at_k_as_its_verdict(): + """run_items has no gate, so nothing sets gate_passed and pass@K decides.""" + item = ItemReport(id="i1", dataset_name="d", test_kind="visualization", question="q", pass_at_k=True, runs=2) + data = build_json_report(EvalReport(model="m", items=[item])) + + assert data["items"]["i1"]["gate_passed"] is True + assert data["summary"]["passed"] == 1 + + +def test_the_console_says_why_a_power_gate_failure_is_a_failure(): + """best_detail describes the best run, which under pass^K passed -- so no check reads + False and Quality prints 100%. Without the count the row is an unexplained FAIL.""" + text = render_console(EvalReport(model="m", gate="power", items=[_flaky_item()])) + + assert "pass^3 failed" in text + assert "2/3 runs passed" in text + assert "did not pass strict checks" not in text + + +# --------------------------------------------------------------------------- # +# ungated kinds — gate_passed must stay None so it keeps meaning "a gate ran" +# --------------------------------------------------------------------------- # +def _run_one(test_kind: str, **dispatch): + """One item through run_agentic_items with _dispatch_agentic stubbed. + + Stubbed at the dispatch seam rather than at an evaluator: the kinds differ in what they + load from the item, and what is under test is how _process_item records the verdict. + """ + with patch("gooddata_eval.cli.agentic_runner._dispatch_agentic", **dispatch): + report = run_agentic_items( + [_item(test_kind, "i1")], + host="http://h", + token="tok", + workspace_id="ws", + k=2, + run_ts="2026-01-01", + gate="power", + ) + return report.items[0] + + +def test_an_ungated_kind_records_no_gate_verdict(): + """_dispatch_agentic passes agentic_conversation no gate, so a Boolean here would claim + a gate decided the item and make an ungated result indistinguishable from a gated one.""" + outcome = AgenticEvalOutcome(reasoning_steps=[], conversation_id="c-1", response_id="r-1", detail={}) + item = _run_one("agentic_conversation", return_value=outcome) + + assert item.error is None + assert item.gate_passed is None + assert (item.pass_at_k, item.passed) == (True, True) # passed falls back to pass@K + + +def test_an_ungated_kind_records_no_gate_verdict_on_failure_either(): + item = _run_one("agentic_conversation", side_effect=AssertionError("nope")) + + assert item.gate_passed is None + assert (item.pass_at_k, item.passed) == (False, False) + + +def test_a_gated_kind_records_the_verdict_the_gate_produced(): + exc = AlertSkillAssertionError("nope") + exc.runs_passed = 1 + exc.detail = {"alert_created": True} + item = _run_one("agentic_alert_skill", side_effect=exc) + + assert item.gate_passed is False + assert item.pass_at_k is True # 1 of 2 runs passed: pass^2 failed, pass@2 did not + assert item.passed is False