From 1d9d7ed03f59f686aa8ed96a59b6028ce42e5d7f Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 10 Sep 2026 08:20:20 +0200 Subject: [PATCH 1/6] feat(gooddata-eval): add the agentic forecasting evaluator The forecasting skill is enabled on the eval org and reachable today (confirmed live: set_skills activates "forecasting"), but nothing evaluates it. Unlike kda_skill this is not limited to "the process ran". The skill refuses to execute unless the visualization carries an AAC forecast config, and that config is exactly where the user's request lands: config.forecast_enabled must be true or execute_forecast errors config.forecast_period "next 3 months" is 3 config.forecast_confidence confidence level config.forecast_seasonal whether seasonality is modelled So a fixture states what it asked for and it is checked exactly, off the numbers in the tool call the agent made -- no judge and no paraphrase tolerance. The measure forecast is checked the same way, from the visualization's own fields. expected_output pins whatever it wants: {"metric": "metric/spend", "forecast_period": 3} An unstated expectation passes rather than fails, and detail["asserted"] records which checks the fixture actually pinned -- otherwise a run that verified nothing reads identically to one where everything matched. The loop follows kda_skill: the agent routinely asks which measure to forecast before building anything (observed live: "your data has two different Spend metrics"), so a simulated user answers from the fixture's own hints, and only hints the fixture supplies reach the prompt. LoopExit is deliberately not used -- it lands with #1789, which is still open. This should gain exit_reason once that merges. 21 tests. 801 passed, lint and format clean. Co-Authored-By: Claude Opus 5 --- .../src/gooddata_eval/cli/agentic_runner.py | 13 + .../gooddata_eval/core/agentic/forecasting.py | 539 ++++++++++++++++++ .../tests/test_agentic_forecasting.py | 295 ++++++++++ .../tests/test_agentic_runner.py | 1 + .../gooddata-eval/tests/test_trace_linker.py | 1 + 5 files changed, 849 insertions(+) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py create mode 100644 packages/gooddata-eval/tests/test_agentic_forecasting.py diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 0bd6f5cf8..cdabd8a7a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -12,6 +12,7 @@ from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation +from gooddata_eval.core.agentic.forecasting import evaluate_agentic_forecasting from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill @@ -44,6 +45,7 @@ class _LfKw(TypedDict, total=False): "agentic_guardrail", "agentic_conversation", "agentic_kda_skill", + "agentic_forecasting", } ) @@ -228,6 +230,17 @@ def _dispatch_agentic( agent_id=agent_id, **lf_kw, ) + elif kind == "agentic_forecasting": + return evaluate_agentic_forecasting( + host=host, + token=token, + workspace_id=workspace_id, + question=item.question, + expected_output=eo if isinstance(eo, dict) else {}, + k=k, + agent_id=agent_id, + **lf_kw, + ) elif kind == "agentic_conversation": fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {} return evaluate_agentic_conversation( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py new file mode 100644 index 000000000..91b2cfc6a --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py @@ -0,0 +1,539 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Agentic forecasting-skill evaluation runner. + +Unlike kda_skill, this is not limited to "the process ran". The forecasting skill refuses to +execute unless the visualization it points at carries an AAC forecast config, and that config +is where the user's request lands: + + config.forecast_enabled must be true or execute_forecast returns an error + config.forecast_period how many periods ahead -- "next 3 months" is 3 + config.forecast_confidence confidence level, 0.95 by default + config.forecast_seasonal whether seasonality is modelled + +So a fixture can state what it asked for and have it checked exactly, with no judge: the +period the agent chose is a number in the tool call it made. The measure it forecast is +checkable the same way, off the visualization's own fields. +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass, field +from typing import Any + +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.render import render_answer_text +from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ChatResult, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, + shift_and_index_events, +) + +_log = logging.getLogger(__name__) + +_DEFAULT_K = 1 +# The agent routinely asks which measure to forecast before it builds anything (observed +# live: "your data has two different Spend metrics"), so the budget covers a couple of +# disambiguation rounds plus slack, matching kda_skill's reasoning. +_DEFAULT_MAX_ITERATIONS = 4 + + +def _build_clarification_prompt(agent_message: str, expected_output: dict) -> str: + """The simulated-user reply, mentioning only the hints the fixture actually supplies. + + An absent hint is dropped from the prompt entirely rather than asserted as a literal + "None", which would answer a question the agent never asked with a wrong value. + """ + hints: list[str] = [] + metric = expected_output.get("metric") + if metric: + hints.append(f"the measure to forecast is {metric}") + period = expected_output.get("forecast_period") + if period is not None: + hints.append(f"the forecast horizon is {period} periods ahead") + granularity = expected_output.get("granularity") + if granularity: + hints.append(f"the time granularity is {granularity}") + reference = "; ".join(hints) + return ( + f"You are simulating a user in a conversation with a BI assistant that forecasts metric " + f"values. The assistant asked: '{agent_message}'. " + + (f"For reference, {reference}. " if reference else "") + + "Reply briefly as the user, answering whichever of those the assistant actually asked about." + ) + + +def generate_simulated_forecast_response(agent_message: str, expected_output: dict) -> str: + """Generate a user reply to keep the forecasting conversation going (gpt-4o-mini). + + Always OpenAI regardless of the workspace's own model: this is harness plumbing, not the + system under test. + """ + try: + from openai import OpenAI # noqa: PLC0415 + except ImportError as exc: + raise RuntimeError("openai package is required for generate_simulated_forecast_response") from exc + + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + raise OSError("OPENAI_API_KEY environment variable is not set") + + client = OpenAI(api_key=api_key) + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[{"role": "user", "content": _build_clarification_prompt(agent_message, expected_output)}], + max_tokens=150, + temperature=0, + timeout=30, + ) + return response.choices[0].message.content or "Please proceed with the most complete option." + + +def _extract_forecast_calls( + tool_call_events: list[ToolCallEvent], +) -> tuple[dict | None, dict | None]: + """Return (visualization_args, execute_result) for the LAST create/execute pair. + + A new create_adhoc_visualization clears any earlier execute result: that result belongs + to the visualization it followed, not to this one. Picking the last of each + independently would pair a fresh chart with a stale forecast. + """ + viz_args: dict | None = None + execute_result: dict | None = None + for tc in tool_call_events: + if tc.function_name == "create_adhoc_visualization": + args = tc.parsed_arguments() or {} + viz = args.get("visualization") + viz_args = viz if isinstance(viz, dict) else args + execute_result = None + elif tc.function_name == "execute_forecast" and tc.result: + execute_result = tc.parsed_result() + return viz_args, execute_result + + +def _forecast_config(viz_args: dict | None) -> dict: + config = (viz_args or {}).get("config") + return config if isinstance(config, dict) else {} + + +def _metric_uris(viz_args: dict | None) -> set[str]: + """The metric URIs the visualization measures, resolved through its field aliases. + + Deliberately not core.scoring.get_metric_uri_set: that takes a parsed + CreatedVisualization, while this reads the raw tool-call arguments, where a field may be + a bare URI string rather than an object. + """ + query = (viz_args or {}).get("query") + fields = query.get("fields") if isinstance(query, dict) else None + if not isinstance(fields, dict): + return set() + uris: set[str] = set() + for alias in (viz_args or {}).get("metrics") or list(fields): + name = alias.get("field") if isinstance(alias, dict) else alias + field_def = fields.get(name) + if isinstance(field_def, dict) and field_def.get("using"): + uris.add(str(field_def["using"])) + elif isinstance(field_def, str): + uris.add(field_def) + return {u for u in uris if u.startswith(("metric/", "fact/"))} + + +@dataclass +class ForecastEvaluation: + """Scores for a single forecasting run. + + ``triggered``/``executed``/``success``/``turn_completed`` are the process checks every + skill kind shares. ``forecast_enabled``, ``period_correct`` and ``metric_correct`` are + content checks, and each is True when the fixture did not ask for it -- an unstated + expectation must not fail a run, and must not silently pass one either, which is why + ``asserted`` records which of them the fixture actually pinned. + """ + + triggered: bool + executed: bool + success: bool + turn_completed: bool + forecast_enabled: bool + period_correct: bool + metric_correct: bool + asserted: list[str] = field(default_factory=list) + disambiguated: bool = False + + @property + def strict_pass(self) -> bool: + return all( + [ + self.triggered, + self.executed, + self.success, + self.turn_completed, + self.forecast_enabled, + self.period_correct, + self.metric_correct, + ] + ) + + +@dataclass +class ForecastRunResult: + """Outcome of one run (one conversation, up to max_iterations messages).""" + + conversation_id: str + evaluation: ForecastEvaluation + actual_visualization: dict | None + actual_execute_result: dict | None + turn_wall_clock_sec: float | None = None + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + + +@dataclass +class AgenticForecastSummary: + """Aggregated outcome of K runs for one forecasting item.""" + + run_results: list[ForecastRunResult] + pass_at_k: bool + pass_power_k: bool + best: ForecastRunResult + + +def _evaluate_run( + viz_args: dict | None, + execute_result: dict | None, + expected_output: dict, + turn_completed: bool, + disambiguated: bool = False, +) -> ForecastEvaluation: + triggered = execute_result is not None or viz_args is not None + executed = execute_result is not None + success = executed and execute_result.get("success") is True + + config = _forecast_config(viz_args) + # None is "not set", which the tool treats as disabled -- only an explicit true enables it. + forecast_enabled = config.get("forecast_enabled") is True + + asserted: list[str] = [] + expected_period = expected_output.get("forecast_period") + if expected_period is None: + period_correct = True + else: + asserted.append("forecast_period") + actual_period = config.get("forecast_period") + period_correct = isinstance(actual_period, int | float) and float(actual_period) == float(expected_period) + + expected_metric = expected_output.get("metric") + if not expected_metric: + metric_correct = True + else: + asserted.append("metric") + wanted = {expected_metric} if isinstance(expected_metric, str) else set(expected_metric) + metric_correct = bool(_metric_uris(viz_args) & wanted) + + return ForecastEvaluation( + triggered=triggered, + executed=executed, + success=success, + turn_completed=turn_completed, + forecast_enabled=forecast_enabled, + period_correct=period_correct, + metric_correct=metric_correct, + asserted=asserted, + disambiguated=disambiguated, + ) + + +def run_agentic_forecasting( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, +) -> AgenticForecastSummary: + """Run the forecasting agentic evaluation K times and return a summary. + + A run ends as soon as execute_forecast returns, which is the goal signal. Short of that + it keeps sending simulated replies up to ``max_iterations``, with no attempt to classify + whether the agent's text was really a question: missing a genuine clarifying question + hard-fails the run, while answering a final answer costs one harmless extra turn. + """ + if k < 1: + raise ValueError(f"k must be >= 1, got {k}") + run_results: list[ForecastRunResult] = [] + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) + + def _run_once(conv_id: str) -> ForecastRunResult: + viz_args: dict | None = None + execute_result: dict | None = None + turn_wall_clock_sec: float | None = None + turn_completed = False + disambiguated = False + current_question = question + reasoning_steps: list[str] = [] + response_id: str | None = None + all_tool_call_events: list[ToolCallEvent] = [] + all_reasoning_step_events: list[ReasoningStepEvent] = [] + turn_offset = 0.0 # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time + tool_index_offset = 0 + reasoning_index_offset = 0 + + def _accumulate(result: ChatResult) -> None: + nonlocal turn_offset, tool_index_offset, reasoning_index_offset + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + result, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) + all_tool_call_events.extend(result.tool_call_events or []) + all_reasoning_step_events.extend(result.reasoning_step_events or []) + + for iteration in range(max_iterations): + try: + chat_result = client.send_message(conv_id, current_question) + except Exception as exc: # noqa: BLE001 -- end this run, not the whole item + _log.warning("Forecast send_message failed for conversation %s: %s", conv_id, exc) + partial = getattr(exc, "partial_result", None) + if partial is not None: + reasoning_steps.extend(partial.reasoning_steps or []) + response_id = partial.response_id or response_id + _accumulate(partial) + viz_args, execute_result = _extract_forecast_calls(partial.tool_call_events or []) + turn_completed = False + break + reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id + _accumulate(chat_result) + viz_args, execute_result = _extract_forecast_calls(chat_result.tool_call_events or []) + response_text = render_answer_text(chat_result) + turn_completed = chat_result.stream_ended and bool(response_text) + if execute_result is not None: + # The turn that actually forecast, not an earlier disambiguation turn. + turn_wall_clock_sec = chat_result.turn_wall_clock_sec + break + if not response_text: + break + if iteration >= max_iterations - 1: + break + try: + current_question = generate_simulated_forecast_response(response_text, expected_output) + disambiguated = True + except Exception as exc: # noqa: BLE001 -- harness-side fault; end only this run + _log.warning("Simulated forecast user reply failed for conversation %s: %s", conv_id, exc) + break + + return ForecastRunResult( + conversation_id=conv_id, + evaluation=_evaluate_run(viz_args, execute_result, expected_output, turn_completed, disambiguated), + actual_visualization=viz_args, + actual_execute_result=execute_result, + turn_wall_clock_sec=turn_wall_clock_sec, + reasoning_steps=reasoning_steps, + response_id=response_id, + tool_call_events=all_tool_call_events, + reasoning_step_events=all_reasoning_step_events, + ) + + try: + conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() + try: + run_results.append(_run_once(conv_id_0)) + finally: + if initial_conversation_id is None: # only delete conversations we created + client.delete_conversation(conv_id_0) + + for _ in range(1, k): + conv_id = client.create_conversation() + try: + run_results.append(_run_once(conv_id)) + finally: + client.delete_conversation(conv_id) + finally: + client.close() + + pass_at_k = any(r.evaluation.strict_pass for r in run_results) + pass_power_k = all(r.evaluation.strict_pass for r in run_results) + best = max( + run_results, + key=lambda r: sum( + [ + r.evaluation.triggered, + r.evaluation.executed, + r.evaluation.success, + r.evaluation.turn_completed, + r.evaluation.forecast_enabled, + r.evaluation.period_correct, + r.evaluation.metric_correct, + ] + ), + ) + return AgenticForecastSummary( + run_results=run_results, + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + best=best, + ) + + +class ForecastingAssertionError(AgenticAssertionError): + """Raised when a forecasting evaluation fails.""" + + +def _detail(best: ForecastRunResult) -> dict[str, Any]: + ev = best.evaluation + return { + "triggered": ev.triggered, + "executed": ev.executed, + "success": ev.success, + "turn_completed": ev.turn_completed, + "forecast_enabled": ev.forecast_enabled, + "period_correct": ev.period_correct, + "metric_correct": ev.metric_correct, + # Which content checks the fixture pinned. Without it a run where nothing was + # asserted is indistinguishable in the report from one where everything matched. + "asserted": ev.asserted, + "disambiguated": ev.disambiguated, + "actual_forecast_config": _forecast_config(best.actual_visualization), + "actual_metrics": sorted(_metric_uris(best.actual_visualization)), + "actual_execute_result": best.actual_execute_result, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + } + + +def evaluate_agentic_forecasting( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + agent_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "forecasting", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, +) -> AgenticEvalOutcome: + """Run forecasting evaluation, log to Langfuse, and raise ForecastingAssertionError on failure.""" + langfuse, window_start = open_trace_window(langfuse) + summary = run_agentic_forecasting( + host=host, + token=token, + workspace_id=workspace_id, + question=question, + expected_output=expected_output, + k=k, + max_iterations=max_iterations, + initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + ) + + if langfuse is not None and dataset_item_id: + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + for run_idx, run in enumerate(summary.run_results): + pt = ctx.trace(run.conversation_id) + ev = run.evaluation + strict_checks = { + "forecast_triggered": ev.triggered, + "forecast_executed": ev.executed, + "forecast_success": ev.success, + "forecast_turn_completed": ev.turn_completed, + "forecast_enabled": ev.forecast_enabled, + "forecast_period_correct": ev.period_correct, + "forecast_metric_correct": ev.metric_correct, + } + with ctx.observe(pt, run_idx) as tid: + for score_name, value in strict_checks.items(): + ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") + ctx.quality( + tid, + strict_checks=strict_checks, + latency_sec=run.turn_wall_clock_sec, + cost_usd=pt.total_cost if pt and ev.triggered else None, + ) + + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in summary.run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, + ) + + best = summary.best + ev = best.evaluation + detail = _detail(best) + runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) + + if not summary.pass_at_k: + message = ( + f"Forecasting assertion failed. strict_pass={ev.strict_pass} " + f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " + f"turn_completed={ev.turn_completed}, forecast_enabled={ev.forecast_enabled}, " + f"period_correct={ev.period_correct}, metric_correct={ev.metric_correct}). " + f"Actual forecast config: {detail['actual_forecast_config']}. " + f"Actual execute result: {best.actual_execute_result}." + ) + exc = ForecastingAssertionError(message) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = len(summary.run_results) + raise exc + + return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=len(summary.run_results), + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + ) diff --git a/packages/gooddata-eval/tests/test_agentic_forecasting.py b/packages/gooddata-eval/tests/test_agentic_forecasting.py new file mode 100644 index 000000000..78222bf50 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_forecasting.py @@ -0,0 +1,295 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic.forecasting import ( + ForecastingAssertionError, + _evaluate_run, + _extract_forecast_calls, + _metric_uris, + evaluate_agentic_forecasting, + run_agentic_forecasting, +) +from gooddata_eval.core.models import ChatResult + +_MODULE = "gooddata_eval.core.agentic.forecasting" + +_EXPECTED = {"metric": "metric/spend", "forecast_period": 3} + + +def _viz_args(*, period=3, enabled=True, metric="metric/spend", confidence=0.95, seasonal=False) -> dict: + return { + "visualization": { + "type": "line_chart", + "title": "Spend forecast", + "query": { + "fields": { + "m_spend": {"using": metric}, + "d_month": {"using": "label/process_date.month"}, + }, + "filter_by": {}, + }, + "metrics": ["m_spend"], + "view_by": ["d_month"], + "config": { + "forecast_enabled": enabled, + "forecast_period": period, + "forecast_confidence": confidence, + "forecast_seasonal": seasonal, + }, + } + } + + +def _tc(name: str, args: dict | None = None, result: str | None = None): + tc = MagicMock() + tc.function_name = name + tc.call_ts = None + tc.result_ts = None + tc.index = None + tc.parsed_arguments = lambda a=args: a or {} + tc.result = result + tc.parsed_result = lambda r=result: __import__("json").loads(r) if r else None + return tc + + +_OK_FORECAST = '{"success": true, "data": {"points": [{"value": 1}], "truncated": false}, "error": null}' + + +def _chat(tool_calls, text="Here is the forecast.", ended=True) -> ChatResult: + """A real ChatResult, so render_answer_text sees every field it reads.""" + result = ChatResult.model_validate({"textResponse": text, "responseId": "resp-1"}) + result.stream_ended = ended + result.turn_wall_clock_sec = 4.2 + # Assigned after validation: these are hand-built tool-call doubles, not payload. + result.tool_call_events = tool_calls + return result + + +# ── extraction ────────────────────────────────────────────────────────────── + + +def test_extract_pairs_the_execute_with_the_visualization_it_followed(): + """A new chart clears the previous forecast: picking the last of each independently + would pair a fresh visualization with a stale result.""" + calls = [ + _tc("create_adhoc_visualization", _viz_args(period=1)), + _tc("execute_forecast", {"visualization_ref": "viz_1"}, _OK_FORECAST), + _tc("create_adhoc_visualization", _viz_args(period=3)), + ] + viz, result = _extract_forecast_calls(calls) + + assert viz["config"]["forecast_period"] == 3 + assert result is None + + +def test_extract_unwraps_the_visualization_argument(): + viz, _ = _extract_forecast_calls([_tc("create_adhoc_visualization", _viz_args())]) + assert viz["type"] == "line_chart" + + +def test_metric_uris_resolves_aliases_and_ignores_dimensions(): + viz, _ = _extract_forecast_calls([_tc("create_adhoc_visualization", _viz_args())]) + assert _metric_uris(viz) == {"metric/spend"} + + +def test_metric_uris_handles_a_bare_uri_field(): + """Tool-call arguments are raw JSON, where a field may be a plain string rather than + the object core.scoring's parsed model always has.""" + viz = {"query": {"fields": {"m": "metric/spend"}}, "metrics": ["m"]} + assert _metric_uris(viz) == {"metric/spend"} + + +# ── scoring ───────────────────────────────────────────────────────────────── + + +def _evaluate(viz=None, result=None, expected=None, turn_completed=True): + return _evaluate_run(viz, result, expected if expected is not None else _EXPECTED, turn_completed) + + +def _viz(**kw): + return _viz_args(**kw)["visualization"] + + +def test_a_correct_forecast_passes_every_check(): + ev = _evaluate(_viz(), {"success": True}) + assert ev.strict_pass is True + assert ev.asserted == ["forecast_period", "metric"] + + +def test_the_wrong_horizon_fails_on_period_alone(): + """The number the agent chose is in the tool call, so 'next 3 months' is checkable + exactly -- no judge, no paraphrase tolerance.""" + ev = _evaluate(_viz(period=6), {"success": True}) + assert ev.period_correct is False + assert ev.metric_correct is True + assert ev.strict_pass is False + + +def test_forecasting_the_wrong_measure_fails_on_metric_alone(): + ev = _evaluate(_viz(metric="metric/orders"), {"success": True}) + assert ev.metric_correct is False + assert ev.period_correct is True + + +def test_forecast_enabled_must_be_explicitly_true(): + """The tool refuses on forecast_enabled=false, and treats an unset value the same way, + so only an explicit true counts.""" + assert _evaluate(_viz(enabled=None), {"success": True}).forecast_enabled is False + assert _evaluate(_viz(enabled=False), {"success": True}).forecast_enabled is False + assert _evaluate(_viz(enabled=True), {"success": True}).forecast_enabled is True + + +def test_an_unstated_expectation_neither_fails_nor_silently_passes(): + """A fixture that pins nothing must not fail, but the report has to say nothing was + checked -- otherwise it reads identically to a run where everything matched.""" + ev = _evaluate(_viz(period=99, metric="metric/anything"), {"success": True}, expected={}) + assert ev.period_correct is True + assert ev.metric_correct is True + assert ev.asserted == [] + + +def test_a_failed_execution_is_not_a_success(): + ev = _evaluate(_viz(), {"success": False, "error": "no forecast for you"}) + assert ev.executed is True + assert ev.success is False + assert ev.strict_pass is False + + +def test_never_reaching_execute_is_not_triggered_by_a_chart_alone(): + ev = _evaluate(_viz(), None) + assert ev.executed is False + assert ev.success is False + + +def test_an_integer_valued_float_period_still_matches(): + """The AAC config is JSON, so 3 may arrive as 3.0.""" + assert _evaluate(_viz(period=3.0), {"success": True}).period_correct is True + + +# ── run loop ──────────────────────────────────────────────────────────────── + + +def _run(side_effect, *, k=1, expected=None, max_iterations=4): + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, k + 2)] + client.send_message.side_effect = side_effect + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.generate_simulated_forecast_response", return_value="use spend, 3 months"), + ): + return run_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=expected if expected is not None else _EXPECTED, + k=k, + max_iterations=max_iterations, + ) + + +def test_a_single_turn_forecast_passes(): + calls = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + summary = _run([_chat(calls)]) + assert summary.pass_at_k is True + assert summary.best.evaluation.disambiguated is False + + +def test_a_clarifying_question_is_answered_and_the_run_continues(): + """Observed live: the agent asks which Spend metric before building anything.""" + calls = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + summary = _run([_chat([], text="Which Spend metric did you mean?"), _chat(calls)]) + assert summary.pass_at_k is True + assert summary.best.evaluation.disambiguated is True + + +def test_the_loop_stops_at_max_iterations_without_a_forecast(): + summary = _run([_chat([], text="Still thinking")] * 4, max_iterations=4) + assert summary.pass_at_k is False + assert summary.best.evaluation.executed is False + + +def test_an_empty_response_ends_the_run_immediately(): + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = _chat([], text="") + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.generate_simulated_forecast_response") as sim, + ): + run_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="q", + expected_output=_EXPECTED, + ) + assert client.send_message.call_count == 1 + sim.assert_not_called() + + +def test_a_chat_error_ends_the_run_and_keeps_what_the_partial_carried(): + error = RuntimeError("stream died") + error.partial_result = _chat( + [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + ) + summary = _run([error]) + # The forecast landed before the stream broke, so it is still scored -- but the turn + # did not complete, which strict_pass requires. + assert summary.best.evaluation.executed is True + assert summary.best.evaluation.turn_completed is False + + +def test_a_chat_error_on_a_later_run_does_not_discard_the_earlier_one(): + calls = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + summary = _run([_chat(calls), RuntimeError("boom")], k=2) + assert len(summary.run_results) == 2 + assert summary.pass_at_k is True + + +def test_k_must_be_at_least_one(): + with pytest.raises(ValueError, match="k must be >= 1"): + run_agentic_forecasting(host="h", token="t", workspace_id="w", question="q", expected_output={}, k=0) + + +# ── evaluate_* wrapper ────────────────────────────────────────────────────── + + +def _evaluate_item(calls): + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = _chat(calls) + with patch(f"{_MODULE}.ChatClient", return_value=client): + return evaluate_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=_EXPECTED, + ) + + +def test_detail_reports_the_config_the_agent_chose(): + outcome = _evaluate_item( + [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + ) + assert outcome.detail["actual_forecast_config"]["forecast_period"] == 3 + assert outcome.detail["actual_metrics"] == ["metric/spend"] + assert outcome.detail["asserted"] == ["forecast_period", "metric"] + assert "latency_breakdown" in outcome.detail + assert outcome.runs_passed == 1 + + +def test_a_wrong_horizon_raises_naming_what_the_agent_actually_did(): + with pytest.raises(ForecastingAssertionError) as exc_info: + _evaluate_item( + [_tc("create_adhoc_visualization", _viz_args(period=12)), _tc("execute_forecast", {}, _OK_FORECAST)] + ) + + error = exc_info.value + assert "period_correct=False" in str(error) + assert error.detail["actual_forecast_config"]["forecast_period"] == 12 + assert error.runs_passed == 0 + assert error.conversation_id == "conv-1" diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 77086d101..a908f7db2 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -85,6 +85,7 @@ def test_dispatch_agentic_omits_agent_id_by_default(): ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), ("agentic_kda_skill", {"Measure": {"type": "metric", "id": "revenue"}}, "evaluate_agentic_kda_skill"), ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), + ("agentic_forecasting", {"forecast_period": 3}, "evaluate_agentic_forecasting"), ] diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py index 4737ee34c..5ca05355a 100644 --- a/packages/gooddata-eval/tests/test_trace_linker.py +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -118,6 +118,7 @@ def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): ("visualization", "evaluate_agentic_visualization"), ("kda_skill", "evaluate_agentic_kda_skill"), ("conversation", "evaluate_agentic_conversation"), + ("forecasting", "evaluate_agentic_forecasting"), ] From 59958ce09da0cb7c431a0382e4f6e24d76574406 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 10 Sep 2026 09:11:40 +0200 Subject: [PATCH 2/6] fix(gooddata-eval): address review on the forecasting evaluator Three findings, all real. Tool calls were extracted from the current turn only. The agent may build the chart on one turn and call execute_forecast on the next -- the common path, since it routinely asks which measure to forecast first -- and reading a single turn dropped the visualization the forecast actually ran on, failing a correct run for an empty config. Extraction now reads every turn accumulated so far. kda_skill does not have this bug only because its create and execute always land in the same turn. forecast_confidence and forecast_seasonal were described as checkable and never checked, so a fixture could pin either, receive something else, and pass. Both are now scored, with an absent forecast_seasonal counting as the tool's own default of false rather than as a mismatch. Unasserted content checks were published to Langfuse as BOOLEAN 1. They are True internally so they cannot fail a run, but reporting that as a score claims the evaluator verified something it never looked at. Only checks named in ev.asserted are now scored. 5 tests added; the cross-turn one verified to fail against the previous version. 808 passed, lint and format clean. Co-Authored-By: Claude Opus 5 --- .../gooddata_eval/core/agentic/forecasting.py | 53 +++++++++++++++++-- .../tests/test_agentic_forecasting.py | 51 ++++++++++++++++++ 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py index 91b2cfc6a..b56dcb927 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py @@ -171,6 +171,8 @@ class ForecastEvaluation: forecast_enabled: bool period_correct: bool metric_correct: bool + confidence_correct: bool + seasonal_correct: bool asserted: list[str] = field(default_factory=list) disambiguated: bool = False @@ -185,6 +187,8 @@ def strict_pass(self) -> bool: self.forecast_enabled, self.period_correct, self.metric_correct, + self.confidence_correct, + self.seasonal_correct, ] ) @@ -246,6 +250,26 @@ def _evaluate_run( wanted = {expected_metric} if isinstance(expected_metric, str) else set(expected_metric) metric_correct = bool(_metric_uris(viz_args) & wanted) + expected_confidence = expected_output.get("forecast_confidence") + if expected_confidence is None: + confidence_correct = True + else: + asserted.append("forecast_confidence") + actual_confidence = config.get("forecast_confidence") + confidence_correct = isinstance(actual_confidence, int | float) and float(actual_confidence) == float( + expected_confidence + ) + + expected_seasonal = expected_output.get("forecast_seasonal") + if expected_seasonal is None: + seasonal_correct = True + else: + asserted.append("forecast_seasonal") + # The tool defaults seasonal to False, so an absent value means "not seasonal" -- + # reading it as a mismatch would fail an agent that simply left the default alone. + actual_seasonal = config.get("forecast_seasonal") + seasonal_correct = bool(actual_seasonal) == bool(expected_seasonal) + return ForecastEvaluation( triggered=triggered, executed=executed, @@ -254,6 +278,8 @@ def _evaluate_run( forecast_enabled=forecast_enabled, period_correct=period_correct, metric_correct=metric_correct, + confidence_correct=confidence_correct, + seasonal_correct=seasonal_correct, asserted=asserted, disambiguated=disambiguated, ) @@ -321,13 +347,16 @@ def _accumulate(result: ChatResult) -> None: reasoning_steps.extend(partial.reasoning_steps or []) response_id = partial.response_id or response_id _accumulate(partial) - viz_args, execute_result = _extract_forecast_calls(partial.tool_call_events or []) + viz_args, execute_result = _extract_forecast_calls(all_tool_call_events) turn_completed = False break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id _accumulate(chat_result) - viz_args, execute_result = _extract_forecast_calls(chat_result.tool_call_events or []) + # Over every turn so far, not just this one: the agent may build the chart on + # one turn and forecast on the next, and reading a single turn would drop the + # visualization the forecast actually ran on. + viz_args, execute_result = _extract_forecast_calls(all_tool_call_events) response_text = render_answer_text(chat_result) turn_completed = chat_result.stream_ended and bool(response_text) if execute_result is not None: @@ -387,6 +416,8 @@ def _accumulate(result: ChatResult) -> None: r.evaluation.forecast_enabled, r.evaluation.period_correct, r.evaluation.metric_correct, + r.evaluation.confidence_correct, + r.evaluation.seasonal_correct, ] ), ) @@ -412,6 +443,8 @@ def _detail(best: ForecastRunResult) -> dict[str, Any]: "forecast_enabled": ev.forecast_enabled, "period_correct": ev.period_correct, "metric_correct": ev.metric_correct, + "confidence_correct": ev.confidence_correct, + "seasonal_correct": ev.seasonal_correct, # Which content checks the fixture pinned. Without it a run where nothing was # asserted is indistinguishable in the report from one where everything matched. "asserted": ev.asserted, @@ -471,9 +504,18 @@ def _write_scores(ctx: RunTraceContext) -> None: "forecast_success": ev.success, "forecast_turn_completed": ev.turn_completed, "forecast_enabled": ev.forecast_enabled, - "forecast_period_correct": ev.period_correct, - "forecast_metric_correct": ev.metric_correct, } + # Only the content checks the fixture actually pinned. An unasserted check + # is True internally so it cannot fail a run, but publishing that as a + # BOOLEAN 1 would claim the evaluator verified something it never looked at. + for name, key, value in ( + ("forecast_period", "forecast_period_correct", ev.period_correct), + ("metric", "forecast_metric_correct", ev.metric_correct), + ("forecast_confidence", "forecast_confidence_correct", ev.confidence_correct), + ("forecast_seasonal", "forecast_seasonal_correct", ev.seasonal_correct), + ): + if name in ev.asserted: + strict_checks[key] = value with ctx.observe(pt, run_idx) as tid: for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") @@ -516,7 +558,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Forecasting assertion failed. strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " f"turn_completed={ev.turn_completed}, forecast_enabled={ev.forecast_enabled}, " - f"period_correct={ev.period_correct}, metric_correct={ev.metric_correct}). " + f"period_correct={ev.period_correct}, metric_correct={ev.metric_correct}, " + f"confidence_correct={ev.confidence_correct}, seasonal_correct={ev.seasonal_correct}). " f"Actual forecast config: {detail['actual_forecast_config']}. " f"Actual execute result: {best.actual_execute_result}." ) diff --git a/packages/gooddata-eval/tests/test_agentic_forecasting.py b/packages/gooddata-eval/tests/test_agentic_forecasting.py index 78222bf50..a7873cbe9 100644 --- a/packages/gooddata-eval/tests/test_agentic_forecasting.py +++ b/packages/gooddata-eval/tests/test_agentic_forecasting.py @@ -293,3 +293,54 @@ def test_a_wrong_horizon_raises_naming_what_the_agent_actually_did(): assert error.detail["actual_forecast_config"]["forecast_period"] == 12 assert error.runs_passed == 0 assert error.conversation_id == "conv-1" + + +# ── confidence and seasonality ────────────────────────────────────────────── + + +def test_confidence_is_checked_when_pinned(): + expected = {"forecast_confidence": 0.99} + assert _evaluate(_viz(confidence=0.99), {"success": True}, expected=expected).confidence_correct is True + assert _evaluate(_viz(confidence=0.95), {"success": True}, expected=expected).confidence_correct is False + + +def test_seasonality_is_checked_when_pinned(): + expected = {"forecast_seasonal": True} + assert _evaluate(_viz(seasonal=True), {"success": True}, expected=expected).seasonal_correct is True + assert _evaluate(_viz(seasonal=False), {"success": True}, expected=expected).seasonal_correct is False + + +def test_an_absent_seasonal_counts_as_the_tools_default(): + """The tool defaults seasonal to false, so an agent that leaves it alone has asked for + a non-seasonal forecast -- failing that would penalise correct behaviour.""" + viz = _viz() + del viz["config"]["forecast_seasonal"] + assert _evaluate(viz, {"success": True}, expected={"forecast_seasonal": False}).seasonal_correct is True + + +def test_confidence_and_seasonality_are_unasserted_by_default(): + ev = _evaluate(_viz(confidence=0.5, seasonal=True), {"success": True}) + assert ev.confidence_correct is True + assert ev.seasonal_correct is True + assert "forecast_confidence" not in ev.asserted + assert "forecast_seasonal" not in ev.asserted + + +# ── cross-turn extraction ─────────────────────────────────────────────────── + + +def test_a_chart_built_on_an_earlier_turn_is_still_the_one_scored(): + """The agent may build the chart on one turn and forecast on the next -- reading only + the current turn's calls would drop the visualization the forecast actually ran on and + fail a correct run for an empty config.""" + summary = _run( + [ + _chat([_tc("create_adhoc_visualization", _viz_args())], text="Building the chart, one moment."), + _chat([_tc("execute_forecast", {"visualization_ref": "viz_1"}, _OK_FORECAST)]), + ] + ) + + assert summary.best.evaluation.executed is True + assert summary.best.evaluation.period_correct is True + assert summary.best.evaluation.metric_correct is True + assert summary.pass_at_k is True From 273b8428d1896116ab6c5b7f24bd000cffa4306e Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 10 Sep 2026 09:18:19 +0200 Subject: [PATCH 3/6] style(gooddata-eval): satisfy PERF403 in the forecasting score assembly Also corrects the previous commit message, which said 808 tests; the suite is at 806. Co-Authored-By: Claude Opus 5 --- .../gooddata_eval/core/agentic/forecasting.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py index b56dcb927..7b35e18e7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py @@ -508,14 +508,18 @@ def _write_scores(ctx: RunTraceContext) -> None: # Only the content checks the fixture actually pinned. An unasserted check # is True internally so it cannot fail a run, but publishing that as a # BOOLEAN 1 would claim the evaluator verified something it never looked at. - for name, key, value in ( - ("forecast_period", "forecast_period_correct", ev.period_correct), - ("metric", "forecast_metric_correct", ev.metric_correct), - ("forecast_confidence", "forecast_confidence_correct", ev.confidence_correct), - ("forecast_seasonal", "forecast_seasonal_correct", ev.seasonal_correct), - ): - if name in ev.asserted: - strict_checks[key] = value + strict_checks.update( + { + key: value + for name, key, value in ( + ("forecast_period", "forecast_period_correct", ev.period_correct), + ("metric", "forecast_metric_correct", ev.metric_correct), + ("forecast_confidence", "forecast_confidence_correct", ev.confidence_correct), + ("forecast_seasonal", "forecast_seasonal_correct", ev.seasonal_correct), + ) + if name in ev.asserted + } + ) with ctx.observe(pt, run_idx) as tid: for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") From c7bef36042930d7b8e568abeb67e87a6025f7ca0 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 10 Sep 2026 09:44:37 +0200 Subject: [PATCH 4/6] fix(gooddata-eval): report whole-conversation latency and ungated cost Third structural finding from #1799's review, applied here. latency_sec used run.turn_wall_clock_sec, which is the goal turn alone and excludes the clarification turns that got there -- understating the item's real elapsed cost on exactly the runs where it matters. It now prefers pt.latency and falls back to the goal turn, which is what 7 of the 8 existing kinds already do; kda_skill is the outlier and documents its own reason, and this copied it without re-checking. cost_usd was gated on ev.triggered, so a run that answered without ever reaching the tool reported no cost despite having spent tokens. The gate is gone. 806 passed, lint and format clean. Co-Authored-By: Claude Opus 5 --- .../src/gooddata_eval/core/agentic/forecasting.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py index 7b35e18e7..47d8e7fde 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py @@ -526,8 +526,15 @@ def _write_scores(ctx: RunTraceContext) -> None: ctx.quality( tid, strict_checks=strict_checks, - latency_sec=run.turn_wall_clock_sec, - cost_usd=pt.total_cost if pt and ev.triggered else None, + # pt.latency covers the whole conversation, which is the item's real + # elapsed cost when the agent needed clarification turns to get + # there; turn_wall_clock_sec (the goal turn alone) is the fallback. + # This is what 7 of the 8 existing kinds do -- kda_skill is the + # outlier and documents its own reason. Cost is not gated on + # ev.triggered: a run that answered without ever reaching the tool + # still spent tokens, and hiding that understates what the item cost. + latency_sec=pt.latency if pt else run.turn_wall_clock_sec, + cost_usd=pt.total_cost if pt else None, ) # Before the pass@K raise: a failing item's scores are the ones worth having. From 9ddc7c1e57f3732f883a8304812ae74ac27d3189 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 10 Sep 2026 12:42:03 +0200 Subject: [PATCH 5/6] test(gooddata-eval): cover the forecasting Langfuse scoring block The ev.asserted gating and the latency/cost change made in response to review were behavioural fixes shipped with no test. Covered now by capturing the deferred callable and running it against a fake context: only pinned checks are scored, every pinned check is scored, and cost is reported even when the tool was never reached. 79% -> 93% on the module. What remains is the clarification prompt builder and the OpenAI call, both exercised through the loop tests via their patch points. 809 passed, lint and format clean. Co-Authored-By: Claude Opus 5 --- .../tests/test_agentic_forecasting.py | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/packages/gooddata-eval/tests/test_agentic_forecasting.py b/packages/gooddata-eval/tests/test_agentic_forecasting.py index a7873cbe9..afcab9166 100644 --- a/packages/gooddata-eval/tests/test_agentic_forecasting.py +++ b/packages/gooddata-eval/tests/test_agentic_forecasting.py @@ -1,5 +1,6 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +from contextlib import contextmanager from unittest.mock import MagicMock, patch import pytest @@ -344,3 +345,93 @@ def test_a_chart_built_on_an_earlier_turn_is_still_the_one_scored(): assert summary.best.evaluation.period_correct is True assert summary.best.evaluation.metric_correct is True assert summary.pass_at_k is True + + +# ── Langfuse scoring ──────────────────────────────────────────────────────── + + +class _FakeCtx: + """Records what the deferred Langfuse block writes, without a Langfuse.""" + + def __init__(self): + self.scores: dict[str, float] = {} + # Not named `quality`: the method below would overwrite itself on first call. + self.quality_call: dict = {} + + def trace(self, _conversation_id): + return None + + @contextmanager + def observe(self, _trace, _run_idx): + yield "trace-id" + + def score(self, _tid, *, name, value, data_type): + self.scores[name] = value + + def quality(self, _tid, *, strict_checks, latency_sec, cost_usd): + self.quality_call = {"strict_checks": strict_checks, "latency_sec": latency_sec, "cost_usd": cost_usd} + + +def _scored(expected_output, calls=None): + """Run one item with Langfuse on, then execute the deferred block against a fake ctx.""" + client = MagicMock() + client.create_conversation.return_value = "conv-1" + default = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + client.send_message.return_value = _chat(calls if calls is not None else default) + captured = {} + + def _capture(_link, _identity, **kwargs): + captured.update(kwargs) + + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.submit_trace_scoring", side_effect=_capture), + ): + try: + evaluate_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=expected_output, + langfuse=MagicMock(), + dataset_item_id="ds-1", + ) + except ForecastingAssertionError: + pass # scores are written before the pass@K raise, which is the point + + ctx = _FakeCtx() + captured["write_scores"](ctx) + return ctx + + +def test_only_the_checks_the_fixture_pinned_are_scored(): + """An unasserted check is True internally so it cannot fail a run. Publishing that as a + BOOLEAN 1 would claim the evaluator verified something it never looked at.""" + ctx = _scored({"forecast_period": 3}) # metric, confidence, seasonality unpinned + + assert ctx.scores["forecast_period_correct"] == 1.0 + for absent in ("forecast_metric_correct", "forecast_confidence_correct", "forecast_seasonal_correct"): + assert absent not in ctx.scores + # The process checks are unconditional -- they are always actually evaluated. + assert set(ctx.scores) >= {"forecast_triggered", "forecast_executed", "forecast_enabled"} + + +def test_every_pinned_check_is_scored(): + ctx = _scored( + {"metric": "metric/spend", "forecast_period": 3, "forecast_confidence": 0.95, "forecast_seasonal": False} + ) + for name in ( + "forecast_period_correct", + "forecast_metric_correct", + "forecast_confidence_correct", + "forecast_seasonal_correct", + ): + assert ctx.scores[name] == 1.0 + + +def test_cost_is_reported_even_when_the_tool_was_never_reached(): + """A run that answered without forecasting still spent tokens; gating cost on + ev.triggered hid that and understated what the item cost.""" + ctx = _scored(_EXPECTED, calls=[]) + assert "cost_usd" in ctx.quality_call From 559c18ac919a9a251436c9c8dd35492879a65be8 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 10 Sep 2026 12:52:10 +0200 Subject: [PATCH 6/6] fix(gooddata-eval): pass item_input when deferring forecasting scores Merges master, which added a guard requiring every kind to hand the scored item's question to the linker -- a score is otherwise readable only by resolving its conversation back to the item. This kind predates the guard and did not. 946 passed, lint and format clean. Co-Authored-By: Claude Opus 5 --- .../src/gooddata_eval/core/agentic/forecasting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py index 47d8e7fde..34043b6b1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py @@ -557,6 +557,9 @@ def _write_scores(ctx: RunTraceContext) -> None: window_end=window_end, suffix_runs=len(summary.run_results) > 1, write_scores=_write_scores, + # The question this run answered, so a score is readable without resolving the + # conversation back to its item. + item_input=question, ) best = summary.best