diff --git a/packages/client/README.md b/packages/client/README.md index 92ca6b1c..c80298f5 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -122,6 +122,8 @@ A judge is shown the tool calls the row made on the way to its output, so a rubr The harness records this itself: it wraps your tool implementations once per row before handing them to the handler, so every handler package is covered without changes and your tools still return and raise exactly what they did before. +**This is not specific to offline evaluations.** Online judges — both the inline ones sampled by `config().invoke()` and the deferred ones you run from a `JudgeTask` on a background thread — are shown the same trajectory, built by the same function. See [Judges see one conversation](#judges-see-one-conversation). + The trajectory is rendered into **`{{message_history}}`** — the row input, then the trajectory, then the generated output, then the formatting instructions, in that order. There is no separate trajectory variable: `message_history` is already the transcript variable every judge reads, and judges built from the AI Library's default templates reference it, so a trajectory rubric can be written against an existing judge template with no new placeholder. ``` @@ -137,6 +139,24 @@ Tool calls made while producing the response, in order: A row with tools that called none of them says so explicitly, which is the finding a tool-selection rubric most needs. A run with no tools adds no block at all, so judges written before trajectories existed read exactly the history they read before. +#### Judges see one conversation + +All three judge paths build `{{message_history}}` through a single function, `judge_scoring.build_message_history`: + +| Path | Entry point | +| --- | --- | +| Online, inline | `config().invoke()` → `run_judges` | +| Online, deferred | `config(skip_judges=True).invoke()` → `run_judge(task, handlers)` on your own thread | +| Offline | `init_evaluations().run(criteria=[Judge(...)])` | + +Each one is the input, then the tool trajectory, then the output, then the `{score, reasoning}` format block, with empty parts skipped. A judge therefore grades the same conversation wherever it runs, which is what makes a rubric portable between a production sample and a dataset replay. + +They did not always agree, and that is why this is a single function now: each path used to join its own history. The offline one carried the row input, the inline one carried the user input, and the deferred one carried **neither** — so a deferred judge graded a response with no request beside it. `JudgeTask` gained `user_input` and `trajectory` to close that. + +For the deferred path those two fields travel on the task, which stays picklable — the trajectory crosses as the rendered string, not the structured record. + +A **graph-level** judge (`graph_judge`) gets no trajectory: it grades a final answer produced across several nodes, and splicing their trajectories together would describe a conversation that never happened. Per-node judges inside a graph do get their own node's. + Two limits keep a trajectory from spending the judge's context window: at most 50 recorded calls per row and 2000 characters per rendered argument bag or result, with anything beyond either reported as a count or marked truncated. Calls past the limit still execute — truncation drops the record, never the work. A `NativeTool` runs inside the provider, so no local wrapper sees it; such a tool is left out of the trajectory and out of the "Tools available" line, since naming a tool whose use cannot be shown would invite a judge to conclude the model ignored it. A tool result is now judge-prompt input. It stays literal for the same reason the generated output does: the judge config is handed to the handler unrendered and the handler makes exactly one template pass, so a `{{...}}` sequence coming back from a tool is never expanded into the judge prompt. diff --git a/packages/client/src/launchdarkly_ai_server/client.py b/packages/client/src/launchdarkly_ai_server/client.py index f64422db..f9f598f2 100644 --- a/packages/client/src/launchdarkly_ai_server/client.py +++ b/packages/client/src/launchdarkly_ai_server/client.py @@ -123,6 +123,8 @@ async def invoke( handlers=resolved_handler_list, llm_response=llm_str, base_track_data=track_data, + user_input=user_input, + trajectory=result.get("trajectory", ""), ) return ProviderResponse( response=parsed_response, @@ -137,6 +139,7 @@ async def invoke( handler=handler, handlers=resolved_handler_list, user_input=user_input, + trajectory=result.get("trajectory", ""), llm_response=llm_str, base_track_data=track_data, tool_handlers=resolved_tools, @@ -215,6 +218,7 @@ async def _stream_events( handler=handler, handlers=resolved_handler_list, user_input=user_input, + trajectory=done_event.get("trajectory", ""), llm_response=done_event.get("response", ""), base_track_data=track_data, tool_handlers=resolved_tools, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 6738d68c..c6da9eca 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -14,10 +14,16 @@ from ..judge_scoring import ( FORMATTING_INSTRUCTIONS, + build_message_history, numeric_score, parse_judge_response, ) from ..lifecycle import extract_variation +from ..trajectory import ( + TrajectoryRecorder, + render_row_trajectory, + row_fields, +) from ..types import NativeTool from ..utils import ( collapse_messages_to_instructions, @@ -35,7 +41,6 @@ LDJudgeCriterionEventPayload, TokenUsage, ) -from .trajectory import TrajectoryRecorder, render_row_trajectory, row_fields from .types import ( DatasetRef, DatasetRow, @@ -712,32 +717,21 @@ def _judge_variables( # reading the history sees the request, what the agent did about it, and # what it finally answered, in order. trajectory = render_row_trajectory(row_result) - # message_history carries FORMATTING_INSTRUCTIONS the same way the - # online path builds it (judges.run_judges), because that -- not the - # standalone formatting_instructions variable below -- is what every - # judge built from the AI Library's default templates (accuracy, - # relevance, toxicity, and any judge cloned from them) actually - # references. A judge authored before this variable existed must keep - # getting scored without edits. - # - # The trajectory goes here and nowhere else. It was briefly also - # exposed as a standalone tool_trajectory variable, which bought - # nothing: this is already the transcript variable every judge reads, - # and two overlapping variables only invited a rubric to interpolate - # both and pay for the trajectory twice. + # Built by the shared builder, not inline here: this path and both + # online paths must show a judge the same conversation, and they did + # not while each one joined its own. The trajectory goes into + # message_history and nowhere else -- it is already the transcript + # variable every judge cloned from the AI Library's default templates + # reads, so a second overlapping variable only invited a rubric to + # interpolate both and pay for the trajectory twice. variables.update( { "input": row_result.get("input") or "", "response_to_evaluate": output if output is not None else "", - "message_history": "\n\n".join( - str(value) - for value in ( - row_result.get("input"), - trajectory, - output, - FORMATTING_INSTRUCTIONS, - ) - if value + "message_history": build_message_history( + user_input=row_result.get("input"), + trajectory=trajectory, + output=output, ), "expected_output": expected if expected is not None else "", "ground_truth_context": ( diff --git a/packages/client/src/launchdarkly_ai_server/graph.py b/packages/client/src/launchdarkly_ai_server/graph.py index e9088e71..2398cdfc 100644 --- a/packages/client/src/launchdarkly_ai_server/graph.py +++ b/packages/client/src/launchdarkly_ai_server/graph.py @@ -225,6 +225,7 @@ async def run_node( base_track_data=result["track_data"], tool_handlers=tool_handlers, graph_key=key, + trajectory=result.get("trajectory", ""), ) if from_node: @@ -379,6 +380,7 @@ def _fn(*a: Any, **kw: Any) -> str: base_track_data=result["track_data"], tool_handlers=tool_handlers, graph_key=key, + trajectory=result.get("trajectory", ""), ) next_node = nodes.get(chosen[0]) if chosen else None diff --git a/packages/client/src/launchdarkly_ai_server/judge_scoring.py b/packages/client/src/launchdarkly_ai_server/judge_scoring.py index 8375b413..2c93298b 100644 --- a/packages/client/src/launchdarkly_ai_server/judge_scoring.py +++ b/packages/client/src/launchdarkly_ai_server/judge_scoring.py @@ -1,10 +1,19 @@ -"""Shared scoring contract for LaunchDarkly AI Judge invocations. +"""Shared contract for LaunchDarkly AI Judge invocations. -Both judge execution paths — the online path (``judges.run_judges``, sampled -per invocation) and the offline evaluations path (``evaluations.runner``) — -prompt a judge model for the same ``{"score": <0-1>, "reasoning": }`` -JSON shape and must parse it the same way. This module owns that contract so -the two paths cannot drift. +Three judge execution paths exist — the online inline path +(``judges.run_judges``, sampled per invocation), the online deferred path +(``judges.run_judge``, from a ``JudgeTask`` on a background thread), and the +offline evaluations path (``evaluations.runner``). All three prompt a judge +model for the same ``{"score": <0-1>, "reasoning": }`` JSON shape, and +all three must show the judge the same conversation. This module owns both +halves of that contract so the paths cannot drift. + +They did drift. Each path built ``message_history`` with its own inline join: +the offline one carried the row input, the inline online one carried the user +input, and the deferred one carried neither -- a judge grading the same +response saw a different conversation depending on which path reached it. The +trajectory landing in only one of the three is what made that visible. +:func:`build_message_history` is now the only place it is built. """ from __future__ import annotations @@ -27,6 +36,32 @@ ) +def build_message_history( + *, + user_input: Any = None, + trajectory: Any = None, + output: Any = None, +) -> str: + """The conversation a judge is shown, as the ``message_history`` variable. + + Ordered the way it happened: what was asked, what the agent did about it, + what it answered, and finally how to format the verdict. Empty parts are + skipped, so a run with no tools produces exactly the history it produced + before trajectories existed and a judge authored against it is unaffected. + + ``FORMATTING_INSTRUCTIONS`` is appended here rather than by each caller, + because every judge built from the AI Library's default templates + references ``{{message_history}}`` and not ``{{formatting_instructions}}`` + -- a judge that stopped being told the JSON shape would start returning + prose, and every one of its results would become an invalid-output error. + """ + return "\n\n".join( + str(part) + for part in (user_input, trajectory, output, FORMATTING_INSTRUCTIONS) + if part + ) + + def numeric_score(score: Any) -> float | None: """Return ``score`` as a float only when it already is a finite number. diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 5878d6b1..bf9f256a 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -7,7 +7,7 @@ from .conversation import with_judge_evaluation from .judge_scoring import ( - FORMATTING_INSTRUCTIONS, + build_message_history, numeric_score, parse_judge_response, ) @@ -53,10 +53,16 @@ async def run_judges( base_track_data: TrackData, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, graph_key: str | None = None, + trajectory: str = "", ) -> dict[str, JudgeResult]: """ Runs any judges configured on ``config['judgeConfiguration']`` against the produced output. Each judge is itself a tracked AI call. + + ``trajectory`` is the rendered tool-call trajectory of the invocation being + judged, from ``execute_and_track``. It defaults to empty so a caller that + has none -- a graph-level judge over several nodes, for instance -- is + unchanged, and so is a judge for a config with no tools. """ from .lifecycle import extract_variation from .tracking import execute_and_track @@ -145,8 +151,10 @@ async def run_judges( else judge_ai_config ) - message_history = "\n\n".join( - filter(None, [user_input, llm_response, FORMATTING_INSTRUCTIONS]) + message_history = build_message_history( + user_input=user_input, + trajectory=trajectory, + output=llm_response, ) async with with_judge_evaluation(judge_key) as record_evaluation: @@ -208,6 +216,8 @@ async def build_judge_tasks( handlers: list[ProviderHandler] | None = None, llm_response: str, base_track_data: TrackData, + user_input: str | None = None, + trajectory: str = "", ) -> list[JudgeTask]: """ Resolves all judges configured on ``config['judgeConfiguration']`` into @@ -306,6 +316,8 @@ async def build_judge_tasks( judge_config=judge_ai_config, judge_meta=judge_meta, actual_output=llm_response, + user_input=user_input, + trajectory=trajectory, user_context=user_context, judge_provider=judge_provider, judge_mode=judge_mode, @@ -374,8 +386,13 @@ def _matches(h: ProviderHandler) -> bool: else task.judge_config ) - message_history = "\n\n".join( - filter(None, [task.actual_output, FORMATTING_INSTRUCTIONS]) + # user_input and trajectory come off the task rather than being omitted: + # this path used to build a history with neither, so a judge grading the + # same response saw a different conversation than the inline path did. + message_history = build_message_history( + user_input=task.user_input, + trajectory=task.trajectory, + output=task.actual_output, ) async with with_judge_evaluation(task.config_key) as record_evaluation: diff --git a/packages/client/src/launchdarkly_ai_server/tracking.py b/packages/client/src/launchdarkly_ai_server/tracking.py index 639ff681..fc99a45e 100644 --- a/packages/client/src/launchdarkly_ai_server/tracking.py +++ b/packages/client/src/launchdarkly_ai_server/tracking.py @@ -7,6 +7,7 @@ from collections.abc import AsyncGenerator, Callable from typing import Any +from .trajectory import TrajectoryRecorder, render_trajectory from .types import ( NATIVE_TOOL_KEY, AiConfigRep, @@ -138,7 +139,16 @@ async def execute_and_track( client = get_client() ld_ctx = to_ld_context(client, user_context) - tracked_tool_handlers = wrap_tool_handlers(tool_handlers, ld_ctx, track_data) + # Recording is composed *inside* the tracking wrapper, on the original map, + # so the recorder still sees a NativeTool as a NativeTool and skips it. If + # it wrapped the tracked map instead it would see the callable stub + # wrap_tool_handlers substitutes for a native tool, and would show a judge + # a tool call with an empty result while the provider's real result stayed + # invisible. One recorder per invocation, since invocations run concurrently. + recorder = TrajectoryRecorder() + tracked_tool_handlers = wrap_tool_handlers( + recorder.wrap(tool_handlers or {}), ld_ctx, track_data + ) merged_variables: dict[str, Any] = { **(variables or {}), "ldContext": {**user_context}, @@ -172,7 +182,19 @@ async def execute_and_track( raw_output = result.get("output") response = raw_output if raw_output is not None else "" - return {"usage": usage, "response": response, "track_data": track_data} + return { + "usage": usage, + "response": response, + "track_data": track_data, + # Rendered here rather than returned structurally: the one consumer is + # message_history, and a JudgeTask has to stay picklable for the + # background path. + "trajectory": render_trajectory( + recorder.invocations, + observable_tools=recorder.observable_tools, + omitted=recorder.omitted, + ), + } async def execute_and_stream( @@ -219,7 +241,10 @@ async def execute_and_stream( client = get_client() ld_ctx = to_ld_context(client, user_context) - tracked_tool_handlers = wrap_tool_handlers(tool_handlers, ld_ctx, track_data) + recorder = TrajectoryRecorder() + tracked_tool_handlers = wrap_tool_handlers( + recorder.wrap(tool_handlers or {}), ld_ctx, track_data + ) merged_variables: dict[str, Any] = { **(variables or {}), "ldContext": {**user_context}, @@ -275,4 +300,9 @@ async def execute_and_stream( "response": full_text, "usage": usage, "track_data": track_data, + "trajectory": render_trajectory( + recorder.invocations, + observable_tools=recorder.observable_tools, + omitted=recorder.omitted, + ), } diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/trajectory.py b/packages/client/src/launchdarkly_ai_server/trajectory.py similarity index 86% rename from packages/client/src/launchdarkly_ai_server/evaluations/trajectory.py rename to packages/client/src/launchdarkly_ai_server/trajectory.py index 10ad15d5..67764c69 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/trajectory.py +++ b/packages/client/src/launchdarkly_ai_server/trajectory.py @@ -1,4 +1,4 @@ -"""Tool-call trajectory capture for the generation phase of an SDK-run evaluation. +"""Tool-call trajectory capture, shared by both judge paths. A judge can only grade what it is shown. Handler packages record tool traffic onto OpenTelemetry spans and return only ``{output, usage}``, so by the time a @@ -7,10 +7,16 @@ right arguments?" an unaskable question of an SDK-run evaluation, even though the evaluation had just run the agent that answered it. -The runner therefore records the trajectory itself, by wrapping the caller's -tool implementations once per row before handing them to the handler. Wrapping -is what makes this work with every handler package without changing any of -them: a handler looks a tool up by its key and calls it, exactly as before. +Both judge paths therefore record the trajectory themselves, by wrapping the +caller's tool implementations before handing them to the handler: once per row +in the offline evaluations runner, and once per invocation in +``tracking.execute_and_track``. Wrapping is what makes this work with every +handler package without changing any of them: a handler looks a tool up by its +key and calls it, exactly as before. + +The recorded trajectory reaches a judge through ``message_history``, built by +:func:`judge_scoring.build_message_history` -- one function for both paths, so +an online judge and an offline one are shown the same shape. Three properties are load-bearing. @@ -20,16 +26,24 @@ truncation drops the *record*, never the work, because an evaluation that changed the agent's behavior would no longer be evaluating the agent. -**A recorder belongs to one row.** ``_run_rows`` runs rows concurrently against -one shared tool map, so a single shared recorder would splice one row's calls -into another row's trajectory and hand the judge a conversation that never -happened. +**A recorder belongs to one invocation.** The offline runner generates rows +concurrently against one shared tool map, so a single shared recorder would +splice one row's calls into another row's trajectory and hand the judge a +conversation that never happened. The same holds for concurrent online +invocations, which is why ``execute_and_track`` builds its own per call. **Only observable tools are described.** A ``NativeTool`` is executed inside the provider, so no local wrapper ever sees it and its calls cannot appear in the trajectory. Such a tool is therefore left out of the rendered "tools available" line as well: naming a tool whose use is invisible would let a judge conclude the model ignored a tool it may well have called. + +Online, ``tracking.wrap_tool_handlers`` does turn a ``NativeTool`` into a +callable tracking stub, so a native call *is* locally observable there -- but it +is still skipped, and deliberately. The stub returns nothing, so recording it +would show a judge a tool call with an empty result while the provider's real +result stayed invisible. Recording is therefore composed *inside* that wrapper, +on the original map, so both paths see natives identically. """ from __future__ import annotations @@ -40,7 +54,7 @@ from dataclasses import dataclass, replace from typing import Any -from ..types import NativeTool +from .types import NativeTool #: How many tool calls one row's trajectory records. A trajectory is #: interpolated into a judge prompt, so an agent that loops over a large tool diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index f3505364..56ff58fa 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -299,6 +299,20 @@ class JudgeTask: variables: dict[str, Any] | None = None """Optional extra template variables for the judge prompt.""" evaluation_metric_key: str | None = None + user_input: str | None = None + """The input that produced ``actual_output``. + + Carried so this path builds the same ``message_history`` as the inline one + (:func:`judge_scoring.build_message_history`). It was previously absent, + which meant a deferred judge was shown the response with no request beside + it. + """ + trajectory: str = "" + """The rendered tool-call trajectory of the invocation being judged. + + A plain string, not the structured record, because every field here has to + stay picklable for the worker thread. + """ """LD metric key to track the score against.""" diff --git a/packages/client/tests/test_judge_message_history.py b/packages/client/tests/test_judge_message_history.py new file mode 100644 index 00000000..56900f52 --- /dev/null +++ b/packages/client/tests/test_judge_message_history.py @@ -0,0 +1,374 @@ +"""One message_history for every judge path. + +The three paths -- online inline (``run_judges``), online deferred +(``run_judge`` from a ``JudgeTask``), and offline evaluations -- each used to +join their own. They disagreed, so a judge grading the same response saw a +different conversation depending on which path reached it. These tests hold +them to :func:`judge_scoring.build_message_history`. +""" + +from __future__ import annotations + +import pickle +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_server import JudgeTask, ProviderHandler, run_judge, run_judges +from launchdarkly_ai_server.judge_scoring import ( + FORMATTING_INSTRUCTIONS, + build_message_history, +) +from launchdarkly_ai_server.tracking import execute_and_track +from launchdarkly_ai_server.trajectory import TrajectoryRecorder, render_trajectory + +CONTEXT = {"kind": "user", "key": "u1"} + +JUDGE_CONFIG = { + "model": {"name": "gpt-4"}, + "provider": {"name": "TestProvider"}, + "instructions": "Judge it", +} + + +@pytest.fixture +def mock_ld_client() -> Any: + client = MagicMock() + client.track = MagicMock() + client.flush = AsyncMock() + client.close = AsyncMock() + client.variation = AsyncMock(return_value=None) + lifecycle_module._set_client_for_testing(client) + yield client + lifecycle_module._reset_for_testing() + + +def capturing_judge_handler(seen: list[dict[str, Any]]) -> ProviderHandler: + async def fn( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict[str, Any]: + seen.append(dict(variables or {})) + return { + "output": '{"score": 1, "reasoning": "ok"}', + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + return ProviderHandler(fn=fn, provides_for=("TestProvider", "messages")) # type: ignore[arg-type] + + +def judged_config() -> dict[str, Any]: + return { + "model": {"name": "gpt-4"}, + "provider": {"name": "TestProvider"}, + "instructions": "hi", + "judgeConfiguration": {"judges": [{"key": "judge-1", "samplingRate": 1}]}, + } + + +@pytest.fixture +def judge_variation(monkeypatch: pytest.MonkeyPatch) -> None: + async def fake_extract_variation(key: str, context: Any) -> dict[str, Any]: + return { + "config": dict(JUDGE_CONFIG), + "meta": {"variationKey": "v", "version": 1}, + } + + monkeypatch.setattr( + "launchdarkly_ai_server.lifecycle.extract_variation", fake_extract_variation + ) + + +# ─── the builder itself ────────────────────────────────────────────────────── + + +def test_builder_orders_the_conversation_and_appends_the_format_block() -> None: + history = build_message_history(user_input="Q", trajectory="T", output="A") + + assert history == f"Q\n\nT\n\nA\n\n{FORMATTING_INSTRUCTIONS}" + + +def test_builder_skips_empty_parts() -> None: + """A run with no tools produces the history it produced before trajectories. + + This is what keeps a judge authored before this feature scoring unchanged. + """ + assert build_message_history(user_input="Q", trajectory="", output="A") == ( + f"Q\n\nA\n\n{FORMATTING_INSTRUCTIONS}" + ) + assert build_message_history(output="A") == f"A\n\n{FORMATTING_INSTRUCTIONS}" + + +def test_builder_always_carries_the_format_block() -> None: + """Judges from the AI Library's templates read the JSON shape from here. + + A history that stopped carrying it would make every such judge return prose, + turning each result into an invalid-output error. + """ + assert FORMATTING_INSTRUCTIONS in build_message_history() + + +# ─── online: inline ────────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_inline_judge_is_shown_the_trajectory( + mock_ld_client: Any, judge_variation: None +) -> None: + seen: list[dict[str, Any]] = [] + await run_judges( + config=judged_config(), + user_context=CONTEXT, + handler=capturing_judge_handler(seen), + user_input="Where is order A1?", + llm_response="It shipped.", + base_track_data={}, + trajectory="Tools available: lookup\n1. lookup\n result: shipped", + ) + + history = seen[0]["message_history"] + assert "1. lookup" in history + assert ( + history.index("Where is order A1?") + < history.index("1. lookup") + < history.index("It shipped.") + ) + + +@pytest.mark.asyncio +async def test_inline_judge_without_a_trajectory_is_unchanged( + mock_ld_client: Any, judge_variation: None +) -> None: + seen: list[dict[str, Any]] = [] + await run_judges( + config=judged_config(), + user_context=CONTEXT, + handler=capturing_judge_handler(seen), + user_input="Q", + llm_response="A", + base_track_data={}, + ) + + assert seen[0]["message_history"] == build_message_history( + user_input="Q", output="A" + ) + + +# ─── online: deferred ──────────────────────────────────────────────────────── + + +def deferred_task(**overrides: Any) -> JudgeTask: + fields: dict[str, Any] = { + "config_key": "judge-1", + "judge_config": dict(JUDGE_CONFIG), + "judge_meta": {"variationKey": "v", "version": 1}, + "actual_output": "It shipped.", + "user_context": CONTEXT, + "judge_provider": "TestProvider", + "judge_mode": "messages", + "collapse_messages": False, + "parent_track_data": {}, + } + fields.update(overrides) + return JudgeTask(**fields) + + +@pytest.mark.asyncio +async def test_deferred_judge_is_shown_the_input_and_the_trajectory( + mock_ld_client: Any, +) -> None: + """This path carried neither before, so it graded a response in isolation.""" + seen: list[dict[str, Any]] = [] + task = deferred_task( + user_input="Where is order A1?", + trajectory="Tools available: lookup\n1. lookup\n result: shipped", + ) + + await run_judge(task, [capturing_judge_handler(seen)]) + + history = seen[0]["message_history"] + assert ( + history.index("Where is order A1?") + < history.index("1. lookup") + < history.index("It shipped.") + ) + + +@pytest.mark.asyncio +async def test_deferred_and_inline_agree_on_the_same_row( + mock_ld_client: Any, judge_variation: None +) -> None: + """The point of the shared builder: same inputs, byte-identical history.""" + inline_seen: list[dict[str, Any]] = [] + deferred_seen: list[dict[str, Any]] = [] + trajectory = "Tools available: lookup\n1. lookup\n result: shipped" + + await run_judges( + config=judged_config(), + user_context=CONTEXT, + handler=capturing_judge_handler(inline_seen), + user_input="Where is order A1?", + llm_response="It shipped.", + base_track_data={}, + trajectory=trajectory, + ) + await run_judge( + deferred_task(user_input="Where is order A1?", trajectory=trajectory), + [capturing_judge_handler(deferred_seen)], + ) + + assert inline_seen[0]["message_history"] == deferred_seen[0]["message_history"] + + +def test_judge_task_stays_picklable_with_the_new_fields() -> None: + """JudgeTask crosses a thread or IPC boundary, so it must stay primitives.""" + task = deferred_task(user_input="Q", trajectory="T") + + assert pickle.loads(pickle.dumps(task)).trajectory == "T" + + +# ─── online: capture ───────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_execute_and_track_records_the_trajectory(mock_ld_client: Any) -> None: + def lookup(args: dict[str, Any]) -> str: + return f"order {args['id']} shipped" + + async def handler( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict[str, Any]: + await tool_handlers["lookup"]({"id": "A1"}) + return {"output": "It shipped.", "usage": {}} + + result = await execute_and_track( + config_key="c", + config={"model": {"name": "m"}, "provider": {"name": "TestProvider"}}, + meta={"variationKey": "v", "version": 1}, + user_context=CONTEXT, + handler=handler, # type: ignore[arg-type] + user_input="Where is order A1?", + tool_handlers={"lookup": lookup}, + ) + + assert "Tools available: lookup" in result["trajectory"] + assert '1. lookup\n arguments: {"id":"A1"}' in result["trajectory"] + assert "result: order A1 shipped" in result["trajectory"] + + +@pytest.mark.asyncio +async def test_a_native_tool_is_not_recorded_online_either( + mock_ld_client: Any, +) -> None: + """wrap_tool_handlers makes a native tool a callable stub, so it *is* + locally observable online -- but the stub returns nothing while the + provider's real result stays invisible, so recording it would show a judge + a call with an empty result. Both paths skip it identically. + """ + from launchdarkly_ai_server import NativeTool + + async def handler( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict[str, Any]: + # Not awaited: wrap_tool_handlers substitutes a *sync* zero-arg stub + # for a native tool (§3.8), unlike the async wrapper it gives a real + # callable. That asymmetry is pre-existing. + tool_handlers["web_search"]({"q": "x"}) + return {"output": "done", "usage": {}} + + result = await execute_and_track( + config_key="c", + config={"model": {"name": "m"}, "provider": {"name": "TestProvider"}}, + meta={"variationKey": "v", "version": 1}, + user_context=CONTEXT, + handler=handler, # type: ignore[arg-type] + user_input="q", + tool_handlers={"web_search": NativeTool("WebSearch")}, + ) + + assert result["trajectory"] == "" + + +@pytest.mark.asyncio +async def test_tool_tracking_still_fires_under_the_recorder( + mock_ld_client: Any, +) -> None: + """Recording is composed inside wrap_tool_handlers, so §3.8 still works.""" + + async def handler( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict[str, Any]: + await tool_handlers["lookup"]({"id": "A1"}) + return {"output": "ok", "usage": {}} + + await execute_and_track( + config_key="c", + config={"model": {"name": "m"}, "provider": {"name": "TestProvider"}}, + meta={"variationKey": "v", "version": 1}, + user_context=CONTEXT, + handler=handler, # type: ignore[arg-type] + user_input="q", + tool_handlers={"lookup": lambda args: "shipped"}, + ) + + tool_events = [ + call.args + for call in mock_ld_client.track.call_args_list + if call.args[0] == "$ld:ai:tool_call" + ] + assert len(tool_events) == 1 + assert tool_events[0][2]["toolKey"] == "lookup" + + +@pytest.mark.asyncio +async def test_a_run_with_no_tools_reports_no_trajectory(mock_ld_client: Any) -> None: + async def handler( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict[str, Any]: + return {"output": "ok", "usage": {}} + + result = await execute_and_track( + config_key="c", + config={"model": {"name": "m"}, "provider": {"name": "TestProvider"}}, + meta={"variationKey": "v", "version": 1}, + user_context=CONTEXT, + handler=handler, # type: ignore[arg-type] + user_input="q", + ) + + assert result["trajectory"] == "" + + +def test_the_recorder_renders_identically_for_both_paths() -> None: + """Both paths call render_trajectory, so one fixture pins the shape.""" + recorder = TrajectoryRecorder() + recorder.wrap({"lookup": lambda args: "shipped"}) + + assert render_trajectory( + recorder.invocations, + observable_tools=recorder.observable_tools, + omitted=recorder.omitted, + ) == ( + "Tools available: lookup\nNo tool calls were made while producing the response." + ) diff --git a/packages/client/tests/test_evaluations_trajectory.py b/packages/client/tests/test_trajectory.py similarity index 99% rename from packages/client/tests/test_evaluations_trajectory.py rename to packages/client/tests/test_trajectory.py index db90192c..4c4328bb 100644 --- a/packages/client/tests/test_evaluations_trajectory.py +++ b/packages/client/tests/test_trajectory.py @@ -5,7 +5,7 @@ import pytest -from launchdarkly_ai_server.evaluations.trajectory import ( +from launchdarkly_ai_server.trajectory import ( MAX_RECORDED_VALUE_CHARS, ToolInvocation, TrajectoryRecorder,