diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c60158f..dd4d8e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,3 +55,24 @@ jobs: python scripts/check_release_artifacts.py dist python scripts/check_dist.py dist/* python scripts/check_source_boundary.py --require-dist + + sandbox-e2e: + # The zero-flag sandbox with real openadapt-flow, a real browser, and the + # synthetic MockMed app: no fakes. Unit jobs above stub Flow's CLI. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + cache: pip + - name: Install with the browser extra + run: pip install -e ".[dev,tutorial]" + - name: Install Chromium + run: python -m playwright install --with-deps chromium + - name: Run every sandbox case through openadapt-flow + env: + OPENADAPT_AGENT_E2E: "1" + DO_NOT_TRACK: "1" + OPENADAPT_TELEMETRY: "0" + run: pytest -q tests/test_sandbox_flow.py diff --git a/README.md b/README.md index cd13b23..a5bc99d 100644 --- a/README.md +++ b/README.md @@ -3,209 +3,274 @@ [![License: MIT](https://img.shields.io/badge/license-MIT-yellow.svg)](LICENSE) [![Python 3.10–3.12](https://img.shields.io/badge/python-3.10%E2%80%933.12-blue)](https://www.python.org/downloads/) -`mcp-name: io.github.OpenAdaptAI/openadapt-agent` +Your AI agent decides what to enter. OpenAdapt enters it in the app and checks that it saved. -OpenAdapt compiles demonstrated GUI workflows into programs. This package invokes them over MCP. +Your agent already reads the referral, the intake form, or the authorization and works out what belongs in the record. Today a person still opens the EMR or the payer portal and types it in again. OpenAdapt takes that last step off your customer's team. It enters the approved values in the app they already use, on a computer they control, and then reads the saved record back. When something doesn't match, it stops and asks a person instead of guessing. -`pip install openadapt` includes this package. Point a local MCP client at `openadapt-agent serve --allow-run` and the calling agent can invoke a compiled GUI program on this computer. Healthy runs make no model calls. Flow still runs the program. Policy, identity, verification, pauses, repair, and audit stay in [`openadapt-flow`](https://github.com/OpenAdaptAI/openadapt-flow). `--allow-run` is an explicit opt-in. Admission stays fail-closed. +## How it fits -The calling agent operates. A named human is authority. They watch if the agent clicked, then they admit. Compliance samples Seals, and that is a different job than running the program. Computer-use agents are the user of OpenAdapt. They are not the executor inside OpenAdapt. +```mermaid +flowchart LR + A["Your agent decides
what to enter"] + B["OpenAdapt enters it in the app
on a computer the customer controls
and reads the record back"] + C["A result returns to your queue
done, needs review, or check the record"] + A -- "workflow, inputs, request id" --> B + B -- "one result per request" --> C +``` -Don't name this skill computer use. +Your agent sends the name of a saved task, the values to enter, and its own id for the piece of work. OpenAdapt does the entry in the same screens staff use, reads the saved record back, and returns one result to your queue. -## Install +## What changes for your customer's team -Python 3.10 through 3.12. Local Claude Code, Cursor, Codex, and Grok CLI speak stdio MCP. Author in the chat you already use. +| Step | Today | With OpenAdapt | +| --- | --- | --- | +| Read the document and decide what to enter | Your agent | Your agent | +| Open the app and find the right record | Staff | OpenAdapt, which confirms it has the right record before it types | +| Type the values and save | Staff, re-keying your agent's output | OpenAdapt, in the same screens staff use | +| Confirm the entry saved | Staff, if they look again after saving | OpenAdapt reads the saved record back before it reports done | +| Something doesn't match | Staff notice it while typing, or it's found later | OpenAdapt stops and a person decides | +| What staff handle | Every item | Only the items that stopped or need a record check | -```bash -claude mcp add openadapt -- \ - uvx --from 'openadapt-agent[tutorial]' openadapt-agent \ - serve --allow-run -``` +For scale, two hospital time studies put keying one faxed referral into an EHR at about 10 to 12 minutes ([UCSF, JAMIA Open 2020](https://pmc.ncbi.nlm.nih.gov/articles/PMC7660949/); [Calderdale and Huddersfield NHS Foundation Trust, HFMA 2024](https://www.hfma.org.uk/system/files/2024-04/Using%20digital%20technologies%20to%20process%20admin%20tasks%20Case%20Study%20v7.pdf)). This page doesn't estimate time saved. Measure it on your own workflow. -The server generates the public synthetic tutorial at serve time. It isn't in the wheel. +## What your agent gets back -```bash -python -m pip install --upgrade openadapt -openadapt flow tutorial -openadapt-agent serve --allow-run -``` +Every request ends in one of four outcomes. Each one tells your agent what to do next. -`openadapt quickstart --break-it` is the halt demo. After the verified run, the same certified bundle hits a backend that paints a success banner and rejects the write. An independent system-of-record read catches the lie. The record did not change. +| `outcome` | Plain label | What it means | What your agent does | +| --- | --- | --- | --- | +| `done` | Done and checked | OpenAdapt saved the entry and read the record back to confirm it. | Marks the item complete and keeps the `run_id`. | +| `needs_review` | Stopped before saving | Something didn't match, so it stopped before changing anything. Nothing was written. A person decides. | Waits. It doesn't start the same work again. | +| `not_sure_if_saved` | Check the record | A save may have gone through. A person checks the record before anything is retried. | Never retries. It routes the item to the person who checks. | +| `did_not_run` | Didn't run | Nothing was written. It didn't start, or a person ended it before it saved anything. | Fixes the cause, such as a bad input, and retries with the same request id when `safe_to_retry` is true. | -Local unsigned replay is free. If the tool returns unsigned success, treat it as failure. Production success without a Seal is failure. HALTED does not mint a Seal. +Each result also carries `safe_to_retry`, one plain sentence in `what_happened`, a `next_action`, and `proof`. `proof` is `local` when the record check ran on the customer's computer. It's never a reason to report a saved entry as a failure. -A first demo over stdio uses a second MCP server. `--authoring` does not enable run tools. First-demo compile needs a human ok before the tool is callable, and that ok is the whole approval. +## Three tools -```bash -claude mcp add openadapt-authoring -- \ - uvx --from 'openadapt-agent[tutorial]' openadapt-agent \ - serve --authoring +Your agent needs three calls. + +1. `list_workflows` returns each saved task's `name`, `purpose`, what `done_means`, and its typed `inputs`. +2. `run_workflow` takes `workflow`, `inputs`, `request_id`, and an optional `wait_seconds`. +3. `get_run` takes a `run_id` and returns the result of a run that was still going. + +`request_id` is your own id for the piece of work, such as the referral id. It's required. If your agent sends the same `request_id` again, it gets the first result back and nothing is entered a second time. A new attempt starts only when the first one proved nothing was written. + +Here is a real result from the sandbox, with some fields left out. The synthetic app saved the note and then showed an error, so OpenAdapt can't confirm the save and says so: + +```json +{ + "outcome": "not_sure_if_saved", + "label": "Check the record", + "safe_to_retry": false, + "what_happened": "OpenAdapt can't confirm whether the change was saved. It may have gone through. This was a sandbox run on a synthetic app; no real record was touched.", + "next_action": "Have a person check the record in the app before anything runs again. Don't retry this request.", + "record_changed": "unknown", + "proof": "none", + "workflow": "add_triage_note", + "request_id": "demo-timeout-after-save", + "run_id": "run-cef0b138a5e1488abbd805f5", + "model_calls": 0 +} ``` -`--bundles` is omitted. Probe tools are `observe`, `start_record`, `click`, and `halt`. This process stays stdio. Pass `--url --headed` to pin a fresh Playwright Chromium with empty cookies. Sign in in that window with `pause_for_input`; `continue_input` records it. `--url` is not the Chrome you already signed into, and it does not attach over CDP. Omit `--url` after you sign in in Chrome, and the unique frontmost window pins on macOS (no DOM identity). Windows native, Citrix, and RDP stay coach-only here, as does any macOS or Linux session without a unique frontmost window. +Measured on synthetic data. The full set, one result per case, is in [`docs/examples/sandbox-results.json`](docs/examples/sandbox-results.json). + +## Who builds what + +| You build | OpenAdapt provides | +| --- | --- | +| The agent that reads documents and decides what to enter | An automation built from one recording of a person doing the task in the real app | +| The call to `run_workflow`, with your own request id | The run on a computer the customer controls, with a right-record check before it types and a record check after it saves | +| What your product does with each outcome | Stops that go to a person, a durable record of every run, and the first result back for a repeated request id | +| Your tests against the four outcomes | A sandbox that returns each outcome on demand | + +Your customer provides a computer that can already open the app and a person who handles the items that stop. A normal run sends nothing to an AI model. -Private customer bundles still use `--bundles`. Those stay on the operator's disk and are never shipped here. +## Try it + +The sandbox needs no account and no flags. It serves one workflow, `add_triage_note`, on MockMed, a synthetic clinic app with made-up patients. Pick a `sandbox_case` to see each outcome. ```bash -uvx openadapt-agent serve --bundles /path/to/bundles -uvx openadapt-agent serve --bundles /path/to/bundles --allow-run +claude mcp add openadapt -- uvx --python 3.12 --from git+https://github.com/OpenAdaptAI/openadapt-agent openadapt-agent serve ``` -## ChatGPT.com and Claude.ai +PyPI has openadapt-agent 2.0.1, which predates the sandbox. From version 2.0.2 on, this shorter line does the same: -Hosted ChatGPT.com and Claude.ai can't talk to localhost stdio. Don't tell those tabs to run `openadapt-agent serve`. Send them to https://openadapt.ai/start. +```bash +claude mcp add openadapt -- uvx --python 3.12 openadapt-agent serve +``` -Those tabs still can't click the user's GUI. +Then ask your agent something like: "Use OpenAdapt to add the triage note 'Synthetic: recheck blood pressure in 2 weeks' with request id demo-0001. Then try the false_saved_banner case with request id demo-0002." -Once a job exists at `https://openadapt.ai/j/{id}`, Desktop **Connect this computer** claims `openadapt://runner`. Copy from that page and paste into the chat. +| `sandbox_case` | `outcome` | What the synthetic app does | +| --- | --- | --- | +| `normal` (default) | `done` | Saves the note. The record check finds it in the app's record store. | +| `duplicate_record` | `needs_review` | Lists the same referral twice, so OpenAdapt can't be sure which record to open. It stops before saving. | +| `false_saved_banner` | `not_sure_if_saved` | Shows a saved message, but its record store rejected the note. | +| `timeout_after_save` | `not_sure_if_saved` | Saves the note, then times out and shows an error. Only a person checking the record can tell it saved. | +| `app_offline` | `did_not_run` | Isn't reachable, so nothing starts. | -Pip users start the mailbox client: +A plain install returns simulated results and says so in every result. Add the `tutorial` extra to drive the synthetic app in a hidden browser with the real OpenAdapt engine. The first run downloads a browser and records the workflow once: ```bash -openadapt-agent authoring connect \ - 'openadapt://runner?pack=p.…&bind=oab_…&origin=https://openadapt.ai' +claude mcp add openadapt -- uvx --python 3.12 --from 'openadapt-agent[tutorial] @ git+https://github.com/OpenAdaptAI/openadapt-agent' openadapt-agent serve ``` -That claims `oab_`, polls `/j/{id}/runner/poll` with `wait_seconds: 0`, prints Allow (`y/n`) per chat account, and on pause prints `Sign in in the app, then press Enter`. Continue records with `record_observed`. It never types your password. Credentials stay on the user's computer. Don't paste a login into a vendor VM. +Pick one task your agent hands to staff today, and [talk to us about connecting it](https://openadapt.ai/qualify?why=embed). Public examples use synthetic apps and data. Each new environment gets a readiness test on its own system before it handles real records. -The job page should offer `openadapt connect ` next to Open OpenAdapt. This repo implements `openadapt-agent authoring connect`. Overlay chrome, launchd, the OS URL handler, and Keychain lease persistence stay Desktop-only. See [`docs/MAILBOX_CLI.md`](docs/MAILBOX_CLI.md). +## Developer setup -## Outcomes +Python 3.10 through 3.12. This package is a local MCP server over stdio. Claude Code, Cursor, Codex, and your own MCP client can use it. It runs on the computer that can open the app. -Every `run_workflow_` call returns one of `success`, `halt`, `refused`, `timeout`, or `error`. +### Modes -| `status` | Meaning | +| Command | What it serves | | --- | --- | -| `success` | The process exited successfully and the persisted report records `execution_outcome: VERIFIED`. Legacy reports must record `success: true`. | -| `halt` | Execution halted, completed without enough verification, or completed a rollback. Not a verified success. If `execution_outcome` is `HALTED`, tell the user the record did not change. Protected evidence stays local. | -| `refused` | A governed admission gate refused the bundle before execution. Nothing ran. | -| `timeout` | The process exceeded its deadline. The target may be partly executed. Inspect it before retrying. | -| `error` | The CLI, report, or other execution infrastructure was inconsistent. | +| `openadapt-agent serve` | The sandbox. Tools: `list_workflows`, `run_workflow`, `get_run`. | +| `openadapt-agent serve --bundles DIR` | Your compiled workflows, read-only: they're listed, not run. | +| `openadapt-agent serve --mode production --bundles DIR` | Your workflows, with runs. | +| `openadapt-agent serve --mode attended --bundles DIR --config deployment.yaml --headed` | Runs, plus decisions on paused runs by a person at this computer. | -Don't summarize any of them as success. +Operator bundles always run through the governed `openadapt-flow run` command, so every admission gate in [openadapt-flow](https://github.com/OpenAdaptAI/openadapt-flow) still applies. Results and run records live under `--runs-dir` (default `./runs`; the sandbox uses `~/.openadapt/agent-sandbox`). -A timeout isn't a rollback. Don't retry until you've looked at the durable run. +If your agent runs in your cloud rather than on the customer's computer, it can't reach this stdio server. [Talk to us](https://openadapt.ai/qualify?why=embed) about the hosted route. -`get_run_report` returns a PHI-safe status and a count-only summary for a run this server created. The persisted report stays in the local operator experience unless you enabled protected export. +### Describe a workflow for agents -## Trust boundary - -Don't expose this process's stdin and stdout as an unauthenticated network service. It inherits the local user's OS permissions, and Flow records that OS account as the operator for attended decisions. +`list_workflows` reads types and choices from the compiled workflow. Add an `openadapt-card.json` next to the bundle's `workflow.json` to give it a name, a purpose, and input descriptions: -By default every MCP response is safe to render outside the protected workflow-data boundary. Labels, recorded values, paths, raw reports, observed text, stdout, stderr, and local exception messages stay on the OpenAdapt machine. The client gets opaque IDs, fixed outcome copy, declared parameter names and types, and count or boolean metrics. +```json +{ + "name": "enter_referral", + "purpose": "Create an outgoing referral for an existing patient in the clinic EMR.", + "done_means": "The referral appears in the EMR's referral report for that patient.", + "inputs": { + "patient_mrn": {"description": "Clinic MRN of the patient.", "pattern": "^[0-9]{6,10}$"}, + "referral_date": {"description": "Date of the referral.", "format": "date"} + } +} +``` -`--allow-protected-export` sends raw local metadata and evidence to the MCP client. Use it only when that client is trusted and inside the same protected data boundary. +Card text goes to the calling agent and, through it, to a hosted model. Never put patient details in it. The server ignores a card that looks like it carries an identifier and logs a warning on this computer. -`--allow-synthetic-recorded-defaults` lets omitted parameters reuse recorded values. It requires `--allow-run`. Synthetic demonstrations only. Production runs require every declared parameter so a wrong-record action cannot hide in a default. +### Technical details -Remote transport, account identity, tenant isolation, fleet policy, and managed execution belong to OpenAdapt Cloud. They aren't duplicated here. +Results come from openadapt-flow's `transaction_outcome`, never from the coarse halt label alone. -The complete contract is in [docs/DESIGN.md](docs/DESIGN.md). +| Flow `transaction_outcome` | `outcome` | `safe_to_retry` | +| --- | --- | --- | +| `VERIFIED` | `done` | false | +| `HALTED_BEFORE_EFFECT` | `needs_review` | false | +| `REJECTED_POLICY`, `CANCELED`, `FAILED_PLATFORM` while a durable pause is open | `needs_review` | false | +| `REJECTED_POLICY`, `CANCELED`, `FAILED_PLATFORM` with no open pause | `did_not_run` | true | +| `RECONCILIATION_REQUIRED`, `COMPLETED_UNVERIFIED`, `ROLLED_BACK` | `not_sure_if_saved` | false | +| A timeout after Flow started, an unreadable report, or a server that stopped mid-run | `not_sure_if_saved` | false | +| An admission refusal before Flow started, with closed `failed_checks` codes | `did_not_run` | true | +| A person ended the run at its pause | `did_not_run` | false | -## Serve a bundle +`reason` names the case from a closed list, and `technical` carries Flow's exact `execution_outcome` and `transaction_outcome`. `needs_attention_id` points at the local Needs Attention item when there is one. `receipt_url` and `review_url` are reserved for the hosted service; this local server leaves them out. `run_workflow` and `get_run` publish an MCP `outputSchema` and return `structuredContent`. The schema is `RUN_RESULT_SCHEMA` in [`src/openadapt_agent/contract.py`](src/openadapt_agent/contract.py). -`--allow-run` with no `--bundles` records and compiles the synthetic MockMed workflow, certifies it, then keeps that app up so a governed run can hit a live system of record. `--tutorial` is the same path without implying run tools. +Each attempt reserves `:` in openadapt-flow's `IdempotencyLedger` before anything runs. Run records hold the result only. Inputs are never stored; a keyed fingerprint catches a `request_id` reused with different inputs. -```bash -openadapt-agent serve --allow-run --runs-dir /tmp/openadapt-runs -``` +### Older tools and flags -The client gets `list_workflows`, `get_workflow`, `get_run_report`, `list_needs_attention`, and `get_attention_item`. Those are read-only until `--allow-run`. The synthetic tutorial registers `run_local_quickstart`. Add `--allow-run` and the server registers one typed `run_workflow_` tool per loadable bundle. Declared parameters are required. Recorded demonstration values never appear in the schema and are not reused unless you ask for that. +Existing clients keep working. With `--bundles`, the server also registers the older tools: -| Tool | Registration | +| Tool | Status | | --- | --- | -| `list_workflows` | Always | -| `get_workflow` | Always | -| `get_run_report` | Always | -| `list_needs_attention` | Always | -| `get_attention_item` | Always | -| `run_workflow_` | `--allow-run` | -| `run_local_quickstart` | `--allow-run` with no `--bundles` | -| `reject_attention`, `teach_attention`, `escalate_attention` | `--allow-attended-actions` | -| `continue_attention`, `skip_attention` | `--allow-attended-actions` plus a qualified deployment `--config` | -| `observe`, `start_record`, `click`, `halt`, `compile`, `admit` | `--authoring` | +| `get_workflow`, `get_run_report`, `list_needs_attention`, `get_attention_item` | Kept. `get_run_report` also finds runs started with `run_workflow`. | +| `run_workflow_` | Deprecated. Use `run_workflow`, which takes a request id. The old tools now return the same `outcome`, `safe_to_retry`, and `what_happened` fields, and a verified run reports `success` with `proof: "local"`. | +| `reject_attention`, `teach_attention`, `escalate_attention` | `--mode attended` (or `--allow-attended-actions`) | +| `continue_attention`, `skip_attention` | `--mode attended` plus a qualified deployment `--config` | + +`openadapt-agent serve --allow-run` is the older form of `--mode production` when you pass `--bundles`, and of the sandbox when you don't. `--tutorial` is the older name for the sandbox. `--allow-attended-actions` is part of `--mode attended`. + +### Trust boundary + +Don't expose this process's stdin and stdout as an unauthenticated network service. It inherits the local user's OS permissions, and Flow records that OS account as the operator for attended decisions. + +By default every MCP response is safe to render outside the protected workflow-data boundary. Labels, recorded values, paths, raw reports, observed text, stdout, stderr, and local exception messages stay on this computer. The client gets workflow cards, fixed outcome copy from a closed list, opaque ids, and count or boolean metrics. + +`--allow-protected-export` sends raw local metadata and evidence to the MCP client. Use it only when that client is trusted and inside the same protected data boundary. + +`--allow-synthetic-recorded-defaults` lets omitted inputs reuse recorded values. It requires runs. Synthetic demonstrations only: production runs require every declared input so a wrong-record entry can't hide in a default. -`--headed` keeps the attended web session visible. The MCP server is stdio. The target app still has a window. +Remote transport, account identity, tenant isolation, fleet policy, and managed execution belong to OpenAdapt Cloud. They aren't duplicated here. The complete design is in [docs/DESIGN.md](docs/DESIGN.md). -## Finish a halt +### Finish a paused run -`continue_attention` does not perform the paused action again. Point the server at Flow's qualified deployment config if a local operator needs to finish an exception and continue the same durable run: +`continue_attention` doesn't perform the paused action again. Point the server at Flow's qualified deployment config when a person at this computer finishes an exception and continues the same durable run: ```bash openadapt-agent serve \ + --mode attended \ --bundles /opt/openadapt/bundles \ --runs-dir /var/lib/openadapt/runs \ - --allow-run \ - --allow-attended-actions \ --config /etc/openadapt/deployment.yaml \ --headed ``` -The queue summary is safe to show in an agent UI. It carries opaque IDs, typed categories, counts, and signed-capability metadata. Observed text, workflow values, reports, and local paths stay off that surface. - | Tool | What happens | | --- | --- | -| `continue_attention` | The operator confirms they completed the paused task in the live app. Flow revalidates postconditions and independent effects, checkpoints it as human-completed, and resumes after it. It does not perform the completed action again. | -| `skip_attention` | Flow applies only an already-declared, non-consequential skip. A stale, undeclared, consequential, or ambiguous skip is refused. | -| `reject_attention` | Ends this run and dispatches no new action. Earlier steps can still have effects, so read the protected local report. Escalate if a qualified operator can still continue. | -| `teach_attention` | Records an audited request for a corrective demonstration. Flow's revision and regression gates decide what can be promoted. | -| `escalate_attention` | Records an audited escalation and leaves the exact durable pause intact. | +| `continue_attention` | The person confirms they completed the paused task in the live app. Flow rechecks the screen and the record, checkpoints it as done by a person, and resumes after it. | +| `skip_attention` | Flow applies only an already-declared skip that changes no record. | +| `reject_attention` | Ends this run and dispatches no new action. Earlier steps can still have effects, so read the protected local report. | +| `teach_attention` | Records an audited request for a corrective demonstration. | +| `escalate_attention` | Records an audited escalation and leaves the pause in place. | -Every mutation needs the opaque queue-item ID, the item's current capability digest, a caller-stable idempotency key, and an action-specific `true` confirmation. Before the server submits that to Flow, it opens an MCP form elicitation and the local operator has to accept it. That's a host-mediated confirmation, not cryptographic proof of who clicked. Flow records the effective local OS account as the operator. - -Clients without form elicitation cannot execute attended actions through this bridge. The same Continue, Skip, Reject, Teach, and Escalate capabilities stay available through Flow's attended console: +Every decision needs the queue item id, its current capability digest, a stable idempotency key, and an action-specific `true` confirmation. The server then asks the person to confirm through MCP form elicitation. Clients without form elicitation use Flow's attended console instead: ```bash -openadapt-flow console \ - --attend \ - --allow-actions \ - --bundles /opt/openadapt/bundles \ - --runs /var/lib/openadapt/runs \ - --config /etc/openadapt/deployment.yaml \ - --headed +openadapt-flow console --attend --allow-actions \ + --bundles /opt/openadapt/bundles --runs /var/lib/openadapt/runs \ + --config /etc/openadapt/deployment.yaml --headed ``` -`--allow-attended-actions` without `--config` still registers Reject, Teach, and Escalate, and it withholds Continue and Skip until Flow can construct the deployment-bound live verifier and backend. `--headed` is required for web Continue and Skip. +After a person decides, `get_run` shows the new outcome. -## Emit an Agent Skill +### Record a first workflow with your agent -`openadapt-agent emit-skill` wraps Flow's own skill emitter and writes a folder you can drop in `~/.claude/skills`. +`--authoring` adds first-demo tools over the same local stdio server: `observe`, `start_record`, `click`, `halt`, `type`, `pause_for_input`, `continue_input`, `stop_record`, `compile`, `admit`, and helpers. It doesn't enable runs. ```bash -openadapt-agent emit-skill \ - /tmp/openadapt-agent-demo/bundle \ - --out ~/.claude/skills +claude mcp add openadapt-authoring -- uvx --python 3.12 --from 'openadapt-agent[tutorial] @ git+https://github.com/OpenAdaptAI/openadapt-agent' openadapt-agent serve --authoring ``` -The wrapper keeps Flow's portable bundle and appends MCP invocation, halt handling, Needs Attention, and attended-action guidance. The frontmatter description is the same sentence as `server.json` and `llms.txt`. The skill is named from the workflow slug, never "computer use". If the tool returns HALTED, tell the user the record did not change. +Pass `--url --headed` to open a fresh Playwright Chromium with empty cookies, and sign in there during `pause_for_input`. Windows native, Citrix, and RDP sessions are coach-only here. Hosted ChatGPT.com and Claude.ai can't reach localhost stdio; send them to https://openadapt.ai/start. Once a job exists at `https://openadapt.ai/j/{id}`, `openadapt-agent authoring connect` claims it over outbound HTTPS (the job page should offer `openadapt connect `). See [docs/MAILBOX_CLI.md](docs/MAILBOX_CLI.md). + +To see the engine on its own without an agent, `openadapt flow tutorial` runs the synthetic app from the OpenAdapt launcher, and `openadapt quickstart --break-it` shows a fake saved message caught by the record check. -A first-party copy lives at [`skills/openadapt-gui-write/SKILL.md`](skills/openadapt-gui-write/SKILL.md). A folder emitted from a private bundle isn't a sanitized derivative. It includes the compiled bundle, so it's protected workflow data. +### Agent Skills -## Product state +`openadapt-agent emit-skill BUNDLE --out ~/.claude/skills` writes a skill for one workflow from its card: purpose, inputs, how to call `run_workflow`, and how to read the four outcomes. A skill is loaded into model context, so it carries no recorded values, step text, or secrets, and each skill's description names its own workflow. `--include-bundle` also copies the compiled bundle, which is protected workflow data. A first-party skill lives at [`skills/openadapt-gui-write/SKILL.md`](skills/openadapt-gui-write/SKILL.md). The skill is named after the workflow, never "computer use". + +### Product state An exact Agent release enters Production only through an active signed admission. That admission expires and can be revoked. A missing, expired, revoked, mismatched, or unverifiable admission produces **not actively admitted**. The validator doesn't restore an older admission or assign a fallback lifecycle label. Check the [current signed Production record](https://openadapt.ai/production-lifecycle.json). That ledger currently has seven target admissions. Evidence class is `remote-safe-synthetic`. -Standard and Regulated also need an active workflow admission for the exact bundle version. Demo and the synthetic tutorial may run without one. The public workflow ledger is [production-workflow-admissions.json](https://openadapt.ai/production-workflow-admissions.json). It lists seven synthetic admissions (`0.0.0-synthetic`). That isn't a customer job. +Standard and Regulated runs also need an active workflow admission for the exact bundle version. Demo and the synthetic sandbox may run without one. The public workflow ledger is [production-workflow-admissions.json](https://openadapt.ai/production-workflow-admissions.json). It lists seven synthetic admissions (`0.0.0-synthetic`). That isn't a customer job. -## Package history +### Package history and listings -Before v2 this repository wrapped model-driven GUI agents. That was the wrong product. The execution path now lives in `openadapt-flow`. The current name stays because the package bridges MCP and Agent Skills. +Before v2 this repository wrapped model-driven GUI agents. The execution path now lives in `openadapt-flow`. The name stays because the package connects agents to it over MCP and Agent Skills. -A user's compiled workflow is their private artifact. Supply it at launch with `--bundles`. It is never embedded in the package or a registry listing. See [`docs/DISTRIBUTION.md`](docs/DISTRIBUTION.md). +A customer's compiled workflow is their private artifact. Supply it at launch with `--bundles`. It is never embedded in the package or a registry listing. See [`docs/DISTRIBUTION.md`](docs/DISTRIBUTION.md). -The MCP registry reads [`server.json`](server.json). Smithery packs from [`manifest.json`](manifest.json). [`llms.txt`](llms.txt) exists because hosted assistants scrape a file, not this README. Registry-launched installs start read-only. Execution tools wait for `--allow-run`. +The MCP registry reads [`server.json`](server.json). Smithery packs from [`manifest.json`](manifest.json). [`llms.txt`](llms.txt) exists because hosted assistants read a file, not this README. Registry-launched installs start read-only. -## Development +`mcp-name: io.github.OpenAdaptAI/openadapt-agent` -`pytest -q` covers README identity and the MCP registry manifests. It also pins the PHI-safe default. +### Development ```bash pip install -e ".[dev]" ruff check src tests scripts pytest -q +OPENADAPT_AGENT_E2E=1 pytest -q tests/test_sandbox_flow.py # needs .[tutorial] and Chromium +python scripts/smoke_client.py # the sandbox over real stdio python -m build python scripts/check_release_artifacts.py dist python scripts/check_dist.py dist/* diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 0a44abf..91a805b 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -12,10 +12,10 @@ workflow ledger is https://openadapt.ai/production-workflow-admissions.json. It currently lists seven synthetic admissions (`0.0.0-synthetic`). That isn't a customer job. Standard and Regulated need an active workflow admission -for the exact bundle version. Demo and the synthetic tutorial may run +for the exact bundle version. Demo and the synthetic sandbox may run without one. -OpenAdapt compiles demonstrated GUI workflows into programs. This package invokes them over MCP. +Your AI agent decides what to enter. OpenAdapt enters it in the app and checks that it saved. `openadapt-agent` is the default runtime interface for a calling agent. It is not a second workflow engine, and it is not "OpenAdapt the agent." Flow @@ -26,19 +26,18 @@ and audit. The CLI remains. Headless does not mean the UI is gone. | Role | Who | What they do | |---|---|---| -| Operator | Calling agent (Claude Code, MCP client, orchestrator) | Discover, bind parameters, invoke, read outcomes | -| Authority | Named human | Admit the compiled program (watch if the agent clicked), certify policy, resolve identity / effect / judgment halts | -| Auditor | Compliance | Sample seals, revoke admission | - -Computer-use agents are the user of OpenAdapt. They are not the executor -inside OpenAdapt. Never summarize halt as success. A local agent may click -during the demonstration while the person watches. The calling agent must -not be the sole source of a production demonstration of a novel GUI path. - -Halt packets and typed agent-continue (missing parameter / retryable -transport only) are not in this package yet. Identity, effect, expired -policy, novel UI, and admission still require a human. Do not let a -planner resolve those classes. +| Calling agent | Your AI agent (Claude Code, an MCP client, an orchestrator) | Chooses a workflow, sends inputs and its own `request_id`, acts on the outcome | +| Operator | A person at the computer that runs workflows | Starts the server, answers paused runs, checks records the result says to check | +| Authority | Named person | Approves the compiled workflow, certifies policy, resolves identity, record-check, and judgment stops | +| Auditor | Compliance | Reviews receipts, revokes admission | + +"Operator" always means a person. Computer-use agents are the user of +OpenAdapt. They are not the executor inside OpenAdapt. A local agent may +click during the demonstration while the person watches. The calling agent +must not be the sole source of a production demonstration of a novel GUI +path. Identity, record-check, expired-policy, novel-UI, and admission stops +still require a person. A calling agent never resolves those classes; it +reads `needs_review` or `not_sure_if_saved` and waits. `openadapt-agent` exposes two complementary interfaces: @@ -50,18 +49,22 @@ planner resolve those classes. ```text MCP client / Agent Skill │ - │ local stdio + exact JSON schemas + │ local stdio + exact JSON schemas (inputSchema / outputSchema) ▼ openadapt_agent.mcp (local stdio only; HTTP shim forbidden) │ - ├── openadapt_agent.bridge - │ ├── bundle discovery and typed run tools + ├── openadapt_agent.bridge tool specs + dispatch + │ ├── list_workflows / run_workflow / get_run (the contract) + │ │ ├── cards workflow cards: name, purpose, done_means, typed inputs + │ │ ├── service validate → request gate → worker thread → wait + │ │ ├── runs durable records + Flow IdempotencyLedger + │ │ └── contract transaction_outcome → four outcomes, closed reasons + │ │ │ + │ │ ├── operator bundles ─► openadapt-flow run subprocess + │ │ └── sandbox ─────────► Flow run gate + Replayer on MockMed + │ ├── older tools: get_workflow, get_run_report, run_ (deprecated) │ ├── PHI-safe Needs Attention projection - │ ├── action-specific operator decisions - │ └── structured success / halt / refusal results - │ │ - │ ├── new run ──► openadapt-flow run subprocess - │ └── attended ─► openadapt-flow durable API + │ └── action-specific attended decisions ─► openadapt-flow durable API │ ├── openadapt_agent.authoring (--authoring first demo, stdio) │ observe / start_record / click / halt @@ -84,50 +87,139 @@ starting stdio. ## Why the package keeps both MCP and Agent Skills -Flow's skill emitter produces the portable workflow folder. This package -wraps it and appends agent-facing invocation, halt, and attended-action -guidance without regenerating the workflow. The resulting skill includes -the compiled bundle and is protected workflow data, not a sanitized -artifact; operators install it only into an agent authorized for the -same data boundary. - -Flow's per-bundle MCP emitter has a different purpose: a self-contained -demo server for one workflow. `openadapt-agent serve` is the governed -multi-bundle local surface. It uses Flow's `run` admission path and -returns durable evidence rather than invoking permissive replay. +`openadapt-agent emit-skill` writes a skill from the workflow card: name, +purpose, what done means, typed inputs, how to call `run_workflow`, and how +to read the four outcomes. A skill is loaded into model context, which for +most clients means a hosted model, so it never carries recorded example +values, step intents, observed text, typed secrets, local paths, or a +replay recipe that would fall back to recorded values. Each description +names its own workflow and inputs. `--include-bundle` copies the compiled +bundle next to `SKILL.md`; that copy is protected workflow data. + +Flow's per-bundle MCP emitter and `openadapt-flow emit-skill` have a +different purpose: self-contained replay demos for one workflow. +`openadapt-agent serve` is the governed multi-bundle local surface. It +uses Flow's `run` admission path and returns durable evidence rather than +invoking permissive replay. The repository therefore remains `openadapt-agent`, not `openadapt-mcp`: MCP is one transport; Agent Skills are an equally supported surface. -## Tool registration +## The result contract -The server always registers: +Every agent-facing result comes from `openadapt_agent.contract`. It has +four terminal outcomes, each mapped to one decision the calling agent +makes: -- `list_workflows` -- `get_workflow` -- `get_run_report` -- `list_needs_attention` -- `get_attention_item` +| `outcome` | Meaning | `safe_to_retry` | +|---|---|---| +| `done` | Saved and checked by reading the record back | false | +| `needs_review` | Stopped before saving; nothing written; a person decides | false | +| `not_sure_if_saved` | A save may have gone through; a person checks the record | false | +| `did_not_run` | Nothing was written | true when a retry may succeed | + +`running` is the one non-terminal value. The plain result is a pure +function of Flow's `transaction_outcome`, the process facts (did Flow +start, did it exit cleanly, did it time out), the state of any durable +pause, and a consistency check of the persisted report: -The first three return PHI-safe projections: opaque IDs, parameter -names/types, availability, status, and count/boolean metrics. They do not +| Flow evidence | `reason` | `outcome` | +|---|---|---| +| consistent `VERIFIED`, exit 0 | `saved_and_checked` | `done` | +| `HALTED_BEFORE_EFFECT` | from the pause category, for example `record_not_confirmed` | `needs_review` | +| `REJECTED_POLICY`, `CANCELED`, `FAILED_PLATFORM` with an open durable pause | from the pause category | `needs_review` | +| the same with no open pause | `policy_refused`, `canceled`, `platform_error` | `did_not_run` | +| any proven-no-effect outcome whose pause a person rejected | `stopped_by_person` | `did_not_run` (no retry) | +| `RECONCILIATION_REQUIRED` | `save_not_confirmed` | `not_sure_if_saved` | +| `COMPLETED_UNVERIFIED` | `not_checked` | `not_sure_if_saved` | +| `ROLLED_BACK` | `change_reversed` | `not_sure_if_saved` | +| timeout after start, inconsistent or missing report, coarse `HALTED` alone | `timed_out`, `result_unreadable` | `not_sure_if_saved` | +| a server that stopped mid-run | `interrupted` | `not_sure_if_saved` | +| exit 2 before execution | `not_ready_to_run` with closed `failed_checks` | `did_not_run` | +| Flow could not be launched | `platform_error` | `did_not_run` | +| the app isn't reachable, checked before anything is sent (sandbox) | `app_unreachable` | `did_not_run` | +| invalid inputs, unknown workflow, runs disabled | `invalid_input`, `unknown_workflow`, `runs_disabled` | `did_not_run` | + +Only `done` says the change was saved. `record_changed: "no"` and +`safe_to_retry: true` appear only when the evidence proves nothing was +written. A coarse `HALTED` without a transaction outcome is uncertain, +never "nothing changed". A failed Needs Attention lookup counts as an open +pause, so the run waits for a person. `proof` (`local`, `sealed`, +`simulated`, or `none`) is separate from the outcome: a verified local run +is `done` with `proof: "local"`, never an error. A partner whose policy +requires a signed receipt checks `proof`. + +Every sentence in `what_happened` and `next_action` is fixed copy keyed by +a closed reason code. No observed screen text, recorded value, input value, +caller-supplied name, or local path reaches a result. + +### Requests, retries, and records + +`request_id` is required on `run_workflow`. Each attempt reserves +`:` in openadapt-flow's `IdempotencyLedger` +(namespace `openadapt-agent/run-workflow/v1`, under +`/openadapt-agent/`) before anything runs. A lost race reads back +the winner. The same `request_id` returns the latest attempt's result with +`replayed: true`. A new attempt starts only when the latest one ended in a +reason from `RETRYABLE_REASONS`, each of which proves nothing was written, +and at most five times. A `request_id` reused with different inputs is +refused as `request_id_conflict`. If Flow's ledger can't be opened, local +exclusive-create claim files give the same at-most-once guarantee for that +runs directory. + +Every call leaves a record at +`/openadapt-agent/runs/.json`, refusals included, so +`get_run` can always answer. Records hold the contract result only. Inputs +are never stored; a keyed fingerprint of `(workflow, inputs)` is. + +`run_workflow` runs a new attempt on a worker thread and waits up to +`wait_seconds` (default 60, maximum 600). A run still going returns +`outcome: "running"` with its `run_id`. `get_run` waits the same way. A +record left `running` by a process that is gone reads as `interrupted`. A +`needs_review` result is re-read from Flow on each `get_run`, so a +person's Continue or Reject shows up. A re-read never turns a reviewed run +into a retryable result: Flow clears a continued pause just before it +replaces the pre-resume report, so the run keeps waiting until the new +report lands. + +### Workflow cards + +`list_workflows` returns one card per workflow. Types and choices come from +Flow's `param_specs`. Words come from an optional operator-written +`openadapt-card.json` next to `workflow.json`. Card text is +length-limited and refused when it looks like it carries an identifier (a +run of five or more digits, an email address, a phone number). A refused +card falls back to the default card with a local warning that never +repeats the text. Every declared input stays required unless +`--allow-synthetic-recorded-defaults` is set. + +## Modes and tool registration + +| Start | Mode | Tools | +|---|---|---| +| `serve` (no flags), `--mode sandbox`, `--tutorial`, `--allow-run` without `--bundles` | sandbox | `list_workflows`, `run_workflow`, `get_run` | +| `--bundles DIR` | production, read-only | `list_workflows`, `get_run`, `get_workflow`, `get_run_report`, `list_needs_attention`, `get_attention_item` | +| `--mode production --bundles DIR` (or `--allow-run`) | production | the above plus `run_workflow` and deprecated `run_` | +| `--mode attended --bundles DIR` (or `--allow-run --allow-attended-actions`) | attended | the above plus Reject, Teach, Escalate; Continue and Skip with a qualified `--config` | +| `--bundles DIR --allow-attended-actions` (older flags, no runs) | attended, read-only | the read-only tools plus the same decisions on paused runs | + +The read-only tools return PHI-safe projections: workflow cards, opaque +ids, availability, status, and count or boolean metrics. They do not return workflow labels, recorded values, step intents, report bodies, observed text, local paths, subprocess output, or exception text. The attention tools use the same boundary plus typed categories, artifact -IDs, and non-authorizing capability metadata. - -`--allow-run` registers one `run_workflow_` tool per loadable -workflow. `--allow-run` with no `--bundles` generates the public synthetic -MockMed bundle at serve time and registers `run_local_quickstart`. -`--tutorial` is the same generator without implying run tools. Every declared parameter -is required by default, and recorded demonstration values never enter the -schema. Missing or unknown parameters are rejected before the subprocess -starts. A per-call URL is accepted only if the operator separately enabled -`--allow-url-override`. The tutorial path still uses Flow's fail-closed -`run` verb. It does not pass `--approve-unverified-writes`. - -Two server-start modes are intentionally separate from ordinary +ids, and non-authorizing capability metadata. + +The deprecated per-workflow `run_` tools keep their schemas and +legacy `status` field and add the contract fields. They take no +`request_id`, so their descriptions say a retry can write twice. Missing +or unknown parameters are rejected before the subprocess starts. A +per-call URL is accepted only if the operator separately enabled +`--allow-url-override`. No run path passes +`--approve-unverified-writes`. + +Two server-start options are intentionally separate from ordinary operation: - `--allow-protected-export` includes raw labels, values, intents, local @@ -137,10 +229,6 @@ operation: use demonstrated values. It requires run authority and is only for synthetic demos; it never places those values in a tool schema. -`--allow-attended-actions` registers Reject, Teach, and Escalate. Continue and -Skip are registered only when a deployment configuration lets Flow -construct its bound live executor. - `--authoring` registers first-demo tools over the same local stdio server. Probe names match hosted MCP: `observe`, `start_record`, `click`, `halt`. Local stdio may also include `type` for agent-driven @@ -154,7 +242,7 @@ digest. If the Flow session has no `admit`, the tool fails closed and does not mint a Seal or write an unsigned ledger row. `--authoring` does not imply `--allow-run`. `--bundles` is optional iff -`--authoring` (or the existing `--tutorial` / implied-tutorial path). The +`--authoring` (or the sandbox). The published run recipe in `server.json` still requires `--bundles` and stays `transport: stdio`. Authoring is a first demo; there is no bundle yet. @@ -178,41 +266,66 @@ session, and the mailbox client against a mocked wait=0 poll. ## Governed runs -Each run tool shells out to the `openadapt-flow` installed in the same -Python environment. The command uses Flow's fail-closed `run` verb, so -its certification, identity, effects, encryption, integrity, and egress -gates remain authoritative. +Each operator-bundle run shells out to the `openadapt-flow` installed in +the same Python environment. The command uses Flow's fail-closed `run` +verb, so its certification, identity, effects, encryption, integrity, and +egress gates remain authoritative. -For a precise Flow report, the bridge reports success only when all conditions hold: +The bridge reports `done` (legacy `status: "success"`) only when all of +these hold: 1. Flow exits with code 0. -2. The persisted report has `execution_outcome: VERIFIED`. -3. The persisted report has a consistent `success: true` value. -4. The report is not production-eligible. Local MCP never mints a Seal, - so production-eligible `VERIFIED` is unsigned success and is failure. - -Write tools advertise `requires_seal: true` on MCP `_meta`. If the tool -returns unsigned success, treat it as failure. - -For a legacy report without `execution_outcome`, the bridge preserves the -compatible rule: exit code 0 plus `success: true`. Local unsigned replay -is free. Production success without a Seal is failure. Standard and -Regulated need an active workflow admission for the exact bundle version. -Demo and the synthetic tutorial may run without one. - -Exit 1 is a halt. Exit 2 is a governed refusal before execution. A -timeout is explicitly uncertain rather than a rollback. Report evidence -always outranks a process exit code. +2. The persisted report has `transaction_outcome: VERIFIED` and + `execution_outcome: VERIFIED`. +3. The persisted report has a consistent `success: true` value and a + consistent outcome envelope. + +A production-eligible `VERIFIED` run is `done` with `proof: "local"`. +Local MCP mints no Seal, and reporting a verified write as an error would +invite a retry and a duplicate write. A legacy report with only the +`success` flag is `not_sure_if_saved` ("finished, not checked"). Standard +and Regulated need an active workflow admission for the exact bundle +version. Demo and the synthetic sandbox may run without one. + +Exit 2 is a governed refusal before execution, mapped to closed +`failed_checks` codes. A timeout is uncertain, never a rollback. Report +evidence always outranks a process exit code. Parameters are passed in a mode-`0600` temporary JSON file and removed after the run. Server-fixed target, policy, deployment configuration, timeout, and extra Flow arguments cannot be changed by an MCP call. The runner retains detailed reports and subprocess diagnostics locally, -but its default MCP projection contains only an opaque workflow/run ID, -fixed status copy, and schema-limited metrics. Success, halt, refusal, -timeout, and infrastructure-error paths share this same projection, so -an exception cannot accidentally turn into an egress channel. +but its default MCP projection contains only the contract fields, opaque +ids, fixed copy, and schema-limited metrics. Every outcome path shares +this projection, so an exception cannot turn into an egress channel. + +## Sandbox + +With no `--bundles`, the server serves one synthetic workflow, +`add_triage_note`, on MockMed, the synthetic clinic app that ships with +openadapt-flow. The `sandbox_case` input selects what the app does: +`normal` (done), `duplicate_record` (needs_review), `false_saved_banner` +and `timeout_after_save` (not_sure_if_saved), and `app_offline` +(did_not_run). Results always carry `mode: "sandbox"` and the engine name. + +- The `flow` engine needs the `tutorial` extra. In a background thread at + start, Flow records the workflow once (reused afterwards), compiles it + with mined record checks, binds the mined `note` read-back to the `note` + parameter (the operator review Flow leaves open after mining), and + certifies it under `clinical-write`. Each run is admitted by Flow's run + gate under the standard profile and executed by Flow's `Replayer` with + the caller's note, a right-record check, and an independent read of + MockMed's record store. This is the same path Flow's own tutorial uses. +- The `simulated` engine returns the same contract results without opening + any app. Results open with "Simulated sandbox result" and `done` carries + `proof: "simulated"`. + +`--sandbox-engine auto` picks `flow` when Playwright and a compatible Flow +are installed, and falls back to `simulated` when the app can't start. The +sandbox keeps its records under `~/.openadapt/agent-sandbox` (or +`OPENADAPT_AGENT_SANDBOX_DIR`, or `--runs-dir`), so it never writes into +the project a client was launched from. ## Needs Attention @@ -375,9 +488,20 @@ Tests cover: - Reject, Teach, and Escalate without a live service; - delegation to Flow's public service context; - compatibility with Flow's public, thread-owned attended service; -- success/halt/refusal/timeout outcome mapping; +- the four-outcome contract: every reason against the output schema, real + Flow 1.35.1 outcome shapes, pause-aware mapping, and the rule that only + proven no-effect results allow a retry; +- `request_id` replay, conflicts, retry limits, interrupted runs, and + reviewed runs a person resolves; +- `outputSchema` and `structuredContent` on MCP SDK 1 and SDK 2; +- workflow cards: Flow types, operator text, identifier guard, closed input + problems; +- every sandbox case with the simulated engine, and, in the opt-in + `sandbox-e2e` job, with real Flow, a real browser, and MockMed; +- success/halt/refusal/timeout outcome mapping for the older tools; - MCP serialization and thread ownership; -- Agent Skill emission; +- Agent Skill emission without recorded values, step intents, or secrets, + and one description per workflow; - `--authoring` probe tools (`observe`, `start_record`, `click`, `halt`) and local `type`; observe projection drops values/titles/screenshots and extra keys, caps the wire at 32 KiB, and uses `n_` + 8 hex node diff --git a/docs/DISTRIBUTION.md b/docs/DISTRIBUTION.md index e143e43..b6b8ed2 100644 --- a/docs/DISTRIBUTION.md +++ b/docs/DISTRIBUTION.md @@ -8,7 +8,7 @@ https://openadapt.ai/production-lifecycle.json (seven target admissions, https://openadapt.ai/production-workflow-admissions.json (seven synthetic admissions, `0.0.0-synthetic`). That isn't a customer job. Standard and Regulated need an active workflow admission for the exact bundle version. -Demo and the synthetic tutorial may run without one. +Demo and the synthetic sandbox may run without one. This document describes how the package is made installable and discoverable as an MCP server, the security-relevant distinction @@ -27,7 +27,7 @@ server". Keep them separate. | **What it is** | the `openadapt-agent` package and its MCP server *program* | a user's compiled `openadapt-flow` workflow *bundle* | | **What it exposes** | PHI-safe workflow and Needs Attention projections plus opt-in governed run and attended-action tools | one customer's specific recorded workflow (its steps, parameters, recorded example values) | | **Where it lives** | PyPI + MCP registries (official, Smithery, mcp.so, Glama, PulseMCP) | the operator's own disk, passed at launch via `--bundles` | -| **Ships in the package?** | yes — code only | **never** — no bundle is embedded in the wheel, `server.json`, or any registry listing. `serve --tutorial` generates the public synthetic MockMed bundle at serve time. | +| **Ships in the package?** | yes — code only | **never** — no bundle is embedded in the wheel, `server.json`, or any registry listing. The zero-flag sandbox generates its synthetic MockMed workflow at serve time. | **Design consequence:** the published server takes the bundle directory as a launch-time argument (`--bundles `) and reads nothing about a @@ -38,12 +38,13 @@ inside their trust boundary. The registry listing advertises the *capability*, not any workflow. **Read-only by default when registry-launched.** The official `server.json` -omits `--allow-run`, and the MCPB `manifest.json` leaves both execution and -attended actions off until the operator enables them in the install form. A -default install yields PHI-safe inspection and Needs Attention tools only -(`list_workflows`, `get_workflow`, `get_run_report`, -`list_needs_attention`, and `get_attention_item`). Enabling workflow runs -adds the dynamic `run_workflow_` tools; enabling attended actions +omits `--mode` and `--allow-run`, and the MCPB `manifest.json` leaves both +execution and attended actions off until the operator enables them in the +install form. A default install yields PHI-safe inspection and Needs +Attention tools only (`list_workflows`, `get_run`, `get_workflow`, +`get_run_report`, `list_needs_attention`, and `get_attention_item`). +Enabling workflow runs adds `run_workflow` (the request-id contract) and the +deprecated per-workflow `run_workflow_` tools; enabling attended actions adds Reject, Teach, and Escalate, while a qualified deployment config also makes Continue and Skip available. Clients without MCP form elicitation use Flow's attended console/CLI, where all five capabilities remain available. This @@ -70,21 +71,24 @@ truth; the machine-readable copies are [`../server.json`](../server.json) and [`../manifest.json`](../manifest.json). - **Name (reverse-DNS, official registry):** `io.github.OpenAdaptAI/openadapt-agent` -- **Display name:** OpenAdapt Agent (openadapt-flow bridge) +- **Display name:** OpenAdapt: your agent decides, OpenAdapt enters it and checks it saved - **PyPI package:** `openadapt-agent` - **Version:** `2.0.2` (staged in this repository; `2.0.1` is the newest version published to PyPI) -- **Description:** OpenAdapt compiles demonstrated GUI workflows into programs. This package invokes them over MCP. +- **Description:** Your AI agent decides what to enter. OpenAdapt enters it in the app and checks that it saved. - **Homepage / docs:** https://docs.openadapt.ai - **Repository:** https://github.com/OpenAdaptAI/openadapt-agent - **License:** MIT - **Transport:** stdio -- **Run command (uvx):** `uvx --from 'openadapt-agent[tutorial]' openadapt-agent serve --allow-run` -- **Config:** `--allow-run` with no `--bundles` (public synthetic bundle, generated at serve time), `--tutorial` (same path without implying run tools), `--authoring` (local Claude Code first demo; not the published registry recipe), `authoring connect` (outbound hosted mailbox; not the registry recipe), or `--bundles` (operator's private artifact), `--runs-dir`, `--allow-attended-actions`, qualified `--config` for Continue/Skip, and optional secret `OPENADAPT_BUNDLE_KEY` +- **Run command (uvx):** `uvx --python 3.12 openadapt-agent serve` (the sandbox; from GitHub before 2.0.2 is on PyPI: `uvx --python 3.12 --from git+https://github.com/OpenAdaptAI/openadapt-agent openadapt-agent serve`) +- **Config:** no flags (the synthetic sandbox; `--allow-run` and `--tutorial` without `--bundles` are older names for it), `--mode production --bundles ` (run the operator's private workflows), `--mode attended` (also answer paused runs), `--bundles` alone (read-only), `--authoring` (local Claude Code first demo; not the published registry recipe), `authoring connect` (outbound hosted mailbox; not the registry recipe), `--runs-dir`, `--allow-attended-actions`, qualified `--config` for Continue/Skip, and optional secret `OPENADAPT_BUNDLE_KEY` - **Tools:** - - `list_workflows` / `get_workflow` — PHI-safe structural bundle projections with opaque IDs. + - `list_workflows` — one card per workflow: name, purpose, what done means, typed inputs. + - `run_workflow` — one request with the caller's `request_id`; returns `done` | `needs_review` | `not_sure_if_saved` | `did_not_run` (or `running` with a `run_id`), with `outputSchema` and `structuredContent`. The same `request_id` never writes twice. + - `get_run` — the result of a run by `run_id`. + - `get_workflow` — PHI-safe structural bundle projection by opaque id. - `get_run_report` — PHI-safe status and count summary; raw evidence stays local unless protected export was explicitly enabled. - `list_needs_attention` / `get_attention_item` — PHI-safe durable-pause cards and current signed-capability metadata. - - `run_workflow_` — execute through the governed `openadapt-flow run` CLI when `--allow-run`; returns `success` | `halt` | `refused` | `timeout` | `error`. + - `run_workflow_` — deprecated; executes through the governed `openadapt-flow run` CLI and returns `success` | `halt` | `refused` | `timeout` | `error` plus the contract fields. A verified run is `success` with `proof: "local"`. - `continue_attention` / `skip_attention` / `reject_attention` / `teach_attention` / `escalate_attention` — exact, elicited attended decisions under Flow's capability, idempotency, verification, and audit contract. - **Categories/tags:** mcp, agent-skills, automation, workflow, gui, governed, healthcare, rpa diff --git a/docs/examples/sandbox-results.json b/docs/examples/sandbox-results.json new file mode 100644 index 0000000..217145c --- /dev/null +++ b/docs/examples/sandbox-results.json @@ -0,0 +1,262 @@ +{ + "captured_at": "2026-10-09T22:46:34+00:00", + "engine": "flow", + "provenance": "Measured on synthetic data: openadapt-agent sandbox, flow engine, MockMed synthetic clinic app. No real records.", + "results": { + "app_offline": { + "contract_version": "openadapt.run-result/v1", + "label": "Didn't run", + "mode": "sandbox", + "next_action": "Try again with the same request_id once the app is back. If it keeps happening, tell the operator.", + "next_step": "retry", + "outcome": "did_not_run", + "proof": "none", + "reason": "app_unreachable", + "record_changed": "no", + "request_id": "demo-app-offline", + "run_id": "run-6314b8b10b93421899360217", + "safe_to_retry": true, + "sandbox": { + "case": "app_offline", + "engine": "flow" + }, + "what_happened": "Didn't start because the app isn't reachable from this computer. Nothing was written. This was a sandbox run on a synthetic app; no real record was touched.", + "workflow": "add_triage_note" + }, + "duplicate_record": { + "contract_version": "openadapt.run-result/v1", + "label": "Stopped before saving", + "mode": "sandbox", + "model_calls": 0, + "needs_attention_id": "77d744a682594d44a3f8810d", + "next_action": "A person reviews this run on the computer that ran it. Call get_run later to see their decision. Don't start another run for this work until they finish.", + "next_step": "wait_for_person", + "outcome": "needs_review", + "proof": "none", + "reason": "record_not_confirmed", + "record_changed": "no", + "request_id": "demo-duplicate-record", + "run_id": "run-0b5813cc521e4854a6c3ffa3", + "safe_to_retry": false, + "sandbox": { + "case": "duplicate_record", + "engine": "flow" + }, + "seconds": 1.2, + "technical": { + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "what_happened": "Stopped before saving. It couldn't confirm it had the right record open, so nothing was written. This was a sandbox run on a synthetic app; no real record was touched.", + "workflow": "add_triage_note" + }, + "false_saved_banner": { + "contract_version": "openadapt.run-result/v1", + "label": "Check the record", + "mode": "sandbox", + "model_calls": 0, + "needs_attention_id": "77917c25aa8da46d4de2826a", + "next_action": "Have a person check the record in the app before anything runs again. Don't retry this request.", + "next_step": "check_record", + "outcome": "not_sure_if_saved", + "proof": "none", + "reason": "save_not_confirmed", + "record_changed": "unknown", + "request_id": "demo-false-saved-banner", + "run_id": "run-5b151caa473e4ab292d7e108", + "safe_to_retry": false, + "sandbox": { + "case": "false_saved_banner", + "engine": "flow" + }, + "seconds": 9.8, + "technical": { + "execution_outcome": "HALTED", + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "what_happened": "OpenAdapt can't confirm whether the change was saved. It may have gone through. This was a sandbox run on a synthetic app; no real record was touched.", + "workflow": "add_triage_note" + }, + "invalid_input": { + "contract_version": "openadapt.run-result/v1", + "invalid_inputs": [ + { + "input": "note", + "problem": "is too short" + } + ], + "label": "Didn't run", + "mode": "sandbox", + "next_action": "Fix the inputs listed in invalid_inputs, then send the request again. You can reuse the same request_id.", + "next_step": "fix_inputs", + "outcome": "did_not_run", + "proof": "none", + "reason": "invalid_input", + "record_changed": "no", + "request_id": "demo-invalid", + "run_id": "run-9d88393a00f54898a3a0af9c", + "safe_to_retry": true, + "what_happened": "Didn't start because some inputs aren't valid. Nothing was written. This was a sandbox run on a synthetic app; no real record was touched.", + "workflow": "add_triage_note" + }, + "list_workflows": { + "how_to_run": "Call run_workflow with a workflow name from this list, its inputs, and your own request_id for this piece of work. Only outcome done means the change was saved and checked. Never retry not_sure_if_saved. Reuse the same request_id when you retry anything.", + "mode": "sandbox", + "run_tools_enabled": true, + "sandbox": { + "app": "MockMed, a synthetic clinic app with made-up patients", + "cases": [ + { + "case": "normal", + "outcome": "done", + "what_it_shows": "The app saves the note and the record check finds it in the app's record store." + }, + { + "case": "duplicate_record", + "outcome": "needs_review", + "what_it_shows": "The queue lists the same referral twice, so OpenAdapt can't be sure which record to open. It stops before saving." + }, + { + "case": "false_saved_banner", + "outcome": "not_sure_if_saved", + "what_it_shows": "The app shows a saved message, but its record store rejected the note. The record check doesn't find it." + }, + { + "case": "timeout_after_save", + "outcome": "not_sure_if_saved", + "what_it_shows": "The record store saves the note, then the app times out and shows an error. Only a person checking the record can tell it saved." + }, + { + "case": "app_offline", + "outcome": "did_not_run", + "what_it_shows": "The app isn't reachable, so OpenAdapt doesn't start." + } + ], + "engine": "flow", + "engine_detail": "Each run drives the synthetic app in a hidden browser with the real OpenAdapt engine. The first run waits while OpenAdapt records the workflow once.", + "ready": false + }, + "workflows": [ + { + "available": true, + "changes_records": true, + "done_means": "The note is in the synthetic app's record store, confirmed by reading the store directly instead of trusting the screen.", + "inputs": { + "additionalProperties": false, + "properties": { + "note": { + "description": "The triage note to add. Use made-up text.", + "maxLength": 500, + "minLength": 1, + "type": "string" + }, + "sandbox_case": { + "default": "normal", + "description": "Which situation the synthetic app shows (normal: done; duplicate_record: needs_review; false_saved_banner: not_sure_if_saved; timeout_after_save: not_sure_if_saved; app_offline: did_not_run).", + "enum": [ + "normal", + "duplicate_record", + "false_saved_banner", + "timeout_after_save", + "app_offline" + ], + "type": "string" + } + }, + "required": [ + "note" + ], + "type": "object" + }, + "name": "add_triage_note", + "purpose": "Sandbox: add a triage note to the first referral in a synthetic clinic app. Nothing real changes." + } + ] + }, + "normal": { + "contract_version": "openadapt.run-result/v1", + "label": "Done and checked", + "mode": "sandbox", + "model_calls": 0, + "next_action": "Nothing more to do for this request. Don't run it again for the same work.", + "next_step": "none", + "outcome": "done", + "proof": "local", + "reason": "saved_and_checked", + "record_changed": "yes", + "request_id": "demo-normal", + "run_id": "run-8c28f0bcac224af9b6947a7d", + "safe_to_retry": false, + "sandbox": { + "case": "normal", + "engine": "flow" + }, + "seconds": 5.0, + "technical": { + "execution_outcome": "VERIFIED", + "transaction_outcome": "VERIFIED" + }, + "what_happened": "Saved the change and read the record back to confirm it. The check ran on this computer, so there's no signed receipt. This was a sandbox run on a synthetic app; no real record was touched.", + "workflow": "add_triage_note" + }, + "normal_again_same_request_id": { + "contract_version": "openadapt.run-result/v1", + "label": "Done and checked", + "mode": "sandbox", + "model_calls": 0, + "next_action": "Nothing more to do for this request. Don't run it again for the same work.", + "next_step": "none", + "outcome": "done", + "proof": "local", + "reason": "saved_and_checked", + "record_changed": "yes", + "replayed": true, + "request_id": "demo-normal", + "run_id": "run-8c28f0bcac224af9b6947a7d", + "safe_to_retry": false, + "sandbox": { + "case": "normal", + "engine": "flow" + }, + "seconds": 5.0, + "technical": { + "execution_outcome": "VERIFIED", + "transaction_outcome": "VERIFIED" + }, + "what_happened": "Saved the change and read the record back to confirm it. The check ran on this computer, so there's no signed receipt. This was a sandbox run on a synthetic app; no real record was touched.", + "workflow": "add_triage_note" + }, + "timeout_after_save": { + "contract_version": "openadapt.run-result/v1", + "label": "Check the record", + "mode": "sandbox", + "model_calls": 0, + "needs_attention_id": "481486193ef0e44c0e2b0dd6", + "next_action": "Have a person check the record in the app before anything runs again. Don't retry this request.", + "next_step": "check_record", + "outcome": "not_sure_if_saved", + "proof": "none", + "reason": "save_not_confirmed", + "record_changed": "unknown", + "request_id": "demo-timeout-after-save", + "run_id": "run-cef0b138a5e1488abbd805f5", + "safe_to_retry": false, + "sandbox": { + "case": "timeout_after_save", + "engine": "flow" + }, + "seconds": 10.2, + "technical": { + "execution_outcome": "HALTED", + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "what_happened": "OpenAdapt can't confirm whether the change was saved. It may have gone through. This was a sandbox run on a synthetic app; no real record was touched.", + "workflow": "add_triage_note" + } + }, + "versions": { + "openadapt-agent": "2.0.2", + "openadapt-flow": "1.35.1", + "python": "3.12.13" + } +} diff --git a/llms.txt b/llms.txt index 7daf0c6..e89cf69 100644 --- a/llms.txt +++ b/llms.txt @@ -1,47 +1,65 @@ # openadapt-agent -> OpenAdapt compiles demonstrated GUI workflows into programs. This package invokes them over MCP. +> Your AI agent decides what to enter. OpenAdapt enters it in the app and checks that it saved. -Default runtime interface for a calling agent. Computer-use agents are the user of OpenAdapt. They are not the executor inside it. Never summarize halt as success. Roles: operator (calling agent), authority (named human admits; a local agent may click while they watch), auditor (samples seals). Hosted ChatGPT.com and Claude.ai can't click the user's GUI; send them to https://openadapt.ai/start. Unsigned success is failure. Standard and Regulated need a workflow admission; Demo and the synthetic tutorial may run without one. Target admissions: https://openadapt.ai/production-lifecycle.json (seven, remote-safe-synthetic). Workflow admissions: https://openadapt.ai/production-workflow-admissions.json (seven, 0.0.0-synthetic, not a customer job). +Use this when an AI agent has decided what to enter in an app with no usable API, such as an EMR, a payer portal, or a desktop or Citrix app. OpenAdapt enters the values in the same screens staff use, on a computer the customer controls, then reads the saved record back. When something doesn't match, it stops and a person decides. A normal run makes no AI model calls. -## What it provides +## The contract: three tools -- `openadapt-agent serve --allow-run`: generate and serve the public synthetic tutorial at serve time. `openadapt-agent serve --tutorial` is the same path without run tools. `openadapt-agent serve --bundles [--allow-run]`: serve a private compiled bundle. `openadapt-agent serve --authoring`: local Claude Code first-demo tools `observe`, `start_record`, `click`, and `halt` over stdio. `--authoring` does not enable run tools and does not open an HTTP listener. `openadapt-agent authoring connect `: outbound mailbox client for hosted ChatGPT.com / Claude.ai (claim `oab_`, poll wait=0, Allow-per-sub, Continue via `record_observed`). Overlay chrome stays Desktop-only. Hosted MCP is `https://openadapt.ai/mcp`. `list_workflows`, `get_workflow`, `get_run_report`, `list_needs_attention`, and `get_attention_item` are always available as PHI-safe read-only projections. `run_workflow_` tools require `--allow-run`. The synthetic tutorial registers `run_local_quickstart`. If a run returns HALTED, tell the user the record did not change. Never summarize halt, refused, timeout, or error as success. -- `--allow-attended-actions` adds exact Reject, Teach, and Escalate tools for signed durable pauses. With a qualified Flow `--config`, the same server also exposes Continue and Skip through Flow's deployment-bound live verifier and deterministic resume path. -- `openadapt-agent emit-skill --out ` wraps Flow's skill emitter and appends MCP, halt, and attended-action guidance. +- `list_workflows`: each workflow's `name`, `purpose`, `done_means`, and typed `inputs` (types, enums, formats). +- `run_workflow(workflow, inputs, request_id, wait_seconds?)`: one piece of work. `request_id` is the caller's own id and is required. The same `request_id` returns the first result and never writes twice. +- `get_run(run_id, wait_seconds?)`: the result of a run that returned `outcome: running`. -## Attended actions +Every result has `outcome`, `safe_to_retry`, `what_happened` (one plain sentence), `next_action`, `proof`, `run_id`, and `request_id`. Tools publish an MCP `outputSchema` and return `structuredContent`. -- `continue_attention`: after the local operator completes the paused task, Flow verifies the exact outcome and resumes without actuating that task again. -- `skip_attention`: applies only a compiled, non-consequential skip that Flow revalidates at decision time. -- `reject_attention`: terminates this run and dispatches no new action. Earlier run actions may have effects, so review the protected local report and transaction outcome. Use escalation when a qualified operator can still continue the run. -- `teach_attention`: records an audited corrective-demonstration request; Flow's revision and regression gates still decide promotion. -- `escalate_attention`: records escalation and preserves the exact durable pause. +## Outcomes (the honesty rules) -All mutations require an exact capability digest, a stable idempotency key, action-specific confirmation, and MCP form elicitation. Clients without form elicitation use Flow's attended console/CLI; the capabilities are not removed or converted to read-only. +- `done`: saved and checked by reading the record back. `proof` is `local` (checked on the customer's computer), `sealed` (signed receipt), or `simulated` (sandbox, no app opened). A verified local run is done; it is never reported as an error. +- `needs_review`: stopped before saving; nothing was written; a person decides. Don't start the same work again. +- `not_sure_if_saved`: a save may have gone through. Never retry. A person checks the record. Never tell the user nothing changed. +- `did_not_run`: nothing was written. Fix the cause and retry with the same `request_id` when `safe_to_retry` is true. -## Run outcomes (honesty invariant) +Only `done` is a success. Results come from openadapt-flow's `transaction_outcome`, never from the coarse halt label alone. -- `success`: exit 0 AND the persisted `report.json` marks the run successful. Only then did the workflow complete. -- `halt`: execution stopped instead of guessing. It is not a success; protected evidence remains in the local operator experience. -- `refused`: a governed admission gate refused the bundle before execution; nothing ran. -- `timeout` / `error`: killed at the deadline / infrastructure problem. +## Try it + +`openadapt-agent serve` with no flags is a sandbox: one workflow, `add_triage_note`, on MockMed, a synthetic clinic app. The `sandbox_case` input picks the outcome: `normal` (done), `duplicate_record` (needs_review), `false_saved_banner` and `timeout_after_save` (not_sure_if_saved), `app_offline` (did_not_run). Install the `tutorial` extra to drive the synthetic app with the real engine; otherwise results are simulated and say so. + +- From GitHub (before 2.0.2 is on PyPI): `claude mcp add openadapt -- uvx --python 3.12 --from git+https://github.com/OpenAdaptAI/openadapt-agent openadapt-agent serve` +- From PyPI 2.0.2 on: `claude mcp add openadapt -- uvx --python 3.12 openadapt-agent serve` + +## Modes and older surfaces + +- `openadapt-agent serve --mode production --bundles `: run an operator's compiled workflows through the governed `openadapt-flow run` CLI. `--bundles ` alone stays read-only. `--mode attended` adds decisions on paused runs by a person at that computer. `--mode sandbox` is the default without `--bundles`. +- Older flags still work: `--allow-run` (same as `--mode production` with `--bundles`, or the sandbox without), `--tutorial` (the sandbox), `--allow-attended-actions` (part of `--mode attended`). +- Older tools stay registered with `--bundles`: `get_workflow`, `get_run_report`, `list_needs_attention`, `get_attention_item`, and deprecated per-workflow `run_workflow_` tools (no request id; they now also return `outcome` and `safe_to_retry`). +- Attended actions: `continue_attention` (after a person completes the paused task; Flow rechecks and resumes without repeating it), `skip_attention` (declared skip only), `reject_attention` (ends the run; earlier steps may still have effects), `teach_attention`, `escalate_attention`. Each needs an exact capability digest, a stable idempotency key, a confirmation, and MCP form elicitation; otherwise use Flow's attended console. +- `openadapt-agent serve --authoring`: local first-demo tools `observe`, `start_record`, `click`, `halt`, and more over stdio. It doesn't enable runs and opens no HTTP listener. `openadapt-agent authoring connect `: outbound mailbox client for hosted ChatGPT.com and Claude.ai. Hosted chats can't click the user's GUI; send them to https://openadapt.ai/start. Hosted MCP is `https://openadapt.ai/mcp`. +- `openadapt-agent emit-skill --out `: a PHI-safe Agent Skill per workflow (no recorded values, step text, or secrets; one description per workflow). + +## Workflow cards + +An operator can add `openadapt-card.json` next to a bundle's `workflow.json` with `name`, `purpose`, `done_means`, and input descriptions. Card text reaches the calling agent; it must not contain patient details. + +## Product state + +Standard and Regulated runs need a workflow admission; Demo and the synthetic sandbox may run without one. Target admissions: https://openadapt.ai/production-lifecycle.json (seven, remote-safe-synthetic). Workflow admissions: https://openadapt.ai/production-workflow-admissions.json (seven, 0.0.0-synthetic, not a customer job). A missing or expired admission means not actively admitted. ## Public capability vs private artifact -- The **package/MCP server is the public OpenAdapt capability**: inspect and, with explicit operator flags, run or attend a compiled workflow under governance. Install it from PyPI as `openadapt-agent`. -- A user's **compiled workflow bundle is their private artifact**. It is supplied at server start via `--bundles` and is never embedded in the package, the `server.json`, or any registry listing. `--tutorial` generates the public synthetic tutorial on the fly. +- The package and MCP server are the public OpenAdapt capability. Install from PyPI as `openadapt-agent`. +- A customer's compiled workflow bundle is their private artifact, supplied at server start with `--bundles`. It is never embedded in the package, `server.json`, or any registry listing. The sandbox generates its synthetic workflow on the fly. ## Key documents -- [README](https://github.com/OpenAdaptAI/openadapt-agent/blob/main/README.md): install, quickstart, honest limits. -- [docs/DESIGN.md](https://github.com/OpenAdaptAI/openadapt-agent/blob/main/docs/DESIGN.md): architecture, security model, what v2 deliberately does not do. -- [docs/DISTRIBUTION.md](https://github.com/OpenAdaptAI/openadapt-agent/blob/main/docs/DISTRIBUTION.md): registry artifacts, the public-vs-private-server distinction, and the submission plan. +- [README](https://github.com/OpenAdaptAI/openadapt-agent/blob/main/README.md): what it does, the contract, the sandbox, developer setup. +- [docs/DESIGN.md](https://github.com/OpenAdaptAI/openadapt-agent/blob/main/docs/DESIGN.md): architecture, the result contract, security model. +- [docs/DISTRIBUTION.md](https://github.com/OpenAdaptAI/openadapt-agent/blob/main/docs/DISTRIBUTION.md): registry artifacts and the public-vs-private distinction. - [server.json](https://github.com/OpenAdaptAI/openadapt-agent/blob/main/server.json): official MCP registry manifest. -- [manifest.json](https://github.com/OpenAdaptAI/openadapt-agent/blob/main/manifest.json): portable MCPB launch config for Smithery and desktop clients. +- [manifest.json](https://github.com/OpenAdaptAI/openadapt-agent/blob/main/manifest.json): portable MCPB launch config. ## Related -- [openadapt-flow](https://github.com/OpenAdaptAI/openadapt-flow): the governed demonstration compiler and runtime this package bridges. +- [openadapt-flow](https://github.com/OpenAdaptAI/openadapt-flow): the demonstration compiler and runtime this package connects to. - [OpenAdapt](https://github.com/OpenAdaptAI/OpenAdapt): the umbrella project. - [docs.openadapt.ai](https://docs.openadapt.ai): OpenAdapt documentation. diff --git a/manifest.json b/manifest.json index 0f6275d..55c6625 100644 --- a/manifest.json +++ b/manifest.json @@ -3,8 +3,8 @@ "name": "openadapt-agent", "display_name": "OpenAdapt", "version": "2.0.2", - "description": "Governed local MCP bridge for compiled OpenAdapt workflows and attended decisions.", - "long_description": "Inspect compiled workflows, run them through OpenAdapt's policy and verification gates, and resolve durable Needs Attention items. The server runs locally; workflow bundles and protected evidence stay on the operator's machine by default.", + "description": "Lets your AI agent enter information in an app with no API and get back whether it saved.", + "long_description": "Your agent sends a workflow name, its inputs, and its own request id. OpenAdapt enters the values in the app's own screens on this computer, reads the saved record back, and returns done, needs review, not sure if saved, or did not run. The same request id never writes twice. Workflow files and evidence stay on this computer.", "author": { "name": "OpenAdaptAI", "email": "contact@openadapt.ai", @@ -44,23 +44,31 @@ "tools": [ { "name": "list_workflows", - "description": "List PHI-safe projections of locally compiled OpenAdapt workflows." + "description": "List each workflow's name, purpose, what done means, and typed inputs." + }, + { + "name": "run_workflow", + "description": "Run one workflow with your request id and get done, needs_review, not_sure_if_saved, or did_not_run." + }, + { + "name": "get_run", + "description": "Get the result of a run that was still going." }, { "name": "get_workflow", - "description": "Inspect one workflow's governed, PHI-safe capability summary." + "description": "Older tool: one workflow's structural summary." }, { "name": "get_run_report", - "description": "Read a PHI-safe status and count summary for a run created by this server." + "description": "Older tool: a status and count summary for a run on this server." }, { "name": "list_needs_attention", - "description": "List durable workflow pauses that need an operator decision." + "description": "List paused runs that need a person's decision." }, { "name": "get_attention_item", - "description": "Inspect one PHI-safe Needs Attention item and its allowed actions." + "description": "Inspect one paused run and its allowed decisions." } ], "tools_generated": true, @@ -69,8 +77,10 @@ "automation", "workflow", "gui", - "governed", - "attended" + "ai-agents", + "ehr", + "rpa", + "human-in-the-loop" ], "license": "MIT", "compatibility": { @@ -99,8 +109,8 @@ }, "allow_run": { "type": "boolean", - "title": "Allow governed workflow runs", - "description": "Add run tools. Every run still passes OpenAdapt's admission, identity, policy, and effect-verification gates.", + "title": "Allow workflow runs", + "description": "Add run_workflow. Every run still passes OpenAdapt's admission, right-record, policy, and record-check gates, and the same request id never writes twice.", "required": false, "default": false }, diff --git a/pyproject.toml b/pyproject.toml index e15595c..0d44e91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "openadapt-agent" version = "2.0.2" -description = "Local agent bridge for governed OpenAdapt workflows, attended actions, MCP, and Agent Skills" +description = "Let your AI agent enter information in apps with no API and get back whether it saved (MCP and Agent Skills)" readme = "README.md" # openadapt-flow pins <3.13 (rapidocr-onnxruntime); this package inherits it. requires-python = ">=3.10,<3.13" @@ -26,7 +26,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development :: Libraries :: Python Modules", ] -keywords = ["mcp", "agent-skills", "gui", "automation", "workflow", "openadapt", "governed"] +keywords = ["mcp", "agent-skills", "gui", "automation", "workflow", "openadapt", "ai-agents", "ehr", "rpa"] dependencies = [ # Governed workflow compiler/runtime this package bridges. Execution @@ -58,7 +58,10 @@ dev = [ "ruff>=0.4.0", ] tutorial = [ - "openadapt-flow[browser]>=1.26.0,<2", + # The sandbox's real engine drives MockMed through Flow's tutorial entry + # query, added in openadapt-flow 1.29.0. Older Flow falls back to the + # simulated engine. + "openadapt-flow[browser]>=1.29.0,<2", ] [project.scripts] diff --git a/scripts/capture_sandbox_examples.py b/scripts/capture_sandbox_examples.py new file mode 100644 index 0000000..8ffc958 --- /dev/null +++ b/scripts/capture_sandbox_examples.py @@ -0,0 +1,93 @@ +"""Record one real sandbox result per case, for the README and docs. + +Drives the zero-flag sandbox with the flow engine (real openadapt-flow, a +hidden browser, and the synthetic MockMed app) and writes every result to +docs/examples/sandbox-results.json. Synthetic data only. Usage: + + pip install -e '.[tutorial]' + python scripts/capture_sandbox_examples.py [--out docs/examples/sandbox-results.json] +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import tempfile +from datetime import datetime, timezone +from importlib.metadata import version +from pathlib import Path + +from openadapt_agent.bridge import AgentBridge +from openadapt_agent.runner import RunnerConfig +from openadapt_agent.sandbox import CASES, SANDBOX_WORKFLOW, SandboxEngine + +NOTE = "Synthetic: recheck blood pressure in 2 weeks" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--out", default="docs/examples/sandbox-results.json") + args = parser.parse_args() + os.environ.setdefault("DO_NOT_TRACK", "1") + os.environ.setdefault("OPENADAPT_TELEMETRY", "0") + with tempfile.TemporaryDirectory(prefix="openadapt-sandbox-") as tmp: + runs = Path(tmp) + engine = SandboxEngine(runs, engine="flow") + bridge = AgentBridge(None, RunnerConfig(runs_dir=runs), sandbox=engine, mode="sandbox") + results: dict[str, object] = {} + try: + results["list_workflows"] = bridge.dispatch("list_workflows", {}) + for case in CASES: + results[case] = bridge.dispatch( + "run_workflow", + { + "workflow": SANDBOX_WORKFLOW, + "inputs": {"note": NOTE, "sandbox_case": case}, + "request_id": f"demo-{case.replace('_', '-')}", + "wait_seconds": 600, + }, + ) + results["normal_again_same_request_id"] = bridge.dispatch( + "run_workflow", + { + "workflow": SANDBOX_WORKFLOW, + "inputs": {"note": NOTE, "sandbox_case": "normal"}, + "request_id": "demo-normal", + "wait_seconds": 600, + }, + ) + results["invalid_input"] = bridge.dispatch( + "run_workflow", + { + "workflow": SANDBOX_WORKFLOW, + "inputs": {"note": ""}, + "request_id": "demo-invalid", + }, + ) + finally: + engine.close() + payload = { + "provenance": ( + "Measured on synthetic data: openadapt-agent sandbox, flow engine, " + "MockMed synthetic clinic app. No real records." + ), + "captured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "versions": { + "openadapt-agent": version("openadapt-agent"), + "openadapt-flow": version("openadapt-flow"), + "python": sys.version.split()[0], + }, + "engine": engine.engine, + "results": results, + } + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"wrote {out} (engine {engine.engine})") + return 0 if engine.engine == "flow" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_client.py b/scripts/smoke_client.py index c058c32..6da5adf 100644 --- a/scripts/smoke_client.py +++ b/scripts/smoke_client.py @@ -1,12 +1,20 @@ -"""Manual smoke test: drive the served MCP server over real stdio. +"""Drive a served OpenAdapt MCP server over real stdio, the way a client does. -Not run in CI (it spawns the real server process). Usage: +Not run in CI (it spawns a real server process). Usage: + # The zero-flag sandbox from this checkout: + python scripts/smoke_client.py + + # Any launch command, for example the documented uvx line: + python scripts/smoke_client.py -- uvx --python 3.12 --from \\ + git+https://github.com/OpenAdaptAI/openadapt-agent openadapt-agent serve + + # Operator bundles, read-only or with runs: python scripts/smoke_client.py --bundles /path/to/bundles [--allow-run] -Performs tools/list plus ``list_workflows``, ``get_workflow``, and the -PHI-safe ``list_needs_attention`` call over stdio. It never invokes a -run or attended-action tool. +In sandbox mode it calls list_workflows, then run_workflow once per +sandbox case, then get_run, and prints each outcome. With --bundles it lists +tools, workflows, and Needs Attention, and never starts a run. """ from __future__ import annotations @@ -14,55 +22,78 @@ import argparse import asyncio import json +import os import sys +import tempfile from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client +def _structured(result): + data = result.model_dump(by_alias=True, mode="json") + return data.get("structuredContent") or json.loads(data["content"][0]["text"]) + + async def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--bundles", required=True) + parser.add_argument("--bundles") parser.add_argument("--allow-run", action="store_true") + parser.add_argument("command", nargs=argparse.REMAINDER) args = parser.parse_args() - server_args = [ - "-m", - "openadapt_agent.mcp", - "--bundles", - args.bundles, - ] - if args.allow_run: - server_args.append("--allow-run") - params = StdioServerParameters(command=sys.executable, args=server_args) + command = [part for part in args.command if part != "--"] + if not command: + command = [sys.executable, "-m", "openadapt_agent.mcp"] + if args.bundles: + command += ["--bundles", args.bundles] + if args.allow_run: + command.append("--allow-run") + sandbox_dir = tempfile.mkdtemp(prefix="openadapt-smoke-") + env = dict(os.environ, OPENADAPT_AGENT_SANDBOX_DIR=sandbox_dir) + params = StdioServerParameters(command=command[0], args=command[1:], env=env) async with stdio_client(params) as (read, write): async with ClientSession(read, write) as session: await session.initialize() - - tools = await session.list_tools() - print("tools/list:") - for tool in tools.tools: - print(f" - {tool.name}") - - listing = await session.call_tool("list_workflows", {}) - payload = json.loads(listing.content[0].text) - print("\nlist_workflows:") - print(json.dumps(payload, indent=2)) - - attention = await session.call_tool("list_needs_attention", {}) - print("\nlist_needs_attention:") - print(attention.content[0].text) - - workflows = [w for w in payload["workflows"] if w["available"]] - if workflows: - workflow_id = workflows[0]["id"] - detail = await session.call_tool( - "get_workflow", - {"workflow": workflow_id}, + tools = [tool.name for tool in (await session.list_tools()).tools] + print("tools:", ", ".join(tools)) + listing = _structured(await session.call_tool("list_workflows", {})) + print("mode:", listing["mode"]) + for workflow in listing["workflows"]: + print(f"workflow: {workflow['name']}: {workflow['purpose']}") + if listing["mode"] != "sandbox": + attention = await session.call_tool("list_needs_attention", {}) + print("needs attention:", attention.content[0].text) + return 0 + print("sandbox engine:", listing["sandbox"]["engine"]) + name = listing["workflows"][0]["name"] + last_run = None + for case in listing["sandbox"]["cases"]: + result = _structured( + await session.call_tool( + "run_workflow", + { + "workflow": name, + "inputs": { + "note": "Synthetic: recheck in 2 weeks", + "sandbox_case": case["case"], + }, + "request_id": f"smoke-{case['case'].replace('_', '-')}", + "wait_seconds": 600, + }, + ) + ) + last_run = result["run_id"] + print( + f"{case['case']}: outcome={result['outcome']} " + f"safe_to_retry={result['safe_to_retry']} :: {result['what_happened']}" ) - print(f"\nget_workflow({workflow_id!r}):") - print(detail.content[0].text) + if result["outcome"] != case["outcome"]: + print(f" expected {case['outcome']}", file=sys.stderr) + return 1 + fetched = _structured(await session.call_tool("get_run", {"run_id": last_run})) + print("get_run:", fetched["outcome"], fetched["run_id"]) return 0 diff --git a/server.json b/server.json index 5d71e74..3318522 100644 --- a/server.json +++ b/server.json @@ -1,8 +1,8 @@ { "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", "name": "io.github.OpenAdaptAI/openadapt-agent", - "title": "OpenAdapt Agent (openadapt-flow bridge)", - "description": "OpenAdapt compiles demonstrated GUI workflows into programs. This package invokes them over MCP.", + "title": "OpenAdapt: your agent decides, OpenAdapt enters it and checks it saved", + "description": "Your AI agent decides what to enter. OpenAdapt enters it in the app and checks that it saved.", "websiteUrl": "https://docs.openadapt.ai", "repository": { "url": "https://github.com/OpenAdaptAI/openadapt-agent", @@ -47,7 +47,7 @@ ], "_meta": { "io.modelcontextprotocol.registry/publisher-provided": { - "notes": "PHI-safe inspection and Needs Attention tools are available by default. Run tools require --allow-run. Reject, Teach, and Escalate require --allow-attended-actions; Continue and Skip also require --allow-attended-actions and a qualified deployment --config. Reject dispatches no new action, but earlier run actions may have effects and require review of the protected local outcome. See docs/DISTRIBUTION.md." + "notes": "Starts read-only with --bundles: list_workflows and get_run plus the older inspection tools. --mode production adds run_workflow, which takes the caller's request_id and returns done, needs_review, not_sure_if_saved, or did_not_run. --mode attended also adds Reject, Teach, and Escalate; Continue and Skip also need a qualified deployment --config. Without --bundles the server is a synthetic sandbox. See docs/DISTRIBUTION.md." } } } diff --git a/skills/openadapt-gui-write/SKILL.md b/skills/openadapt-gui-write/SKILL.md index b3e59ef..1fa6f15 100644 --- a/skills/openadapt-gui-write/SKILL.md +++ b/skills/openadapt-gui-write/SKILL.md @@ -1,24 +1,43 @@ --- name: openadapt-gui-write -description: "OpenAdapt compiles demonstrated GUI workflows into programs. This package invokes them over MCP." +description: "Your AI agent decides what to enter. OpenAdapt enters it in the app and checks that it saved." --- -# OpenAdapt GUI write +# OpenAdapt: enter it in the app and check that it saved -When the user needs a repeating GUI write with no API and must prove persistence, call run_. If the tool returns HALTED, tell the user the record did not change. +Use this when your agent has decided what to enter and the app has no usable API. Call list_workflows, then run_workflow with the workflow name, its inputs, and your own request_id. -Never summarize halt, refused, timeout, or error as success. A local unsigned replay may complete. If the tool returns unsigned success, treat it as failure. Production success without a Seal is failure. +## The three tools -The MCP server is this package. OpenAdapt compiles the program. The skill name is openadapt-gui-write. +1. `list_workflows` returns each workflow's `name`, `purpose`, `done_means`, and typed `inputs`. +2. `run_workflow` takes `workflow`, `inputs`, `request_id`, and optional `wait_seconds`. Use your own id for the piece of work, such as a referral id, and send the same id if you retry. The same id never writes twice. Don't put patient details in it. +3. `get_run` takes a `run_id` and returns the result when a run was still going. -Serve the public synthetic tutorial with: +## What the outcome means + +| outcome | What happened | What to do | +| --- | --- | --- | +| `done` | Saved and checked. | Nothing more for this request. | +| `needs_review` | Stopped before saving. Nothing was written. A person decides. | Don't start the same work again. Check back with `get_run`. | +| `not_sure_if_saved` | It may or may not have saved. | Never retry. A person checks the record. | +| `did_not_run` | Nothing was written. | Fix the problem in `what_happened`, then retry with the same `request_id` if `safe_to_retry` is true. | + +Only outcome done means the change was saved and checked. needs_review means it stopped before saving and a person decides, so don't start the same work again. not_sure_if_saved means a person must check the record, so never retry it. did_not_run means nothing was written: fix the problem and retry with the same request_id when safe_to_retry is true. + +`proof` says how strong the evidence behind `done` is: `local` means the record check ran on the computer that ran the workflow, `sealed` means a signed receipt exists, and `simulated` means a sandbox result that opened no app. + +## Try it + +The sandbox serves one synthetic workflow, `add_triage_note`. Its `sandbox_case` input shows each outcome: `normal`, `duplicate_record`, `false_saved_banner`, `timeout_after_save`, and `app_offline`. ```bash -claude mcp add openadapt -- \ - uvx --from 'openadapt-agent[tutorial]' openadapt-agent \ - serve --allow-run +claude mcp add openadapt -- uvx --python 3.12 --from git+https://github.com/OpenAdaptAI/openadapt-agent openadapt-agent serve ``` -`openadapt quickstart --break-it` is the halt demo: the independent system-of-record check rejects a fake success banner, and the record did not change. +PyPI has openadapt-agent 2.0.1, which predates the sandbox. From version 2.0.2 on, this shorter line does the same: + +```bash +claude mcp add openadapt -- uvx --python 3.12 openadapt-agent serve +``` -A `success` status requires a persisted `execution_outcome: VERIFIED`. HALTED, refused, timeout, and error are not that. +The skill name is openadapt-gui-write. It is never called computer use: your agent decides, and OpenAdapt does the entry. diff --git a/src/openadapt_agent/__init__.py b/src/openadapt_agent/__init__.py index 724f098..7afa0e3 100644 --- a/src/openadapt_agent/__init__.py +++ b/src/openadapt_agent/__init__.py @@ -1,22 +1,24 @@ -"""openadapt-agent — the local agent bridge for openadapt-flow workflows. +"""openadapt-agent: your AI agent decides what to enter; OpenAdapt enters it. -This package exposes compiled -`openadapt-flow `_ workflow -bundles and PHI-safe Needs Attention items to other agents: +This package connects a calling agent to +`openadapt-flow `_, which +enters values in an app's own screens and reads the saved record back: - an **MCP server** (``openadapt-agent serve`` / ``python -m - openadapt_agent.mcp``) that presents each bundle as a tool whose - execution shells out to the governed ``openadapt-flow run`` CLI — the - server never reimplements or bypasses flow's policy / identity / effect - gates; and -- an **Agent Skills emitter** (``openadapt-agent emit-skill``) that wraps - flow's own ``emit-skill`` and appends agent-facing MCP invocation and - halt and attended-action guidance. + openadapt_agent.mcp``) with three contract tools, ``list_workflows``, + ``run_workflow``, and ``get_run``. Every result is ``done``, + ``needs_review``, ``not_sure_if_saved``, or ``did_not_run``, derived from + Flow's transaction outcome (:mod:`openadapt_agent.contract`). Operator + bundles run through the governed ``openadapt-flow run`` CLI; the server + never reimplements or bypasses Flow's policy, identity, or record checks. + With no flags it serves a synthetic sandbox (:mod:`openadapt_agent.sandbox`). +- an **Agent Skills emitter** (``openadapt-agent emit-skill``) that writes a + PHI-safe skill per workflow from its card. -A halted or refused run is always surfaced as a PHI-safe structured -halt/refusal, never as success. Protected report evidence remains local -unless the operator explicitly enables protected export for a trusted -client in the same data boundary. +Only a verified write is reported as done. Uncertain delivery is never safe +to retry, and the same ``request_id`` never writes twice. Protected report +evidence stays local unless the operator explicitly enables protected export +for a trusted client in the same data boundary. """ __version__ = "2.0.2" diff --git a/src/openadapt_agent/attended.py b/src/openadapt_agent/attended.py index 211d7a9..07fc204 100644 --- a/src/openadapt_agent/attended.py +++ b/src/openadapt_agent/attended.py @@ -305,7 +305,10 @@ def for_run_dir(self, run_dir: Path | str | None) -> Optional[dict[str, Any]]: return None from openadapt_flow.console.attention import attention_item - item = attention_item(self.runs_dir, path) + # Flow derives the queue id from the path relative to the root, so + # both must be resolved the same way (a runs directory under a + # symlink, such as /var on macOS, otherwise fails). + item = attention_item(root, path) return item.model_dump(mode="json") if item is not None else None def act(self, tool_name: str, arguments: dict[str, Any]) -> dict[str, Any]: diff --git a/src/openadapt_agent/bridge.py b/src/openadapt_agent/bridge.py index a39daa6..fc72545 100644 --- a/src/openadapt_agent/bridge.py +++ b/src/openadapt_agent/bridge.py @@ -1,20 +1,30 @@ -"""Transport-agnostic bridge: tool specs + dispatch over discovered bundles. +"""Transport-agnostic bridge: tool specs + dispatch over discovered workflows. Kept free of ``mcp`` imports so the tool surface (schema generation, gating, outcome mapping) is unit-testable without an MCP transport. The thin MCP wiring lives in :mod:`openadapt_agent.mcp`. +The partner contract is three tools: + +- ``list_workflows``: names, purposes, what done means, typed inputs. +- ``run_workflow``: one request with the caller's ``request_id``; returns + ``done``, ``needs_review``, ``not_sure_if_saved``, ``did_not_run``, or + ``running`` with a ``run_id``. +- ``get_run``: the result of a run by ``run_id``. + Safety model (documented in ``docs/DESIGN.md``): -- PHI-safe read-only tools (workflow inspection, run summaries, and Needs - Attention projections) are always registered. -- ``run_`` tools are registered ONLY when the operator started the - server with ``--allow-run``; even then, every call shells out to the - governed ``openadapt-flow run`` CLI, so flow's fail-closed admission - gates cannot be bypassed from here. -- Needs Attention projections are read-only and PHI-safe. Attended mutations - are registered only under ``--allow-attended-actions`` and are submitted to - Flow's signed capability/idempotency/audit contract. +- Results come from :mod:`openadapt_agent.contract`, which reads Flow's + ``transaction_outcome``. Only a verified write is ``done``; uncertain + delivery is ``not_sure_if_saved`` and is never safe to retry. +- ``run_workflow`` is registered only in sandbox mode or when the operator + enabled runs. Operator bundles always run through the governed + ``openadapt-flow run`` CLI, so Flow's fail-closed admission gates apply. +- The same ``request_id`` never starts a second write (Flow's idempotency + ledger, see :mod:`openadapt_agent.runs`). +- The older tools (``get_workflow``, ``get_run_report``, Needs Attention, + attended actions, and per-workflow ``run_``) stay registered for + operator bundles so existing clients keep working. - Remote authentication is outside this local stdio process. The local OS user is the operator of record for attended decisions. """ @@ -38,20 +48,39 @@ discover_bundles, tool_input_schema, ) -from openadapt_agent.copy import REQUIRES_SEAL_META +from openadapt_agent.cards import card_for_bundle, inputs_schema +from openadapt_agent.contract import ( + RUN_RESULT_SCHEMA, + WORKFLOW_LIST_SCHEMA, + reason_for_run, +) from openadapt_agent.runner import ( FlowRunner, RunnerConfig, + RunOutcome, classify_report_status, is_safe_run_id, - public_outcome_message, public_report_summary, + status_for_reason, +) +from openadapt_agent.runs import RunStore, read_record +from openadapt_agent.service import ( + DEFAULT_WAIT_SECONDS, + MAX_WAIT_SECONDS, + REQUEST_ID_PATTERN, + CatalogEntry, + FlowCliEngine, + RunService, + ServiceError, + attention_for, ) -__all__ = ["AgentBridge", "BridgeError", "ToolSpec"] +__all__ = ["AgentBridge", "BridgeError", "ToolSpec", "REQUEST_ID_PATTERN"] _LOG = logging.getLogger(__name__) +_RUN_ID_PATTERN = r"^run-[A-Za-z0-9._-]{1,124}$" + class BridgeError(Exception): """A tool-level error the transport should surface to the caller.""" @@ -64,6 +93,9 @@ class ToolSpec: input_schema: dict annotations: Optional[dict[str, Any]] = None meta: Optional[dict[str, Any]] = None + output_schema: Optional[dict[str, Any]] = None + #: Fixed guidance returned when the arguments don't match the schema. + usage: Optional[str] = None _READ_ONLY_ANNOTATIONS = { @@ -78,14 +110,49 @@ class ToolSpec: "idempotentHint": False, "openWorldHint": True, } +#: run_workflow writes, but the same request_id never writes twice. +_RUN_WORKFLOW_ANNOTATIONS = { + "readOnlyHint": False, + "destructiveHint": True, + "idempotentHint": True, + "openWorldHint": True, +} +_DEPRECATED_META = {"deprecated": True, "use_instead": "run_workflow"} + +_HOW_TO_RUN = ( + "Call run_workflow with a workflow name from this list, its inputs, and " + "your own request_id for this piece of work. Only outcome done means the " + "change was saved and checked. Never retry not_sure_if_saved. Reuse the " + "same request_id when you retry anything." +) +_RUN_WORKFLOW_DESCRIPTION = ( + "Do one workflow in an app on this computer, for example enter a referral " + "in a clinic's EMR, and check that it saved. Send the workflow name from " + "list_workflows, its inputs, and your own request_id for this piece of " + "work. The outcome is done (saved and checked), needs_review (stopped " + "before saving; a person decides), not_sure_if_saved (a person must check " + "the record; never retry), or did_not_run (nothing was written; fix it " + "and retry when safe_to_retry is true). Reuse the same request_id when " + "you retry: the same id never writes twice. If the outcome is running, " + "call get_run with the run_id." +) +_RUN_WORKFLOW_USAGE = ( + "run_workflow needs workflow (a name from list_workflows), inputs (an " + "object), and request_id (4 to 128 letters, digits, '.', '_', ':' or '-'). " + "wait_seconds is optional, 0 to 600. Nothing ran." +) +_GET_RUN_USAGE = ( + "get_run needs run_id (the run_id from run_workflow). wait_seconds is " + "optional, 0 to 600." +) class AgentBridge: - """Expose a directory of compiled workflow bundles as agent tools.""" + """Expose workflows as agent tools: operator bundles or the sandbox.""" def __init__( self, - bundles_dir: Path, + bundles_dir: Optional[Path], runner_config: RunnerConfig, *, allow_run: bool = False, @@ -96,9 +163,13 @@ def __init__( allow_protected_export: bool = False, allow_recorded_defaults: bool = False, public_synthetic: bool = False, + sandbox: Optional[Any] = None, + mode: Optional[str] = None, + store: Optional[RunStore] = None, ): - self.bundles_dir = Path(bundles_dir) - self.allow_run = allow_run + self.bundles_dir = Path(bundles_dir) if bundles_dir is not None else None + self.sandbox = sandbox + self.allow_run = allow_run or sandbox is not None self.allow_protected_export = allow_protected_export self.allow_recorded_defaults = allow_recorded_defaults self.public_synthetic = public_synthetic @@ -109,27 +180,78 @@ def __init__( allow_actions=allow_attended_actions, service=attended_service, ) + if mode is None: + if sandbox is not None: + mode = "sandbox" + elif allow_attended_actions: + mode = "attended" + else: + mode = "production" + self.mode = mode + #: Older per-bundle tools stay for operator bundles only. + self.legacy_tools = sandbox is None + self._store = store + self._service: Optional[RunService] = None self.workflows: dict[str, WorkflowInfo] = {} - for info in discover_bundles(self.bundles_dir): - base_id = info.slug if public_synthetic else info.public_id - workflow_id = base_id - suffix = 2 - while workflow_id in self.workflows: - workflow_id = f"{base_id}_{suffix}" - suffix += 1 - self.workflows[workflow_id] = info + if self.bundles_dir is not None: + for info in discover_bundles(self.bundles_dir): + base_id = info.slug if public_synthetic else info.public_id + workflow_id = base_id + suffix = 2 + while workflow_id in self.workflows: + workflow_id = f"{base_id}_{suffix}" + suffix += 1 + self.workflows[workflow_id] = info + self.catalog: dict[str, CatalogEntry] = {} + if sandbox is not None: + for entry in sandbox.entries(): + self.catalog[entry.name] = entry + else: + engine = FlowCliEngine(self.runner, self.attended) + for workflow_id, info in self.workflows.items(): + card = card_for_bundle(info, default_name=workflow_id) + name = card.name + suffix = 2 + while name in self.catalog: + name = f"{card.name}_{suffix}" + suffix += 1 + card.name = name + self.catalog[name] = CatalogEntry( + card=card, + engine=engine, + available=info.ok, + info=info, + legacy_id=workflow_id, + ) + self._entries_by_id = { + entry.legacy_id: entry for entry in self.catalog.values() if entry.legacy_id + } + + @property + def service(self) -> RunService: + """The run service, created on first use so read-only servers stay inert.""" + if self._service is None: + store = self._store or RunStore(self.runner_config.runs_dir) + self._service = RunService( + store, + self.catalog, + mode=self.mode, + allow_run=self.allow_run, + require_all_inputs=not self.allow_recorded_defaults, + ) + return self._service # -- tool surface ------------------------------------------------------ - def list_tool_specs(self) -> list[ToolSpec]: + def _contract_specs(self) -> list[ToolSpec]: + names = sorted(self.catalog) specs = [ ToolSpec( name="list_workflows", description=( - "List workflows by opaque id with parameter names/types, " - "step count, availability, and whether run tools are " - "enabled. Demonstration labels, values, intents, paths, " - "and load exceptions stay local by default." + "List the workflows this computer can do: each one's name, " + "purpose, what done means, and its inputs. Start here, then " + "call run_workflow." ), input_schema={ "type": "object", @@ -137,14 +259,105 @@ def list_tool_specs(self) -> list[ToolSpec]: "additionalProperties": False, }, annotations=_READ_ONLY_ANNOTATIONS, - ), + output_schema=WORKFLOW_LIST_SCHEMA, + ) + ] + if self.allow_run: + description = _RUN_WORKFLOW_DESCRIPTION + if self.sandbox is not None: + description = ( + "Sandbox: runs against a synthetic app, so no real record " + "changes. " + description + ) + workflow_schema: dict[str, Any] = { + "type": "string", + "description": "A workflow name from list_workflows.", + } + if names: + workflow_schema["enum"] = names + specs.append( + ToolSpec( + name="run_workflow", + description=description, + input_schema={ + "type": "object", + "properties": { + "workflow": workflow_schema, + "inputs": { + "type": "object", + "description": ( + "The inputs list_workflows shows for this workflow." + ), + }, + "request_id": { + "type": "string", + "pattern": REQUEST_ID_PATTERN, + "description": ( + "Your own id for this piece of work, for " + "example the referral id. Send the same id " + "when you retry. Don't put patient details in it." + ), + }, + "wait_seconds": { + "type": "integer", + "minimum": 0, + "maximum": MAX_WAIT_SECONDS, + "default": DEFAULT_WAIT_SECONDS, + "description": ( + "How long to wait for the result before " + "returning outcome running." + ), + }, + }, + "required": ["workflow", "inputs", "request_id"], + "additionalProperties": False, + }, + annotations=_RUN_WORKFLOW_ANNOTATIONS, + output_schema=RUN_RESULT_SCHEMA, + usage=_RUN_WORKFLOW_USAGE, + ) + ) + specs.append( + ToolSpec( + name="get_run", + description=( + "Get the result of a run started with run_workflow. Waits up " + "to wait_seconds while it is still running. Returns the same " + "result shape as run_workflow." + ), + input_schema={ + "type": "object", + "properties": { + "run_id": {"type": "string", "pattern": _RUN_ID_PATTERN}, + "wait_seconds": { + "type": "integer", + "minimum": 0, + "maximum": MAX_WAIT_SECONDS, + "default": DEFAULT_WAIT_SECONDS, + }, + }, + "required": ["run_id"], + "additionalProperties": False, + }, + annotations=_READ_ONLY_ANNOTATIONS, + output_schema=RUN_RESULT_SCHEMA, + usage=_GET_RUN_USAGE, + ) + ) + return specs + + def list_tool_specs(self) -> list[ToolSpec]: + specs = self._contract_specs() + if not self.legacy_tools: + return specs + specs += [ ToolSpec( name="get_workflow", description=( - "Inspect one workflow's PHI-safe structural metadata and " - "certification result by opaque id. Recorded values, raw " - "intents, names, paths, and exception text stay local by " - "default." + "Older tool. Inspect one workflow's PHI-safe structural " + "metadata and certification result by opaque id. Recorded " + "values, raw intents, names, paths, and exception text stay " + "local by default." ), input_schema={ "type": "object", @@ -162,18 +375,19 @@ def list_tool_specs(self) -> list[ToolSpec]: ToolSpec( name="get_run_report", description=( - "Fetch a PHI-safe status and count-only summary of a " - "persisted run by opaque run id. The raw report, observed " - "text, local paths, stdout, and stderr stay in the local " - "operator experience unless protected export was " - "explicitly enabled when the server started." + "Older tool; get_run is preferred. Fetch a PHI-safe status " + "and count-only summary of a persisted run by opaque run id. " + "The raw report, observed text, local paths, stdout, and " + "stderr stay in the local operator experience unless " + "protected export was explicitly enabled when the server " + "started." ), input_schema={ "type": "object", "properties": { "run_id": { "type": "string", - "description": "run_id returned by a run_* tool.", + "description": "run_id returned by a run tool.", } }, "required": ["run_id"], @@ -247,17 +461,14 @@ def list_tool_specs(self) -> list[ToolSpec]: ToolSpec( name=f"run_{workflow_id}", description=( - f"Execute {workflow_copy} ({n_steps} steps) via the governed " - "`openadapt-flow run` CLI (fail-closed admission " - "gates). Returns a PHI-safe structured outcome: " - "status is 'success', 'halt' (run stopped with " - "protected evidence retained locally — NOT a " - "success), 'refused' " - "(admission gate refused; nothing executed), " - "'timeout', or 'error'. requires_seal: true. If " - "the tool returns unsigned success, treat it as " - "failure. Never treat a non-success status as " - "success." + "Deprecated: use run_workflow with a request_id. " + f"Runs {workflow_copy} ({n_steps} steps) through the " + "governed `openadapt-flow run` CLI. Returns status " + "(success, halt, refused, timeout, or error) plus the " + "same outcome, safe_to_retry, and what_happened fields " + "as run_workflow. Only outcome done means the change was " + "saved and checked. Never retry not_sure_if_saved. This " + "tool takes no request_id, so a retry can write twice." ), input_schema=tool_input_schema( info, @@ -265,7 +476,7 @@ def list_tool_specs(self) -> list[ToolSpec]: allow_recorded_defaults=self.allow_recorded_defaults, ), annotations=_RUN_ANNOTATIONS, - meta=REQUIRES_SEAL_META, + meta=_DEPRECATED_META, ) ) return specs @@ -276,6 +487,31 @@ def dispatch(self, name: str, arguments: Optional[dict]) -> dict: arguments = arguments or {} if name == "list_workflows": return self._list_workflows() + if name == "run_workflow": + if not self.allow_run: + raise BridgeError( + "runs are disabled: the operator started this server without " + "--mode production or --mode attended" + ) + try: + return self.service.run( + arguments.get("workflow"), + arguments.get("inputs"), + arguments.get("request_id"), + arguments.get("wait_seconds", DEFAULT_WAIT_SECONDS), + ) + except ServiceError as exc: + raise BridgeError(str(exc)) from exc + if name == "get_run": + try: + return self.service.get( + arguments.get("run_id"), + arguments.get("wait_seconds", DEFAULT_WAIT_SECONDS), + ) + except ServiceError as exc: + raise BridgeError(str(exc)) from exc + if not self.legacy_tools: + raise BridgeError("unknown tool name") if name == "get_workflow": return self._get_workflow(arguments.get("workflow", "")) if name == "get_run_report": @@ -297,22 +533,39 @@ def dispatch(self, name: str, arguments: Optional[dict]) -> dict: raise BridgeError("unknown tool name") def _list_workflows(self) -> dict: - result = { - "schema_version": 1, - "lifecycle": "admission-derived", + workflows = [] + for name, entry in self.catalog.items(): + item = { + **entry.card.projection(), + "available": entry.available, + } + item["inputs"] = inputs_schema(entry.card, require_all=not self.allow_recorded_defaults) + if entry.legacy_id is not None and entry.info is not None: + item.update(self._workflow_projection(entry.legacy_id, entry.info, entry)) + workflows.append(item) + result: dict[str, Any] = { + "mode": self.mode, "run_tools_enabled": self.allow_run, - "protected_export_enabled": self.allow_protected_export, - "synthetic_recorded_defaults_enabled": self.allow_recorded_defaults, - "workflows": [ - self._workflow_projection(workflow_id, info) - for workflow_id, info in self.workflows.items() - ], - "note": ( - "Run tools are disabled unless the operator started the server with --allow-run." - if not self.allow_run - else None - ), + "how_to_run": _HOW_TO_RUN, + "workflows": workflows, } + if self.sandbox is not None: + result["sandbox"] = self.sandbox.describe() + return result + result.update( + { + "schema_version": 1, + "lifecycle": "admission-derived", + "protected_export_enabled": self.allow_protected_export, + "synthetic_recorded_defaults_enabled": self.allow_recorded_defaults, + "note": ( + "Runs are disabled. The operator starts the server with " + "--mode production (or the older --allow-run) to enable them." + if not self.allow_run + else None + ), + } + ) if self.allow_protected_export: result["protected"] = {"bundles_dir": str(self.bundles_dir)} return result @@ -321,7 +574,11 @@ def _workflow_projection( self, workflow_id: str, info: WorkflowInfo, + entry: Optional[CatalogEntry] = None, ) -> dict: + if entry is None: + entry = self._entries_by_id.get(workflow_id) + properties = entry.card.inputs if entry is not None else {} result = { "id": workflow_id, "available": info.ok, @@ -329,7 +586,7 @@ def _workflow_projection( "parameters": [ { "name": name, - "type": "string", + "type": (properties.get(name) or {}).get("type", "string"), "required": not self.allow_recorded_defaults, } for name in sorted(info.params) @@ -402,6 +659,9 @@ def _get_run_report(self, run_id: str) -> dict: raise BridgeError("run_id resolves outside the server's runs directory") report_path = run_dir / "report.json" if report_path.is_symlink() or not report_path.is_file(): + record = read_record(self.runner_config.runs_dir, run_id) + if record is not None and record.get("state") == "finished": + return self._legacy_from_record(run_id, record) raise BridgeError("no local report exists for that run id") try: report = json.loads(report_path.read_text()) @@ -410,20 +670,24 @@ def _get_run_report(self, run_id: str) -> dict: raise BridgeError("the local report could not be read safely") from exc if not isinstance(report, dict): raise BridgeError("the local report has no trustworthy terminal structure") - status, execution_outcome = classify_report_status(report) - result = { - "schema_version": 1, - "run_id": run_id, - "status": status, - "success": status == "success", - "sealed": False, - "requires_seal": True, - "frames_included": False, - "message": public_outcome_message(status, execution_outcome), - "summary": public_report_summary(report), - } - if execution_outcome is not None: - result["execution_outcome"] = execution_outcome + _status, execution_outcome = classify_report_status(report) + transaction = report.get("transaction_outcome") + reason = reason_for_run( + exit_code=None, + report=report, + attention=attention_for(self.attended, run_dir), + ) + outcome = RunOutcome( + status=status_for_reason(reason), + workflow="", + run_id=run_id, + execution_outcome=execution_outcome, + transaction_outcome=transaction if isinstance(transaction, str) else None, + reason=reason, + summary=public_report_summary(report), + ) + result = outcome.to_dict() + result.pop("workflow_id", None) if self.allow_protected_export: result["protected"] = { "run_dir": str(run_dir), @@ -431,6 +695,29 @@ def _get_run_report(self, run_id: str) -> dict: } return result + @staticmethod + def _legacy_from_record(run_id: str, record: dict) -> dict: + """Older get_run_report shape for a run that left no Flow report.""" + result = dict(record.get("result") or {}) + reason = str(result.get("reason") or "result_unreadable") + technical = result.get("technical") or {} + outcome = RunOutcome( + status=status_for_reason( + reason, + refused=result.get("outcome") == "did_not_run" + and not technical.get("transaction_outcome"), + ), + workflow="", + run_id=run_id, + reason=reason, + failed_checks=list(result.get("failed_checks") or []), + execution_outcome=technical.get("execution_outcome"), + transaction_outcome=technical.get("transaction_outcome"), + ) + payload = outcome.to_dict() + payload.pop("workflow_id", None) + return payload + def _run(self, workflow_id: str, arguments: dict) -> dict: info = self._require_workflow(workflow_id) if not self.allow_run: @@ -461,9 +748,13 @@ def _run(self, workflow_id: str, arguments: dict) -> dict: params=params, url_override=url_override, ) + attention = ( + attention_for(self.attended, outcome.run_dir) if outcome.status == "halt" else None + ) + outcome.apply_attention(attention) result = outcome.to_dict( include_protected=self.allow_protected_export, ) if outcome.status == "halt": - result["needs_attention"] = self.attended.for_run_dir(outcome.run_dir) + result["needs_attention"] = attention return result diff --git a/src/openadapt_agent/bundles.py b/src/openadapt_agent/bundles.py index b846ef7..247d555 100644 --- a/src/openadapt_agent/bundles.py +++ b/src/openadapt_agent/bundles.py @@ -59,6 +59,12 @@ class WorkflowInfo: schema_version: int | None = None encrypted: bool = False load_error: str | None = None + #: Typed parameter facts from Flow's ``param_specs``: name -> ``{type, + #: choices}``. Recorded example values are deliberately not copied here. + param_types: dict[str, dict[str, Any]] = field(default_factory=dict) + #: Whether any step may change a record (Flow's fail-closed + #: ``is_consequential``). Unknown counts as True. + changes_records: bool = True @property def ok(self) -> bool: @@ -96,9 +102,35 @@ def load_workflow_info(bundle_dir: Path, slug: str | None = None) -> WorkflowInf step_intents=[step.intent for step in wf.steps], schema_version=wf.schema_version, encrypted=encrypted, + param_types=_param_types(wf), + changes_records=_changes_records(wf), ) +def _param_types(workflow: Any) -> dict[str, dict[str, Any]]: + """Copy each declared parameter's type and choices, never its example.""" + specs = getattr(workflow, "param_specs", None) or {} + types: dict[str, dict[str, Any]] = {} + for name, spec in specs.items(): + kind = getattr(getattr(spec, "type", None), "value", getattr(spec, "type", None)) + choices = [str(choice) for choice in (getattr(spec, "choices", None) or [])] + types[str(name)] = { + "type": str(kind) if kind else "string", + "choices": choices, + } + return types + + +def _changes_records(workflow: Any) -> bool: + """Fail-closed: True unless Flow says no step is consequential.""" + try: + from openadapt_flow.run_gate import is_consequential + + return any(is_consequential(step, workflow) for step in workflow.steps) + except Exception: + return True + + def discover_bundles(root: Path) -> list[WorkflowInfo]: """Discover bundles under *root*. diff --git a/src/openadapt_agent/cards.py b/src/openadapt_agent/cards.py new file mode 100644 index 0000000..2bceea7 --- /dev/null +++ b/src/openadapt_agent/cards.py @@ -0,0 +1,346 @@ +"""Workflow cards: what a calling agent needs to choose and call a workflow. + +A card gives each workflow a stable ``name``, a one-sentence ``purpose``, what +``done_means`` for it, and a JSON Schema for its ``inputs``. Types and choices +come from Flow's ``param_specs``. Words come from an optional operator-written +``openadapt-card.json`` file next to the bundle's ``workflow.json``:: + + { + "name": "enter_referral", + "purpose": "Create an outgoing referral for an existing patient in the clinic EMR.", + "done_means": "The referral appears on the patient's chart in the EMR's referral report.", + "inputs": { + "patient_mrn": {"description": "Clinic MRN of the patient.", "pattern": "^[0-9]{6,10}$"}, + "referral_date": {"description": "Date of the referral.", "format": "date"} + } + } + +Card text reaches the calling agent and, through it, a hosted model. It is +operator-authored, length-limited, and refused when it looks like it carries +an identifier. Recorded example values and step intents never enter a card. +""" + +from __future__ import annotations + +import datetime as _dt +import json +import logging +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping, Optional + +__all__ = [ + "CARD_FILENAME", + "CardError", + "WorkflowCard", + "card_for_bundle", + "coerce_inputs", + "default_purpose", + "input_problems", + "inputs_schema", +] + +CARD_FILENAME = "openadapt-card.json" +_LOG = logging.getLogger(__name__) +_NAME_RE = re.compile(r"^[a-z][a-z0-9_]{2,63}$") +_TEXT_LIMITS = {"purpose": 300, "done_means": 300, "description": 200} +# Operator text is checked for the shapes identifiers usually take. This is a +# guard against accidents, not a PHI classifier. +_IDENTIFIER_SHAPES = ( + re.compile(r"\d{5,}"), + re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"), + re.compile(r"\b\d{3}[-. )]+\d{3}[-. ]+\d{4}\b"), +) +_ALLOWED_INPUT_KEYS = { + "description", + "format", + "pattern", + "enum", + "minLength", + "maxLength", + "minimum", + "maximum", +} +_FORMATS = {"date", "date-time"} +_FLOW_TYPES = { + "string": {"type": "string"}, + "entity_ref": {"type": "string"}, + "date": {"type": "string", "format": "date"}, + "enum": {"type": "string"}, + "number": {"type": "number"}, + "boolean": {"type": "boolean"}, +} +DEFAULT_DONE_MEANS = ( + "OpenAdapt saved the change and its record check confirmed it in the app's records." +) + + +def default_purpose() -> str: + return ( + "A workflow recorded on this computer. Its owner hasn't described it yet, " + "so ask a person what it does before you run it." + ) + + +class CardError(ValueError): + """An operator-written card is invalid. The message never repeats card text.""" + + +@dataclass +class WorkflowCard: + """The agent-facing description of one workflow.""" + + name: str + purpose: str + done_means: str + inputs: dict[str, dict[str, Any]] = field(default_factory=dict) + required: list[str] = field(default_factory=list) + changes_records: bool = True + authored: bool = False + + def projection(self) -> dict[str, Any]: + return { + "name": self.name, + "purpose": self.purpose, + "done_means": self.done_means, + "inputs": inputs_schema(self), + "changes_records": self.changes_records, + } + + +def _check_text(field_name: str, value: Any) -> str: + limit = _TEXT_LIMITS[field_name] + if not isinstance(value, str) or not value.strip(): + raise CardError(f"{field_name} must be non-empty text") + text = " ".join(value.split()) + if len(text) > limit: + raise CardError(f"{field_name} must be {limit} characters or fewer") + if any(shape.search(text) for shape in _IDENTIFIER_SHAPES): + raise CardError(f"{field_name} looks like it contains an identifier") + return text + + +def _flow_schema(param_type: Optional[Mapping[str, Any]]) -> dict[str, Any]: + kind = (param_type or {}).get("type") or "string" + schema = dict(_FLOW_TYPES.get(kind, {"type": "string"})) + choices = [str(choice) for choice in (param_type or {}).get("choices") or []] + if choices: + schema["enum"] = choices + return schema + + +def _default_description(name: str) -> str: + return f"The {name.replace('_', ' ')} to enter." + + +def _merge_input(name: str, base: dict[str, Any], authored: Any) -> dict[str, Any]: + if not isinstance(authored, Mapping): + raise CardError("each card input must be an object") + unknown = set(authored) - _ALLOWED_INPUT_KEYS + if unknown: + raise CardError("a card input uses an unsupported key") + schema = dict(base) + if "description" in authored: + schema["description"] = _check_text("description", authored["description"]) + if "format" in authored: + if authored["format"] not in _FORMATS or schema.get("type") != "string": + raise CardError("format must be date or date-time on a text input") + schema["format"] = authored["format"] + if "pattern" in authored: + pattern = authored["pattern"] + if not isinstance(pattern, str) or len(pattern) > 200: + raise CardError("pattern must be a short regular expression") + try: + re.compile(pattern) + except re.error as exc: + raise CardError("pattern is not a valid regular expression") from exc + schema["pattern"] = pattern + if "enum" in authored: + values = authored["enum"] + if ( + not isinstance(values, list) + or not values + or not all(isinstance(value, str) and value for value in values) + ): + raise CardError("enum must be a non-empty list of text values") + if "enum" in schema and not set(values) <= set(schema["enum"]): + raise CardError("enum can only narrow the workflow's own choices") + schema["enum"] = list(values) + for key in ("minLength", "maxLength"): + if key in authored: + value = authored[key] + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise CardError(f"{key} must be a whole number") + schema[key] = value + for key in ("minimum", "maximum"): + if key in authored: + value = authored[key] + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise CardError(f"{key} must be a number") + schema[key] = value + return schema + + +def card_for_bundle(info: Any, *, default_name: str) -> WorkflowCard: + """Build the card for one discovered bundle. + + ``info`` is a :class:`openadapt_agent.bundles.WorkflowInfo`. A missing card + file gives a plain default card. An invalid one is ignored with a local + warning, so a typo can't stop the server or leak its text. + """ + params = sorted(getattr(info, "params", {}) or {}) + param_types = getattr(info, "param_types", {}) or {} + inputs = { + name: {**_flow_schema(param_types.get(name)), "description": _default_description(name)} + for name in params + } + card = WorkflowCard( + name=default_name, + purpose=default_purpose(), + done_means=DEFAULT_DONE_MEANS, + inputs=inputs, + required=list(params), + changes_records=bool(getattr(info, "changes_records", True)), + ) + path = Path(getattr(info, "bundle_dir", ".")) / CARD_FILENAME + if not path.is_file() or path.is_symlink(): + return card + try: + authored = json.loads(path.read_text(encoding="utf-8")) + return _apply_card(card, authored) + except (OSError, ValueError, CardError) as exc: + reason = str(exc) if isinstance(exc, CardError) else type(exc).__name__ + _LOG.warning("ignoring workflow card for %s: %s", default_name, reason) + return card + + +def _apply_card(card: WorkflowCard, authored: Any) -> WorkflowCard: + if not isinstance(authored, Mapping): + raise CardError("the card must be a JSON object") + unknown = set(authored) - {"name", "purpose", "done_means", "inputs"} + if unknown: + raise CardError("the card uses an unsupported key") + name = authored.get("name", card.name) + if not isinstance(name, str) or not _NAME_RE.fullmatch(name): + raise CardError("name must be 3-64 lowercase letters, digits, or underscores") + inputs = dict(card.inputs) + authored_inputs = authored.get("inputs", {}) + if not isinstance(authored_inputs, Mapping): + raise CardError("inputs must be an object") + if set(authored_inputs) - set(inputs): + raise CardError("the card describes an input the workflow doesn't have") + for input_name, spec in authored_inputs.items(): + inputs[input_name] = _merge_input(input_name, inputs[input_name], spec) + return WorkflowCard( + name=name, + purpose=_check_text("purpose", authored["purpose"]) + if "purpose" in authored + else card.purpose, + done_means=_check_text("done_means", authored["done_means"]) + if "done_means" in authored + else card.done_means, + inputs=inputs, + required=list(card.required), + changes_records=card.changes_records, + authored=True, + ) + + +def inputs_schema(card: WorkflowCard, *, require_all: bool = True) -> dict[str, Any]: + """JSON Schema for the ``inputs`` object of ``run_workflow``.""" + return { + "type": "object", + "properties": {name: dict(schema) for name, schema in card.inputs.items()}, + "required": sorted(card.required) if require_all else [], + "additionalProperties": False, + } + + +def _valid_date(value: str) -> bool: + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): + return False + try: + _dt.date.fromisoformat(value) + except ValueError: + return False + return True + + +def _valid_datetime(value: str) -> bool: + try: + _dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return False + return "T" in value or " " in value + + +def _problem(schema: Mapping[str, Any], value: Any) -> Optional[str]: + kind = schema.get("type", "string") + if kind == "string": + if not isinstance(value, str): + return "must be text" + if "enum" in schema and value not in schema["enum"]: + return "must be one of the listed values" + if "minLength" in schema and len(value) < schema["minLength"]: + return "is too short" + if "maxLength" in schema and len(value) > schema["maxLength"]: + return "is too long" + if schema.get("format") == "date" and not _valid_date(value): + return "must be a date like 2026-10-08" + if schema.get("format") == "date-time" and not _valid_datetime(value): + return "must be a date and time like 2026-10-08T09:30:00" + if "pattern" in schema and re.search(schema["pattern"], value) is None: + return "doesn't match the expected format" + return None + if kind == "number": + if not isinstance(value, (int, float)) or isinstance(value, bool): + return "must be a number" + elif kind == "boolean": + if not isinstance(value, bool): + return "must be true or false" + return None + if "minimum" in schema and value < schema["minimum"]: + return "is too small" + if "maximum" in schema and value > schema["maximum"]: + return "is too large" + return None + + +def input_problems( + card: WorkflowCard, inputs: Any, *, require_all: bool = True +) -> list[dict[str, str]]: + """Closed-vocabulary problems with ``inputs``. Values are never repeated.""" + if not isinstance(inputs, Mapping): + return [{"input": "inputs", "problem": "must be an object"}] + problems: list[dict[str, str]] = [] + if set(inputs) - set(card.inputs): + # Never echo a caller-supplied key: it could itself carry a value. + problems.append( + { + "input": "(unknown)", + "problem": "isn't an input of this workflow; list_workflows has the names", + } + ) + for name, schema in sorted(card.inputs.items()): + if name not in inputs: + if require_all and name in card.required: + problems.append({"input": name, "problem": "is missing"}) + continue + problem = _problem(schema, inputs[name]) + if problem is not None: + problems.append({"input": name, "problem": problem}) + return problems + + +def coerce_inputs(card: WorkflowCard, inputs: Mapping[str, Any]) -> dict[str, str]: + """Render validated inputs as the strings Flow's ``--params-file`` takes.""" + rendered: dict[str, str] = {} + for name, value in inputs.items(): + if isinstance(value, bool): + rendered[name] = "true" if value else "false" + elif isinstance(value, float) and value.is_integer(): + rendered[name] = str(int(value)) + else: + rendered[name] = str(value) + return rendered diff --git a/src/openadapt_agent/cli.py b/src/openadapt_agent/cli.py index f42242f..ba24d82 100644 --- a/src/openadapt_agent/cli.py +++ b/src/openadapt_agent/cli.py @@ -1,17 +1,18 @@ -"""``openadapt-agent`` CLI: serve bundles over MCP, emit Agent Skills. +"""``openadapt-agent`` CLI: serve workflows over MCP, emit Agent Skills. Subcommands: -- ``serve`` — expose compiled openadapt-flow bundles and the local Needs - Attention queue over MCP stdio. PHI-safe read-only tools are always on; - workflow runs and attended decisions require separate operator flags. - ``--authoring`` adds first-demo stdio tools and does not imply - ``--allow-run``. +- ``serve`` — with no flags, a sandbox: one synthetic workflow that shows + every result your agent can get back. ``--mode production --bundles DIR`` + serves an operator's compiled workflows; ``--mode attended`` also lets a + person at this computer answer paused runs. ``--bundles DIR`` alone stays + read-only. ``--authoring`` adds first-demo stdio tools and does not imply + runs. - ``authoring connect`` — outbound mailbox client for hosted ChatGPT.com / Claude.ai (claim ``oab_``, poll wait=0, Allow-per-sub). Not an HTTP listener. Overlay chrome stays Desktop-only. -- ``emit-skill`` — emit a Claude Agent Skill folder for one bundle - (wraps ``openadapt-flow emit-skill`` and appends MCP + halt guidance). +- ``emit-skill`` — write a PHI-safe Agent Skill for one workflow from its + card: purpose, inputs, and how to read the four outcomes. ``serve`` stays local stdio. Hosted ChatGPT.com reaches this computer through ``authoring connect`` (outbound HTTPS), not a port-forwarded MCP server. @@ -35,9 +36,8 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="openadapt-agent", description=( - "Agent-facing bridge for openadapt-flow: expose compiled workflow " - "bundles, Needs Attention, and governed operator decisions as " - "local MCP tools and Agent Skills." + "Your AI agent decides what to enter. OpenAdapt enters it in the app " + "and checks that it saved. 'serve' with no flags starts a sandbox." ), ) parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") @@ -45,17 +45,39 @@ def build_parser() -> argparse.ArgumentParser: p = sub.add_parser( "serve", - help="Serve a directory of workflow bundles as an MCP server (stdio)", + help=( + "Serve workflows to your AI agent over MCP (stdio). With no flags, " + "serve the sandbox." + ), + ) + p.add_argument( + "--mode", + choices=("sandbox", "attended", "production"), + default=None, + help=( + "sandbox (default with no --bundles): one synthetic workflow that " + "shows every result. production: run the compiled workflows in " + "--bundles. attended: production plus decisions on paused runs by a " + "person at this computer. --bundles without --mode stays read-only." + ), ) p.add_argument( "--bundles", default=None, help=( "Bundle directory: either one compiled bundle, or a directory " - "whose immediate subdirectories are bundles. Required unless " - "--authoring or --tutorial is set, or --allow-run is set with no " - "--bundles (synthetic tutorial). The published run recipe still " - "requires --bundles." + "whose immediate subdirectories are bundles. Needed for " + "--mode production and --mode attended." + ), + ) + p.add_argument( + "--sandbox-engine", + choices=("auto", "flow", "simulated"), + default="auto", + help=( + "Advanced. flow drives the synthetic app in a hidden browser (needs " + "the tutorial extra); simulated returns the same results without " + "opening it. auto picks flow when it can." ), ) p.add_argument( @@ -77,19 +99,16 @@ def build_parser() -> argparse.ArgumentParser: "--tutorial", action="store_true", help=( - "Generate and serve the public synthetic tutorial (MockMed). " - "The bundle is created at serve time and is not vendored. " - "Implied by --allow-run when --bundles is omitted. " - "Synthetic only. Cannot be combined with --bundles, --url, " - "or --config." + "Older name for --mode sandbox. Synthetic only. Cannot be combined " + "with --bundles, --url, or --config." ), ) p.add_argument( "--allow-run", action="store_true", help=( - "Register run_* tools (default: read-only tools only). Without " - "this flag no MCP client can execute anything." + "Older name for --mode production (with --bundles) or --mode sandbox " + "(without). Without a mode or this flag, --bundles stays read-only." ), ) p.add_argument( @@ -116,9 +135,10 @@ def build_parser() -> argparse.ArgumentParser: "--allow-attended-actions", action="store_true", help=( - "Register governed Reject/Teach/Escalate tools for signed durable pauses. " - "With --config, also register Continue/Skip through Flow's " - "deployment-bound live verifier and deterministic resume path." + "Part of --mode attended. Register governed Reject/Teach/Escalate tools " + "for signed durable pauses. With --config, also register Continue/Skip " + "through Flow's deployment-bound live verifier and deterministic " + "resume path." ), ) p.add_argument( @@ -143,8 +163,11 @@ def build_parser() -> argparse.ArgumentParser: ) p.add_argument( "--runs-dir", - default="runs", - help="Directory for per-run output directories (default: ./runs)", + default=None, + help=( + "Directory for run records and evidence (default: ./runs; the " + "sandbox uses ~/.openadapt/agent-sandbox)" + ), ) p.add_argument( "--timeout", @@ -246,82 +269,118 @@ def build_parser() -> argparse.ArgumentParser: p = sub.add_parser( "emit-skill", help=( - "Emit a Claude Agent Skill folder for a bundle (wraps " - "`openadapt-flow emit-skill`, appends MCP + halt guidance)" + "Write an Agent Skill for one workflow: what it does, its inputs, and " + "how to read the result. No recorded values, step text, or secrets." ), ) p.add_argument("bundle", help="Workflow bundle directory") p.add_argument("--out", required=True, help="Parent directory for the skill folder") + p.add_argument( + "--include-bundle", + action="store_true", + help=( + "Also copy the compiled bundle next to SKILL.md. The copy is protected " + "workflow data; install it only where that data may live." + ), + ) p.set_defaults(func=_cmd_emit_skill) return parser +def _resolve_mode(args: argparse.Namespace) -> Optional[str]: + """Pick the server mode from --mode and the older flags it replaces. + + Returns None for an authoring-only server. Raises ValueError with the + message to print when the flags conflict. + """ + if args.mode is not None: + if args.mode == "sandbox": + if args.bundles: + raise ValueError("--mode sandbox cannot be combined with --bundles") + return "sandbox" + if not args.bundles: + raise ValueError(f"--mode {args.mode} needs --bundles") + args.allow_run = True + if args.mode == "attended": + args.allow_attended_actions = True + return args.mode + if args.tutorial: + return "sandbox" + if args.bundles: + # Older flags: a person can answer paused runs with or without runs. + return "attended" if args.allow_attended_actions else "production" + if args.authoring: + return None + return "sandbox" + + def _cmd_serve(args: argparse.Namespace) -> int: from openadapt_agent.bridge import AgentBridge from openadapt_agent.flow_service import open_attended_service from openadapt_agent.mcp import serve - from openadapt_agent.tutorial import TutorialError, prepare_tutorial_session - from openadapt_agent.runner import default_flow_cli + from openadapt_agent.tutorial import TutorialError - if args.allow_attended_actions and args.flow_cli: + if args.authoring and args.tutorial: + print("serve: --authoring cannot be combined with --tutorial", file=sys.stderr) + return 2 + if args.authoring and args.allow_run and not args.bundles: print( - "serve: attended actions require the openadapt-flow installed in " - "this interpreter; --flow-cli cannot select a different runtime", + "serve: --authoring does not imply --allow-run; --allow-run still " + "requires --bundles", file=sys.stderr, ) return 2 - if args.allow_synthetic_recorded_defaults and not args.allow_run: + if args.tutorial and (args.bundles or args.url or args.config): print( - "serve: --allow-synthetic-recorded-defaults requires --allow-run", + "serve: --tutorial cannot be combined with --bundles, --url, or --config", file=sys.stderr, ) return 2 - if args.authoring and args.tutorial: - print("serve: --authoring cannot be combined with --tutorial", file=sys.stderr) + try: + mode = _resolve_mode(args) + except ValueError as exc: + print(f"serve: {exc}", file=sys.stderr) return 2 - if args.authoring and args.allow_run and not args.bundles: + # After mode resolution, so --mode attended is covered too. + if args.allow_attended_actions and args.flow_cli: print( - "serve: --authoring does not imply --allow-run; --allow-run still " - "requires --bundles", + "serve: attended actions require the openadapt-flow installed in " + "this interpreter; --flow-cli cannot select a different runtime", file=sys.stderr, ) return 2 - if not args.tutorial and not args.bundles and not args.authoring: - if args.allow_run: - # Day-1 partner kit: `openadapt-agent serve --allow-run` hosts - # the synthetic tutorial. Registry installs still pass --bundles - # and stay read-only until this flag is set. - args.tutorial = True - else: - print( - "serve: provide --bundles, --tutorial, or --authoring " - "(or --allow-run for the synthetic tutorial)", - file=sys.stderr, - ) - return 2 - if args.tutorial and (args.bundles or args.url or args.config): + if mode == "sandbox" and (args.url or args.config): print( - "serve: --tutorial cannot be combined with --bundles, --url, or --config", + "serve: the sandbox runs a synthetic app; --url and --config need " + "--mode production --bundles DIR", + file=sys.stderr, + ) + return 2 + if args.allow_synthetic_recorded_defaults and not (args.allow_run and args.bundles): + print( + "serve: --allow-synthetic-recorded-defaults requires --allow-run", file=sys.stderr, ) return 2 - extra_run_args = list(args.extra_run_arg) - tutorial_session = None + sandbox_engine = None authoring_bridge = None - bundles_dir = args.bundles - url = args.url - deployment_config = args.config - policy = args.policy - public_synthetic = False try: if args.authoring: from openadapt_agent.authoring import AuthoringBridge, AuthoringError from openadapt_agent.authoring import open_authoring_session - authoring_dir = Path(args.runs_dir).expanduser().resolve() / "authoring" + if args.runs_dir: + authoring_root = Path(args.runs_dir) + elif mode == "sandbox": + from openadapt_agent.sandbox import default_sandbox_dir + + authoring_root = default_sandbox_dir() + else: + authoring_root = Path("runs") + authoring_dir = authoring_root.expanduser().resolve() / "authoring" try: authoring_bridge = AuthoringBridge( open_authoring_session( @@ -334,23 +393,8 @@ def _cmd_serve(args: argparse.Namespace) -> int: except AuthoringError as exc: print(f"serve: {exc}", file=sys.stderr) return 2 - if args.tutorial: - work_dir = Path(args.runs_dir).expanduser().resolve() / "synthetic-tutorial" - tutorial_session = prepare_tutorial_session( - work_dir, - headed=args.headed, - ) - bundles_dir = str(tutorial_session.bundle_dir) - url = tutorial_session.url - deployment_config = str(tutorial_session.deployment_config) - policy = policy or "clinical-write" - public_synthetic = True - if "--profile" not in extra_run_args: - extra_run_args.extend(["--profile", "standard"]) - if args.headed and "--headed" not in extra_run_args: - extra_run_args.append("--headed") - - if bundles_dir is None and authoring_bridge is not None: + + if mode is None: print( f"openadapt-agent {__version__}: authoring tools enabled over " "local stdio; run tools disabled; --authoring does not imply " @@ -358,53 +402,82 @@ def _cmd_serve(args: argparse.Namespace) -> int: file=sys.stderr, ) _serve(serve, None, authoring_bridge) - else: - runner_config = RunnerConfig( - flow_cli=( - tuple(shlex.split(args.flow_cli)) if args.flow_cli else default_flow_cli() - ), - runs_dir=Path(args.runs_dir), - url=url, - deployment_config=deployment_config, - policy=policy, - timeout_s=args.timeout, - allow_url_override=args.allow_url_override, - extra_run_args=tuple(extra_run_args), + return 0 + + if mode == "sandbox": + from openadapt_agent.sandbox import SandboxEngine, default_sandbox_dir + + runs_dir = Path(args.runs_dir).expanduser() if args.runs_dir else default_sandbox_dir() + runs_dir.mkdir(parents=True, exist_ok=True) + sandbox_engine = SandboxEngine( + runs_dir, engine=args.sandbox_engine, headed=args.headed ) - with open_attended_service( - enabled=args.allow_attended_actions, - deployment_config=deployment_config, - url=url, - headed=args.headed, - allow_model_grounding=args.allow_model_grounding, - ) as attended_service: - bridge = AgentBridge( - Path(bundles_dir), - runner_config, - allow_run=args.allow_run, - allow_attended_actions=args.allow_attended_actions, - attended_service=attended_service, - allow_protected_export=args.allow_protected_export, - allow_recorded_defaults=args.allow_synthetic_recorded_defaults, - public_synthetic=public_synthetic, - ) - n = len(bridge.workflows) - print( - f"openadapt-agent {__version__}: serving {n} workflow(s) " - "over local stdio; run tools " - f"{'enabled' if args.allow_run else 'disabled'}; attended " - f"decisions {'enabled' if args.allow_attended_actions else 'disabled'}; " - "live Continue/Skip " - f"{'ready' if bridge.attended.live_actions_ready else 'not configured'}; " - "protected MCP export " - f"{'ENABLED' if args.allow_protected_export else 'disabled'}; " - "synthetic recorded defaults " - f"{'ENABLED' if args.allow_synthetic_recorded_defaults else 'disabled'}; " - f"tutorial {'enabled' if args.tutorial else 'disabled'}; " - f"authoring {'enabled' if authoring_bridge is not None else 'disabled'}", - file=sys.stderr, + bridge = AgentBridge( + None, + RunnerConfig(runs_dir=runs_dir, timeout_s=args.timeout), + sandbox=sandbox_engine, + mode="sandbox", + ) + note = "" + if sandbox_engine.unavailable_reason == "browser_extra_missing": + note = ( + " To drive the synthetic app in a hidden browser instead, " + "install the tutorial extra." ) - _serve(serve, bridge, authoring_bridge) + print( + f"openadapt-agent {__version__}: sandbox mode. One workflow " + f"({next(iter(bridge.catalog))}) on a synthetic app; engine " + f"{sandbox_engine.engine}. Tools: list_workflows, run_workflow, " + f"get_run. Nothing real changes.{note}", + file=sys.stderr, + ) + _serve(serve, bridge, authoring_bridge) + return 0 + + extra_run_args = list(args.extra_run_arg) + runner_config = RunnerConfig( + flow_cli=(tuple(shlex.split(args.flow_cli)) if args.flow_cli else default_flow_cli()), + runs_dir=Path(args.runs_dir or "runs"), + url=args.url, + deployment_config=args.config, + policy=args.policy, + timeout_s=args.timeout, + allow_url_override=args.allow_url_override, + extra_run_args=tuple(extra_run_args), + ) + with open_attended_service( + enabled=args.allow_attended_actions, + deployment_config=args.config, + url=args.url, + headed=args.headed, + allow_model_grounding=args.allow_model_grounding, + ) as attended_service: + bridge = AgentBridge( + Path(args.bundles), + runner_config, + allow_run=args.allow_run, + allow_attended_actions=args.allow_attended_actions, + attended_service=attended_service, + allow_protected_export=args.allow_protected_export, + allow_recorded_defaults=args.allow_synthetic_recorded_defaults, + mode=mode, + ) + n = len(bridge.workflows) + print( + f"openadapt-agent {__version__}: {mode} mode; serving {n} workflow(s) " + "over local stdio; run tools " + f"{'enabled' if args.allow_run else 'disabled (read-only)'}; attended " + f"decisions {'enabled' if args.allow_attended_actions else 'disabled'}; " + "live Continue/Skip " + f"{'ready' if bridge.attended.live_actions_ready else 'not configured'}; " + "protected MCP export " + f"{'ENABLED' if args.allow_protected_export else 'disabled'}; " + "synthetic recorded defaults " + f"{'ENABLED' if args.allow_synthetic_recorded_defaults else 'disabled'}; " + f"authoring {'enabled' if authoring_bridge is not None else 'disabled'}", + file=sys.stderr, + ) + _serve(serve, bridge, authoring_bridge) except TutorialError as exc: print(f"serve: {exc}", file=sys.stderr) return 2 @@ -412,8 +485,8 @@ def _cmd_serve(args: argparse.Namespace) -> int: print(f"serve: {exc}", file=sys.stderr) return 2 finally: - if tutorial_session is not None: - tutorial_session.close() + if sandbox_engine is not None: + sandbox_engine.close() if authoring_bridge is not None: closer = getattr(authoring_bridge.session, "close", None) if callable(closer): @@ -446,7 +519,13 @@ def _cmd_authoring_connect(args: argparse.Namespace) -> int: def _cmd_emit_skill(args: argparse.Namespace) -> int: from openadapt_agent.skill import emit_agent_skill - skill_dir = emit_agent_skill(Path(args.bundle), Path(args.out)) + try: + skill_dir = emit_agent_skill( + Path(args.bundle), Path(args.out), include_bundle=args.include_bundle + ) + except ValueError as exc: + print(f"emit-skill: {exc}", file=sys.stderr) + return 2 print(f"Wrote Agent Skill folder: {skill_dir}") return 0 diff --git a/src/openadapt_agent/contract.py b/src/openadapt_agent/contract.py new file mode 100644 index 0000000..4e87a1e --- /dev/null +++ b/src/openadapt_agent/contract.py @@ -0,0 +1,828 @@ +"""The partner result contract: one shape, four outcomes, plain words. + +Every agent-facing result in this package is derived here: ``run_workflow``, +``get_run``, the deprecated ``run_`` tools, ``get_run_report``, and the +Agent Skill text. A calling agent decides one thing from a result: move on, +wait for a person, have a person check the record, or fix something and try +again. The four outcomes map one-to-one onto those decisions. + +Safety rules this module enforces by construction: + +- The plain result is a pure function of Flow's ``transaction_outcome``, the + process facts (did Flow start, did it exit cleanly, did it time out), the + state of any durable pause, and a consistency check of the persisted report. + The coarse ``HALTED`` label alone never selects a result. +- Only ``done`` says the change was saved, and only a consistent ``VERIFIED`` + report can produce it. How strong the proof is (``local``, ``sealed``, or + ``simulated`` in the sandbox) is a separate field. A verified write is never + reported as an error. +- ``record_changed: "no"`` and ``safe_to_retry: true`` appear only for results + whose evidence proves nothing was written. Uncertain delivery (including a + timeout after Flow started) is ``not_sure_if_saved`` with + ``safe_to_retry: false``. +- A run with an open durable pause is ``needs_review``: a person holds the + decision, so the caller must not start the same work again. +- Every sentence is fixed copy keyed by a closed reason code. No observed screen + text, recorded value, input value, or local path can reach a result. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any, Iterable, Mapping, Optional + +__all__ = [ + "CONTRACT_VERSION", + "FAILED_CHECKS", + "LABELS", + "MODES", + "NEXT_STEPS", + "OUTCOMES", + "PROOF", + "REASONS", + "RETRYABLE_REASONS", + "RUNNING", + "RUN_RESULT_SCHEMA", + "TRANSACTION_OUTCOMES", + "TRANSACTION_REASONS", + "WORKFLOW_LIST_SCHEMA", + "Reason", + "build_result", + "ledger_outcome", + "parse_failed_checks", + "pause_state", + "reason_for_run", + "report_consistency", +] + +#: Version of the result shape. Bump only for an incompatible change. +CONTRACT_VERSION = "openadapt.run-result/v1" + +#: The four terminal outcomes, in the order a reader should learn them. +OUTCOMES = ("done", "needs_review", "not_sure_if_saved", "did_not_run") +#: The one non-terminal value. ``get_run`` resolves it later. +RUNNING = "running" +#: Who the server is serving. ``sandbox`` results come from a synthetic app. +MODES = ("sandbox", "attended", "production") +#: Closed machine-readable next steps, mirrored by the plain ``next_action``. +NEXT_STEPS = ( + "none", + "wait_for_person", + "check_record", + "fix_inputs", + "fix_setup", + "retry", + "call_get_run", +) +#: How strong the evidence behind ``done`` is. ``sealed`` means a signed +#: receipt exists; ``local`` means the record check ran on this computer and +#: its evidence stays here; ``simulated`` means a sandbox result produced +#: without opening any app; ``none`` means no saved change is being claimed. +PROOF = ("sealed", "local", "simulated", "none") +#: What is known about the target record after this run. +RECORD_CHANGED = ("yes", "no", "unknown") + +#: Flow ``TransactionOutcome`` values this bridge understands. +TRANSACTION_OUTCOMES = frozenset( + { + "VERIFIED", + "HALTED_BEFORE_EFFECT", + "RECONCILIATION_REQUIRED", + "FAILED_PLATFORM", + "CANCELED", + "REJECTED_POLICY", + "COMPLETED_UNVERIFIED", + "ROLLED_BACK", + } +) +#: Flow transaction outcomes that prove no business effect occurred. +_NO_EFFECT_OUTCOMES = frozenset( + {"HALTED_BEFORE_EFFECT", "REJECTED_POLICY", "CANCELED", "FAILED_PLATFORM"} +) +_EXECUTION_OUTCOMES = frozenset( + {"VERIFIED", "COMPLETED_UNVERIFIED", "HALTED", "FAILED", "ROLLED_BACK"} +) + +#: Short plain label per outcome group, for a UI badge or a log line. +LABELS = { + "done": "Done and checked", + "needs_review": "Stopped before saving", + "not_sure_if_saved": "Check the record", + "did_not_run": "Didn't run", + RUNNING: "Running", +} + + +@dataclass(frozen=True) +class Reason: + """Fixed copy and semantics for one closed reason code.""" + + outcome: str + safe_to_retry: bool + record_changed: str + next_step: str + what_happened: str + next_action: str + label: Optional[str] = None + + +_CHECK_RECORD = ( + "Have a person check the record in the app before anything runs again. " + "Don't retry this request." +) +_WAIT_FOR_PERSON = ( + "A person reviews this run on the computer that ran it. Call get_run " + "later to see their decision. Don't start another run for this work " + "until they finish." +) + +REASONS: dict[str, Reason] = { + # -- done ------------------------------------------------------------ + "saved_and_checked": Reason( + "done", + False, + "yes", + "none", + "Saved the change and read the record back to confirm it.", + "Nothing more to do for this request. Don't run it again for the same work.", + ), + # -- needs_review: stopped before saving, nothing written ------------- + "unexpected_screen": Reason( + "needs_review", + False, + "no", + "wait_for_person", + "Stopped before saving. The app showed a screen the workflow didn't " + "expect, so nothing was written.", + _WAIT_FOR_PERSON, + ), + "record_not_confirmed": Reason( + "needs_review", + False, + "no", + "wait_for_person", + "Stopped before saving. It couldn't confirm it had the right record " + "open, so nothing was written.", + _WAIT_FOR_PERSON, + ), + "person_needed": Reason( + "needs_review", + False, + "no", + "wait_for_person", + "Stopped before saving. The app needs a person, for example to sign " + "in, so nothing was written.", + _WAIT_FOR_PERSON, + ), + "record_check_not_set_up": Reason( + "needs_review", + False, + "no", + "wait_for_person", + "Stopped before saving. This workflow's record check isn't set up " + "yet, so nothing was written.", + _WAIT_FOR_PERSON, + ), + "stopped_for_review": Reason( + "needs_review", + False, + "no", + "wait_for_person", + "Stopped before saving because something didn't match, so nothing was written.", + _WAIT_FOR_PERSON, + ), + # -- not_sure_if_saved: a person checks the record -------------------- + "save_not_confirmed": Reason( + "not_sure_if_saved", + False, + "unknown", + "check_record", + "OpenAdapt can't confirm whether the change was saved. It may have gone through.", + _CHECK_RECORD, + ), + "not_checked": Reason( + "not_sure_if_saved", + False, + "unknown", + "check_record", + "It finished the steps but didn't check the saved record, so the " + "change may or may not be saved.", + _CHECK_RECORD, + label="Finished, not checked", + ), + "change_reversed": Reason( + "not_sure_if_saved", + False, + "unknown", + "check_record", + "OpenAdapt found an extra change and reversed it. The record still " + "needs a person to confirm it's right.", + _CHECK_RECORD, + ), + "timed_out": Reason( + "not_sure_if_saved", + False, + "unknown", + "check_record", + "The run didn't finish in time. The change may or may not be saved.", + _CHECK_RECORD, + ), + "result_unreadable": Reason( + "not_sure_if_saved", + False, + "unknown", + "check_record", + "The run ended without a result OpenAdapt can trust. The change may " + "or may not be saved.", + _CHECK_RECORD, + ), + "interrupted": Reason( + "not_sure_if_saved", + False, + "unknown", + "check_record", + "The run stopped before it reported a result, for example because " + "the server closed. The change may or may not be saved.", + _CHECK_RECORD, + ), + # -- did_not_run: proven nothing written ------------------------------- + "invalid_input": Reason( + "did_not_run", + True, + "no", + "fix_inputs", + "Didn't start because some inputs aren't valid. Nothing was written.", + "Fix the inputs listed in invalid_inputs, then send the request again. " + "You can reuse the same request_id.", + ), + "unknown_workflow": Reason( + "did_not_run", + True, + "no", + "fix_inputs", + "Didn't start because there's no workflow with that name here. " + "Nothing was written.", + "Call list_workflows and use one of the names it returns.", + ), + "workflow_unavailable": Reason( + "did_not_run", + True, + "no", + "fix_setup", + "Didn't start because this workflow can't be loaded on this computer. " + "Nothing was written.", + "Ask the operator to check the workflow files on this computer, then " + "send the same request again.", + ), + "runs_disabled": Reason( + "did_not_run", + True, + "no", + "fix_setup", + "Didn't start because this server only lists workflows. Nothing was written.", + "Ask the operator to start the server with --mode production or " + "--mode attended, then send the same request again.", + ), + "not_ready_to_run": Reason( + "did_not_run", + True, + "no", + "fix_setup", + "Didn't start because the workflow hasn't passed the checks it needs " + "before it can run here. Nothing was written.", + "Ask the operator to finish setup (see failed_checks), then send the " + "same request again with the same request_id.", + ), + "policy_refused": Reason( + "did_not_run", + True, + "no", + "fix_setup", + "Didn't start because a policy, sign-off, or environment check " + "refused it. Nothing was written.", + "Ask the operator why this run isn't allowed, then send the same " + "request again with the same request_id.", + ), + "canceled": Reason( + "did_not_run", + True, + "no", + "retry", + "The run was canceled before it changed anything. Nothing was written.", + "If the work is still needed, send the same request again with the " + "same request_id.", + ), + "platform_error": Reason( + "did_not_run", + True, + "no", + "retry", + "OpenAdapt hit a problem on this computer before it changed anything. " + "Nothing was written.", + "Try again with the same request_id. If it keeps happening, tell the operator.", + ), + "app_unreachable": Reason( + "did_not_run", + True, + "no", + "retry", + "Didn't start because the app isn't reachable from this computer. " + "Nothing was written.", + "Try again with the same request_id once the app is back. If it keeps " + "happening, tell the operator.", + ), + "stopped_by_person": Reason( + "did_not_run", + False, + "no", + "none", + "A person ended this run before it saved anything. Nothing was written.", + "Don't retry this request. If the work is still needed, a person " + "decides how, and a new attempt uses a new request_id.", + ), + "request_id_conflict": Reason( + "did_not_run", + False, + "no", + "fix_inputs", + "Didn't start because this request_id was already used for a " + "different request. Nothing new ran.", + "Use a new request_id for new work. To see the earlier result, call " + "get_run with first_run_id.", + ), + "retry_limit": Reason( + "did_not_run", + False, + "no", + "fix_setup", + "Didn't start because this request_id was already tried too many " + "times. Nothing new ran.", + "Ask the operator to look at the earlier runs for this request before " + "trying again.", + ), + # -- not terminal -------------------------------------------------------- + "running": Reason( + RUNNING, + False, + "unknown", + "call_get_run", + "Still running.", + "Call get_run with this run_id to get the result. Don't start the " + "same work again.", + ), +} + +#: Results that let the same ``request_id`` start a fresh attempt. Each one +#: proves nothing was written AND says the retry may now succeed. +RETRYABLE_REASONS = frozenset( + { + "not_ready_to_run", + "policy_refused", + "canceled", + "platform_error", + "app_unreachable", + "workflow_unavailable", + "runs_disabled", + } +) + +#: Closed vocabulary for why Flow refused to start a run. +FAILED_CHECKS: dict[str, str] = { + "no_readiness_test": "The workflow has no signed readiness test for this setup.", + "readiness_test_invalid": "The readiness test is invalid, expired, or doesn't match this run.", + "not_certified": "The workflow isn't certified under the required policy.", + "profile_not_met": "The workflow doesn't meet the selected run profile.", + "right_record_check_missing": "A step that needs a right-record check doesn't have one.", + "record_check_missing": "A step that saves doesn't have a record check.", + "approval_missing": "A write without a record check has no approval.", + "popup_handling_not_approved": "Automatic pop-up handling isn't approved.", + "not_encrypted": "The workflow files must be encrypted for this setup.", + "integrity_check_failed": "The workflow files changed or their integrity check failed.", + "workflow_unloadable": "The workflow files couldn't be loaded safely.", + "app_target_missing": "The run setup doesn't say which app or window to use.", + "dispatch_mismatch": "The hosted request doesn't match this exact run.", + "url_override_not_allowed": "This server doesn't let callers change the app address.", + "other": "Another setup check refused the run. The operator can see which one.", +} + +# Flow prints one ``[REFUSE] : ...`` line per failing admission gate. +_GATE_TITLE_CHECKS = ( + ("Execution profile", "profile_not_met"), + ("Certification passed", "not_certified"), + ("Identity coverage", "right_record_check_missing"), + ("Effect coverage", "record_check_missing"), + ("Approval fallback", "approval_missing"), + ("Interstitial admission", "popup_handling_not_approved"), + ("Encrypted bundle", "not_encrypted"), + ("Sealed manifest", "integrity_check_failed"), +) +# Refusals Flow prints before its gate report. Order matters: the first +# marker found wins, so specific phrases come before general ones. +_PREGATE_CHECKS = ( + ("signed qualification admission", "no_readiness_test"), + ("qualification authority is invalid", "readiness_test_invalid"), + ("qualification campaign permit", "readiness_test_invalid"), + ("bundle could not be loaded safely", "workflow_unloadable"), + ("explicit execution surface", "app_target_missing"), + ("execution surface", "app_target_missing"), + ("cannot weaken a named profile", "not_encrypted"), + ("requires encrypted bundles", "not_encrypted"), + ("managed dispatch", "dispatch_mismatch"), + ("profile", "profile_not_met"), +) +_REFUSE_LINE = re.compile(r"^\s*\[REFUSE\]\s+([^:]+):", re.MULTILINE) + + +def parse_failed_checks(text: str) -> list[str]: + """Map Flow's refusal output to closed check codes. Never returns text.""" + found: list[str] = [] + if not isinstance(text, str): + return ["other"] + for title in _REFUSE_LINE.findall(text): + for prefix, code in _GATE_TITLE_CHECKS: + if title.strip().startswith(prefix) and code not in found: + found.append(code) + if not found: + lowered = text.lower() + for marker, code in _PREGATE_CHECKS: + if marker.lower() in lowered: + found.append(code) + break + return found or ["other"] + + +def report_consistency(report: object) -> tuple[bool, Optional[str], Optional[str]]: + """Return ``(consistent, execution_outcome, transaction_outcome)``. + + A report is consistent only when its precise fields agree with each other + and with the legacy ``success`` flag. An inconsistent report can never + produce ``done`` or a "nothing was written" result. + """ + if not isinstance(report, dict): + return False, None, None + execution = report.get("execution_outcome") + transaction = report.get("transaction_outcome") + if execution is not None and ( + not isinstance(execution, str) or execution not in _EXECUTION_OUTCOMES + ): + return False, None, None + if transaction is not None and ( + not isinstance(transaction, str) or transaction not in TRANSACTION_OUTCOMES + ): + return False, execution, None + success = report.get("success") + if execution is not None and not isinstance(success, bool): + return False, execution, transaction + if execution == "VERIFIED" and success is not True: + return False, execution, transaction + if execution in {"HALTED", "FAILED", "ROLLED_BACK"} and success is True: + return False, execution, transaction + if transaction is not None and execution is not None: + if (transaction == "VERIFIED") != (execution == "VERIFIED"): + return False, execution, transaction + profile = report.get("execution_profile") + if profile is not None and profile not in {"demo", "standard", "regulated"}: + return False, execution, transaction + eligible = report.get("production_eligible") + if eligible is not None and not isinstance(eligible, bool): + return False, execution, transaction + if eligible is True and ( + execution != "VERIFIED" or profile not in {"standard", "regulated"} + ): + return False, execution, transaction + envelope = report.get("outcome_envelope") + if envelope is not None: + if not isinstance(envelope, dict) or envelope.get("outcome") != execution: + return False, execution, transaction + for report_key, envelope_key in ( + ("execution_profile", "profile"), + ("production_eligible", "production_eligible"), + ("execution_completed", "execution_completed"), + ("model_calls", "model_calls"), + ("external_network_calls", "external_network_calls"), + ): + if report_key in report and envelope.get(envelope_key) != report.get(report_key): + return False, execution, transaction + return True, execution, transaction + + +# Flow's PHI-safe Needs Attention categories -> needs_review reasons. +_ATTENTION_REASONS = { + "identity": "record_not_confirmed", + "disambiguation": "record_not_confirmed", + "human_required": "person_needed", + "effect_unverifiable": "record_check_not_set_up", + "placeholder_effect": "record_check_not_set_up", + "unmet_guard": "unexpected_screen", + "postcondition": "unexpected_screen", + "resolution": "unexpected_screen", +} +_NO_EFFECT_DID_NOT_RUN = { + "REJECTED_POLICY": "policy_refused", + "CANCELED": "canceled", + "FAILED_PLATFORM": "platform_error", +} +#: Reason for each non-VERIFIED transaction outcome when nothing else is known. +#: Proven-no-effect outcomes assume a pause may be open, so they wait for a +#: person rather than invite a retry; ``reason_for_run`` refines them. +TRANSACTION_REASONS = { + "RECONCILIATION_REQUIRED": "save_not_confirmed", + "COMPLETED_UNVERIFIED": "not_checked", + "ROLLED_BACK": "change_reversed", + **{outcome: "stopped_for_review" for outcome in _NO_EFFECT_OUTCOMES}, +} + + +def pause_state(attention: Optional[Mapping[str, Any]]) -> Optional[str]: + """Reduce a Flow Needs Attention projection to ``open``, ``rejected``, or None.""" + if not isinstance(attention, Mapping): + return None + if attention.get("status") == "rejected": + return "rejected" + if attention.get("durably_paused") is True: + return "open" + return None + + +def reason_for_run( + *, + exit_code: Optional[int], + report: object, + timed_out: bool = False, + process_started: Optional[bool] = True, + refused_before_start: bool = False, + attention: Optional[Mapping[str, Any]] = None, +) -> str: + """Pick the one closed reason code for a finished run attempt. + + ``refused_before_start`` is True when Flow's admission refused the run + (exit code 2) and wrote no report. ``process_started`` is False only when + Flow provably never started. ``attention`` is Flow's PHI-safe Needs + Attention projection for the run, when one exists. + """ + report_present = isinstance(report, dict) + if refused_before_start and not report_present: + return "not_ready_to_run" + if timed_out: + return "timed_out" + if not report_present: + return "platform_error" if process_started is False else "result_unreadable" + consistent, execution, tx = report_consistency(report) + if not consistent: + return "result_unreadable" + if tx is None: + # Older report shape without a transaction outcome. Only a precise, + # consistent VERIFIED can be done; anything else is unproven. + if execution == "VERIFIED" and exit_code in (0, None): + return "saved_and_checked" + if execution is None and report.get("success") is True and exit_code in (0, None): + return "not_checked" + if execution == "COMPLETED_UNVERIFIED": + return "not_checked" + return "result_unreadable" + if tx == "VERIFIED": + return "saved_and_checked" if exit_code in (0, None) else "result_unreadable" + if tx in _NO_EFFECT_OUTCOMES: + state = pause_state(attention) + if state == "rejected": + return "stopped_by_person" + category = attention.get("category") if isinstance(attention, Mapping) else None + if state == "open" or tx == "HALTED_BEFORE_EFFECT": + return _ATTENTION_REASONS.get(category or "", "stopped_for_review") + return _NO_EFFECT_DID_NOT_RUN[tx] + return TRANSACTION_REASONS.get(tx, "result_unreadable") + + +#: Flow ledger value recorded for each reason, so the idempotency ledger keeps +#: a terminal outcome for every attempt. Uncertain results stay uncertain. +_LEDGER_OUTCOMES = { + "saved_and_checked": "VERIFIED", + "save_not_confirmed": "RECONCILIATION_REQUIRED", + "not_checked": "COMPLETED_UNVERIFIED", + "change_reversed": "ROLLED_BACK", + "timed_out": "RECONCILIATION_REQUIRED", + "result_unreadable": "RECONCILIATION_REQUIRED", + "interrupted": "RECONCILIATION_REQUIRED", + "canceled": "CANCELED", + "platform_error": "FAILED_PLATFORM", + "app_unreachable": "FAILED_PLATFORM", + "not_ready_to_run": "REJECTED_POLICY", + "policy_refused": "REJECTED_POLICY", + "workflow_unavailable": "FAILED_PLATFORM", + "runs_disabled": "REJECTED_POLICY", + "stopped_by_person": "HALTED_BEFORE_EFFECT", +} + + +def ledger_outcome(reason: str, transaction_outcome: Optional[str] = None) -> Optional[str]: + """Return the Flow ``TransactionOutcome`` value to record for a reason.""" + if transaction_outcome in TRANSACTION_OUTCOMES: + return transaction_outcome + spec = REASONS.get(reason) + if spec is not None and spec.outcome == "needs_review": + return "HALTED_BEFORE_EFFECT" + return _LEDGER_OUTCOMES.get(reason) + + +_SANDBOX_NOTE = " This was a sandbox run on a synthetic app; no real record was touched." +_SIMULATED_PREFIX = "Simulated sandbox result (no app was opened): " + + +def build_result( + reason: str, + *, + workflow: Optional[str], + run_id: Optional[str], + request_id: Optional[str], + mode: str, + proof: Optional[str] = None, + execution_outcome: Optional[str] = None, + transaction_outcome: Optional[str] = None, + failed_checks: Iterable[str] = (), + invalid_inputs: Iterable[Mapping[str, str]] = (), + needs_attention_id: Optional[str] = None, + first_run_id: Optional[str] = None, + replayed: bool = False, + seconds: Optional[float] = None, + model_calls: Optional[int] = None, + sandbox: Optional[Mapping[str, str]] = None, +) -> dict[str, Any]: + """Render one PHI-safe result object from a closed reason code.""" + spec = REASONS.get(reason) + if spec is None: + reason = "result_unreadable" + spec = REASONS[reason] + simulated = bool(sandbox) and sandbox.get("engine") == "simulated" + if spec.outcome == "done": + if simulated: + proof = "simulated" + elif proof not in {"sealed", "local"}: + proof = "local" + else: + proof = "none" + what_happened = spec.what_happened + if spec.outcome == "done" and proof == "local": + what_happened += " The check ran on this computer, so there's no signed receipt." + if simulated and spec.outcome != RUNNING: + what_happened = _SIMULATED_PREFIX + what_happened + elif mode == "sandbox" and spec.outcome != RUNNING: + what_happened += _SANDBOX_NOTE + result: dict[str, Any] = { + "outcome": spec.outcome, + "label": spec.label or LABELS[spec.outcome], + "safe_to_retry": spec.safe_to_retry, + "what_happened": what_happened, + "next_action": spec.next_action, + "next_step": spec.next_step, + "reason": reason, + "record_changed": spec.record_changed, + "proof": proof, + "run_id": run_id, + "request_id": request_id, + "workflow": workflow, + "mode": mode, + "contract_version": CONTRACT_VERSION, + } + checks = [code for code in failed_checks if code in FAILED_CHECKS] + if reason == "not_ready_to_run": + result["failed_checks"] = checks or ["other"] + if reason == "invalid_input": + result["invalid_inputs"] = [ + {"input": str(item["input"]), "problem": str(item["problem"])} + for item in invalid_inputs + if isinstance(item, Mapping) and "input" in item and "problem" in item + ] + if needs_attention_id and spec.outcome in {"needs_review", "not_sure_if_saved"}: + result["needs_attention_id"] = needs_attention_id + if first_run_id and reason == "request_id_conflict": + result["first_run_id"] = first_run_id + if replayed: + result["replayed"] = True + if isinstance(seconds, (int, float)) and not isinstance(seconds, bool) and seconds >= 0: + result["seconds"] = round(float(seconds), 1) + if isinstance(model_calls, int) and not isinstance(model_calls, bool) and model_calls >= 0: + result["model_calls"] = model_calls + if sandbox: + result["sandbox"] = { + key: str(value) for key, value in sandbox.items() if key in {"case", "engine"} + } + technical: dict[str, Any] = {} + if execution_outcome in _EXECUTION_OUTCOMES: + technical["execution_outcome"] = execution_outcome + if transaction_outcome in TRANSACTION_OUTCOMES: + technical["transaction_outcome"] = transaction_outcome + if technical: + result["technical"] = technical + return result + + +_OUTCOME_ENUM = [*OUTCOMES, RUNNING] + +#: MCP ``outputSchema`` for ``run_workflow`` and ``get_run``. +RUN_RESULT_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "outcome": { + "type": "string", + "enum": _OUTCOME_ENUM, + "description": ( + "done: saved and checked. needs_review: stopped before saving; a " + "person decides. not_sure_if_saved: a person must check the record " + "before anything runs again; never retry. did_not_run: nothing was " + "written. running: call get_run later." + ), + }, + "label": {"type": "string"}, + "safe_to_retry": { + "type": "boolean", + "description": ( + "True only when nothing was written. Reuse the same request_id " + "when you retry." + ), + }, + "what_happened": {"type": "string"}, + "next_action": {"type": "string"}, + "next_step": {"type": "string", "enum": list(NEXT_STEPS)}, + "reason": {"type": "string", "enum": sorted(REASONS)}, + "record_changed": {"type": "string", "enum": list(RECORD_CHANGED)}, + "proof": {"type": "string", "enum": list(PROOF)}, + "run_id": {"type": ["string", "null"]}, + "request_id": {"type": ["string", "null"]}, + "workflow": {"type": ["string", "null"]}, + "mode": {"type": "string", "enum": list(MODES)}, + "contract_version": {"type": "string"}, + "failed_checks": { + "type": "array", + "items": {"type": "string", "enum": sorted(FAILED_CHECKS)}, + }, + "invalid_inputs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "input": {"type": "string"}, + "problem": {"type": "string"}, + }, + "required": ["input", "problem"], + }, + }, + "needs_attention_id": {"type": "string"}, + "first_run_id": {"type": "string"}, + "replayed": {"type": "boolean"}, + "receipt_url": {"type": "string"}, + "review_url": {"type": "string"}, + "seconds": {"type": "number"}, + "model_calls": {"type": "integer"}, + "sandbox": { + "type": "object", + "properties": { + "case": {"type": "string"}, + "engine": {"type": "string", "enum": ["flow", "simulated"]}, + }, + }, + "technical": {"type": "object"}, + }, + "required": [ + "outcome", + "label", + "safe_to_retry", + "what_happened", + "next_action", + "next_step", + "reason", + "record_changed", + "proof", + "run_id", + "request_id", + "workflow", + "mode", + "contract_version", + ], +} + +#: MCP ``outputSchema`` for ``list_workflows``. +WORKFLOW_LIST_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "mode": {"type": "string", "enum": list(MODES)}, + "run_tools_enabled": {"type": "boolean"}, + "how_to_run": {"type": "string"}, + "workflows": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "purpose": {"type": "string"}, + "done_means": {"type": "string"}, + "inputs": {"type": "object"}, + "available": {"type": "boolean"}, + "changes_records": {"type": "boolean"}, + }, + "required": ["name", "purpose", "done_means", "inputs", "available"], + }, + }, + }, + "required": ["mode", "run_tools_enabled", "workflows"], +} diff --git a/src/openadapt_agent/copy.py b/src/openadapt_agent/copy.py index 26f803c..df57aa8 100644 --- a/src/openadapt_agent/copy.py +++ b/src/openadapt_agent/copy.py @@ -1,35 +1,40 @@ """Canonical public sentences this package repeats. -README, server.json, llms.txt, and skill frontmatter share -``IDENTITY_SENTENCE``. The skill body adds ``SKILL_WHEN_TO_USE``. +README, server.json, llms.txt, and the first-party skill frontmatter share +``IDENTITY_SENTENCE`` (100 characters or fewer, the MCP registry limit). Skill +bodies add ``SKILL_WHEN_TO_USE`` and ``OUTCOME_RULES``. """ from __future__ import annotations IDENTITY_SENTENCE = ( - "OpenAdapt compiles demonstrated GUI workflows into programs. " - "This package invokes them over MCP." + "Your AI agent decides what to enter. OpenAdapt enters it in the app and checks that it saved." ) SKILL_WHEN_TO_USE = ( - "When the user needs a repeating GUI write with no API and must prove " - "persistence, call run_<slug>. If the tool returns HALTED, tell the user " - "the record did not change." + "Use this when your agent has decided what to enter and the app has no usable " + "API. Call list_workflows, then run_workflow with the workflow name, its inputs, " + "and your own request_id." ) -SKILL_HONESTY = ( - "Never summarize halt, refused, timeout, or error as success. " - "A local unsigned replay may complete. If the tool returns unsigned " - "success, treat it as failure. Production success without a Seal is " - "failure." +OUTCOME_RULES = ( + "Only outcome done means the change was saved and checked. needs_review means " + "it stopped before saving and a person decides, so don't start the same work " + "again. not_sure_if_saved means a person must check the record, so never retry " + "it. did_not_run means nothing was written: fix the problem and retry with the " + "same request_id when safe_to_retry is true." ) - -REQUIRES_SEAL_META = {"requires_seal": True} +#: Older name for ``OUTCOME_RULES``. +SKILL_HONESTY = OUTCOME_RULES SKILL_NAME = "openadapt-gui-write" -THREE_LINE_INSTALL = ( - "claude mcp add openadapt -- \\\n" - " uvx --from 'openadapt-agent[tutorial]' openadapt-agent \\\n" - " serve --allow-run" +#: The first command once this version is on PyPI: the zero-flag sandbox. +FIRST_COMMAND = "claude mcp add openadapt -- uvx --python 3.12 openadapt-agent serve" +#: The same command from the GitHub source, for use before the release. +PREVIEW_COMMAND = ( + "claude mcp add openadapt -- uvx --python 3.12 --from " + "git+https://github.com/OpenAdaptAI/openadapt-agent openadapt-agent serve" ) +#: Older name for ``FIRST_COMMAND``. +THREE_LINE_INSTALL = FIRST_COMMAND diff --git a/src/openadapt_agent/mcp.py b/src/openadapt_agent/mcp.py index 59cae7b..1929d26 100644 --- a/src/openadapt_agent/mcp.py +++ b/src/openadapt_agent/mcp.py @@ -14,11 +14,16 @@ from __future__ import annotations +import io import json import logging -from typing import Any +import os +import sys +from contextlib import contextmanager +from typing import Any, Iterator import anyio +import mcp.server.stdio as _mcp_stdio import mcp.types as types from jsonschema import ValidationError, validate from mcp.server.lowlevel import Server @@ -98,27 +103,43 @@ async def _confirm_attended_action(context: Any, name: str) -> None: ) -def _server_instructions(authoring: AuthoringBridge | None) -> str: - text = ( - "Local bridge exposing compiled openadapt-flow workflow bundles " - "and PHI-safe Needs Attention items as tools. run_* tools execute via the governed " - "`openadapt-flow run` CLI and return a structured outcome: only " - "status 'success' means the workflow completed and verified. " - "'halt' means the run stopped and protected evidence remains in " - "the local operator experience; get_run_report returns only a " - "PHI-safe status/count summary by default. 'refused' means an " - "admission gate refused the bundle and nothing executed. " - "Continue/Skip require a human action " - "plus protocol-native operator elicitation, an exact signed " - "capability, live revalidation, and a stable idempotency key; they " - "never re-actuate the human-completed step. " - "Reject terminates the run and dispatches no new action, but earlier " - "run effects still require review of the protected local outcome. " - "Never report a halted, refused, timed-out, or error run as a success. " - "If execution_outcome is HALTED, tell the user the record did not change. " - "Write tools advertise requires_seal: true. If a write tool returns " - "unsigned success, treat it as failure." - ) +def _server_instructions( + authoring: AuthoringBridge | None, bridge: AgentBridge | None = None +) -> str: + text = "" + if bridge is not None: + text = ( + "OpenAdapt enters information into apps on this computer and checks " + "that it saved. Call list_workflows to see what it can do. Then call " + "run_workflow with a workflow name, its inputs, and your own " + "request_id for this piece of work. Act on the outcome: done means " + "saved and checked; needs_review means it stopped before saving and " + "a person decides, so don't start the same work again; " + "not_sure_if_saved means a person must check the record, so never " + "retry it; did_not_run means nothing was written, so fix the problem " + "and retry only when safe_to_retry is true. Reuse the same request_id " + "when you retry: the same id never writes twice. If the outcome is " + "running, call get_run with the run_id. Tell the user what_happened " + "in your own words and never call anything but done a success." + ) + if bridge.mode == "sandbox": + text += ( + " This server is a sandbox: it runs against a synthetic app, so " + "no real record changes. Use the sandbox_case input to see each " + "outcome." + ) + if bridge.legacy_tools: + text += ( + " Older tools stay available for existing clients: get_workflow, " + "get_run_report, list_needs_attention, get_attention_item, and " + "per-workflow run_<id> tools (deprecated; they take no " + "request_id). Attended Continue/Skip require a human action plus " + "protocol-native operator elicitation, an exact signed " + "capability, live revalidation, and a stable idempotency key; " + "they never re-actuate the human-completed step. Reject ends the " + "run and dispatches no new action, but earlier run effects still " + "require review of the protected local outcome." + ) if authoring is not None: text += ( " --authoring adds first-demo tools observe, start_record, click, " @@ -130,7 +151,7 @@ def _server_instructions(authoring: AuthoringBridge | None) -> str: "enable run tools. This process must not be port-forwarded or " "served over HTTP." ) - return text + return text.strip() def build_server( @@ -158,6 +179,7 @@ async def _list_tools() -> list[types.Tool]: if spec.annotations is not None else None ), + **({"outputSchema": spec.output_schema} if spec.output_schema else {}), **({"_meta": spec.meta} if spec.meta is not None else {}), ) for spec in _tool_specs() @@ -176,7 +198,10 @@ async def _call_tool( try: validate(arguments or {}, spec.input_schema) except ValidationError: - raise BridgeError("tool arguments do not match the input schema") from None + raise BridgeError( + getattr(spec, "usage", None) + or "tool arguments do not match the input schema" + ) from None if name in ATTENDED_TOOLS: await _confirm_attended_action(context, name) @@ -222,12 +247,13 @@ def call() -> dict[str, Any]: ) # Both SDKs accept wire aliases in constructors. Always return the # explicit result type: MCP 2 no longer wraps a bare content list. - return types.CallToolResult( - content=[types.TextContent(type="text", text=json.dumps(result, indent=2))] - ) + text = [types.TextContent(type="text", text=json.dumps(result, indent=2))] + if getattr(spec, "output_schema", None): + return types.CallToolResult(content=text, structuredContent=result) + return types.CallToolResult(content=text) if hasattr(Server, "list_tools"): - server = Server(SERVER_NAME, instructions=_server_instructions(authoring)) + server = Server(SERVER_NAME, instructions=_server_instructions(authoring, bridge)) server.list_tools()(_list_tools) async def _call_v1(name: str, arguments: dict[str, Any] | None): @@ -245,20 +271,50 @@ async def _call_v2(context: Any, params: types.CallToolRequestParams): server = Server( SERVER_NAME, - instructions=_server_instructions(authoring), + instructions=_server_instructions(authoring, bridge), on_list_tools=_list_v2, on_call_tool=_call_v2, ) return server +@contextmanager +def _protected_stdout() -> Iterator[Any]: + """Keep stray writes off the MCP wire on SDKs that don't divert fd 1. + + Background runs call Flow and browser tooling, which may print. MCP 2's + stdio transport already points fd 1 at stderr while it serves; MCP 1's + does not, so this does it here and hands the transport a private copy. + """ + if hasattr(_mcp_stdio, "_claim_fd"): + yield None + return + wire_fd = os.dup(1) + saved_fd = os.dup(1) + os.dup2(2, 1) + wire = os.fdopen(wire_fd, "wb") + try: + yield anyio.wrap_file(io.TextIOWrapper(wire, encoding="utf-8")) + finally: + try: + sys.stdout.flush() + except (OSError, ValueError): + pass + os.dup2(saved_fd, 1) + os.close(saved_fd) + + async def _run_stdio( bridge: AgentBridge | None, authoring: AuthoringBridge | None = None, ) -> None: server = build_server(bridge, authoring=authoring) - async with stdio_server() as (read_stream, write_stream): - await server.run(read_stream, write_stream, server.create_initialization_options()) + with _protected_stdout() as wire: + kwargs = {"stdout": wire} if wire is not None else {} + async with stdio_server(**kwargs) as (read_stream, write_stream): + await server.run( + read_stream, write_stream, server.create_initialization_options() + ) def serve( diff --git a/src/openadapt_agent/runner.py b/src/openadapt_agent/runner.py index d46461a..9f9f15d 100644 --- a/src/openadapt_agent/runner.py +++ b/src/openadapt_agent/runner.py @@ -1,10 +1,11 @@ """Shell out to the governed ``openadapt-flow run`` CLI and map the outcome. -This module is the only new-run path in openadapt-agent. It NEVER -reimplements replay: every ``run_*`` call is a subprocess invocation of -``openadapt-flow run`` (the fail-closed deployment verb), so flow's -admission gates (certification, identity arming, effect contracts, -encryption, integrity pinning) apply exactly as they would from a terminal. +This module is the run path for operator bundles. It NEVER reimplements +replay: every bundle run is a subprocess invocation of ``openadapt-flow run`` +(the fail-closed deployment verb), so Flow's admission gates (certification, +identity arming, effect contracts, encryption, integrity pinning) apply +exactly as they would from a terminal. The synthetic sandbox calls the same +Flow run gate in-process (see :mod:`openadapt_agent.sandbox`). Exit-code contract of ``openadapt-flow run``: @@ -16,7 +17,9 @@ - ``2`` — **governed refusal**: an admission gate refused the bundle (or the bundle could not be loaded safely). Nothing was executed. -A halt or refusal is always surfaced as such — never as success. Even an +Results are derived by :mod:`openadapt_agent.contract` from Flow's +``transaction_outcome``. A halt or refusal is never success, a verified run +is never an error, and an uncertain save is never "nothing changed". Even an exit code of 0 is cross-checked against the persisted ``report.json``. """ @@ -32,7 +35,15 @@ import uuid from dataclasses import dataclass, field from pathlib import Path -from typing import Optional +from typing import Any, Mapping, Optional + +from openadapt_agent.contract import ( + REASONS, + TRANSACTION_REASONS, + build_result, + parse_failed_checks, + reason_for_run, +) __all__ = [ "FlowRunner", @@ -42,8 +53,11 @@ "classify_report_status", "default_flow_cli", "is_safe_run_id", + "legacy_reason", + "new_run_id", "public_outcome_message", "public_report_summary", + "status_for_reason", ] _PRECISE_EXECUTION_OUTCOMES = frozenset( @@ -59,47 +73,56 @@ _RUN_ID_RE = re.compile(r"^[A-Za-z0-9._-]+$") _TAIL_CHARS = 4000 _LOG = logging.getLogger(__name__) -_PUBLIC_MESSAGES = { - "success": ("The governed run completed and its persisted report confirms success."), - "halt": ( - "The governed run stopped safely instead of guessing. Review the local " - "Needs Attention experience for protected evidence." - ), - "refused": ("A governed admission check refused the run. The target workflow did not start."), - "timeout": ( - "The run exceeded its deadline. Its live effect is uncertain; inspect " - "the protected local run before retrying." - ), - "error": ( - "The run could not produce a trustworthy terminal result. Inspect the " - "protected local logs and run evidence." - ), +# Legacy ``status`` values a caller may still hold, mapped to the contract +# reason that makes no claim beyond what the status itself proves. +_STATUS_FALLBACK_REASONS = { + "success": "saved_and_checked", + "halt": "result_unreadable", + "refused": "not_ready_to_run", + "timeout": "timed_out", + "error": "result_unreadable", } +def legacy_reason( + status: str, + execution_outcome: Optional[str] = None, + transaction_outcome: Optional[str] = None, +) -> str: + """Contract reason for a legacy status when no fuller evidence is at hand. + + The coarse ``HALTED`` label never selects "nothing was written": without a + transaction outcome a halt is reported as uncertain. + """ + if status == "halt": + if transaction_outcome in TRANSACTION_REASONS: + return TRANSACTION_REASONS[transaction_outcome] + if execution_outcome == "COMPLETED_UNVERIFIED": + return "not_checked" + if execution_outcome == "ROLLED_BACK": + return "change_reversed" + return _STATUS_FALLBACK_REASONS.get(status, "result_unreadable") + + +def public_outcome_message( + status: str, + execution_outcome: Optional[str], + transaction_outcome: Optional[str] = None, +) -> str: + """Return fixed public copy for one classified terminal result.""" + reason = legacy_reason(status, execution_outcome, transaction_outcome) + return REASONS[reason].what_happened -def public_outcome_message(status: str, execution_outcome: Optional[str]) -> str: - """Return fixed public copy for one classified terminal result.""" - if status == "error" and execution_outcome == "VERIFIED": - return ( - "Unsigned production success is failure. A Seal is required " - "before this write can be treated as complete." - ) - if status == "halt" and execution_outcome == "HALTED": - return ( - "HALTED. The independent check did not confirm the write. " - "Tell the user the record did not change." - ) - if status == "halt" and execution_outcome == "COMPLETED_UNVERIFIED": - return ( - "The run completed, but its persisted evidence did not prove " - "verified success. Review it locally before any retry." - ) - if status == "halt" and execution_outcome == "ROLLED_BACK": - return ( - "The configured compensating action completed. Review the local " - "evidence before further action." - ) - return _PUBLIC_MESSAGES.get(status, _PUBLIC_MESSAGES["error"]) +def status_for_reason(reason: str, *, refused: bool = False, timed_out: bool = False) -> str: + """Legacy ``status`` that agrees with a contract reason.""" + if refused: + return "refused" + if timed_out: + return "timeout" + if reason == "saved_and_checked": + return "success" + if reason in {"result_unreadable", "platform_error", "app_unreachable"}: + return "error" + return "halt" def default_flow_cli() -> tuple[str, ...]: @@ -144,23 +167,107 @@ class RunOutcome: stdout_tail: str = "" stderr_tail: str = "" execution_outcome: Optional[str] = None + transaction_outcome: Optional[str] = None + #: Closed contract reason (``openadapt_agent.contract.REASONS``). + reason: Optional[str] = None + #: Closed refusal check codes, never Flow's own text. + failed_checks: list[str] = field(default_factory=list) + #: The persisted report, kept so a later attention lookup can refine the + #: reason. Never exported unless protected export is enabled. + report: Optional[dict] = field(default=None, repr=False) + refused_before_start: bool = False + timed_out: bool = False + process_started: Optional[bool] = True + + def contract_reason(self) -> str: + """The contract reason for this outcome, falling back to the status.""" + if self.reason in REASONS: + return self.reason + return legacy_reason(self.status, self.execution_outcome, self.transaction_outcome) + + def apply_attention(self, attention: Optional[Mapping[str, Any]]) -> None: + """Refine the reason with Flow's Needs Attention state for this run.""" + if attention is None or self.refused_before_start or self.timed_out: + return + if not isinstance(self.report, dict): + return + self.reason = reason_for_run( + exit_code=self.exit_code, + report=self.report, + process_started=self.process_started, + attention=attention, + ) + + def contract( + self, + *, + mode: str = "production", + request_id: Optional[str] = None, + workflow: Optional[str] = None, + needs_attention_id: Optional[str] = None, + ) -> dict: + """Project the partner contract result for this outcome.""" + model_calls = self.summary.get("model_calls") if isinstance(self.summary, dict) else None + total_ms = self.summary.get("total_ms") if isinstance(self.summary, dict) else None + return build_result( + self.contract_reason(), + workflow=workflow if workflow is not None else self.workflow, + run_id=self.run_id, + request_id=request_id, + mode=mode, + execution_outcome=self.execution_outcome, + transaction_outcome=self.transaction_outcome, + failed_checks=self.failed_checks, + needs_attention_id=needs_attention_id, + model_calls=model_calls if isinstance(model_calls, int) else None, + seconds=( + total_ms / 1000.0 + if isinstance(total_ms, (int, float)) and not isinstance(total_ms, bool) + else None + ), + ) def to_dict(self, *, include_protected: bool = False) -> dict: - """Project an MCP-safe result; raw local evidence is explicit opt-in.""" + """Project an MCP-safe legacy result; raw local evidence is explicit opt-in. + + The legacy keys stay for existing callers. The partner contract fields + (``outcome``, ``safe_to_retry``, ``what_happened``, ...) are added + alongside them, and ``message`` is the contract's plain sentence. + """ + contract = self.contract() result = { "schema_version": 1, "status": self.status, "success": self.status == "success", "sealed": False, - "requires_seal": True, + # Kept for older callers. A verified local run is reported as done + # with ``proof: "local"``; it is never turned into a failure. + "requires_seal": False, "frames_included": False, "workflow_id": self.workflow, "run_id": self.run_id, - "message": public_outcome_message(self.status, self.execution_outcome), + "message": contract["what_happened"], "summary": _sanitize_public_summary(self.summary), } + for key in ( + "outcome", + "label", + "safe_to_retry", + "what_happened", + "next_action", + "next_step", + "reason", + "record_changed", + "proof", + "contract_version", + "failed_checks", + ): + if key in contract: + result[key] = contract[key] if self.execution_outcome is not None: result["execution_outcome"] = self.execution_outcome + if self.transaction_outcome is not None: + result["transaction_outcome"] = self.transaction_outcome if include_protected: result["protected"] = { "workflow": self.workflow, @@ -306,11 +413,9 @@ def classify_report_status(report: object) -> tuple[str, Optional[str]]: return "error", precise if precise == "VERIFIED": - if production_eligible is True: - # Local MCP never mints a Seal. Production-eligible VERIFIED - # without one is unsigned success, which this adapter treats - # as failure. - return "error", precise + # A production-eligible VERIFIED run saved the change and read it back. + # Local MCP mints no Seal, so the proof is local; the run is still a + # success. Reporting it as an error would invite a duplicate write. return ("success" if success is True else "error"), precise if precise in {"HALTED", "FAILED", "ROLLED_BACK"} and success is True: return "error", precise @@ -332,12 +437,11 @@ def classify_outcome( ) -> RunOutcome: """Map a finished ``openadapt-flow run`` process to a :class:`RunOutcome`. - Pure function (no I/O) so the mapping is unit-testable. The invariant: - ``status == "success"`` requires BOTH exit code 0 AND a persisted - report that has a verified terminal outcome. Legacy reports require a true - ``success`` flag. Precise reports require ``execution_outcome=VERIFIED`` - and a consistent true legacy flag. Anything else surfaces as a halt, - refusal, or error while protected evidence remains local. + Pure function (no I/O) so the mapping is unit-testable. The contract + reason comes from :func:`openadapt_agent.contract.reason_for_run`, which + reads Flow's ``transaction_outcome``; the legacy ``status`` is derived from + that reason so the two never disagree. ``status == "success"`` requires + exit code 0 AND a consistent persisted ``VERIFIED`` report. """ outcome = RunOutcome( status="error", @@ -348,9 +452,24 @@ def classify_outcome( exit_code=exit_code, stdout_tail=_tail(stdout), stderr_tail=_tail(stderr), + report=report if isinstance(report, dict) else None, + ) + if isinstance(report, dict): + _status, outcome.execution_outcome = classify_report_status(report) + transaction = report.get("transaction_outcome") + if isinstance(transaction, str): + outcome.transaction_outcome = transaction + + refused = exit_code == 2 and not isinstance(report, dict) + outcome.refused_before_start = refused + outcome.reason = reason_for_run( + exit_code=exit_code, + report=report if isinstance(report, dict) else None, + refused_before_start=refused, ) - if exit_code == 2: + outcome.failed_checks = parse_failed_checks(f"{stdout}\n{stderr}") + if refused: outcome.status = "refused" outcome.detail = ( "Governed refusal: an openadapt-flow admission gate refused this " @@ -360,78 +479,45 @@ def classify_outcome( ) return outcome - if exit_code == 0: - if report is None: - outcome.status = "error" - outcome.detail = ( - "openadapt-flow run exited 0 but no report.json was found in " - "the run directory; refusing to report success without " - "evidence." - ) - return outcome - report_status, precise_outcome = classify_report_status(report) - outcome.execution_outcome = precise_outcome - if report_status == "success": - outcome.status = "success" - outcome.summary = _report_summary(report) - outcome.detail = "Run completed; every executed step verified." - return outcome - # Exit 0 with a non-success report: trust the report, not the code. A - # Demo completion can carry the legacy success flag while its precise - # evidence outcome is explicitly COMPLETED_UNVERIFIED. - outcome.status = report_status - outcome.summary = _report_summary(report) - outcome.halt = report.get("halt") if isinstance(report, dict) else None - if precise_outcome == "COMPLETED_UNVERIFIED": - outcome.detail = ( - "Execution completed, but the persisted evidence did not prove " - "VERIFIED success; review the local run before any retry." - ) - elif report_status == "error": - outcome.detail = ( - "Process exited 0, but the persisted report has no consistent " - "verified terminal outcome." - ) - else: - outcome.detail = ( - "Process exited 0 but the persisted run report does not mark the " - "run as VERIFIED; treating it as a halt." - ) - return outcome - - # Any other nonzero exit (canonically 1): the run executed and stopped. - if report is not None: - report_status, precise_outcome = classify_report_status(report) - outcome.execution_outcome = precise_outcome - outcome.status = "error" if report_status in {"success", "error"} else "halt" - else: - outcome.status = "halt" + outcome.status = status_for_reason(outcome.reason) if isinstance(report, dict): outcome.summary = _report_summary(report) - outcome.halt = report.get("halt") - failing = _failing_step(report) - if outcome.halt: - outcome.detail = ( - f"Run halted at state {outcome.halt.get('state_id')!r} " - f"({outcome.halt.get('intent')!r}): " - f"{outcome.halt.get('reason')!r}. Evidence: report.json / " - "REPORT.md in run_dir." - ) - elif failing is not None: - outcome.detail = ( - f"Run failed at step {failing.get('step_id')!r} " - f"({failing.get('intent')!r}): {failing.get('error')!r}. " - "Evidence: report.json / REPORT.md in run_dir." - ) - outcome.halt = failing - else: - outcome.detail = ( - "Run did not complete successfully; see report.json in " - "run_dir for step-level evidence." - ) + outcome.halt = report.get("halt") if outcome.status != "success" else None + if outcome.status == "success": + outcome.detail = "Run completed; every executed step verified." + return outcome + if report is None: + outcome.detail = ( + "openadapt-flow run exited without a report.json in the run " + "directory; refusing to report success without evidence. See " + "stdout_tail / stderr_tail." + ) + return outcome + if outcome.execution_outcome == "COMPLETED_UNVERIFIED": + outcome.detail = ( + "Execution completed, but the persisted evidence did not prove " + "VERIFIED success; review the local run before any retry." + ) + return outcome + failing = _failing_step(report) if isinstance(report, dict) else None + if outcome.halt: + outcome.detail = ( + f"Run halted at state {outcome.halt.get('state_id')!r} " + f"({outcome.halt.get('intent')!r}): " + f"{outcome.halt.get('reason')!r}. Evidence: report.json / " + "REPORT.md in run_dir." + ) + elif failing is not None: + outcome.detail = ( + f"Run failed at step {failing.get('step_id')!r} " + f"({failing.get('intent')!r}): {failing.get('error')!r}. " + "Evidence: report.json / REPORT.md in run_dir." + ) + outcome.halt = failing else: outcome.detail = ( - "Run exited nonzero before a report.json was written; see stdout_tail / stderr_tail." + "The persisted report has no consistent verified terminal outcome; " + "see report.json in run_dir for step-level evidence." ) return outcome @@ -470,14 +556,26 @@ def run( bundle_dir: Path, params: dict[str, str], url_override: Optional[str] = None, + run_id: Optional[str] = None, ) -> RunOutcome: - """Run the bundle once. Params travel via ``--params-file`` (never argv).""" + """Run the bundle once. Params travel via ``--params-file`` (never argv). + + ``run_id`` lets a caller hand out the id before the run finishes. It + must be a single safe path component; a fresh one is made otherwise. + """ url = self.config.url + if run_id is None or not is_safe_run_id(run_id): + run_id = new_run_id() if url_override: if not self.config.allow_url_override: return RunOutcome( status="refused", workflow=workflow, + run_id=run_id, + reason="not_ready_to_run", + failed_checks=["url_override_not_allowed"], + refused_before_start=True, + process_started=False, detail=( "URL override rejected: the server was not started " "with --allow-url-override. The target URL is fixed " @@ -488,7 +586,6 @@ def run( runs_root = Path(self.config.runs_dir) runs_root.mkdir(parents=True, exist_ok=True) - run_id = f"run-{uuid.uuid4().hex[:24]}" run_dir = runs_root / run_id report_path = run_dir / "report.json" @@ -499,10 +596,14 @@ def run( json.dump({k: str(v) for k, v in (params or {}).items()}, fh) cmd = self._build_command(Path(bundle_dir), run_dir, params_file, url) try: + # Decode with replacement: a strict decode error would surface + # after Flow already ran and read as a launch failure. proc = subprocess.run( cmd, capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=self.config.timeout_s, ) except subprocess.TimeoutExpired as exc: @@ -512,36 +613,43 @@ def run( run_id=run_id, run_dir=str(run_dir), report_path=str(report_path) if report_path.exists() else None, + reason="timed_out", + timed_out=True, detail=( f"Run exceeded the per-call timeout of " f"{self.config.timeout_s:.0f}s and was killed. The " "target system may be in a partially-executed state; " "inspect the run directory before retrying." ), - stdout_tail=_tail( - exc.stdout.decode() if isinstance(exc.stdout, bytes) else (exc.stdout or "") - ), - stderr_tail=_tail( - exc.stderr.decode() if isinstance(exc.stderr, bytes) else (exc.stderr or "") - ), + stdout_tail=_tail(_text(exc.stdout)), + stderr_tail=_tail(_text(exc.stderr)), ) except FileNotFoundError: return RunOutcome( status="error", workflow=workflow, + run_id=run_id, + reason="platform_error", + process_started=False, detail=( f"openadapt-flow CLI not found ({self.config.flow_cli[0]!r}); " "install openadapt-flow in the server's environment." ), ) except (OSError, subprocess.SubprocessError, ValueError) as exc: + # subprocess.run raises these while creating the child, so + # the governed run normally never started. If Flow already + # made its run directory, it did start and may have acted. _LOG.exception("governed Flow subprocess failed locally") + started = run_dir.exists() return RunOutcome( status="error", workflow=workflow, run_id=run_id, run_dir=str(run_dir), report_path=(str(report_path) if report_path.exists() else None), + reason="result_unreadable" if started else "platform_error", + process_started=None if started else False, detail=f"{type(exc).__name__}: {exc}", ) finally: @@ -610,6 +718,23 @@ def certify(self, bundle_dir: Path) -> dict: } +def _text(value: object) -> str: + """Captured process output as text, never raising on odd bytes.""" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return value if isinstance(value, str) else "" + + def is_safe_run_id(run_id: str) -> bool: """Run ids are single path components — no separators or traversal.""" - return bool(_RUN_ID_RE.match(run_id)) and ".." not in run_id + return ( + isinstance(run_id, str) + and len(run_id) <= 128 + and bool(_RUN_ID_RE.match(run_id)) + and ".." not in run_id + ) + + +def new_run_id() -> str: + """A fresh opaque run id (``run-`` plus 24 hex characters).""" + return f"run-{uuid.uuid4().hex[:24]}" diff --git a/src/openadapt_agent/runs.py b/src/openadapt_agent/runs.py new file mode 100644 index 0000000..5a4ee15 --- /dev/null +++ b/src/openadapt_agent/runs.py @@ -0,0 +1,381 @@ +"""Durable run records and the at-most-once gate behind ``request_id``. + +Every ``run_workflow`` call leaves a record under +``<runs-dir>/openadapt-agent/``, refusals included, so ``get_run`` can always +answer. Records hold the PHI-safe contract result only. Inputs are never +stored; a keyed fingerprint of ``(workflow, inputs)`` is, so a reused +``request_id`` with different inputs is refused instead of silently replayed. + +The at-most-once authority is openadapt-flow's ``IdempotencyLedger``: each +attempt reserves ``<request>:<attempt>`` before anything runs, and a lost race +reads back the winner instead of acting. A request may start a new attempt +only when its latest attempt proved nothing was written (see +``contract.RETRYABLE_REASONS``). If Flow's ledger can't be opened, a local +exclusive-create claim file gives the same guarantee for this runs directory. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import logging +import os +import secrets +import socket +import tempfile +import threading +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +from openadapt_agent.contract import RETRYABLE_REASONS, ledger_outcome +from openadapt_agent.runner import is_safe_run_id, new_run_id + +__all__ = ["Begin", "MAX_ATTEMPTS", "RunStore", "pid_alive", "read_record"] + +_LOG = logging.getLogger(__name__) +_RECORD_SCHEMA = "openadapt-agent.run/v1" +_REQUEST_SCHEMA = "openadapt-agent.request/v1" +_LEDGER_NAMESPACE = "openadapt-agent/run-workflow/v1" +#: Attempts one request_id may make when earlier ones proved nothing was written. +MAX_ATTEMPTS = 5 +#: How long a reservation may sit without its run record before it is treated +#: as left behind by a process that died between the two writes. +ORPHAN_GRACE_S = 120.0 + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + """Atomic, owner-only write.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=".tmp-", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(payload, handle, sort_keys=True) + os.chmod(tmp, 0o600) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + + +def _read_json(path: Path) -> Optional[dict[str, Any]]: + if path.is_symlink() or not path.is_file(): + return None + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return value if isinstance(value, dict) else None + + +class _Duplicate(Exception): + """The ledger key was already reserved by another attempt.""" + + +class _ClaimFiles: + """Exclusive-create claim files: the fallback at-most-once gate.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.root.mkdir(parents=True, exist_ok=True) + + def _path(self, key: str) -> Path: + return self.root / (hashlib.sha256(key.encode("utf-8")).hexdigest() + ".claim") + + def reserve(self, key: str, *, run_id: str) -> None: + try: + fd = os.open(self._path(key), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError as exc: + raise _Duplicate(key) from exc + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump({"run_id": run_id, "reserved_at": _now()}, handle) + + def lookup(self, key: str) -> Optional[dict[str, Any]]: + return _read_json(self._path(key)) + + def record_outcome(self, key: str, outcome: str, *, run_id: str) -> None: + return None + + +class _FlowLedger: + """openadapt-flow's IdempotencyLedger behind the same small interface.""" + + def __init__(self, path: Path) -> None: + from openadapt_flow.transaction import DuplicateActuation, IdempotencyLedger + + self._duplicate = DuplicateActuation + self._ledger = IdempotencyLedger(path, namespace=_LEDGER_NAMESPACE) + + def reserve(self, key: str, *, run_id: str) -> None: + try: + self._ledger.reserve(key, run_id=run_id) + except self._duplicate as exc: + raise _Duplicate(key) from exc + + def lookup(self, key: str) -> Optional[dict[str, Any]]: + return self._ledger.lookup(key) + + def record_outcome(self, key: str, outcome: str, *, run_id: str) -> None: + self._ledger.record_outcome(key, outcome, run_id=run_id) + + +@dataclass +class Begin: + """What ``RunStore.begin`` decided for one request.""" + + kind: str # "new" | "replay" | "conflict" | "retry_limit" + run_id: str + record: Optional[dict[str, Any]] = None + first_run_id: Optional[str] = None + + +class RunStore: + """PHI-safe run records plus the request index and idempotency ledger.""" + + def __init__(self, runs_dir: Path | str, *, max_attempts: int = MAX_ATTEMPTS) -> None: + # Canonical path: Flow's ledger refuses symlinked components, and two + # servers must agree on one ledger for the same directory. + self.root = Path(runs_dir).expanduser().resolve() / "openadapt-agent" + self.runs = self.root / "runs" + self.requests = self.root / "requests" + for directory in (self.root, self.runs, self.requests): + directory.mkdir(parents=True, exist_ok=True) + try: + os.chmod(directory, 0o700) + except OSError: + pass + self.max_attempts = max_attempts + self._lock = threading.RLock() + self._key = self._fingerprint_key() + try: + self._ledger: Any = _FlowLedger(self.root / "ledger.sqlite") + self.ledger_kind = "openadapt-flow" + except Exception: # missing symbol, old Flow, unsupported filesystem + _LOG.warning("openadapt-flow idempotency ledger unavailable; using claim files") + self._ledger = _ClaimFiles(self.root / "claims") + self.ledger_kind = "claim-files" + + # -- records ----------------------------------------------------------- + + def _fingerprint_key(self) -> bytes: + path = self.root / "fingerprint.key" + existing = path.read_bytes() if path.is_file() and not path.is_symlink() else b"" + if len(existing) >= 32: + return existing + key = secrets.token_bytes(32) + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + return path.read_bytes() + with os.fdopen(fd, "wb") as handle: + handle.write(key) + return key + + def fingerprint(self, workflow: str, inputs: Any) -> str: + material = json.dumps({"workflow": workflow, "inputs": inputs}, sort_keys=True, default=str) + return hmac.new(self._key, material.encode("utf-8"), hashlib.sha256).hexdigest() + + def _request_hash(self, request_id: str) -> str: + return hashlib.sha256(request_id.encode("utf-8")).hexdigest()[:40] + + def read(self, run_id: str) -> Optional[dict[str, Any]]: + if not is_safe_run_id(run_id): + return None + record = _read_json(self.runs / f"{run_id}.json") + if record is None or record.get("schema") != _RECORD_SCHEMA: + return None + return record + + def write(self, record: dict[str, Any]) -> None: + run_id = record["run_id"] + if not is_safe_run_id(run_id): + raise ValueError("unsafe run id") + _write_json(self.runs / f"{run_id}.json", record) + + def new_record( + self, + *, + request_id: Optional[str], + workflow: Optional[str], + mode: str, + run_id: Optional[str] = None, + ) -> dict[str, Any]: + return { + "schema": _RECORD_SCHEMA, + "run_id": run_id or new_run_id(), + "request_id": request_id, + "workflow": workflow, + "mode": mode, + "state": "running", + "started_at": _now(), + "pid": os.getpid(), + "host": socket.gethostname(), + } + + def _orphan( + self, run_id: str, request_id: str, workflow: str, mode: str + ) -> dict[str, Any]: + """A running record with no owner process: get_run marks it interrupted.""" + record = self.new_record( + request_id=request_id, workflow=workflow, mode=mode, run_id=run_id + ) + record["pid"] = None + self.write(record) + return record + + def save_refusal(self, record: dict[str, Any], result: dict[str, Any]) -> dict[str, Any]: + """Persist a result that never reserved the request (bad input, conflict).""" + record = dict(record, state="finished", finished_at=_now(), result=result) + self.write(record) + return record + + def finish( + self, + record: dict[str, Any], + result: dict[str, Any], + *, + transaction_outcome: Optional[str] = None, + ) -> dict[str, Any]: + """Persist a terminal result and record it in the ledger.""" + record = dict(record, state="finished", finished_at=_now(), result=result) + self.write(record) + key = record.get("ledger_key") + outcome = ledger_outcome(str(result.get("reason")), transaction_outcome) + if key and outcome: + try: + self._ledger.record_outcome(key, outcome, run_id=record["run_id"]) + except Exception: + _LOG.warning("could not record a ledger outcome for %s", record["run_id"]) + return record + + def update_result(self, record: dict[str, Any], result: dict[str, Any]) -> dict[str, Any]: + """Replace a finished result after a person resolved its review.""" + record = dict(record, result=result, updated_at=_now()) + self.write(record) + return record + + # -- requests ---------------------------------------------------------- + + def begin( + self, *, request_id: str, workflow: str, inputs: Any, mode: str + ) -> Begin: + """Decide whether this request starts a new attempt or replays one.""" + fingerprint = self.fingerprint(workflow, inputs) + request_hash = self._request_hash(request_id) + entry_path = self.requests / f"{request_hash}.json" + with self._lock: + entry = _read_json(entry_path) or { + "schema": _REQUEST_SCHEMA, + "fingerprint": fingerprint, + "workflow": workflow, + "attempts": [], + } + attempts = [run for run in entry.get("attempts", []) if isinstance(run, str)] + if attempts and entry.get("fingerprint") != fingerprint: + return Begin("conflict", run_id=new_run_id(), first_run_id=attempts[0]) + if attempts: + latest = self.read(attempts[-1]) + if latest is None: + # The entry is written only after the record, so a missing + # or unreadable record was lost. Leave an orphan that + # get_run reports as interrupted, never as running forever. + latest = self._orphan(attempts[-1], request_id, workflow, mode) + reason = ((latest or {}).get("result") or {}).get("reason") + retryable = ( + latest is not None + and latest.get("state") == "finished" + and reason in RETRYABLE_REASONS + ) + if not retryable: + return Begin("replay", run_id=attempts[-1], record=latest) + if len(attempts) >= self.max_attempts: + return Begin("retry_limit", run_id=new_run_id(), first_run_id=attempts[0]) + attempt = len(attempts) + 1 + key = f"{request_hash}:{attempt}" + record = self.new_record(request_id=request_id, workflow=workflow, mode=mode) + record["attempt"] = attempt + record["ledger_key"] = key + record["fingerprint"] = fingerprint + try: + self._ledger.reserve(key, run_id=record["run_id"]) + except _Duplicate: + # Another process won this attempt. Read its result instead. + claim = self._ledger.lookup(key) or {} + owner = claim.get("run_id") + if not isinstance(owner, str) or not is_safe_run_id(owner): + raise + record = self.read(owner) + if record is not None and record.get("fingerprint") not in (None, fingerprint): + # Another process took this request_id for different inputs. + return Begin("conflict", run_id=new_run_id(), first_run_id=owner) + if record is None and _older_than(claim.get("reserved_at"), ORPHAN_GRACE_S): + # Reserved, then the process died before writing the record. + record = self._orphan(owner, request_id, workflow, mode) + return Begin("replay", run_id=owner, record=record) + self.write(record) + entry["attempts"] = [*attempts, record["run_id"]] + entry["fingerprint"] = fingerprint + _write_json(entry_path, entry) + return Begin("new", run_id=record["run_id"], record=record) + + +def _older_than(timestamp: Any, seconds: float) -> bool: + """Whether an ISO timestamp is more than ``seconds`` old. Unknown is old.""" + if not isinstance(timestamp, str): + return True + try: + then = datetime.fromisoformat(timestamp) + except ValueError: + return True + if then.tzinfo is None: + then = then.replace(tzinfo=timezone.utc) + return (datetime.now(timezone.utc) - then).total_seconds() > seconds + + +def read_record(runs_dir: Path | str, run_id: str) -> Optional[dict[str, Any]]: + """Read one run record without creating anything on disk.""" + if not is_safe_run_id(run_id): + return None + record = _read_json(Path(runs_dir) / "openadapt-agent" / "runs" / f"{run_id}.json") + if record is None or record.get("schema") != _RECORD_SCHEMA: + return None + return record + + +def pid_alive(pid: Any) -> bool: + """Whether a process with this id still exists on this computer.""" + if not isinstance(pid, int) or pid <= 0: + return False + if os.name == "nt": + # os.kill would terminate the process on Windows. Ask instead, and + # treat any doubt as "gone": a run then reads as interrupted, which + # asks a person to check the record and never invites a retry. + try: + import ctypes + + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] + handle = kernel32.OpenProcess(0x1000, False, pid) + if not handle: + return False + code = ctypes.c_ulong() + ok = kernel32.GetExitCodeProcess(handle, ctypes.byref(code)) + kernel32.CloseHandle(handle) + return bool(ok) and code.value == 259 + except Exception: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except (PermissionError, OSError): + return True + return True diff --git a/src/openadapt_agent/sandbox.py b/src/openadapt_agent/sandbox.py new file mode 100644 index 0000000..3cb0d08 --- /dev/null +++ b/src/openadapt_agent/sandbox.py @@ -0,0 +1,511 @@ +"""The zero-flag sandbox: one synthetic workflow that shows every outcome. + +``openadapt-agent serve`` with no flags serves one workflow, +``add_triage_note``, on MockMed, a synthetic clinic app that ships with +openadapt-flow. The caller picks a ``sandbox_case`` to see each outcome the +partner contract can return, the way a payments sandbox uses test card +numbers: + +===================== =================== ========================================= +sandbox_case outcome what the synthetic app does +===================== =================== ========================================= +normal done saves the note; the record check finds it +duplicate_record needs_review lists the same referral twice; it stops + before saving +false_saved_banner not_sure_if_saved shows "saved" but its record store + rejected the note +timeout_after_save not_sure_if_saved saves the note, then reports an error +app_offline did_not_run isn't reachable, so nothing starts +===================== =================== ========================================= + +Two engines produce these results: + +- ``flow``: openadapt-flow records the workflow once, then every run drives + MockMed in a hidden browser under the standard profile, with the same run + gate, right-record checks, and independent record-store read a production + run uses. Needs the ``tutorial`` extra (Playwright and Chromium). +- ``simulated``: the same contract results without opening any app, for a + plain install. Results say ``proof: "simulated"``. + +Results always say ``mode: "sandbox"`` and name the engine. Nothing here +touches a real record. +""" + +from __future__ import annotations + +import inspect +import json +import logging +import os +import socket +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Optional +from urllib.request import Request, urlopen + +from openadapt_agent.cards import WorkflowCard +from openadapt_agent.contract import parse_failed_checks, reason_for_run +from openadapt_agent.service import ( + CatalogEntry, + EngineResult, + attention_for, + reason_from_run_dir, +) + +__all__ = [ + "CASES", + "SANDBOX_WORKFLOW", + "SandboxEngine", + "flow_engine_available", +] + +_LOG = logging.getLogger(__name__) + +SANDBOX_WORKFLOW = "add_triage_note" +#: How long a first run waits for the synthetic app to be recorded and compiled. +PREPARE_TIMEOUT_S = 900.0 +_APPROVAL_SOURCE = "openadapt-agent-sandbox" +_POLICY = "clinical-write" + + +@dataclass(frozen=True) +class SandboxCase: + name: str + outcome: str + shows: str + #: MockMed entry query: fault mode and optional drift. None = app offline. + query: Optional[str] + #: Outcome fields a real openadapt-flow 1.35.1 run produced for this case. + execution: Optional[str] = None + transaction: Optional[str] = None + attention_category: Optional[str] = None + + +CASES: dict[str, SandboxCase] = { + case.name: case + for case in ( + SandboxCase( + "normal", + "done", + "The app saves the note and the record check finds it in the app's record store.", + "?fault=ok&idempotency=demo#tasks", + "VERIFIED", + "VERIFIED", + ), + SandboxCase( + "duplicate_record", + "needs_review", + "The queue lists the same referral twice, so OpenAdapt can't be sure " + "which record to open. It stops before saving.", + "?fault=ok&idempotency=demo&drift=ambiguous#tasks", + "HALTED", + "HALTED_BEFORE_EFFECT", + "disambiguation", + ), + SandboxCase( + "false_saved_banner", + "not_sure_if_saved", + "The app shows a saved message, but its record store rejected the note. " + "The record check doesn't find it.", + "?fault=optimistic&idempotency=demo#tasks", + "HALTED", + "RECONCILIATION_REQUIRED", + "effect_escalated", + ), + SandboxCase( + "timeout_after_save", + "not_sure_if_saved", + "The record store saves the note, then the app times out and shows an " + "error. Only a person checking the record can tell it saved.", + "?fault=timeout&idempotency=demo#tasks", + "HALTED", + "RECONCILIATION_REQUIRED", + "postcondition", + ), + SandboxCase( + "app_offline", + "did_not_run", + "The app isn't reachable, so OpenAdapt doesn't start.", + None, + ), + ) +} + + +def _card() -> WorkflowCard: + case_list = "; ".join(f"{case.name}: {case.outcome}" for case in CASES.values()) + return WorkflowCard( + name=SANDBOX_WORKFLOW, + purpose=( + "Sandbox: add a triage note to the first referral in a synthetic clinic " + "app. Nothing real changes." + ), + done_means=( + "The note is in the synthetic app's record store, confirmed by reading " + "the store directly instead of trusting the screen." + ), + inputs={ + "note": { + "type": "string", + "minLength": 1, + "maxLength": 500, + "description": "The triage note to add. Use made-up text.", + }, + "sandbox_case": { + "type": "string", + "enum": list(CASES), + "default": "normal", + "description": f"Which situation the synthetic app shows ({case_list}).", + }, + }, + required=["note"], + changes_records=True, + authored=True, + ) + + +def flow_engine_available() -> tuple[bool, str]: + """Whether this install can drive MockMed with real openadapt-flow.""" + try: + import importlib.util + + if importlib.util.find_spec("playwright.sync_api") is None: + return False, "browser_extra_missing" + from openadapt_flow import tutorial as flow_tutorial + + if "entry_query" not in inspect.signature(flow_tutorial.run_tutorial_workflow).parameters: + return False, "flow_too_old" + from openadapt_flow.run_gate import ( # noqa: F401 + build_runtime_authorization, + evaluate_run_gate, + ) + from openadapt_flow.runtime.effects import RestRecordVerifier # noqa: F401 + except Exception: + return False, "flow_unavailable" + return True, "ok" + + +def _closed_local_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _reachable(base_url: str) -> bool: + try: + with urlopen(f"{base_url.rstrip('/')}/api/db", timeout=2.0) as response: + return 200 <= response.status < 300 + except Exception: + return False + + +def _reset(base_url: str) -> None: + request = Request( + f"{base_url.rstrip('/')}/api/reset", + data=b"{}", + method="POST", + headers={"Content-Type": "application/json"}, + ) + with urlopen(request, timeout=5.0): + pass + + +class SandboxEngine: + """Run the sandbox workflow and report through the partner contract.""" + + def __init__( + self, + runs_dir: Path | str, + *, + engine: str = "auto", + headed: bool = False, + attended: Any = None, + ) -> None: + if engine not in {"auto", "flow", "simulated"}: + raise ValueError("sandbox engine must be auto, flow, or simulated") + self.runs_dir = Path(runs_dir).expanduser().resolve() + self.headed = headed + self.unavailable_reason: Optional[str] = None + if engine == "simulated": + self.engine = "simulated" + else: + available, why = flow_engine_available() + if available: + self.engine = "flow" + elif engine == "flow": + raise RuntimeError( + "the flow sandbox engine needs the tutorial extra: " + "pip install 'openadapt-agent[tutorial]'" + ) + else: + self.engine = "simulated" + self.unavailable_reason = why + if attended is None: + from openadapt_agent.attended import AttendedBridge + + attended = AttendedBridge(self.runs_dir) + self.attended = attended + self._lock = threading.Lock() + self._ready = threading.Event() + self._session: Any = None + self._workflow: Any = None + self._closed = False + self._entry = CatalogEntry(card=_card(), engine=self) + if self.engine == "flow": + threading.Thread( + target=self._prepare, name="openadapt-sandbox-prepare", daemon=True + ).start() + else: + self._ready.set() + + # -- catalog ------------------------------------------------------------- + + def entries(self) -> list[CatalogEntry]: + return [self._entry] + + def describe(self) -> dict[str, Any]: + info: dict[str, Any] = { + "engine": self.engine, + "app": "MockMed, a synthetic clinic app with made-up patients", + "ready": self._ready.is_set(), + "cases": [ + {"case": case.name, "outcome": case.outcome, "what_it_shows": case.shows} + for case in CASES.values() + ], + } + if self.engine == "flow": + info["engine_detail"] = ( + "Each run drives the synthetic app in a hidden browser with the real " + "OpenAdapt engine. The first run waits while OpenAdapt records the " + "workflow once." + ) + else: + info["engine_detail"] = ( + "Results are simulated without opening the app. To drive the " + "synthetic app in a hidden browser, install the tutorial extra: " + "uvx --python 3.12 --from 'openadapt-agent[tutorial]' openadapt-agent serve" + ) + return info + + # -- preparation --------------------------------------------------------- + + def _prepare(self) -> None: + try: + from openadapt_flow.ir import Workflow + + from openadapt_agent.tutorial import prepare_tutorial_session + + session = prepare_tutorial_session( + self.runs_dir / "sandbox-app", headed=self.headed, reuse_bundle=True + ) + if self._closed: + session.close() + return + self._session = session + self._workflow = Workflow.load(session.bundle_dir) + except Exception as exc: + _LOG.warning( + "the synthetic app could not start (%s); sandbox results are now simulated", + type(exc).__name__, + ) + self.engine = "simulated" + self.unavailable_reason = "app_failed_to_start" + finally: + self._ready.set() + + def close(self) -> None: + self._closed = True + session, self._session = self._session, None + if session is not None: + try: + session.close() + except Exception: + _LOG.warning("the synthetic app did not stop cleanly") + + # -- runs ------------------------------------------------------------------ + + def _sandbox(self, case: SandboxCase, engine: Optional[str] = None) -> dict[str, str]: + return {"case": case.name, "engine": engine or self.engine} + + def execute( + self, + entry: CatalogEntry, + run_id: str, + inputs: dict[str, str], + raw: Mapping[str, Any], + ) -> EngineResult: + case = CASES[str(raw.get("sandbox_case") or "normal")] + if self.engine == "flow" and not self._ready.wait(PREPARE_TIMEOUT_S): + return EngineResult("platform_error", sandbox=self._sandbox(case)) + if self.engine != "flow": + return self._simulate(case) + return self._run_flow(case, run_id, {"note": inputs["note"]}) + + def _simulate(self, case: SandboxCase) -> EngineResult: + sandbox = self._sandbox(case, "simulated") + if case.query is None: + return EngineResult("app_unreachable", sandbox=sandbox) + verified = case.execution == "VERIFIED" + report = { + "success": verified, + "execution_outcome": case.execution, + "transaction_outcome": case.transaction, + "execution_profile": "standard", + "production_eligible": verified, + "model_calls": 0, + } + attention = ( + {"status": "pending", "durably_paused": True, "category": case.attention_category} + if case.attention_category + else None + ) + return EngineResult( + reason_for_run(exit_code=None, report=report, attention=attention), + execution_outcome=case.execution, + transaction_outcome=case.transaction, + model_calls=0, + sandbox=sandbox, + ) + + def _run_flow(self, case: SandboxCase, run_id: str, params: dict[str, str]) -> EngineResult: + sandbox = self._sandbox(case, "flow") + session = self._session + if session is None or self._workflow is None: + return EngineResult("platform_error", sandbox=sandbox) + base_url = session.base_url or session.url.split("?", 1)[0] + if case.query is None: + base_url = f"http://127.0.0.1:{_closed_local_port()}/" + started = time.monotonic() + with self._lock: + # Pre-flight: nothing has been sent to the app yet. + if not _reachable(base_url): + return EngineResult("app_unreachable", sandbox=sandbox) + return self._governed_run(case, run_id, params, base_url, started, sandbox) + + def _governed_run( + self, + case: SandboxCase, + run_id: str, + params: dict[str, str], + base_url: str, + started: float, + sandbox: dict[str, str], + ) -> EngineResult: + """Admit and run once, the way Flow's tutorial does, with the caller's note.""" + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from openadapt_flow.deployment import DeploymentConfig, PolicySection + from openadapt_flow.execution_profiles import ( + ExecutionProfile, + execution_profile_contract, + ) + from openadapt_flow.run_gate import build_runtime_authorization, evaluate_run_gate + from openadapt_flow.runtime import Replayer + from openadapt_flow.runtime.effects import RestRecordVerifier + + workflow = self._workflow + bundle_dir = self._session.bundle_dir + run_dir = self.runs_dir / run_id + entry_url = f"{base_url.rstrip('/')}/{case.query}" + try: + _reset(base_url) + verifier = RestRecordVerifier( + base_url, + records_path="/api/db", + records_key="records", + timeout_s=2.0, + poll_interval_s=0.05, + ) + gate = evaluate_run_gate( + workflow, + bundle_dir=bundle_dir, + deployment=DeploymentConfig(policy=PolicySection(policy=_POLICY)), + effect_verifier=verifier, + profile_contract=execution_profile_contract(ExecutionProfile.STANDARD), + effective_durable=True, + effective_require_settled=True, + ) + if not gate.passed: + return EngineResult( + "not_ready_to_run", + failed_checks=parse_failed_checks(gate.render()), + sandbox=sandbox, + ) + authorization = build_runtime_authorization( + workflow, gate, approval_source=_APPROVAL_SOURCE, params=params + ) + backend, close = PlaywrightBackend.launch(entry_url, headless=not self.headed) + except Exception: + # Nothing reached the app: the gate, the authorization, or the + # browser launch failed before any action. + _LOG.exception("sandbox run could not start") + return EngineResult("platform_error", sandbox=sandbox) + try: + replayer = Replayer( + backend, + effect_verifier=verifier, + governed_authorization=authorization, + durable=True, + require_settled=True, + ) + report_model = replayer.run( + workflow.model_copy(deep=True), + params=params, + bundle_dir=bundle_dir, + run_dir=run_dir, + execution_target_kind="web", + execution_origin=base_url.rstrip("/"), + execution_entry_url=entry_url, + ) + except Exception: + _LOG.exception("sandbox run failed after it started") + return EngineResult("result_unreadable", sandbox=sandbox) + finally: + try: + close() + except Exception: + pass + report = self._report(run_dir, report_model) + verified = isinstance(report, dict) and report.get("transaction_outcome") == "VERIFIED" + attention = None if verified else attention_for(self.attended, run_dir) + model_calls = report.get("model_calls") if isinstance(report, dict) else None + return EngineResult( + reason_for_run(exit_code=None, report=report, attention=attention), + execution_outcome=report.get("execution_outcome") if report else None, + transaction_outcome=report.get("transaction_outcome") if report else None, + needs_attention_id=(attention or {}).get("id") if attention else None, + model_calls=model_calls if isinstance(model_calls, int) else None, + seconds=time.monotonic() - started, + sandbox=sandbox, + ) + + @staticmethod + def _report(run_dir: Path, report_model: Any) -> Optional[dict[str, Any]]: + path = run_dir / "report.json" + if path.is_file() and not path.is_symlink(): + try: + value = json.loads(path.read_text()) + if isinstance(value, dict): + return value + except (OSError, ValueError): + pass + dump = getattr(report_model, "model_dump", None) + if callable(dump): + value = dump(mode="json") + return value if isinstance(value, dict) else None + return None + + def refresh(self, entry: CatalogEntry, run_id: str) -> Optional[EngineResult]: + """Re-read a reviewed flow-engine run; simulated runs never change.""" + if self.engine != "flow": + return None + return reason_from_run_dir(self.attended, self.runs_dir / run_id) + + +def default_sandbox_dir() -> Path: + """A stable per-user directory, so the sandbox never writes into a project.""" + base = os.environ.get("OPENADAPT_AGENT_SANDBOX_DIR") + if base: + return Path(base).expanduser() + return Path.home() / ".openadapt" / "agent-sandbox" diff --git a/src/openadapt_agent/service.py b/src/openadapt_agent/service.py new file mode 100644 index 0000000..2d8c647 --- /dev/null +++ b/src/openadapt_agent/service.py @@ -0,0 +1,423 @@ +"""``run_workflow`` and ``get_run``: one request, one durable run, one result. + +The service validates a request, asks :class:`~openadapt_agent.runs.RunStore` +whether it starts a new attempt or replays an earlier one, runs new attempts on +a worker thread, and waits up to ``wait_seconds`` for the result. A run that is +still going returns ``outcome: "running"`` with its ``run_id``; ``get_run`` +picks it up later, from this process or after a restart. + +Engines do the work. :class:`FlowCliEngine` shells out to the governed +``openadapt-flow run`` CLI for an operator's bundles. The sandbox engine in +:mod:`openadapt_agent.sandbox` drives Flow against a synthetic app. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import socket +import threading +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping, Optional, Protocol + +from openadapt_agent.cards import WorkflowCard, coerce_inputs, input_problems +from openadapt_agent.contract import REASONS, build_result, reason_for_run +from openadapt_agent.runner import FlowRunner, is_safe_run_id +from openadapt_agent.runs import RunStore, pid_alive + +__all__ = [ + "REQUEST_ID_PATTERN", + "attention_for", + "reason_from_run_dir", + "CatalogEntry", + "EngineResult", + "FlowCliEngine", + "RunService", + "ServiceError", + "MAX_WAIT_SECONDS", + "DEFAULT_WAIT_SECONDS", +] + +_LOG = logging.getLogger(__name__) +#: The caller's id for one piece of work: no spaces, so free text can't ride in it. +REQUEST_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:-]{3,127}$" +REQUEST_ID_RE = re.compile(REQUEST_ID_PATTERN) +MAX_WAIT_SECONDS = 600 +DEFAULT_WAIT_SECONDS = 60 +#: Outcomes a person can still change after the run reports. +_REFRESHABLE = {"needs_review"} + + +class ServiceError(Exception): + """A request the service can't turn into a run record (bad run_id).""" + + +@dataclass +class EngineResult: + """What an engine learned from one attempt. Closed values only.""" + + reason: str + execution_outcome: Optional[str] = None + transaction_outcome: Optional[str] = None + failed_checks: list[str] = field(default_factory=list) + needs_attention_id: Optional[str] = None + model_calls: Optional[int] = None + seconds: Optional[float] = None + sandbox: Optional[dict[str, str]] = None + + +class Engine(Protocol): + def execute( + self, entry: "CatalogEntry", run_id: str, inputs: dict[str, str], raw: Mapping[str, Any] + ) -> EngineResult: ... + + def refresh(self, entry: "CatalogEntry", run_id: str) -> Optional[EngineResult]: ... + + +@dataclass +class CatalogEntry: + """One workflow the server can list and, when allowed, run.""" + + card: WorkflowCard + engine: Any + available: bool = True + info: Any = None # bundles.WorkflowInfo for an operator bundle + legacy_id: Optional[str] = None + + @property + def name(self) -> str: + return self.card.name + + +#: What a failed attention lookup stands for: a pause may be open, so the +#: run waits for a person instead of inviting a retry. +_UNKNOWN_PAUSE = {"status": "pending", "durably_paused": True, "category": None} + + +def attention_for(attended: Any, run_dir: Any) -> Optional[dict[str, Any]]: + """Flow's Needs Attention projection for a run, failing toward caution.""" + try: + return attended.for_run_dir(run_dir) + except Exception: + _LOG.exception("could not read the Needs Attention state for a run") + return dict(_UNKNOWN_PAUSE) + + +class FlowCliEngine: + """Run an operator bundle through the governed ``openadapt-flow run`` CLI.""" + + def __init__(self, runner: FlowRunner, attended: Any) -> None: + self.runner = runner + self.attended = attended + + def execute( + self, entry: CatalogEntry, run_id: str, inputs: dict[str, str], raw: Mapping[str, Any] + ) -> EngineResult: + outcome = self.runner.run( + workflow=entry.legacy_id or entry.name, + bundle_dir=entry.info.bundle_dir, + params=inputs, + run_id=run_id, + ) + attention = None + if outcome.status == "halt": + attention = attention_for(self.attended, outcome.run_dir) + outcome.apply_attention(attention) + summary = outcome.summary if isinstance(outcome.summary, dict) else {} + total_ms = summary.get("total_ms") + return EngineResult( + reason=outcome.contract_reason(), + execution_outcome=outcome.execution_outcome, + transaction_outcome=outcome.transaction_outcome, + failed_checks=list(outcome.failed_checks), + needs_attention_id=(attention or {}).get("id"), + model_calls=summary.get("model_calls"), + seconds=total_ms / 1000.0 if isinstance(total_ms, (int, float)) else None, + ) + + def refresh(self, entry: CatalogEntry, run_id: str) -> Optional[EngineResult]: + """Re-read a reviewed run: a person may have continued or ended it.""" + return reason_from_run_dir(self.attended, Path(self.runner.config.runs_dir) / run_id) + + +def _owned_elsewhere(record: Mapping[str, Any]) -> bool: + """Whether another live process may still be running this record. + + A record from another computer (a shared runs directory) can't be + checked from here, so it counts as running rather than being overwritten. + """ + host = record.get("host") + if isinstance(host, str) and host and host != socket.gethostname(): + return True + pid = record.get("pid") + return pid != os.getpid() and pid_alive(pid) + + +def reason_from_run_dir(attended: Any, run_dir: Path) -> Optional[EngineResult]: + """Re-derive a run's result from Flow's report and review state on disk.""" + report_path = run_dir / "report.json" + if run_dir.is_symlink() or report_path.is_symlink() or not report_path.is_file(): + return None + try: + report = json.loads(report_path.read_text()) + except (OSError, ValueError): + return None + if not isinstance(report, dict): + return None + attention = attention_for(attended, run_dir) + transaction = report.get("transaction_outcome") + execution = report.get("execution_outcome") + return EngineResult( + reason=reason_for_run(exit_code=None, report=report, attention=attention), + execution_outcome=execution if isinstance(execution, str) else None, + transaction_outcome=transaction if isinstance(transaction, str) else None, + needs_attention_id=(attention or {}).get("id"), + ) + + +class RunService: + """Validate, deduplicate, run, and report workflow requests.""" + + def __init__( + self, + store: RunStore, + catalog: Mapping[str, CatalogEntry], + *, + mode: str, + allow_run: bool, + require_all_inputs: bool = True, + ) -> None: + self.store = store + self.catalog = dict(catalog) + self.mode = mode + self.allow_run = allow_run + self.require_all_inputs = require_all_inputs + self._active: dict[str, threading.Event] = {} + self._lock = threading.Lock() + # Reserving a request and marking its run active happen together, so a + # concurrent call with the same request_id never finds the new run + # unowned in the moment before its worker starts. + self._begin_lock = threading.Lock() + + # -- helpers ----------------------------------------------------------- + + def _result(self, reason: str, record: Mapping[str, Any], **extra: Any) -> dict[str, Any]: + return build_result( + reason, + workflow=record.get("workflow"), + run_id=record.get("run_id"), + request_id=record.get("request_id"), + mode=self.mode, + **extra, + ) + + def _refuse( + self, + reason: str, + *, + request_id: Optional[str], + workflow: Optional[str], + **extra: Any, + ) -> dict[str, Any]: + record = self.store.new_record(request_id=request_id, workflow=workflow, mode=self.mode) + result = self._result(reason, record, **extra) + self.store.save_refusal(record, result) + return result + + @staticmethod + def _wait_seconds(value: Any, default: int) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return float(default) + return float(min(max(value, 0), MAX_WAIT_SECONDS)) + + # -- run_workflow ---------------------------------------------------------- + + def run( + self, + workflow: Any, + inputs: Any, + request_id: Any, + wait_seconds: Any = DEFAULT_WAIT_SECONDS, + ) -> dict[str, Any]: + if not isinstance(request_id, str) or not REQUEST_ID_RE.fullmatch(request_id): + raise ServiceError( + "request_id must be 4 to 128 letters, digits, '.', '_', ':' or '-'. " + "Nothing ran." + ) + wait = self._wait_seconds(wait_seconds, DEFAULT_WAIT_SECONDS) + entry = self.catalog.get(workflow) if isinstance(workflow, str) else None + if entry is None: + # Never echo an unknown workflow name back: it is caller text. + return self._refuse("unknown_workflow", request_id=request_id, workflow=None) + if not entry.available: + return self._refuse("workflow_unavailable", request_id=request_id, workflow=entry.name) + if not self.allow_run: + return self._refuse("runs_disabled", request_id=request_id, workflow=entry.name) + inputs = {} if inputs is None else inputs + if isinstance(inputs, Mapping): + # Fill declared defaults so "omitted" and "sent the default" are + # the same request for request_id purposes. + defaults = { + name: schema["default"] + for name, schema in entry.card.inputs.items() + if "default" in schema and name not in inputs + } + inputs = {**defaults, **inputs} + problems = input_problems(entry.card, inputs, require_all=self.require_all_inputs) + if problems: + return self._refuse( + "invalid_input", + request_id=request_id, + workflow=entry.name, + invalid_inputs=problems, + ) + event = threading.Event() + with self._begin_lock: + begin = self.store.begin( + request_id=request_id, workflow=entry.name, inputs=inputs, mode=self.mode + ) + if begin.kind == "new": + with self._lock: + self._active[begin.run_id] = event + if begin.kind == "conflict": + return self._refuse( + "request_id_conflict", + request_id=request_id, + workflow=entry.name, + first_run_id=begin.first_run_id, + ) + if begin.kind == "retry_limit": + return self._refuse("retry_limit", request_id=request_id, workflow=entry.name) + if begin.kind == "replay": + if begin.record is None and self.store.read(begin.run_id) is None: + # Another process won the reservation and hasn't written its + # record yet. It is running; never start a second attempt. + placeholder = { + "run_id": begin.run_id, + "request_id": request_id, + "workflow": entry.name, + } + return {**self._result("running", placeholder), "replayed": True} + return self.get(begin.run_id, wait_seconds=wait, replayed=True) + try: + self._start(entry, begin.record, coerce_inputs(entry.card, inputs), inputs, event) + except BaseException: + # No worker owns the run, so get_run reports it as interrupted + # (check the record), never as running forever. + with self._lock: + self._active.pop(begin.run_id, None) + event.set() + raise + return self.get(begin.run_id, wait_seconds=wait) + + def _start( + self, + entry: CatalogEntry, + record: dict[str, Any], + rendered: dict[str, str], + raw: Mapping[str, Any], + event: threading.Event, + ) -> None: + """Run one reserved attempt on a worker; ``run`` already marked it active.""" + run_id = record["run_id"] + + def work() -> None: + try: + try: + outcome = entry.engine.execute(entry, run_id, rendered, raw) + except Exception: + # The engine may have acted before it failed: uncertain. + _LOG.exception("workflow engine failed inside the protected boundary") + outcome = EngineResult(reason="result_unreadable") + result = self._result( + outcome.reason, + record, + execution_outcome=outcome.execution_outcome, + transaction_outcome=outcome.transaction_outcome, + failed_checks=outcome.failed_checks, + needs_attention_id=outcome.needs_attention_id, + model_calls=outcome.model_calls, + seconds=outcome.seconds, + sandbox=outcome.sandbox, + ) + self.store.finish( + record, result, transaction_outcome=outcome.transaction_outcome + ) + except Exception: + _LOG.exception("could not persist a run result") + finally: + with self._lock: + self._active.pop(run_id, None) + event.set() + + thread = threading.Thread(target=work, name=f"openadapt-run-{run_id}", daemon=True) + thread.start() + + # -- get_run ----------------------------------------------------------- + + def get( + self, run_id: Any, wait_seconds: Any = DEFAULT_WAIT_SECONDS, *, replayed: bool = False + ) -> dict[str, Any]: + if not isinstance(run_id, str) or not is_safe_run_id(run_id): + raise ServiceError("run_id must be a run id returned by run_workflow") + with self._lock: + event = self._active.get(run_id) + if event is not None: + event.wait(self._wait_seconds(wait_seconds, DEFAULT_WAIT_SECONDS)) + # Check activity BEFORE reading the record. A worker writes its result + # and only then leaves the active set, so "not active" followed by an + # unfinished record really means nobody in this process is running it. + with self._lock: + active = run_id in self._active + record = self.store.read(run_id) + if record is None: + raise ServiceError("no run with that run_id on this server") + if record.get("state") != "finished": + other_process = _owned_elsewhere(record) + if active or other_process: + result = self._result("running", record) + if replayed: + result["replayed"] = True + return result + # The process that ran it is gone without a result. A write may + # have landed, so this is uncertain and must never be retried. + interrupted = self._result("interrupted", record) + record = self.store.finish(record, interrupted) + result = dict(record.get("result") or {}) + if result.get("outcome") in _REFRESHABLE: + result = self._refresh(record, result) + if replayed: + result["replayed"] = True + return result + + def _refresh(self, record: dict[str, Any], result: dict[str, Any]) -> dict[str, Any]: + entry = self.catalog.get(record.get("workflow") or "") + refresh = getattr(getattr(entry, "engine", None), "refresh", None) + if entry is None or refresh is None: + return result + try: + update = refresh(entry, record["run_id"]) + except Exception: + _LOG.exception("could not refresh a reviewed run") + return result + if update is None or update.reason == result.get("reason") or update.reason not in REASONS: + return result + if REASONS[update.reason].safe_to_retry: + # A reviewed run only leaves review through a person's decision. + # Flow clears a continued pause just before it replaces the + # pre-resume report, so a no-effect report with no pause can be + # stale. Keep waiting rather than invite a second write. + return result + fresh = self._result( + update.reason, + record, + execution_outcome=update.execution_outcome, + transaction_outcome=update.transaction_outcome, + needs_attention_id=update.needs_attention_id, + sandbox=result.get("sandbox"), + ) + self.store.update_result(record, fresh) + return fresh diff --git a/src/openadapt_agent/skill.py b/src/openadapt_agent/skill.py index f61d189..8f52f4e 100644 --- a/src/openadapt_agent/skill.py +++ b/src/openadapt_agent/skill.py @@ -1,163 +1,171 @@ -"""Agent Skills emitter — WRAPS ``openadapt_flow.emit.skill.emit_skill``. - -openadapt-flow already generates a portable skill folder -(``<slug>/SKILL.md`` + ``<slug>/bundle/``). This module does not -regenerate any of that; it calls flow's emitter and then APPENDS two -agent-facing sections the flow emitter does not cover: - -1. how to invoke the workflow through this package's MCP server - (``openadapt-agent serve``) instead of the raw CLI, and -2. the halt and attended-action semantics an agent must respect. +"""Agent Skill export: what a workflow does and how to call it, nothing more. + +``openadapt-agent emit-skill BUNDLE --out DIR`` writes ``DIR/<name>/SKILL.md`` +from the workflow card (see :mod:`openadapt_agent.cards`): its name, purpose, +what done means, typed inputs, and how to read the four outcomes. A skill is +loaded into model context, which for most clients means a hosted model, so the +file never carries recorded example values, step intents, observed text, +secrets, or local paths. Each skill's description names its own workflow and +inputs, so an agent with several skills can tell them apart. + +The compiled bundle stays with the operator's server. ``--include-bundle`` +copies it next to ``SKILL.md`` for an operator who wants one portable folder; +that copy is protected workflow data and SKILL.md tells the agent not to read +it. Flow's own ``openadapt-flow emit-skill`` remains available for replay +demos. """ from __future__ import annotations import json -import re +import shutil from pathlib import Path +from typing import Any from openadapt_agent.bundles import load_workflow_info -from openadapt_agent.copy import ( - IDENTITY_SENTENCE, - SKILL_HONESTY, - SKILL_NAME, - SKILL_WHEN_TO_USE, -) +from openadapt_agent.cards import WorkflowCard, card_for_bundle, inputs_schema +from openadapt_agent.copy import OUTCOME_RULES, SKILL_NAME -__all__ = ["emit_agent_skill"] +__all__ = ["emit_agent_skill", "render_skill"] -_FRONTMATTER = re.compile(r"\A---\n(.*?)\n---\n", re.DOTALL) _FORBIDDEN_SKILL_NAMES = frozenset({"computer-use", "computer_use", "computeruse"}) +_DESCRIPTION_LIMIT = 1024 -_APPENDIX_TEMPLATE = """\ -## Invoking via MCP (openadapt-agent) +def _skill_name(card_name: str) -> str: + name = card_name.replace("_", "-").lower()[:64].strip("-") + if name in _FORBIDDEN_SKILL_NAMES or name.replace("-", "") == "computeruse": + return SKILL_NAME + return name or SKILL_NAME + + +def _input_line(name: str, schema: dict[str, Any], required: bool) -> str: + kind = schema.get("type", "string") + shape = {"string": "text", "number": "number", "boolean": "true or false"}.get(kind, kind) + if schema.get("format") == "date": + shape = "date, YYYY-MM-DD" + elif schema.get("format") == "date-time": + shape = "date and time, ISO 8601" + if schema.get("enum"): + shape = "one of " + ", ".join(f"`{value}`" for value in schema["enum"]) + details = [shape] + if "maxLength" in schema: + details.append(f"up to {schema['maxLength']} characters") + if "pattern" in schema: + details.append(f"matches `{schema['pattern']}`") + if not required: + details.append("optional") + description = schema.get("description", "") + return f"- `{name}` ({'; '.join(details)}): {description}".rstrip(": ") + + +def _description(card: WorkflowCard) -> str: + inputs = ", ".join(sorted(card.inputs)) or "none" + text = ( + f"{card.purpose} Runs the OpenAdapt workflow {card.name} through the " + f"run_workflow tool and reports done, needs_review, not_sure_if_saved, " + f"or did_not_run. Inputs: {inputs}." + ) + return text[:_DESCRIPTION_LIMIT] + + +def render_skill(card: WorkflowCard, *, include_bundle: bool = False) -> str: + """Render SKILL.md for one workflow card. Pure: no I/O.""" + schema = inputs_schema(card) + required = set(schema["required"]) + inputs = "\n".join( + _input_line(name, prop, name in required) for name, prop in schema["properties"].items() + ) or "- This workflow takes no inputs. Send an empty object." + example_inputs = {name: f"<{name}>" for name in sorted(required)} + call = json.dumps( + { + "workflow": card.name, + "inputs": example_inputs, + "request_id": "<your id for this piece of work>", + }, + indent=2, + ) + bundle_note = "" + if include_bundle: + bundle_note = ( + "\nThe `bundle` folder next to this file is the compiled workflow for the " + "operator's server. It is protected workflow data. Don't open or quote it.\n" + ) + return f"""--- +name: {_skill_name(card.name)} +description: {json.dumps(_description(card))} +--- -{when_to_use} +# {card.name} -{honesty} +{card.purpose} -If the operator runs the local `openadapt-agent` MCP server over this -skill's bundle directory: +Done means: {card.done_means} -```bash -openadapt-agent serve --bundles {bundle_ref} --allow-run -``` +## When to use it -this workflow appears as the opaque MCP tool `run_{workflow_id}` -(required parameters: {param_list}). Recorded demonstration values are -never placed in the tool schema and every declared parameter is required -by default. Prefer the MCP tool when it is available: it executes -through the governed `openadapt-flow run` path (fail-closed admission -gates) rather than the permissive `replay` demo path, and it returns a -PHI-safe structured outcome instead of raw CLI output. Inspection and -PHI-safe Needs Attention tools are always available; `run_{workflow_id}` exists only -when the operator started the server with `--allow-run`. The public -synthetic tutorial is `openadapt-agent serve --allow-run`. - -## Halt semantics (IMPORTANT) - -A run has exactly one of these outcomes — report it faithfully: - -- **success** — exit code 0 AND a precise `report.json` records - `execution_outcome: VERIFIED` with a consistent success flag. A legacy - report without the precise field must record `success: true`. Only then may - you tell the user the workflow completed. -- **halt** (MCP status `halt`, `execution_outcome: HALTED`) — If the tool - returns HALTED, tell the user the record did not change. - `execution_outcome` says whether the run halted, completed without - sufficient verification, or completed a rollback. None is a verified success. - Protected evidence remains in the local OpenAdapt operator - experience; default MCP results contain only opaque IDs, fixed messages, - and count/boolean metrics. Surface the exact outcome to the user; do not - infer the business effect or retry blindly. -- **governed refusal** (exit code 2 / MCP status `refused`, `openadapt-flow - run` only) — an admission gate refused the bundle before execution; - NOTHING was executed. The printed coverage report names the failing - gate. - -{honesty} - -## Needs Attention - -When a run halts, `list_needs_attention` and `get_attention_item` return -an opaque, PHI-safe queue card. Do not ask for or place credentials, -challenge answers, screenshots, observed text, or other protected values -in an attended-action payload. - -If the operator enabled attended actions, use only the exact action tool -that matches their explicit decision: - -- `continue_attention` only after the local operator says they completed - the paused task in the live application. Flow revalidates the outcome - and resumes after it; it never performs that completed action again. -- `skip_attention` only for an allowed, declared skip. -- `reject_attention` to terminate this run without dispatching a new - action. Earlier run actions may have effects. Review the protected local - report and transaction outcome. Use escalation if a qualified operator - can still continue the run. -- `teach_attention` to request a corrective demonstration. -- `escalate_attention` to preserve the pause for qualified assistance. - -Reload the item immediately before acting, pass its exact capability -digest, and use one stable idempotency key for retries of that same -request. The MCP server will separately elicit the person's confirmation; -a client without form elicitation must direct the operator to Flow's -attended console/CLI instead. Elicitation is host-mediated explicit -confirmation, not cryptographic human-presence or identity proof. Never -infer or answer that elicitation yourself, auto-retry an uncertain -delivery, or substitute one action for another. -""" +Use this skill when you have decided what to enter and the user wants it entered +with the `{card.name}` workflow. OpenAdapt does the entry in the app on the +computer that runs it, then reads the record back to check that it saved. +## How to run it -def _format_params(params: dict[str, str]) -> str: - if not params: - return "none" - return ", ".join(f"`{name}`" for name in sorted(params)) +1. Call `run_workflow` on the OpenAdapt MCP server: + ```json +{_indent(call, 3)} + ``` -def _skill_name(slug: str) -> str: - compact = slug.replace("_", "-").replace(" ", "-").lower() - if compact in _FORBIDDEN_SKILL_NAMES: - return SKILL_NAME - return slug - - -def apply_skill_frontmatter(text: str, *, name: str, description: str) -> str: - """Replace Flow's frontmatter description with the shared identity sentence.""" - match = _FRONTMATTER.match(text) - if match is None: - raise ValueError("emitted skill is missing YAML frontmatter") - body = text[match.end() :] - front = ( - f"---\nname: {_skill_name(name)}\n" - f"description: {json.dumps(description)}\n---\n" - ) - return front + body + Inputs: +{_indent(inputs, 3)} -def emit_agent_skill(bundle_dir: Path | str, out_dir: Path | str) -> Path: - """Emit flow's skill folder for *bundle_dir*, then append MCP guidance. + `request_id` is your own id for this piece of work, such as the referral id. + Send the same id if you retry; the same id never writes twice. Don't put + patient details in it. +2. If the outcome is `running`, call `get_run` with the `run_id` until it isn't. +3. Tell the user `what_happened` and follow `next_action`. - Returns the skill folder path (containing ``SKILL.md`` and - ``bundle/``), exactly as flow's emitter lays it out. - """ - from openadapt_flow.emit import emit_skill # WRAP, don't duplicate +## What the outcome means - skill_dir = emit_skill(bundle_dir, out_dir) - info = load_workflow_info(Path(bundle_dir)) - appendix = _APPENDIX_TEMPLATE.format( - workflow_id=info.public_id, - bundle_ref="<path-to-this-skill-folder>/bundle", - param_list=_format_params(info.params), - when_to_use=SKILL_WHEN_TO_USE, - honesty=SKILL_HONESTY, - ) - skill_md = skill_dir / "SKILL.md" - rewritten = apply_skill_frontmatter( - skill_md.read_text(encoding="utf-8"), - name=skill_dir.name, - description=IDENTITY_SENTENCE, +| outcome | What happened | What to do | +| --- | --- | --- | +| `done` | Saved and checked. | Nothing more for this request. | +| `needs_review` | Stopped before saving. Nothing was written. A person decides. | Don't start the same work again. Check back with `get_run`. | +| `not_sure_if_saved` | It may or may not have saved. | Never retry. A person checks the record. | +| `did_not_run` | Nothing was written. | Fix the problem in `what_happened`, then retry with the same `request_id` if `safe_to_retry` is true. | + +{OUTCOME_RULES} + +## Setup for the operator + +The operator serves this workflow on the computer that can open the app: + +```bash +openadapt-agent serve --mode production --bundles <bundles-dir> +``` +{bundle_note}""" + + +def _indent(text: str, spaces: int) -> str: + pad = " " * spaces + return "\n".join(pad + line if line else line for line in text.splitlines()) + + +def emit_agent_skill( + bundle_dir: Path | str, out_dir: Path | str, *, include_bundle: bool = False +) -> Path: + """Write a PHI-safe skill folder for one bundle and return its path.""" + bundle_dir = Path(bundle_dir) + info = load_workflow_info(bundle_dir) + if not info.ok: + raise ValueError("the workflow bundle could not be loaded safely") + card = card_for_bundle(info, default_name=info.public_id) + skill_dir = Path(out_dir) / _skill_name(card.name) + skill_dir.mkdir(parents=True, exist_ok=True) + (skill_dir / "SKILL.md").write_text( + render_skill(card, include_bundle=include_bundle), encoding="utf-8" ) - skill_md.write_text(rewritten + appendix, encoding="utf-8") + if include_bundle: + shutil.copytree(bundle_dir, skill_dir / "bundle", dirs_exist_ok=True) return skill_dir diff --git a/src/openadapt_agent/tutorial.py b/src/openadapt_agent/tutorial.py index 9506b7f..2354580 100644 --- a/src/openadapt_agent/tutorial.py +++ b/src/openadapt_agent/tutorial.py @@ -1,8 +1,8 @@ -"""Generate and host the public synthetic tutorial for ``serve --tutorial``. +"""Generate and host the synthetic MockMed workflow the sandbox runs. The compiled bundle is not vendored. Record, compile, and certify go through Flow's public tutorial APIs. MockMed stays up for the life of the MCP server -so a governed ``run`` can hit a live system of record. Synthetic data only. +so every sandbox run hits a live system of record. Synthetic data only. """ from __future__ import annotations @@ -11,7 +11,12 @@ from pathlib import Path from typing import Callable -__all__ = ["TutorialError", "TutorialSession", "prepare_tutorial_session"] +__all__ = [ + "TutorialError", + "TutorialSession", + "bind_param_effects", + "prepare_tutorial_session", +] class TutorialError(RuntimeError): @@ -25,6 +30,8 @@ class TutorialSession: deployment_config: Path work_dir: Path close: Callable[[], None] + #: MockMed's origin, without the tutorial entry query. + base_url: str = "" def _deployment_yaml(base_url: str, entry_url: str) -> str: @@ -32,6 +39,7 @@ def _deployment_yaml(base_url: str, entry_url: str) -> str: return ( "name: synthetic-tutorial\n" "backend:\n" + " kind: web\n" f' url: "{entry_url}"\n' " headed: false\n" "effects:\n" @@ -62,6 +70,39 @@ def _bundle_is_loadable(bundle_dir: Path) -> bool: return True +def bind_param_effects(bundle_dir: Path) -> object: + """Bind mined read-back values to the parameters that typed them. + + Flow mines each ``field_equals`` check with the demonstrated value as a + literal and leaves binding it to the run's parameter to the operator's + review. This is that review for the synthetic workflow: a check whose + literal equals a parameter's recorded value reads that parameter instead, + so the record check confirms the caller's value, not the recorded one. + Saving re-seals the bundle manifest. Returns the loaded workflow. + """ + from openadapt_flow.ir import Workflow + + workflow = Workflow.load(bundle_dir) + try: + from openadapt_flow.runtime.effects.effect import ValueExpr + except ImportError: + return workflow + by_value = {str(value): name for name, value in (workflow.params or {}).items()} + changed = False + for step in workflow.steps: + for effect in getattr(step, "effects", None) or []: + kind = getattr(effect.kind, "value", effect.kind) + value = getattr(effect, "value", None) + literal = getattr(value, "literal", None) + if kind == "field_equals" and literal is not None and literal in by_value: + effect.value = ValueExpr(param=by_value[literal]) + changed = True + if changed: + workflow.save(bundle_dir) + workflow = Workflow.load(bundle_dir) + return workflow + + def prepare_tutorial_session( work_dir: Path, *, @@ -101,13 +142,13 @@ def prepare_tutorial_session( try: if not (reuse_bundle and _bundle_is_loadable(bundle_dir)): record_tutorial(base_url, recording_dir, headed=headed) - workflow = compile_recording( + compile_recording( recording_dir, bundle_dir, name=TUTORIAL_WORKFLOW_NAME, mine_effects=True, ) - certify_tutorial(workflow) + certify_tutorial(bind_param_effects(bundle_dir)) entry_url = f"{base_url.rstrip('/')}/{TUTORIAL_ENTRY_QUERY}" config_path.write_text(_deployment_yaml(base_url, entry_url), encoding="utf-8") except TutorialError: @@ -123,4 +164,5 @@ def prepare_tutorial_session( deployment_config=config_path, work_dir=root, close=stop, + base_url=base_url, ) diff --git a/tests/conftest.py b/tests/conftest.py index 9f3bcdb..fefcc9d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -70,7 +70,7 @@ def __init__(self, exit_code: int = 0, report: dict | None = None): self.stdout = "stub stdout" self.stderr = "" - def __call__(self, cmd, capture_output=True, text=True, timeout=None): + def __call__(self, cmd, capture_output=True, text=True, timeout=None, **kwargs): import subprocess self.calls.append(list(cmd)) @@ -84,11 +84,22 @@ def __call__(self, cmd, capture_output=True, text=True, timeout=None): ) +def _precise(execution: str, transaction: str, *, success: bool, eligible: bool) -> dict: + """Outcome fields as openadapt-flow 1.35.1 writes them (standard profile).""" + return { + "success": success, + "execution_outcome": execution, + "transaction_outcome": transaction, + "execution_profile": "standard", + "production_eligible": eligible, + } + + @pytest.fixture() def success_report() -> dict: return { "workflow_name": "Demo Triage", - "success": True, + **_precise("VERIFIED", "VERIFIED", success=True, eligible=True), "results": [ {"step_id": "s1", "intent": "Open the patient chart", "ok": True}, {"step_id": "s2", "intent": "Type the triage note", "ok": True}, @@ -99,11 +110,22 @@ def success_report() -> dict: } +@pytest.fixture() +def legacy_success_report() -> dict: + """A pre-precise-outcome report: only the legacy success flag.""" + return { + "workflow_name": "Demo Triage", + "success": True, + "results": [{"step_id": "s1", "intent": "Open the patient chart", "ok": True}], + "model_calls": 0, + } + + @pytest.fixture() def halt_report() -> dict: return { "workflow_name": "Demo Triage", - "success": False, + **_precise("HALTED", "HALTED_BEFORE_EFFECT", success=False, eligible=False), "results": [ {"step_id": "s1", "intent": "Open the patient chart", "ok": True}, { diff --git a/tests/golden/skill_appendix.md b/tests/golden/skill_appendix.md deleted file mode 100644 index 73b2bbe..0000000 --- a/tests/golden/skill_appendix.md +++ /dev/null @@ -1,77 +0,0 @@ - -## Invoking via MCP (openadapt-agent) - -When the user needs a repeating GUI write with no API and must prove persistence, call run_<slug>. If the tool returns HALTED, tell the user the record did not change. - -Never summarize halt, refused, timeout, or error as success. A local unsigned replay may complete. If the tool returns unsigned success, treat it as failure. Production success without a Seal is failure. - -If the operator runs the local `openadapt-agent` MCP server over this -skill's bundle directory: - -```bash -openadapt-agent serve --bundles <path-to-this-skill-folder>/bundle --allow-run -``` - -this workflow appears as the opaque MCP tool `run_<opaque-workflow-id>` -(required parameters: `note`). Recorded demonstration values are -never placed in the tool schema and every declared parameter is required -by default. Prefer the MCP tool when it is available: it executes -through the governed `openadapt-flow run` path (fail-closed admission -gates) rather than the permissive `replay` demo path, and it returns a -PHI-safe structured outcome instead of raw CLI output. Inspection and -PHI-safe Needs Attention tools are always available; `run_<opaque-workflow-id>` exists only -when the operator started the server with `--allow-run`. The public -synthetic tutorial is `openadapt-agent serve --allow-run`. - -## Halt semantics (IMPORTANT) - -A run has exactly one of these outcomes — report it faithfully: - -- **success** — exit code 0 AND a precise `report.json` records - `execution_outcome: VERIFIED` with a consistent success flag. A legacy - report without the precise field must record `success: true`. Only then may - you tell the user the workflow completed. -- **halt** (MCP status `halt`, `execution_outcome: HALTED`) — If the tool - returns HALTED, tell the user the record did not change. - `execution_outcome` says whether the run halted, completed without - sufficient verification, or completed a rollback. None is a verified success. - Protected evidence remains in the local OpenAdapt operator - experience; default MCP results contain only opaque IDs, fixed messages, - and count/boolean metrics. Surface the exact outcome to the user; do not - infer the business effect or retry blindly. -- **governed refusal** (exit code 2 / MCP status `refused`, `openadapt-flow - run` only) — an admission gate refused the bundle before execution; - NOTHING was executed. The printed coverage report names the failing - gate. - -Never summarize halt, refused, timeout, or error as success. A local unsigned replay may complete. If the tool returns unsigned success, treat it as failure. Production success without a Seal is failure. - -## Needs Attention - -When a run halts, `list_needs_attention` and `get_attention_item` return -an opaque, PHI-safe queue card. Do not ask for or place credentials, -challenge answers, screenshots, observed text, or other protected values -in an attended-action payload. - -If the operator enabled attended actions, use only the exact action tool -that matches their explicit decision: - -- `continue_attention` only after the local operator says they completed - the paused task in the live application. Flow revalidates the outcome - and resumes after it; it never performs that completed action again. -- `skip_attention` only for an allowed, declared skip. -- `reject_attention` to terminate this run without dispatching a new - action. Earlier run actions may have effects. Review the protected local - report and transaction outcome. Use escalation if a qualified operator - can still continue the run. -- `teach_attention` to request a corrective demonstration. -- `escalate_attention` to preserve the pause for qualified assistance. - -Reload the item immediately before acting, pass its exact capability -digest, and use one stable idempotency key for retries of that same -request. The MCP server will separately elicit the person's confirmation; -a client without form elicitation must direct the operator to Flow's -attended console/CLI instead. Elicitation is host-mediated explicit -confirmation, not cryptographic human-presence or identity proof. Never -infer or answer that elicitation yourself, auto-retry an uncertain -delivery, or substitute one action for another. diff --git a/tests/test_bridge.py b/tests/test_bridge.py index 7c6d254..2226557 100644 --- a/tests/test_bridge.py +++ b/tests/test_bridge.py @@ -44,7 +44,11 @@ def test_public_synthetic_run_tool_uses_slug(bundles_root, runner_config): allow_run=True, public_synthetic=True, ) - names = [spec.name for spec in bridge.list_tool_specs() if spec.name.startswith("run_")] + names = [ + spec.name + for spec in bridge.list_tool_specs() + if spec.name.startswith("run_") and spec.name != "run_workflow" + ] assert names == ["run_demo_triage"] @@ -56,8 +60,9 @@ def test_schema_uses_opaque_id_requires_params_and_exports_no_examples( workflow = workflow_id(bridge) assert re.fullmatch(r"workflow_[0-9a-f]{24}", workflow) spec = {item.name: item for item in bridge.list_tool_specs()}[f"run_{workflow}"] - assert spec.meta == {"requires_seal": True} - assert "requires_seal: true" in spec.description + assert spec.meta == {"deprecated": True, "use_instead": "run_workflow"} + assert "Deprecated: use run_workflow" in spec.description + assert "unsigned success" not in spec.description schema = spec.input_schema assert schema["type"] == "object" assert schema["additionalProperties"] is False @@ -89,12 +94,14 @@ def test_read_only_tools_and_workflow_listing_are_phi_safe( names = [spec.name for spec in bridge.list_tool_specs()] assert names == [ "list_workflows", + "get_run", "get_workflow", "get_run_report", "list_needs_attention", "get_attention_item", ] listing = bridge.dispatch("list_workflows", {}) + assert listing["mode"] == "production" assert listing["lifecycle"] == "admission-derived" assert listing["run_tools_enabled"] is False assert listing["protected_export_enabled"] is False @@ -175,6 +182,7 @@ def test_get_run_report_never_turns_demo_completion_into_verified_success( { "execution_profile": "demo", "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", "production_eligible": False, } ) @@ -186,7 +194,7 @@ def test_get_run_report_never_turns_demo_completion_into_verified_success( assert fetched["status"] == "halt" assert fetched["success"] is False assert fetched["execution_outcome"] == "COMPLETED_UNVERIFIED" - assert "completed" in fetched["message"] + assert "finished the steps" in fetched["message"] assert "did not complete" not in fetched["message"] assert str(runner_config.runs_dir) not in json.dumps(fetched) @@ -201,6 +209,7 @@ def test_run_and_report_lookup_use_the_same_public_outcome_message( { "execution_profile": "demo", "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", "production_eligible": False, } ) @@ -400,7 +409,14 @@ def test_unloadable_bundle_is_safe_by_default_and_not_runnable( assert secret not in json.dumps(listing) assert str(bad) not in json.dumps(listing) names = [spec.name for spec in bridge.list_tool_specs()] - assert not any(name.startswith("run_") for name in names) + assert [name for name in names if name.startswith("run_")] == ["run_workflow"] + result = bridge.dispatch( + "run_workflow", + {"workflow": listing["workflows"][0]["name"], "inputs": {}, "request_id": "req-0001"}, + ) + assert result["outcome"] == "did_not_run" + assert result["reason"] == "workflow_unavailable" + assert secret not in json.dumps(result) def test_tool_result_is_json_serializable( diff --git a/tests/test_cards.py b/tests/test_cards.py new file mode 100644 index 0000000..f24a022 --- /dev/null +++ b/tests/test_cards.py @@ -0,0 +1,159 @@ +"""Workflow cards: typed inputs from Flow, words from the operator, no PHI.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from openadapt_agent.bundles import load_workflow_info +from openadapt_agent.cards import ( + CARD_FILENAME, + card_for_bundle, + coerce_inputs, + input_problems, + inputs_schema, +) + + +def typed_bundle(tmp_path: Path) -> Path: + from openadapt_flow.ir import ActionKind, ParamKind, ParamSpec, Step, Workflow + + bundle = tmp_path / "bundles" / "referral" + Workflow( + name="Referral entry", + params={ + "patient_mrn": "1234567", + "specialty": "Cardiology", + "referral_date": "2026-01-02", + "urgent": "false", + }, + param_specs={ + "patient_mrn": ParamSpec(name="patient_mrn", example="1234567"), + "specialty": ParamSpec( + name="specialty", + type=ParamKind.ENUM, + example="Cardiology", + choices=["Cardiology", "Dermatology"], + ), + "referral_date": ParamSpec( + name="referral_date", type=ParamKind.DATE, example="2026-01-02" + ), + # BOOLEAN arrived after Flow 1.26; NUMBER covers the floor. + "urgent": ParamSpec( + name="urgent", + type=getattr(ParamKind, "BOOLEAN", ParamKind.NUMBER), + example=False if hasattr(ParamKind, "BOOLEAN") else "0", + ), + }, + steps=[ + Step(id="s1", intent="Open the chart for 1234567", action=ActionKind.CLICK), + ], + ).save(bundle) + return bundle + + +def write_card(bundle: Path, card: dict) -> None: + (bundle / CARD_FILENAME).write_text(json.dumps(card), encoding="utf-8") + + +def test_default_card_uses_flow_types_without_examples(tmp_path): + info = load_workflow_info(typed_bundle(tmp_path)) + card = card_for_bundle(info, default_name="workflow_abc") + schema = inputs_schema(card) + assert card.name == "workflow_abc" + assert card.authored is False + assert schema["properties"]["specialty"]["enum"] == ["Cardiology", "Dermatology"] + assert schema["properties"]["referral_date"]["format"] == "date" + assert schema["properties"]["urgent"]["type"] in {"boolean", "number"} + assert schema["required"] == ["patient_mrn", "referral_date", "specialty", "urgent"] + serialized = json.dumps(card.projection()) + for recorded in ("1234567", "2026-01-02", "Open the chart"): + assert recorded not in serialized + + +def test_operator_card_names_and_describes_the_workflow(tmp_path): + bundle = typed_bundle(tmp_path) + write_card( + bundle, + { + "name": "enter_referral", + "purpose": "Create an outgoing referral for an existing patient in the clinic EMR.", + "done_means": "The referral appears in the EMR's referral report.", + "inputs": { + "patient_mrn": {"description": "Clinic MRN.", "pattern": "^[0-9]{6,10}$"}, + "specialty": {"enum": ["Cardiology"]}, + }, + }, + ) + card = card_for_bundle(load_workflow_info(bundle), default_name="workflow_abc") + assert card.authored is True + assert card.name == "enter_referral" + schema = inputs_schema(card) + assert schema["properties"]["patient_mrn"]["pattern"] == "^[0-9]{6,10}$" + assert schema["properties"]["specialty"]["enum"] == ["Cardiology"] + + +def test_sidecar_card_does_not_break_flow_bundle_loading(tmp_path): + bundle = typed_bundle(tmp_path) + write_card(bundle, {"name": "enter_referral", "purpose": "Enter a referral."}) + assert load_workflow_info(bundle).ok + + +@pytest.mark.parametrize( + "card", + [ + {"name": "Enter Referral"}, + {"purpose": "Referral for MRN 8812345."}, + {"purpose": "Email jane.roe@example.com when done."}, + {"inputs": {"not_a_param": {"description": "x"}}}, + {"inputs": {"specialty": {"enum": ["Oncology"]}}}, + {"inputs": {"patient_mrn": {"pattern": "("}}}, + {"unexpected": True}, + ], +) +def test_invalid_card_falls_back_to_default_without_echoing(tmp_path, caplog, card): + bundle = typed_bundle(tmp_path) + write_card(bundle, card) + result = card_for_bundle(load_workflow_info(bundle), default_name="workflow_abc") + assert result.authored is False + assert result.name == "workflow_abc" + for text in ("8812345", "jane.roe", "Oncology"): + assert text not in caplog.text + + +def test_input_problems_use_closed_words_and_never_echo_values(tmp_path): + card = card_for_bundle(load_workflow_info(typed_bundle(tmp_path)), default_name="w_x") + problems = input_problems( + card, + { + "patient_mrn": 12, + "specialty": "Oncology", + "referral_date": "Jan 2", + "Jane Roe 555-123-4567": "x", + }, + ) + by_input = {item["input"]: item["problem"] for item in problems} + assert by_input["patient_mrn"] == "must be text" + assert by_input["specialty"] == "must be one of the listed values" + assert by_input["referral_date"].startswith("must be a date") + assert by_input["urgent"] == "is missing" + assert "(unknown)" in by_input + serialized = json.dumps(problems) + for value in ("Oncology", "Jan 2", "Jane Roe", "555-123-4567"): + assert value not in serialized + + +def test_valid_inputs_pass_and_render_for_flow(tmp_path): + card = card_for_bundle(load_workflow_info(typed_bundle(tmp_path)), default_name="w_x") + inputs = { + "patient_mrn": "7654321", + "specialty": "Dermatology", + "referral_date": "2026-10-08", + "urgent": True, + } + if card.inputs["urgent"]["type"] == "number": + inputs["urgent"] = 1 + assert input_problems(card, inputs) == [] + assert coerce_inputs(card, inputs)["urgent"] in {"true", "1"} diff --git a/tests/test_cli.py b/tests/test_cli.py index 969d60a..a5f2c22 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -77,12 +77,89 @@ def test_tutorial_flag_does_not_require_bundles(tmp_path): assert args.allow_run is False -def test_serve_requires_tutorial_or_bundles(capsys): - result = main(["serve"]) - assert result == 2 +def _capture_serve(monkeypatch): + captured: dict = {} + + def fake_serve(bridge, authoring=None): + captured["bridge"] = bridge + captured["tools"] = [spec.name for spec in bridge.list_tool_specs()] + captured["listing"] = bridge.dispatch("list_workflows", {}) + + monkeypatch.setattr("openadapt_agent.mcp.serve", fake_serve) + return captured + + +def test_serve_with_no_flags_starts_the_sandbox(monkeypatch, tmp_path, capsys): + monkeypatch.setenv("OPENADAPT_AGENT_SANDBOX_DIR", str(tmp_path / "sandbox")) + captured = _capture_serve(monkeypatch) + assert main(["serve", "--sandbox-engine", "simulated"]) == 0 + bridge = captured["bridge"] + assert bridge.mode == "sandbox" + assert captured["tools"] == ["list_workflows", "run_workflow", "get_run"] + assert captured["listing"]["workflows"][0]["name"] == "add_triage_note" + assert bridge.runner_config.runs_dir == tmp_path / "sandbox" err = capsys.readouterr().err - assert "provide --bundles, --tutorial, or --authoring" in err - assert "--allow-run" in err + assert "sandbox mode" in err + assert "Nothing real changes" in err + + +@pytest.mark.parametrize( + "argv", + [ + ["serve", "--tutorial"], + ["serve", "--allow-run"], + ["serve", "--mode", "sandbox"], + ], +) +def test_older_sandbox_flags_are_aliases(monkeypatch, tmp_path, argv): + captured = _capture_serve(monkeypatch) + argv = [*argv, "--sandbox-engine", "simulated", "--runs-dir", str(tmp_path / "runs")] + assert main(argv) == 0 + assert captured["bridge"].mode == "sandbox" + + +@pytest.mark.parametrize( + "argv, message", + [ + (["serve", "--mode", "production"], "--mode production needs --bundles"), + (["serve", "--mode", "attended"], "--mode attended needs --bundles"), + (["serve", "--mode", "sandbox", "--bundles", "b"], "cannot be combined with --bundles"), + (["serve", "--url", "https://app.example"], "--url and --config need"), + ], +) +def test_mode_conflicts_fail_closed(argv, message, capsys): + assert main(argv) == 2 + assert message in capsys.readouterr().err + + +def test_mode_production_enables_runs_and_attended_adds_decisions( + monkeypatch, bundles_root, tmp_path +): + from contextlib import contextmanager + + @contextmanager + def fake_attended(**kwargs): + yield None + + monkeypatch.setattr("openadapt_agent.flow_service.open_attended_service", fake_attended) + captured = _capture_serve(monkeypatch) + runs = str(tmp_path / "runs") + assert main(["serve", "--mode", "production", "--bundles", str(bundles_root), "--runs-dir", runs]) == 0 + assert captured["bridge"].mode == "production" + assert "run_workflow" in captured["tools"] + assert "reject_attention" not in captured["tools"] + assert main(["serve", "--mode", "attended", "--bundles", str(bundles_root), "--runs-dir", runs]) == 0 + assert captured["bridge"].mode == "attended" + assert "reject_attention" in captured["tools"] + assert main(["serve", "--bundles", str(bundles_root), "--runs-dir", runs]) == 0 + assert "run_workflow" not in captured["tools"] + assert captured["listing"]["run_tools_enabled"] is False + # Older flags: decisions on paused runs without runs still label as attended. + argv = ["serve", "--bundles", str(bundles_root), "--runs-dir", runs, "--allow-attended-actions"] + assert main(argv) == 0 + assert captured["bridge"].mode == "attended" + assert "run_workflow" not in captured["tools"] + assert "reject_attention" in captured["tools"] def test_authoring_flag_does_not_require_bundles_or_imply_allow_run(tmp_path): @@ -223,130 +300,3 @@ def test_tutorial_rejects_private_bundle_path(tmp_path, capsys): ) assert result == 2 assert "cannot be combined" in capsys.readouterr().err - - -def test_tutorial_serve_uses_slug_tool_and_standard_profile(monkeypatch, tmp_path, capsys): - from contextlib import contextmanager - from pathlib import Path - - from openadapt_agent.tutorial import TutorialSession - from openadapt_flow.ir import ActionKind, Step, Workflow - - captured: dict = {} - work = tmp_path / "runs" / "synthetic-tutorial" - bundle = work / "bundle" - bundle.mkdir(parents=True) - Workflow( - name="local-quickstart", - params={"note": "Synthetic follow-up in two weeks"}, - steps=[Step(id="s1", intent="Save the synthetic note", action=ActionKind.CLICK)], - ).save(bundle) - config = work / "deployment.yaml" - config.write_text("name: synthetic-tutorial\n", encoding="utf-8") - - def fake_prepare(work_dir, *, headed=False, reuse_bundle=True): - def close(): - captured["closed"] = True - - return TutorialSession( - bundle_dir=bundle, - url="http://127.0.0.1:9/?fault=ok&idempotency=demo#tasks", - deployment_config=config, - work_dir=Path(work_dir), - close=close, - ) - - def fake_serve(bridge): - captured["bridge"] = bridge - - @contextmanager - def fake_attended(**kwargs): - captured["attended"] = kwargs - yield None - - monkeypatch.setattr("openadapt_agent.tutorial.prepare_tutorial_session", fake_prepare) - monkeypatch.setattr("openadapt_agent.mcp.serve", fake_serve) - monkeypatch.setattr("openadapt_agent.flow_service.open_attended_service", fake_attended) - - result = main( - [ - "serve", - "--tutorial", - "--allow-run", - "--runs-dir", - str(tmp_path / "runs"), - ] - ) - assert result == 0 - bridge = captured["bridge"] - assert bridge.public_synthetic is True - names = [spec.name for spec in bridge.list_tool_specs() if spec.name.startswith("run_")] - assert names == ["run_local_quickstart"] - assert "--profile" in bridge.runner_config.extra_run_args - assert "standard" in bridge.runner_config.extra_run_args - assert "--approve-unverified-writes" not in bridge.runner_config.extra_run_args - assert captured.get("closed") is True - assert "tutorial enabled" in capsys.readouterr().err - - -def test_serve_allow_run_without_bundles_starts_the_synthetic_tutorial( - monkeypatch, tmp_path, capsys -): - from contextlib import contextmanager - from pathlib import Path - - from openadapt_agent.tutorial import TutorialSession - from openadapt_flow.ir import ActionKind, Step, Workflow - - captured: dict = {} - work = tmp_path / "runs" / "synthetic-tutorial" - bundle = work / "bundle" - bundle.mkdir(parents=True) - Workflow( - name="local-quickstart", - params={"note": "Synthetic follow-up in two weeks"}, - steps=[Step(id="s1", intent="Save the synthetic note", action=ActionKind.CLICK)], - ).save(bundle) - config = work / "deployment.yaml" - config.write_text("name: synthetic-tutorial\n", encoding="utf-8") - - def fake_prepare(work_dir, *, headed=False, reuse_bundle=True): - def close(): - captured["closed"] = True - - return TutorialSession( - bundle_dir=bundle, - url="http://127.0.0.1:9/?fault=ok&idempotency=demo#tasks", - deployment_config=config, - work_dir=Path(work_dir), - close=close, - ) - - def fake_serve(bridge): - captured["bridge"] = bridge - - @contextmanager - def fake_attended(**kwargs): - captured["attended"] = kwargs - yield None - - monkeypatch.setattr("openadapt_agent.tutorial.prepare_tutorial_session", fake_prepare) - monkeypatch.setattr("openadapt_agent.mcp.serve", fake_serve) - monkeypatch.setattr("openadapt_agent.flow_service.open_attended_service", fake_attended) - - result = main( - [ - "serve", - "--allow-run", - "--runs-dir", - str(tmp_path / "runs"), - ] - ) - assert result == 0 - bridge = captured["bridge"] - assert bridge.public_synthetic is True - names = [spec.name for spec in bridge.list_tool_specs() if spec.name.startswith("run_")] - assert names == ["run_local_quickstart"] - assert "--approve-unverified-writes" not in bridge.runner_config.extra_run_args - assert captured.get("closed") is True - assert "tutorial enabled" in capsys.readouterr().err diff --git a/tests/test_contract.py b/tests/test_contract.py new file mode 100644 index 0000000..358a90b --- /dev/null +++ b/tests/test_contract.py @@ -0,0 +1,238 @@ +"""The four-outcome result contract and its safety rules. + +Report fixtures copy the outcome fields of real openadapt-flow 1.35.1 runs on +the synthetic MockMed app (clean save, optimistic banner, timeout after save, +duplicate referral, missing referral). Only the fields the classifier reads are +kept. +""" + +from __future__ import annotations + +import json + +import jsonschema +import pytest + +from openadapt_agent.contract import ( + OUTCOMES, + REASONS, + RETRYABLE_REASONS, + RUN_RESULT_SCHEMA, + build_result, + ledger_outcome, + parse_failed_checks, + reason_for_run, +) + + +def flow_report(execution: str, transaction: str | None, *, success: bool, eligible: bool): + report = { + "success": success, + "execution_outcome": execution, + "execution_profile": "standard", + "production_eligible": eligible, + "execution_completed": execution == "VERIFIED", + "model_calls": 0, + "external_network_calls": "observed", + "results": [], + } + if transaction is not None: + report["transaction_outcome"] = transaction + report["outcome_envelope"] = { + "outcome": execution, + "profile": "standard", + "production_eligible": eligible, + "execution_completed": execution == "VERIFIED", + "model_calls": 0, + "external_network_calls": "observed", + } + return report + + +VERIFIED = flow_report("VERIFIED", "VERIFIED", success=True, eligible=True) +BANNER_LIE = flow_report("HALTED", "RECONCILIATION_REQUIRED", success=False, eligible=False) +DUPLICATE_REFERRAL = flow_report("HALTED", "HALTED_BEFORE_EFFECT", success=False, eligible=False) +MISSING_REFERRAL = flow_report("HALTED", "REJECTED_POLICY", success=False, eligible=False) +OPEN_PAUSE = {"status": "pending", "durably_paused": True, "category": "disambiguation"} + + +def test_production_verified_run_is_done_with_local_proof_not_an_error(): + reason = reason_for_run(exit_code=0, report=VERIFIED) + result = build_result( + reason, workflow="enter_referral", run_id="run-1", request_id="ref-1", mode="production" + ) + assert result["outcome"] == "done" + assert result["label"] == "Done and checked" + assert result["proof"] == "local" + assert result["record_changed"] == "yes" + assert result["safe_to_retry"] is False + assert "no signed receipt" in result["what_happened"] + + +def test_uncertain_delivery_is_never_reported_as_nothing_changed(): + attention = {"status": "pending", "durably_paused": True, "category": "postcondition"} + reason = reason_for_run(exit_code=1, report=BANNER_LIE, attention=attention) + result = build_result( + reason, + workflow="enter_referral", + run_id="run-2", + request_id="ref-2", + mode="production", + transaction_outcome="RECONCILIATION_REQUIRED", + ) + assert result["outcome"] == "not_sure_if_saved" + assert result["label"] == "Check the record" + assert result["safe_to_retry"] is False + assert result["record_changed"] == "unknown" + assert result["next_step"] == "check_record" + text = (result["what_happened"] + result["next_action"]).lower() + assert "did not change" not in text + assert "nothing was written" not in text + assert "don't retry" in text + + +@pytest.mark.parametrize("timed_out,report", [(True, None), (True, VERIFIED)]) +def test_timeout_after_start_is_not_sure_if_saved(timed_out, report): + reason = reason_for_run(exit_code=None, report=report, timed_out=timed_out) + assert reason == "timed_out" + assert REASONS[reason].outcome == "not_sure_if_saved" + assert REASONS[reason].safe_to_retry is False + + +def test_halt_before_effect_with_open_pause_needs_review(): + reason = reason_for_run(exit_code=1, report=DUPLICATE_REFERRAL, attention=OPEN_PAUSE) + assert reason == "record_not_confirmed" + spec = REASONS[reason] + assert spec.outcome == "needs_review" + assert spec.safe_to_retry is False + assert spec.record_changed == "no" + + +def test_policy_rejection_with_open_pause_waits_for_the_person(): + attention = {"status": "pending", "durably_paused": True, "category": "identity"} + reason = reason_for_run(exit_code=1, report=MISSING_REFERRAL, attention=attention) + assert REASONS[reason].outcome == "needs_review" + + +def test_policy_rejection_without_a_pause_did_not_run_and_may_retry(): + reason = reason_for_run(exit_code=1, report=MISSING_REFERRAL) + assert reason == "policy_refused" + assert REASONS[reason].outcome == "did_not_run" + assert reason in RETRYABLE_REASONS + + +def test_person_rejected_pause_ends_the_run_without_retry(): + attention = {"status": "rejected", "durably_paused": False, "category": "identity"} + reason = reason_for_run(exit_code=1, report=DUPLICATE_REFERRAL, attention=attention) + assert reason == "stopped_by_person" + assert REASONS[reason].safe_to_retry is False + assert reason not in RETRYABLE_REASONS + + +def test_rejecting_an_uncertain_run_keeps_it_uncertain(): + attention = {"status": "rejected", "durably_paused": False, "category": "effect_escalated"} + reason = reason_for_run(exit_code=1, report=BANNER_LIE, attention=attention) + assert REASONS[reason].outcome == "not_sure_if_saved" + + +def test_coarse_halted_alone_never_says_nothing_was_written(): + coarse = {"success": False, "execution_outcome": "HALTED"} + reason = reason_for_run(exit_code=1, report=coarse) + assert REASONS[reason].outcome == "not_sure_if_saved" + assert REASONS[reason].record_changed == "unknown" + + +def test_inconsistent_verified_report_is_never_done(): + lying = dict(VERIFIED, success=False) + assert reason_for_run(exit_code=0, report=lying) == "result_unreadable" + nonzero = reason_for_run(exit_code=1, report=VERIFIED) + assert REASONS[nonzero].outcome == "not_sure_if_saved" + + +def test_refusal_before_start_did_not_run_with_closed_checks(): + output = ( + " [REFUSE] Certification passed: policy 'clinical-write' has 8 violations\n" + " [REFUSE] Encrypted bundle: plaintext Jane Roe bundle\n" + ) + checks = parse_failed_checks(output) + assert checks == ["not_certified", "not_encrypted"] + reason = reason_for_run(exit_code=2, report=None, refused_before_start=True) + result = build_result( + reason, + workflow="enter_referral", + run_id="run-3", + request_id="ref-3", + mode="production", + failed_checks=checks, + ) + assert result["outcome"] == "did_not_run" + assert result["failed_checks"] == ["not_certified", "not_encrypted"] + assert "Jane Roe" not in json.dumps(result) + + +def test_pregate_refusal_text_maps_to_one_check(): + text = "run REFUSED: Standard and Regulated actuation requires a signed qualification admission." + assert parse_failed_checks(text) == ["no_readiness_test"] + assert parse_failed_checks("something new") == ["other"] + + +def test_process_that_never_started_is_a_platform_error(): + reason = reason_for_run(exit_code=None, report=None, process_started=False) + assert reason == "platform_error" + assert REASONS[reason].outcome == "did_not_run" + + +def test_every_result_validates_against_the_output_schema(): + for reason in REASONS: + result = build_result( + reason, + workflow="w", + run_id="run-x", + request_id="req-x", + mode="sandbox", + sandbox={"case": "normal", "engine": "flow"}, + ) + jsonschema.validate(result, RUN_RESULT_SCHEMA) + + +def test_only_done_claims_a_save_and_only_proven_cases_allow_retry(): + for name, spec in REASONS.items(): + if spec.record_changed == "yes": + assert spec.outcome == "done", name + if spec.safe_to_retry: + assert spec.record_changed == "no", name + assert spec.outcome == "did_not_run", name + if spec.outcome == "not_sure_if_saved": + assert spec.safe_to_retry is False, name + assert spec.next_step == "check_record", name + assert set(OUTCOMES) <= {spec.outcome for spec in REASONS.values()} + + +def test_simulated_sandbox_done_is_labelled_simulated(): + result = build_result( + "saved_and_checked", + workflow="add_triage_note", + run_id="run-4", + request_id="demo-4", + mode="sandbox", + sandbox={"case": "normal", "engine": "simulated"}, + ) + assert result["proof"] == "simulated" + assert result["what_happened"].startswith("Simulated sandbox result") + assert result["sandbox"] == {"case": "normal", "engine": "simulated"} + + +def test_ledger_outcome_keeps_uncertain_results_uncertain(): + assert ledger_outcome("timed_out") == "RECONCILIATION_REQUIRED" + assert ledger_outcome("interrupted") == "RECONCILIATION_REQUIRED" + assert ledger_outcome("record_not_confirmed") == "HALTED_BEFORE_EFFECT" + assert ledger_outcome("saved_and_checked", "VERIFIED") == "VERIFIED" + assert ledger_outcome("invalid_input") is None + + +def test_unknown_reason_falls_back_to_uncertain(): + result = build_result( + "made_up", workflow=None, run_id=None, request_id=None, mode="production" + ) + assert result["reason"] == "result_unreadable" + assert result["outcome"] == "not_sure_if_saved" diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 5f266c3..cde77a7 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -20,7 +20,13 @@ import tomli as tomllib from openadapt_agent import __version__ -from openadapt_agent.copy import IDENTITY_SENTENCE, SKILL_WHEN_TO_USE, THREE_LINE_INSTALL +from openadapt_agent.copy import ( + FIRST_COMMAND, + IDENTITY_SENTENCE, + OUTCOME_RULES, + PREVIEW_COMMAND, + SKILL_WHEN_TO_USE, +) REPO_ROOT = Path(__file__).resolve().parents[1] SERVER_JSON = REPO_ROOT / "server.json" @@ -212,12 +218,18 @@ def test_llms_txt_lists_the_tool_surface() -> None: text = LLMS_TXT.read_text(encoding="utf-8") for token in ( "list_workflows", + "run_workflow(workflow, inputs, request_id", + "get_run(run_id", + "not_sure_if_saved", + "safe_to_retry", + "add_triage_note", + "sandbox_case", + "--mode production", "get_workflow", "get_run_report", "list_needs_attention", "get_attention_item", "run_workflow_<opaque-id>", - "run_local_quickstart", "--tutorial", "--authoring", "observe", @@ -227,9 +239,11 @@ def test_llms_txt_lists_the_tool_surface() -> None: "teach_attention", "escalate_attention", "docs.openadapt.ai", + PREVIEW_COMMAND.split(" -- ", 1)[1], + FIRST_COMMAND.split(" -- ", 1)[1], IDENTITY_SENTENCE, ): - assert token in text + assert token in text, token def _readme_first_paragraph() -> str: @@ -256,37 +270,103 @@ def _readme_first_paragraph() -> str: def test_identity_sentence_is_shared() -> None: # Contracts, not marketing copy: MCP registry namespace via the PyPI # readme, the 100-char identity sentence shared with server.json / llms.txt - # / skill frontmatter, the findable install commands, Seal/unsigned honesty, - # and the ban on naming the skill "computer use". - # MCP registry server.json description maxLength is 100. + # / skill frontmatter, the findable commands, and the ban on naming the + # skill "computer use". MCP registry server.json description maxLength is 100. assert len(IDENTITY_SENTENCE) <= 100 assert _readme_first_paragraph() == IDENTITY_SENTENCE assert IDENTITY_SENTENCE == _server_json()["description"] - assert IDENTITY_SENTENCE in LLMS_TXT.read_text(encoding="utf-8") - skill = REPO_ROOT / "skills" / "openadapt-gui-write" / "SKILL.md" - text = skill.read_text(encoding="utf-8") - assert f'description: "{IDENTITY_SENTENCE}"' in text - assert SKILL_WHEN_TO_USE in text - assert "name: computer-use" not in text.lower() - assert THREE_LINE_INSTALL in README.read_text(encoding="utf-8") - assert "serve --tutorial --allow-run" not in THREE_LINE_INSTALL - assert "openadapt-agent serve --allow-run" in README.read_text(encoding="utf-8") - assert "serve --authoring" in README.read_text(encoding="utf-8") - assert "serve --authoring" in LLMS_TXT.read_text(encoding="utf-8") - assert "authoring connect" in README.read_text(encoding="utf-8") - assert "authoring connect" in LLMS_TXT.read_text(encoding="utf-8") - assert "openadapt connect" in README.read_text(encoding="utf-8") - assert "https://openadapt.ai/start" in README.read_text(encoding="utf-8") - assert "https://openadapt.ai/start" in LLMS_TXT.read_text(encoding="utf-8") - assert "openadapt flow tutorial" in README.read_text(encoding="utf-8") - assert "openadapt quickstart --break-it" in README.read_text(encoding="utf-8") - assert "If the tool returns unsigned success, treat it as failure" in README.read_text( + llms = LLMS_TXT.read_text(encoding="utf-8") + readme = README.read_text(encoding="utf-8") + assert IDENTITY_SENTENCE in llms + skill = (REPO_ROOT / "skills" / "openadapt-gui-write" / "SKILL.md").read_text( encoding="utf-8" ) - assert "Production success without a Seal is failure" in README.read_text( - encoding="utf-8" - ) - assert "name: computer-use" not in README.read_text(encoding="utf-8").lower() + assert f'description: "{IDENTITY_SENTENCE}"' in skill + assert SKILL_WHEN_TO_USE in skill + assert OUTCOME_RULES in skill + assert "name: computer-use" not in skill.lower() + for command in (FIRST_COMMAND, PREVIEW_COMMAND): + assert command in readme + assert command in skill + assert "openadapt-agent serve --allow-run" in readme + assert "serve --authoring" in readme + assert "serve --authoring" in llms + assert "authoring connect" in readme + assert "authoring connect" in llms + assert "openadapt connect" in readme + assert "https://openadapt.ai/start" in readme + assert "https://openadapt.ai/start" in llms + assert "openadapt flow tutorial" in readme + assert "openadapt quickstart --break-it" in readme + assert "name: computer-use" not in readme.lower() + + +def test_readme_leads_with_the_partner_contract() -> None: + readme = README.read_text(encoding="utf-8") + for token in ( + "```mermaid", + "## What changes for your customer's team", + "## What your agent gets back", + "## Three tools", + "## Who builds what", + "list_workflows", + "run_workflow", + "get_run", + "request_id", + "done", + "needs_review", + "not_sure_if_saved", + "did_not_run", + "safe_to_retry", + ): + assert token in readme, token + assert readme.index("## What your agent gets back") < readme.index("## Developer setup") + + +def test_retired_unsafe_claims_never_come_back() -> None: + surfaces = { + "README.md": README.read_text(encoding="utf-8"), + "llms.txt": LLMS_TXT.read_text(encoding="utf-8"), + "SKILL.md": (REPO_ROOT / "skills" / "openadapt-gui-write" / "SKILL.md").read_text( + encoding="utf-8" + ), + "server.json": SERVER_JSON.read_text(encoding="utf-8"), + } + for name, text in surfaces.items(): + for retired in ( + "treat it as failure", + "record did not change", + "Production success without a Seal is failure", + "Unsigned production success is failure", + ): + assert retired not in text, (name, retired) + + +def _without_code(text: str) -> str: + return re.sub(r"```.*?```", "", text, flags=re.DOTALL) + + +def test_business_section_of_readme_uses_plain_words() -> None: + """The partner-facing top avoids engine jargon; engineers get it below.""" + readme = README.read_text(encoding="utf-8") + business = _without_code(readme.split("## Developer setup", 1)[0]) + for jargon in ( + "Seal", + "admission", + "admit", + "governed", + "oracle", + "effect contract", + "RECONCILIATION_REQUIRED", + "VERIFIED", + "HALTED", + "runner", + "MCP", + "substrate", + "fixture", + "\u2014", + ): + assert jargon not in business, jargon def test_mcpb_checker_allows_skill_markdown_not_bundles() -> None: diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index d714d0f..eb1d72c 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -42,6 +42,10 @@ async def list_tools(): names = [t["name"] for t in tools] assert {"list_workflows", "get_run_report", "list_needs_attention"} <= set(names) run_tools = [t for t in tools if t["name"].startswith("run_workflow_")] + contract = next(t for t in tools if t["name"] == "run_workflow") + assert contract["outputSchema"]["required"][0] == "outcome" + assert contract["inputSchema"]["required"] == ["workflow", "inputs", "request_id"] + assert contract["annotations"]["idempotentHint"] is True assert len(run_tools) == 1 run_tool = run_tools[0] assert run_tool["inputSchema"]["properties"]["note"]["type"] == "string" @@ -50,9 +54,8 @@ async def list_tools(): assert "governed" in run_tool["description"] assert run_tool["annotations"]["readOnlyHint"] is False assert run_tool["annotations"]["destructiveHint"] is True - assert run_tool["_meta"] == {"requires_seal": True} - assert "requires_seal: true" in run_tool["description"] - assert "unsigned success" in run_tool["description"] + assert run_tool["_meta"] == {"deprecated": True, "use_instead": "run_workflow"} + assert "Deprecated" in run_tool["description"] list_tool = next(t for t in tools if t["name"] == "list_needs_attention") assert list_tool["annotations"]["readOnlyHint"] is True assert list_tool.get("_meta") is None @@ -71,6 +74,7 @@ async def probe(): names, payload, result = anyio.run(probe) assert names == [ "list_workflows", + "get_run", "get_workflow", "get_run_report", "list_needs_attention", diff --git a/tests/test_runner.py b/tests/test_runner.py index 1d1ce7b..75acd86 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -4,6 +4,7 @@ import json import subprocess +from pathlib import Path import pytest from conftest import FlowCliStub @@ -35,7 +36,10 @@ def test_success_mapping(monkeypatch, runner_config, bundle_dir, success_report) public = outcome.to_dict() assert public["success"] is True assert public["sealed"] is False - assert public["requires_seal"] is True + assert public["requires_seal"] is False + assert public["outcome"] == "done" + assert public["proof"] == "local" + assert public["safe_to_retry"] is False assert public["frames_included"] is False blob = json.dumps(public) assert ".png" not in blob @@ -63,19 +67,44 @@ def test_halt_maps_to_structured_halt_not_success( assert "report.json" in outcome.detail -def test_halted_execution_outcome_tells_the_caller_the_record_did_not_change( +def test_halt_before_effect_says_nothing_was_written( monkeypatch, runner_config, bundle_dir, halt_report ): - halt_report["execution_outcome"] = "HALTED" - halt_report["success"] = False stub = FlowCliStub(exit_code=1, report=halt_report) outcome = _run(monkeypatch, runner_config, stub, bundle_dir=bundle_dir) payload = outcome.to_dict() assert payload["status"] == "halt" assert payload["success"] is False assert payload["execution_outcome"] == "HALTED" - assert payload["message"].startswith("HALTED.") - assert "record did not change" in payload["message"] + assert payload["transaction_outcome"] == "HALTED_BEFORE_EFFECT" + assert payload["outcome"] == "needs_review" + assert "nothing was written" in payload["message"] + + +def test_uncertain_delivery_never_says_the_record_did_not_change( + monkeypatch, runner_config, bundle_dir, halt_report +): + halt_report["transaction_outcome"] = "RECONCILIATION_REQUIRED" + stub = FlowCliStub(exit_code=1, report=halt_report) + outcome = _run(monkeypatch, runner_config, stub, bundle_dir=bundle_dir) + payload = outcome.to_dict() + assert payload["status"] == "halt" + assert payload["outcome"] == "not_sure_if_saved" + assert payload["safe_to_retry"] is False + assert payload["record_changed"] == "unknown" + assert "did not change" not in payload["message"] + assert "nothing was written" not in payload["message"].lower() + assert "Don't retry" in payload["next_action"] + + +def test_coarse_halt_without_transaction_outcome_makes_no_claim( + monkeypatch, runner_config, bundle_dir, halt_report +): + del halt_report["transaction_outcome"] + stub = FlowCliStub(exit_code=1, report=halt_report) + payload = _run(monkeypatch, runner_config, stub, bundle_dir=bundle_dir).to_dict() + assert payload["outcome"] == "not_sure_if_saved" + assert "may or may not be saved" in payload["message"] def test_exit_zero_with_failed_report_is_never_success( @@ -95,6 +124,7 @@ def test_demo_completed_unverified_is_never_agent_success( { "execution_profile": "demo", "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", "production_eligible": False, } ) @@ -106,12 +136,12 @@ def test_demo_completed_unverified_is_never_agent_success( assert outcome.execution_outcome == "COMPLETED_UNVERIFIED" assert outcome.to_dict()["success"] is False assert outcome.to_dict()["execution_outcome"] == "COMPLETED_UNVERIFIED" - assert "completed" in outcome.to_dict()["message"] + assert "finished the steps" in outcome.to_dict()["message"] assert "stopped safely" not in outcome.to_dict()["message"] assert "did not prove VERIFIED success" in outcome.detail -def test_production_eligible_verified_without_a_seal_is_failure( +def test_production_verified_without_a_seal_is_success_with_local_proof( monkeypatch, runner_config, bundle_dir, success_report ): success_report.update( @@ -126,12 +156,14 @@ def test_production_eligible_verified_without_a_seal_is_failure( outcome = _run(monkeypatch, runner_config, stub, bundle_dir=bundle_dir) payload = outcome.to_dict() - assert payload["status"] == "error" - assert payload["success"] is False + assert payload["status"] == "success" + assert payload["success"] is True assert payload["sealed"] is False - assert payload["requires_seal"] is True + assert payload["proof"] == "local" + assert payload["outcome"] == "done" assert payload["execution_outcome"] == "VERIFIED" - assert "Unsigned production success is failure" in payload["message"] + assert "read the record back" in payload["message"] + assert "failure" not in payload["message"] def test_precise_verified_requires_consistent_legacy_success( @@ -157,6 +189,7 @@ def test_precise_failed_report_maps_to_error_even_on_exit_zero( monkeypatch, runner_config, bundle_dir, halt_report ): halt_report["execution_outcome"] = "FAILED" + halt_report["transaction_outcome"] = "FAILED_PLATFORM" stub = FlowCliStub(exit_code=0, report=halt_report) outcome = _run(monkeypatch, runner_config, stub, bundle_dir=bundle_dir) @@ -238,7 +271,8 @@ def test_inconsistent_precise_outcome_uses_the_public_error_message(): public = outcome.to_dict() assert public["success"] is False - assert "trustworthy terminal result" in public["message"] + assert "can trust" in public["message"] + assert public["outcome"] == "not_sure_if_saved" assert "completed" not in public["message"] @@ -259,7 +293,7 @@ def test_exit_two_is_governed_refusal(monkeypatch, runner_config, bundle_dir): def test_timeout_maps_to_timeout(monkeypatch, runner_config, bundle_dir): - def raise_timeout(cmd, capture_output=True, text=True, timeout=None): + def raise_timeout(cmd, capture_output=True, text=True, timeout=None, **kwargs): raise subprocess.TimeoutExpired(cmd, timeout, output=b"partial", stderr=b"") outcome = _run(monkeypatch, runner_config, raise_timeout, bundle_dir=bundle_dir) @@ -269,7 +303,7 @@ def raise_timeout(cmd, capture_output=True, text=True, timeout=None): def test_missing_cli_maps_to_error(monkeypatch, runner_config, bundle_dir): - def raise_missing(cmd, capture_output=True, text=True, timeout=None): + def raise_missing(cmd, capture_output=True, text=True, timeout=None, **kwargs): raise FileNotFoundError(cmd[0]) outcome = _run(monkeypatch, runner_config, raise_missing, bundle_dir=bundle_dir) @@ -329,10 +363,80 @@ def test_operator_fixed_args_forwarded(monkeypatch, tmp_path, bundle_dir, succes @pytest.mark.parametrize("exit_code", [3, 130]) -def test_other_nonzero_exit_is_halt_with_evidence_pointer(exit_code): +def test_other_nonzero_exit_without_report_is_uncertain(exit_code): outcome = classify_outcome("w", exit_code, None, stdout="boom", stderr="") - assert outcome.status == "halt" - assert outcome.to_dict()["success"] is False + assert outcome.status == "error" + public = outcome.to_dict() + assert public["success"] is False + assert public["outcome"] == "not_sure_if_saved" + assert public["safe_to_retry"] is False + + +def test_legacy_success_flag_alone_is_finished_not_checked(legacy_success_report): + outcome = classify_outcome("w", 0, legacy_success_report) + public = outcome.to_dict() + assert public["success"] is False + assert public["outcome"] == "not_sure_if_saved" + assert public["label"] == "Finished, not checked" + + +def test_refusal_carries_closed_failed_checks_not_flow_text(): + stdout = " [REFUSE] Certification passed: Jane Roe chart has 8 violations\n" + outcome = classify_outcome("w", 2, None, stdout=stdout) + public = outcome.to_dict() + assert public["status"] == "refused" + assert public["outcome"] == "did_not_run" + assert public["failed_checks"] == ["not_certified"] + assert "Jane Roe" not in str(public) + + +def test_launch_failure_is_a_platform_error_that_may_retry( + monkeypatch, runner_config, bundle_dir +): + def raise_missing(cmd, capture_output=True, text=True, timeout=None, **kwargs): + raise FileNotFoundError(cmd[0]) + + public = _run(monkeypatch, runner_config, raise_missing, bundle_dir=bundle_dir).to_dict() + assert public["outcome"] == "did_not_run" + assert public["reason"] == "platform_error" + assert public["safe_to_retry"] is True + + +def test_failure_after_flow_made_its_run_dir_is_uncertain_not_retryable( + monkeypatch, runner_config, bundle_dir +): + def fail_after_start(cmd, capture_output=True, text=True, timeout=None, **kwargs): + # Flow created its run directory (so it started), then reading its + # output failed in this process. + Path(cmd[cmd.index("--run-dir") + 1]).mkdir(parents=True) + raise UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte") + + public = _run(monkeypatch, runner_config, fail_after_start, bundle_dir=bundle_dir).to_dict() + assert public["outcome"] == "not_sure_if_saved" + assert public["safe_to_retry"] is False + + +def test_flow_output_is_decoded_without_strict_errors( + monkeypatch, runner_config, bundle_dir, success_report +): + stub = FlowCliStub(exit_code=0, report=success_report) + seen = {} + + def capture(cmd, **kwargs): + seen.update(kwargs) + return stub(cmd) + + _run(monkeypatch, runner_config, capture, bundle_dir=bundle_dir) + assert seen.get("errors") == "replace" + + +def test_pre_allocated_run_id_is_used(monkeypatch, runner_config, bundle_dir, success_report): + stub = FlowCliStub(exit_code=0, report=success_report) + outcome = _run( + monkeypatch, runner_config, stub, bundle_dir=bundle_dir, run_id="run-" + "c" * 24 + ) + assert outcome.run_id == "run-" + "c" * 24 + assert stub.calls[0][stub.calls[0].index("--run-dir") + 1].endswith("run-" + "c" * 24) @pytest.mark.parametrize( diff --git a/tests/test_runs.py b/tests/test_runs.py new file mode 100644 index 0000000..bfd383a --- /dev/null +++ b/tests/test_runs.py @@ -0,0 +1,156 @@ +"""Run records and the request_id at-most-once gate.""" + +from __future__ import annotations + +import json + +import pytest + +from openadapt_agent.contract import build_result +from openadapt_agent.runs import RunStore, pid_alive + + +def finish(store, begin, reason): + result = build_result( + reason, + workflow="w", + run_id=begin.run_id, + request_id="req-0001", + mode="production", + ) + return store.finish(begin.record, result) + + +@pytest.fixture() +def store(tmp_path): + return RunStore(tmp_path / "runs") + + +def test_uses_flow_idempotency_ledger(store): + assert store.ledger_kind == "openadapt-flow" + + +def test_same_request_replays_the_first_result_and_never_starts_twice(store): + first = store.begin(request_id="req-0001", workflow="w", inputs={"a": "1"}, mode="production") + assert first.kind == "new" + finish(store, first, "saved_and_checked") + again = store.begin(request_id="req-0001", workflow="w", inputs={"a": "1"}, mode="production") + assert again.kind == "replay" + assert again.run_id == first.run_id + assert again.record["result"]["outcome"] == "done" + + +def test_a_running_request_replays_instead_of_starting_again(store): + first = store.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + again = store.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + assert again.kind == "replay" + assert again.run_id == first.run_id + + +@pytest.mark.parametrize( + "reason", ["save_not_confirmed", "timed_out", "record_not_confirmed", "stopped_by_person"] +) +def test_uncertain_or_reviewed_results_are_never_retried(store, reason): + first = store.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + finish(store, first, reason) + again = store.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + assert again.kind == "replay" + assert again.run_id == first.run_id + + +def test_proven_no_write_result_allows_a_new_attempt_until_the_limit(tmp_path): + store = RunStore(tmp_path / "runs", max_attempts=2) + first = store.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + finish(store, first, "platform_error") + second = store.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + assert second.kind == "new" + assert second.run_id != first.run_id + finish(store, second, "not_ready_to_run") + third = store.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + assert third.kind == "retry_limit" + assert third.first_run_id == first.run_id + + +def test_reused_request_id_with_different_inputs_is_a_conflict(store): + first = store.begin(request_id="req-0001", workflow="w", inputs={"a": "1"}, mode="production") + conflict = store.begin( + request_id="req-0001", workflow="w", inputs={"a": "2"}, mode="production" + ) + assert conflict.kind == "conflict" + assert conflict.first_run_id == first.run_id + + +def test_records_never_hold_input_values(store, tmp_path): + secret = "Jane Roe MRN 8812345" + first = store.begin(request_id="req-0001", workflow="w", inputs={"a": secret}, mode="production") + finish(store, first, "saved_and_checked") + for path in (tmp_path / "runs" / "openadapt-agent").rglob("*.json"): + assert secret not in path.read_text() + + +def test_two_stores_on_one_directory_share_the_gate(tmp_path): + one = RunStore(tmp_path / "runs") + two = RunStore(tmp_path / "runs") + first = one.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + second = two.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + assert second.kind == "replay" + assert second.run_id == first.run_id + + +def test_a_race_with_different_inputs_is_a_conflict_not_a_replay(tmp_path): + one = RunStore(tmp_path / "runs") + two = RunStore(tmp_path / "runs") + first = one.begin(request_id="req-0001", workflow="w", inputs={"a": "1"}, mode="production") + # The first process reserved and wrote its record but not yet its request + # entry, so the second one only meets the ledger reservation. + for entry in one.requests.glob("*.json"): + entry.unlink() + second = two.begin(request_id="req-0001", workflow="w", inputs={"a": "2"}, mode="production") + assert second.kind == "conflict" + assert second.first_run_id == first.run_id + same = two.begin(request_id="req-0001", workflow="w", inputs={"a": "1"}, mode="production") + assert same.kind == "replay" + assert same.run_id == first.run_id + + +def test_unsafe_run_ids_read_nothing(store): + assert store.read("../etc/passwd") is None + assert store.read("run-" + "z" * 200) is None + + +def test_record_files_are_owner_only(store, tmp_path): + begin = store.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + path = tmp_path / "runs" / "openadapt-agent" / "runs" / f"{begin.run_id}.json" + assert json.loads(path.read_text())["state"] == "running" + assert path.stat().st_mode & 0o077 == 0 + + +def test_pid_alive_detects_this_process_and_rejects_nonsense(): + import os + + assert pid_alive(os.getpid()) is True + assert pid_alive(None) is False + assert pid_alive(-1) is False + + +def test_lost_run_record_resolves_to_an_orphan_not_running_forever(store, tmp_path): + first = store.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + (tmp_path / "runs" / "openadapt-agent" / "runs" / f"{first.run_id}.json").unlink() + again = store.begin(request_id="req-0001", workflow="w", inputs={}, mode="production") + assert again.kind == "replay" + assert again.run_id == first.run_id + assert again.record["state"] == "running" + assert again.record["pid"] is None + + +def test_stale_reservation_without_a_record_becomes_an_orphan(store, monkeypatch): + import openadapt_agent.runs as runs_mod + + run_id = "run-" + "d" * 24 + request_hash = store._request_hash("req-0002") + store._ledger.reserve(f"{request_hash}:1", run_id=run_id) + monkeypatch.setattr(runs_mod, "ORPHAN_GRACE_S", -1.0) + begin = store.begin(request_id="req-0002", workflow="w", inputs={}, mode="production") + assert begin.kind == "replay" + assert begin.run_id == run_id + assert begin.record["pid"] is None diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py new file mode 100644 index 0000000..a36e861 --- /dev/null +++ b/tests/test_sandbox.py @@ -0,0 +1,117 @@ +"""The zero-flag sandbox shows every outcome (simulated engine; no browser). + +tests/test_sandbox_flow.py drives the same cases with real openadapt-flow. +""" + +from __future__ import annotations + +import json + +import jsonschema +import pytest + +from openadapt_agent.bridge import AgentBridge +from openadapt_agent.contract import RUN_RESULT_SCHEMA +from openadapt_agent.runner import RunnerConfig +from openadapt_agent.sandbox import CASES, SANDBOX_WORKFLOW, SandboxEngine + + +@pytest.fixture() +def sandbox_bridge(tmp_path): + runs = tmp_path / "sandbox" + engine = SandboxEngine(runs, engine="simulated") + bridge = AgentBridge(None, RunnerConfig(runs_dir=runs), sandbox=engine, mode="sandbox") + yield bridge + engine.close() + + +def run(bridge, case=None, request_id="demo-0001", note="Synthetic follow-up in 2 weeks"): + inputs = {"note": note} + if case is not None: + inputs["sandbox_case"] = case + return bridge.dispatch( + "run_workflow", + {"workflow": SANDBOX_WORKFLOW, "inputs": inputs, "request_id": request_id}, + ) + + +def test_sandbox_lists_one_workflow_with_cases(sandbox_bridge): + names = [spec.name for spec in sandbox_bridge.list_tool_specs()] + assert names == ["list_workflows", "run_workflow", "get_run"] + listing = sandbox_bridge.dispatch("list_workflows", {}) + assert listing["mode"] == "sandbox" + workflow = listing["workflows"][0] + assert workflow["name"] == SANDBOX_WORKFLOW + assert workflow["inputs"]["required"] == ["note"] + assert workflow["inputs"]["properties"]["sandbox_case"]["enum"] == list(CASES) + assert listing["sandbox"]["engine"] == "simulated" + assert {case["outcome"] for case in listing["sandbox"]["cases"]} == { + "done", + "needs_review", + "not_sure_if_saved", + "did_not_run", + } + + +@pytest.mark.parametrize("case", list(CASES)) +def test_each_case_returns_its_outcome(sandbox_bridge, case): + result = run(sandbox_bridge, case=case, request_id=f"demo-{case}") + jsonschema.validate(result, RUN_RESULT_SCHEMA) + assert result["outcome"] == CASES[case].outcome + assert result["mode"] == "sandbox" + assert result["sandbox"] == {"case": case, "engine": "simulated"} + assert result["what_happened"].startswith("Simulated sandbox result") + if result["outcome"] == "done": + assert result["proof"] == "simulated" + if result["outcome"] == "not_sure_if_saved": + assert result["safe_to_retry"] is False + assert "did not change" not in result["what_happened"] + + +def test_app_offline_says_the_app_is_unreachable(sandbox_bridge): + result = run(sandbox_bridge, case="app_offline", request_id="demo-offline") + assert result["reason"] == "app_unreachable" + assert result["safe_to_retry"] is True + assert "isn't reachable" in result["what_happened"] + + +def test_default_case_is_normal(sandbox_bridge): + assert run(sandbox_bridge)["outcome"] == "done" + + +@pytest.mark.parametrize("note", ["", "x" * 501]) +def test_invalid_note_did_not_run(sandbox_bridge, note): + result = run(sandbox_bridge, note=note) + assert result["outcome"] == "did_not_run" + assert result["reason"] == "invalid_input" + assert result["invalid_inputs"][0]["input"] == "note" + + +def test_same_request_id_replays(sandbox_bridge): + first = run(sandbox_bridge, case="false_saved_banner") + again = run(sandbox_bridge, case="false_saved_banner") + assert again["run_id"] == first["run_id"] + assert again["replayed"] is True + changed = run(sandbox_bridge, case="normal") + assert changed["reason"] == "request_id_conflict" + + +def test_get_run_returns_the_stored_result(sandbox_bridge): + first = run(sandbox_bridge, case="duplicate_record") + fetched = sandbox_bridge.dispatch("get_run", {"run_id": first["run_id"]}) + assert fetched == first + assert json.dumps(fetched) + + +def test_legacy_tools_are_not_offered_in_the_sandbox(sandbox_bridge): + from openadapt_agent.bridge import BridgeError + + with pytest.raises(BridgeError): + sandbox_bridge.dispatch("list_needs_attention", {}) + + +def test_omitted_default_and_explicit_default_are_the_same_request(sandbox_bridge): + first = run(sandbox_bridge, request_id="demo-default") + explicit = run(sandbox_bridge, case="normal", request_id="demo-default") + assert explicit["run_id"] == first["run_id"] + assert explicit["replayed"] is True diff --git a/tests/test_sandbox_flow.py b/tests/test_sandbox_flow.py new file mode 100644 index 0000000..f48a1d4 --- /dev/null +++ b/tests/test_sandbox_flow.py @@ -0,0 +1,117 @@ +"""The sandbox path with real openadapt-flow, a real browser, and MockMed. + +No fakes: Flow records the synthetic workflow once, then each case runs under +the standard profile with Flow's run gate, right-record checks, and an +independent read of MockMed's record store. Opt in with OPENADAPT_AGENT_E2E=1 +(it takes a few minutes and needs the tutorial extra plus Chromium). CI runs +it in the sandbox-e2e job. +""" + +from __future__ import annotations + +import json +import os +from urllib.request import urlopen + +import jsonschema +import pytest + +from openadapt_agent.bridge import AgentBridge +from openadapt_agent.contract import RUN_RESULT_SCHEMA +from openadapt_agent.runner import RunnerConfig +from openadapt_agent.sandbox import SANDBOX_WORKFLOW, SandboxEngine, flow_engine_available + +pytestmark = pytest.mark.skipif( + os.environ.get("OPENADAPT_AGENT_E2E") != "1", + reason="set OPENADAPT_AGENT_E2E=1 to drive MockMed with real openadapt-flow", +) + +NOTE = "Synthetic: recheck blood pressure in 2 weeks" + + +@pytest.fixture(scope="module") +def flow_sandbox(tmp_path_factory): + available, why = flow_engine_available() + if not available: + pytest.skip(f"flow sandbox engine unavailable: {why}") + os.environ.setdefault("DO_NOT_TRACK", "1") + os.environ.setdefault("OPENADAPT_TELEMETRY", "0") + runs = tmp_path_factory.mktemp("flow-sandbox") + engine = SandboxEngine(runs, engine="flow") + bridge = AgentBridge(None, RunnerConfig(runs_dir=runs), sandbox=engine, mode="sandbox") + yield bridge, engine + engine.close() + + +def call(bridge, case, request_id, note=NOTE): + result = bridge.dispatch( + "run_workflow", + { + "workflow": SANDBOX_WORKFLOW, + "inputs": {"note": note, "sandbox_case": case}, + "request_id": request_id, + "wait_seconds": 600, + }, + ) + jsonschema.validate(result, RUN_RESULT_SCHEMA) + return result + + +def store_records(engine): + base = engine._session.base_url.rstrip("/") + with urlopen(f"{base}/api/db", timeout=5) as response: + return json.loads(response.read().decode("utf-8"))["records"] + + +def test_normal_case_saves_the_callers_note_and_checks_it(flow_sandbox): + bridge, engine = flow_sandbox + result = call(bridge, "normal", "e2e-normal") + assert engine.engine == "flow" + assert result["outcome"] == "done", result + assert result["proof"] == "local" + assert result["sandbox"] == {"case": "normal", "engine": "flow"} + assert result["technical"]["transaction_outcome"] == "VERIFIED" + assert result["model_calls"] == 0 + records = store_records(engine) + assert [record["note"] for record in records] == [NOTE] + + again = call(bridge, "normal", "e2e-normal") + assert again["run_id"] == result["run_id"] + assert again["replayed"] is True + assert len(store_records(engine)) == 1 + + +def test_duplicate_record_stops_before_saving(flow_sandbox): + bridge, engine = flow_sandbox + result = call(bridge, "duplicate_record", "e2e-duplicate") + assert result["outcome"] == "needs_review", result + assert result["record_changed"] == "no" + assert result["safe_to_retry"] is False + assert result["technical"]["transaction_outcome"] == "HALTED_BEFORE_EFFECT" + assert store_records(engine) == [] + + +def test_false_saved_banner_is_not_sure_if_saved(flow_sandbox): + bridge, engine = flow_sandbox + result = call(bridge, "false_saved_banner", "e2e-banner") + assert result["outcome"] == "not_sure_if_saved", result + assert result["safe_to_retry"] is False + assert result["technical"]["transaction_outcome"] == "RECONCILIATION_REQUIRED" + assert store_records(engine) == [] + + +def test_timeout_after_save_is_not_sure_even_though_it_saved(flow_sandbox): + bridge, engine = flow_sandbox + result = call(bridge, "timeout_after_save", "e2e-timeout") + assert result["outcome"] == "not_sure_if_saved", result + assert result["safe_to_retry"] is False + # The write landed. Saying "nothing changed" here would be false. + assert [record["note"] for record in store_records(engine)] == [NOTE] + + +def test_app_offline_did_not_run(flow_sandbox): + bridge, _engine = flow_sandbox + result = call(bridge, "app_offline", "e2e-offline") + assert result["outcome"] == "did_not_run", result + assert result["reason"] == "app_unreachable" + assert result["safe_to_retry"] is True diff --git a/tests/test_service.py b/tests/test_service.py new file mode 100644 index 0000000..48dd152 --- /dev/null +++ b/tests/test_service.py @@ -0,0 +1,449 @@ +"""run_workflow / get_run over operator bundles, with a stubbed Flow CLI.""" + +from __future__ import annotations + +import json +import os +import time + +import anyio +import jsonschema +import pytest +from conftest import FlowCliStub + +import openadapt_agent.runner as runner_mod +from openadapt_agent.bridge import AgentBridge, BridgeError +from openadapt_agent.contract import RUN_RESULT_SCHEMA +from openadapt_agent.mcp import build_server +from openadapt_agent.runs import RunStore + + +def wire(model): + """Both SDK generations serialize the same protocol aliases.""" + return model.model_dump(by_alias=True, mode="json") + + +def make_bridge(bundles_root, runner_config, **kwargs): + kwargs.setdefault("allow_run", True) + return AgentBridge(bundles_root, runner_config, **kwargs) + + +def workflow_name(bridge): + return next(iter(bridge.catalog)) + + +def run(bridge, request_id="ref-0001", inputs=None, wait=10, workflow=None): + return bridge.dispatch( + "run_workflow", + { + "workflow": workflow or workflow_name(bridge), + "inputs": {"note": "fresh note"} if inputs is None else inputs, + "request_id": request_id, + "wait_seconds": wait, + }, + ) + + +def test_verified_run_is_done_and_the_same_request_never_runs_twice( + monkeypatch, bundles_root, runner_config, success_report +): + stub = FlowCliStub(exit_code=0, report=success_report) + monkeypatch.setattr(runner_mod.subprocess, "run", stub) + bridge = make_bridge(bundles_root, runner_config) + + first = run(bridge) + assert first["outcome"] == "done" + assert first["proof"] == "local" + assert first["request_id"] == "ref-0001" + assert first["workflow"] == workflow_name(bridge) + jsonschema.validate(first, RUN_RESULT_SCHEMA) + + again = run(bridge) + assert again["run_id"] == first["run_id"] + assert again["replayed"] is True + assert len(stub.calls) == 1 + + +def test_list_workflows_returns_cards_and_legacy_fields(bundles_root, runner_config): + listing = make_bridge(bundles_root, runner_config).dispatch("list_workflows", {}) + item = listing["workflows"][0] + assert item["name"].startswith("workflow_") + assert item["id"] == item["name"] + assert item["purpose"] + assert item["done_means"] + assert item["inputs"]["required"] == ["note"] + assert item["inputs"]["properties"]["note"]["type"] == "string" + assert "Follow-up in 2 weeks" not in json.dumps(listing) + assert listing["how_to_run"].startswith("Call run_workflow") + + +def test_invalid_input_did_not_run_and_leaves_a_retrievable_record( + monkeypatch, bundles_root, runner_config +): + stub = FlowCliStub(exit_code=0) + monkeypatch.setattr(runner_mod.subprocess, "run", stub) + bridge = make_bridge(bundles_root, runner_config) + secret = "Jane Roe MRN 8812345" + result = run(bridge, inputs={"note": 5, "patient": secret}) + assert result["outcome"] == "did_not_run" + assert result["reason"] == "invalid_input" + assert result["safe_to_retry"] is True + assert {"input": "note", "problem": "must be text"} in result["invalid_inputs"] + assert secret not in json.dumps(result) + assert stub.calls == [] + fetched = bridge.dispatch("get_run", {"run_id": result["run_id"]}) + assert fetched == result + # The fixed request may reuse the same request_id. + stub.report = None + stub.exit_code = 2 + retried = run(bridge) + assert retried["reason"] == "not_ready_to_run" + assert len(stub.calls) == 1 + + +def test_unknown_workflow_never_echoes_the_name(bundles_root, runner_config): + bridge = make_bridge(bundles_root, runner_config) + result = run(bridge, workflow="Jane Roe chart") + assert result["reason"] == "unknown_workflow" + assert result["workflow"] is None + assert "Jane Roe" not in json.dumps(result) + + +def test_refusal_is_retryable_with_the_same_request_id( + monkeypatch, bundles_root, runner_config, success_report +): + stub = FlowCliStub(exit_code=2, report=None) + stub.stdout = " [REFUSE] Certification passed: 8 violations\n" + monkeypatch.setattr(runner_mod.subprocess, "run", stub) + bridge = make_bridge(bundles_root, runner_config) + refused = run(bridge) + assert refused["outcome"] == "did_not_run" + assert refused["failed_checks"] == ["not_certified"] + assert bridge.dispatch("get_run_report", {"run_id": refused["run_id"]})["status"] == "refused" + + stub.exit_code = 0 + stub.report = success_report + done = run(bridge) + assert done["outcome"] == "done" + assert done["run_id"] != refused["run_id"] + assert len(stub.calls) == 2 + + +@pytest.mark.parametrize("transaction", ["RECONCILIATION_REQUIRED", "HALTED_BEFORE_EFFECT"]) +def test_uncertain_or_reviewed_runs_are_never_run_again( + monkeypatch, bundles_root, runner_config, halt_report, transaction +): + halt_report["transaction_outcome"] = transaction + stub = FlowCliStub(exit_code=1, report=halt_report) + monkeypatch.setattr(runner_mod.subprocess, "run", stub) + bridge = make_bridge(bundles_root, runner_config) + first = run(bridge) + assert first["safe_to_retry"] is False + assert first["outcome"] in {"not_sure_if_saved", "needs_review"} + again = run(bridge) + assert again["run_id"] == first["run_id"] + assert len(stub.calls) == 1 + + +def test_reused_request_id_with_new_inputs_is_refused( + monkeypatch, bundles_root, runner_config, success_report +): + monkeypatch.setattr( + runner_mod.subprocess, "run", FlowCliStub(exit_code=0, report=success_report) + ) + bridge = make_bridge(bundles_root, runner_config) + first = run(bridge) + conflict = run(bridge, inputs={"note": "a different note"}) + assert conflict["reason"] == "request_id_conflict" + assert conflict["first_run_id"] == first["run_id"] + assert conflict["safe_to_retry"] is False + + +def test_slow_run_returns_running_then_get_run_waits_for_it( + monkeypatch, bundles_root, runner_config, success_report +): + stub = FlowCliStub(exit_code=0, report=success_report) + + def slow(*args, **kwargs): + time.sleep(0.5) + return stub(*args, **kwargs) + + monkeypatch.setattr(runner_mod.subprocess, "run", slow) + bridge = make_bridge(bundles_root, runner_config) + started = run(bridge, wait=0) + assert started["outcome"] == "running" + assert started["next_step"] == "call_get_run" + jsonschema.validate(started, RUN_RESULT_SCHEMA) + finished = bridge.dispatch("get_run", {"run_id": started["run_id"], "wait_seconds": 10}) + assert finished["outcome"] == "done" + + +def test_run_left_by_a_dead_server_reads_as_interrupted(bundles_root, runner_config): + store = RunStore(runner_config.runs_dir) + begin = store.begin(request_id="ref-0009", workflow="w", inputs={}, mode="production") + record = dict(begin.record, pid=2**22 + 12345) + store.write(record) + bridge = make_bridge(bundles_root, runner_config) + result = bridge.dispatch("get_run", {"run_id": begin.run_id, "wait_seconds": 0}) + assert result["outcome"] == "not_sure_if_saved" + assert result["reason"] == "interrupted" + assert result["safe_to_retry"] is False + + +def test_review_resolved_by_a_person_updates_get_run( + monkeypatch, bundles_root, runner_config, halt_report, success_report +): + monkeypatch.setattr(runner_mod.subprocess, "run", FlowCliStub(exit_code=1, report=halt_report)) + bridge = make_bridge(bundles_root, runner_config) + paused = run(bridge) + assert paused["outcome"] == "needs_review" + # A person finishes the step; Flow resumes the durable run and rewrites it. + report_path = runner_config.runs_dir / paused["run_id"] / "report.json" + report_path.write_text(json.dumps(success_report)) + resolved = bridge.dispatch("get_run", {"run_id": paused["run_id"]}) + assert resolved["outcome"] == "done" + assert run(bridge)["outcome"] == "done" + + +def test_continued_pause_never_turns_into_a_retryable_result( + monkeypatch, bundles_root, runner_config, halt_report, success_report +): + """Regression: a reviewed run must never invite a second write. + + When a person continues a pause, Flow clears the pause just before it + replaces the pre-resume report. A get_run in that moment sees no pause + and the old REJECTED_POLICY report. That must not become a retryable + did_not_run, or the same request_id would start a second attempt while + the resumed run writes. + """ + halt_report["transaction_outcome"] = "REJECTED_POLICY" + stub = FlowCliStub(exit_code=1, report=halt_report) + monkeypatch.setattr(runner_mod.subprocess, "run", stub) + bridge = make_bridge(bundles_root, runner_config) + pause = { + "item": {"id": "a1", "status": "pending", "durably_paused": True, "category": "identity"} + } + monkeypatch.setattr(bridge.attended, "for_run_dir", lambda run_dir: pause["item"]) + paused = run(bridge) + assert paused["outcome"] == "needs_review" + + pause["item"] = {"id": "a1", "status": "halted", "durably_paused": False, "category": "identity"} + during = bridge.dispatch("get_run", {"run_id": paused["run_id"]}) + assert during["outcome"] == "needs_review" + assert during["safe_to_retry"] is False + assert run(bridge)["run_id"] == paused["run_id"] + assert len(stub.calls) == 1 + + # The resumed run's report lands, and get_run reports it. + report_path = runner_config.runs_dir / paused["run_id"] / "report.json" + report_path.write_text(json.dumps(success_report)) + pause["item"] = None + assert bridge.dispatch("get_run", {"run_id": paused["run_id"]})["outcome"] == "done" + + +def test_get_run_rejects_unknown_and_unsafe_ids(bundles_root, runner_config): + bridge = make_bridge(bundles_root, runner_config) + with pytest.raises(BridgeError): + bridge.dispatch("get_run", {"run_id": "../secrets"}) + with pytest.raises(BridgeError, match="no run"): + bridge.dispatch("get_run", {"run_id": "run-" + "a" * 24}) + + +def test_read_only_server_has_no_run_workflow_and_creates_nothing(bundles_root, runner_config): + bridge = AgentBridge(bundles_root, runner_config) + assert "run_workflow" not in [spec.name for spec in bridge.list_tool_specs()] + with pytest.raises(BridgeError, match="runs are disabled"): + run(bridge) + bridge.dispatch("list_workflows", {}) + assert not (runner_config.runs_dir / "openadapt-agent").exists() + + +def test_mcp_returns_structured_content_that_the_client_validates( + monkeypatch, bundles_root, runner_config, success_report, mcp_client +): + monkeypatch.setattr( + runner_mod.subprocess, "run", FlowCliStub(exit_code=0, report=success_report) + ) + bridge = make_bridge(bundles_root, runner_config) + server = build_server(bridge) + + async def call(): + async with mcp_client(server) as client: + listing = wire(await client.call_tool("list_workflows", {})) + name = listing["structuredContent"]["workflows"][0]["name"] + result = await client.call_tool( + "run_workflow", + {"workflow": name, "inputs": {"note": "x"}, "request_id": "ref-0042"}, + ) + missing = await client.call_tool( + "run_workflow", {"workflow": name, "inputs": {"note": "x"}} + ) + return wire(result), wire(missing) + + result, missing = anyio.run(call) + assert result["isError"] is False + assert result["structuredContent"]["outcome"] == "done" + assert json.loads(result["content"][0]["text"]) == result["structuredContent"] + assert missing["isError"] is True + assert "request_id" in missing["content"][0]["text"] + + +def test_records_hold_no_input_values(monkeypatch, bundles_root, runner_config, success_report): + monkeypatch.setattr( + runner_mod.subprocess, "run", FlowCliStub(exit_code=0, report=success_report) + ) + bridge = make_bridge(bundles_root, runner_config) + secret = "Jane Roe MRN 8812345" + run(bridge, inputs={"note": secret}) + for root, _dirs, files in os.walk(runner_config.runs_dir / "openadapt-agent"): + for name in files: + if name.endswith(".json"): + assert secret not in open(os.path.join(root, name)).read() + + +def test_attention_lookup_works_under_a_symlinked_runs_directory(monkeypatch, tmp_path): + from openadapt_agent.attended import AttendedBridge + + real = tmp_path / "real" + (real / "runs" / "run-abc").mkdir(parents=True) + (tmp_path / "link").symlink_to(real, target_is_directory=True) + seen = {} + + def fake_attention_item(root, path): + seen["relative"] = path.relative_to(root) + return None + + monkeypatch.setattr("openadapt_flow.console.attention.attention_item", fake_attention_item) + bridge = AttendedBridge(tmp_path / "link" / "runs") + assert bridge.for_run_dir(tmp_path / "link" / "runs" / "run-abc") is None + assert str(seen["relative"]) == "run-abc" + + +def test_failed_attention_lookup_waits_for_a_person_instead_of_inviting_retry(tmp_path): + from openadapt_agent.contract import REASONS, reason_for_run + from openadapt_agent.service import attention_for + + class Broken: + def for_run_dir(self, run_dir): + raise ValueError("boom") + + attention = attention_for(Broken(), tmp_path) + report = { + "success": False, + "execution_outcome": "HALTED", + "transaction_outcome": "REJECTED_POLICY", + "execution_profile": "standard", + "production_eligible": False, + } + reason = reason_for_run(exit_code=1, report=report, attention=attention) + assert REASONS[reason].outcome == "needs_review" + assert REASONS[reason].safe_to_retry is False + + +@pytest.mark.parametrize("request_id", ["", "abc", "has space", "x" * 200, None, 42]) +def test_bad_request_id_is_refused_before_anything_runs( + monkeypatch, bundles_root, runner_config, request_id +): + stub = FlowCliStub(exit_code=0) + monkeypatch.setattr(runner_mod.subprocess, "run", stub) + bridge = make_bridge(bundles_root, runner_config) + with pytest.raises(BridgeError, match="request_id"): + bridge.dispatch( + "run_workflow", + {"workflow": workflow_name(bridge), "inputs": {"note": "x"}, "request_id": request_id}, + ) + assert stub.calls == [] + + +def test_a_run_finishing_during_get_is_never_marked_interrupted( + monkeypatch, bundles_root, runner_config, success_report +): + """Regression: the finish/active race must not overwrite a real result.""" + import threading + + gate = threading.Event() + stub = FlowCliStub(exit_code=0, report=success_report) + + def held(*args, **kwargs): + gate.wait(10) + return stub(*args, **kwargs) + + monkeypatch.setattr(runner_mod.subprocess, "run", held) + bridge = make_bridge(bundles_root, runner_config) + started = run(bridge, wait=0) + service = bridge.service + real_read = service.store.read + + def read_then_finish(run_id): + record = real_read(run_id) + gate.set() + time.sleep(0.5) # let the worker write its result and leave + return record + + monkeypatch.setattr(service.store, "read", read_then_finish) + first = bridge.dispatch("get_run", {"run_id": started["run_id"], "wait_seconds": 0}) + monkeypatch.setattr(service.store, "read", real_read) + assert first["outcome"] in {"running", "done"} + final = bridge.dispatch("get_run", {"run_id": started["run_id"], "wait_seconds": 10}) + assert final["outcome"] == "done" + + +def test_same_request_id_while_a_run_is_starting_waits_instead_of_interrupting( + monkeypatch, bundles_root, runner_config, success_report +): + """Regression: a concurrent replay must not see a new run as abandoned.""" + import threading + + stub = FlowCliStub(exit_code=0, report=success_report) + monkeypatch.setattr(runner_mod.subprocess, "run", stub) + bridge = make_bridge(bundles_root, runner_config) + service = bridge.service + real_start = service._start + starting = threading.Event() + + def slow_start(*args, **kwargs): + starting.set() + time.sleep(0.3) # the moment between reserving and starting the worker + return real_start(*args, **kwargs) + + monkeypatch.setattr(service, "_start", slow_start) + results = {} + first = threading.Thread(target=lambda: results.setdefault("first", run(bridge))) + first.start() + assert starting.wait(5) + second = run(bridge) + first.join(10) + assert results["first"]["outcome"] == "done" + assert second["outcome"] == "done", second + assert second["replayed"] is True + assert second["run_id"] == results["first"]["run_id"] + assert len(stub.calls) == 1 + + +def test_orphaned_run_reads_as_interrupted_through_get_run(bundles_root, runner_config): + bridge = make_bridge(bundles_root, runner_config) + store = bridge.service.store + record = store.new_record(request_id="ref-0010", workflow="w", mode="production") + record["pid"] = None + store.write(record) + result = bridge.dispatch("get_run", {"run_id": record["run_id"], "wait_seconds": 0}) + assert result["reason"] == "interrupted" + + +def test_run_owned_by_another_computer_is_not_overwritten(bundles_root, runner_config): + bridge = make_bridge(bundles_root, runner_config) + store = bridge.service.store + record = store.new_record(request_id="ref-0011", workflow="w", mode="production") + record.update(pid=2**22 + 7, host="another-computer.invalid") + store.write(record) + result = bridge.dispatch("get_run", {"run_id": record["run_id"], "wait_seconds": 0}) + assert result["outcome"] == "running" + assert store.read(record["run_id"])["state"] == "running" + + +def test_attended_mode_with_another_flow_cli_is_refused(bundles_root, capsys): + from openadapt_agent.cli import main + + argv = ["serve", "--mode", "attended", "--bundles", str(bundles_root), "--flow-cli", "x"] + assert main(argv) == 2 + assert "cannot select a different runtime" in capsys.readouterr().err diff --git a/tests/test_skill.py b/tests/test_skill.py index 9de59c9..6a79883 100644 --- a/tests/test_skill.py +++ b/tests/test_skill.py @@ -1,56 +1,118 @@ -"""Skill emission: wraps flow's emitter, appends MCP + halt guidance.""" +"""Agent Skill export keeps recorded values, step text, and secrets out.""" from __future__ import annotations -import re +import json from pathlib import Path -from openadapt_agent.bundles import load_workflow_info -from openadapt_agent.copy import IDENTITY_SENTENCE, SKILL_HONESTY, SKILL_WHEN_TO_USE +from openadapt_agent.cards import CARD_FILENAME +from openadapt_agent.cli import main +from openadapt_agent.copy import OUTCOME_RULES from openadapt_agent.skill import emit_agent_skill -GOLDEN = Path(__file__).parent / "golden" / "skill_appendix.md" +SECRET_VALUES = ("Follow-up in 2 weeks", "Open the patient chart", "Type the triage note") -def test_emit_skill_wraps_flow_and_appends_guidance(bundle_dir, tmp_path): +def frontmatter(text: str) -> dict[str, str]: + block = text.split("---", 2)[1] + fields = {} + for line in block.strip().splitlines(): + key, value = line.split(":", 1) + fields[key.strip()] = value.strip() + return fields + + +def test_skill_has_no_recorded_values_intents_or_bundle(bundle_dir, tmp_path): skill_dir = emit_agent_skill(bundle_dir, tmp_path / "skills") - assert skill_dir.name == "demo-triage" # flow's slug convention text = (skill_dir / "SKILL.md").read_text() + for value in SECRET_VALUES: + assert value not in text + assert "Demo Triage" not in text + assert "replay" not in text + assert str(bundle_dir) not in text + assert not (skill_dir / "bundle").exists() + assert "`note`" in text + assert "run_workflow" in text + assert "not_sure_if_saved" in text + assert OUTCOME_RULES in text + assert "name: computer-use" not in text.lower() - # flow-emitted content is intact (frontmatter + CLI usage). - assert text.startswith("---\nname: demo-triage\n") - assert "openadapt-flow replay bundle" in text - # Bundle copied alongside (flow's portability contract). - assert (skill_dir / "bundle" / "workflow.json").is_file() - # Our appendix is present, exactly as the golden file specifies. - golden = GOLDEN.read_text() - normalized = re.sub( - r"run_workflow_[0-9a-f]{24}", - "run_<opaque-workflow-id>", - text, - ) - assert normalized.endswith(golden) +def test_typed_password_step_never_reaches_the_skill(tmp_path): + from openadapt_flow.ir import ActionKind, Step, Workflow + + bundle = tmp_path / "bundles" / "login" + Workflow( + name="Login and note", + params={"note": "Recorded note"}, + steps=[ + Step(id="s1", intent="type 'mockmed-demo-pass'", action=ActionKind.TYPE, + text="mockmed-demo-pass"), + Step(id="s2", intent="type the note", action=ActionKind.TYPE, + text="Recorded note", param="note"), + ], + ).save(bundle) + text = (emit_agent_skill(bundle, tmp_path / "skills") / "SKILL.md").read_text() + assert "mockmed-demo-pass" not in text + assert "Recorded note" not in text + + +def test_each_skill_gets_its_own_description(tmp_path): + from openadapt_flow.ir import ActionKind, Step, Workflow + + descriptions = set() + for name, param in (("one", "note"), ("two", "referral_reason")): + bundle = tmp_path / "bundles" / name + Workflow( + name=name, + params={param: "x"}, + steps=[Step(id="s1", intent="click", action=ActionKind.CLICK)], + ).save(bundle) + skill = emit_agent_skill(bundle, tmp_path / "skills") + descriptions.add(frontmatter((skill / "SKILL.md").read_text())["description"]) + assert len(descriptions) == 2 -def test_appendix_names_the_mcp_tool_and_params(bundle_dir, tmp_path): +def test_card_names_and_describes_the_skill(bundle_dir, tmp_path): + (bundle_dir / CARD_FILENAME).write_text( + json.dumps( + { + "name": "add_triage_note", + "purpose": "Add a triage note to a patient's chart.", + "done_means": "The note shows on the chart's encounter list.", + "inputs": {"note": {"description": "The note text.", "maxLength": 500}}, + } + ) + ) skill_dir = emit_agent_skill(bundle_dir, tmp_path / "skills") + assert skill_dir.name == "add-triage-note" text = (skill_dir / "SKILL.md").read_text() - public_id = load_workflow_info(skill_dir / "bundle").public_id - assert f"run_{public_id}" in text - assert "`note`" in text - assert "every declared parameter is required" in text - assert "Follow-up in 2 weeks" not in text.split("## Invoking via MCP", 1)[1] - # Honesty requirements: halt is never success; refusal executes nothing. - assert "None is a verified success" in text - assert "NOTHING was executed" in text - assert "continue_attention" in text - assert "never performs that completed action again" in text - assert "reject_attention" in text - assert "Earlier run actions may have effects" in text - assert SKILL_WHEN_TO_USE in text - assert SKILL_HONESTY in text - assert "If the tool returns unsigned success, treat it as failure" in text - assert IDENTITY_SENTENCE in text - assert "name: computer-use" not in text.lower() - assert text.split("---", 2)[1].count(IDENTITY_SENTENCE) == 1 + fields = frontmatter(text) + assert fields["name"] == "add-triage-note" + assert "Add a triage note" in fields["description"] + assert "up to 500 characters" in text + assert "The note shows on the chart's encounter list." in text + + +def test_include_bundle_is_explicit_and_labelled(bundle_dir, tmp_path): + skill_dir = emit_agent_skill(bundle_dir, tmp_path / "skills", include_bundle=True) + assert (skill_dir / "bundle" / "workflow.json").is_file() + assert "protected workflow data" in (skill_dir / "SKILL.md").read_text() + + +def test_cli_emit_skill(bundle_dir, tmp_path, capsys): + assert main(["emit-skill", str(bundle_dir), "--out", str(tmp_path / "out")]) == 0 + assert "Wrote Agent Skill folder" in capsys.readouterr().out + broken = tmp_path / "broken" + broken.mkdir() + (broken / "workflow.json").write_text("{not json") + assert main(["emit-skill", str(broken), "--out", str(tmp_path / "out")]) == 2 + + +def test_skill_folder_is_valid_markdown_with_frontmatter(bundle_dir, tmp_path): + text = (emit_agent_skill(bundle_dir, tmp_path / "s") / "SKILL.md").read_text() + assert text.startswith("---\nname: ") + fields = frontmatter(text) + assert len(fields["name"]) <= 64 + assert len(json.loads(fields["description"])) <= 1024 + assert Path(tmp_path / "s").is_dir() diff --git a/tests/test_tutorial.py b/tests/test_tutorial.py index 27418d7..7710a2f 100644 --- a/tests/test_tutorial.py +++ b/tests/test_tutorial.py @@ -55,3 +55,40 @@ def fake_compile(recording_dir, bundle_dir, **kwargs): assert "phi" not in yaml_text.lower() session.close() assert stopped["n"] == 1 + + +def test_bind_param_effects_reads_the_callers_value(tmp_path: Path) -> None: + import pytest + + effect_module = pytest.importorskip("openadapt_flow.runtime.effects.effect") + from openadapt_agent.tutorial import bind_param_effects + + try: + effect = effect_module.Effect( + kind=effect_module.EffectKind.FIELD_EQUALS, + match={"type": effect_module.ValueExpr(literal="Triage")}, + field="note", + value=effect_module.ValueExpr(literal="Synthetic follow-up in two weeks"), + risk="irreversible", + ) + except Exception as exc: # older Flow effect model + pytest.skip(f"effect model differs on this Flow: {type(exc).__name__}") + bundle = tmp_path / "bundle" + Workflow( + name="local-quickstart", + params={"note": "Synthetic follow-up in two weeks"}, + steps=[ + Step( + id="save", + intent="Save the synthetic note", + action=ActionKind.CLICK, + effects=[effect], + ) + ], + ).save(bundle) + workflow = bind_param_effects(bundle) + bound = workflow.steps[0].effects[0].value + assert bound.param == "note" + assert bound.literal is None + # The re-sealed bundle still loads, and binding twice changes nothing. + assert bind_param_effects(bundle).steps[0].effects[0].value.param == "note" diff --git a/uv.lock b/uv.lock index 3b8eca9..7f05dde 100644 --- a/uv.lock +++ b/uv.lock @@ -509,7 +509,7 @@ requires-dist = [ { name = "jsonschema", specifier = ">=4.20.0" }, { name = "mcp", specifier = ">=1.28,<3" }, { name = "openadapt-flow", specifier = ">=1.26.0,<2" }, - { name = "openadapt-flow", extras = ["browser"], marker = "extra == 'tutorial'", specifier = ">=1.26.0,<2" }, + { name = "openadapt-flow", extras = ["browser"], marker = "extra == 'tutorial'", specifier = ">=1.29.0,<2" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4.0" }, ]