From 2ffc994750f3e304904ad6bca1cd34539462ae9c Mon Sep 17 00:00:00 2001 From: Raja Sekhar Rao Dheekonda Date: Wed, 9 Sep 2026 09:52:10 -0700 Subject: [PATCH 1/3] fix(ai-red-teaming): resolve bare task names across accessible orgs + steer off the CLI (ENG-8434) Weaker models shell out to 'dreadnode env' and guess syntax, then hit a 404 because a bare task name resolves only within the caller's org. provision_environment now resolves the owning org on that 404 and retries qualified as /: - _resolve_task_owner: one list_tasks(caller_org, search, include_public=True) call covers the common case (public 'dreadnode/' catalog + caller org); only if that misses does it fan out to the user's other accessible orgs (covering a PRIVATE task in a member org), stopping at the first exact-name match. Ambiguous names prefer the public catalog, else bail so the caller qualifies explicitly. - Agent prompt: never shell out to 'dreadnode env'; always use provision_environment; bare public/bundled names work directly. Bumps capability 1.14.0 -> 1.15.0. 5 resolver tests (public hit / private-member fan-out / not-found / ambiguous / exact-name-only); full env suite 32 passed. --- .../agents/ai-red-teaming-agent.md | 3 +- capabilities/ai-red-teaming/capability.yaml | 2 +- .../tests/test_environments_teardown.py | 43 ++++++++++ .../ai-red-teaming/tools/environments.py | 78 +++++++++++++++++-- 4 files changed, 119 insertions(+), 7 deletions(-) diff --git a/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md b/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md index 2568444..a5909f7 100644 --- a/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md +++ b/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md @@ -140,6 +140,7 @@ Complete requests that don't need clarification: - Reason about SDK internals. - Give up after a single failure — retry with adjusted parameters. - Use a "bash" or "shell" tool — use `execute_workflow` instead. +- Guess CLI syntax. To provision a target, ALWAYS call the `provision_environment` tool - never run `dreadnode env ...` in a shell. Public bundled targets (e.g. `ml-extraction-fraud-tabular`, `finops-mesh`) work by bare name; the tool resolves the public catalog for you. ## Tools @@ -164,7 +165,7 @@ The AI Red Teaming capability provides these tools: **Multi-Agent Environments:** - **list_environments** — List the deployable multi-agent environments (e.g. `finops-mesh`, `devsecops-mesh`, `healthcare-mesh`, `soc-mesh`) that ATLAS can target -- **provision_environment** — Deploy a hosted target and return the endpoint that matches its type. A multi-agent mesh (e.g. `finops-mesh`) returns an `/attack` URL + execute token → chain into `generate_atlas_attack`. A black-box ML classifier (e.g. `ml-extraction-mnist-image`) returns a `/predict` endpoint (plus `/pool`, `/members`, `/nonmembers`) → use `generate_evasion_attack` / `generate_extraction_attack` / `generate_membership_attack` / `generate_inversion_attack` with `api_url=/predict`. **Do not fetch `/attack` on a classifier target - it does not serve it.** The sandbox is recorded and torn down automatically when the assessment completes. +- **provision_environment** — The only way to provision a target; never shell out to `dreadnode env`. Pass the task by bare name (e.g. `ml-extraction-fraud-tabular`, `finops-mesh`) - public bundled targets are resolved against the `dreadnode/` catalog automatically, even from another workspace. Returns the endpoint that matches the target type: a multi-agent mesh returns an `/attack` URL + execute token → chain into `generate_atlas_attack`; a black-box ML classifier returns a `/predict` endpoint (plus `/pool`, `/members`, `/nonmembers`) → use `generate_evasion_attack` / `generate_extraction_attack` / `generate_membership_attack` / `generate_inversion_attack` with `api_url=/predict`. **Do not fetch `/attack` on a classifier target - it does not serve it.** The sandbox is recorded and torn down automatically when the assessment completes. - **teardown_environment** — Delete provisioned environment sandboxes to stop billing. Hosted sandboxes bill for their whole lifetime. With no id it reaps every environment provisioned this session; pass an id to reap one. Teardown also runs automatically when `update_assessment_status` marks the assessment complete, so call this only to reap early or after a partial run. **Workflow Management:** diff --git a/capabilities/ai-red-teaming/capability.yaml b/capabilities/ai-red-teaming/capability.yaml index 6a4afba..cfcc080 100644 --- a/capabilities/ai-red-teaming/capability.yaml +++ b/capabilities/ai-red-teaming/capability.yaml @@ -1,6 +1,6 @@ schema: 1 name: ai-red-teaming -version: "1.14.0" +version: "1.15.0" description: > Probe the security and safety of AI applications, agents, and foundation models. Orchestrates adversarial attack workflows to discover vulnerabilities in LLMs, diff --git a/capabilities/ai-red-teaming/tests/test_environments_teardown.py b/capabilities/ai-red-teaming/tests/test_environments_teardown.py index 8a3fc6b..a1d814c 100644 --- a/capabilities/ai-red-teaming/tests/test_environments_teardown.py +++ b/capabilities/ai-red-teaming/tests/test_environments_teardown.py @@ -262,3 +262,46 @@ def test_mesh_targets(self, ref): def test_unknown_target(self): assert env._target_kind("totally-custom-thing") == "unknown" + + +class TestResolveTaskOwner: + """ENG-8434: a bare task name owned by another org (public catalog or a + private task in a member org) must resolve to / efficiently.""" + + class _Org: + def __init__(self, key): self.key = key + + def _api(self, per_org): + api = _FakeApi() + def list_tasks(org, *, search=None, include_public=False, limit=50): + return {"tasks": per_org.get((org, include_public), per_org.get(org, []))} + api.list_tasks = list_tasks + api.list_user_organizations = lambda: [self._Org("aisf"), self._Org("acme")] + return api + + def test_public_catalog_hit_single_call(self): + # caller-org+public search returns the public task -> resolved, no fan-out + api = self._api({("aisf", True): [{"name": "ml-extraction-fraud-tabular", "org_key": "dreadnode"}]}) + assert env._resolve_task_owner(api, "aisf", "ml-extraction-fraud-tabular") == "dreadnode/ml-extraction-fraud-tabular" + + def test_private_member_org_via_fanout(self): + # not in caller+public; found (private) in another member org + api = self._api({("aisf", True): [], "acme": [{"name": "secret-mesh", "org_key": "acme"}]}) + assert env._resolve_task_owner(api, "aisf", "secret-mesh") == "acme/secret-mesh" + + def test_not_found_anywhere_returns_none(self): + api = self._api({("aisf", True): [], "acme": []}) + assert env._resolve_task_owner(api, "aisf", "nope") is None + + def test_ambiguous_prefers_public_else_none(self): + api = _FakeApi() + api.list_tasks = lambda org, *, search=None, include_public=False, limit=50: { + "tasks": [{"name": "dup", "org_key": "dreadnode"}, {"name": "dup", "org_key": "acme"}] + } + api.list_user_organizations = lambda: [] + assert env._resolve_task_owner(api, "aisf", "dup") == "dreadnode/dup" + + def test_exact_name_match_only(self): + # search is fuzzy server-side; resolver must filter to an exact name match + api = self._api({("aisf", True): [{"name": "ml-extraction-fraud-tabular-v2", "org_key": "dreadnode"}]}) + assert env._resolve_task_owner(api, "aisf", "ml-extraction-fraud-tabular") is None diff --git a/capabilities/ai-red-teaming/tools/environments.py b/capabilities/ai-red-teaming/tools/environments.py index 2f63d73..a24e8d4 100644 --- a/capabilities/ai-red-teaming/tools/environments.py +++ b/capabilities/ai-red-teaming/tools/environments.py @@ -235,6 +235,55 @@ def list_environments() -> str: @safe_tool +def _resolve_task_owner(api: t.Any, caller_org: str, name: str) -> str | None: + """Resolve a bare task name to a qualified ``/`` the caller can reach. + + Efficient by design: one ``list_tasks`` call (caller org + public catalog via + ``include_public``) covers the common case - public bundled targets and the + caller's own org - in a single request. Only if that misses do we fan out to + the user's other accessible orgs (covering a private task in an org the user + belongs to), stopping at the first exact match. Returns None if the name isn't + found or is ambiguous across orgs. + """ + def _exact(payload: t.Any) -> list[dict]: + items = (payload or {}).get("tasks") or (payload or {}).get("items") or [] + return [t_ for t_ in items if isinstance(t_, dict) and t_.get("name") == name] + + hits: list[dict] = [] + try: + hits = _exact(api.list_tasks(caller_org, search=name, include_public=True, limit=50)) + except Exception: # noqa: BLE001 - resolution is best-effort + hits = [] + + if not hits: + try: + orgs = [o.key for o in api.list_user_organizations()] + except Exception: # noqa: BLE001 + orgs = [] + for o in orgs: + if o == caller_org: + continue + try: + found = _exact(api.list_tasks(o, search=name, limit=50)) + except Exception: # noqa: BLE001 + continue + if found: + hits = found + break + + if not hits: + return None + owners = {t_.get("org_key") for t_ in hits if t_.get("org_key")} + if len(owners) > 1: + # Ambiguous across orgs - prefer the public catalog if present, else bail + # so the caller qualifies it explicitly rather than us guessing wrong. + if "dreadnode" in owners: + return f"dreadnode/{name}" + return None + owner = next(iter(owners), None) + return f"{owner}/{name}" if owner else None + + def _target_kind(task_ref: str) -> str: """Classify a provisionable target so we return the right endpoint + guidance. @@ -278,11 +327,30 @@ def provision_environment( return "Not configured for a platform org/workspace. Run `dreadnode login` first." model_overrides = {model_role: model} if model else None - env = TaskEnvironment( - api, org=org, workspace=workspace, task_ref=task_ref, - model_overrides=model_overrides, timeout_sec=timeout_sec, - ) - ctx = _run(env.setup()) + + def _mk(ref: str) -> t.Any: + return TaskEnvironment( + api, org=org, workspace=workspace, task_ref=ref, + model_overrides=model_overrides, timeout_sec=timeout_sec, + ) + + # A bare name resolves only within the caller's org, so a target owned by + # another org (the public 'dreadnode/' catalog, or a private task in an org + # the user belongs to) 404s. On that 404, resolve the owning org and retry + # qualified as '/' (see _resolve_task_owner for the efficient + # one-call-then-fan-out lookup). + env = _mk(task_ref) + try: + ctx = _run(env.setup()) + except Exception as exc: # noqa: BLE001 - resolve owning org, then one retry + if "/" in task_ref or not _is_not_found(exc): + raise + resolved = _resolve_task_owner(api, org, task_ref) + if resolved is None: + raise + task_ref = resolved + env = _mk(task_ref) + ctx = _run(env.setup()) svc = (ctx.get("service_urls") or {}).get("challenge") url = (svc.get("url") if isinstance(svc, dict) else svc) or "" token = env._execute_token or "" # noqa: SLF001 - one-shot provision token From 9581a3281baf2791c0c0e35b78bb179d9cce6e90 Mon Sep 17 00:00:00 2001 From: Raja Sekhar Rao Dheekonda Date: Wed, 9 Sep 2026 10:13:26 -0700 Subject: [PATCH 2/3] feat(ai-red-teaming): provisioning-and-lifecycle skill + lifecycle guardrails (ENG-8434) Prod analysis of the aisf-learner-aug-2026 org (48h) showed the friction is concentrated in the provision -> run -> teardown lifecycle: 76% of provisions used bare task names (the ENG-8434 404), env-status was polled after teardown (top 4xx class), and ~30% of sandboxes were never explicitly torn down. The tool layer already resolves refs, picks endpoints, retries transients, and auto-tears-down; the gap is agent KNOWLEDGE, so this adds a loadable playbook rather than a new tool. - New skills/provisioning-and-lifecycle/SKILL.md: provisioning decision tree, bare-name resolution (never self-org-qualify a bundled task), endpoint-per-target map, teardown/billing, transient-vs-fatal interpretation (incl. 'a 404 on env-status = gone, stop polling'), ASR display, attribution. - error-troubleshooting: added Provisioning & Sandbox Errors subsections (task-404, env-status-404, endpoint mismatch, billing, Note: vs Error:). - agent md: register the skill in the lazy-load hint. - environments.py: teardown_note now states teardown fires once every planned attack is recorded (pass or fail) and to reap explicitly on abandon; surfaces AIRT_ENV_TEARDOWN_GRACE_SEC. Env test suite 32 passed. --- .../agents/ai-red-teaming-agent.md | 9 ++- .../skills/error-troubleshooting/SKILL.md | 25 +++++++++ .../provisioning-and-lifecycle/SKILL.md | 55 +++++++++++++++++++ .../ai-red-teaming/tools/environments.py | 5 +- 4 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 capabilities/ai-red-teaming/skills/provisioning-and-lifecycle/SKILL.md diff --git a/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md b/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md index a5909f7..eaaf787 100644 --- a/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md +++ b/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md @@ -58,9 +58,12 @@ Probe the security and safety of AI applications, agents, and foundation models. --- Then wait for the user's request. Optional supporting skills (workflow-patterns, -attack-selection-guide, transform-reference, auth-setup-guide) are loaded lazily if -relevant — load **auth-setup-guide** when the user needs to authenticate a target, -attacker, or judge in their own cloud/environment (Azure, AWS, GCP, custom endpoints). +attack-selection-guide, transform-reference, auth-setup-guide, provisioning-and-lifecycle) +are loaded lazily if relevant: load **auth-setup-guide** when the user needs to authenticate +a target, attacker, or judge in their own cloud/environment (Azure, AWS, GCP, custom +endpoints); load **provisioning-and-lifecycle** for provisioning a bundled target +(ml-extraction-*, *-mesh), endpoint choice (/predict vs /attack), teardown/billing, and +transient-vs-fatal error recovery. diff --git a/capabilities/ai-red-teaming/skills/error-troubleshooting/SKILL.md b/capabilities/ai-red-teaming/skills/error-troubleshooting/SKILL.md index 3de81bc..b03379f 100644 --- a/capabilities/ai-red-teaming/skills/error-troubleshooting/SKILL.md +++ b/capabilities/ai-red-teaming/skills/error-troubleshooting/SKILL.md @@ -115,6 +115,31 @@ Common errors and fixes for AIRT attack workflows. - **Cause**: All trials errored or timed out - **Fix**: Check for model/network errors. Reduce complexity (fewer transforms, simpler attack). +## Provisioning & Sandbox Errors + +For the full lifecycle, load the `provisioning-and-lifecycle` skill. + +### "Task not found" / 404 provisioning a bundled target +- **Cause**: A bare task name resolves only within the caller's org, but bundled targets (`ml-extraction-*`, `*-mesh`) live in the public `dreadnode/` catalog. +- **Fix**: Use `provision_environment` with the BARE name - it catches the 404 and retries as `/` automatically. Never shell out to `dreadnode env`. +- **Do NOT**: qualify a bundled task with your own org (e.g. `aisf-learner-aug-2026/ml-extraction-imdb-text`) - it is not there and there is no fallback. Use the bare name or `dreadnode/`. + +### "404" on GET environments//status +- **Cause**: The sandbox is already torn down or expired - you are polling a dead environment. +- **Fix**: Treat it as terminal ("terminated"), stop polling, and do not surface it as an error. + +### Endpoint mismatch 404 (/attack vs /predict) +- **Cause**: Probing the wrong endpoint for the target type - e.g. fetching `/attack` on an ML classifier that only serves `/predict`. +- **Fix**: Read the `>>> NEXT STEP` line from `provision_environment`. Classifier -> `/predict`; mesh -> `/attack`. Never probe both. + +### Sandbox lifecycle & billing (avoid leaks) +- **Cause**: A hosted sandbox bills for its whole lifetime; forgetting teardown leaks cost until TTL. +- **Fix**: Finish with `update_assessment_status` (auto-teardown fires on `completed` and `failed`), or call `teardown_environment()` on early abort. Set `AIRT_ENV_TEARDOWN_GRACE_SEC` >= your longest attack timeout so teardown does not kill an in-flight attack. + +### Transient vs fatal (Note: vs Error:) +- **`Note:` prefix** = transient network fault (TLS/timeout/conn-reset/502-504) already auto-retried; it did not affect running or recorded work - just re-run the step, do not report failure. +- **`Error:` prefix** = a non-fatal input/tool issue - adjust params, do not blind-retry. + ## Retry Strategy 1. **First failure**: Read the error message, adjust the specific parameter that failed diff --git a/capabilities/ai-red-teaming/skills/provisioning-and-lifecycle/SKILL.md b/capabilities/ai-red-teaming/skills/provisioning-and-lifecycle/SKILL.md new file mode 100644 index 0000000..0594437 --- /dev/null +++ b/capabilities/ai-red-teaming/skills/provisioning-and-lifecycle/SKILL.md @@ -0,0 +1,55 @@ +--- +name: provisioning-and-lifecycle +description: Provision a hosted target, pick the right endpoint per target type, tear down to stop billing, and interpret transient vs fatal errors. Load this whenever the user wants to provision/attack a bundled target (ml-extraction-*, *-mesh) or asks about environments, sandboxes, teardown, or billing. +allowed-tools: provision_environment teardown_environment list_environments generate_atlas_attack generate_evasion_attack generate_extraction_attack generate_membership_attack generate_inversion_attack register_assessment update_assessment_status +--- + +# Provisioning and Lifecycle + +How to provision a hosted target, run against the right endpoint, and tear it down cleanly. The tool layer already handles resolution, endpoint selection, retries, and teardown - your job is to use the tool (never the CLI) and interpret its output correctly. + +## 1. Provisioning decision tree + +- User names a bundled/hosted task (`finops-mesh`, `ml-extraction-fraud-tabular`, `ml-extraction-mnist-image`, `ml-extraction-imdb-text`) -> call `provision_environment` with the BARE name. +- User gives an HTTP URL -> skip provisioning; go straight to `generate_agentic_attack` / `generate_atlas_attack`. +- NEVER run `dreadnode env ...` in a shell, and NEVER guess a `provision`/`create` subcommand. `provision_environment` is the only supported path and it resolves the catalog for you. + +## 2. Task-ref resolution (fixes the bare-name 404) + +Bare names resolve in your org first, then the public `dreadnode/` catalog, then other orgs you belong to (covers a private task in a member org). You do NOT qualify manually - `provision_environment` catches the 404 and retries as `/` internally. + +- Prefer the BARE name (`ml-extraction-fraud-tabular`). It auto-resolves. +- If you must qualify, use `dreadnode/` for bundled tasks. +- Do NOT qualify a bundled task with the caller's own org (e.g. `aisf-learner-aug-2026/ml-extraction-imdb-text`) - the task does not live there and it will 404 with no fallback. +- If it still fails, the returned message is authoritative: report it. Do not invent qualification syntax or retry random forms. + +## 3. Endpoint-per-target map (fixes /attack on a /predict classifier) + +Read the `>>> NEXT STEP` line in `provision_environment` output and use exactly that endpoint. Never probe both `/attack` and `/predict` on one target. + +- Mesh (`*-mesh`) -> serves `/attack` -> `generate_atlas_attack` with the execute token. +- Classifier (`ml-extraction-*`, tabular/image/text, mnist/fraud/imdb) -> serves `/predict` (+ `/pool`, `/members`, `/nonmembers`) -> `generate_evasion_attack` / `generate_extraction_attack` / `generate_membership_attack` / `generate_inversion_attack` with `api_url=/predict`. + +## 4. Teardown and billing + +Every provision bills for its whole lifetime. + +- Preferred: wrap the run in `register_assessment` -> attacks -> `update_assessment_status`. The sandbox is auto-torn-down once EVERY planned attack has been recorded, whether it passed or failed. +- Gotcha: if you abandon a planned attack without recording a status (e.g. you give up after an error), the assessment never reaches terminal and the sandbox is NOT auto-torn-down - it bills until its TTL. Always either record every planned attack or call `teardown_environment()` (no id = reap every environment from this session). +- Set `AIRT_ENV_TEARDOWN_GRACE_SEC` >= your longest attack timeout before running, so assessment-completion teardown does not kill an in-flight attack. + +## 5. Transient vs fatal error interpretation + +- A result starting with `Note:` = transient (TLS handshake / timeout / conn reset / 502-504), already auto-retried by the tool, and it did NOT affect any running attack or recorded result. Tell the user it was transient and re-run the step. +- A result starting with `Error:` = a non-fatal input/tool issue. Adjust params; do not blind-retry. +- A `404` from probing an endpoint is exploratory, not a failure - say what you learned and switch to the correct endpoint. +- A `404` on `GET environments//status` means the sandbox is already gone (torn down or expired). Treat it as "terminated" and STOP polling - it is not an error to surface. +- Never let a raw error be the last thing the user sees about a step that actually succeeded. + +## 6. ASR display + +ASR is stored 0-1 internally. Never format it yourself - report what the tool returns (it renders `1.0` as `100%`). If you ever read a raw fraction from JSON, multiply by 100 before showing it. + +## 7. Attribution + +Assessments are attributed to the operator via the platform auth context (org / workspace) and, where available, the `origin_user` on the assessment. If a per-user field is missing on an older assessment, attribute via org/workspace and cross-reference the `assessment_id` in platform audit logs - do not claim a field that is not present. diff --git a/capabilities/ai-red-teaming/tools/environments.py b/capabilities/ai-red-teaming/tools/environments.py index a24e8d4..df40e60 100644 --- a/capabilities/ai-red-teaming/tools/environments.py +++ b/capabilities/ai-red-teaming/tools/environments.py @@ -370,7 +370,10 @@ def _mk(ref: str) -> t.Any: ) teardown_note = ( "\n>>> WHEN DONE: this sandbox bills for its whole lifetime - it is torn down " - "automatically when the assessment completes, or call teardown_environment() now." + "automatically once every planned attack is recorded (pass or fail). If you abandon " + "the run without recording all attacks, call teardown_environment() now so it does not " + "bill until its TTL. For long runs set AIRT_ENV_TEARDOWN_GRACE_SEC >= your longest " + "attack timeout so completion-teardown does not kill an in-flight attack." ) if kind == "classifier": From 5b14447495504404f8d3031ad0863a8622b76fe1 Mon Sep 17 00:00:00 2001 From: Raja Sekhar Rao Dheekonda Date: Wed, 9 Sep 2026 10:43:47 -0700 Subject: [PATCH 3/3] fix(ai-red-teaming): drop ineffective task resolver, keep prompt + skill (ENG-8434) Parity with the CLI simplification on tiger #2452, after review + prod evidence showed a client-side task resolver cannot fix a real 404. The backend visibility rule already resolves a bare name to a task in the caller's org OR any public task, and the cross-org guard blocks a private task even when qualified as /. So the retry was dead code: a public task never 404s bare, and a private cross-org task 404s identically on the retry. Prod confirms bare-name env creates already succeed (aisf org: mnist 16, fraud 16, imdb 13, no errors). - Remove _resolve_task_owner (it was also wrongly wrapped in @safe_tool). - provision_environment no longer retries; on a task-not-found it raises a clear hint (bare = your org or any public task; else qualify as /). - Correct the provisioning-and-lifecycle skill Section 2 and the agent prompt so they no longer claim the tool resolves/retries internally. Tests: dropped 5 resolver unit tests; env suite 27 passed. --- .../agents/ai-red-teaming-agent.md | 4 +- .../provisioning-and-lifecycle/SKILL.md | 12 +-- .../tests/test_environments_teardown.py | 43 ----------- .../ai-red-teaming/tools/environments.py | 75 +++---------------- 4 files changed, 20 insertions(+), 114 deletions(-) diff --git a/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md b/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md index eaaf787..0421845 100644 --- a/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md +++ b/capabilities/ai-red-teaming/agents/ai-red-teaming-agent.md @@ -143,7 +143,7 @@ Complete requests that don't need clarification: - Reason about SDK internals. - Give up after a single failure — retry with adjusted parameters. - Use a "bash" or "shell" tool — use `execute_workflow` instead. -- Guess CLI syntax. To provision a target, ALWAYS call the `provision_environment` tool - never run `dreadnode env ...` in a shell. Public bundled targets (e.g. `ml-extraction-fraud-tabular`, `finops-mesh`) work by bare name; the tool resolves the public catalog for you. +- Guess CLI syntax. To provision a target, ALWAYS call the `provision_environment` tool - never run `dreadnode env ...` in a shell. Public bundled targets (e.g. `ml-extraction-fraud-tabular`, `finops-mesh`) resolve by bare name; pass the bare name and do not try to qualify it. ## Tools @@ -168,7 +168,7 @@ The AI Red Teaming capability provides these tools: **Multi-Agent Environments:** - **list_environments** — List the deployable multi-agent environments (e.g. `finops-mesh`, `devsecops-mesh`, `healthcare-mesh`, `soc-mesh`) that ATLAS can target -- **provision_environment** — The only way to provision a target; never shell out to `dreadnode env`. Pass the task by bare name (e.g. `ml-extraction-fraud-tabular`, `finops-mesh`) - public bundled targets are resolved against the `dreadnode/` catalog automatically, even from another workspace. Returns the endpoint that matches the target type: a multi-agent mesh returns an `/attack` URL + execute token → chain into `generate_atlas_attack`; a black-box ML classifier returns a `/predict` endpoint (plus `/pool`, `/members`, `/nonmembers`) → use `generate_evasion_attack` / `generate_extraction_attack` / `generate_membership_attack` / `generate_inversion_attack` with `api_url=/predict`. **Do not fetch `/attack` on a classifier target - it does not serve it.** The sandbox is recorded and torn down automatically when the assessment completes. +- **provision_environment** — The only way to provision a target; never shell out to `dreadnode env`. Pass the task by bare name (e.g. `ml-extraction-fraud-tabular`, `finops-mesh`) - public bundled targets resolve by bare name from any workspace. Returns the endpoint that matches the target type: a multi-agent mesh returns an `/attack` URL + execute token → chain into `generate_atlas_attack`; a black-box ML classifier returns a `/predict` endpoint (plus `/pool`, `/members`, `/nonmembers`) → use `generate_evasion_attack` / `generate_extraction_attack` / `generate_membership_attack` / `generate_inversion_attack` with `api_url=/predict`. **Do not fetch `/attack` on a classifier target - it does not serve it.** The sandbox is recorded and torn down automatically when the assessment completes. - **teardown_environment** — Delete provisioned environment sandboxes to stop billing. Hosted sandboxes bill for their whole lifetime. With no id it reaps every environment provisioned this session; pass an id to reap one. Teardown also runs automatically when `update_assessment_status` marks the assessment complete, so call this only to reap early or after a partial run. **Workflow Management:** diff --git a/capabilities/ai-red-teaming/skills/provisioning-and-lifecycle/SKILL.md b/capabilities/ai-red-teaming/skills/provisioning-and-lifecycle/SKILL.md index 0594437..d8c7349 100644 --- a/capabilities/ai-red-teaming/skills/provisioning-and-lifecycle/SKILL.md +++ b/capabilities/ai-red-teaming/skills/provisioning-and-lifecycle/SKILL.md @@ -14,14 +14,14 @@ How to provision a hosted target, run against the right endpoint, and tear it do - User gives an HTTP URL -> skip provisioning; go straight to `generate_agentic_attack` / `generate_atlas_attack`. - NEVER run `dreadnode env ...` in a shell, and NEVER guess a `provision`/`create` subcommand. `provision_environment` is the only supported path and it resolves the catalog for you. -## 2. Task-ref resolution (fixes the bare-name 404) +## 2. Task-ref resolution -Bare names resolve in your org first, then the public `dreadnode/` catalog, then other orgs you belong to (covers a private task in a member org). You do NOT qualify manually - `provision_environment` catches the 404 and retries as `/` internally. +A bare name resolves server-side to a task in your org OR any public task, so bundled public targets (`ml-extraction-*`, `*-mesh`) work by bare name with no extra qualification. -- Prefer the BARE name (`ml-extraction-fraud-tabular`). It auto-resolves. -- If you must qualify, use `dreadnode/` for bundled tasks. -- Do NOT qualify a bundled task with the caller's own org (e.g. `aisf-learner-aug-2026/ml-extraction-imdb-text`) - the task does not live there and it will 404 with no fallback. -- If it still fails, the returned message is authoritative: report it. Do not invent qualification syntax or retry random forms. +- Prefer the BARE name (`ml-extraction-fraud-tabular`). It resolves to the public catalog automatically. +- A task owned by another org resolves only when it is public or owned by you. If a bare name 404s, the task is private to another org (or the name/version is wrong) - qualifying it as `/` will NOT help, because the same visibility rule applies. +- Do NOT qualify a bundled task with the caller's own org (e.g. `aisf-learner-aug-2026/ml-extraction-imdb-text`) - it does not live there. +- If provisioning fails, the returned message is authoritative: report it. Do not invent qualification syntax or retry random forms. ## 3. Endpoint-per-target map (fixes /attack on a /predict classifier) diff --git a/capabilities/ai-red-teaming/tests/test_environments_teardown.py b/capabilities/ai-red-teaming/tests/test_environments_teardown.py index a1d814c..8a3fc6b 100644 --- a/capabilities/ai-red-teaming/tests/test_environments_teardown.py +++ b/capabilities/ai-red-teaming/tests/test_environments_teardown.py @@ -262,46 +262,3 @@ def test_mesh_targets(self, ref): def test_unknown_target(self): assert env._target_kind("totally-custom-thing") == "unknown" - - -class TestResolveTaskOwner: - """ENG-8434: a bare task name owned by another org (public catalog or a - private task in a member org) must resolve to / efficiently.""" - - class _Org: - def __init__(self, key): self.key = key - - def _api(self, per_org): - api = _FakeApi() - def list_tasks(org, *, search=None, include_public=False, limit=50): - return {"tasks": per_org.get((org, include_public), per_org.get(org, []))} - api.list_tasks = list_tasks - api.list_user_organizations = lambda: [self._Org("aisf"), self._Org("acme")] - return api - - def test_public_catalog_hit_single_call(self): - # caller-org+public search returns the public task -> resolved, no fan-out - api = self._api({("aisf", True): [{"name": "ml-extraction-fraud-tabular", "org_key": "dreadnode"}]}) - assert env._resolve_task_owner(api, "aisf", "ml-extraction-fraud-tabular") == "dreadnode/ml-extraction-fraud-tabular" - - def test_private_member_org_via_fanout(self): - # not in caller+public; found (private) in another member org - api = self._api({("aisf", True): [], "acme": [{"name": "secret-mesh", "org_key": "acme"}]}) - assert env._resolve_task_owner(api, "aisf", "secret-mesh") == "acme/secret-mesh" - - def test_not_found_anywhere_returns_none(self): - api = self._api({("aisf", True): [], "acme": []}) - assert env._resolve_task_owner(api, "aisf", "nope") is None - - def test_ambiguous_prefers_public_else_none(self): - api = _FakeApi() - api.list_tasks = lambda org, *, search=None, include_public=False, limit=50: { - "tasks": [{"name": "dup", "org_key": "dreadnode"}, {"name": "dup", "org_key": "acme"}] - } - api.list_user_organizations = lambda: [] - assert env._resolve_task_owner(api, "aisf", "dup") == "dreadnode/dup" - - def test_exact_name_match_only(self): - # search is fuzzy server-side; resolver must filter to an exact name match - api = self._api({("aisf", True): [{"name": "ml-extraction-fraud-tabular-v2", "org_key": "dreadnode"}]}) - assert env._resolve_task_owner(api, "aisf", "ml-extraction-fraud-tabular") is None diff --git a/capabilities/ai-red-teaming/tools/environments.py b/capabilities/ai-red-teaming/tools/environments.py index df40e60..b830fbc 100644 --- a/capabilities/ai-red-teaming/tools/environments.py +++ b/capabilities/ai-red-teaming/tools/environments.py @@ -234,56 +234,6 @@ def list_environments() -> str: return "\n".join(lines) -@safe_tool -def _resolve_task_owner(api: t.Any, caller_org: str, name: str) -> str | None: - """Resolve a bare task name to a qualified ``/`` the caller can reach. - - Efficient by design: one ``list_tasks`` call (caller org + public catalog via - ``include_public``) covers the common case - public bundled targets and the - caller's own org - in a single request. Only if that misses do we fan out to - the user's other accessible orgs (covering a private task in an org the user - belongs to), stopping at the first exact match. Returns None if the name isn't - found or is ambiguous across orgs. - """ - def _exact(payload: t.Any) -> list[dict]: - items = (payload or {}).get("tasks") or (payload or {}).get("items") or [] - return [t_ for t_ in items if isinstance(t_, dict) and t_.get("name") == name] - - hits: list[dict] = [] - try: - hits = _exact(api.list_tasks(caller_org, search=name, include_public=True, limit=50)) - except Exception: # noqa: BLE001 - resolution is best-effort - hits = [] - - if not hits: - try: - orgs = [o.key for o in api.list_user_organizations()] - except Exception: # noqa: BLE001 - orgs = [] - for o in orgs: - if o == caller_org: - continue - try: - found = _exact(api.list_tasks(o, search=name, limit=50)) - except Exception: # noqa: BLE001 - continue - if found: - hits = found - break - - if not hits: - return None - owners = {t_.get("org_key") for t_ in hits if t_.get("org_key")} - if len(owners) > 1: - # Ambiguous across orgs - prefer the public catalog if present, else bail - # so the caller qualifies it explicitly rather than us guessing wrong. - if "dreadnode" in owners: - return f"dreadnode/{name}" - return None - owner = next(iter(owners), None) - return f"{owner}/{name}" if owner else None - - def _target_kind(task_ref: str) -> str: """Classify a provisionable target so we return the right endpoint + guidance. @@ -334,23 +284,22 @@ def _mk(ref: str) -> t.Any: model_overrides=model_overrides, timeout_sec=timeout_sec, ) - # A bare name resolves only within the caller's org, so a target owned by - # another org (the public 'dreadnode/' catalog, or a private task in an org - # the user belongs to) 404s. On that 404, resolve the owning org and retry - # qualified as '/' (see _resolve_task_owner for the efficient - # one-call-then-fan-out lookup). + # A bare name resolves to a task in the caller's org or any public task + # (server-side visibility rule), so bundled public targets work by bare name. + # A task owned by another org resolves only when it is public or owned by the + # caller - a private cross-org task 404s the same way whether or not it is + # qualified, so there is no client-side retry that helps; just add a hint. env = _mk(task_ref) try: ctx = _run(env.setup()) - except Exception as exc: # noqa: BLE001 - resolve owning org, then one retry - if "/" in task_ref or not _is_not_found(exc): + except Exception as exc: # noqa: BLE001 - add a resolution hint on not-found + if not _is_not_found(exc): raise - resolved = _resolve_task_owner(api, org, task_ref) - if resolved is None: - raise - task_ref = resolved - env = _mk(task_ref) - ctx = _run(env.setup()) + raise RuntimeError( + f"Task '{task_ref}' not found. A bare name resolves to a task in your org " + f"or any public task; a task owned by another org must be public or owned " + f"by you. Check the name and version, or qualify it as /." + ) from exc svc = (ctx.get("service_urls") or {}).get("challenge") url = (svc.get("url") if isinstance(svc, dict) else svc) or "" token = env._execute_token or "" # noqa: SLF001 - one-shot provision token