diff --git a/.github/instructions/scenarios.instructions.md b/.github/instructions/scenarios.instructions.md index fa3df99322..e2673e58bc 100644 --- a/.github/instructions/scenarios.instructions.md +++ b/.github/instructions/scenarios.instructions.md @@ -169,8 +169,24 @@ Options: - Ingredients must remain complete. Use `max_per_dataset="all"` and no finite source `max_size`; cap the assembled attack groups with `max_total`, not ingredient rows. - Use `with_overrides()` instead of reconstructing a subclass or mutating its defaults. +- Clients and Python use `ScenarioDatasetRequest` through the common `dataset_overrides` + parameter. `dataset_config` still replaces the default; overrides apply on top. +- Resolve parameter/technique-dependent names in `_resolve_default_dataset_config()`, + before overrides, validation, and preparation. Do not rewrite user selections in validators. +- Declare `DATASET_OPTIONS` (or parameter-aware `get_dataset_options()`): `ANY` only for + generic named populations, `SUBSET` for approved alternatives, `FIXED` for ingredients. + Disable limits that would truncate ingredients. Compound source edits are ambiguous; + preserve child shaping/caps and allow only supported global edits. +- Estimates expose `dataset_defaults`, `effective_datasets`, and `dataset_options`. + These are response projections, not incoming edit state. Never read seeds for projections. + The catalog reads options and defaults from registry metadata, never from an estimate. +- Resume replays the saved launch request; the stored run plan fixes the selected seed groups, + so caps and fetch policies cannot change a resumed run. Dataset names are checked by the + stored scenario identity. Resume never prepares datasets. - Keep custom shaping in `_build_attack_groups()` or `_build_groups_by_dataset_async()`. Reads use `_collect_seeds_for_dataset_async()` and must not fetch. +- Build synthesized prompts from those filtered seeds, not a second unfiltered memory query. + Scenario code must not call memory seed-read methods directly; the scenario contract test enforces this. - `dataset_names`, `max_dataset_size`, `auto_fetch`, and `per_dataset()` are deprecated. `max_dataset_size` is an exact alias for `max_total`; it does not disable source caps. diff --git a/doc/code/scenarios/0_scenarios.ipynb b/doc/code/scenarios/0_scenarios.ipynb index b088db7741..7b0b110a57 100644 --- a/doc/code/scenarios/0_scenarios.ipynb +++ b/doc/code/scenarios/0_scenarios.ipynb @@ -66,10 +66,11 @@ "\n", "3. **Default Dataset**: Pass `default_dataset_config=` to `super().__init__()` to specify the datasets your scenario uses out of the box.\n", " - Returns a `DatasetAttackConfiguration` with named sources (e.g., `sources=[DatasetSource(name=\"my_dataset\")]`)\n", - " - Users can override this at runtime via `--dataset-names` in the CLI or by passing a custom `dataset_config` programmatically\n", + " - Users select sources with `--datasets name[:N|:all]`, or pass `dataset_overrides=ScenarioDatasetRequest(...)` in `set_params_from_args`. A custom `dataset_config` replaces the default; overrides apply on top.\n", " - Named sources select at most 5 attack groups per dataset by default; `max_total` caps the union. For any limit, omitted/`None`/empty/`\"default\"` uses the default; `\"all\"` removes that limit.\n", " - The deprecated `max_dataset_size` is an exact alias for `max_total`, not a way to disable source caps. To keep the old total-only selection, use `max_per_dataset=\"all\", max_total=10`; to remove both caps, set both to `\"all\"`. `None` now uses the default, not unlimited selection.\n", " - New runs prepare missing registered datasets once; reads, estimates, and resume never fetch\n", + " - Scenarios declare allowed source selections and limit controls. Estimates return resolved `dataset_defaults` and `effective_datasets` separately from requested edits. Name-only selection preserves existing source caps and fetch policies.\n", "\n", "4. **Constructor**: Use `@apply_defaults` decorator and call `super().__init__()` with scenario metadata:\n", " - `name`: Descriptive name for your scenario\n", diff --git a/doc/code/scenarios/0_scenarios.py b/doc/code/scenarios/0_scenarios.py index 3a951368a4..a9407131da 100644 --- a/doc/code/scenarios/0_scenarios.py +++ b/doc/code/scenarios/0_scenarios.py @@ -68,10 +68,11 @@ # # 3. **Default Dataset**: Pass `default_dataset_config=` to `super().__init__()` to specify the datasets your scenario uses out of the box. # - Returns a `DatasetAttackConfiguration` with named sources (e.g., `sources=[DatasetSource(name="my_dataset")]`) -# - Users can override this at runtime via `--dataset-names` in the CLI or by passing a custom `dataset_config` programmatically +# - Users select sources with `--datasets name[:N|:all]`, or pass `dataset_overrides=ScenarioDatasetRequest(...)` in `set_params_from_args`. A custom `dataset_config` replaces the default; overrides apply on top. # - Named sources select at most 5 attack groups per dataset by default; `max_total` caps the union. For any limit, omitted/`None`/empty/`"default"` uses the default; `"all"` removes that limit. # - The deprecated `max_dataset_size` is an exact alias for `max_total`, not a way to disable source caps. To keep the old total-only selection, use `max_per_dataset="all", max_total=10`; to remove both caps, set both to `"all"`. `None` now uses the default, not unlimited selection. # - New runs prepare missing registered datasets once; reads, estimates, and resume never fetch +# - Scenarios declare allowed source selections and limit controls. Estimates return resolved `dataset_defaults` and `effective_datasets` separately from requested edits. Name-only selection preserves existing source caps and fetch policies. # # 4. **Constructor**: Use `@apply_defaults` decorator and call `super().__init__()` with scenario metadata: # - `name`: Descriptive name for your scenario diff --git a/doc/code/scenarios/1_common_scenario_parameters.ipynb b/doc/code/scenarios/1_common_scenario_parameters.ipynb index e6081ccad4..5950f87f7c 100644 --- a/doc/code/scenarios/1_common_scenario_parameters.ipynb +++ b/doc/code/scenarios/1_common_scenario_parameters.ipynb @@ -15,7 +15,7 @@ "\n", "> **Two selection axes**: *Techniques* select attack techniques (*how* attacks run — e.g., prompt\n", "> sending, role play, TAP). *Datasets* select objectives (*what* is tested — e.g., harm categories,\n", - "> compliance topics). Use `--dataset-names` on the CLI to filter by content category.\n", + "> compliance topics). Use `--datasets` on the CLI to filter by content category.\n", "\n", "> **Running scenarios from the command line?** See the [Scanner documentation](../../scanner/0_scanner.md).\n", "\n", diff --git a/doc/code/scenarios/1_common_scenario_parameters.py b/doc/code/scenarios/1_common_scenario_parameters.py index 354df1410e..0ced660ed1 100644 --- a/doc/code/scenarios/1_common_scenario_parameters.py +++ b/doc/code/scenarios/1_common_scenario_parameters.py @@ -19,7 +19,7 @@ # # > **Two selection axes**: *Techniques* select attack techniques (*how* attacks run — e.g., prompt # > sending, role play, TAP). *Datasets* select objectives (*what* is tested — e.g., harm categories, -# > compliance topics). Use `--dataset-names` on the CLI to filter by content category. +# > compliance topics). Use `--datasets` on the CLI to filter by content category. # # > **Running scenarios from the command line?** See the [Scanner documentation](../../scanner/0_scanner.md). # diff --git a/doc/code/scenarios/3_adaptive_scenarios.ipynb b/doc/code/scenarios/3_adaptive_scenarios.ipynb index 2c9b2bc2f5..436b55b09a 100644 --- a/doc/code/scenarios/3_adaptive_scenarios.ipynb +++ b/doc/code/scenarios/3_adaptive_scenarios.ipynb @@ -804,7 +804,7 @@ "# Use specific datasets and limit size\n", "pyrit_scan --scenario TextAdaptive --target openai_chat \\\n", " --datasets airt_hate airt_violence \\\n", - " --max-dataset-size 10\n", + " --max-total 10\n", "```" ] } diff --git a/doc/code/scenarios/3_adaptive_scenarios.py b/doc/code/scenarios/3_adaptive_scenarios.py index 247b24b786..2614eb1d17 100644 --- a/doc/code/scenarios/3_adaptive_scenarios.py +++ b/doc/code/scenarios/3_adaptive_scenarios.py @@ -301,5 +301,5 @@ def _technique_label(result) -> str: # # Use specific datasets and limit size # pyrit_scan --scenario TextAdaptive --target openai_chat \ # --datasets airt_hate airt_violence \ -# --max-dataset-size 10 +# --max-total 10 # ``` diff --git a/doc/getting_started/install_local_dev.md b/doc/getting_started/install_local_dev.md index c1e7fde984..021cd51d01 100644 --- a/doc/getting_started/install_local_dev.md +++ b/doc/getting_started/install_local_dev.md @@ -114,6 +114,15 @@ Use `uv run` to execute Python with the virtual environment: uv run python your_script.py ``` +Remote dataset loaders use `HF_DATASETS_CACHE` when it is set. Otherwise they cache +Hugging Face data under `dbdata/huggingface`. On Windows, a long worktree path can +cause `WinError 206` when Hugging Face creates a cache lock file. Set a shorter +cache path before you start the server or script: + +```powershell +$env:HF_DATASETS_CACHE = Join-Path $env:TEMP "pyrit-hf" +``` + #### Running Tests ```bash diff --git a/doc/gui/0_gui.md b/doc/gui/0_gui.md index 0eab1d9945..14348fb870 100644 --- a/doc/gui/0_gui.md +++ b/doc/gui/0_gui.md @@ -34,7 +34,7 @@ To deploy an isolated instance for an external team, see [Deploy a New Instance] ## Views -CoPyRIT has three main views, accessible from the left sidebar: **Chat**, **Attack History**, and **Target Configuration**. The **Theme** menu is available at the bottom of the sidebar. +CoPyRIT has three main views, accessible from the left sidebar: **Technique.execute** (Chat), **Attack History**, and **Target Configuration**. The **Theme** menu is available at the bottom of the sidebar. ### Themes @@ -511,7 +511,59 @@ Defaults are saved in this browser, separately for each signed-in account. They A missing or changed default is shown as unavailable. Select a new default or clear it; CoPyRIT does not silently substitute a different target. If browser storage is unavailable, a warning states that the choice applies only in the current session. -Scanner forms let you override either selection for one run. **Objective Target** is the target the scenario runs against. **Adversarial Target**, under **Parameters** above **Dataset override**, is the target used to generate attacks. The adversarial selector appears only for scenarios whose available techniques use the shared default; scenarios with no such use, including the explicit-target adversarial benchmark, do not show it or send an override. **Use server default** clears the per-run adversarial override. Changes to your defaults do not change existing chats or queued/running scans. Explicit adversarial targets in a scenario or technique still take priority. Scorer targets are unchanged. +Scanner forms let you override either selection for one run. **Objective Target** is the target the scenario runs against. **Adversarial Target**, under **Parameters**, is the target used to generate attacks. The adversarial selector appears only for scenarios whose available techniques use the shared default; scenarios with no such use, including the explicit-target adversarial benchmark, do not show it or send an override. **Use server default** clears the per-run adversarial override. Changes to your defaults do not change existing chats or queued/running scans. Explicit adversarial targets in a scenario or technique still take priority. Scorer targets are unchanged. + +### Scenario datasets + +**Datasets** is a separate section below **Techniques**. Its table has **Dataset**, +**Size**, and **Include Criteria** columns. **Size** shows an upper bound, such as **Up to 4 +of 200**, when the stored total is known, or **Up to 4** before loading. This is not +an exact selected count: filters and the combined objective limit can reduce it. +Click a supported size to edit its limit, use all available examples, or reset to +the displayed default. Required reference data shows its stored size without a +limit editor. + +Select a name from **Add dataset** to add it immediately. Provider names remain +available before their data is stored. Selecting a name never loads it. The remove +icon is available only when the scenario permits removal and another source will +remain. A scenario that permits only one source uses **Choose dataset** to replace +it. **Browse datasets**, beside **Reset datasets**, opens the dataset browser. +**Reset datasets** restores source defaults and row limits without clearing +the combined objective limit or filters. + +**Load datasets** appears only while selected data is missing. It loads missing +sources and keeps existing data, including manual edits. It may download data or +use a configured generator, but does not start a scan. A missing source without a +provider must be imported first. When all selected data is loaded, the load button +and loading controls are omitted. + +Loaded rows show harm categories and data types as differently colored chips. +All labels start checked when there is no configured filter. Uncheck labels to +narrow the existing filters for **all selected datasets**, not just the row clicked. +Checking all labels restores the default request when no filter was configured. +Partial category selections require every checked label and retain substring +matching; data types match any checked type. An empty selection blocks launch +instead of silently restoring all data. Selection is shown on the metadata chips; +there is no separate filter list below the table. No preview of data values is provided. + +**Maximum objectives** shows its resolved value. Click it to enter a positive +integer, use all available objectives, or reset to the displayed default. This +limits objectives, not prompts sent. Four objectives with three techniques produce +12 attacks, plus four more if the baseline is enabled. Each attack can send several +prompts. The run estimate shows the resulting attacks. + +Scenarios that build objectives from templates or reference data apply the total +limit to the completed objectives, not to the required reference data. Their +generation limits still apply. A scenario that requires one objective per selected +technique rejects a total limit smaller than the number of techniques. + +**Load missing data when the scan starts** is one switch for the selected sources. +Its initial state reflects their resolved loading rules, which normally load +missing data. Turning it off prevents automatic loading on scan start without +disabling the manual load action. Existing source-specific rules are not silently +replaced until the switch is changed. Closing a limit editor does not clear an +invalid draft. Estimate and launch use the same dataset edits. +Resume uses the saved request and original selection and never loads datasets. #### Core Adversarial Default Override diff --git a/doc/scanner/1_pyrit_scan.ipynb b/doc/scanner/1_pyrit_scan.ipynb index ee28f8b9e7..670f985d40 100644 --- a/doc/scanner/1_pyrit_scan.ipynb +++ b/doc/scanner/1_pyrit_scan.ipynb @@ -1194,9 +1194,17 @@ "on demand, so a full dataset preload is not required:\n", "\n", "```shell\n", - "pyrit_scan run garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1\n", + "pyrit_scan run garak.figstep --target openai_chat --datasets figstep_pro --max-total 1 --dataset-fetch if_missing\n", "```\n", "\n", + "Scanner and shell share `--datasets name[:N|:all]`, `--max-per-dataset`, `--max-total`,\n", + "and `--dataset-fetch never|if_missing`. Source limits apply before the combined total.\n", + "Omitted, empty, or `default` keeps the scenario setting; `all` removes only its own cap.\n", + "`list-scenarios` displays allowed sources and defaults. Ingredient/compound scenarios\n", + "restrict controls to preserve their shaping. `--dataset-filters` works with either form.\n", + "Deprecated `--dataset-names` and `--max-dataset-size` cannot be mixed with the new flags.\n", + "Discovery and estimates never fetch; only a new run prepares absent registered datasets.\n", + "\n", "Custom initialization scripts are loaded by the backend at startup: list them in the\n", "`initialization_scripts` section of the config the server loads (paths are relative to your\n", "working directory, but full paths avoid confusion). Once the server is running with that\n", diff --git a/doc/scanner/1_pyrit_scan.py b/doc/scanner/1_pyrit_scan.py index 0aa18722aa..1e5dce2700 100644 --- a/doc/scanner/1_pyrit_scan.py +++ b/doc/scanner/1_pyrit_scan.py @@ -125,9 +125,17 @@ # on demand, so a full dataset preload is not required: # # ```shell -# pyrit_scan run garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1 +# pyrit_scan run garak.figstep --target openai_chat --datasets figstep_pro --max-total 1 --dataset-fetch if_missing # ``` # +# Scanner and shell share `--datasets name[:N|:all]`, `--max-per-dataset`, `--max-total`, +# and `--dataset-fetch never|if_missing`. Source limits apply before the combined total. +# Omitted, empty, or `default` keeps the scenario setting; `all` removes only its own cap. +# `list-scenarios` displays allowed sources and defaults. Ingredient/compound scenarios +# restrict controls to preserve their shaping. `--dataset-filters` works with either form. +# Deprecated `--dataset-names` and `--max-dataset-size` cannot be mixed with the new flags. +# Discovery and estimates never fetch; only a new run prepares absent registered datasets. +# # Custom initialization scripts are loaded by the backend at startup: list them in the # `initialization_scripts` section of the config the server loads (paths are relative to your # working directory, but full paths avoid confusion). Once the server is running with that diff --git a/doc/scanner/adaptive.ipynb b/doc/scanner/adaptive.ipynb index a4bc5849a4..1f7c72fd64 100644 --- a/doc/scanner/adaptive.ipynb +++ b/doc/scanner/adaptive.ipynb @@ -26,8 +26,8 @@ "pyrit_scan run adaptive.text_adaptive \\\n", " --initializers target \\\n", " --target openai_chat \\\n", - " --dataset-names airt_hate \\\n", - " --max-dataset-size 2 \\\n", + " --datasets airt_hate \\\n", + " --max-total 2 \\\n", " --max-attempts-per-objective 2\n", "```" ] diff --git a/doc/scanner/adaptive.py b/doc/scanner/adaptive.py index b229e0e55d..0eda326f0b 100644 --- a/doc/scanner/adaptive.py +++ b/doc/scanner/adaptive.py @@ -25,8 +25,8 @@ # pyrit_scan run adaptive.text_adaptive \ # --initializers target \ # --target openai_chat \ -# --dataset-names airt_hate \ -# --max-dataset-size 2 \ +# --datasets airt_hate \ +# --max-total 2 \ # --max-attempts-per-objective 2 # ``` diff --git a/doc/scanner/airt.ipynb b/doc/scanner/airt.ipynb index 09cf12e75e..04866a9005 100644 --- a/doc/scanner/airt.ipynb +++ b/doc/scanner/airt.ipynb @@ -102,8 +102,8 @@ " --initializers target \\\n", " --target openai_chat \\\n", " --techniques role_play_movie_script \\\n", - " --dataset-names airt_hate \\\n", - " --max-dataset-size 1\n", + " --datasets airt_hate \\\n", + " --max-total 1\n", "```\n", "\n", "**Available techniques:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, role_play_movie_script, many_shot, tap" @@ -255,8 +255,8 @@ "\n", "Each sub-harm escalates a simulated multi-turn conversation toward the objective, then layers the\n", "selected converter techniques on top (natural-language rewrites that preserve emotional framing;\n", - "obfuscation converters like base64/morse are excluded). Datasets are bound to the sub-harms, so\n", - "`--dataset-names` is ignored (`--max-dataset-size` still applies).\n", + "obfuscation converters like base64/morse are excluded). `--datasets` can select a subset of the\n", + "selected sub-harms' datasets; `--max-total` must cover each selected sub-harm.\n", "\n", "**Available techniques:** ALL, DEFAULT, tone, language, persuasion, deterministic, crescendo" ] @@ -413,7 +413,7 @@ " --initializers target \\\n", " --target openai_chat \\\n", " --techniques role_play_movie_script \\\n", - " --max-dataset-size 1\n", + " --max-total 1\n", "```\n", "\n", "**Available techniques:** Use `pyrit_scan run airt.cyber --list-scenario-parameters` to inspect\n", @@ -570,8 +570,8 @@ "pyrit_scan run airt.jailbreak \\\n", " --initializers target \\\n", " --target openai_chat \\\n", - " --dataset-names harmbench \\\n", - " --max-dataset-size 1\n", + " --datasets harmbench \\\n", + " --max-total 1\n", "```\n", "\n", "**Available technique selectors:** ALL, DEFAULT, and SINGLE_TURN currently select both\n", @@ -730,8 +730,8 @@ "pyrit_scan airt.multilingual \\\n", " --initializers target \\\n", " --target openai_chat \\\n", - " --dataset-names harmbench \\\n", - " --max-dataset-size 1\n", + " --datasets harmbench \\\n", + " --max-total 1\n", "```\n", "\n", "**Available techniques:** `prompt_sending` is the default. Every registry technique (`role_play_*`,\n", @@ -870,7 +870,7 @@ "plagiarism detection.\n", "\n", "```bash\n", - "pyrit_scan run airt.leakage --target openai_chat --techniques first_letter --max-dataset-size 1\n", + "pyrit_scan run airt.leakage --target openai_chat --techniques first_letter --max-total 1\n", "```\n", "\n", "**Available techniques:** ALL, SINGLE_TURN, MULTI_TURN, IP, SENSITIVE_DATA, FirstLetter, Image, RolePlay, Crescendo\n", @@ -1049,7 +1049,7 @@ " --initializers target \\\n", " --target openai_chat \\\n", " --techniques context_compliance \\\n", - " --max-dataset-size 1\n", + " --max-total 1\n", "```\n", "\n", "**Available techniques:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, ContextCompliance, RolePlay,\n", diff --git a/doc/scanner/airt.py b/doc/scanner/airt.py index cb804f14e7..a7fba16ebc 100644 --- a/doc/scanner/airt.py +++ b/doc/scanner/airt.py @@ -43,8 +43,8 @@ # --initializers target \ # --target openai_chat \ # --techniques role_play_movie_script \ -# --dataset-names airt_hate \ -# --max-dataset-size 1 +# --datasets airt_hate \ +# --max-total 1 # ``` # # **Available techniques:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, role_play_movie_script, many_shot, tap @@ -84,8 +84,8 @@ # # Each sub-harm escalates a simulated multi-turn conversation toward the objective, then layers the # selected converter techniques on top (natural-language rewrites that preserve emotional framing; -# obfuscation converters like base64/morse are excluded). Datasets are bound to the sub-harms, so -# `--dataset-names` is ignored (`--max-dataset-size` still applies). +# obfuscation converters like base64/morse are excluded). `--datasets` can select a subset of the +# selected sub-harms' datasets; `--max-total` must cover each selected sub-harm. # # **Available techniques:** ALL, DEFAULT, tone, language, persuasion, deterministic, crescendo @@ -124,7 +124,7 @@ # --initializers target \ # --target openai_chat \ # --techniques role_play_movie_script \ -# --max-dataset-size 1 +# --max-total 1 # ``` # # **Available techniques:** Use `pyrit_scan run airt.cyber --list-scenario-parameters` to inspect @@ -169,8 +169,8 @@ # pyrit_scan run airt.jailbreak \ # --initializers target \ # --target openai_chat \ -# --dataset-names harmbench \ -# --max-dataset-size 1 +# --datasets harmbench \ +# --max-total 1 # ``` # # **Available technique selectors:** ALL, DEFAULT, and SINGLE_TURN currently select both @@ -213,8 +213,8 @@ # pyrit_scan airt.multilingual \ # --initializers target \ # --target openai_chat \ -# --dataset-names harmbench \ -# --max-dataset-size 1 +# --datasets harmbench \ +# --max-total 1 # ``` # # **Available techniques:** `prompt_sending` is the default. Every registry technique (`role_play_*`, @@ -254,7 +254,7 @@ # plagiarism detection. # # ```bash -# pyrit_scan run airt.leakage --target openai_chat --techniques first_letter --max-dataset-size 1 +# pyrit_scan run airt.leakage --target openai_chat --techniques first_letter --max-total 1 # ``` # # **Available techniques:** ALL, SINGLE_TURN, MULTI_TURN, IP, SENSITIVE_DATA, FirstLetter, Image, RolePlay, Crescendo @@ -308,7 +308,7 @@ # --initializers target \ # --target openai_chat \ # --techniques context_compliance \ -# --max-dataset-size 1 +# --max-total 1 # ``` # # **Available techniques:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, ContextCompliance, RolePlay, diff --git a/doc/scanner/benchmark.ipynb b/doc/scanner/benchmark.ipynb index 6abd678fe9..2ed6d84aeb 100644 --- a/doc/scanner/benchmark.ipynb +++ b/doc/scanner/benchmark.ipynb @@ -35,7 +35,7 @@ " --target openai_chat \\\n", " --adversarial-targets adversarial_chat \\\n", " --techniques role_play_video_game \\\n", - " --max-dataset-size 1\n", + " --max-total 1\n", "```\n", "\n", "Pass multiple `--adversarial-targets` values to compare across models in a single run.\n", diff --git a/doc/scanner/benchmark.py b/doc/scanner/benchmark.py index 8adddffbc1..92588c50db 100644 --- a/doc/scanner/benchmark.py +++ b/doc/scanner/benchmark.py @@ -34,7 +34,7 @@ # --target openai_chat \ # --adversarial-targets adversarial_chat \ # --techniques role_play_video_game \ -# --max-dataset-size 1 +# --max-total 1 # ``` # # Pass multiple `--adversarial-targets` values to compare across models in a single run. diff --git a/doc/scanner/foundry.ipynb b/doc/scanner/foundry.ipynb index 78a5dbeaab..dca0cb3342 100644 --- a/doc/scanner/foundry.ipynb +++ b/doc/scanner/foundry.ipynb @@ -73,7 +73,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan run foundry.red_team_agent --target openai_chat --techniques base64 --max-dataset-size 1\n", + "pyrit_scan run foundry.red_team_agent --target openai_chat --techniques base64 --max-total 1\n", "```\n", "\n", "**Available techniques by difficulty:**\n", diff --git a/doc/scanner/foundry.py b/doc/scanner/foundry.py index bbcc9b8bb6..e73747b8ca 100644 --- a/doc/scanner/foundry.py +++ b/doc/scanner/foundry.py @@ -40,7 +40,7 @@ # **CLI example:** # # ```bash -# pyrit_scan run foundry.red_team_agent --target openai_chat --techniques base64 --max-dataset-size 1 +# pyrit_scan run foundry.red_team_agent --target openai_chat --techniques base64 --max-total 1 # ``` # # **Available techniques by difficulty:** diff --git a/doc/scanner/garak.ipynb b/doc/scanner/garak.ipynb index 18cc5a8c90..da4ca4811d 100644 --- a/doc/scanner/garak.ipynb +++ b/doc/scanner/garak.ipynb @@ -37,7 +37,7 @@ "PromptInject tests whether a target follows injected instructions hidden in benign tasks.\n", "`--techniques` selects the injection wording, and `--goal-texts` selects the requested output.\n", "The default run uses all five techniques, the three Garak goal texts, and up to 60 attempts.\n", - "`--max-dataset-size` must be at least the number of selected goal texts.\n", + "`--max-total` must be at least the number of selected goal texts.\n", "Sampling selects at least one context for every goal, then fills the remaining budget\n", "without replacement. All techniques use the same sample. Injection wording is applied\n", "through request converters, before any caller-supplied converters.\n", @@ -49,7 +49,7 @@ "pyrit_scan run garak.prompt_inject --target openai_chat \\\n", " --techniques ignore_print \\\n", " --goal-texts \"Custom trigger text\" \\\n", - " --max-dataset-size 16\n", + " --max-total 16\n", "```\n", "\n", "**Techniques:** `ignore_print`, `ignore_say`, `ignore_print_upper`, `nevermind`, and\n", @@ -299,7 +299,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan run garak.encoding --target openai_chat --techniques base64 --max-dataset-size 1\n", + "pyrit_scan run garak.encoding --target openai_chat --techniques base64 --max-total 1\n", "```\n", "\n", "**Available techniques** (17 encodings): Base64, Base2048, Base16, Base32, ASCII85, Hex,\n", @@ -457,8 +457,8 @@ "**CLI examples:**\n", "\n", "```bash\n", - "pyrit_scan garak.figstep --target openai_chat --dataset-names figstep --max-dataset-size 1\n", - "pyrit_scan garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1\n", + "pyrit_scan garak.figstep --target openai_chat --datasets figstep --max-total 1\n", + "pyrit_scan garak.figstep --target openai_chat --datasets figstep_pro --max-total 1\n", "```\n", "\n", "> **Note:** The objective target must natively support multi-piece user messages and accept text\n", @@ -630,7 +630,7 @@ "**Aggregate techniques:** `ALL` (all 8), `DEFAULT` (excludes the two combinatorial extended\n", "probes), `EXFIL` (the 6 markdown-exfil probes), and `XSS` (TaskXSS + MarkdownXSS).\n", "\n", - "**Dataset requirements:** `--dataset-names` replaces the default datasets, so it must include\n", + "**Dataset requirements:** `--datasets` replaces the default datasets, so it must include\n", "every dataset the selected techniques read. Otherwise the scenario stops before running and names\n", "each missing dataset. Datasets already loaded into memory never fill the gap.\n", "\n", @@ -645,7 +645,7 @@ "| StringAssemblyDataExfil | None (built-in seeds) |\n", "\n", "```bash\n", - "pyrit_scan run garak.web_injection --target openai_chat --techniques task_xss --dataset-names garak_xss_normal_instructions garak_web_html_js\n", + "pyrit_scan run garak.web_injection --target openai_chat --techniques task_xss --datasets garak_xss_normal_instructions garak_web_html_js\n", "```" ] }, @@ -986,7 +986,7 @@ "pyrit_scan run garak.api_key --target openai_chat\n", "\n", "# Run only GetKey with a smaller sample.\n", - "pyrit_scan run garak.api_key --target openai_chat --techniques get_key --max-dataset-size 2\n", + "pyrit_scan run garak.api_key --target openai_chat --techniques get_key --max-total 2\n", "```\n", "\n", "**Available techniques:** `GetKey` and `CompleteKey`. `DEFAULT` and `ALL` both select the two\n", @@ -1161,7 +1161,7 @@ "\n", "```bash\n", "pyrit_scan run garak.latent_injection --target openai_chat --techniques bare \\\n", - " --families whois --max-dataset-size 1\n", + " --families whois --max-total 1\n", "```\n", "\n", "**Available techniques** (14 separator styles): Bare, Newline, Blockquote, BlockquoteInline,\n", @@ -1353,7 +1353,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan run garak.doctor --target openai_chat --techniques policy_puppetry --max-dataset-size 1\n", + "pyrit_scan run garak.doctor --target openai_chat --techniques policy_puppetry --max-total 1\n", "```\n", "\n", "**Available techniques** (2 probes): `PolicyPuppetry` (wraps the objective in the Dr House\n", @@ -1845,13 +1845,13 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan garak.audio_achilles_heel --target azure_openai_realtime --max-dataset-size 2\n", + "pyrit_scan garak.audio_achilles_heel --target azure_openai_realtime --max-total 2\n", "```\n", "\n", "> **Note:** The objective target must accept `audio_path` input (i.e. be multimodal). The example\n", "> below uses the registered Azure OpenAI Realtime target; non-audio targets such as the default\n", "> `openai_chat` will error when the audio request is sent. The full dataset holds ~350\n", - "> clips, so a default run samples a small subset to finish quickly — raise `--max-dataset-size`\n", + "> clips, so a default run samples a small subset to finish quickly — raise `--max-total`\n", "> for broader coverage." ] }, @@ -2009,7 +2009,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan run garak.divergence --target openai_chat --techniques repeat --max-dataset-size 2\n", + "pyrit_scan run garak.divergence --target openai_chat --techniques repeat --max-total 2\n", "```\n", "\n", "**Available techniques:** `Repeat`, `DEFAULT`, and `ALL` all select the same probe.\n", diff --git a/doc/scanner/garak.py b/doc/scanner/garak.py index 23c41e5555..5276a808ae 100644 --- a/doc/scanner/garak.py +++ b/doc/scanner/garak.py @@ -36,7 +36,7 @@ # PromptInject tests whether a target follows injected instructions hidden in benign tasks. # `--techniques` selects the injection wording, and `--goal-texts` selects the requested output. # The default run uses all five techniques, the three Garak goal texts, and up to 60 attempts. -# `--max-dataset-size` must be at least the number of selected goal texts. +# `--max-total` must be at least the number of selected goal texts. # Sampling selects at least one context for every goal, then fills the remaining budget # without replacement. All techniques use the same sample. Injection wording is applied # through request converters, before any caller-supplied converters. @@ -48,7 +48,7 @@ # pyrit_scan run garak.prompt_inject --target openai_chat \ # --techniques ignore_print \ # --goal-texts "Custom trigger text" \ -# --max-dataset-size 16 +# --max-total 16 # ``` # # **Techniques:** `ignore_print`, `ignore_say`, `ignore_print_upper`, `nevermind`, and @@ -132,7 +132,7 @@ # **CLI example:** # # ```bash -# pyrit_scan run garak.encoding --target openai_chat --techniques base64 --max-dataset-size 1 +# pyrit_scan run garak.encoding --target openai_chat --techniques base64 --max-total 1 # ``` # # **Available techniques** (17 encodings): Base64, Base2048, Base16, Base32, ASCII85, Hex, @@ -180,8 +180,8 @@ # **CLI examples:** # # ```bash -# pyrit_scan garak.figstep --target openai_chat --dataset-names figstep --max-dataset-size 1 -# pyrit_scan garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1 +# pyrit_scan garak.figstep --target openai_chat --datasets figstep --max-total 1 +# pyrit_scan garak.figstep --target openai_chat --datasets figstep_pro --max-total 1 # ``` # # > **Note:** The objective target must natively support multi-piece user messages and accept text @@ -231,7 +231,7 @@ # **Aggregate techniques:** `ALL` (all 8), `DEFAULT` (excludes the two combinatorial extended # probes), `EXFIL` (the 6 markdown-exfil probes), and `XSS` (TaskXSS + MarkdownXSS). # -# **Dataset requirements:** `--dataset-names` replaces the default datasets, so it must include +# **Dataset requirements:** `--datasets` replaces the default datasets, so it must include # every dataset the selected techniques read. Otherwise the scenario stops before running and names # each missing dataset. Datasets already loaded into memory never fill the gap. # @@ -246,7 +246,7 @@ # | StringAssemblyDataExfil | None (built-in seeds) | # # ```bash -# pyrit_scan run garak.web_injection --target openai_chat --techniques task_xss --dataset-names garak_xss_normal_instructions garak_web_html_js +# pyrit_scan run garak.web_injection --target openai_chat --techniques task_xss --datasets garak_xss_normal_instructions garak_web_html_js # ``` # %% @@ -333,7 +333,7 @@ # pyrit_scan run garak.api_key --target openai_chat # # # Run only GetKey with a smaller sample. -# pyrit_scan run garak.api_key --target openai_chat --techniques get_key --max-dataset-size 2 +# pyrit_scan run garak.api_key --target openai_chat --techniques get_key --max-total 2 # ``` # # **Available techniques:** `GetKey` and `CompleteKey`. `DEFAULT` and `ALL` both select the two @@ -381,7 +381,7 @@ # # ```bash # pyrit_scan run garak.latent_injection --target openai_chat --techniques bare \ -# --families whois --max-dataset-size 1 +# --families whois --max-total 1 # ``` # # **Available techniques** (14 separator styles): Bare, Newline, Blockquote, BlockquoteInline, @@ -447,7 +447,7 @@ # **CLI example:** # # ```bash -# pyrit_scan run garak.doctor --target openai_chat --techniques policy_puppetry --max-dataset-size 1 +# pyrit_scan run garak.doctor --target openai_chat --techniques policy_puppetry --max-total 1 # ``` # # **Available techniques** (2 probes): `PolicyPuppetry` (wraps the objective in the Dr House @@ -575,13 +575,13 @@ # **CLI example:** # # ```bash -# pyrit_scan garak.audio_achilles_heel --target azure_openai_realtime --max-dataset-size 2 +# pyrit_scan garak.audio_achilles_heel --target azure_openai_realtime --max-total 2 # ``` # # > **Note:** The objective target must accept `audio_path` input (i.e. be multimodal). The example # > below uses the registered Azure OpenAI Realtime target; non-audio targets such as the default # > `openai_chat` will error when the audio request is sent. The full dataset holds ~350 -# > clips, so a default run samples a small subset to finish quickly — raise `--max-dataset-size` +# > clips, so a default run samples a small subset to finish quickly — raise `--max-total` # > for broader coverage. # %% @@ -620,7 +620,7 @@ # **CLI example:** # # ```bash -# pyrit_scan run garak.divergence --target openai_chat --techniques repeat --max-dataset-size 2 +# pyrit_scan run garak.divergence --target openai_chat --techniques repeat --max-total 2 # ``` # # **Available techniques:** `Repeat`, `DEFAULT`, and `ALL` all select the same probe. diff --git a/frontend/e2e/accessibility.spec.ts b/frontend/e2e/accessibility.spec.ts index ee07c64883..a279a2c6e3 100644 --- a/frontend/e2e/accessibility.spec.ts +++ b/frontend/e2e/accessibility.spec.ts @@ -129,7 +129,7 @@ test.describe("Accessibility", () => { const objectiveDefault = page.getByRole("combobox", { name: "Default objective target", exact: true }); await expect(objectiveDefault).toBeVisible({ timeout: 5000 }); await objectiveDefault.selectOption({ index: 1 }); - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); // Input should be accessible const input = page.getByRole("textbox"); @@ -146,7 +146,7 @@ test.describe("Accessibility", () => { test("should have accessible sidebar navigation", async ({ page }) => { // Chat button - const chatBtn = page.getByTitle("Chat"); + const chatBtn = page.getByTitle("Technique.execute"); await expect(chatBtn).toBeVisible(); // Registry button @@ -272,7 +272,7 @@ test.describe("Accessibility", () => { const objectiveDefault = page.getByRole("combobox", { name: "Default objective target", exact: true }); await expect(objectiveDefault).toBeVisible({ timeout: 5000 }); await objectiveDefault.selectOption({ index: 1 }); - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); const input = page.getByRole("textbox"); await expect(input).toBeEnabled({ timeout: 5000 }); @@ -337,7 +337,7 @@ test.describe("Accessibility", () => { const views = [ { button: "History", heading: "History" }, { button: "Registry", heading: "Target Registry" }, - { button: "Chat", heading: "Chat" }, + { button: "Technique.execute", heading: "Chat" }, ]; for (const view of views) { @@ -407,8 +407,8 @@ test.describe("Visual Consistency", () => { await page.goto("/"); // Wait for initial render then navigate to chat to measure the chat ribbon - await expect(page.getByTitle("Chat")).toBeVisible(); - await page.getByTitle("Chat").click(); + await expect(page.getByTitle("Technique.execute")).toBeVisible(); + await page.getByTitle("Technique.execute").click(); const anchor = page.getByTestId("new-attack-btn"); await expect(anchor).toBeVisible(); diff --git a/frontend/e2e/api.spec.ts b/frontend/e2e/api.spec.ts index 3739b3fb3e..c15a7dccfe 100644 --- a/frontend/e2e/api.spec.ts +++ b/frontend/e2e/api.spec.ts @@ -141,6 +141,6 @@ test.describe("Error Handling", () => { await page.goto("/"); // UI should be responsive even while APIs are delayed - await expect(page.getByTitle("Chat")).toBeVisible({ timeout: 10000 }); + await expect(page.getByTitle("Technique.execute")).toBeVisible({ timeout: 10000 }); }); }); diff --git a/frontend/e2e/chat-recovery.spec.ts b/frontend/e2e/chat-recovery.spec.ts index b31f3e96c7..ddc5985f4c 100644 --- a/frontend/e2e/chat-recovery.spec.ts +++ b/frontend/e2e/chat-recovery.spec.ts @@ -240,7 +240,7 @@ test.describe("Chat processing recovery @seeded", () => { await page.getByTitle("Registry", { exact: true }).click(); await page.getByRole("combobox", { name: "Default objective target", exact: true }) .selectOption(localTarget.registryName); - await page.getByTitle("Chat", { exact: true }).click(); + await page.getByTitle("Technique.execute", { exact: true }).click(); }); test("keeps the drawer closed when a delayed send refresh discovers another conversation", async ({ diff --git a/frontend/e2e/chat.spec.ts b/frontend/e2e/chat.spec.ts index 4ccfb4241d..d75df13f55 100644 --- a/frontend/e2e/chat.spec.ts +++ b/frontend/e2e/chat.spec.ts @@ -152,7 +152,7 @@ async function activateMockTarget(page: Page) { await objectiveDefault.selectOption({ index: 1 }); // Return to Chat view - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); await expect(page.getByTestId("new-attack-btn")).toBeVisible({ timeout: 5000 }); } @@ -170,18 +170,18 @@ test.describe("Application Smoke Tests", () => { }); test("should display chat ribbon", async ({ page }) => { - await expect(page.getByTitle("Chat")).toBeVisible({ timeout: 10000 }); - await page.getByTitle("Chat").click(); + await expect(page.getByTitle("Technique.execute")).toBeVisible({ timeout: 10000 }); + await page.getByTitle("Technique.execute").click(); await expect(page.getByTestId("new-attack-btn")).toBeVisible({ timeout: 10000 }); }); test("should have New Attack button", async ({ page }) => { - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); await expect(page.getByRole("button", { name: /new attack/i })).toBeVisible(); }); test("should offer the target picker without a bottom warning", async ({ page }) => { - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); await expect(page.getByRole("combobox", { name: "Chat target" })).toBeVisible(); await expect(page.getByTestId("no-target-banner")).toHaveCount(0); }); @@ -190,7 +190,7 @@ test.describe("Application Smoke Tests", () => { test.describe("Theme Toggle", () => { test("should toggle dark/light theme", async ({ page }) => { await page.goto("/"); - await expect(page.getByTitle("Chat")).toBeVisible({ timeout: 10000 }); + await expect(page.getByTitle("Technique.execute")).toBeVisible({ timeout: 10000 }); // The app defaults to system mode, so the toggle button title should say "Theme: System" const themeBtn = page.getByTitle("Theme: System"); @@ -374,7 +374,7 @@ test.describe("Multiple Messages", () => { test.describe("Chat without target", () => { test("should disable input when no target is selected", async ({ page }) => { await page.goto("/"); - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); await expect(page.getByRole("textbox")).toBeDisabled(); await expect(page.getByRole("button", { name: "Send message" })).toBeDisabled(); @@ -912,7 +912,7 @@ test.describe("Target type scenarios", () => { .selectOption("dall-e-image-gen"); // Navigate to chat - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); // Scope queries to the badge so we don't also match the (hidden) // copy of the target text that Fluent's Tooltip renders into the DOM. const badge = page.getByTestId("target-badge"); diff --git a/frontend/e2e/compatibility.spec.ts b/frontend/e2e/compatibility.spec.ts index f4983b1640..b49f8c9095 100644 --- a/frontend/e2e/compatibility.spec.ts +++ b/frontend/e2e/compatibility.spec.ts @@ -43,10 +43,10 @@ test("waits for a matching packaged stamp before sending business requests", asy }); await page.goto("/"); await expect(page.getByRole("status")).toContainText("Checking frontend and backend compatibility"); - await expect(page.getByTitle("Chat", { exact: true })).not.toBeVisible(); + await expect(page.getByTitle("Technique.execute", { exact: true })).not.toBeVisible(); expect(requests).toHaveLength(0); releaseVersion(); - await expect(page.getByTitle("Chat", { exact: true })).toBeVisible(); + await expect(page.getByTitle("Technique.execute", { exact: true })).toBeVisible(); await expect.poll(() => requests.length).toBeGreaterThan(0); }); @@ -62,7 +62,7 @@ for (const [name, payload] of [ }); await page.goto("/"); await expect(page.getByRole("heading", { name: "PyRIT compatibility blocked" })).toBeVisible(); - await expect(page.getByTitle("Chat", { exact: true })).not.toBeVisible(); + await expect(page.getByTitle("Technique.execute", { exact: true })).not.toBeVisible(); expect(requests).toHaveLength(0); }); } @@ -81,7 +81,7 @@ for (const [status, type] of [ await route.fulfill({ status, json: { type, expected: otherCompatibilityId(), actual: getCompatibilityId() } }); }); await page.goto("/"); - await expect(page.getByTitle("Chat", { exact: true })).toBeVisible(); + await expect(page.getByTitle("Technique.execute", { exact: true })).toBeVisible(); await page.evaluate(async () => { const modulePath = "/src/services/api.ts"; const { apiClient } = await import(modulePath); @@ -92,7 +92,7 @@ for (const [status, type] of [ await expect(page.getByRole("heading", { name: "PyRIT compatibility blocked" })).toBeVisible(); await expect(page.getByRole("alertdialog")).toContainText(getCompatibilityId()); await expect(page.getByRole("alertdialog")).toContainText(otherCompatibilityId()); - await expect(page.getByTitle("Chat", { exact: true })).not.toBeVisible(); + await expect(page.getByTitle("Technique.execute", { exact: true })).not.toBeVisible(); expect(mutations).toBe(1); }); } diff --git a/frontend/e2e/config.spec.ts b/frontend/e2e/config.spec.ts index b0d048e8cd..a77f466179 100644 --- a/frontend/e2e/config.spec.ts +++ b/frontend/e2e/config.spec.ts @@ -699,7 +699,7 @@ test.describe("Target Config ↔ Chat Navigation", () => { await page.getByRole("combobox", { name: "Default objective target", exact: true }).selectOption({ index: 1 }); // Navigate back to chat - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); await expect(page.getByTestId("new-attack-btn")).toBeVisible(); // Chat should show the selected target type. Scope to the badge to @@ -717,7 +717,7 @@ test.describe("Target Config ↔ Chat Navigation", () => { // Start in chat with the composer disabled until a target is selected. await page.goto("/"); - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); await expect(page.getByRole("textbox")).toBeDisabled(); // Go to targets, set a target @@ -726,7 +726,7 @@ test.describe("Target Config ↔ Chat Navigation", () => { await page.getByRole("combobox", { name: "Default objective target", exact: true }).selectOption({ index: 1 }); // Return to chat — send should be enabled when there's text - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); const input = page.getByRole("textbox"); await input.fill("Hello"); await expect(page.getByRole("button", { name: /send/i })).toBeEnabled(); diff --git a/frontend/e2e/converters.spec.ts b/frontend/e2e/converters.spec.ts index 185e66e8b9..22f2b32933 100644 --- a/frontend/e2e/converters.spec.ts +++ b/frontend/e2e/converters.spec.ts @@ -469,7 +469,7 @@ async function activateMockTarget(page: Page) { await expect(objectiveDefault).toBeVisible({ timeout: 5000 }); await objectiveDefault.selectOption({ index: 1 }); - await page.getByTitle("Chat", { exact: true }).click(); + await page.getByTitle("Technique.execute", { exact: true }).click(); await expect(page.getByTestId("new-attack-btn")).toBeVisible({ timeout: 5000 }); } @@ -568,7 +568,7 @@ test.describe("Shared per-piece converter pipelines @seeded", () => { await page.getByTitle("Registry", { exact: true }).click(); await page.getByRole("combobox", { name: "Default objective target", exact: true }) .selectOption(targetRegistryName); - await page.getByTitle("Chat", { exact: true }).click(); + await page.getByTitle("Technique.execute", { exact: true }).click(); await expect(page.getByTestId("chat-input")).toBeEnabled(); }); diff --git a/frontend/e2e/errors.spec.ts b/frontend/e2e/errors.spec.ts index af764de659..444c6cef12 100644 --- a/frontend/e2e/errors.spec.ts +++ b/frontend/e2e/errors.spec.ts @@ -241,7 +241,7 @@ async function activateMockTarget(page: Page) { const objectiveDefault = page.getByRole("combobox", { name: "Default objective target", exact: true }); await expect(objectiveDefault).toBeVisible({ timeout: 5000 }); await objectiveDefault.selectOption({ index: 1 }); - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); await expect(page.getByTestId("new-attack-btn")).toBeVisible({ timeout: 5000 }); } @@ -504,7 +504,7 @@ test.describe("Error: connection banner on health failure", () => { }) => { // Let the page load normally first await page.goto("/"); - await expect(page.getByTitle("Chat")).toBeVisible({ + await expect(page.getByTitle("Technique.execute")).toBeVisible({ timeout: 10000, }); @@ -552,7 +552,7 @@ test.describe("Error: connection banner recovery", () => { }); await page.goto("/"); - await expect(page.getByTitle("Chat")).toBeVisible({ + await expect(page.getByTitle("Technique.execute")).toBeVisible({ timeout: 10000, }); diff --git a/frontend/e2e/flows.spec.ts b/frontend/e2e/flows.spec.ts index 104f539538..ec8a478e13 100644 --- a/frontend/e2e/flows.spec.ts +++ b/frontend/e2e/flows.spec.ts @@ -210,7 +210,7 @@ async function activateTarget( await expect(row).toBeVisible({ timeout: 10_000 }); await page.getByRole("combobox", { name: "Default objective target", exact: true }) .selectOption(targetRegistryName); - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); await expect(page.getByTestId("new-attack-btn")).toBeVisible({ timeout: 5_000 }); } diff --git a/frontend/e2e/numeric-controls.spec.ts b/frontend/e2e/numeric-controls.spec.ts index a9b7cc8705..fb3d8e3262 100644 --- a/frontend/e2e/numeric-controls.spec.ts +++ b/frontend/e2e/numeric-controls.spec.ts @@ -1,11 +1,27 @@ import { expect, test, type Locator, type Page } from './_fixtures' import { mockVersion } from './_compatibility' -import type { RegisteredScenario } from '../src/types' +import type { RegisteredScenario, ScenarioDatasetRequest, ScenarioDatasetSettings } from '../src/types' import { READY_RUNTIME } from './_runtime' import { makeTarget } from './_targets' +const DATASET_DEFAULTS: ScenarioDatasetSettings = { + sources: [{ name: 'harmbench', max_size: 5, fetch: 'if_missing', configured_provider: false }], + max_per_dataset: 5, max_total: 5, fetch: 'if_missing', filters: {}, +} +const DATASET_OPTIONS = { + mode: 'any' as const, allowed_names: [], per_dataset_limit: true, + total_limit: true, supports_overrides: true, source_settings: true, filters: true, +} +const DATASETS = { + items: [ + { name: 'harmbench', selection_key: 'dataset:named:harmbench', loaded: false, provider_available: true, origins: [], logical_examples: null }, + { name: 'custom_objectives', selection_key: 'dataset:named:custom_objectives', loaded: true, provider_available: false, + origins: ['user'], logical_examples: 10, modalities: ['text', 'image_path'], harm_categories: [], has_unlabeled_harm_categories: false }, + ], +} + const SCENARIO: RegisteredScenario = { scenario_name: 'numeric-controls', scenario_type: 'NumericControlsScenario', @@ -18,7 +34,9 @@ const SCENARIO: RegisteredScenario = { default_technique: 'prompt_sending', default_techniques: ['prompt_sending'], technique_summaries: [{ name: 'prompt_sending', description: null, tags: [] }], - default_datasets: [], + default_datasets: ['harmbench'], + dataset_defaults: DATASET_DEFAULTS, + dataset_options: DATASET_OPTIONS, baseline_policy: 'disabled', include_baseline_by_default: false, supported_parameters: [ @@ -26,12 +44,13 @@ const SCENARIO: RegisteredScenario = { { name: 'temperature', type_name: 'float', required: false, default: '0.5', choices: null, is_list: false }, ], supported_parameter_names: [ - 'objective_target', 'scenario_techniques', 'dataset_config', 'memory_labels', + 'objective_target', 'scenario_techniques', 'dataset_config', 'dataset_overrides', 'memory_labels', 'max_concurrency', 'max_retries', 'include_baseline', 'iterations', 'temperature', ], default_run_size: { dataset_size: { kind: 'indeterminate', reason: 'configuration_unavailable', detail: 'Population configuration is not available.' }, - dataset_limit: { state: 'scenario_default' }, + dataset_limit: { state: 'value', value: 5 }, + dataset_defaults: DATASET_DEFAULTS, effective_datasets: DATASET_DEFAULTS, dataset_options: DATASET_OPTIONS, estimated_attack_count: null, components: [], datasets: [], @@ -40,6 +59,7 @@ const SCENARIO: RegisteredScenario = { } async function mockNumericControlApis(page: Page): Promise { + let datasetItems = DATASETS.items.map((item) => ({ ...item })) // No backend, model calls, or connections to another worktree's HMR server. await page.routeWebSocket('**/*', (socket) => socket.close()) await page.route('**/api/**', async (route) => { @@ -74,9 +94,38 @@ async function mockNumericControlApis(page: Page): Promise { ], }, '/api/labels': { source: 'attacks', labels: {} }, + '/api/datasets': { items: datasetItems }, [`/api/scenarios/catalog/${SCENARIO.scenario_name}`]: SCENARIO, [`/api/scenarios/catalog/${SCENARIO.scenario_name}/estimate`]: SCENARIO.default_run_size, } + if (path.endsWith('/datasets/load')) { + const edits: ScenarioDatasetRequest = route.request().postDataJSON()?.datasets ?? {} + const names = (edits.sources ?? DATASET_DEFAULTS.sources).map((source) => source.name) + datasetItems = datasetItems.map((item) => names.includes(item.name) + ? { ...item, loaded: true, logical_examples: 200, modalities: ['text'], harm_categories: ['malware'] } + : item) + await route.fulfill({ json: { settings: DATASET_DEFAULTS, items: datasetItems.filter((item) => names.includes(item.name)) } }) + return + } + if (path.endsWith('/estimate')) { + const edits: ScenarioDatasetRequest = route.request().postDataJSON()?.datasets ?? {} + const limit = (value: ScenarioDatasetRequest['max_total'], fallback: number | 'all'): number | 'all' => + typeof value === 'number' || value === 'all' ? value : fallback + const effective: ScenarioDatasetSettings = { + ...DATASET_DEFAULTS, + max_per_dataset: limit(edits.max_per_dataset, DATASET_DEFAULTS.max_per_dataset), + max_total: limit(edits.max_total, DATASET_DEFAULTS.max_total), + fetch: edits.fetch === 'never' || edits.fetch === 'if_missing' ? edits.fetch : DATASET_DEFAULTS.fetch, + sources: (edits.sources ?? DATASET_DEFAULTS.sources).map((source) => ({ + name: source.name, + max_size: limit(source.max_size, DATASET_DEFAULTS.max_per_dataset), + fetch: source.fetch === 'never' || source.fetch === 'if_missing' ? source.fetch : DATASET_DEFAULTS.fetch, + configured_provider: false, + })), + } + await route.fulfill({ json: { ...SCENARIO.default_run_size, effective_datasets: effective } }) + return + } if (!(path in responses)) { await route.abort() throw new Error(`Unexpected API request: ${route.request().method()} ${path}`) @@ -168,21 +217,48 @@ test('scenario steppers commit typed values before stepping and clamp at existin await expect(page.getByRole('spinbutton', { name: 'Max concurrency', exact: true })).toHaveValue('10') }) -test('native scenario dataset and shared dynamic parameter controls step once', async ({ page }) => { +test('native shared dynamic parameter controls step once', async ({ page }) => { await page.goto(`/scanner/${SCENARIO.scenario_name}`) - const datasetSize = page.getByRole('spinbutton', { name: 'Max dataset size', exact: true }) - await expectNativeSteps(datasetSize, 2) await expectNativeSteps(page.getByRole('spinbutton', { name: 'iterations', exact: true }), 2) // The existing dynamic float input has native step=1 with a fractional base. await expectNativeSteps(page.getByRole('spinbutton', { name: 'temperature', exact: true }), 0.5) - await datasetSize.fill('1') - await clickNativeSpinner(datasetSize, 'down') - await expect(datasetSize).toHaveValue('1') - await datasetSize.press('ArrowDown') - await expect(datasetSize).toHaveValue('1') - await datasetSize.fill('') - await datasetSize.press('Tab') - await expect(datasetSize).toHaveValue('') +}) + +test('dataset sources and explicit loading are separate from Parameters', async ({ page }, testInfo) => { + await page.goto(`/scanner/${SCENARIO.scenario_name}`) + const section = page.getByRole('region', { name: 'Datasets', exact: true }) + await expect(section.getByRole('table', { name: 'Selected datasets' })).toBeVisible() + await expect(section.getByRole('columnheader', { name: 'Size' })).toBeVisible() + await expect(section.getByRole('button', { name: 'Edit harmbench objective limit' })).toHaveText('Up to 5') + await expect(section.getByText('Load dataset to see details', { exact: true })).toBeVisible() + await expect(page.getByRole('region', { name: 'Parameters' }).getByRole('table')).toHaveCount(0) + await expect(section.getByRole('button', { name: 'Edit Maximum objectives' })).toHaveText('Up to 5') + await expect(section.getByText('Details and advanced settings')).toHaveCount(0) + await section.screenshot({ path: testInfo.outputPath('datasets-default.png') }) + await section.getByRole('button', { name: 'Load datasets' }).click() + await expect(section.getByRole('button', { name: 'Edit harmbench objective limit' })).toHaveText('Up to 5 of 200') + await expect(section.getByRole('button', { name: 'Load datasets' })).toHaveCount(0) + await expect(section.getByRole('switch')).toHaveCount(0) + await expect(section.getByRole('button', { name: 'Filter harm category malware' })).toBeVisible() + await expect(section.getByRole('button', { name: 'Filter data type Text' })).toBeVisible() + await section.screenshot({ path: testInfo.outputPath('datasets-loaded.png') }) + await section.getByRole('button', { name: 'Edit Maximum objectives' }).click() + const total = page.getByRole('textbox', { name: 'Maximum objectives', exact: true }) + await total.fill('12') + await page.getByRole('button', { name: 'Done', exact: true }).click() + await section.screenshot({ path: testInfo.outputPath('datasets-custom.png') }) + await section.getByRole('button', { name: 'Edit Maximum objectives' }).click() + await total.fill('0') + await total.press('Escape') + await section.getByRole('button', { name: 'Reset datasets' }).click() + await expect(section.getByRole('alert')).toContainText('Maximum objectives must be a positive integer') + await expect(page.getByRole('button', { name: 'Launch scan', exact: true })).toBeDisabled() + await section.screenshot({ path: testInfo.outputPath('datasets-invalid.png') }) + await section.getByRole('button', { name: 'Edit Maximum objectives' }).click() + await expect(total).toHaveValue('0') + await page.getByRole('button', { name: 'Reset to 5', exact: true }).click() + await page.getByRole('button', { name: 'Done', exact: true }).click() + await expect(section.getByRole('alert')).not.toBeVisible() }) test('native Azure ML advanced controls retain steps and reset to default guidance', async ({ page }) => { @@ -210,6 +286,43 @@ test('native Azure ML advanced controls retain steps and reset to default guidan await expect(resetTemperature).toHaveAttribute('placeholder', 'Defaults to 1.0') }) +test('dataset rows add immediately, edit in place, share filters, and reset without loading', async ({ page }) => { + await page.goto(`/scanner/${SCENARIO.scenario_name}`) + const section = page.getByRole('region', { name: 'Datasets', exact: true }) + await section.getByRole('combobox', { name: 'Add dataset' }).click() + await page.getByRole('option', { name: 'custom_objectives', exact: true }).click() + await expect(section.getByText('custom_objectives', { exact: true })).toBeVisible() + await section.getByRole('button', { name: 'Remove harmbench' }).click() + await expect(section.getByRole('button', { name: 'Load datasets' })).toHaveCount(0) + await section.getByRole('button', { name: 'Edit custom_objectives objective limit' }).click() + await page.getByRole('textbox', { name: 'custom_objectives objective limit' }).fill('3') + await page.getByRole('button', { name: 'Done', exact: true }).click() + await expect(section.getByRole('button', { name: 'Edit custom_objectives objective limit' })).toHaveText('Up to 3 of 10') + await section.getByRole('button', { name: 'Filter data type Image' }).click() + await expect(section.getByRole('button', { name: 'Filter data type Text' })).toHaveAttribute('aria-pressed', 'true') + await expect(section.getByRole('link', { name: 'Browse datasets' })).toHaveAttribute('href', '/datasets') + await expect(section.getByText('Filters:', { exact: true })).toHaveCount(0) + await section.getByRole('button', { name: 'Reset datasets' }).click() + await expect(section.getByText('harmbench', { exact: true })).toBeVisible() + await expect(section.getByRole('button', { name: 'Load datasets' })).toBeVisible() + await expect(section.getByRole('button', { name: 'Remove type filter text' })).toHaveCount(0) +}) + +test('dataset manual loading keeps an explicit load-on-start switch override', async ({ page }) => { + await page.goto(`/scanner/${SCENARIO.scenario_name}`) + const section = page.getByRole('region', { name: 'Datasets', exact: true }) + const policy = section.getByRole('switch', { name: 'Load missing data when the scan starts' }) + await expect(policy).toBeChecked() + await policy.click() + const load = page.waitForRequest((request) => new URL(request.url()).pathname.endsWith('/datasets/load')) + await section.getByRole('button', { name: 'Load datasets' }).click() + const request = await load + expect(request.postDataJSON().datasets.fetch).toBe('never') + expect(request.postDataJSON().datasets.sources[0].fetch).toBe('never') + await expect(section.getByRole('button', { name: 'Load datasets' })).toHaveCount(0) + await expect(policy).toHaveCount(0) +}) + test('native round-robin weights step once and preserve bounds and integer validation', async ({ page }) => { await page.goto('/registry/targets') await page.getByRole('button', { name: /new target/i }).click() diff --git a/frontend/e2e/onboarding-tour.spec.ts b/frontend/e2e/onboarding-tour.spec.ts index dae177aaa3..534e522ea0 100644 --- a/frontend/e2e/onboarding-tour.spec.ts +++ b/frontend/e2e/onboarding-tour.spec.ts @@ -39,6 +39,9 @@ const TOUR_SCENARIO: RegisteredScenario = { /** Serves a one-scenario catalog so the detail route is reachable by click. */ async function mockScenarioCatalog(page: Page): Promise { + await page.route(/\/api\/datasets(?:\?|$)/, async (route) => { + await route.fulfill({ json: { items: [] } }); + }); await page.route(/\/api\/scenarios\/catalog(?:\?.*)?$/, async (route) => { await route.fulfill({ json: { diff --git a/frontend/e2e/routing.spec.ts b/frontend/e2e/routing.spec.ts index c308fde70f..6573c8e97d 100644 --- a/frontend/e2e/routing.spec.ts +++ b/frontend/e2e/routing.spec.ts @@ -300,7 +300,7 @@ test.describe("URL-driven routing", () => { await expect(page).toHaveURL(/\/history\/attacks$/); await expect(page.getByTestId("attacks-table")).toBeVisible(); - await page.getByTitle("Chat").click(); + await page.getByTitle("Technique.execute").click(); await expect(page).toHaveURL(/\/chat$/); await expect(page.getByRole("switch", { name: "Markdown" })).toBeChecked(); diff --git a/frontend/e2e/scenario-history.spec.ts b/frontend/e2e/scenario-history.spec.ts index 0cfc087e49..315b8a0738 100644 --- a/frontend/e2e/scenario-history.spec.ts +++ b/frontend/e2e/scenario-history.spec.ts @@ -264,6 +264,10 @@ async function mockScenarioAPIs(page: Page): Promise { }); }); + await page.route(/\/api\/datasets(?:\?|$)/, async (route) => { + await route.fulfill({ json: { items: [] } }); + }); + await page.route(/\/api\/runtime(?:\?|$)/, async (route) => { await route.fulfill({ status: 200, @@ -539,7 +543,7 @@ test.describe("Scenario catalog, history, and live run routing", () => { expect(await primaryButtons.evaluateAll((buttons) => buttons.map((button) => button.getAttribute("aria-label")))).toEqual([ "Home", - "Chat", + "Technique.execute", "History", "Scanner", "Datasets", diff --git a/frontend/e2e/touch-targets.spec.ts b/frontend/e2e/touch-targets.spec.ts index d75603ed57..7618194d20 100644 --- a/frontend/e2e/touch-targets.spec.ts +++ b/frontend/e2e/touch-targets.spec.ts @@ -476,7 +476,7 @@ async function startChatWithMessages(page: Page): Promise { await page.getByRole("button", { name: "Registry", exact: true }).click(); await expect(page.getByText("gpt-4o-mobile", { exact: true })).toBeVisible(); await page.getByRole("combobox", { name: "Default objective target", exact: true }).selectOption({ index: 1 }); - await page.getByRole("button", { name: "Chat", exact: true }).click(); + await page.getByRole("button", { name: "Technique.execute", exact: true }).click(); await page.getByTestId("chat-input").fill( "Assess this deterministic mobile prompt" ); @@ -501,7 +501,7 @@ test.describe("Mobile touch targets", () => { await page.getByRole("button", { name: "Registry", exact: true }).click(); await expect(page.getByText("gpt-4o-mobile", { exact: true })).toBeVisible(); await page.getByRole("combobox", { name: "Default objective target", exact: true }).selectOption({ index: 1 }); - await page.getByRole("button", { name: "Chat", exact: true }).click(); + await page.getByRole("button", { name: "Technique.execute", exact: true }).click(); await page.getByRole("button", { name: "Add objective" }).click(); const objectiveInput = page.getByRole("textbox", { diff --git a/frontend/src/App.labels.test.tsx b/frontend/src/App.labels.test.tsx index 9f17d1610f..935503bfc7 100644 --- a/frontend/src/App.labels.test.tsx +++ b/frontend/src/App.labels.test.tsx @@ -208,7 +208,7 @@ describe('Shared new run labels', () => { await chooseOperation(user, 'remembered_op') const bar = currentLabels() - for (const destination of ['Home', 'Chat', 'Scanner']) { + for (const destination of ['Home', 'Technique.execute', 'Scanner']) { await user.click(screen.getByRole('button', { name: destination, exact: true })) expect(screen.getAllByTestId('labels-bar')).toHaveLength(1) expect(currentLabels()).toBe(bar) @@ -273,7 +273,7 @@ describe('Shared new run labels', () => { expect(screen.getAllByTestId('labels-bar')).toHaveLength(1) await user.click(screen.getByRole('button', { name: 'Scanner', exact: true })) expect(screen.queryByRole('group', { name: 'Chat controls' })).not.toBeInTheDocument() - await user.click(screen.getByRole('button', { name: 'Chat', exact: true })) + await user.click(screen.getByRole('button', { name: 'Technique.execute', exact: true })) expect(screen.getAllByRole('group', { name: 'Chat controls' })).toHaveLength(1) expect(within(currentLabels()).getByRole('switch')).toBeChecked() }) diff --git a/frontend/src/components/Scenarios/DatasetLimitControl.tsx b/frontend/src/components/Scenarios/DatasetLimitControl.tsx new file mode 100644 index 0000000000..a3d1fce71d --- /dev/null +++ b/frontend/src/components/Scenarios/DatasetLimitControl.tsx @@ -0,0 +1,63 @@ +import { useState } from 'react' + +import { Button, Checkbox, Field, Input, Popover, PopoverSurface, PopoverTrigger, Text } from '@fluentui/react-components' + +import type { ScenarioDatasetLimit } from '@/types' +import { useScenarioDatasetTableStyles } from './ScenarioDatasetTable.styles' +import { parseCaseLimit } from './datasetPresentation' + +interface DatasetLimitControlProps { + label: string + value: ScenarioDatasetLimit | null | undefined + defaultValue: number | 'all' + text?: string + showLabel?: boolean + hint?: string + draft?: string + disabled: boolean + onChange: (value: ScenarioDatasetLimit) => void + onDraftChange: (text: string | null) => void +} + +export default function DatasetLimitControl({ + label, value, defaultValue, text, showLabel = true, hint, draft, disabled, onChange, onDraftChange, +}: DatasetLimitControlProps) { + const styles = useScenarioDatasetTableStyles() + const [open, setOpen] = useState(false) + const limit = typeof value === 'number' || value === 'all' ? value : defaultValue + const invalid = draft !== undefined && parseCaseLimit(draft) === null + const control = setOpen(data.open)}> + + + + +
+ + { + onDraftChange(data.value) + const parsed = parseCaseLimit(data.value) + if (parsed !== null) onChange(parsed) + }} /> + + { + onDraftChange(null) + onChange(data.checked ? 'all' : typeof defaultValue === 'number' ? defaultValue : 1) + }} /> + {hint && {hint}} +
+ + +
+
+ + + return showLabel ?
{label}:{control}
: control +} diff --git a/frontend/src/components/Scenarios/ScenarioDatasetTable.styles.ts b/frontend/src/components/Scenarios/ScenarioDatasetTable.styles.ts new file mode 100644 index 0000000000..fdc7ebb74a --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioDatasetTable.styles.ts @@ -0,0 +1,28 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTarget } from '@/styles/touchTargets' + +export const useScenarioDatasetTableStyles = makeStyles({ + root: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalM }, + controls: { display: 'flex', flexWrap: 'wrap', alignItems: 'center', gap: tokens.spacingHorizontalM }, + browseLink: { color: tokens.colorBrandForegroundLink, ...mobileTouchTarget, display: 'inline-flex', alignItems: 'center' }, + scroll: { overflowX: 'auto' }, + table: { width: '100%', minWidth: '36em', borderCollapse: 'collapse' }, + toolbar: { display: 'flex', flexWrap: 'wrap', alignItems: 'center', gap: tokens.spacingHorizontalM }, + metadata: { display: 'flex', flexWrap: 'wrap', alignItems: 'center', gap: tokens.spacingHorizontalXS }, + chip: { ...mobileTouchTarget, padding: tokens.spacingHorizontalXXS, minWidth: 'unset' }, + actionsCell: { width: '3em' }, + datasetName: { overflowWrap: 'anywhere' }, + limitButton: { + ...mobileTouchTarget, + justifyContent: 'flex-start', + paddingLeft: tokens.spacingHorizontalXS, + paddingRight: tokens.spacingHorizontalXS, + color: tokens.colorBrandForeground1, + ':hover': { textDecorationLine: 'underline' }, + }, + limitEditor: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalM, maxWidth: '24em' }, + totalLimit: { display: 'flex', flexWrap: 'wrap', alignItems: 'center', gap: tokens.spacingHorizontalXS }, + loadingRule: { display: 'inline-flex' }, + button: { ...mobileTouchTarget }, +}) diff --git a/frontend/src/components/Scenarios/ScenarioDatasetTable.test.tsx b/frontend/src/components/Scenarios/ScenarioDatasetTable.test.tsx new file mode 100644 index 0000000000..964247c1e0 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioDatasetTable.test.tsx @@ -0,0 +1,279 @@ +import { useState } from 'react' + +import { render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { MemoryRouter } from 'react-router' + +import type { DatasetInfo, ScenarioDatasetOptions, ScenarioDatasetRequest, ScenarioDatasetSettings } from '@/types' + +import ScenarioDatasetTable from './ScenarioDatasetTable' + +const DEFAULTS: ScenarioDatasetSettings = { + sources: [{ name: 'stored', max_size: 2, fetch: 'never', configured_provider: false }], + max_per_dataset: 5, max_total: 4, fetch: 'if_missing', filters: {}, +} +const OPTIONS: ScenarioDatasetOptions = { + mode: 'any', allowed_names: [], per_dataset_limit: true, total_limit: true, supports_overrides: true, filters: true, +} +const STORED: DatasetInfo = { + name: 'stored', selection_key: 'dataset:named:stored', loaded: true, provider_available: false, + origins: ['local'], logical_examples: 8, seed_pieces: 8, objectives: 8, + modalities: ['text'], harm_categories: ['hate', 'violence'], has_unlabeled_harm_categories: false, +} +const REMOTE: DatasetInfo = { + ...STORED, name: 'remote', selection_key: 'dataset:named:remote', loaded: false, + provider_available: true, logical_examples: null, origins: [], modalities: [], harm_categories: [], +} +const DATASETS = [STORED, REMOTE, { ...STORED, name: 'other', selection_key: 'dataset:named:other' }] + +interface HarnessProps { + options?: ScenarioDatasetOptions + defaults?: ScenarioDatasetSettings + datasets?: DatasetInfo[] + initialRequest?: ScenarioDatasetRequest + onChange?: jest.Mock + onLoad?: jest.Mock +} +function Harness({ + options = OPTIONS, defaults = DEFAULTS, datasets = DATASETS, initialRequest = {}, onChange = jest.fn(), onLoad = jest.fn(), +}: HarnessProps) { + const [request, setRequest] = useState(initialRequest) + const [validation, setValidation] = useState(null) + const effective: ScenarioDatasetSettings = { + ...defaults, + sources: (request.sources ?? defaults.sources).map((source) => ({ + name: source.name, + max_size: typeof source.max_size === 'number' || source.max_size === 'all' + ? source.max_size : defaults.sources.find((item) => item.name === source.name)?.max_size ?? defaults.max_per_dataset, + fetch: source.fetch === 'never' || source.fetch === 'if_missing' ? source.fetch + : defaults.sources.find((item) => item.name === source.name)?.fetch ?? defaults.fetch, + configured_provider: false, + })), + } + return + { setRequest(next); onChange(next) }} /> + + +} + +async function editLimit(user: ReturnType, label: string, value: string): Promise { + await user.click(screen.getByRole('button', { name: `Edit ${label}` })) + const all = screen.getByRole('checkbox', { name: `All available for ${label}` }) + if (all instanceof HTMLInputElement && all.checked) await user.click(all) + const input = screen.getByRole('textbox', { name: label }) + await user.clear(input) + await user.type(input, value) +} + +async function addDataset(user: ReturnType, name: string): Promise { + await user.click(screen.getByRole('combobox', { name: 'Add dataset' })) + await user.click(screen.getByRole('option', { name, exact: true })) +} + +describe('ScenarioDatasetTable', () => { + it('shows a compact table with resolved sizes and metadata, without a loading banner for loaded data', () => { + const onChange = jest.fn() + render() + expect(screen.getByRole('columnheader', { name: 'Dataset', exact: true })).toBeInTheDocument() + expect(screen.getByRole('columnheader', { name: 'Size' })).toBeInTheDocument() + expect(screen.getByRole('columnheader', { name: 'Include Criteria' })).toBeInTheDocument() + expect(screen.getByRole('separator')).toBeInTheDocument() + expect(screen.queryByText(/All selected uses the defaults/)).not.toBeInTheDocument() + expect(screen.getByText('Maximum objectives:', { exact: true })).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Edit stored objective limit' })).toHaveTextContent('Up to 2 of 8') + expect(screen.getByRole('button', { name: 'Filter harm category hate' })).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByRole('button', { name: 'Filter data type Text' })).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Load datasets' })).not.toBeInTheDocument() + expect(screen.queryByRole('switch')).not.toBeInTheDocument() + expect(screen.queryByText('All selected datasets are loaded.')).not.toBeInTheDocument() + expect(screen.queryByText('Details and advanced settings')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Edit Maximum objectives' })).toHaveTextContent('Up to 4') + expect(onChange).not.toHaveBeenCalled() + }) + + it('adds a provider immediately from a name-only dropdown without fetching it', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + const onLoad = jest.fn() + render() + await addDataset(user, 'remote') + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({ sources: [{ name: 'stored' }, { name: 'remote' }] }) + expect(screen.getByRole('button', { name: 'Edit remote objective limit' })).toHaveTextContent('Up to 4') + expect(screen.getByText('Load dataset to see details')).toBeInTheDocument() + expect(onLoad).not.toHaveBeenCalled() + await user.click(screen.getByRole('button', { name: 'Load datasets' })) + expect(onLoad).toHaveBeenCalledTimes(1) + }) + + it('keeps required reference sources complete and does not expose source edits', () => { + render() + expect(screen.getByText('8 examples')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Edit stored objective limit' })).not.toBeInTheDocument() + expect(screen.queryByRole('combobox', { name: 'Add dataset' })).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Remove stored' })).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Edit Maximum objectives' })).toBeInTheDocument() + }) + + it('supports inline numeric, all, and inherited objective limits', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + await editLimit(user, 'Maximum objectives', '12') + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({ max_total: 12 }) + await user.click(screen.getByRole('checkbox', { name: 'All available for Maximum objectives' })) + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({ max_total: 'all' }) + await user.click(screen.getByRole('button', { name: 'Reset to 4' })) + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({}) + }) + + it('keeps invalid global drafts blocking when datasets reset', async () => { + const user = userEvent.setup() + render() + await editLimit(user, 'Maximum objectives', '0') + await user.keyboard('{Escape}') + await user.click(screen.getByRole('button', { name: 'Reset datasets' })) + expect(screen.getByRole('button', { name: 'Launch' })).toBeDisabled() + expect(screen.getByRole('alert')).toHaveTextContent('Maximum objectives must be a positive integer') + await user.click(screen.getByRole('button', { name: 'Edit Maximum objectives' })) + expect(screen.getByRole('textbox', { name: 'Maximum objectives' })).toHaveValue('0') + await user.click(screen.getByRole('button', { name: 'Reset to 4' })) + expect(screen.getByRole('button', { name: 'Launch' })).toBeEnabled() + }) + + it('removes only the invalid row draft and restores other rows and limits on reset', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + await addDataset(user, 'other') + await editLimit(user, 'other objective limit', '0') + await user.keyboard('{Escape}') + expect(screen.getByRole('button', { name: 'Launch' })).toBeDisabled() + await user.click(screen.getByRole('button', { name: 'Remove other' })) + expect(screen.getByRole('button', { name: 'Launch' })).toBeEnabled() + expect(onChange.mock.calls.at(-1)?.[0].sources).toEqual([{ name: 'stored' }]) + expect(screen.queryByRole('button', { name: 'Remove stored' })).not.toBeInTheDocument() + await editLimit(user, 'stored objective limit', '3') + await user.keyboard('{Escape}') + await user.click(screen.getByRole('button', { name: 'Reset datasets' })) + expect(screen.getByRole('button', { name: 'Edit stored objective limit' })).toHaveTextContent('Up to 2 of 8') + }) + + it('replaces a one-source subset without permitting invalid or additional sources', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + await user.click(screen.getByRole('combobox', { name: 'Choose dataset' })) + expect(screen.queryByRole('option', { name: 'other', exact: true })).not.toBeInTheDocument() + await user.click(screen.getByRole('option', { name: 'remote', exact: true })) + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({ sources: [{ name: 'remote' }] }) + }) + + it('shares category and type filters across rows and preserves inherited filters when deselected', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + await addDataset(user, 'other') + const hate = screen.getAllByRole('button', { name: 'Filter harm category hate' }) + expect(hate.every((button) => button.getAttribute('aria-pressed') === 'true')).toBe(true) + await user.click(screen.getAllByRole('button', { name: 'Filter harm category violence' })[0]) + await user.click(hate[0]) + expect(onChange.mock.calls.at(-1)?.[0].filters).toEqual({ harm_categories: ['violence'] }) + expect(hate.every((button) => button.getAttribute('aria-pressed') === 'false')).toBe(true) + await user.click(hate[0]) + expect(onChange.mock.calls.at(-1)?.[0].filters.harm_categories).toEqual([]) + expect(screen.queryByText('Filters:')).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Remove type filter text' })).not.toBeInTheDocument() + }) + + it('preserves custom inherited filters without rendering a separate filter list', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + await user.click(screen.getByRole('button', { name: 'Filter harm category hate' })) + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({ filters: { harm_categories: ['custom-substring', 'hate'] } }) + expect(screen.queryByText('Filters:')).not.toBeInTheDocument() + }) + + it('starts with every label checked and restores the default request after reselecting all labels', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + expect(screen.getByRole('button', { name: 'Filter data type Text' })).toHaveAttribute('aria-pressed', 'true') + await user.click(screen.getByRole('button', { name: 'Filter harm category hate' })) + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({ filters: { harm_categories: ['violence'] } }) + await user.click(screen.getByRole('button', { name: 'Filter harm category hate' })) + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({}) + }) + + it('links to the dataset browser next to reset and for required sources', () => { + const view = render() + expect(screen.getByRole('link', { name: 'Browse datasets' })).toHaveAttribute('href', '/datasets') + expect(screen.getByRole('button', { name: 'Reset datasets' })).toBeInTheDocument() + view.rerender() + expect(screen.getByRole('link', { name: 'Browse datasets' })).toHaveAttribute('href', '/datasets') + }) + + it('blocks an empty selection instead of sending an empty list that means include all', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + await user.click(screen.getByRole('button', { name: 'Filter data type Text' })) + expect(screen.getByRole('button', { name: 'Filter data type Text' })).toHaveAttribute('aria-pressed', 'false') + expect(screen.getByRole('button', { name: 'Launch' })).toBeDisabled() + expect(screen.getByRole('alert')).toHaveTextContent('Select at least one data type.') + expect(onChange).not.toHaveBeenCalled() + await user.click(screen.getByRole('button', { name: 'Filter data type Text' })) + expect(screen.getByRole('button', { name: 'Launch' })).toBeEnabled() + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({}) + }) + + it('uses a single switch that updates all selected source loading policies', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + await addDataset(user, 'remote') + await user.click(screen.getByRole('switch', { name: 'Load missing data when the scan starts' })) + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({ + fetch: 'if_missing', sources: [{ name: 'stored', fetch: 'if_missing' }, { name: 'remote', fetch: 'if_missing' }], + }) + await user.click(screen.getByRole('switch', { name: 'Load missing data when the scan starts' })) + expect(onChange.mock.calls.at(-1)?.[0].sources.every((source: { fetch: string }) => source.fetch === 'never')).toBe(true) + expect(screen.getByRole('button', { name: 'Load datasets' })).toBeEnabled() + }) + + it('keeps missing-data loading on by default and hides all loading controls once data is loaded', () => { + const defaults = { ...DEFAULTS, sources: [{ name: 'remote', max_size: 4 as const, fetch: 'if_missing' as const, configured_provider: false }] } + const view = render() + expect(screen.getByRole('switch', { name: 'Load missing data when the scan starts' })).toBeChecked() + view.rerender() + expect(screen.queryByRole('button', { name: 'Load datasets' })).not.toBeInTheDocument() + expect(screen.queryByRole('switch')).not.toBeInTheDocument() + expect(screen.queryByText(/All selected datasets are loaded/)).not.toBeInTheDocument() + }) + + it('does not assume stored counts are exact selected sizes after filters and a total cap', () => { + render() + expect(screen.getByRole('button', { name: 'Edit stored objective limit' })).toHaveTextContent('Up to 1 of 8') + const row = screen.getByRole('row', { name: /stored/ }) + expect(within(row).queryByText('1 of 8', { exact: true })).not.toBeInTheDocument() + }) + + it('does not expose loading or source selection for an inline-only scenario', () => { + render() + expect(screen.getByText('This scenario has no named datasets.')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Load datasets' })).not.toBeInTheDocument() + expect(screen.queryByRole('combobox', { name: 'Add dataset' })).not.toBeInTheDocument() + }) + + it('does not pretend a global policy can replace protected source-specific loading rules', () => { + render() + expect(screen.getByRole('switch', { name: 'Load missing data when the scan starts' })).not.toBeChecked() + expect(screen.getByRole('switch', { name: 'Load missing data when the scan starts' })).toBeDisabled() + expect(screen.getByRole('button', { name: 'Load datasets' })).toBeEnabled() + }) +}) diff --git a/frontend/src/components/Scenarios/ScenarioDatasetTable.tsx b/frontend/src/components/Scenarios/ScenarioDatasetTable.tsx new file mode 100644 index 0000000000..ffe344634c --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioDatasetTable.tsx @@ -0,0 +1,253 @@ +import { useEffect, useState } from 'react' + +import { + Badge, Button, Combobox, Divider, MessageBar, MessageBarBody, Option, Switch, + Table, TableBody, TableCell, TableHeader, TableHeaderCell, TableRow, Text, ToggleButton, Tooltip, +} from '@fluentui/react-components' +import { DismissRegular } from '@fluentui/react-icons' +import { Link } from 'react-router' + +import { MODALITY_LABELS } from '@/components/Config/targetFilters' +import type { + DatasetInfo, DatasetFetchPolicy, ScenarioDatasetLimit, ScenarioDatasetOptions, ScenarioDatasetRequest, + ScenarioDatasetSettings, ScenarioDatasetSourceInput, +} from '@/types' + +import DatasetLimitControl from './DatasetLimitControl' +import { datasetChoices, datasetFacts, datasetSize, parseCaseLimit } from './datasetPresentation' +import { useScenarioDatasetTableStyles } from './ScenarioDatasetTable.styles' + +interface ScenarioDatasetTableProps { + request: ScenarioDatasetRequest + defaults: ScenarioDatasetSettings + effective: ScenarioDatasetSettings + options: ScenarioDatasetOptions + datasets: DatasetInfo[] + disabled: boolean + loading: boolean + metadataError: string | null + preparing?: boolean + loadError?: string | null + loadEnabled?: boolean + onLoad?: () => void + onReload: () => void + onChange: (request: ScenarioDatasetRequest) => void + onValidationChange: (error: string | null) => void +} + +export default function ScenarioDatasetTable({ + request, defaults, effective, options, datasets, disabled, loading, metadataError, + preparing = false, loadError, loadEnabled = false, onLoad, onReload, onChange, onValidationChange, +}: ScenarioDatasetTableProps) { + const styles = useScenarioDatasetTableStyles() + const [search, setSearch] = useState('') + const [drafts, setDrafts] = useState>({}) + const [emptySelections, setEmptySelections] = useState([]) + const selected: ScenarioDatasetSourceInput[] = request.sources ?? defaults.sources.map((source) => ({ name: source.name })) + const canSelect = options.supports_overrides && options.mode !== 'fixed' && selected.length > 0 + const canEditSources = options.supports_overrides && options.source_settings !== false + const canLimit = canEditSources && options.per_dataset_limit + const canFilter = options.supports_overrides && options.filters + const canAdd = options.max_selected == null || selected.length < options.max_selected + const replaceSingleSource = options.max_selected === 1 + const busy = disabled || preparing + const total = typeof request.max_total === 'number' || request.max_total === 'all' + ? request.max_total : defaults.max_total + const filters = { ...defaults.filters, ...request.filters } + const candidates = datasetChoices(datasets, defaults.sources, options) + .filter((name) => !selected.some((source) => source.name === name) && name.toLowerCase().includes(search.toLowerCase())) + const { loaded, allLoaded } = datasetFacts(datasets, selected.map((source) => source.name)) + const availableMetadata: Record = { + harm_categories: [...new Set(loaded.flatMap((dataset) => dataset.harm_categories))], + data_types: [...new Set(loaded.flatMap((dataset) => dataset.modalities))], + } + const selectedMetadata = (key: string): string[] => emptySelections.includes(key) + ? [] : filters[key]?.length ? filters[key] : availableMetadata[key] ?? [] + const metadataSelected = (key: string, value: string): boolean => selectedMetadata(key).includes(value) + const settingsFor = (name: string) => effective.sources.find((source) => source.name === name) + ?? defaults.sources.find((source) => source.name === name) + const requiresImport = !loading && !metadataError && selected.some((source) => { + const metadata = datasets.find((item) => item.name === source.name) + return !metadata?.loaded && !metadata?.provider_available && !settingsFor(source.name)?.configured_provider + }) + const updateDraft = (key: string, text: string | null): void => { + setDrafts((previous) => { + const next = { ...previous } + if (text === null) delete next[key] + else next[key] = text + return next + }) + } + const clearSourceDrafts = (name?: string): void => { + setDrafts((previous) => Object.fromEntries(Object.entries(previous).filter(([key]) => + name === undefined ? !key.startsWith('source:') : key !== `source:${name}`, + ))) + } + const updateTotal = (value: ScenarioDatasetLimit): void => { + const edits = { ...request } + if (value === 'default') delete edits.max_total + else edits.max_total = value + onChange(edits) + } + const toggleFilter = (key: string, value: string): void => { + const current = selectedMetadata(key) + const next = current.includes(value) ? current.filter((item) => item !== value) : [...current, value] + if (next.length === 0) { + setEmptySelections((previous) => [...new Set([...previous, key])]) + return + } + setEmptySelections((previous) => previous.filter((item) => item !== key)) + const edits = { ...request.filters } + const allSelected = (availableMetadata[key] ?? []).every((item) => next.includes(item)) + && next.every((item) => (availableMetadata[key] ?? []).includes(item)) + if (allSelected && !(defaults.filters[key]?.length)) delete edits[key] + else edits[key] = allSelected ? [] : next + const result = { ...request } + if (Object.keys(edits).length > 0) result.filters = edits + else delete result.filters + onChange(result) + } + const labels: Record = {} + if (options.supports_overrides && options.total_limit) labels.total = 'Maximum objectives' + if (canLimit) for (const source of selected) labels[`source:${source.name}`] = `${source.name} objective limit` + const errors = Object.entries(labels) + .filter(([key]) => drafts[key] !== undefined && parseCaseLimit(drafts[key]) === null) + .map(([, label]) => `${label} must be a positive integer.`) + if (canFilter) for (const key of emptySelections) { + if (availableMetadata[key]?.length) errors.push(`Select at least one ${key === 'harm_categories' ? 'harm category' : 'data type'}.`) + } + if (options.mode === 'subset') { + const invalid = selected.filter((source) => !options.allowed_names.includes(source.name)) + if (invalid.length > 0) errors.push(`Select a permitted source: ${invalid.map((source) => source.name).join(', ')}`) + } + const validationError = errors[0] ?? null + useEffect(() => { onValidationChange(validationError) }, [validationError, onValidationChange]) + + const policies = selected.map((source) => !canEditSources + ? settingsFor(source.name)?.fetch ?? defaults.fetch + : source.fetch ?? ( + request.fetch === 'never' || request.fetch === 'if_missing' ? request.fetch : settingsFor(source.name)?.fetch ?? defaults.fetch + )) + const mixedPolicies = policies.some((policy) => policy !== policies[0]) + const canSetLoading = options.supports_overrides && (canEditSources || ( + !mixedPolicies && policies.every((policy) => policy === effective.fetch) + )) + const changeLoading = (policy: DatasetFetchPolicy): void => onChange({ + ...request, fetch: policy, + ...(canEditSources ? { sources: selected.map((source) => ({ ...source, fetch: policy })) } : {}), + }) + const loadingDisabledReason = !options.supports_overrides + ? 'Loading rules are fixed by this scenario.' + : !canSetLoading ? 'This scenario uses source-specific loading rules.' : '' + + return ( +
+ {!allLoaded && selected.length > 0 &&
+ {onLoad && } + {requiresImport && Import the missing datasets into PyRIT first.} + {metadataError && } +
} + {[metadataError, loadError, ...errors].filter((error): error is string => Boolean(error)) + .map((error) => {error})} + {selected.length === 0 ? This scenario has no named datasets. :
+ + + Dataset + Size + Include Criteria + + + {selected.map((source) => { + const metadata = datasets.find((item) => item.name === source.name) + const sourceDefault = defaults.sources.find((item) => item.name === source.name)?.max_size ?? defaults.max_per_dataset + const inherited = request.max_per_dataset != null && request.max_per_dataset !== 'default' + ? settingsFor(source.name)?.max_size ?? sourceDefault : sourceDefault + const cap = typeof source.max_size === 'number' || source.max_size === 'all' ? source.max_size : inherited + const displayedCap = !canLimit || total === 'all' ? cap : cap === 'all' ? total : Math.min(cap, total) + const size = datasetSize(metadata, canLimit ? displayedCap : 'all') + return + {source.name} + {canLimit ? updateDraft(`source:${source.name}`, text)} + onChange={(value) => onChange({ ...request, sources: selected.map((item) => + item.name === source.name ? { ...item, max_size: value } : item) })} /> : {size}} + {metadata?.loaded ?
+ {metadata.harm_categories.map((category) => canFilter + ? toggleFilter('harm_categories', category)}> + {category} + + : {category})} + {metadata.modalities.map((type) => canFilter + ? toggleFilter('data_types', type)}> + {MODALITY_LABELS[type] ?? type} + + : {MODALITY_LABELS[type] ?? type})} + {metadata.has_unlabeled_harm_categories && + Unlabeled + } + {metadata.harm_categories.length === 0 && metadata.modalities.length === 0 && No metadata labels.} +
: {loading ? 'Checking datasets...' : 'Load dataset to see details'}} + + {canSelect && selected.length > 1 &&
+
} + {selected.length > 0 &&
+ {canSelect && <> + {(canAdd || replaceSingleSource) && { if (data.open) onReload() }} + onChange={(event) => setSearch(event.target.value)} + onOptionSelect={(_, data) => { + const name = data.optionValue + if (!name || !candidates.includes(name)) return + if (replaceSingleSource) clearSourceDrafts() + onChange({ ...request, sources: replaceSingleSource ? [{ name }] : [...selected, { name }] }) + setSearch('') + }}> + {candidates.map((name) => )} + } + + } + Browse datasets +
} + {selected.length > 0 && } + {canFilter && emptySelections.length > 0 && } + {!allLoaded && selected.length > 0 && + + policy === 'if_missing')} + disabled={busy || !canSetLoading} onChange={(_, data) => changeLoading(data.checked ? 'if_missing' : 'never')} /> + + } + {!allLoaded && mixedPolicies && The selected datasets currently use different loading rules.} + {options.supports_overrides && options.total_limit && updateDraft('total', text)} onChange={updateTotal} />} + {options.total_limit && Each objective can run through several techniques} +
+ ) +} diff --git a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx index f63684c869..767130382c 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx @@ -7,6 +7,7 @@ import { scenariosApi, targetsApi } from '@/services/api' import { useRuntime } from '@/hooks/useRuntime' import type { RegisteredScenario, + ScenarioRunSizeEstimateRequest, ScenarioRunSizeEstimateResponse, TargetInstance, } from '@/types' @@ -16,6 +17,11 @@ import ScenarioDetail from './ScenarioDetail' jest.mock('@/hooks/useRuntime', () => ({ useRuntime: jest.fn() })) jest.mock('@/services/api', () => ({ + datasetsApi: { listDatasets: jest.fn().mockResolvedValue({ items: ['harmbench', 'ds_a', 'ds_b', 'remote', 'stale-dataset'].map((name) => ({ + name, selection_key: `dataset:named:${name}`, loaded: true, provider_available: true, + logical_examples: 8, seed_pieces: null, objectives: null, modalities: ['text', 'image_path', 'audio_path'], + harm_categories: ['cyber', 'violence', 'other'], has_unlabeled_harm_categories: false, + })) }) }, scenariosApi: { estimateRun: jest.fn(), getScenario: jest.fn(), @@ -162,6 +168,20 @@ async function confirmRunPreview(user: ReturnType): Prom await user.click(screen.getByTestId('confirm-launch-scenario-btn')) } +async function openObjectiveLimit(user: ReturnType): Promise { + await user.click(await screen.findByRole('button', { name: 'Edit Maximum objectives' })) +} + +async function setObjectiveLimit(user: ReturnType, value: string): Promise { + await openObjectiveLimit(user) + const all = screen.getByRole('checkbox', { name: 'All available for Maximum objectives' }) + if (all instanceof HTMLInputElement && all.checked) await user.click(all) + const input = screen.getByRole('textbox', { name: 'Maximum objectives' }) + await user.clear(input) + await user.type(input, value) + await user.keyboard('{Escape}') +} + function renderDetail( path: string, props: Partial> = {}, @@ -306,18 +326,19 @@ describe('ScenarioDetail', () => { await flushRenderedPromises() await user.type(screen.getByLabelText(/^iterations/), '1.5') await user.type(screen.getByLabelText(/^note/), 'Keep this value') - await user.type(screen.getByLabelText('Dataset override'), 'stale-dataset') - await user.type(screen.getByLabelText('Max dataset size'), '0') - await user.type(screen.getByLabelText('Harm categories'), 'stale-harm') - await user.type(screen.getByLabelText('Data types'), 'stale-type') + await user.click(screen.getByRole('combobox', { name: 'Add dataset' })) + await user.click(screen.getByRole('option', { name: 'stale-dataset', exact: true })) + await setObjectiveLimit(user, '0') + await user.click(screen.getAllByRole('button', { name: 'Filter harm category cyber' })[0]) + await user.click(screen.getAllByRole('button', { name: 'Filter data type Text' })[0]) for (const label of ['Max concurrency', 'Max retries']) { const input = screen.getByRole('spinbutton', { name: label }) await user.clear(input) await user.type(input, '500') await user.tab() } - await user.click(screen.getByRole('button', { name: 'Launch scan' })) - expect(screen.getByRole('alert')).toHaveTextContent('iterations must be an integer.') + expect(screen.getByRole('button', { name: 'Launch scan' })).toBeDisabled() + expect(screen.getByRole('alert')).toHaveTextContent('Maximum objectives must be a positive integer') mockGetScenario.mockResolvedValue(makeScenario({ supported_parameter_names: ['note'], @@ -476,8 +497,9 @@ describe('ScenarioDetail', () => { expect(within(screen.getByRole('region', { name: 'Objective Target' })).queryByRole( 'combobox', { name: 'Adversarial Target' }, )).not.toBeInTheDocument() - expect(fallback.compareDocumentPosition(within(parameters).getByRole('textbox', { name: 'Dataset override' }))) - .toBe(Node.DOCUMENT_POSITION_FOLLOWING) + const datasetsSection = screen.getByRole('region', { name: 'Datasets' }) + expect(within(datasetsSection).getByRole('table', { name: 'Selected datasets' })).toBeInTheDocument() + expect(datasetsSection.compareDocumentPosition(parameters)).toBe(Node.DOCUMENT_POSITION_FOLLOWING) expect(fallback).toHaveAccessibleDescription('The registered target this scenario uses to generate attacks.') expect(fallback).toHaveValue('adversary') expect(within(fallback).queryByRole('option', { name: 'target-a' })).not.toBeInTheDocument() @@ -1165,10 +1187,7 @@ describe('ScenarioDetail', () => { expect(await screen.findByRole('heading', { name: 'Parameters' })).toBeInTheDocument() expect(screen.queryByText('Advanced options')).not.toBeInTheDocument() - expect(screen.getByTestId('max-dataset-size-input')).toHaveValue(8) - expect(screen.getByText( - 'The scenario default is 8. Edit it to override the default.', - )).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Edit Maximum objectives' })).toHaveTextContent('Up to 8') const estimate = screen.getByTestId('run-estimate') expect(within(estimate).getByText('Dataset size')).toBeInTheDocument() expect(within(estimate).getByText('8')).toBeInTheDocument() @@ -1198,7 +1217,7 @@ describe('ScenarioDetail', () => { renderDetail('/scanner/foundry.red_team_agent') - expect(await screen.findByTestId('max-dataset-size-input')).toHaveValue(5) + expect(await screen.findByRole('button', { name: 'Edit Maximum objectives' })).toHaveTextContent('Up to 5') expect(await screen.findByText('Up to 10')).toBeInTheDocument() expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size') }) @@ -1212,7 +1231,7 @@ describe('ScenarioDetail', () => { }, })) renderDetail('/scanner/foundry.red_team_agent') - expect(await screen.findByRole('spinbutton', { name: 'Max dataset size' })).toHaveValue(null) + expect(await screen.findByRole('button', { name: 'Edit Maximum objectives' })).toHaveTextContent('All available') await waitFor(() => expect(mockEstimateRun).toHaveBeenCalled()) expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size') }) @@ -1226,9 +1245,8 @@ describe('ScenarioDetail', () => { }, })) renderDetail('/scanner/foundry.red_team_agent') - const input = await screen.findByRole('spinbutton', { name: 'Max dataset size' }) - expect(input).toBeDisabled() - expect(input).toHaveValue(null) + await screen.findByRole('table', { name: 'Selected datasets' }) + expect(screen.queryByRole('textbox', { name: 'Maximum objectives' })).not.toBeInTheDocument() expect(screen.queryByRole('button', { name: 'Use all data' })).not.toBeInTheDocument() }) @@ -1242,9 +1260,10 @@ describe('ScenarioDetail', () => { }, })) renderDetail('/scanner/foundry.red_team_agent') - const input = await screen.findByRole('spinbutton', { name: 'Max dataset size' }) - expect(input).toHaveValue(5) - await user.clear(input) + await setObjectiveLimit(user, '2') + await openObjectiveLimit(user) + await user.click(screen.getByRole('button', { name: 'Reset to 5' })) + await user.keyboard('{Escape}') expect(screen.queryByRole('button', { name: 'Use all data' })).not.toBeInTheDocument() await waitFor(() => expect(mockEstimateRun).toHaveBeenCalled()) expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size') @@ -1256,13 +1275,14 @@ describe('ScenarioDetail', () => { it('sends an explicit all total for both estimation and launch', async () => { const user = userEvent.setup() renderDetail('/scanner/foundry.red_team_agent') - const unlimited = await screen.findByRole('checkbox', { name: 'No total dataset limit' }) - await user.click(unlimited) - expect(screen.getByRole('spinbutton', { name: 'Max dataset size' })).toBeDisabled() - await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toHaveProperty('max_dataset_size', 'all')) + await setObjectiveLimit(user, '2') + await openObjectiveLimit(user) + await user.click(screen.getByRole('checkbox', { name: 'All available for Maximum objectives' })) + await user.keyboard('{Escape}') + await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toHaveProperty('datasets.max_total', 'all')) await confirmRunPreview(user) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) - expect(mockStartRun.mock.calls[0][0]).toHaveProperty('max_dataset_size', 'all') + expect(mockStartRun.mock.calls[0][0]).toHaveProperty('datasets.max_total', 'all') }) it('restores the scenario default when no-total-limit is unchecked', async () => { @@ -1274,11 +1294,11 @@ describe('ScenarioDetail', () => { }, })) renderDetail('/scanner/foundry.red_team_agent') - const unlimited = await screen.findByRole('checkbox', { name: 'No total dataset limit' }) - await user.click(unlimited) - await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toHaveProperty('max_dataset_size', 'all')) - await user.click(unlimited) - expect(screen.getByRole('spinbutton', { name: 'Max dataset size' })).toHaveValue(5) + await openObjectiveLimit(user) + await user.click(screen.getByRole('checkbox', { name: 'All available for Maximum objectives' })) + await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toHaveProperty('datasets.max_total', 'all')) + await user.click(screen.getByRole('button', { name: 'Reset to 5' })) + await user.keyboard('{Escape}') await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size')) await confirmRunPreview(user) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) @@ -1290,17 +1310,22 @@ describe('ScenarioDetail', () => { renderDetail('/scanner/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') - await user.type(screen.getByTestId('dataset-override-input'), 'ds_a, ds_b') - await user.type(screen.getByTestId('max-dataset-size-input'), '25') - await user.type(screen.getByTestId('harm-categories-filter-input'), 'cyber, violence') - await user.type(screen.getByTestId('data-types-filter-input'), 'text, image_path') + const addDataset = screen.getByRole('combobox', { name: 'Add dataset' }) + await user.click(addDataset) + await user.click(screen.getByRole('option', { name: 'ds_a', exact: true })) + await user.click(addDataset) + await user.click(screen.getByRole('option', { name: 'ds_b', exact: true })) + await user.click(screen.getByRole('button', { name: 'Remove harmbench' })) + await setObjectiveLimit(user, '25') + await user.click(screen.getAllByRole('button', { name: 'Filter harm category other' })[0]) + await user.click(screen.getAllByRole('button', { name: 'Filter data type Audio' })[0]) await confirmRunPreview(user) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) const request = mockStartRun.mock.calls[0][0] - expect(request.dataset_names).toEqual(['ds_a', 'ds_b']) - expect(request.max_dataset_size).toBe(25) - expect(request.dataset_filters).toEqual({ + expect(request.datasets.sources).toEqual([{ name: 'ds_a' }, { name: 'ds_b' }]) + expect(request.datasets.max_total).toBe(25) + expect(request.datasets.filters).toEqual({ harm_categories: ['cyber', 'violence'], data_types: ['text', 'image_path'], }) @@ -1309,11 +1334,13 @@ describe('ScenarioDetail', () => { expect.objectContaining({ target_name: 'target-a', techniques: ['default_technique'], - dataset_names: ['ds_a', 'ds_b'], - max_dataset_size: 25, - dataset_filters: { + datasets: { + sources: [{ name: 'ds_a' }, { name: 'ds_b' }], + max_total: 25, + filters: { harm_categories: ['cyber', 'violence'], data_types: ['text', 'image_path'], + }, }, include_baseline: true, }), @@ -1322,17 +1349,62 @@ describe('ScenarioDetail', () => { expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('labels') }) + it('refreshes parameter-aware defaults after invalid source edits without silently dropping them', async () => { + const user = userEvent.setup() + mockGetScenario.mockResolvedValueOnce(makeScenario({ + supported_parameters: [ + { name: 'scope', type_name: 'str', required: false, default: 'all', choices: null, is_list: false }, + ], + })) + mockEstimateRun.mockImplementation(async (_name: string, request: ScenarioRunSizeEstimateRequest) => { + const narrowed = request.scenario_params?.scope === 'one' + if (narrowed && request.datasets?.sources) { + throw { isAxiosError: true, response: { status: 400, data: { detail: 'Dataset is outside the selected scope.' } } } + } + const name = narrowed ? 'remote' : 'harmbench' + const settings = { + sources: [{ name, max_size: 5, fetch: 'never', configured_provider: false }], + max_per_dataset: 5, max_total: 5, fetch: 'never', filters: {}, + } + return { + ...makeEstimate(10), dataset_defaults: settings, effective_datasets: settings, + dataset_options: { + mode: 'subset', allowed_names: [name], per_dataset_limit: true, total_limit: true, + supports_overrides: true, filters: true, + }, + } + }) + renderDetail('/scanner/foundry.red_team_agent') + await screen.findByRole('table', { name: 'Selected datasets' }) + await waitFor(() => expect(screen.getByRole('button', { name: 'Edit harmbench objective limit' })).toHaveTextContent('Up to 5')) + await user.click(screen.getByRole('button', { name: 'Edit harmbench objective limit' })) + const all = screen.getByRole('checkbox', { name: 'All available for harmbench objective limit' }) + if (all instanceof HTMLInputElement && all.checked) await user.click(all) + await user.clear(screen.getByRole('textbox', { name: 'harmbench objective limit' })) + await user.type(screen.getByRole('textbox', { name: 'harmbench objective limit' }), '1') + await user.keyboard('{Escape}') + await user.clear(screen.getByTestId('scenario-param-scope')) + await user.type(screen.getByTestId('scenario-param-scope'), 'one') + await screen.findByText('Select a permitted source: harmbench') + expect(screen.getByRole('button', { name: 'Edit harmbench objective limit' })).toHaveTextContent('Up to 1') + expect(mockEstimateRun.mock.calls.some((call) => + call[1].scenario_params?.scope === 'one' && call[1].datasets === undefined, + )).toBe(true) + await user.click(screen.getByRole('button', { name: 'Reset datasets' })) + await screen.findByText('remote', { exact: true }) + }) + it('rejects a non-positive-integer max dataset size', async () => { const user = userEvent.setup() renderDetail('/scanner/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') - await user.type(screen.getByTestId('max-dataset-size-input'), '0') + await setObjectiveLimit(user, '0') await user.click(screen.getByTestId('launch-scenario-btn')) - expect(await screen.findByRole('alert')).toHaveTextContent( - 'Max dataset size must be a positive integer.', - ) + expect(await screen.findAllByRole('alert')).toEqual(expect.arrayContaining([ + expect.objectContaining({ textContent: expect.stringContaining('Maximum objectives must be a positive integer.') }), + ])) expect(mockStartRun).not.toHaveBeenCalled() }) diff --git a/frontend/src/components/Scenarios/ScenarioDetail.tsx b/frontend/src/components/Scenarios/ScenarioDetail.tsx index afdacba996..cb1215b1fb 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.tsx @@ -11,7 +11,6 @@ import { DialogSurface, DialogTitle, Field, - Input, Radio, MessageBar, MessageBarBody, @@ -33,6 +32,7 @@ import { Link, useNavigate, useParams } from 'react-router' import MarkdownContent from '@/components/Markdown/MarkdownContent' import TargetSelect from '@/components/Config/TargetSelect' import { useRuntime } from '@/hooks/useRuntime' +import { useScenarioDatasets } from '@/hooks/useScenarioDatasets' import ParameterField from '@/components/Parameters/ParameterField' import SingleStepSpinButton from '@/components/Parameters/SingleStepSpinButton' import { @@ -52,6 +52,10 @@ import type { ScenarioRunSizeEstimateRequest, ScenarioRunEstimateState, ScenarioTechniqueSummary, + ScenarioDatasetRequest, + ScenarioDatasetSettings, + ScenarioDatasetOptions, + ScenarioRunSizeEstimateResponse, TargetInstance, } from '@/types' import { routerPathParamValue, scenarioRunRoutePath } from '@/utils/routeParams' @@ -62,6 +66,7 @@ import { ScenarioRunEstimateDetails } from './ScenarioRunEstimate' import { normalizeScenarioMarkdown } from './scenarioMarkdown' import { mapScenarioRunEstimate } from './scenarioRunEstimateAdapter' import { techniqueSetName } from './scenarioTechniqueSets' +import ScenarioDatasetTable from './ScenarioDatasetTable' /** * Common/opaque parameters the base scenario declares via @@ -75,6 +80,7 @@ const COMMON_SCENARIO_PARAMETER_NAMES = new Set([ 'scenario_techniques', 'technique_converters', 'dataset_config', + 'dataset_overrides', 'memory_labels', 'max_concurrency', 'max_retries', @@ -100,11 +106,6 @@ function targetOptionLabel(target: TargetInstance): string { : target.target_registry_name } -function defaultMaxDatasetSize(scenario: RegisteredScenario): string { - const limit = scenario.default_run_size.dataset_limit - return limit.state === 'value' ? String(limit.value) : '' -} - /** Resolves a Fluent `SpinButton` change event to a numeric value, preferring the parsed `value` over the raw `displayValue`. */ function resolveSpinButtonValue(data: { value?: number | null; displayValue?: string }, previous: number): number { if (typeof data.value === 'number') { @@ -153,13 +154,6 @@ const BASELINE_TECHNIQUE: ScenarioTechniqueSummary = { tags: ['baseline', 'single_turn'], } -function parseDatasetNames(datasetOverride: string): string[] { - return datasetOverride - .split(',') - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0) -} - function formatParameterPreview(value: ParameterFormValue | undefined): string { if (Array.isArray(value)) { return value.length > 0 ? value.join(', ') : 'Not set' @@ -239,10 +233,8 @@ interface BuildEstimateRequestInput { techniques: string[] dynamicParameters: Parameter[] scenarioParamValues: Record - datasetOverride: string - maxDatasetSize: string - harmCategoriesFilter: string - dataTypesFilter: string + datasetEdits: ScenarioDatasetRequest + datasetError: string | null includeBaseline: boolean } @@ -296,10 +288,8 @@ function buildEstimateRequest({ techniques, dynamicParameters, scenarioParamValues, - datasetOverride, - maxDatasetSize, - harmCategoriesFilter, - dataTypesFilter, + datasetEdits, + datasetError, includeBaseline, }: BuildEstimateRequestInput): BuildEstimateRequestResult { if (supportsParameter(scenario, 'scenario_techniques') && techniques.length === 0) { @@ -319,19 +309,8 @@ function buildEstimateRequest({ scenarioParams = result.parameters } - let maxDatasetSizeValue: number | 'all' | undefined const supportsDatasets = supportsParameter(scenario, 'dataset_config') - const trimmedMaxDatasetSize = supportsDatasets ? maxDatasetSize.trim() : '' - if (trimmedMaxDatasetSize === 'all') { - maxDatasetSizeValue = 'all' - } else if (trimmedMaxDatasetSize.length > 0) { - const parsed = Number(trimmedMaxDatasetSize) - if (!Number.isInteger(parsed) || parsed < 1) { - return { ok: false, error: 'Max dataset size must be a positive integer.' } - } - maxDatasetSizeValue = parsed - } - const datasetNames = supportsDatasets ? parseDatasetNames(datasetOverride) : [] + if (supportsDatasets && datasetError) return { ok: false, error: datasetError } const request: ScenarioRunSizeEstimateRequest = {} if (supportsParameter(scenario, 'scenario_techniques')) { request.techniques = techniques @@ -345,19 +324,8 @@ function buildEstimateRequest({ if (scenario.uses_default_adversarial_target && adversarialTargetName) { request.adversarial_target_name = adversarialTargetName } - if (datasetNames.length > 0) { - request.dataset_names = datasetNames - } - if (maxDatasetSizeValue !== undefined) { - request.max_dataset_size = maxDatasetSizeValue - } - const harmCategories = supportsDatasets ? parseDatasetNames(harmCategoriesFilter) : [] - const dataTypes = supportsDatasets ? parseDatasetNames(dataTypesFilter) : [] - if (harmCategories.length > 0 || dataTypes.length > 0) { - request.dataset_filters = { - ...(harmCategories.length > 0 ? { harm_categories: harmCategories } : {}), - ...(dataTypes.length > 0 ? { data_types: dataTypes } : {}), - } + if (supportsDatasets && Object.keys(datasetEdits).length > 0) { + request.datasets = datasetEdits } if (scenarioParams) { request.scenario_params = scenarioParams @@ -634,14 +602,31 @@ function ScenarioLaunchForm({ const [baselineChecked, setBaselineChecked] = useState( () => !isBaselineForbidden && scenario.include_baseline_by_default, ) - const [datasetOverride, setDatasetOverride] = useState('') - const configuredDefaultMaxDatasetSize = useMemo( - () => defaultMaxDatasetSize(scenario), - [scenario], - ) - const [maxDatasetSize, setMaxDatasetSize] = useState(configuredDefaultMaxDatasetSize) - const [harmCategoriesFilter, setHarmCategoriesFilter] = useState('') - const [dataTypesFilter, setDataTypesFilter] = useState('') + const [datasetEdits, setDatasetEdits] = useState({}) + const [datasetError, setDatasetError] = useState(null) + const datasetMetadata = useScenarioDatasets() + const [datasetProjection, setDatasetProjection] = useState(null) + const datasetDefaults: ScenarioDatasetSettings = datasetProjection?.dataset_defaults ?? scenario.dataset_defaults ?? { + sources: scenario.default_datasets.map((name) => ({ name, max_size: 'all', fetch: 'if_missing', configured_provider: false })), + max_per_dataset: 'all', max_total: scenario.default_run_size.dataset_limit.value ?? 'all', + fetch: 'if_missing', filters: {}, + } + const effectiveDatasets = datasetProjection?.effective_datasets ?? datasetDefaults + const datasetOptions: ScenarioDatasetOptions = datasetProjection?.dataset_options ?? scenario.dataset_options ?? { + mode: 'any', allowed_names: [], per_dataset_limit: true, + total_limit: scenario.default_run_size.dataset_limit.state !== 'not_applicable', + supports_overrides: true, filters: true, + } + const selectedDatasetNames = (datasetEdits.sources ?? datasetDefaults.sources).map((source) => source.name) + const datasetSelectionKey = JSON.stringify(selectedDatasetNames) + const reloadDatasetMetadata = datasetMetadata.reload + const previousDatasetSelection = useRef(datasetSelectionKey) + useEffect(() => { + if (previousDatasetSelection.current !== datasetSelectionKey) { + previousDatasetSelection.current = datasetSelectionKey + reloadDatasetMetadata() + } + }, [datasetSelectionKey, reloadDatasetMetadata]) const [maxConcurrency, setMaxConcurrency] = useState(DEFAULT_MAX_CONCURRENCY) const [maxRetries, setMaxRetries] = useState(DEFAULT_MAX_RETRIES) const [scenarioParamValues, setScenarioParamValues] = useState>(() => @@ -692,15 +677,8 @@ function ScenarioLaunchForm({ [isBaselineForbidden, supportsBaseline, supportsTechniques, techniqueOptions], ) const techniques = selectedTechniques - const maxDatasetSizeOverride = maxDatasetSize.trim() - && maxDatasetSize !== configuredDefaultMaxDatasetSize - ? maxDatasetSize - : '' - const datasetSizeLabel = scenario.default_run_size.dataset_limit.state === 'not_applicable' - ? 'Not applicable' - : maxDatasetSize === 'all' - ? 'No total limit' - : maxDatasetSize.trim() || configuredDefaultMaxDatasetSize || 'Scenario default' + const datasetSizeLabel = !datasetOptions.total_limit ? 'Not applicable' + : effectiveDatasets.max_total === 'all' ? 'No total limit' : String(effectiveDatasets.max_total) const estimateResult = useMemo( () => buildEstimateRequest({ scenario, @@ -709,21 +687,17 @@ function ScenarioLaunchForm({ techniques, dynamicParameters, scenarioParamValues, - datasetOverride, - maxDatasetSize: maxDatasetSizeOverride, - harmCategoriesFilter, - dataTypesFilter, + datasetEdits, + datasetError, includeBaseline: isBaselineForbidden ? false : baselineChecked, }), [ adversarialTargetName, baselineChecked, - datasetOverride, - dataTypesFilter, + datasetEdits, + datasetError, dynamicParameters, - harmCategoriesFilter, isBaselineForbidden, - maxDatasetSizeOverride, scenario, scenarioParamValues, targetName, @@ -738,10 +712,8 @@ function ScenarioLaunchForm({ techniques, dynamicParameters, scenarioParamValues, - datasetOverride, - maxDatasetSize: maxDatasetSizeOverride, - harmCategoriesFilter, - dataTypesFilter, + datasetEdits, + datasetError, maxConcurrency, maxRetries, includeBaseline: isBaselineForbidden ? false : baselineChecked, @@ -750,14 +722,12 @@ function ScenarioLaunchForm({ [ adversarialTargetName, baselineChecked, - datasetOverride, - dataTypesFilter, + datasetEdits, + datasetError, dynamicParameters, - harmCategoriesFilter, isBaselineForbidden, labels, maxConcurrency, - maxDatasetSizeOverride, maxRetries, scenario, scenarioParamValues, @@ -796,6 +766,7 @@ function ScenarioLaunchForm({ return } const result = mapScenarioRunEstimate(response, 'request') + setDatasetProjection(response) setEstimateRequestState({ status: 'resolved', requestKey: estimateRequestKey, @@ -812,11 +783,26 @@ function ScenarioLaunchForm({ ) { return } + const error = toApiError(err) setEstimateRequestState({ status: 'error', requestKey: estimateRequestKey, - error: toApiError(err).detail, + error: error.detail, }) + if (estimateRequest.datasets && (error.status === 400 || error.status === 422)) { + const { datasets, ...defaultRequest } = estimateRequest + void datasets + // Keep invalid edits visible while refreshing parameter-aware allowed sources. + void scenariosApi.estimateRun(scenario.scenario_name, defaultRequest, controller.signal) + .then((response) => { + if (controller.signal.aborted || requestSequence !== estimateSequenceRef.current) return + setDatasetProjection((previous) => ({ + ...response, + effective_datasets: previous?.effective_datasets ?? response.effective_datasets, + })) + }) + .catch(() => { /* The original validation error remains authoritative. */ }) + } }) }, ESTIMATE_DEBOUNCE_MS) @@ -938,6 +924,7 @@ function ScenarioLaunchForm({ try { const summary = await scenariosApi.startRun(requestResult.request) + datasetMetadata.reload() setPreviewOpen(false) navigate(scenarioRunRoutePath(summary.scenario_result_id), { state: { scenarioName: scenario.scenario_name }, @@ -972,10 +959,11 @@ function ScenarioLaunchForm({ const displayedEstimate = estimateFromState(estimateState) const selectedTechniqueCount = (supportsTechniques ? selectedTechniques.length : 0) + (supportsBaseline && baselineChecked && !isBaselineForbidden ? 1 : 0) - const previewDatasets = parseDatasetNames(datasetOverride) - const effectiveDatasets = previewDatasets.length > 0 ? previewDatasets : scenario.default_datasets - const previewHarmCategories = parseDatasetNames(harmCategoriesFilter) - const previewDataTypes = parseDatasetNames(dataTypesFilter) + const previewDatasets = (datasetEdits.sources ?? datasetDefaults.sources).map((source) => source.name) + const previewDatasetNames = previewDatasets.length > 0 ? previewDatasets : scenario.default_datasets + const previewFilters = { ...datasetDefaults.filters, ...datasetEdits.filters } + const previewHarmCategories = previewFilters.harm_categories ?? [] + const previewDataTypes = previewFilters.data_types ?? [] return (
)} + {(supportsDatasets || datasetDefaults.sources.length > 0) && ( +
+ Datasets + { if (estimateRequest) void datasetMetadata.loadSelected(scenario.scenario_name, estimateRequest) }} + onReload={datasetMetadata.reload} onChange={setDatasetEdits} onValidationChange={setDatasetError} + /> +
+ )} + {(dynamicParameters.length > 0 || scenario.uses_default_adversarial_target - || supportsDatasets || supportsConcurrency || supportsRetries) && ( + || supportsConcurrency || supportsRetries) && (
Parameters @@ -1196,75 +1200,6 @@ function ScenarioLaunchForm({ disabled={submitting} /> )} - {supportsDatasets && ( - <> - - setDatasetOverride(data.value)} - placeholder={scenario.default_datasets.join(', ') || undefined} - data-testid="dataset-override-input" - /> - - - setMaxDatasetSize(data.value)} - data-testid="max-dataset-size-input" - /> - - setMaxDatasetSize(data.checked === true ? 'all' : configuredDefaultMaxDatasetSize)} - /> - Per-dataset limits still apply. An empty input uses the scenario default total. - - setHarmCategoriesFilter(data.value)} - data-testid="harm-categories-filter-input" - /> - - - setDataTypesFilter(data.value)} - data-testid="data-types-filter-input" - /> - - - )} {supportsConcurrency && Launch scan @@ -1416,8 +1351,8 @@ function ScenarioLaunchForm({
- {effectiveDatasets.length > 0 - ? effectiveDatasets.join(', ') + {previewDatasetNames.length > 0 + ? previewDatasetNames.join(', ') : 'No datasets declared'} @@ -1487,7 +1422,7 @@ function ScenarioLaunchForm({