Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ impl Engine {
///
/// Returns the content for the AWF `-- '<command>'` argument, including the
/// binary path, prompt delivery flag, MCP config flag, and all CLI arguments.
/// The engine controls how the prompt is provided (e.g., `--prompt "$(cat ...)"`
/// The engine controls how the prompt is provided (e.g., `--prompt="$(cat ...)"`
/// for Copilot) and how MCP config is referenced.
///
/// `prompt_path` is the path to the prompt file inside the AWF container.
Expand Down Expand Up @@ -1355,7 +1355,7 @@ fn copilot_invocation(
) -> String {
let mut parts = vec![
command_path.to_string(),
format!("--prompt \"$(cat {prompt_path})\""),
format!("--prompt=\"$(cat {prompt_path})\""),
];

if let Some(mcp_path) = mcp_config_path {
Expand Down
2 changes: 1 addition & 1 deletion tests/awf-copilot-safeoutputs/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ chmod 600 "${TOOLS_DIR}/agent-prompt.md"

readonly ALLOWED_DOMAINS="api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,config.edge.skype.com,copilot-proxy.githubusercontent.com,github.com,telemetry.enterprise.githubcopilot.com,*.copilot.github.com,*.githubcopilot.com"
# shellcheck disable=SC2016 # AWF expands the engine command inside the sandbox.
readonly ENGINE_RUN='export NO_PROXY="${NO_PROXY:+$NO_PROXY,}awmg-mcpg"; export no_proxy="$NO_PROXY"; /tmp/awf-tools/copilot --prompt "$(cat /tmp/awf-tools/agent-prompt.md)" --additional-mcp-config @/tmp/awf-tools/mcp-config.json --model gpt-5-mini --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-tool safeoutputs --allow-all-paths'
readonly ENGINE_RUN='export NO_PROXY="${NO_PROXY:+$NO_PROXY,}awmg-mcpg"; export no_proxy="$NO_PROXY"; /tmp/awf-tools/copilot --prompt="$(cat /tmp/awf-tools/agent-prompt.md)" --additional-mcp-config @/tmp/awf-tools/mcp-config.json --model gpt-5-mini --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-tool safeoutputs --allow-all-paths'

set +e
"${AWF_BIN}" \
Expand Down
161 changes: 159 additions & 2 deletions tests/compiler_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1890,11 +1890,26 @@ Call the noop tool exactly once.

assert!(
agent.contains(
"/tmp/awf-tools/copilot --prompt \"$(cat /tmp/awf-tools/agent-prompt.md)\" \
"/tmp/awf-tools/copilot --prompt=\"$(cat /tmp/awf-tools/agent-prompt.md)\" \
--additional-mcp-config @/tmp/awf-tools/mcp-config.json"
),
"agent job should pass compiler-emitted MCP config to Copilot CLI: {agent}"
);
assert!(
!agent.contains("--prompt \"$(cat "),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No test actually reproduces the reported regression

This assertion (and the twin at line 1911) only pins the literal generated string; nothing in this PR exercises the actual failure mode from #2186 — a prompt value that begins with --- being mis-parsed as another CLI option. The PR description even says the real verification was "Manual parse check with pinned Copilot CLI and a prompt beginning with ---", which never lands in CI.

A future refactor of copilot_invocation (e.g. reintroducing a space, or changing the quoting/escaping) could still reintroduce the bug while this string-equality assertion stays green if the refactor happens to produce a different-but-still-broken shape.

💡 Suggested strengthening

Add a case where the compiled prompt content itself starts with --- (e.g. a markdown body with YAML-frontmatter-like content) and assert the emitted invocation still has exactly one --prompt= token — ideally by shell-tokenizing the ENGINE_RUN-equivalent string (e.g. via shlex/shell-words in a small helper) and asserting the prompt argument round-trips as a single token, rather than only substring-matching the exact literal.

"agent job should not pass prompt as a separate option value: {agent}"
);
assert!(
detection.contains(
"/tmp/awf-tools/copilot --prompt=\"$(cat \
/tmp/awf-tools/threat-analysis-prompt.md)\""
),
"detection job should pass prompt using attached form: {detection}"
);
assert!(
!detection.contains("--prompt \"$(cat "),
"detection job should not pass prompt as a separate option value: {detection}"
);
assert!(
agent.contains("--allow-all-tools"),
"default unrestricted tools path should emit --allow-all-tools: {agent}"
Expand Down Expand Up @@ -1923,6 +1938,45 @@ Call the noop tool exactly once.
);
}

#[test]
fn test_runtime_import_frontmatter_prompt_uses_attached_copilot_prompt_flag() {
let fixture_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests")
.join("fixtures")
.join("runtime_imports_standalone.md");
let fixture = fs::read_to_string(&fixture_path).expect("fixture should be readable");
assert!(
fixture.starts_with("---"),
"regression fixture should produce a runtime-import prompt beginning with frontmatter"
);

let compiled = compile_fixture("runtime_imports_standalone.md");
let agent = extract_job_block(&compiled, "Agent").expect("Agent job should exist");
let detection = extract_job_block(&compiled, "Detection").expect("Detection job should exist");

assert!(
compiled.contains("{{#runtime-import runtime_imports_standalone.md}}"),
"agent prompt should runtime-import the full markdown fixture: {compiled}"
);
assert!(
agent.contains("/tmp/awf-tools/copilot --prompt=\"$(cat /tmp/awf-tools/agent-prompt.md)\""),
"agent job should pass prompt using attached form: {agent}"
);
assert!(
detection.contains(
"/tmp/awf-tools/copilot --prompt=\"$(cat \
/tmp/awf-tools/threat-analysis-prompt.md)\""
),
"detection job should pass prompt using attached form: {detection}"
);
assert!(
!compiled.contains("--prompt \"$(cat "),
"compiled pipeline should not pass prompt as a separate option value: {compiled}"
);

exercise_attached_prompt_with_pinned_copilot_cli(&fixture);
}

#[test]
fn test_compiled_agent_job_copilot_invocation_supports_representative_cli_variants() {
let compiled = compile_inline_agent(
Expand Down Expand Up @@ -5093,7 +5147,7 @@ fn test_1es_compiled_output_is_valid_yaml() {
"1ES output should contain SafeOutputs references"
);
assert!(
compiled.contains("copilot --prompt"),
compiled.contains("copilot --prompt="),
"1ES output should contain copilot invocation (engine_run substituted)"
);
assert!(
Expand Down Expand Up @@ -9506,6 +9560,109 @@ fn compile_inline_agent_with_env(tag: &str, content: &str, env: &[(&str, &str)])
compiled
}

fn exercise_attached_prompt_with_pinned_copilot_cli(prompt: &str) {
let temp_dir = std::env::temp_dir().join(format!(
"ado-aw-copilot-cli-prompt-regression-{}",
std::process::id()
));
let _ = fs::remove_dir_all(&temp_dir);
fs::create_dir_all(&temp_dir).expect("Failed to create Copilot CLI regression temp dir");

let prompt_path = temp_dir.join("agent-prompt.md");
fs::write(&prompt_path, prompt).expect("Failed to write Copilot CLI regression prompt");

let Some(copilot_path) = pinned_copilot_cli_for_test(&temp_dir) else {
let _ = fs::remove_dir_all(&temp_dir);
return;
};

let command = format!(
"{} --prompt=\"$(cat {})\" --allow-all-tools --version",
shell_single_quote(copilot_path.to_str().expect("Copilot path should be UTF-8")),
shell_single_quote(prompt_path.to_str().expect("prompt path should be UTF-8")),
);
let output = std::process::Command::new("bash")
.args(["-c", &command])
.output()
.expect("Failed to run Copilot CLI regression command");
let _ = fs::remove_dir_all(&temp_dir);

assert!(
output.status.success(),
"attached --prompt= form should parse a prompt beginning with '---'.\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}

fn pinned_copilot_cli_for_test(temp_dir: &std::path::Path) -> Option<PathBuf> {
if let Ok(path) = std::env::var("ADO_AW_TEST_COPILOT_BIN") {
let path = PathBuf::from(path);
if path.is_file() {
return Some(path);
}
eprintln!(
"Skipping pinned Copilot CLI regression: ADO_AW_TEST_COPILOT_BIN does not point to a file: {}",
path.display()
);
return None;
}

let version = pinned_copilot_cli_version_for_test();
let base_url = format!("https://github.com/github/copilot-cli/releases/download/v{version}");
let script = format!(
r#"set -euo pipefail
download_dir="$1"
tarball="$download_dir/copilot-linux-x64.tar.gz"
checksums="$download_dir/SHA256SUMS.txt"
curl -fsSL "{base_url}/copilot-linux-x64.tar.gz" -o "$tarball"
curl -fsSL "{base_url}/SHA256SUMS.txt" -o "$checksums"
expected="$(grep 'copilot-linux-x64.tar.gz$' "$checksums" | awk '{{print $1}}' | tr 'A-F' 'a-f')"
actual="$(sha256sum "$tarball" | awk '{{print $1}}' | tr 'A-F' 'a-f')"
test -n "$expected"
test "$expected" = "$actual"
tar -xzf "$tarball" -C "$download_dir"
test -x "$download_dir/copilot"
"#
);
let output = std::process::Command::new("bash")
.args(["-c", &script, "download-pinned-copilot", temp_dir.to_str()?])
.output()
.expect("Failed to run pinned Copilot CLI download command");
if !output.status.success() {
eprintln!(
"Skipping pinned Copilot CLI regression; unable to download v{version}.\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
return None;
}

Some(temp_dir.join("copilot"))
}

fn pinned_copilot_cli_version_for_test() -> String {
let engine_rs = fs::read_to_string(
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("src")
.join("engine.rs"),
)
.expect("engine.rs should be readable");
let marker = "pub const COPILOT_CLI_VERSION: &str = \"";
let start = engine_rs
.find(marker)
.expect("COPILOT_CLI_VERSION should be declared")
+ marker.len();
let end = engine_rs[start..]
.find('"')
.expect("COPILOT_CLI_VERSION should terminate");
engine_rs[start..start + end].to_string()
}

fn shell_single_quote(value: &str) -> String {
format!("'{}'", value.replace('\'', "'\\''"))
}

const GITHUB_APP_TOKEN_FM: &str = r#"
engine:
id: copilot
Expand Down
Loading