phoebus_os/installer-rs/crates/phoebus-installer/tests/sys_ai_e2e.rs
Phoebus 9d6781e300
Some checks are pending
ci / test (push) Waiting to run
test(sys): cover the connected-model AI path end to end
2026-09-29 00:23:44 -07:00

249 lines
8.4 KiB
Rust

//! End-to-end tests for the `sys` AI path over a **real** local TCP stub and
//! the **real** curl transport.
//!
//! The deterministic tests cover the `--legacy` path and the no-model fail-closed
//! routing. These cover the connected-model path the fake transports cannot:
//! the `sys` binary gathers bounded facts, sends them with curl, and parses the
//! model reply into the exact typed-action schema. Hostile and malformed replies
//! must fail closed, and no personal path may reach the request.
//!
//! No real model and no credential are used: the stub returns a fixed body.
use std::io::{Read, Write};
use std::net::TcpListener;
use std::path::Path;
use std::process::{Command, Output};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::Duration;
/// A minimal OpenAI-compatible stub that always returns `content`.
struct Stub {
port: u16,
stop: Arc<AtomicBool>,
requests: Arc<AtomicUsize>,
bodies: Arc<Mutex<Vec<String>>>,
}
impl Drop for Stub {
fn drop(&mut self) {
self.stop.store(true, Ordering::SeqCst);
}
}
impl Stub {
fn start(content: &str) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind stub");
let port = listener.local_addr().expect("stub addr").port();
listener.set_nonblocking(true).expect("nonblocking");
let stop = Arc::new(AtomicBool::new(false));
let requests = Arc::new(AtomicUsize::new(0));
let bodies = Arc::new(Mutex::new(Vec::new()));
let content = content.to_string();
let stop_thread = stop.clone();
let requests_thread = requests.clone();
let bodies_thread = bodies.clone();
thread::spawn(move || {
while !stop_thread.load(Ordering::SeqCst) {
match listener.accept() {
Ok((mut stream, _)) => {
let _ = stream.set_read_timeout(Some(Duration::from_secs(5)));
let _ = serve(&mut stream, &content, &requests_thread, &bodies_thread);
}
Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => {
thread::sleep(Duration::from_millis(5));
}
Err(_) => break,
}
}
});
Self {
port,
stop,
requests,
bodies,
}
}
fn endpoint(&self) -> String {
format!("http://127.0.0.1:{}/v1/chat/completions", self.port)
}
fn request_count(&self) -> usize {
self.requests.load(Ordering::SeqCst)
}
/// The first request body the model received.
fn first_request(&self) -> String {
self.bodies
.lock()
.expect("stub bodies")
.first()
.cloned()
.unwrap_or_default()
}
}
fn serve(
stream: &mut std::net::TcpStream,
content: &str,
requests: &Arc<AtomicUsize>,
bodies: &Arc<Mutex<Vec<String>>>,
) -> std::io::Result<()> {
let mut buf = Vec::new();
let mut chunk = [0u8; 8192];
let mut body_len = None;
loop {
let n = stream.read(&mut chunk)?;
if n == 0 {
break;
}
buf.extend_from_slice(&chunk[..n]);
if body_len.is_none() {
if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") {
let headers = String::from_utf8_lossy(&buf[..pos]).to_string();
body_len = Some(content_length(&headers));
}
}
if let Some(len) = body_len {
if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") {
if buf.len() >= pos + 4 + len {
break;
}
}
}
}
let text = String::from_utf8_lossy(&buf).to_string();
let body = text
.split_once("\r\n\r\n")
.map(|(_, b)| b.to_string())
.unwrap_or_default();
bodies.lock().expect("stub bodies").push(body);
requests.fetch_add(1, Ordering::SeqCst);
let envelope = format!(
"{{\"choices\":[{{\"message\":{{\"content\":{}}}}}]}}",
serde_json::to_string(content).expect("encode content")
);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
envelope.len(),
envelope
);
stream.write_all(response.as_bytes())?;
stream.flush()
}
fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack
.windows(needle.len())
.position(|window| window == needle)
}
fn content_length(headers: &str) -> usize {
headers
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().ok())
.flatten()
})
.unwrap_or(0)
}
/// Run the real `sys` binary against a local stub, sandbox disabled so the test
/// does not need a systemd seat.
fn sys_ask(root: &Path, endpoint: &str, instruction: &str) -> Output {
Command::new(env!("CARGO_BIN_EXE_sys"))
.arg("--root")
.arg(root)
.env("PHOEBUS_AI_PROVIDER", "")
.env("PHOEBUS_AI_API_KEY_FILE", "")
.env("PHOEBUS_AI_BACKEND", "direct")
.args(["ask", instruction, "--ai-provider", "local"])
.args(["--ai-endpoint", endpoint, "--no-sandbox"])
.output()
.expect("run sys")
}
#[test]
fn a_valid_model_plan_runs_through_the_connected_ai_path() {
let root = tempfile::tempdir().expect("temp root");
let stub = Stub::start(
r#"{"version":1,"summary":"Show history","actions":[{"action":"history"}],"rationale":[{"action":"history","reason":"the user asked"}]}"#,
);
let out = sys_ask(root.path(), &stub.endpoint(), "what changed?");
assert!(
out.status.success(),
"a valid plan must execute: {}",
String::from_utf8_lossy(&out.stderr)
);
let stdout = String::from_utf8_lossy(&out.stdout);
assert!(stdout.contains("Assistant Plan"), "{stdout}");
assert_eq!(stub.request_count(), 1, "a valid plan needs one turn");
// The bounded facts actually sent must not contain a personal path or a
// credential, and the request must be the assistant schema.
let sent = stub.first_request();
assert!(sent.contains("messages"), "{sent}");
assert!(!sent.contains("/home/"), "personal path reached the model");
assert!(!sent.contains("Bearer"), "a credential reached the model");
}
#[test]
fn a_shell_action_from_the_model_fails_closed() {
let root = tempfile::tempdir().expect("temp root");
let stub = Stub::start(
r#"{"version":1,"summary":"x","actions":[{"action":"run_shell","command":"rm -rf /"}],"rationale":[{"action":"run_shell","reason":"x"}]}"#,
);
let out = sys_ask(root.path(), &stub.endpoint(), "do it");
assert!(!out.status.success(), "a shell action must be refused");
assert_eq!(
stub.request_count(),
2,
"one corrective retry, then refusal"
);
assert!(
!root
.path()
.join("var/lib/phoebus/transactions.toml")
.exists(),
"a refused plan must not execute or record a transaction"
);
}
#[test]
fn a_smuggled_unit_variant_extra_key_fails_closed() {
let root = tempfile::tempdir().expect("temp root");
// `depclean` is a unit variant; `deny_unknown_fields` does not cover it, so
// this must be caught by the exact schema validator.
let stub = Stub::start(
r#"{"version":1,"summary":"x","actions":[{"action":"depclean","shell":"sh"}],"rationale":[{"action":"depclean","reason":"x"}]}"#,
);
let out = sys_ask(root.path(), &stub.endpoint(), "do it");
assert!(!out.status.success(), "a smuggled field must be refused");
let stderr = String::from_utf8_lossy(&out.stderr);
assert!(
stderr.contains("unknown field `shell`"),
"the schema must reject the extra key: {stderr}"
);
}
#[test]
fn malformed_model_output_fails_closed_after_one_correction() {
let root = tempfile::tempdir().expect("temp root");
let stub = Stub::start("not-json-at-all");
let out = sys_ask(root.path(), &stub.endpoint(), "do it");
assert!(!out.status.success());
assert_eq!(
stub.request_count(),
2,
"one corrective retry, then refusal"
);
assert!(!root
.path()
.join("var/lib/phoebus/transactions.toml")
.exists());
}