//! End-to-end tests for the `sys` AI path over a **real** local TCP stub and //! the **real** curl transport. //! //! The deterministic tests cover the `--legacy` path and the no-model fail-closed //! routing. These cover the connected-model path the fake transports cannot: //! the `sys` binary gathers bounded facts, sends them with curl, and parses the //! model reply into the exact typed-action schema. Hostile and malformed replies //! must fail closed, and no personal path may reach the request. //! //! No real model and no credential are used: the stub returns a fixed body. use std::io::{Read, Write}; use std::net::TcpListener; use std::path::Path; use std::process::{Command, Output}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::Duration; /// A minimal OpenAI-compatible stub that always returns `content`. struct Stub { port: u16, stop: Arc, requests: Arc, bodies: Arc>>, } impl Drop for Stub { fn drop(&mut self) { self.stop.store(true, Ordering::SeqCst); } } impl Stub { fn start(content: &str) -> Self { let listener = TcpListener::bind("127.0.0.1:0").expect("bind stub"); let port = listener.local_addr().expect("stub addr").port(); listener.set_nonblocking(true).expect("nonblocking"); let stop = Arc::new(AtomicBool::new(false)); let requests = Arc::new(AtomicUsize::new(0)); let bodies = Arc::new(Mutex::new(Vec::new())); let content = content.to_string(); let stop_thread = stop.clone(); let requests_thread = requests.clone(); let bodies_thread = bodies.clone(); thread::spawn(move || { while !stop_thread.load(Ordering::SeqCst) { match listener.accept() { Ok((mut stream, _)) => { let _ = stream.set_read_timeout(Some(Duration::from_secs(5))); let _ = serve(&mut stream, &content, &requests_thread, &bodies_thread); } Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => { thread::sleep(Duration::from_millis(5)); } Err(_) => break, } } }); Self { port, stop, requests, bodies, } } fn endpoint(&self) -> String { format!("http://127.0.0.1:{}/v1/chat/completions", self.port) } fn request_count(&self) -> usize { self.requests.load(Ordering::SeqCst) } /// The first request body the model received. fn first_request(&self) -> String { self.bodies .lock() .expect("stub bodies") .first() .cloned() .unwrap_or_default() } } fn serve( stream: &mut std::net::TcpStream, content: &str, requests: &Arc, bodies: &Arc>>, ) -> std::io::Result<()> { let mut buf = Vec::new(); let mut chunk = [0u8; 8192]; let mut body_len = None; loop { let n = stream.read(&mut chunk)?; if n == 0 { break; } buf.extend_from_slice(&chunk[..n]); if body_len.is_none() { if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") { let headers = String::from_utf8_lossy(&buf[..pos]).to_string(); body_len = Some(content_length(&headers)); } } if let Some(len) = body_len { if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") { if buf.len() >= pos + 4 + len { break; } } } } let text = String::from_utf8_lossy(&buf).to_string(); let body = text .split_once("\r\n\r\n") .map(|(_, b)| b.to_string()) .unwrap_or_default(); bodies.lock().expect("stub bodies").push(body); requests.fetch_add(1, Ordering::SeqCst); let envelope = format!( "{{\"choices\":[{{\"message\":{{\"content\":{}}}}}]}}", serde_json::to_string(content).expect("encode content") ); let response = format!( "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", envelope.len(), envelope ); stream.write_all(response.as_bytes())?; stream.flush() } fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { haystack .windows(needle.len()) .position(|window| window == needle) } fn content_length(headers: &str) -> usize { headers .lines() .find_map(|line| { let (name, value) = line.split_once(':')?; name.eq_ignore_ascii_case("content-length") .then(|| value.trim().parse::().ok()) .flatten() }) .unwrap_or(0) } /// Run the real `sys` binary against a local stub, sandbox disabled so the test /// does not need a systemd seat. fn sys_ask(root: &Path, endpoint: &str, instruction: &str) -> Output { Command::new(env!("CARGO_BIN_EXE_sys")) .arg("--root") .arg(root) .env("PHOEBUS_AI_PROVIDER", "") .env("PHOEBUS_AI_API_KEY_FILE", "") .env("PHOEBUS_AI_BACKEND", "direct") .args(["ask", instruction, "--ai-provider", "local"]) .args(["--ai-endpoint", endpoint, "--no-sandbox"]) .output() .expect("run sys") } #[test] fn a_valid_model_plan_runs_through_the_connected_ai_path() { let root = tempfile::tempdir().expect("temp root"); let stub = Stub::start( r#"{"version":1,"summary":"Show history","actions":[{"action":"history"}],"rationale":[{"action":"history","reason":"the user asked"}]}"#, ); let out = sys_ask(root.path(), &stub.endpoint(), "what changed?"); assert!( out.status.success(), "a valid plan must execute: {}", String::from_utf8_lossy(&out.stderr) ); let stdout = String::from_utf8_lossy(&out.stdout); assert!(stdout.contains("Assistant Plan"), "{stdout}"); let stderr = String::from_utf8_lossy(&out.stderr); assert!( stderr.contains("Checking this system and its package sources"), "{stderr}" ); assert!(stderr.contains("validating its answer"), "{stderr}"); assert!( stderr.contains("Running the approved plan and verifying the result"), "{stderr}" ); assert_eq!(stub.request_count(), 1, "a valid plan needs one turn"); // The bounded facts actually sent must not contain a personal path or a // credential, and the request must be the assistant schema. let sent = stub.first_request(); assert!(sent.contains("messages"), "{sent}"); assert!(!sent.contains("/home/"), "personal path reached the model"); assert!(!sent.contains("Bearer"), "a credential reached the model"); } #[test] fn an_informational_answer_reads_as_prose_without_an_approval_digest() { let root = tempfile::tempdir().expect("temp root"); let stub = Stub::start( r#"{"version":1,"summary":"This system has a stable desktop and an encrypted root. Review the pending Portage updates before changing system packages.","actions":[],"rationale":[]}"#, ); let out = sys_ask(root.path(), &stub.endpoint(), "how is this system?"); assert!( out.status.success(), "{}", String::from_utf8_lossy(&out.stderr) ); let stdout = String::from_utf8_lossy(&out.stdout); assert!(stdout.starts_with("Phoebus\n\n"), "{stdout}"); assert!(stdout.contains("encrypted root"), "{stdout}"); assert!(!stdout.contains("Approval digest"), "{stdout}"); assert!( stdout.lines().all(|line| line.chars().count() <= 88), "{stdout}" ); } #[test] fn a_shell_action_from_the_model_fails_closed() { let root = tempfile::tempdir().expect("temp root"); let stub = Stub::start( r#"{"version":1,"summary":"x","actions":[{"action":"run_shell","command":"rm -rf /"}],"rationale":[{"action":"run_shell","reason":"x"}]}"#, ); let out = sys_ask(root.path(), &stub.endpoint(), "do it"); assert!(!out.status.success(), "a shell action must be refused"); assert_eq!( stub.request_count(), 2, "one corrective retry, then refusal" ); assert!( !root .path() .join("var/lib/phoebus/transactions.toml") .exists(), "a refused plan must not execute or record a transaction" ); } #[test] fn a_smuggled_unit_variant_extra_key_fails_closed() { let root = tempfile::tempdir().expect("temp root"); // `depclean` is a unit variant; `deny_unknown_fields` does not cover it, so // this must be caught by the exact schema validator. let stub = Stub::start( r#"{"version":1,"summary":"x","actions":[{"action":"depclean","shell":"sh"}],"rationale":[{"action":"depclean","reason":"x"}]}"#, ); let out = sys_ask(root.path(), &stub.endpoint(), "do it"); assert!(!out.status.success(), "a smuggled field must be refused"); let stderr = String::from_utf8_lossy(&out.stderr); assert!( stderr.contains("unknown field `shell`"), "the schema must reject the extra key: {stderr}" ); } #[test] fn malformed_model_output_fails_closed_after_one_correction() { let root = tempfile::tempdir().expect("temp root"); let stub = Stub::start("not-json-at-all"); let out = sys_ask(root.path(), &stub.endpoint(), "do it"); assert!(!out.status.success()); assert_eq!( stub.request_count(), 2, "one corrective retry, then refusal" ); assert!(!root .path() .join("var/lib/phoebus/transactions.toml") .exists()); }