249 lines
8.4 KiB
Rust
249 lines
8.4 KiB
Rust
//! End-to-end tests for the `sys` AI path over a **real** local TCP stub and
|
|
//! the **real** curl transport.
|
|
//!
|
|
//! The deterministic tests cover the `--legacy` path and the no-model fail-closed
|
|
//! routing. These cover the connected-model path the fake transports cannot:
|
|
//! the `sys` binary gathers bounded facts, sends them with curl, and parses the
|
|
//! model reply into the exact typed-action schema. Hostile and malformed replies
|
|
//! must fail closed, and no personal path may reach the request.
|
|
//!
|
|
//! No real model and no credential are used: the stub returns a fixed body.
|
|
|
|
use std::io::{Read, Write};
|
|
use std::net::TcpListener;
|
|
use std::path::Path;
|
|
use std::process::{Command, Output};
|
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
|
use std::sync::{Arc, Mutex};
|
|
use std::thread;
|
|
use std::time::Duration;
|
|
|
|
/// A minimal OpenAI-compatible stub that always returns `content`.
|
|
struct Stub {
|
|
port: u16,
|
|
stop: Arc<AtomicBool>,
|
|
requests: Arc<AtomicUsize>,
|
|
bodies: Arc<Mutex<Vec<String>>>,
|
|
}
|
|
|
|
impl Drop for Stub {
|
|
fn drop(&mut self) {
|
|
self.stop.store(true, Ordering::SeqCst);
|
|
}
|
|
}
|
|
|
|
impl Stub {
|
|
fn start(content: &str) -> Self {
|
|
let listener = TcpListener::bind("127.0.0.1:0").expect("bind stub");
|
|
let port = listener.local_addr().expect("stub addr").port();
|
|
listener.set_nonblocking(true).expect("nonblocking");
|
|
let stop = Arc::new(AtomicBool::new(false));
|
|
let requests = Arc::new(AtomicUsize::new(0));
|
|
let bodies = Arc::new(Mutex::new(Vec::new()));
|
|
let content = content.to_string();
|
|
let stop_thread = stop.clone();
|
|
let requests_thread = requests.clone();
|
|
let bodies_thread = bodies.clone();
|
|
thread::spawn(move || {
|
|
while !stop_thread.load(Ordering::SeqCst) {
|
|
match listener.accept() {
|
|
Ok((mut stream, _)) => {
|
|
let _ = stream.set_read_timeout(Some(Duration::from_secs(5)));
|
|
let _ = serve(&mut stream, &content, &requests_thread, &bodies_thread);
|
|
}
|
|
Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => {
|
|
thread::sleep(Duration::from_millis(5));
|
|
}
|
|
Err(_) => break,
|
|
}
|
|
}
|
|
});
|
|
Self {
|
|
port,
|
|
stop,
|
|
requests,
|
|
bodies,
|
|
}
|
|
}
|
|
|
|
fn endpoint(&self) -> String {
|
|
format!("http://127.0.0.1:{}/v1/chat/completions", self.port)
|
|
}
|
|
|
|
fn request_count(&self) -> usize {
|
|
self.requests.load(Ordering::SeqCst)
|
|
}
|
|
|
|
/// The first request body the model received.
|
|
fn first_request(&self) -> String {
|
|
self.bodies
|
|
.lock()
|
|
.expect("stub bodies")
|
|
.first()
|
|
.cloned()
|
|
.unwrap_or_default()
|
|
}
|
|
}
|
|
|
|
fn serve(
|
|
stream: &mut std::net::TcpStream,
|
|
content: &str,
|
|
requests: &Arc<AtomicUsize>,
|
|
bodies: &Arc<Mutex<Vec<String>>>,
|
|
) -> std::io::Result<()> {
|
|
let mut buf = Vec::new();
|
|
let mut chunk = [0u8; 8192];
|
|
let mut body_len = None;
|
|
loop {
|
|
let n = stream.read(&mut chunk)?;
|
|
if n == 0 {
|
|
break;
|
|
}
|
|
buf.extend_from_slice(&chunk[..n]);
|
|
if body_len.is_none() {
|
|
if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") {
|
|
let headers = String::from_utf8_lossy(&buf[..pos]).to_string();
|
|
body_len = Some(content_length(&headers));
|
|
}
|
|
}
|
|
if let Some(len) = body_len {
|
|
if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") {
|
|
if buf.len() >= pos + 4 + len {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
let text = String::from_utf8_lossy(&buf).to_string();
|
|
let body = text
|
|
.split_once("\r\n\r\n")
|
|
.map(|(_, b)| b.to_string())
|
|
.unwrap_or_default();
|
|
bodies.lock().expect("stub bodies").push(body);
|
|
requests.fetch_add(1, Ordering::SeqCst);
|
|
|
|
let envelope = format!(
|
|
"{{\"choices\":[{{\"message\":{{\"content\":{}}}}}]}}",
|
|
serde_json::to_string(content).expect("encode content")
|
|
);
|
|
let response = format!(
|
|
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
|
|
envelope.len(),
|
|
envelope
|
|
);
|
|
stream.write_all(response.as_bytes())?;
|
|
stream.flush()
|
|
}
|
|
|
|
fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
|
haystack
|
|
.windows(needle.len())
|
|
.position(|window| window == needle)
|
|
}
|
|
|
|
fn content_length(headers: &str) -> usize {
|
|
headers
|
|
.lines()
|
|
.find_map(|line| {
|
|
let (name, value) = line.split_once(':')?;
|
|
name.eq_ignore_ascii_case("content-length")
|
|
.then(|| value.trim().parse::<usize>().ok())
|
|
.flatten()
|
|
})
|
|
.unwrap_or(0)
|
|
}
|
|
|
|
/// Run the real `sys` binary against a local stub, sandbox disabled so the test
|
|
/// does not need a systemd seat.
|
|
fn sys_ask(root: &Path, endpoint: &str, instruction: &str) -> Output {
|
|
Command::new(env!("CARGO_BIN_EXE_sys"))
|
|
.arg("--root")
|
|
.arg(root)
|
|
.env("PHOEBUS_AI_PROVIDER", "")
|
|
.env("PHOEBUS_AI_API_KEY_FILE", "")
|
|
.env("PHOEBUS_AI_BACKEND", "direct")
|
|
.args(["ask", instruction, "--ai-provider", "local"])
|
|
.args(["--ai-endpoint", endpoint, "--no-sandbox"])
|
|
.output()
|
|
.expect("run sys")
|
|
}
|
|
|
|
#[test]
|
|
fn a_valid_model_plan_runs_through_the_connected_ai_path() {
|
|
let root = tempfile::tempdir().expect("temp root");
|
|
let stub = Stub::start(
|
|
r#"{"version":1,"summary":"Show history","actions":[{"action":"history"}],"rationale":[{"action":"history","reason":"the user asked"}]}"#,
|
|
);
|
|
let out = sys_ask(root.path(), &stub.endpoint(), "what changed?");
|
|
assert!(
|
|
out.status.success(),
|
|
"a valid plan must execute: {}",
|
|
String::from_utf8_lossy(&out.stderr)
|
|
);
|
|
let stdout = String::from_utf8_lossy(&out.stdout);
|
|
assert!(stdout.contains("Assistant Plan"), "{stdout}");
|
|
assert_eq!(stub.request_count(), 1, "a valid plan needs one turn");
|
|
|
|
// The bounded facts actually sent must not contain a personal path or a
|
|
// credential, and the request must be the assistant schema.
|
|
let sent = stub.first_request();
|
|
assert!(sent.contains("messages"), "{sent}");
|
|
assert!(!sent.contains("/home/"), "personal path reached the model");
|
|
assert!(!sent.contains("Bearer"), "a credential reached the model");
|
|
}
|
|
|
|
#[test]
|
|
fn a_shell_action_from_the_model_fails_closed() {
|
|
let root = tempfile::tempdir().expect("temp root");
|
|
let stub = Stub::start(
|
|
r#"{"version":1,"summary":"x","actions":[{"action":"run_shell","command":"rm -rf /"}],"rationale":[{"action":"run_shell","reason":"x"}]}"#,
|
|
);
|
|
let out = sys_ask(root.path(), &stub.endpoint(), "do it");
|
|
assert!(!out.status.success(), "a shell action must be refused");
|
|
assert_eq!(
|
|
stub.request_count(),
|
|
2,
|
|
"one corrective retry, then refusal"
|
|
);
|
|
assert!(
|
|
!root
|
|
.path()
|
|
.join("var/lib/phoebus/transactions.toml")
|
|
.exists(),
|
|
"a refused plan must not execute or record a transaction"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_smuggled_unit_variant_extra_key_fails_closed() {
|
|
let root = tempfile::tempdir().expect("temp root");
|
|
// `depclean` is a unit variant; `deny_unknown_fields` does not cover it, so
|
|
// this must be caught by the exact schema validator.
|
|
let stub = Stub::start(
|
|
r#"{"version":1,"summary":"x","actions":[{"action":"depclean","shell":"sh"}],"rationale":[{"action":"depclean","reason":"x"}]}"#,
|
|
);
|
|
let out = sys_ask(root.path(), &stub.endpoint(), "do it");
|
|
assert!(!out.status.success(), "a smuggled field must be refused");
|
|
let stderr = String::from_utf8_lossy(&out.stderr);
|
|
assert!(
|
|
stderr.contains("unknown field `shell`"),
|
|
"the schema must reject the extra key: {stderr}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn malformed_model_output_fails_closed_after_one_correction() {
|
|
let root = tempfile::tempdir().expect("temp root");
|
|
let stub = Stub::start("not-json-at-all");
|
|
let out = sys_ask(root.path(), &stub.endpoint(), "do it");
|
|
assert!(!out.status.success());
|
|
assert_eq!(
|
|
stub.request_count(),
|
|
2,
|
|
"one corrective retry, then refusal"
|
|
);
|
|
assert!(!root
|
|
.path()
|
|
.join("var/lib/phoebus/transactions.toml")
|
|
.exists());
|
|
}
|