diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cf4769c..9ba9c93 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,19 @@ this file is the curated, human-readable summary. ## Unreleased — Fast defaults with user-controlled source builds +- **End-to-end AI-path test over a real local stub (2026-09-28).** The + deterministic tests and the no-model fail-closed routing left the + connected-model path covered only by fake transports. New + `installer-rs/crates/phoebus-installer/tests/sys_ai_e2e.rs` drives the real + `sys` binary against a real local OpenAI-compatible TCP stub and the real curl + transport, with the sandbox disabled (`--no-sandbox`) so no seat is needed. + Four cases: a valid model plan executes end-to-end and the request actually + sent contains no `/home/` path and no `Bearer` credential; a model `run_shell` + action fails closed after one corrective retry with no transaction written; a + `depclean` with a smuggled `shell` field is refused by the exact schema + (`unknown field \`shell\``); and malformed model output fails closed after one + correction. Gates: `cargo fmt`, `clippy --workspace --all-targets -- -D + warnings`, `cargo test --workspace` 841 passed / 0 failed. - **`sys`/AI boundary hardening pass (2026-09-28).** A read-audit of every mutation route plus the AI path found six host-provable defects, all fixed with regression tests. (1) **Root-gate backdoor removed:** the service layer diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 997d586..5ff5745 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -3,6 +3,17 @@ Rolling note between work sessions. **Update this file before you finish**, then commit it. It should be short and current; use `git log` for detail. +- **End-to-end AI-path test (2026-09-28, IMPLEMENTED + UNIT-TESTED, no live + model):** new `installer-rs/crates/phoebus-installer/tests/sys_ai_e2e.rs` + drives the real `sys` binary against a real local OpenAI-compatible TCP stub + and the real curl transport (`--no-sandbox`, no seat). Four cases: a valid + model plan executes end-to-end; the request sent contains no `/home/` path and + no credential; a model `run_shell` fails closed after one corrective retry; + a `depclean` with a smuggled `shell` key is refused by the exact schema; and + malformed output fails closed after one correction. This is not a live-model + call (`sys ask` with a real provider is still VM/live NotRun) but it closes + the "connected-model path only tested with fake transports" gap. Gates: + `cargo test --workspace` 841 passed / 0 failed. - **Boundary-hardening pass (2026-09-28, IMPLEMENTED + UNIT-TESTED, VM NOT RUN):** six host-provable defects fixed with regression tests. (1) Removed the `PHOEBUS_ALLOW_UNPRIVILEGED_EXEC=1` env backdoor: it made diff --git a/docs/STATUS.md b/docs/STATUS.md index 4d0867c..88b4cf3 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -97,6 +97,7 @@ Booting that recovery entry offline remains a manual gate. | AI privacy boundary (facts only) | yes | yes (every string field scanned, incl. `cpu_features`; forbidden path in `cpu_features` refused) | no | n/a | | Provider credential + endpoint boundary | yes | yes (unsafe key characters refused before curl config; userinfo endpoint refused; derived base64 token redacted) | no | n/a | | AI typed-action schema (exact, incl. unit-variant extra keys) | yes | yes (plan/step/investigate actions via `validate_action_value`; smuggled extra key refused) | no | n/a | +| AI connected-model path over a real local stub | yes | yes (4 e2e cases: valid plan executes; request has no home path/credential; shell + smuggled key + malformed fail closed) | no (local stub, not a live model) | n/a | | Privilege derived only from the process identity | yes | yes (an environment variable cannot forge uid 0; the removed `PHOEBUS_ALLOW_UNPRIVILEGED_EXEC` regression test) | no | n/a | | AI OS sandbox (systemd hardened unit) | yes | yes (argv + sandboxed-send via `systemd-run`, key/body on stdin) | no — **host probe 2026-09-28: system `manage-units` needs polkit auth and the full property set is invalid for `--user`; VM seat run required to settle `systemd-run` vs `--user`** | no | | Installed assistant (`sys ask`) | yes | yes | yes (live call 2026-09-28: `sys ask "list installed packages"` via `--ai-backend opencode` to host `opencode serve` through a guest loopback tunnel, model `opencode-go/muse-spark-1.3-contributor`, valid Assistant Plan + approval digest, `AI_LIVE_RC=0`; the `show system status` phrasing elicits schema-noncompliant JSON from this model and is fail-closed rejected) | n/a | diff --git a/installer-rs/crates/phoebus-installer/tests/sys_ai_e2e.rs b/installer-rs/crates/phoebus-installer/tests/sys_ai_e2e.rs new file mode 100644 index 0000000..ea472be --- /dev/null +++ b/installer-rs/crates/phoebus-installer/tests/sys_ai_e2e.rs @@ -0,0 +1,249 @@ +//! End-to-end tests for the `sys` AI path over a **real** local TCP stub and +//! the **real** curl transport. +//! +//! The deterministic tests cover the `--legacy` path and the no-model fail-closed +//! routing. These cover the connected-model path the fake transports cannot: +//! the `sys` binary gathers bounded facts, sends them with curl, and parses the +//! model reply into the exact typed-action schema. Hostile and malformed replies +//! must fail closed, and no personal path may reach the request. +//! +//! No real model and no credential are used: the stub returns a fixed body. + +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::path::Path; +use std::process::{Command, Output}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +/// A minimal OpenAI-compatible stub that always returns `content`. +struct Stub { + port: u16, + stop: Arc, + requests: Arc, + bodies: Arc>>, +} + +impl Drop for Stub { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + } +} + +impl Stub { + fn start(content: &str) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind stub"); + let port = listener.local_addr().expect("stub addr").port(); + listener.set_nonblocking(true).expect("nonblocking"); + let stop = Arc::new(AtomicBool::new(false)); + let requests = Arc::new(AtomicUsize::new(0)); + let bodies = Arc::new(Mutex::new(Vec::new())); + let content = content.to_string(); + let stop_thread = stop.clone(); + let requests_thread = requests.clone(); + let bodies_thread = bodies.clone(); + thread::spawn(move || { + while !stop_thread.load(Ordering::SeqCst) { + match listener.accept() { + Ok((mut stream, _)) => { + let _ = stream.set_read_timeout(Some(Duration::from_secs(5))); + let _ = serve(&mut stream, &content, &requests_thread, &bodies_thread); + } + Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(_) => break, + } + } + }); + Self { + port, + stop, + requests, + bodies, + } + } + + fn endpoint(&self) -> String { + format!("http://127.0.0.1:{}/v1/chat/completions", self.port) + } + + fn request_count(&self) -> usize { + self.requests.load(Ordering::SeqCst) + } + + /// The first request body the model received. + fn first_request(&self) -> String { + self.bodies + .lock() + .expect("stub bodies") + .first() + .cloned() + .unwrap_or_default() + } +} + +fn serve( + stream: &mut std::net::TcpStream, + content: &str, + requests: &Arc, + bodies: &Arc>>, +) -> std::io::Result<()> { + let mut buf = Vec::new(); + let mut chunk = [0u8; 8192]; + let mut body_len = None; + loop { + let n = stream.read(&mut chunk)?; + if n == 0 { + break; + } + buf.extend_from_slice(&chunk[..n]); + if body_len.is_none() { + if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&buf[..pos]).to_string(); + body_len = Some(content_length(&headers)); + } + } + if let Some(len) = body_len { + if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") { + if buf.len() >= pos + 4 + len { + break; + } + } + } + } + let text = String::from_utf8_lossy(&buf).to_string(); + let body = text + .split_once("\r\n\r\n") + .map(|(_, b)| b.to_string()) + .unwrap_or_default(); + bodies.lock().expect("stub bodies").push(body); + requests.fetch_add(1, Ordering::SeqCst); + + let envelope = format!( + "{{\"choices\":[{{\"message\":{{\"content\":{}}}}}]}}", + serde_json::to_string(content).expect("encode content") + ); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + envelope.len(), + envelope + ); + stream.write_all(response.as_bytes())?; + stream.flush() +} + +fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn content_length(headers: &str) -> usize { + headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0) +} + +/// Run the real `sys` binary against a local stub, sandbox disabled so the test +/// does not need a systemd seat. +fn sys_ask(root: &Path, endpoint: &str, instruction: &str) -> Output { + Command::new(env!("CARGO_BIN_EXE_sys")) + .arg("--root") + .arg(root) + .env("PHOEBUS_AI_PROVIDER", "") + .env("PHOEBUS_AI_API_KEY_FILE", "") + .env("PHOEBUS_AI_BACKEND", "direct") + .args(["ask", instruction, "--ai-provider", "local"]) + .args(["--ai-endpoint", endpoint, "--no-sandbox"]) + .output() + .expect("run sys") +} + +#[test] +fn a_valid_model_plan_runs_through_the_connected_ai_path() { + let root = tempfile::tempdir().expect("temp root"); + let stub = Stub::start( + r#"{"version":1,"summary":"Show history","actions":[{"action":"history"}],"rationale":[{"action":"history","reason":"the user asked"}]}"#, + ); + let out = sys_ask(root.path(), &stub.endpoint(), "what changed?"); + assert!( + out.status.success(), + "a valid plan must execute: {}", + String::from_utf8_lossy(&out.stderr) + ); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!(stdout.contains("Assistant Plan"), "{stdout}"); + assert_eq!(stub.request_count(), 1, "a valid plan needs one turn"); + + // The bounded facts actually sent must not contain a personal path or a + // credential, and the request must be the assistant schema. + let sent = stub.first_request(); + assert!(sent.contains("messages"), "{sent}"); + assert!(!sent.contains("/home/"), "personal path reached the model"); + assert!(!sent.contains("Bearer"), "a credential reached the model"); +} + +#[test] +fn a_shell_action_from_the_model_fails_closed() { + let root = tempfile::tempdir().expect("temp root"); + let stub = Stub::start( + r#"{"version":1,"summary":"x","actions":[{"action":"run_shell","command":"rm -rf /"}],"rationale":[{"action":"run_shell","reason":"x"}]}"#, + ); + let out = sys_ask(root.path(), &stub.endpoint(), "do it"); + assert!(!out.status.success(), "a shell action must be refused"); + assert_eq!( + stub.request_count(), + 2, + "one corrective retry, then refusal" + ); + assert!( + !root + .path() + .join("var/lib/phoebus/transactions.toml") + .exists(), + "a refused plan must not execute or record a transaction" + ); +} + +#[test] +fn a_smuggled_unit_variant_extra_key_fails_closed() { + let root = tempfile::tempdir().expect("temp root"); + // `depclean` is a unit variant; `deny_unknown_fields` does not cover it, so + // this must be caught by the exact schema validator. + let stub = Stub::start( + r#"{"version":1,"summary":"x","actions":[{"action":"depclean","shell":"sh"}],"rationale":[{"action":"depclean","reason":"x"}]}"#, + ); + let out = sys_ask(root.path(), &stub.endpoint(), "do it"); + assert!(!out.status.success(), "a smuggled field must be refused"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("unknown field `shell`"), + "the schema must reject the extra key: {stderr}" + ); +} + +#[test] +fn malformed_model_output_fails_closed_after_one_correction() { + let root = tempfile::tempdir().expect("temp root"); + let stub = Stub::start("not-json-at-all"); + let out = sys_ask(root.path(), &stub.endpoint(), "do it"); + assert!(!out.status.success()); + assert_eq!( + stub.request_count(), + 2, + "one corrective retry, then refusal" + ); + assert!(!root + .path() + .join("var/lib/phoebus/transactions.toml") + .exists()); +}