From a224bb72e8164a36adab5e09476d00d2105a514d Mon Sep 17 00:00:00 2001 From: Phoebus Date: Sat, 3 Oct 2026 10:06:32 -0700 Subject: [PATCH] fix(assistant): handle oversized search investigations --- docs/CHANGELOG.md | 10 ++ docs/HANDOFF.md | 20 +++ docs/STATUS.md | 15 +++ .../crates/phoebus-ai/src/assistant.rs | 119 ++++++++++++++++-- .../phoebus-installer/src/assistant_bridge.rs | 66 +++++++++- .../crates/phoebus-installer/src/bin/sys.rs | 7 +- .../phoebus-installer/src/service/plan.rs | 4 +- .../phoebus-installer/src/service/tests.rs | 4 + 8 files changed, 230 insertions(+), 15 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5306ae6..8afcc87 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,16 @@ this file is the curated, human-readable summary. ## Unreleased — Fast defaults with user-controlled source builds +- **Bounded Assistant searches (2026-10-03).** A general search reply with more + than six typed read-only checks now runs the first six and tells the model + which checks were skipped, so it can continue on the next turn. The full + reply is validated before any check runs; a mutating action anywhere in the + batch is still refused. Informational answers with no actions now appear as + information in the desktop Assistant, with no approval button or staged + plan. The running VM answered the exact Balena Etcher package question with + the AppImage catalog lead instead of the previous `max 6` rejection, and + the final information card has no approval button. + - **Steady assistant display and missing-package result (2026-10-03).** Progress events now update a small activity list inside the current message instead of recreating the whole conversation and scrolling it on every diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 27fa54c..bafcfca 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -3,6 +3,26 @@ Rolling note between work sessions. **Update this file before you finish**, then commit it. It should be short and current; use `git log` for detail. +- **General Assistant search cap (2026-10-03):** The exact VM question `can you + find a package for balena etcher for me` previously failed with `too many + investigation requests in one turn (max 6)`. The general loop now validates + every typed action, executes only the first six read-only checks, and feeds + evidence plus a skipped-check note back to the model. The live VM answered + with an AppImage catalog lead and no verified installable package + (`/var/tmp/phoebus-vm/etcher-search-new2.png`). That run exposed a desktop + bridge issue: empty-action answers showed `Approve & Execute`. The bridge + now sends an information result with no plan ID or digest, and the service + does not stage an empty plan. Tests cover both behavior and the read-only + boundary. The final user-local static `sys` was installed in the VM (SHA-256 + `6c7226433af167377d2002e2aba9682914e99691c501c61bb2c04681b4e586ee`). + Repeating the exact question returned an Information card, displayed its + read-only `sys` checks, and showed no approval button + (`/var/tmp/phoebus-vm/final-question-progress.png`). The VM's root-owned + binary remains older until a fresh install. A fresh ISO is still gated by + `make doctor` reporting 9 GiB free against its 20 GiB VM requirement. Rust + format, strict clippy, workspace tests, `make check`, and dynamic/static + release builds pass. + - **Assistant flicker and Etcher follow-up (2026-10-03):** Progress used to replace the entire QML conversation model and scroll the full view on every event. It now updates a fixed-height activity list inside the message card. diff --git a/docs/STATUS.md b/docs/STATUS.md index 0b1e6dd..31b9a4a 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -48,6 +48,21 @@ disposable VM; `serial-driver.py recovery` now automates the same check. ## Matrix +General Assistant search bounds (2026-10-03): an oversized model investigation +is validated in full and only its first six read-only requests are executed; +the model receives the evidence and a clear note about the skipped checks. +Unit tests cover the cap and a mutating seventh request. In the running VM, +the exact `can you find a package for balena etcher for me` request returned a +package search summary with an AppImage catalog lead, instead of the previous +`max 6` error (`/var/tmp/phoebus-vm/etcher-search-new2.png`). That run also +showed an approval button for a no-action answer; the bridge now returns an +information card for such answers and does not stage an empty plan. The final +user-local static binary was installed in the VM (SHA-256 +`6c7226433af167377d2002e2aba9682914e99691c501c61bb2c04681b4e586ee`). +The exact question then returned an Information card with visible read-only +checks and no approval button (`/var/tmp/phoebus-vm/final-question-progress.png`). +The root-owned installed binary remains older until a fresh ISO/install. + Assistant display and unavailable plan (2026-10-03): QML progress updates modify only the current card's fixed-height activity list; the conversation model is not replaced for every phase. The Mango Assistant window stays opaque diff --git a/installer-rs/crates/phoebus-ai/src/assistant.rs b/installer-rs/crates/phoebus-ai/src/assistant.rs index 00c34a2..5369ad3 100644 --- a/installer-rs/crates/phoebus-ai/src/assistant.rs +++ b/installer-rs/crates/phoebus-ai/src/assistant.rs @@ -92,7 +92,8 @@ real evidence. Useful tools: search_packages (query, optional backend), \ discover_candidates, inspect_candidate, check_backend, inspect_installation, \ list_packages (optional backend/filter), package_info, check_updates \ (optional backend), package_owners, package_dependencies. Never request a \ -mutating action as evidence. +mutating action as evidence. Request at most six read-only actions per turn; \ +you can ask for more evidence on the next turn. 2. Plan or answer: {\"version\":1,\"summary\":\"...\",\"actions\":\ [{\"action\":\"list_packages\"}],\ @@ -402,12 +403,15 @@ pub fn run_assistant( match step { AssistantStep::Investigate { requests } => { - if requests.len() > limits.max_requests_per_turn { - return Err(AiError::Rejected(format!( - "too many investigation requests in one turn (max {})", - limits.max_requests_per_turn - ))); - } + // The whole reply has already passed the typed, read-only + // validator. Run only the bounded prefix and let the model + // request any omitted evidence on its next turn. An oversized + // batch is a model mistake, not a user-facing failure. + let requested_count = requests.len(); + let requests: Vec<_> = requests + .into_iter() + .take(limits.max_requests_per_turn) + .collect(); let mut keys: Vec = requests .iter() .filter_map(|r| serde_json::to_string(r).ok()) @@ -444,6 +448,16 @@ pub fn run_assistant( payload.truncate(limits.max_evidence_bytes); payload.push_str("\n[evidence truncated]"); } + if requested_count > requests.len() { + payload.push_str(&format!( + "\nOnly the first {} of {} requested checks ran. The remaining \ + checks were not performed. Request any still needed evidence in \ + another turn, with at most {} checks, or answer from the evidence above.", + requests.len(), + requested_count, + limits.max_requests_per_turn + )); + } chat.messages.push(ChatMessage { role: "user".into(), content: format!("Evidence:\n{payload}"), @@ -736,6 +750,97 @@ mod tests { assert!(requests[1].contains("com.google.Chrome")); } + #[test] + fn oversized_investigation_runs_only_the_bounded_prefix() { + use crate::provider::{chat_response, ScriptedTransport}; + let requests: Vec<_> = (0..7) + .map(|index| serde_json::json!({"action":"search_packages","query":format!("etcher{index}")})) + .collect(); + let transport = ScriptedTransport::new(vec![ + chat_response( + &serde_json::json!({"version":1,"kind":"investigate","requests":requests}) + .to_string(), + ), + chat_response( + r#"{"version":1,"summary":"No package found in the checked sources.","actions":[]}"#, + ), + ]); + let investigate = |requests: &[SystemAction]| -> Vec { + assert_eq!(requests.len(), 6); + let queries: Vec<_> = requests + .iter() + .map(|r| { + serde_json::to_value(r).unwrap()["query"] + .as_str() + .unwrap() + .to_string() + }) + .collect(); + assert_eq!( + queries, + (0..6) + .map(|index| format!("etcher{index}")) + .collect::>() + ); + requests + .iter() + .map(|r| Evidence { + request: r.describe(), + output: "[]".into(), + }) + .collect() + }; + let plan = run_assistant( + ProviderKind::Local, + "http://127.0.0.1:11434/v1/chat/completions", + "m", + &facts(), + "find etcher", + &transport, + &investigate, + &crate::install::Limits::default(), + ) + .unwrap(); + assert!(plan.actions.is_empty()); + let sent = transport.requests(); + assert_eq!(sent.len(), 2); + assert!(sent[1].contains("Only the first 6 of 7 requested checks ran")); + assert!(sent[1].contains("remaining checks were not performed")); + } + + #[test] + fn oversized_investigation_still_rejects_a_mutation_after_the_limit() { + use crate::provider::{chat_response, ScriptedTransport}; + let mut requests: Vec<_> = (0..6) + .map(|index| serde_json::json!({"action":"search_packages","query":format!("etcher{index}")})) + .collect(); + requests.push(serde_json::json!({"action":"install_package","atom":"vim"})); + let transport = ScriptedTransport::new(vec![ + chat_response( + &serde_json::json!({"version":1,"kind":"investigate","requests":requests}) + .to_string(), + ), + chat_response( + r#"{"version":1,"summary":"I can search without changing the system.","actions":[]}"#, + ), + ]); + let investigate = |_: &[SystemAction]| -> Vec { + panic!("no checks should run when any request is mutating") + }; + let plan = run_assistant( + ProviderKind::Local, + "http://127.0.0.1:11434/v1/chat/completions", + "m", + &facts(), + "find etcher", + &transport, + &investigate, + &crate::install::Limits::default(), + ) + .unwrap(); + assert!(plan.actions.is_empty()); + } + #[test] fn run_assistant_never_executes_a_mutating_investigation_request() { use crate::provider::{chat_response, ScriptedTransport}; diff --git a/installer-rs/crates/phoebus-installer/src/assistant_bridge.rs b/installer-rs/crates/phoebus-installer/src/assistant_bridge.rs index 1c3ce4d..ebba93b 100644 --- a/installer-rs/crates/phoebus-installer/src/assistant_bridge.rs +++ b/installer-rs/crates/phoebus-installer/src/assistant_bridge.rs @@ -267,6 +267,21 @@ fn plan_response(id: u64, kind: &str, plan: &PreparedPlan) -> BridgeResponse { ) } +fn general_response(id: u64, plan: &PreparedPlan) -> BridgeResponse { + if plan.actions.is_empty() { + return BridgeResponse::ok( + id, + json!({ + "kind": "info", + "state": "info", + "summary": plan.summary, + "recommendation": plan.recommendation, + }), + ); + } + plan_response(id, "plan", plan) +} + pub fn handle_request( exec: &Exec, root: &Path, @@ -513,6 +528,7 @@ pub fn handle_request( }); match response { Ok(Ok(plan)) => { + let informational = plan.actions.is_empty(); record_turn( exec, &conv_path, @@ -520,16 +536,16 @@ pub fn handle_request( timestamp_unix: now_unix(), // Structured plan summary, not the raw request. query: plan.summary.clone(), - response_kind: "plan".into(), + response_kind: if informational { "info" } else { "plan" }.into(), application: None, chosen_candidate: None, alternatives: Vec::new(), - plan_id: Some(plan.id.clone()), - plan_digest: Some(plan.digest.clone()), + plan_id: (!informational).then(|| plan.id.clone()), + plan_digest: (!informational).then(|| plan.digest.clone()), rationale: plan.rationale.clone(), }, ); - plan_response(id, "plan", &plan) + general_response(id, &plan) } Ok(Err(e)) => BridgeResponse::err(id, e.to_string()), Err(e) => BridgeResponse::err(id, e), @@ -1102,6 +1118,48 @@ mod tests { assert_eq!(parse_install_phrase("why is audio broken"), None); } + #[test] + fn general_answer_without_actions_has_no_approval_controls() { + use phoebus_ai::provider::{chat_response, ProviderKind, ScriptedTransport}; + + let root = tempdir().unwrap(); + let exec = make_test_exec(root.path()); + let plans = PlanStore::with_dir(tempdir().unwrap().keep()); + let transport = ScriptedTransport::new(vec![chat_response( + r#"{"version":1,"summary":"An AppImage catalog lead was found; no installable package was verified.","actions":[]}"#, + )]); + let ai = AiInstall { + provider: ProviderKind::Local, + endpoint: "http://127.0.0.1:11434/v1/chat/completions".into(), + model: "m".into(), + transport: &transport, + limits: phoebus_ai::install::Limits::default(), + }; + let plan = service::plan_general( + &plans, + &exec, + root.path(), + "find Etcher", + &SystemFacts::default(), + &ai, + None, + ) + .unwrap(); + + let response = general_response(5, &plan); + assert!(response.ok); + let result = response.result.unwrap(); + assert_eq!(result["kind"], "info"); + assert_eq!(result["state"], "info"); + assert!(result["recommendation"] + .as_str() + .unwrap() + .contains("AppImage")); + assert!(result.get("prepared").is_none()); + assert!(result.get("plan_id").is_none()); + assert!(result.get("digest").is_none()); + } + #[test] fn appimage_install_request_returns_catalog_information_without_a_plan() { let root = tempdir().unwrap(); diff --git a/installer-rs/crates/phoebus-installer/src/bin/sys.rs b/installer-rs/crates/phoebus-installer/src/bin/sys.rs index 935afb2..c3c181c 100644 --- a/installer-rs/crates/phoebus-installer/src/bin/sys.rs +++ b/installer-rs/crates/phoebus-installer/src/bin/sys.rs @@ -1474,6 +1474,7 @@ fn run_ask( } }; + let informational = plan.actions.is_empty(); record_turn( exec, &conv_path, @@ -1482,12 +1483,12 @@ fn run_ask( // Persist the plan's own summary, not the raw request: structured // context is sufficient for follow-ups and minimizes exposure. query: plan.summary.clone(), - response_kind: "plan".into(), + response_kind: if informational { "info" } else { "plan" }.into(), application: None, chosen_candidate: None, alternatives: Vec::new(), - plan_id: Some(plan.id.clone()), - plan_digest: Some(plan.digest.clone()), + plan_id: (!informational).then(|| plan.id.clone()), + plan_digest: (!informational).then(|| plan.digest.clone()), rationale: plan.rationale.clone(), }, ); diff --git a/installer-rs/crates/phoebus-installer/src/service/plan.rs b/installer-rs/crates/phoebus-installer/src/service/plan.rs index 2b69e87..a9c399c 100644 --- a/installer-rs/crates/phoebus-installer/src/service/plan.rs +++ b/installer-rs/crates/phoebus-installer/src/service/plan.rs @@ -882,6 +882,8 @@ pub fn plan_general( integrity: String::new(), }; - plans.save(&mut plan)?; + if !plan.actions.is_empty() { + plans.save(&mut plan)?; + } Ok(plan) } diff --git a/installer-rs/crates/phoebus-installer/src/service/tests.rs b/installer-rs/crates/phoebus-installer/src/service/tests.rs index 82a3d63..926599e 100644 --- a/installer-rs/crates/phoebus-installer/src/service/tests.rs +++ b/installer-rs/crates/phoebus-installer/src/service/tests.rs @@ -1967,6 +1967,10 @@ fn plan_general_investigates_with_read_only_tools_then_answers() { assert!(plan.actions.is_empty(), "an answer has no actions"); assert_eq!(plan.summary, "Two real candidates found."); assert!(!plan.requires_root, "a read-only answer needs no elevation"); + assert!( + store.load(&plan.id).is_err(), + "an answer must not stage a plan" + ); // Two model turns: investigate, then answer. assert_eq!(transport.requests().len(), 2); assert!(transport.requests()[1].contains("Package catalogue"));