From 802b843bc9484ed725c73a7e703c74918a5da411 Mon Sep 17 00:00:00 2001 From: Phoebus Date: Tue, 22 Sep 2026 03:32:07 -0700 Subject: [PATCH] fix(ai): retry a rejected proposal once with a correction The shipped DeepSeek models occasionally key a change or rationale wrongly. Retry once with a corrective turn; the strict parser remains the final authority and a retry that still fails is still rejected. Add scripted transport tests for both the proposal and assistant paths. --- .../crates/phoebus-ai/src/assistant.rs | 72 +++++++++- .../crates/phoebus-ai/src/provider.rs | 128 +++++++++++++++++- 2 files changed, 188 insertions(+), 12 deletions(-) diff --git a/installer-rs/crates/phoebus-ai/src/assistant.rs b/installer-rs/crates/phoebus-ai/src/assistant.rs index 2eb46ed..4906ab7 100644 --- a/installer-rs/crates/phoebus-ai/src/assistant.rs +++ b/installer-rs/crates/phoebus-ai/src/assistant.rs @@ -145,7 +145,27 @@ pub fn build_request( } } -/// Ask the provider for a plan and parse it, fail-closed at every step. +/// The request with a corrective turn appended after a rejected plan. +pub fn build_corrected_request( + provider: ProviderKind, + model: &str, + facts: &SystemFacts, + instruction: &str, + correction: &str, +) -> ChatRequest { + let mut request = build_request(provider, model, facts, instruction); + request.messages.push(ChatMessage { + role: "user".into(), + content: format!( + "Your previous reply was rejected: {correction}. Reply with ONLY the \ + corrected JSON object, with no prose and no code fence." + ), + }); + request +} + +/// Ask the provider for a plan and parse it, fail-closed at every step. A +/// rejected reply is retried once with a corrective turn. pub fn complete_plan( provider: ProviderKind, endpoint: &str, @@ -165,11 +185,27 @@ pub fn complete_plan( )); } facts.validate_no_forbidden()?; - let request = build_request(provider, model, facts, instruction); - let body = serde_json::to_string(&request)?; - let response = transport.send(provider, endpoint, &body)?; - let content = crate::provider::extract_content(&response)?; - parse_plan(&content) + + let attempt = |request: &ChatRequest| -> Result { + let body = serde_json::to_string(request)?; + let response = transport.send(provider, endpoint, &body)?; + let content = crate::provider::extract_content(&response)?; + parse_plan(&content) + }; + + match attempt(&build_request(provider, model, facts, instruction)) { + Ok(plan) => Ok(plan), + Err(first) => { + let correction = first.to_string(); + attempt(&build_corrected_request( + provider, + model, + facts, + instruction, + &correction, + )) + } + } } #[cfg(test)] @@ -279,6 +315,30 @@ mod tests { .is_err()); } + #[test] + fn a_rejected_plan_is_retried_once_with_a_correction() { + use crate::provider::{chat_response, ScriptedTransport}; + let bad = + chat_response(r#"{"version":1,"summary":"x","actions":[{"action":"run_shell"}]}"#); + let good = chat_response( + r#"{"version":1,"summary":"look","actions":[{"action":"history"}],"rationale":[{"action":"history","reason":"asked"}]}"#, + ); + let transport = ScriptedTransport::new(vec![bad, good]); + let plan = complete_plan( + ProviderKind::Local, + "http://127.0.0.1:11434/v1/chat/completions", + "m", + &facts(), + "what changed?", + &transport, + ) + .unwrap(); + assert_eq!(plan.actions.len(), 1); + let requests = transport.requests(); + assert_eq!(requests.len(), 2); + assert!(requests[1].contains("rejected")); + } + #[test] fn action_tag_matches_the_schema() { assert_eq!(action_tag(&SystemAction::History), "history"); diff --git a/installer-rs/crates/phoebus-ai/src/provider.rs b/installer-rs/crates/phoebus-ai/src/provider.rs index ed9f3d5..ca50f17 100644 --- a/installer-rs/crates/phoebus-ai/src/provider.rs +++ b/installer-rs/crates/phoebus-ai/src/provider.rs @@ -375,6 +375,42 @@ impl ModelTransport for TestTransport { } } +/// A transport that returns a scripted sequence of responses, one per call +/// (tests of retry and multi-turn behaviour). +pub struct ScriptedTransport { + responses: std::sync::Mutex>, + requests: std::sync::Mutex>, +} + +impl ScriptedTransport { + pub fn new(responses: Vec) -> Self { + Self { + responses: std::sync::Mutex::new(responses.into()), + requests: std::sync::Mutex::new(Vec::new()), + } + } + + pub fn requests(&self) -> Vec { + self.requests.lock().unwrap().clone() + } +} + +impl ModelTransport for ScriptedTransport { + fn send(&self, _provider: ProviderKind, _url: &str, body: &str) -> Result { + self.requests.lock().unwrap().push(body.to_string()); + self.responses + .lock() + .unwrap() + .pop_front() + .ok_or_else(|| AiError::Provider("no scripted response left".into())) + } +} + +/// Wrap a bare proposal in an OpenAI-style chat response (test helper). +pub fn chat_response(content: &str) -> String { + serde_json::json!({ "choices": [{ "message": { "content": content } }] }).to_string() +} + /// Ask a provider for a proposal and parse it, fail-closed at every step. pub fn complete_proposal( provider: ProviderKind, @@ -385,8 +421,30 @@ pub fn complete_proposal( complete_proposal_at(provider, provider.endpoint(), model, facts, transport) } +/// Build the request again, appending a corrective turn after a rejected reply. +/// Models occasionally key a change by the action name or add prose; one +/// bounded corrective round-trip meaningfully improves reliability without +/// weakening the parser. +pub fn build_corrected_request( + provider: ProviderKind, + model: &str, + facts: &SystemFacts, + correction: &str, +) -> ChatRequest { + let mut request = build_request(provider, model, facts); + request.messages.push(ChatMessage { + role: "user".into(), + content: format!( + "Your previous reply was rejected: {correction}. Reply with ONLY the \ + corrected JSON object, with no prose and no code fence." + ), + }); + request +} + /// As `complete_proposal`, but with an explicit endpoint (`--ai-endpoint`), for -/// a self-hosted model on a non-default port. +/// a self-hosted model on a non-default port. A rejected reply is retried once +/// with a corrective turn; the strict parser is still the final authority. pub fn complete_proposal_at( provider: ProviderKind, endpoint: &str, @@ -401,11 +459,26 @@ pub fn complete_proposal_at( } validate_endpoint(endpoint)?; facts.validate_no_forbidden()?; - let request = build_request(provider, model, facts); - let body = serde_json::to_string(&request)?; - let response = transport.send(provider, endpoint, &body)?; - let content = extract_content(&response)?; - parse_proposal(&content) + + let attempt = |request: &ChatRequest| -> Result { + let body = serde_json::to_string(request)?; + let response = transport.send(provider, endpoint, &body)?; + let content = extract_content(&response)?; + parse_proposal(&content) + }; + + match attempt(&build_request(provider, model, facts)) { + Ok(proposal) => Ok(proposal), + Err(first) => { + let correction = first.to_string(); + attempt(&build_corrected_request( + provider, + model, + facts, + &correction, + )) + } + } } #[cfg(test)] @@ -569,6 +642,49 @@ mod tests { assert_eq!(parsed.changes.len(), 1); } + #[test] + fn a_rejected_proposal_is_retried_once_with_a_correction() { + // First response uses the wrong key (the failure seen with a real + // model); the second is corrected. + let bad = chat_response( + r#"{"version":1,"changes":[{"action":"set_kernel_profile","profile":"safe"}],"rationale":[]}"#, + ); + let good = chat_response( + r#"{"version":1,"changes":[{"action":"set_kernel_profile","value":"safe"}],"rationale":[{"action":"set_kernel_profile","reason":"x"}]}"#, + ); + let transport = ScriptedTransport::new(vec![bad, good]); + let proposal = complete_proposal_at( + ProviderKind::Local, + "http://127.0.0.1:8080/v1/chat/completions", + "m", + &facts(), + &transport, + ) + .unwrap(); + assert_eq!(proposal.changes.len(), 1); + let requests = transport.requests(); + assert_eq!(requests.len(), 2, "exactly one corrective retry"); + assert!( + requests[1].contains("rejected"), + "the retry carries the reason" + ); + } + + #[test] + fn a_retry_that_still_fails_is_still_rejected() { + let bad = chat_response(r#"{"version":1,"changes":[{"action":"nope"}]}"#); + let transport = ScriptedTransport::new(vec![bad.clone(), bad]); + assert!(complete_proposal_at( + ProviderKind::Local, + "http://127.0.0.1:8080/v1/chat/completions", + "m", + &facts(), + &transport + ) + .is_err()); + assert_eq!(transport.requests().len(), 2); + } + #[test] fn unavailable_provider_is_explicit() { let transport = TestTransport::new("{}");