fix(ai): retry a rejected proposal once with a correction

The shipped DeepSeek models occasionally key a change or rationale wrongly.
Retry once with a corrective turn; the strict parser remains the final
authority and a retry that still fails is still rejected. Add scripted
transport tests for both the proposal and assistant paths.
This commit is contained in:
Phoebus 2026-09-22 03:32:07 -07:00
parent c16014ade8
commit 802b843bc9
2 changed files with 188 additions and 12 deletions

View file

@ -145,7 +145,27 @@ pub fn build_request(
}
}
/// Ask the provider for a plan and parse it, fail-closed at every step.
/// The request with a corrective turn appended after a rejected plan.
pub fn build_corrected_request(
provider: ProviderKind,
model: &str,
facts: &SystemFacts,
instruction: &str,
correction: &str,
) -> ChatRequest {
let mut request = build_request(provider, model, facts, instruction);
request.messages.push(ChatMessage {
role: "user".into(),
content: format!(
"Your previous reply was rejected: {correction}. Reply with ONLY the \
corrected JSON object, with no prose and no code fence."
),
});
request
}
/// Ask the provider for a plan and parse it, fail-closed at every step. A
/// rejected reply is retried once with a corrective turn.
pub fn complete_plan(
provider: ProviderKind,
endpoint: &str,
@ -165,11 +185,27 @@ pub fn complete_plan(
));
}
facts.validate_no_forbidden()?;
let request = build_request(provider, model, facts, instruction);
let body = serde_json::to_string(&request)?;
let attempt = |request: &ChatRequest| -> Result<AssistantPlan, AiError> {
let body = serde_json::to_string(request)?;
let response = transport.send(provider, endpoint, &body)?;
let content = crate::provider::extract_content(&response)?;
parse_plan(&content)
};
match attempt(&build_request(provider, model, facts, instruction)) {
Ok(plan) => Ok(plan),
Err(first) => {
let correction = first.to_string();
attempt(&build_corrected_request(
provider,
model,
facts,
instruction,
&correction,
))
}
}
}
#[cfg(test)]
@ -279,6 +315,30 @@ mod tests {
.is_err());
}
#[test]
fn a_rejected_plan_is_retried_once_with_a_correction() {
use crate::provider::{chat_response, ScriptedTransport};
let bad =
chat_response(r#"{"version":1,"summary":"x","actions":[{"action":"run_shell"}]}"#);
let good = chat_response(
r#"{"version":1,"summary":"look","actions":[{"action":"history"}],"rationale":[{"action":"history","reason":"asked"}]}"#,
);
let transport = ScriptedTransport::new(vec![bad, good]);
let plan = complete_plan(
ProviderKind::Local,
"http://127.0.0.1:11434/v1/chat/completions",
"m",
&facts(),
"what changed?",
&transport,
)
.unwrap();
assert_eq!(plan.actions.len(), 1);
let requests = transport.requests();
assert_eq!(requests.len(), 2);
assert!(requests[1].contains("rejected"));
}
#[test]
fn action_tag_matches_the_schema() {
assert_eq!(action_tag(&SystemAction::History), "history");

View file

@ -375,6 +375,42 @@ impl ModelTransport for TestTransport {
}
}
/// A transport that returns a scripted sequence of responses, one per call
/// (tests of retry and multi-turn behaviour).
pub struct ScriptedTransport {
responses: std::sync::Mutex<std::collections::VecDeque<String>>,
requests: std::sync::Mutex<Vec<String>>,
}
impl ScriptedTransport {
pub fn new(responses: Vec<String>) -> Self {
Self {
responses: std::sync::Mutex::new(responses.into()),
requests: std::sync::Mutex::new(Vec::new()),
}
}
pub fn requests(&self) -> Vec<String> {
self.requests.lock().unwrap().clone()
}
}
impl ModelTransport for ScriptedTransport {
fn send(&self, _provider: ProviderKind, _url: &str, body: &str) -> Result<String, AiError> {
self.requests.lock().unwrap().push(body.to_string());
self.responses
.lock()
.unwrap()
.pop_front()
.ok_or_else(|| AiError::Provider("no scripted response left".into()))
}
}
/// Wrap a bare proposal in an OpenAI-style chat response (test helper).
pub fn chat_response(content: &str) -> String {
serde_json::json!({ "choices": [{ "message": { "content": content } }] }).to_string()
}
/// Ask a provider for a proposal and parse it, fail-closed at every step.
pub fn complete_proposal(
provider: ProviderKind,
@ -385,8 +421,30 @@ pub fn complete_proposal(
complete_proposal_at(provider, provider.endpoint(), model, facts, transport)
}
/// Build the request again, appending a corrective turn after a rejected reply.
/// Models occasionally key a change by the action name or add prose; one
/// bounded corrective round-trip meaningfully improves reliability without
/// weakening the parser.
pub fn build_corrected_request(
provider: ProviderKind,
model: &str,
facts: &SystemFacts,
correction: &str,
) -> ChatRequest {
let mut request = build_request(provider, model, facts);
request.messages.push(ChatMessage {
role: "user".into(),
content: format!(
"Your previous reply was rejected: {correction}. Reply with ONLY the \
corrected JSON object, with no prose and no code fence."
),
});
request
}
/// As `complete_proposal`, but with an explicit endpoint (`--ai-endpoint`), for
/// a self-hosted model on a non-default port.
/// a self-hosted model on a non-default port. A rejected reply is retried once
/// with a corrective turn; the strict parser is still the final authority.
pub fn complete_proposal_at(
provider: ProviderKind,
endpoint: &str,
@ -401,11 +459,26 @@ pub fn complete_proposal_at(
}
validate_endpoint(endpoint)?;
facts.validate_no_forbidden()?;
let request = build_request(provider, model, facts);
let body = serde_json::to_string(&request)?;
let attempt = |request: &ChatRequest| -> Result<AiProposal, AiError> {
let body = serde_json::to_string(request)?;
let response = transport.send(provider, endpoint, &body)?;
let content = extract_content(&response)?;
parse_proposal(&content)
};
match attempt(&build_request(provider, model, facts)) {
Ok(proposal) => Ok(proposal),
Err(first) => {
let correction = first.to_string();
attempt(&build_corrected_request(
provider,
model,
facts,
&correction,
))
}
}
}
#[cfg(test)]
@ -569,6 +642,49 @@ mod tests {
assert_eq!(parsed.changes.len(), 1);
}
#[test]
fn a_rejected_proposal_is_retried_once_with_a_correction() {
// First response uses the wrong key (the failure seen with a real
// model); the second is corrected.
let bad = chat_response(
r#"{"version":1,"changes":[{"action":"set_kernel_profile","profile":"safe"}],"rationale":[]}"#,
);
let good = chat_response(
r#"{"version":1,"changes":[{"action":"set_kernel_profile","value":"safe"}],"rationale":[{"action":"set_kernel_profile","reason":"x"}]}"#,
);
let transport = ScriptedTransport::new(vec![bad, good]);
let proposal = complete_proposal_at(
ProviderKind::Local,
"http://127.0.0.1:8080/v1/chat/completions",
"m",
&facts(),
&transport,
)
.unwrap();
assert_eq!(proposal.changes.len(), 1);
let requests = transport.requests();
assert_eq!(requests.len(), 2, "exactly one corrective retry");
assert!(
requests[1].contains("rejected"),
"the retry carries the reason"
);
}
#[test]
fn a_retry_that_still_fails_is_still_rejected() {
let bad = chat_response(r#"{"version":1,"changes":[{"action":"nope"}]}"#);
let transport = ScriptedTransport::new(vec![bad.clone(), bad]);
assert!(complete_proposal_at(
ProviderKind::Local,
"http://127.0.0.1:8080/v1/chat/completions",
"m",
&facts(),
&transport
)
.is_err());
assert_eq!(transport.requests().len(), 2);
}
#[test]
fn unavailable_provider_is_explicit() {
let transport = TestTransport::new("{}");