fix(ai): retry a rejected proposal once with a correction
The shipped DeepSeek models occasionally key a change or rationale wrongly. Retry once with a corrective turn; the strict parser remains the final authority and a retry that still fails is still rejected. Add scripted transport tests for both the proposal and assistant paths.
This commit is contained in:
parent
c16014ade8
commit
802b843bc9
2 changed files with 188 additions and 12 deletions
|
|
@ -145,7 +145,27 @@ pub fn build_request(
|
|||
}
|
||||
}
|
||||
|
||||
/// Ask the provider for a plan and parse it, fail-closed at every step.
|
||||
/// The request with a corrective turn appended after a rejected plan.
|
||||
pub fn build_corrected_request(
|
||||
provider: ProviderKind,
|
||||
model: &str,
|
||||
facts: &SystemFacts,
|
||||
instruction: &str,
|
||||
correction: &str,
|
||||
) -> ChatRequest {
|
||||
let mut request = build_request(provider, model, facts, instruction);
|
||||
request.messages.push(ChatMessage {
|
||||
role: "user".into(),
|
||||
content: format!(
|
||||
"Your previous reply was rejected: {correction}. Reply with ONLY the \
|
||||
corrected JSON object, with no prose and no code fence."
|
||||
),
|
||||
});
|
||||
request
|
||||
}
|
||||
|
||||
/// Ask the provider for a plan and parse it, fail-closed at every step. A
|
||||
/// rejected reply is retried once with a corrective turn.
|
||||
pub fn complete_plan(
|
||||
provider: ProviderKind,
|
||||
endpoint: &str,
|
||||
|
|
@ -165,11 +185,27 @@ pub fn complete_plan(
|
|||
));
|
||||
}
|
||||
facts.validate_no_forbidden()?;
|
||||
let request = build_request(provider, model, facts, instruction);
|
||||
let body = serde_json::to_string(&request)?;
|
||||
|
||||
let attempt = |request: &ChatRequest| -> Result<AssistantPlan, AiError> {
|
||||
let body = serde_json::to_string(request)?;
|
||||
let response = transport.send(provider, endpoint, &body)?;
|
||||
let content = crate::provider::extract_content(&response)?;
|
||||
parse_plan(&content)
|
||||
};
|
||||
|
||||
match attempt(&build_request(provider, model, facts, instruction)) {
|
||||
Ok(plan) => Ok(plan),
|
||||
Err(first) => {
|
||||
let correction = first.to_string();
|
||||
attempt(&build_corrected_request(
|
||||
provider,
|
||||
model,
|
||||
facts,
|
||||
instruction,
|
||||
&correction,
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -279,6 +315,30 @@ mod tests {
|
|||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rejected_plan_is_retried_once_with_a_correction() {
|
||||
use crate::provider::{chat_response, ScriptedTransport};
|
||||
let bad =
|
||||
chat_response(r#"{"version":1,"summary":"x","actions":[{"action":"run_shell"}]}"#);
|
||||
let good = chat_response(
|
||||
r#"{"version":1,"summary":"look","actions":[{"action":"history"}],"rationale":[{"action":"history","reason":"asked"}]}"#,
|
||||
);
|
||||
let transport = ScriptedTransport::new(vec![bad, good]);
|
||||
let plan = complete_plan(
|
||||
ProviderKind::Local,
|
||||
"http://127.0.0.1:11434/v1/chat/completions",
|
||||
"m",
|
||||
&facts(),
|
||||
"what changed?",
|
||||
&transport,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(plan.actions.len(), 1);
|
||||
let requests = transport.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert!(requests[1].contains("rejected"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn action_tag_matches_the_schema() {
|
||||
assert_eq!(action_tag(&SystemAction::History), "history");
|
||||
|
|
|
|||
|
|
@ -375,6 +375,42 @@ impl ModelTransport for TestTransport {
|
|||
}
|
||||
}
|
||||
|
||||
/// A transport that returns a scripted sequence of responses, one per call
|
||||
/// (tests of retry and multi-turn behaviour).
|
||||
pub struct ScriptedTransport {
|
||||
responses: std::sync::Mutex<std::collections::VecDeque<String>>,
|
||||
requests: std::sync::Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl ScriptedTransport {
|
||||
pub fn new(responses: Vec<String>) -> Self {
|
||||
Self {
|
||||
responses: std::sync::Mutex::new(responses.into()),
|
||||
requests: std::sync::Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn requests(&self) -> Vec<String> {
|
||||
self.requests.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelTransport for ScriptedTransport {
|
||||
fn send(&self, _provider: ProviderKind, _url: &str, body: &str) -> Result<String, AiError> {
|
||||
self.requests.lock().unwrap().push(body.to_string());
|
||||
self.responses
|
||||
.lock()
|
||||
.unwrap()
|
||||
.pop_front()
|
||||
.ok_or_else(|| AiError::Provider("no scripted response left".into()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrap a bare proposal in an OpenAI-style chat response (test helper).
|
||||
pub fn chat_response(content: &str) -> String {
|
||||
serde_json::json!({ "choices": [{ "message": { "content": content } }] }).to_string()
|
||||
}
|
||||
|
||||
/// Ask a provider for a proposal and parse it, fail-closed at every step.
|
||||
pub fn complete_proposal(
|
||||
provider: ProviderKind,
|
||||
|
|
@ -385,8 +421,30 @@ pub fn complete_proposal(
|
|||
complete_proposal_at(provider, provider.endpoint(), model, facts, transport)
|
||||
}
|
||||
|
||||
/// Build the request again, appending a corrective turn after a rejected reply.
|
||||
/// Models occasionally key a change by the action name or add prose; one
|
||||
/// bounded corrective round-trip meaningfully improves reliability without
|
||||
/// weakening the parser.
|
||||
pub fn build_corrected_request(
|
||||
provider: ProviderKind,
|
||||
model: &str,
|
||||
facts: &SystemFacts,
|
||||
correction: &str,
|
||||
) -> ChatRequest {
|
||||
let mut request = build_request(provider, model, facts);
|
||||
request.messages.push(ChatMessage {
|
||||
role: "user".into(),
|
||||
content: format!(
|
||||
"Your previous reply was rejected: {correction}. Reply with ONLY the \
|
||||
corrected JSON object, with no prose and no code fence."
|
||||
),
|
||||
});
|
||||
request
|
||||
}
|
||||
|
||||
/// As `complete_proposal`, but with an explicit endpoint (`--ai-endpoint`), for
|
||||
/// a self-hosted model on a non-default port.
|
||||
/// a self-hosted model on a non-default port. A rejected reply is retried once
|
||||
/// with a corrective turn; the strict parser is still the final authority.
|
||||
pub fn complete_proposal_at(
|
||||
provider: ProviderKind,
|
||||
endpoint: &str,
|
||||
|
|
@ -401,11 +459,26 @@ pub fn complete_proposal_at(
|
|||
}
|
||||
validate_endpoint(endpoint)?;
|
||||
facts.validate_no_forbidden()?;
|
||||
let request = build_request(provider, model, facts);
|
||||
let body = serde_json::to_string(&request)?;
|
||||
|
||||
let attempt = |request: &ChatRequest| -> Result<AiProposal, AiError> {
|
||||
let body = serde_json::to_string(request)?;
|
||||
let response = transport.send(provider, endpoint, &body)?;
|
||||
let content = extract_content(&response)?;
|
||||
parse_proposal(&content)
|
||||
};
|
||||
|
||||
match attempt(&build_request(provider, model, facts)) {
|
||||
Ok(proposal) => Ok(proposal),
|
||||
Err(first) => {
|
||||
let correction = first.to_string();
|
||||
attempt(&build_corrected_request(
|
||||
provider,
|
||||
model,
|
||||
facts,
|
||||
&correction,
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -569,6 +642,49 @@ mod tests {
|
|||
assert_eq!(parsed.changes.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rejected_proposal_is_retried_once_with_a_correction() {
|
||||
// First response uses the wrong key (the failure seen with a real
|
||||
// model); the second is corrected.
|
||||
let bad = chat_response(
|
||||
r#"{"version":1,"changes":[{"action":"set_kernel_profile","profile":"safe"}],"rationale":[]}"#,
|
||||
);
|
||||
let good = chat_response(
|
||||
r#"{"version":1,"changes":[{"action":"set_kernel_profile","value":"safe"}],"rationale":[{"action":"set_kernel_profile","reason":"x"}]}"#,
|
||||
);
|
||||
let transport = ScriptedTransport::new(vec![bad, good]);
|
||||
let proposal = complete_proposal_at(
|
||||
ProviderKind::Local,
|
||||
"http://127.0.0.1:8080/v1/chat/completions",
|
||||
"m",
|
||||
&facts(),
|
||||
&transport,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(proposal.changes.len(), 1);
|
||||
let requests = transport.requests();
|
||||
assert_eq!(requests.len(), 2, "exactly one corrective retry");
|
||||
assert!(
|
||||
requests[1].contains("rejected"),
|
||||
"the retry carries the reason"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_retry_that_still_fails_is_still_rejected() {
|
||||
let bad = chat_response(r#"{"version":1,"changes":[{"action":"nope"}]}"#);
|
||||
let transport = ScriptedTransport::new(vec![bad.clone(), bad]);
|
||||
assert!(complete_proposal_at(
|
||||
ProviderKind::Local,
|
||||
"http://127.0.0.1:8080/v1/chat/completions",
|
||||
"m",
|
||||
&facts(),
|
||||
&transport
|
||||
)
|
||||
.is_err());
|
||||
assert_eq!(transport.requests().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_provider_is_explicit() {
|
||||
let transport = TestTransport::new("{}");
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue