fix(ai): retry a rejected proposal once with a correction
The shipped DeepSeek models occasionally key a change or rationale wrongly. Retry once with a corrective turn; the strict parser remains the final authority and a retry that still fails is still rejected. Add scripted transport tests for both the proposal and assistant paths.
This commit is contained in:
parent
c16014ade8
commit
802b843bc9
2 changed files with 188 additions and 12 deletions
|
|
@ -145,7 +145,27 @@ pub fn build_request(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Ask the provider for a plan and parse it, fail-closed at every step.
|
/// The request with a corrective turn appended after a rejected plan.
|
||||||
|
pub fn build_corrected_request(
|
||||||
|
provider: ProviderKind,
|
||||||
|
model: &str,
|
||||||
|
facts: &SystemFacts,
|
||||||
|
instruction: &str,
|
||||||
|
correction: &str,
|
||||||
|
) -> ChatRequest {
|
||||||
|
let mut request = build_request(provider, model, facts, instruction);
|
||||||
|
request.messages.push(ChatMessage {
|
||||||
|
role: "user".into(),
|
||||||
|
content: format!(
|
||||||
|
"Your previous reply was rejected: {correction}. Reply with ONLY the \
|
||||||
|
corrected JSON object, with no prose and no code fence."
|
||||||
|
),
|
||||||
|
});
|
||||||
|
request
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ask the provider for a plan and parse it, fail-closed at every step. A
|
||||||
|
/// rejected reply is retried once with a corrective turn.
|
||||||
pub fn complete_plan(
|
pub fn complete_plan(
|
||||||
provider: ProviderKind,
|
provider: ProviderKind,
|
||||||
endpoint: &str,
|
endpoint: &str,
|
||||||
|
|
@ -165,11 +185,27 @@ pub fn complete_plan(
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
facts.validate_no_forbidden()?;
|
facts.validate_no_forbidden()?;
|
||||||
let request = build_request(provider, model, facts, instruction);
|
|
||||||
let body = serde_json::to_string(&request)?;
|
let attempt = |request: &ChatRequest| -> Result<AssistantPlan, AiError> {
|
||||||
let response = transport.send(provider, endpoint, &body)?;
|
let body = serde_json::to_string(request)?;
|
||||||
let content = crate::provider::extract_content(&response)?;
|
let response = transport.send(provider, endpoint, &body)?;
|
||||||
parse_plan(&content)
|
let content = crate::provider::extract_content(&response)?;
|
||||||
|
parse_plan(&content)
|
||||||
|
};
|
||||||
|
|
||||||
|
match attempt(&build_request(provider, model, facts, instruction)) {
|
||||||
|
Ok(plan) => Ok(plan),
|
||||||
|
Err(first) => {
|
||||||
|
let correction = first.to_string();
|
||||||
|
attempt(&build_corrected_request(
|
||||||
|
provider,
|
||||||
|
model,
|
||||||
|
facts,
|
||||||
|
instruction,
|
||||||
|
&correction,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
@ -279,6 +315,30 @@ mod tests {
|
||||||
.is_err());
|
.is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_rejected_plan_is_retried_once_with_a_correction() {
|
||||||
|
use crate::provider::{chat_response, ScriptedTransport};
|
||||||
|
let bad =
|
||||||
|
chat_response(r#"{"version":1,"summary":"x","actions":[{"action":"run_shell"}]}"#);
|
||||||
|
let good = chat_response(
|
||||||
|
r#"{"version":1,"summary":"look","actions":[{"action":"history"}],"rationale":[{"action":"history","reason":"asked"}]}"#,
|
||||||
|
);
|
||||||
|
let transport = ScriptedTransport::new(vec![bad, good]);
|
||||||
|
let plan = complete_plan(
|
||||||
|
ProviderKind::Local,
|
||||||
|
"http://127.0.0.1:11434/v1/chat/completions",
|
||||||
|
"m",
|
||||||
|
&facts(),
|
||||||
|
"what changed?",
|
||||||
|
&transport,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(plan.actions.len(), 1);
|
||||||
|
let requests = transport.requests();
|
||||||
|
assert_eq!(requests.len(), 2);
|
||||||
|
assert!(requests[1].contains("rejected"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn action_tag_matches_the_schema() {
|
fn action_tag_matches_the_schema() {
|
||||||
assert_eq!(action_tag(&SystemAction::History), "history");
|
assert_eq!(action_tag(&SystemAction::History), "history");
|
||||||
|
|
|
||||||
|
|
@ -375,6 +375,42 @@ impl ModelTransport for TestTransport {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A transport that returns a scripted sequence of responses, one per call
|
||||||
|
/// (tests of retry and multi-turn behaviour).
|
||||||
|
pub struct ScriptedTransport {
|
||||||
|
responses: std::sync::Mutex<std::collections::VecDeque<String>>,
|
||||||
|
requests: std::sync::Mutex<Vec<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScriptedTransport {
|
||||||
|
pub fn new(responses: Vec<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
responses: std::sync::Mutex::new(responses.into()),
|
||||||
|
requests: std::sync::Mutex::new(Vec::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn requests(&self) -> Vec<String> {
|
||||||
|
self.requests.lock().unwrap().clone()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ModelTransport for ScriptedTransport {
|
||||||
|
fn send(&self, _provider: ProviderKind, _url: &str, body: &str) -> Result<String, AiError> {
|
||||||
|
self.requests.lock().unwrap().push(body.to_string());
|
||||||
|
self.responses
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.pop_front()
|
||||||
|
.ok_or_else(|| AiError::Provider("no scripted response left".into()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wrap a bare proposal in an OpenAI-style chat response (test helper).
|
||||||
|
pub fn chat_response(content: &str) -> String {
|
||||||
|
serde_json::json!({ "choices": [{ "message": { "content": content } }] }).to_string()
|
||||||
|
}
|
||||||
|
|
||||||
/// Ask a provider for a proposal and parse it, fail-closed at every step.
|
/// Ask a provider for a proposal and parse it, fail-closed at every step.
|
||||||
pub fn complete_proposal(
|
pub fn complete_proposal(
|
||||||
provider: ProviderKind,
|
provider: ProviderKind,
|
||||||
|
|
@ -385,8 +421,30 @@ pub fn complete_proposal(
|
||||||
complete_proposal_at(provider, provider.endpoint(), model, facts, transport)
|
complete_proposal_at(provider, provider.endpoint(), model, facts, transport)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Build the request again, appending a corrective turn after a rejected reply.
|
||||||
|
/// Models occasionally key a change by the action name or add prose; one
|
||||||
|
/// bounded corrective round-trip meaningfully improves reliability without
|
||||||
|
/// weakening the parser.
|
||||||
|
pub fn build_corrected_request(
|
||||||
|
provider: ProviderKind,
|
||||||
|
model: &str,
|
||||||
|
facts: &SystemFacts,
|
||||||
|
correction: &str,
|
||||||
|
) -> ChatRequest {
|
||||||
|
let mut request = build_request(provider, model, facts);
|
||||||
|
request.messages.push(ChatMessage {
|
||||||
|
role: "user".into(),
|
||||||
|
content: format!(
|
||||||
|
"Your previous reply was rejected: {correction}. Reply with ONLY the \
|
||||||
|
corrected JSON object, with no prose and no code fence."
|
||||||
|
),
|
||||||
|
});
|
||||||
|
request
|
||||||
|
}
|
||||||
|
|
||||||
/// As `complete_proposal`, but with an explicit endpoint (`--ai-endpoint`), for
|
/// As `complete_proposal`, but with an explicit endpoint (`--ai-endpoint`), for
|
||||||
/// a self-hosted model on a non-default port.
|
/// a self-hosted model on a non-default port. A rejected reply is retried once
|
||||||
|
/// with a corrective turn; the strict parser is still the final authority.
|
||||||
pub fn complete_proposal_at(
|
pub fn complete_proposal_at(
|
||||||
provider: ProviderKind,
|
provider: ProviderKind,
|
||||||
endpoint: &str,
|
endpoint: &str,
|
||||||
|
|
@ -401,11 +459,26 @@ pub fn complete_proposal_at(
|
||||||
}
|
}
|
||||||
validate_endpoint(endpoint)?;
|
validate_endpoint(endpoint)?;
|
||||||
facts.validate_no_forbidden()?;
|
facts.validate_no_forbidden()?;
|
||||||
let request = build_request(provider, model, facts);
|
|
||||||
let body = serde_json::to_string(&request)?;
|
let attempt = |request: &ChatRequest| -> Result<AiProposal, AiError> {
|
||||||
let response = transport.send(provider, endpoint, &body)?;
|
let body = serde_json::to_string(request)?;
|
||||||
let content = extract_content(&response)?;
|
let response = transport.send(provider, endpoint, &body)?;
|
||||||
parse_proposal(&content)
|
let content = extract_content(&response)?;
|
||||||
|
parse_proposal(&content)
|
||||||
|
};
|
||||||
|
|
||||||
|
match attempt(&build_request(provider, model, facts)) {
|
||||||
|
Ok(proposal) => Ok(proposal),
|
||||||
|
Err(first) => {
|
||||||
|
let correction = first.to_string();
|
||||||
|
attempt(&build_corrected_request(
|
||||||
|
provider,
|
||||||
|
model,
|
||||||
|
facts,
|
||||||
|
&correction,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
@ -569,6 +642,49 @@ mod tests {
|
||||||
assert_eq!(parsed.changes.len(), 1);
|
assert_eq!(parsed.changes.len(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_rejected_proposal_is_retried_once_with_a_correction() {
|
||||||
|
// First response uses the wrong key (the failure seen with a real
|
||||||
|
// model); the second is corrected.
|
||||||
|
let bad = chat_response(
|
||||||
|
r#"{"version":1,"changes":[{"action":"set_kernel_profile","profile":"safe"}],"rationale":[]}"#,
|
||||||
|
);
|
||||||
|
let good = chat_response(
|
||||||
|
r#"{"version":1,"changes":[{"action":"set_kernel_profile","value":"safe"}],"rationale":[{"action":"set_kernel_profile","reason":"x"}]}"#,
|
||||||
|
);
|
||||||
|
let transport = ScriptedTransport::new(vec![bad, good]);
|
||||||
|
let proposal = complete_proposal_at(
|
||||||
|
ProviderKind::Local,
|
||||||
|
"http://127.0.0.1:8080/v1/chat/completions",
|
||||||
|
"m",
|
||||||
|
&facts(),
|
||||||
|
&transport,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(proposal.changes.len(), 1);
|
||||||
|
let requests = transport.requests();
|
||||||
|
assert_eq!(requests.len(), 2, "exactly one corrective retry");
|
||||||
|
assert!(
|
||||||
|
requests[1].contains("rejected"),
|
||||||
|
"the retry carries the reason"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_retry_that_still_fails_is_still_rejected() {
|
||||||
|
let bad = chat_response(r#"{"version":1,"changes":[{"action":"nope"}]}"#);
|
||||||
|
let transport = ScriptedTransport::new(vec![bad.clone(), bad]);
|
||||||
|
assert!(complete_proposal_at(
|
||||||
|
ProviderKind::Local,
|
||||||
|
"http://127.0.0.1:8080/v1/chat/completions",
|
||||||
|
"m",
|
||||||
|
&facts(),
|
||||||
|
&transport
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
assert_eq!(transport.requests().len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn unavailable_provider_is_explicit() {
|
fn unavailable_provider_is_explicit() {
|
||||||
let transport = TestTransport::new("{}");
|
let transport = TestTransport::new("{}");
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue