Integration tests drive the curl client against a local TCP server (status preservation, chunked multibyte bodies, oversized bodies, redirects, abrupt disconnects, authentication). The live tests obtain a real model response when PHOEBUS_OPENCODE_LIVE_* is set and fail as BLOCKED when a live run is required but not configured.
140 lines
5.1 KiB
Rust
140 lines
5.1 KiB
Rust
//! Live OpenCode acceptance test.
|
|
//!
|
|
//! This is the only test that proves **real model inference** through OpenCode.
|
|
//! It is skipped unless the environment names a server, provider and model:
|
|
//!
|
|
//! ```sh
|
|
//! PHOEBUS_OPENCODE_LIVE_URL=http://127.0.0.1:4096 \
|
|
//! PHOEBUS_OPENCODE_LIVE_PROVIDER=deepseek \
|
|
//! PHOEBUS_OPENCODE_LIVE_MODEL=deepseek-chat \
|
|
//! cargo test -p phoebus-executor --test opencode_live -- --nocapture
|
|
//! ```
|
|
//!
|
|
//! When `PHOEBUS_OPENCODE_REQUIRE_LIVE=1` is set (as `make opencode-acceptance`
|
|
//! does), a missing configuration is a hard failure, so an acceptance command
|
|
//! cannot pass by silently not running.
|
|
|
|
use std::time::Duration;
|
|
|
|
use phoebus_core::SecretString;
|
|
use phoebus_executor::opencode_client::{CurlHttpClient, OpenCodeClient};
|
|
use phoebus_executor::opencode_server::{probe, ConnectionState, Endpoint, ProbeRequest};
|
|
use phoebus_executor::{Exec, Logger, RealRunner};
|
|
|
|
struct Live {
|
|
url: String,
|
|
provider: String,
|
|
model: String,
|
|
password: Option<SecretString>,
|
|
}
|
|
|
|
fn live() -> Option<Live> {
|
|
let url = std::env::var("PHOEBUS_OPENCODE_LIVE_URL").unwrap_or_default();
|
|
let provider = std::env::var("PHOEBUS_OPENCODE_LIVE_PROVIDER").unwrap_or_default();
|
|
let model = std::env::var("PHOEBUS_OPENCODE_LIVE_MODEL").unwrap_or_default();
|
|
let required = std::env::var("PHOEBUS_OPENCODE_REQUIRE_LIVE")
|
|
.map(|v| matches!(v.as_str(), "1" | "yes" | "true"))
|
|
.unwrap_or(false);
|
|
if url.trim().is_empty() || provider.trim().is_empty() || model.trim().is_empty() {
|
|
if required {
|
|
panic!(
|
|
"BLOCKED: a live OpenCode acceptance run was required, but \
|
|
PHOEBUS_OPENCODE_LIVE_URL/PROVIDER/MODEL are not all set"
|
|
);
|
|
}
|
|
eprintln!(
|
|
"SKIPPED: set PHOEBUS_OPENCODE_LIVE_URL, PHOEBUS_OPENCODE_LIVE_PROVIDER and \
|
|
PHOEBUS_OPENCODE_LIVE_MODEL to run the live acceptance test"
|
|
);
|
|
return None;
|
|
}
|
|
let password = match std::env::var("PHOEBUS_OPENCODE_LIVE_PASSWORD_FILE") {
|
|
Ok(path) if !path.is_empty() => {
|
|
Some(phoebus_executor::ai::load_api_key(std::path::Path::new(&path)).unwrap())
|
|
}
|
|
_ => None,
|
|
};
|
|
Some(Live {
|
|
url,
|
|
provider,
|
|
model,
|
|
password,
|
|
})
|
|
}
|
|
|
|
fn exec() -> Exec {
|
|
let dir = tempfile::tempdir().unwrap().keep();
|
|
let logger = Logger::open(
|
|
dir.join("opencode-live.log"),
|
|
phoebus_core::Redactor::new(),
|
|
false,
|
|
);
|
|
Exec::new(Box::new(RealRunner), logger, false, dir.clone(), dir)
|
|
}
|
|
|
|
#[test]
|
|
fn live_inference_through_opencode() {
|
|
let Some(live) = live() else { return };
|
|
let exec = exec();
|
|
let http = CurlHttpClient::new(&exec);
|
|
let endpoint = Endpoint::parse(&live.url).expect("a valid live endpoint");
|
|
|
|
// 1. The catalog must list the selected model.
|
|
let mut request = ProbeRequest::new();
|
|
request.provider = Some(live.provider.clone());
|
|
request.model = Some(live.model.clone());
|
|
request.auth = live.password.clone();
|
|
let capability = probe(&http, &endpoint, &request);
|
|
eprintln!("live capability: {capability:?}");
|
|
assert!(
|
|
matches!(
|
|
capability.state,
|
|
ConnectionState::ModelCatalog | ConnectionState::SelectedModelUnavailable
|
|
),
|
|
"the live server is not usable: {capability:?}"
|
|
);
|
|
assert!(
|
|
capability.selected_model_available,
|
|
"the selected model {}:{} is not in the catalog: {capability:?}",
|
|
live.provider, live.model
|
|
);
|
|
|
|
// 2. A real inference round-trip must return a valid JSON response.
|
|
let client = OpenCodeClient::new(&http, endpoint.base(), &live.provider, &live.model)
|
|
.with_auth(live.password.clone())
|
|
.with_timeout(Duration::from_secs(120));
|
|
let reply = client
|
|
.one_shot(
|
|
"You are a connectivity test. Reply with JSON only.",
|
|
"Reply with exactly {\"phoebus\":\"ok\"} and nothing else.",
|
|
)
|
|
.unwrap_or_else(|e| panic!("live inference failed: {e}"));
|
|
let value: serde_json::Value =
|
|
serde_json::from_str(&reply.content).expect("the live model returned non-JSON");
|
|
assert_eq!(
|
|
value.get("phoebus").and_then(|v| v.as_str()),
|
|
Some("ok"),
|
|
"the live model did not follow the instruction: {}",
|
|
reply.content
|
|
);
|
|
eprintln!("live inference OK: {}", reply.content);
|
|
}
|
|
|
|
#[test]
|
|
fn live_endpoint_is_reported_truthfully() {
|
|
let Some(live) = live() else { return };
|
|
let exec = exec();
|
|
let http = CurlHttpClient::new(&exec);
|
|
let endpoint = Endpoint::parse(&live.url).unwrap();
|
|
// A deliberate wrong model must be reported as unavailable, not accepted.
|
|
let mut request = ProbeRequest::new();
|
|
request.provider = Some(live.provider.clone());
|
|
request.model = Some("definitely-not-a-real-model-xyz".into());
|
|
request.auth = live.password.clone();
|
|
let capability = probe(&http, &endpoint, &request);
|
|
assert_eq!(
|
|
capability.state,
|
|
ConnectionState::SelectedModelUnavailable,
|
|
"an unknown model was not reported as unavailable: {capability:?}"
|
|
);
|
|
}
|