phoebus_os/installer-rs/crates/phoebus-executor/tests/opencode_live.rs
Phoebus 5c3674e3c2 test(opencode): real-HTTP and live inference tests
Integration tests drive the curl client against a local TCP server
(status preservation, chunked multibyte bodies, oversized bodies,
redirects, abrupt disconnects, authentication). The live tests obtain a
real model response when PHOEBUS_OPENCODE_LIVE_* is set and fail as
BLOCKED when a live run is required but not configured.
2026-09-22 02:36:28 -07:00

140 lines
5.1 KiB
Rust

//! Live OpenCode acceptance test.
//!
//! This is the only test that proves **real model inference** through OpenCode.
//! It is skipped unless the environment names a server, provider and model:
//!
//! ```sh
//! PHOEBUS_OPENCODE_LIVE_URL=http://127.0.0.1:4096 \
//! PHOEBUS_OPENCODE_LIVE_PROVIDER=deepseek \
//! PHOEBUS_OPENCODE_LIVE_MODEL=deepseek-chat \
//! cargo test -p phoebus-executor --test opencode_live -- --nocapture
//! ```
//!
//! When `PHOEBUS_OPENCODE_REQUIRE_LIVE=1` is set (as `make opencode-acceptance`
//! does), a missing configuration is a hard failure, so an acceptance command
//! cannot pass by silently not running.
use std::time::Duration;
use phoebus_core::SecretString;
use phoebus_executor::opencode_client::{CurlHttpClient, OpenCodeClient};
use phoebus_executor::opencode_server::{probe, ConnectionState, Endpoint, ProbeRequest};
use phoebus_executor::{Exec, Logger, RealRunner};
struct Live {
url: String,
provider: String,
model: String,
password: Option<SecretString>,
}
fn live() -> Option<Live> {
let url = std::env::var("PHOEBUS_OPENCODE_LIVE_URL").unwrap_or_default();
let provider = std::env::var("PHOEBUS_OPENCODE_LIVE_PROVIDER").unwrap_or_default();
let model = std::env::var("PHOEBUS_OPENCODE_LIVE_MODEL").unwrap_or_default();
let required = std::env::var("PHOEBUS_OPENCODE_REQUIRE_LIVE")
.map(|v| matches!(v.as_str(), "1" | "yes" | "true"))
.unwrap_or(false);
if url.trim().is_empty() || provider.trim().is_empty() || model.trim().is_empty() {
if required {
panic!(
"BLOCKED: a live OpenCode acceptance run was required, but \
PHOEBUS_OPENCODE_LIVE_URL/PROVIDER/MODEL are not all set"
);
}
eprintln!(
"SKIPPED: set PHOEBUS_OPENCODE_LIVE_URL, PHOEBUS_OPENCODE_LIVE_PROVIDER and \
PHOEBUS_OPENCODE_LIVE_MODEL to run the live acceptance test"
);
return None;
}
let password = match std::env::var("PHOEBUS_OPENCODE_LIVE_PASSWORD_FILE") {
Ok(path) if !path.is_empty() => {
Some(phoebus_executor::ai::load_api_key(std::path::Path::new(&path)).unwrap())
}
_ => None,
};
Some(Live {
url,
provider,
model,
password,
})
}
fn exec() -> Exec {
let dir = tempfile::tempdir().unwrap().keep();
let logger = Logger::open(
dir.join("opencode-live.log"),
phoebus_core::Redactor::new(),
false,
);
Exec::new(Box::new(RealRunner), logger, false, dir.clone(), dir)
}
#[test]
fn live_inference_through_opencode() {
let Some(live) = live() else { return };
let exec = exec();
let http = CurlHttpClient::new(&exec);
let endpoint = Endpoint::parse(&live.url).expect("a valid live endpoint");
// 1. The catalog must list the selected model.
let mut request = ProbeRequest::new();
request.provider = Some(live.provider.clone());
request.model = Some(live.model.clone());
request.auth = live.password.clone();
let capability = probe(&http, &endpoint, &request);
eprintln!("live capability: {capability:?}");
assert!(
matches!(
capability.state,
ConnectionState::ModelCatalog | ConnectionState::SelectedModelUnavailable
),
"the live server is not usable: {capability:?}"
);
assert!(
capability.selected_model_available,
"the selected model {}:{} is not in the catalog: {capability:?}",
live.provider, live.model
);
// 2. A real inference round-trip must return a valid JSON response.
let client = OpenCodeClient::new(&http, endpoint.base(), &live.provider, &live.model)
.with_auth(live.password.clone())
.with_timeout(Duration::from_secs(120));
let reply = client
.one_shot(
"You are a connectivity test. Reply with JSON only.",
"Reply with exactly {\"phoebus\":\"ok\"} and nothing else.",
)
.unwrap_or_else(|e| panic!("live inference failed: {e}"));
let value: serde_json::Value =
serde_json::from_str(&reply.content).expect("the live model returned non-JSON");
assert_eq!(
value.get("phoebus").and_then(|v| v.as_str()),
Some("ok"),
"the live model did not follow the instruction: {}",
reply.content
);
eprintln!("live inference OK: {}", reply.content);
}
#[test]
fn live_endpoint_is_reported_truthfully() {
let Some(live) = live() else { return };
let exec = exec();
let http = CurlHttpClient::new(&exec);
let endpoint = Endpoint::parse(&live.url).unwrap();
// A deliberate wrong model must be reported as unavailable, not accepted.
let mut request = ProbeRequest::new();
request.provider = Some(live.provider.clone());
request.model = Some("definitely-not-a-real-model-xyz".into());
request.auth = live.password.clone();
let capability = probe(&http, &endpoint, &request);
assert_eq!(
capability.state,
ConnectionState::SelectedModelUnavailable,
"an unknown model was not reported as unavailable: {capability:?}"
);
}