From d4617211527b3059c8ecc444c6de6c9532405b82 Mon Sep 17 00:00:00 2001 From: Phoebus Date: Fri, 2 Oct 2026 09:12:31 -0700 Subject: [PATCH] feat(sys): discover external Portage overlay leads --- docs/CHANGELOG.md | 7 + docs/HANDOFF.md | 12 ++ docs/STATUS.md | 11 ++ docs/SYS.md | 10 ++ .../crates/phoebus-executor/src/query.rs | 153 +++++++++++++++++- .../crates/phoebus-installer/src/bin/sys.rs | 66 ++++++++ .../crates/phoebus-installer/tests/sys_cli.rs | 26 +++ 7 files changed, 284 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e3e537b..fd9659f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,13 @@ this file is the curated, human-readable summary. ## Unreleased — Fast defaults with user-controlled source builds +- **External Portage overlay discovery (2026-10-02).** `sys search + --legacy --overlays` adds read-only GPO leads to the configured package + catalogue. Leads are visibly unverified and never become install candidates + merely because a remote index lists them. The network lookup is opt-in and + bounded; the existing offline search remains local. Host and running-VM + lookups found `app-misc/chatgpt-desktop` for `chatgpt desktop`. + - **Natural install lane parsing (2026-10-02).** `sys install chatgpt desktop through nix` now separates the Nix lane from the descriptive application name. If an AI plan fails, `sys` searches that lane and suggests a verified diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 1a39e61..41069ee 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -3,6 +3,18 @@ Rolling note between work sessions. **Update this file before you finish**, then commit it. It should be short and current; use `git log` for detail. +- **External overlay leads (2026-10-02):** GPO lists + `app-misc/chatgpt-desktop` in Gentoo community overlays, but `sys search` + previously queried only Portage repositories enabled locally. The new + `sys search chatgpt --legacy --overlays --portage` keeps those local results + and adds a bounded read-only GPO lookup with clearly unverified leads. A + host lookup returned the ChatGPT desktop ebuild. The updated static binary + is installed at `~/.local/bin/sys` in the running playground; a VM query + displayed the external lead beneath an empty configured-Portage result. + No overlay is enabled by this search, and the ebuild is not treated as an + install candidate until a reviewed overlay is configured and Portage + resolves it. Host checks, release and static builds passed. + - **`sys` natural install error (2026-10-02):** The phrase `sys install chatgpt desktop through nix` used to pass the entire phrase as a package name and print invalid legacy examples. The parser now extracts the requested lane; diff --git a/docs/STATUS.md b/docs/STATUS.md index 682cd32..d469430 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -48,6 +48,17 @@ disposable VM; `serial-driver.py recovery` now automates the same check. ## Matrix +External overlay discovery (2026-10-02): local Portage search now labels its +scope as configured repositories. `sys search --legacy --overlays` +adds a bounded, opt-in read-only lookup of GPO and labels its results as +unverified external leads. No repository is enabled or package installed by +search. Parser, offline failure, CLI guard, and live GPO lookup checks pass; +the live query found `app-misc/chatgpt-desktop` for `chatgpt`. The updated +static binary was installed as the running VM user's `~/.local/bin/sys` and +`sys search chatgpt desktop --legacy --overlays --portage` displayed the +external lead after an empty configured-Portage result. No overlay or package +was installed. + Natural install lane parsing (2026-10-02): `sys` recognizes trailing `through/via/using/with ` as a backend constraint. A model failure for a descriptive package name searches the chosen lane and offers an exact diff --git a/docs/SYS.md b/docs/SYS.md index ccc64c8..87c4601 100644 --- a/docs/SYS.md +++ b/docs/SYS.md @@ -333,6 +333,7 @@ separate updater widget. ```sh sys search firefox --legacy # every source sys search firefox --legacy --flatpak # one source +sys search chatgpt --legacy --overlays --portage # configured Portage + external GPO leads sys list --legacy # installed inventory, all lanes sys list vim --legacy --portage # inventory, source + text filter sys info firefox --legacy # version/source/status/deps/size @@ -345,6 +346,15 @@ sys deps vim --legacy # forward dependencies sys rdeps vim --legacy # reverse dependencies (equery) ``` +Portage searches cover repositories configured on this machine. The optional +`--overlays` flag sends the search term to the unofficial GPO overlay index and +lists matching ebuild names as **external leads** below the local catalogue. +Those leads are not validated install candidates: review and sync a repository +first, and accept `~amd64` only for a specific package if needed. The external +lookup has an eight-second limit; a network failure leaves local results +available. `--overlays` is restricted to deterministic `search --legacy` and +cannot be combined with a non-Portage source flag. + During a terminal mutation, `sys` shows the package manager's output as it arrives: Portage build and merge logs, Flatpak transaction progress, Snap operations, Nix realisation, custom ebuild builds, and AppImage downloads. diff --git a/installer-rs/crates/phoebus-executor/src/query.rs b/installer-rs/crates/phoebus-executor/src/query.rs index 62e82f0..904fccb 100644 --- a/installer-rs/crates/phoebus-executor/src/query.rs +++ b/installer-rs/crates/phoebus-executor/src/query.rs @@ -155,7 +155,11 @@ fn search_portage(exec: &Exec, root: &Path, query: &str, backend: Backend) -> So entry }) .collect(); - SourceReport::ok(backend, "Gentoo tree (emerge --search)", entries) + SourceReport::ok( + backend, + "configured Portage repositories (emerge --search)", + entries, + ) } Ok(out) => SourceReport::failed( backend, @@ -171,6 +175,119 @@ fn search_portage(exec: &Exec, root: &Path, query: &str, backend: Backend) -> So } } +/// An external overlay index result is a lead for review, never an install +/// candidate. GPO is an unofficial index and the named repository may not be +/// enabled, synced, trusted, or compatible with this system. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExternalOverlayLead { + pub atom: String, +} + +fn encode_query(query: &str) -> String { + let mut encoded = String::new(); + for byte in query.bytes() { + if byte.is_ascii_alphanumeric() || b"-._~".contains(&byte) { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +fn valid_gpo_atom(atom: &str) -> bool { + let Some((category, package)) = atom.split_once('/') else { + return false; + }; + !category.is_empty() + && !package.is_empty() + && !package.contains('/') + && [category, package].iter().all(|part| { + part.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"+_.-".contains(&byte) + }) + }) +} + +fn parse_gpo_search(html: &str, query: &str) -> Vec { + let terms: Vec = query + .split_whitespace() + .map(str::to_ascii_lowercase) + .collect(); + let mut atoms: Vec = html + .split("href=\"/") + .skip(1) + .filter_map(|fragment| fragment.split_once('"').map(|(path, _)| path)) + .filter(|atom| valid_gpo_atom(atom)) + .filter(|atom| { + terms + .iter() + .all(|term| atom.to_ascii_lowercase().contains(term)) + }) + .map(str::to_string) + .collect(); + let normalized_query = terms.join("-"); + atoms.sort_by_key(|atom| { + let package = atom.rsplit('/').next().unwrap_or(atom); + let rank = if package == normalized_query { + 0 + } else if package == format!("{normalized_query}-desktop") { + 1 + } else if package.starts_with(&format!("{normalized_query}-")) { + 2 + } else { + 3 + }; + (rank, atom.len(), atom.clone()) + }); + atoms.dedup(); + atoms.truncate(12); + atoms + .into_iter() + .map(|atom| ExternalOverlayLead { atom }) + .collect() +} + +/// Opt-in, read-only discovery of packages indexed on GPO. Results never enter +/// the verified local catalogue or the installation candidate set. +pub fn search_external_overlays( + exec: &Exec, + root: &Path, + query: &str, +) -> std::result::Result, String> { + if !phoebus_core::action::valid_search_query(query) { + return Err("invalid overlay search query".into()); + } + let url = format!( + "https://gpo.zugaina.org/Search?search={}", + encode_query(query) + ); + let out = capture( + exec, + root, + "curl", + &[ + "--fail", + "--silent", + "--show-error", + "--proto", + "=https", + "--connect-timeout", + "3", + "--max-time", + "8", + "--max-filesize", + "524288", + &url, + ], + ) + .map_err(|_| "GPO lookup unavailable (curl or network error)".to_string())?; + if !out.success { + return Err("GPO lookup unavailable (service or network error)".into()); + } + Ok(parse_gpo_search(&out.stdout_lossy(), query)) +} + fn parse_flatpak_lines(text: &str) -> Vec> { text.lines() .filter(|l| !l.trim().is_empty()) @@ -1665,6 +1782,40 @@ mod tests { assert_eq!(source.entries[0].version.as_deref(), Some("128.0")); } + #[test] + fn external_overlay_search_reports_leads_without_install_candidates() { + let runner = RecordingRunner::new(); + runner.set_response( + "curl", + ok(r#"nav +
app-misc/chatgpt-desktop +
Desktop application for ChatGPT and Codex
+duplicate +unsafe"#), + ); + let exec = make_exec(runner.clone()); + let leads = search_external_overlays(&exec, Path::new("/"), "chatgpt desktop") + .expect("external search response"); + assert_eq!(leads.len(), 1); + assert_eq!(leads[0].atom, "app-misc/chatgpt-desktop"); + let calls = runner.recorded(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].program, "curl"); + assert!(calls[0].args.iter().any(|arg| arg == "--max-time")); + assert!(calls[0] + .args + .iter() + .any(|arg| arg.ends_with("Search?search=chatgpt%20desktop"))); + } + + #[test] + fn external_overlay_lookup_fails_closed_when_unavailable() { + let runner = RecordingRunner::new(); + runner.fail("curl", "offline"); + let exec = make_exec(runner); + assert!(search_external_overlays(&exec, Path::new("/"), "chatgpt").is_err()); + } + #[test] fn a_failed_search_is_not_reported_as_empty() { let runner = RecordingRunner::new(); diff --git a/installer-rs/crates/phoebus-installer/src/bin/sys.rs b/installer-rs/crates/phoebus-installer/src/bin/sys.rs index 4af8ec6..5c521a9 100644 --- a/installer-rs/crates/phoebus-installer/src/bin/sys.rs +++ b/installer-rs/crates/phoebus-installer/src/bin/sys.rs @@ -38,6 +38,7 @@ mod progress; administration that never contacts a model.\n\n\ Examples (AI default -> deterministic equivalent):\n \ sys search chrome -> sys search chrome --legacy\n \ +sys search chatgpt --legacy --overlays -> include external overlay leads\n \ sys list -> sys list --legacy\n \ sys info firefox -> sys info firefox --legacy\n \ sys updates -> sys updates --legacy\n \ @@ -61,6 +62,12 @@ struct Cli { #[arg(long)] legacy: bool, + /// For `search --legacy`, also query GPO for external Portage overlay leads. + /// Sends the search terms to gpo.zugaina.org; results are not installable + /// until a reviewed repository is enabled and synced. + #[arg(long)] + overlays: bool, + /// Internal broker invocation by Phoebus OS application layer. #[arg(long)] broker: bool, @@ -248,6 +255,16 @@ fn main() -> ExitCode { fn run() -> std::result::Result<(), Box> { let cli = Cli::parse(); + if cli.overlays + && (cli.broker + || cli.assistant_bridge + || cli.action.is_some() + || cli.action_file.is_some() + || cli.execute_plan.is_some() + || cli.propose) + { + return Err("--overlays applies only to an interactive `sys search --legacy`".into()); + } // Broker stdout is a JSON protocol. The assistant bridge is also a stdio // protocol, so neither entry point gets human progress messages. @@ -371,6 +388,9 @@ fn run() -> std::result::Result<(), Box> { // 4. Explicit legacy mode (--legacy) if cli.legacy { + if cli.overlays { + return run_overlay_search(&cli, &exec, &full_phrase, backend); + } if let Some((application, source)) = parse_install_text(&full_phrase) { let backend = combine_install_backend(backend, source)?; if let Some(b) = backend { @@ -473,6 +493,10 @@ fn run() -> std::result::Result<(), Box> { }; } + if cli.overlays { + return Err("--overlays requires `sys search --legacy --overlays`".into()); + } + // 5. Default AI mode (natural language) if cli.text.is_empty() { eprintln!("sys: no instruction provided."); @@ -514,6 +538,48 @@ fn run() -> std::result::Result<(), Box> { run_ask(&cli, &exec, &plans, &full_phrase, backend) } +fn run_overlay_search( + cli: &Cli, + exec: &Exec, + instruction: &str, + backend: Option, +) -> std::result::Result<(), Box> { + let Some(SystemAction::SearchPackages { query, .. }) = parse_natural(instruction) else { + return Err("--overlays applies only to `sys search --legacy`".into()); + }; + let action = SystemAction::SearchPackages { + query: query.clone(), + backend, + }; + action.validate()?; + if backend.is_some_and(|b| !b.is_portage()) { + return Err("--overlays can only be combined with --portage".into()); + } + + let phase = exec.progress_phase("Searching configured package sources"); + let local = query::search(exec, &cli.root, &query, backend); + phase.complete(); + println!("{}", local.render()); + + let phase = exec.progress_phase("Checking the external Portage overlay index"); + let external = query::search_external_overlays(exec, &cli.root, &query); + phase.complete(); + match external { + Ok(leads) if leads.is_empty() => { + println!("External Portage overlays: no matching leads on GPO.") + } + Ok(leads) => { + println!("External Portage overlay leads (not enabled or verified locally):"); + for lead in leads { + println!(" {} — https://gpo.zugaina.org/{}", lead.atom, lead.atom); + } + println!("Review and sync an overlay before installing any of these packages."); + } + Err(reason) => println!("External Portage overlays: {reason}."), + } + Ok(()) +} + /// Overlay an explicit source flag onto a deterministic legacy action. Only /// commands whose source can be selected accept a flag; anything else is /// refused, so a flag can never silently change an unrelated command. diff --git a/installer-rs/crates/phoebus-installer/tests/sys_cli.rs b/installer-rs/crates/phoebus-installer/tests/sys_cli.rs index cadf505..f54923d 100644 --- a/installer-rs/crates/phoebus-installer/tests/sys_cli.rs +++ b/installer-rs/crates/phoebus-installer/tests/sys_cli.rs @@ -430,6 +430,32 @@ fn natural_install_lane_is_not_part_of_the_package_name() { ); } +#[test] +fn external_overlay_search_requires_explicit_legacy_search() { + let root = tempfile::tempdir().expect("temp root"); + for args in [ + vec!["search", "chatgpt", "--overlays"], + vec!["install", "chatgpt", "--legacy", "--overlays"], + vec!["search", "chatgpt", "--legacy", "--overlays", "--nix"], + vec![ + "--legacy", + "--action", + r#"{"action":"history"}"#, + "--overlays", + ], + ] { + let out = sys() + .arg("--root") + .arg(root.path()) + .args(&args) + .output() + .expect("run sys"); + assert!(!out.status.success(), "{args:?}"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(stderr.contains("--overlays"), "{args:?}: {stderr}"); + } +} + #[test] fn an_explicit_backend_never_falls_back_to_another_lane() { if effective_uid_is_root() {