phoebus_os/installer/tests/vm/sys-acceptance.sh
Phoebus 7ebb616c99
Some checks are pending
ci / test (push) Waiting to run
feat(ai,desktop): unify administration layer, provision QML, and add conversation continuity
Unifies AI-first administration across the CLI (sys) and desktop GUI
(phoebus-assistant) through a shared service layer, fixes installed
desktop QML asset provisioning, adds Polkit elevation, and introduces
contextual conversation continuity.

- Extract phoebus_installer::service shared by CLI and desktop assistant bridge:
  - Reviewable plan staging (PreparedPlan) with SHA-256 cryptographic digests,
    15-minute TTL expiration, and replay prevention (PlanStore).
  - Pre-mutation system state revalidation binding execution strictly to the reviewed plan.
  - Honest recommendations displaying genuine provenance and unmeasured compile times.
  - Two-phase transaction logging (begin before mutation, finish with verification).
- Provision installed desktop assets and Polkit policy in phoebus-executor::desktop:
  - Added gui-apps/quickshell to canonical Mango desktop packages.
  - Embed 13 QML and policy files into target root under /usr/share/phoebus/assistant/.
  - Installed Polkit policy (club.phoebus.sys.policy) allowing unprivileged users to
    authenticate mutating plans via pkexec without blanket passwordless sudo.
- Eliminate startup freeze and polling in phoebus-assistant:
  - Multi-source update queries moved to on-demand tab switching.
  - Handshake timer reduced to one-shot 100ms with fast deterministic check.
- Add phoebus_installer::conversation:
  - Bounded 10-turn history with credential redaction.
  - Contextual follow-up resolution for comparisons, backend switching, and confirmation.
- Implement unified cross-source removal (plan_removal) preserving user data by default.
- Verified: all 486 workspace tests pass, release and crt-static builds succeed,
  and 23/23 VM acceptance cases pass (PASS=23 FAIL=0).
2026-09-24 17:21:16 -07:00

166 lines
7.2 KiB
Bash
Executable file

#!/bin/bash
# Guest-side `sys` acceptance matrix for a booted, installed Phoebus system.
#
# Runs the deterministic phrase path and the typed JSON path, plus failure
# cases, and prints one machine-readable line per case. Read-only by default;
# set SYS_ACCEPTANCE_MUTATE=1 to also exercise install/remove/service/snapshot
# on a disposable VM only. Never run the mutate group on a physical host.
#
# The driver matches `===SYS-ACC-(START|END)===` and `CASE <id> <PASS|FAIL>`.
set -uo pipefail
SYS="${SYS_BIN:-/usr/local/bin/sys}"
[[ -x "$SYS" ]] || SYS=/tmp/sys
MUTATE="${SYS_ACCEPTANCE_MUTATE:-0}"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
PASS=0
FAIL=0
report() {
# report <id> <PASS|FAIL> <detail>
printf 'CASE %s %s %s\n' "$1" "$2" "$3"
if [[ "$2" == PASS ]]; then PASS=$((PASS + 1)); else FAIL=$((FAIL + 1)); fi
}
# run_expect_zero <id> <args...>
run_expect_zero() {
local id="$1"; shift
if "$SYS" "$@" >"$WORK/$id.out" 2>"$WORK/$id.err"; then
report "$id" PASS "$(head -c 200 "$WORK/$id.out" | tr '\n' ' ')"
else
report "$id" FAIL "rc=$? stderr=$(head -c 200 "$WORK/$id.err" | tr '\n' ' ')"
fi
}
# run_expect_nonzero <id> <needle> <args...>
run_expect_nonzero() {
local id="$1" needle="$2"; shift 2
"$SYS" "$@" >"$WORK/$id.out" 2>"$WORK/$id.err"
local rc=$?
if [[ $rc -ne 0 ]] && grep -qi -- "$needle" "$WORK/$id.err" "$WORK/$id.out"; then
report "$id" PASS "rejected: $(head -c 160 "$WORK/$id.err" | tr '\n' ' ')"
else
report "$id" FAIL "rc=$rc expected nonzero and '$needle'"
fi
}
# json_nonzero <id> <needle> <json>
json_nonzero() {
local id="$1" needle="$2" json="$3"
"$SYS" --legacy --action "$json" >"$WORK/$id.out" 2>"$WORK/$id.err"
local rc=$?
if [[ $rc -ne 0 ]] && grep -qi -- "$needle" "$WORK/$id.err" "$WORK/$id.out"; then
report "$id" PASS "rejected: $(head -c 160 "$WORK/$id.err" | tr '\n' ' ')"
else
report "$id" FAIL "rc=$rc expected nonzero and '$needle'"
fi
}
echo "===SYS-ACC-START==="
echo "sys-acceptance: SYS=$SYS MUTATE=$MUTATE"
"$SYS" --version 2>&1 | head -1
# --- Read-only deterministic and JSON paths -------------------------------
run_expect_zero identity --version
run_expect_zero history-legacy history --legacy
run_expect_zero undo-status-legacy undo status --legacy
if [[ "$(id -u)" -ne 0 ]]; then
run_expect_nonzero nonroot-undo "mutating actions require root" undo --legacy
fi
run_expect_zero appimage-empty --root / --legacy appimage list
run_expect_zero ebuild-empty --root / --legacy ebuilds
# --- Failure cases ---------------------------------------------------------
run_expect_nonzero invalid-natural "could not parse" --legacy -- install --color=n
run_expect_nonzero unknown-phrase "unsupported instruction" --legacy frobnicate
run_expect_nonzero action-without-legacy-or-broker "require --legacy" --action '{"action":"history"}'
run_expect_nonzero install-legacy-nosource "requires an explicit package source flag" install firefox --legacy
json_nonzero unknown-action "invalid action" '{"action":"run_shell","command":"rm -rf /"}'
json_nonzero unknown-field "invalid action" '{"action":"history","x":1}'
json_nonzero option-operand "invalid" '{"action":"install_package","atom":"-C"}'
json_nonzero bad-snap-name "invalid" '{"action":"snap_install","name":"Bad Name"}'
json_nonzero bad-appimage-hash "SHA-256" '{"action":"appimage_acquire","name":"x","url":"https://e/x","sha256":"00"}'
json_nonzero ebuild-traversal "invalid" '{"action":"custom_ebuild_prepare","atom":"../../x","version":"1","rationale":"r","ebuild":"EAPI=8"}'
# --- Offline / model-free behaviour ---------------------------------------
if [[ "$(id -u)" -ne 0 ]]; then
run_expect_nonzero nonroot-mutation "root" --legacy --action '{"action":"set_package_preference","name":"firefox","backend":"flatpak"}'
fi
# With no provider, `ask` must fall back deterministically and exit nonzero.
PHOEBUS_AI_PROVIDER="" PHOEBUS_AI_BACKEND=direct \
run_expect_nonzero ask-offline "no AI provider" ask "install vim"
# --- Choosing a source (never invents a candidate) ------------------------
"$SYS" --root / choose unused-package-xyz --legacy >"$WORK/choose.out" 2>"$WORK/choose.err"
if grep -q "Selected package:" "$WORK/choose.out"; then
report choose PASS "selection: $(grep -m1 'Selected package:' "$WORK/choose.out")"
elif grep -qi "no compatible\|no AI provider\|no real candidate" "$WORK/choose.err"; then
report choose PASS "no candidate: $(head -c 160 "$WORK/choose.err" | tr '\n' ' ')"
else
report choose FAIL "unexpected output: $(head -c 200 "$WORK/choose.err")"
fi
# --- AI-first installation workflow (read-only cases) ---------------------
# Conflicting backend flags must be rejected by the CLI.
if ! "$SYS" install firefox --flatpak --nix >"$WORK/conflict.out" 2>"$WORK/conflict.err" && \
grep -qi "cannot be used with\|conflict\|not allowed" "$WORK/conflict.err"; then
report install-flag-conflict PASS "rejected"
else
report install-flag-conflict FAIL "$(head -c 160 "$WORK/conflict.err")"
fi
# A backend flag without an install is refused.
if ! "$SYS" --flatpak history --legacy >"$WORK/flag.out" 2>"$WORK/flag.err" && \
grep -q "only apply to" "$WORK/flag.err"; then
report install-flag-alone PASS "refused"
else
report install-flag-alone FAIL "$(head -c 160 "$WORK/flag.err")"
fi
# An unqualified install with no provider must offer the explicit commands and
# install nothing.
if ! PHOEBUS_AI_PROVIDER="" PHOEBUS_AI_BACKEND=direct \
"$SYS" install unused-package-xyz >"$WORK/noai.out" 2>"$WORK/noai.err" && \
grep -q -- "--flatpak" "$WORK/noai.err"; then
report install-no-model-offer PASS "no model: deterministic offer shown"
else
report install-no-model-offer FAIL "$(head -c 200 "$WORK/noai.err")"
fi
# Read-only investigation returns structured evidence.
"$SYS" --legacy --action '{"action":"check_backend","backend":"flatpak"}' >"$WORK/be.out" 2>"$WORK/be.err"
if grep -q '"backend"' "$WORK/be.out"; then
report investigate-check-backend PASS "$(head -c 120 "$WORK/be.out" | tr '\n' ' ')"
else
report investigate-check-backend FAIL "$(head -c 160 "$WORK/be.err")"
fi
# --- Optional mutating group (disposable VM only) -------------------------
if [[ "$MUTATE" == "1" ]]; then
PKG="${SYS_ACCEPTANCE_PACKAGE:-app-misc/hello}"
if "$SYS" -y install "$PKG" --legacy --portage >"$WORK/install.out" 2>&1; then
if emerge --pretend --quiet "$PKG" >/dev/null 2>&1; then
report mutate-install PASS "installed $PKG"
else
report mutate-install FAIL "install reported ok but system does not know $PKG"
fi
else
report mutate-install FAIL "sys install failed: $(head -c 160 "$WORK/install.out")"
fi
if "$SYS" -y remove "$PKG" --legacy >"$WORK/remove.out" 2>&1; then
report mutate-remove PASS "removed $PKG"
else
report mutate-remove FAIL "sys remove failed: $(head -c 160 "$WORK/remove.out")"
fi
"$SYS" history --legacy >"$WORK/hist.out" 2>&1
if grep -q "$PKG" "$WORK/hist.out"; then
report mutate-history PASS "transaction recorded"
else
report mutate-history FAIL "install/removal missing from history"
fi
fi
echo "===SYS-ACC-END=== SUMMARY PASS=$PASS FAIL=$FAIL"
[[ "$FAIL" -eq 0 ]]