phoebus_os/installer/tests/vm/sys-acceptance.sh
Phoebus 297f3201ff
Some checks are pending
ci / test (push) Waiting to run
feat(sys): hardware-aware lane selection and new lanes
Replace the fixed category routing (ADR 0006) with a shared, inspectable,
hardware- and preference-aware selection policy over resolved candidates from
Portage (source/binary/vendor), Nix, Flatpak, Snap, AppImage and the Phoebus
overlay. The policy keeps base components under Portage, separates hard
constraints from preferences, honours and persists explicit choices, errors
instead of silently switching lanes, and reuses an already-installed lane.

- phoebus-core::selection: the pure policy (16 tests: base-system, low-memory
  vs high-core, sandbox requirement, GPU/arch/storage rejection, explicit
  preference, unavailable backend, reproducibility, render).
- phoebus-executor::chooser: resolves real metadata and hardware facts.
- phoebus-executor::{snap,appimage,ebuild}: new lanes. Snap detects daemon and
  confinement and installs app-containers/snapd through Portage; AppImage
  adopts images with a pinned SHA-256, checks the ELF architecture without
  executing, integrates a desktop entry and preserves user data on removal; the
  custom ebuild overlay binds approval to the ebuild content digest and builds
  through Portage's sandbox.
- Typed actions choose_package / set_package_preference / snap_* / appimage_* /
  custom_ebuild_* with strict validators; the assistant prompt and schema are
  reconciled (no category routing; custom ebuilds are untrusted).
- Installed sys: the installer locates the trusted static binary, copies it to
  /usr/local/bin/sys and proves it runs in the target chroot before the
  recovery snapshot, failing closed if absent; the VM harness fetches sys.
- docs/SYS.md: command/action reference and acceptance matrix; ADR 0006 revised.

Host gates pass: fmt, clippy (no new warnings), workspace tests, make test,
release and static sys builds. VM lane journeys remain pending (see HANDOFF).
2026-09-23 18:35:26 -07:00

125 lines
5.1 KiB
Bash
Executable file

#!/bin/bash
# Guest-side `sys` acceptance matrix for a booted, installed Phoebus system.
#
# Runs the deterministic phrase path and the typed JSON path, plus failure
# cases, and prints one machine-readable line per case. Read-only by default;
# set SYS_ACCEPTANCE_MUTATE=1 to also exercise install/remove/service/snapshot
# on a disposable VM only. Never run the mutate group on a physical host.
#
# The driver matches `===SYS-ACC-(START|END)===` and `CASE <id> <PASS|FAIL>`.
set -uo pipefail
SYS="${SYS_BIN:-/usr/local/bin/sys}"
[[ -x "$SYS" ]] || SYS=/tmp/sys
MUTATE="${SYS_ACCEPTANCE_MUTATE:-0}"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
PASS=0
FAIL=0
report() {
# report <id> <PASS|FAIL> <detail>
printf 'CASE %s %s %s\n' "$1" "$2" "$3"
if [[ "$2" == PASS ]]; then PASS=$((PASS + 1)); else FAIL=$((FAIL + 1)); fi
}
# run_expect_zero <id> <args...>
run_expect_zero() {
local id="$1"; shift
if "$SYS" "$@" >"$WORK/$id.out" 2>"$WORK/$id.err"; then
report "$id" PASS "$(head -c 200 "$WORK/$id.out" | tr '\n' ' ')"
else
report "$id" FAIL "rc=$? stderr=$(head -c 200 "$WORK/$id.err" | tr '\n' ' ')"
fi
}
# run_expect_nonzero <id> <needle> <args...>
run_expect_nonzero() {
local id="$1" needle="$2"; shift 2
"$SYS" "$@" >"$WORK/$id.out" 2>"$WORK/$id.err"
local rc=$?
if [[ $rc -ne 0 ]] && grep -qi -- "$needle" "$WORK/$id.err" "$WORK/$id.out"; then
report "$id" PASS "rejected: $(head -c 160 "$WORK/$id.err" | tr '\n' ' ')"
else
report "$id" FAIL "rc=$rc expected nonzero and '$needle'"
fi
}
# json_nonzero <id> <needle> <json>
json_nonzero() {
local id="$1" needle="$2" json="$3"
"$SYS" --action "$json" >"$WORK/$id.out" 2>"$WORK/$id.err"
local rc=$?
if [[ $rc -ne 0 ]] && grep -qi -- "$needle" "$WORK/$id.err" "$WORK/$id.out"; then
report "$id" PASS "rejected: $(head -c 160 "$WORK/$id.err" | tr '\n' ' ')"
else
report "$id" FAIL "rc=$rc expected nonzero and '$needle'"
fi
}
echo "===SYS-ACC-START==="
echo "sys-acceptance: SYS=$SYS MUTATE=$MUTATE"
"$SYS" --version 2>&1 | head -1
# --- Read-only deterministic and JSON paths -------------------------------
run_expect_zero identity --version
run_expect_zero history history
run_expect_zero appimage-empty --root / appimage list
run_expect_zero ebuild-empty --root / ebuilds
# --- Failure cases ---------------------------------------------------------
run_expect_nonzero invalid-natural "could not parse" -- install --color=n
run_expect_nonzero unknown-phrase "unsupported instruction" frobnicate
json_nonzero unknown-action "invalid action" '{"action":"run_shell","command":"rm -rf /"}'
json_nonzero unknown-field "invalid action" '{"action":"history","x":1}'
json_nonzero option-operand "invalid action" '{"action":"install_package","atom":"-C"}'
json_nonzero bad-snap-name "invalid" '{"action":"snap_install","name":"Bad Name"}'
json_nonzero bad-appimage-hash "SHA-256" '{"action":"appimage_acquire","name":"x","url":"https://e/x","sha256":"00"}'
json_nonzero ebuild-traversal "invalid" '{"action":"custom_ebuild_prepare","atom":"../../x","version":"1","rationale":"r","ebuild":"EAPI=8"}'
# --- Offline / model-free behaviour ---------------------------------------
if [[ "$(id -u)" -ne 0 ]]; then
run_expect_nonzero nonroot-mutation "root" --action '{"action":"set_package_preference","name":"firefox","backend":"flatpak"}'
fi
# With no provider, `ask` must fall back deterministically and exit nonzero.
PHOEBUS_AI_PROVIDER= PHOEBUS_AI_BACKEND=direct \
run_expect_nonzero ask-offline "no AI provider" ask "install vim"
# --- Choosing a source (never invents a candidate) ------------------------
"$SYS" --root / choose unused-package-xyz >"$WORK/choose.out" 2>"$WORK/choose.err"
if grep -q "Selected package:" "$WORK/choose.out"; then
report choose PASS "selection: $(grep -m1 'Selected package:' "$WORK/choose.out")"
elif grep -qi "no compatible\|no AI provider" "$WORK/choose.err"; then
report choose PASS "no candidate: $(head -c 160 "$WORK/choose.err" | tr '\n' ' ')"
else
report choose FAIL "unexpected output: $(head -c 200 "$WORK/choose.err")"
fi
# --- Optional mutating group (disposable VM only) -------------------------
if [[ "$MUTATE" == "1" ]]; then
PKG="${SYS_ACCEPTANCE_PACKAGE:-app-misc/hello}"
if "$SYS" -y install "$PKG" >"$WORK/install.out" 2>&1; then
if emerge --pretend --quiet "$PKG" >/dev/null 2>&1; then
report mutate-install PASS "installed $PKG"
else
report mutate-install FAIL "install reported ok but system does not know $PKG"
fi
else
report mutate-install FAIL "sys install failed: $(head -c 160 "$WORK/install.out")"
fi
if "$SYS" -y remove "$PKG" >"$WORK/remove.out" 2>&1; then
report mutate-remove PASS "removed $PKG"
else
report mutate-remove FAIL "sys remove failed: $(head -c 160 "$WORK/remove.out")"
fi
"$SYS" history >"$WORK/hist.out" 2>&1
if grep -q "$PKG" "$WORK/hist.out"; then
report mutate-history PASS "transaction recorded"
else
report mutate-history FAIL "install/removal missing from history"
fi
fi
echo "===SYS-ACC-END=== SUMMARY PASS=$PASS FAIL=$FAIL"
[[ "$FAIL" -eq 0 ]]