290 lines
14 KiB
Bash
Executable file
290 lines
14 KiB
Bash
Executable file
#!/bin/bash
|
|
# Guest-side `sys` acceptance matrix for a booted, installed Phoebus system.
|
|
#
|
|
# Runs the deterministic phrase path and the typed JSON path, plus failure
|
|
# cases, and prints one machine-readable line per case. Read-only by default;
|
|
# set SYS_ACCEPTANCE_MUTATE=1 to also exercise install/remove/service/snapshot
|
|
# on a disposable VM only. Never run the mutate group on a physical host.
|
|
#
|
|
# The driver matches `===SYS-ACC-(START|END)===` and `CASE <id> <PASS|FAIL>`.
|
|
set -uo pipefail
|
|
|
|
SYS="${SYS_BIN:-/usr/local/bin/sys}"
|
|
[[ -x "$SYS" ]] || SYS=/tmp/sys
|
|
MUTATE="${SYS_ACCEPTANCE_MUTATE:-0}"
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
report() {
|
|
# report <id> <PASS|FAIL> <detail>
|
|
printf 'CASE %s %s %s\n' "$1" "$2" "$3"
|
|
if [[ "$2" == PASS ]]; then PASS=$((PASS + 1)); else FAIL=$((FAIL + 1)); fi
|
|
}
|
|
|
|
# run_expect_zero <id> <args...>
|
|
run_expect_zero() {
|
|
local id="$1"; shift
|
|
if "$SYS" "$@" >"$WORK/$id.out" 2>"$WORK/$id.err"; then
|
|
report "$id" PASS "$(head -c 200 "$WORK/$id.out" | tr '\n' ' ')"
|
|
else
|
|
report "$id" FAIL "rc=$? stderr=$(head -c 200 "$WORK/$id.err" | tr '\n' ' ')"
|
|
fi
|
|
}
|
|
|
|
# run_expect_nonzero <id> <needle> <args...>
|
|
run_expect_nonzero() {
|
|
local id="$1" needle="$2"; shift 2
|
|
"$SYS" "$@" >"$WORK/$id.out" 2>"$WORK/$id.err"
|
|
local rc=$?
|
|
if [[ $rc -ne 0 ]] && grep -qi -- "$needle" "$WORK/$id.err" "$WORK/$id.out"; then
|
|
report "$id" PASS "rejected: $(head -c 160 "$WORK/$id.err" | tr '\n' ' ')"
|
|
else
|
|
report "$id" FAIL "rc=$rc expected nonzero and '$needle'"
|
|
fi
|
|
}
|
|
|
|
# run_expect_zero_contains <id> <needle> <args...>
|
|
run_expect_zero_contains() {
|
|
local id="$1" needle="$2"; shift 2
|
|
if "$SYS" "$@" >"$WORK/$id.out" 2>"$WORK/$id.err" && grep -q -- "$needle" "$WORK/$id.out"; then
|
|
report "$id" PASS "$(head -c 160 "$WORK/$id.out" | tr '\n' ' ')"
|
|
else
|
|
report "$id" FAIL "rc=$? expected 0 and '$needle' in stdout"
|
|
fi
|
|
}
|
|
|
|
# json_nonzero <id> <needle> <json>
|
|
json_nonzero() {
|
|
local id="$1" needle="$2" json="$3"
|
|
"$SYS" --legacy --action "$json" >"$WORK/$id.out" 2>"$WORK/$id.err"
|
|
local rc=$?
|
|
if [[ $rc -ne 0 ]] && grep -qi -- "$needle" "$WORK/$id.err" "$WORK/$id.out"; then
|
|
report "$id" PASS "rejected: $(head -c 160 "$WORK/$id.err" | tr '\n' ' ')"
|
|
else
|
|
report "$id" FAIL "rc=$rc expected nonzero and '$needle'"
|
|
fi
|
|
}
|
|
|
|
echo "===SYS-ACC-START==="
|
|
echo "sys-acceptance: SYS=$SYS MUTATE=$MUTATE"
|
|
"$SYS" --version 2>&1 | head -1
|
|
|
|
# --- Read-only deterministic and JSON paths -------------------------------
|
|
run_expect_zero identity --version
|
|
run_expect_zero history-legacy history --legacy
|
|
run_expect_zero undo-status-legacy undo status --legacy
|
|
if [[ "$(id -u)" -ne 0 ]]; then
|
|
run_expect_nonzero nonroot-undo "mutating actions require root" undo --legacy
|
|
fi
|
|
run_expect_zero appimage-empty --root / --legacy appimage list
|
|
run_expect_zero ebuild-empty --root / --legacy ebuilds
|
|
|
|
# --- Failure cases ---------------------------------------------------------
|
|
run_expect_nonzero invalid-natural "could not parse" --legacy -- install --color=n
|
|
run_expect_nonzero unknown-phrase "unsupported instruction" --legacy frobnicate
|
|
run_expect_nonzero action-without-legacy-or-broker "require --legacy" --action '{"action":"history"}'
|
|
run_expect_nonzero install-legacy-nosource "requires an explicit package source flag" install firefox --legacy
|
|
json_nonzero unknown-action "invalid action" '{"action":"run_shell","command":"rm -rf /"}'
|
|
json_nonzero unknown-field "invalid action" '{"action":"history","x":1}'
|
|
json_nonzero option-operand "invalid" '{"action":"install_package","atom":"-C"}'
|
|
json_nonzero bad-snap-name "invalid" '{"action":"snap_install","name":"Bad Name"}'
|
|
json_nonzero bad-appimage-hash "SHA-256" '{"action":"appimage_acquire","name":"x","url":"https://e/x","sha256":"00"}'
|
|
json_nonzero ebuild-traversal "invalid" '{"action":"custom_ebuild_prepare","atom":"../../x","version":"1","rationale":"r","ebuild":"EAPI=8"}'
|
|
|
|
# --- Offline / model-free behaviour ---------------------------------------
|
|
# A system-wide policy change is privileged; a personal preference is not.
|
|
if [[ "$(id -u)" -ne 0 ]]; then
|
|
run_expect_nonzero nonroot-systemwide "root" --legacy --action '{"action":"set_package_preference","name":"firefox","backend":"flatpak","system_wide":true}'
|
|
fi
|
|
# A normal account can save a personal preference to its own storage.
|
|
PersonalDir="$WORK/personal"
|
|
mkdir -p "$PersonalDir"
|
|
if PHOEBUS_PERSONAL_DIR="$PersonalDir" "$SYS" --legacy --action '{"action":"set_package_preference","name":"firefox","backend":"flatpak"}' >"$WORK/personal-pref.out" 2>"$WORK/personal-pref.err"; then
|
|
if [[ -f "$PersonalDir/.config/phoebus/package-preferences.toml" ]]; then
|
|
report personal-preference PASS "saved to user config"
|
|
else
|
|
report personal-preference FAIL "reported success without writing user config"
|
|
fi
|
|
else
|
|
report personal-preference FAIL "rc=$? stderr=$(head -c 160 "$WORK/personal-pref.err" | tr '\n' ' ')"
|
|
fi
|
|
# With no provider, `ask` must fall back deterministically and exit nonzero.
|
|
PHOEBUS_AI_PROVIDER="" PHOEBUS_AI_BACKEND=direct \
|
|
run_expect_nonzero ask-offline "no AI provider" ask "install vim"
|
|
|
|
# --- Choosing a source (never invents a candidate) ------------------------
|
|
"$SYS" --root / choose unused-package-xyz --legacy >"$WORK/choose.out" 2>"$WORK/choose.err"
|
|
if grep -q "Selected package:" "$WORK/choose.out"; then
|
|
report choose PASS "selection: $(grep -m1 'Selected package:' "$WORK/choose.out")"
|
|
elif grep -qi "no compatible\|no AI provider\|no real candidate" "$WORK/choose.err"; then
|
|
report choose PASS "no candidate: $(head -c 160 "$WORK/choose.err" | tr '\n' ' ')"
|
|
else
|
|
report choose FAIL "unexpected output: $(head -c 200 "$WORK/choose.err")"
|
|
fi
|
|
|
|
# --- AI-first installation workflow (read-only cases) ---------------------
|
|
# Conflicting backend flags must be rejected by the CLI.
|
|
if ! "$SYS" install firefox --flatpak --nix >"$WORK/conflict.out" 2>"$WORK/conflict.err" && \
|
|
grep -qi "cannot be used with\|conflict\|not allowed" "$WORK/conflict.err"; then
|
|
report install-flag-conflict PASS "rejected"
|
|
else
|
|
report install-flag-conflict FAIL "$(head -c 160 "$WORK/conflict.err")"
|
|
fi
|
|
|
|
# A backend flag without an install is refused.
|
|
if ! "$SYS" --flatpak history --legacy >"$WORK/flag.out" 2>"$WORK/flag.err" && \
|
|
grep -q "only apply to" "$WORK/flag.err"; then
|
|
report install-flag-alone PASS "refused"
|
|
else
|
|
report install-flag-alone FAIL "$(head -c 160 "$WORK/flag.err")"
|
|
fi
|
|
|
|
# An unqualified install with no provider must offer the explicit commands and
|
|
# install nothing.
|
|
if ! PHOEBUS_AI_PROVIDER="" PHOEBUS_AI_BACKEND=direct \
|
|
"$SYS" install unused-package-xyz >"$WORK/noai.out" 2>"$WORK/noai.err" && \
|
|
grep -q -- "--flatpak" "$WORK/noai.err"; then
|
|
report install-no-model-offer PASS "no model: deterministic offer shown"
|
|
else
|
|
report install-no-model-offer FAIL "$(head -c 200 "$WORK/noai.err")"
|
|
fi
|
|
|
|
# Read-only investigation returns structured evidence.
|
|
"$SYS" --legacy --action '{"action":"check_backend","backend":"flatpak"}' >"$WORK/be.out" 2>"$WORK/be.err"
|
|
if grep -q '"backend"' "$WORK/be.out"; then
|
|
report investigate-check-backend PASS "$(head -c 120 "$WORK/be.out" | tr '\n' ' ')"
|
|
else
|
|
report investigate-check-backend FAIL "$(head -c 160 "$WORK/be.err")"
|
|
fi
|
|
|
|
# --- Package catalogue (read-only, --legacy) ------------------------------
|
|
run_expect_zero_contains search-legacy "Package catalogue" --legacy search firefox
|
|
run_expect_zero_contains list-legacy "Package catalogue" --legacy list
|
|
run_expect_zero_contains info-legacy "Package information" --legacy info bash
|
|
run_expect_zero_contains updates-legacy "Unified Update Overview" --legacy updates
|
|
run_expect_zero_contains json-search-legacy "Package catalogue" --legacy --action '{"action":"search_packages","query":"firefox"}'
|
|
|
|
# Regression: an alternate root must never be answered by the host's Snap lane.
|
|
AltRoot="$WORK/altroot"
|
|
mkdir -p "$AltRoot"
|
|
run_expect_zero_contains snap-alternate-root "alternate root" \
|
|
--root "$AltRoot" --legacy search firefox --snap
|
|
# Regression: Nix inventory enumerates active profiles, never host commands.
|
|
run_expect_zero_contains list-nix-legacy "[nix]" --legacy list --nix
|
|
# Regression: dependency inspection uses declared metadata, not a merge plan.
|
|
"$SYS" --legacy --action \
|
|
'{"action":"package_dependencies","package":"bash","reverse":false}' \
|
|
>"$WORK/deps.out" 2>"$WORK/deps.err"
|
|
if grep -qE "declares|dependency token|has no declared dependencies" "$WORK/deps.out"; then
|
|
report deps-legacy PASS "$(head -c 160 "$WORK/deps.out" | tr '\n' ' ')"
|
|
elif grep -qi "app-portage/portage-utils\|qdepends\|not installed" "$WORK/deps.err" "$WORK/deps.out"; then
|
|
report deps-legacy PASS "honest limitation: $(head -c 120 "$WORK/deps.err" | tr '\n' ' ')"
|
|
else
|
|
report deps-legacy FAIL "rc=$? $(head -c 200 "$WORK/deps.err" | tr '\n' ' ')"
|
|
fi
|
|
# Regression: reverse dependencies of an installed library must include its
|
|
# dependents (a self-reference bug previously reported none).
|
|
"$SYS" --legacy --action \
|
|
'{"action":"package_dependencies","package":"sys-libs/glibc","reverse":true}' \
|
|
>"$WORK/rdeps.out" 2>"$WORK/rdeps.err"
|
|
if grep -q "installed packages depending on sys-libs/glibc" "$WORK/rdeps.out"; then
|
|
report rdeps-legacy PASS "$(head -c 140 "$WORK/rdeps.out" | tr '\n' ' ')"
|
|
else
|
|
report rdeps-legacy FAIL "$(head -c 200 "$WORK/rdeps.out$WORK/rdeps.err" | tr '\n' ' ')"
|
|
fi
|
|
|
|
# A recognizable catalogue command without --legacy must enter the AI workflow
|
|
# and must not run deterministically when no model is available.
|
|
if ! PHOEBUS_AI_PROVIDER="" PHOEBUS_AI_BACKEND=direct \
|
|
"$SYS" search firefox >"$WORK/srch-noai.out" 2>"$WORK/srch-noai.err" && \
|
|
grep -q "sys search firefox --legacy" "$WORK/srch-noai.err" && \
|
|
! grep -q "Package catalogue" "$WORK/srch-noai.out"; then
|
|
report search-no-model-offer PASS "offered the exact legacy alternative"
|
|
else
|
|
report search-no-model-offer FAIL "$(head -c 200 "$WORK/srch-noai.err")"
|
|
fi
|
|
|
|
# A source flag alone constrains the AI; it never bypasses it.
|
|
if ! PHOEBUS_AI_PROVIDER="" PHOEBUS_AI_BACKEND=direct \
|
|
"$SYS" search firefox --flatpak >"$WORK/srch-flag.out" 2>"$WORK/srch-flag.err" && \
|
|
grep -q "sys search firefox --legacy --flatpak" "$WORK/srch-flag.err" && \
|
|
! grep -q "Package catalogue" "$WORK/srch-flag.out"; then
|
|
report search-flag-constraint PASS "source flag constrained the AI workflow"
|
|
else
|
|
report search-flag-constraint FAIL "$(head -c 200 "$WORK/srch-flag.err")"
|
|
fi
|
|
|
|
# Destructive maintenance is privileged.
|
|
if [[ "$(id -u)" -ne 0 ]]; then
|
|
run_expect_nonzero nonroot-depclean "require root" --legacy depclean
|
|
run_expect_nonzero nonroot-apply-updates "require root" apply updates --legacy
|
|
run_expect_nonzero nonroot-clean-cache "require root" clean cache --legacy
|
|
run_expect_nonzero nonroot-source-build "require root" --legacy --action '{"action":"convert_to_source","app":"firefox"}'
|
|
run_expect_nonzero nonroot-tuned-kernel "require root" --legacy build tuned kernel
|
|
fi
|
|
|
|
# User-initiated source builds fail closed before any mutation, even as root:
|
|
# Brave has no trustworthy source route, and unknown apps are never guessed.
|
|
# Both are refused during effect resolution, before approval and commands.
|
|
if [[ "$(id -u)" -eq 0 ]]; then
|
|
json_nonzero source-brave "no trustworthy" '{"action":"convert_to_source","app":"brave"}'
|
|
json_nonzero source-unknown "no supported source build" '{"action":"convert_to_source","app":"definitely-not-an-app-xyz"}'
|
|
fi
|
|
|
|
# --- Service lifecycle (read-only + non-root gates) -----------------------
|
|
run_expect_zero inspect-service-legacy --legacy inspect sshd
|
|
if [[ "$(id -u)" -ne 0 ]]; then
|
|
run_expect_nonzero nonroot-enable "root" --legacy enable sshd
|
|
run_expect_nonzero nonroot-disable "root" --legacy disable sshd
|
|
run_expect_nonzero nonroot-start "root" --legacy start sshd
|
|
run_expect_nonzero nonroot-stop "root" --legacy stop sshd
|
|
run_expect_nonzero nonroot-restart "root" --legacy restart sshd
|
|
fi
|
|
|
|
# --- Versioned atom validation (Flaw 1 regression) ------------------------
|
|
# Versioned/slotted atoms must pass the validator and reach the tool.
|
|
run_expect_zero versioned-atom-inspect --legacy --action \
|
|
'{"action":"inspect_package","atom":"=app-misc/hello-2.10"}'
|
|
run_expect_zero slot-atom-inspect --legacy --action \
|
|
'{"action":"inspect_package","atom":"dev-lang/python:3.12"}'
|
|
|
|
# --- Path normalization (Flaw 5 regression) -------------------------------
|
|
# A relative path in "who owns" must be normalized, never rejected as an
|
|
# invalid file path. (The ownership tool itself may be absent.)
|
|
"$SYS" who owns usr/bin/bash --legacy >"$WORK/who.out" 2>"$WORK/who.err"
|
|
if ! grep -qi "invalid file path\|could not parse" "$WORK/who.err" "$WORK/who.out"; then
|
|
report who-owns-relative PASS "$(head -c 140 "$WORK/who.out" | tr '\n' ' ')"
|
|
else
|
|
report who-owns-relative FAIL "$(head -c 200 "$WORK/who.err" | tr '\n' ' ')"
|
|
fi
|
|
|
|
# --- Optional mutating group (disposable VM only) -------------------------
|
|
if [[ "$MUTATE" == "1" ]]; then
|
|
PKG="${SYS_ACCEPTANCE_PACKAGE:-app-misc/hello}"
|
|
if "$SYS" -y install "$PKG" --legacy --portage >"$WORK/install.out" 2>&1; then
|
|
if emerge --pretend --quiet "$PKG" >/dev/null 2>&1; then
|
|
report mutate-install PASS "installed $PKG"
|
|
else
|
|
report mutate-install FAIL "install reported ok but system does not know $PKG"
|
|
fi
|
|
else
|
|
report mutate-install FAIL "sys install failed: $(head -c 160 "$WORK/install.out")"
|
|
fi
|
|
if "$SYS" -y remove "$PKG" --legacy >"$WORK/remove.out" 2>&1; then
|
|
report mutate-remove PASS "removed $PKG"
|
|
else
|
|
report mutate-remove FAIL "sys remove failed: $(head -c 160 "$WORK/remove.out")"
|
|
fi
|
|
"$SYS" history --legacy >"$WORK/hist.out" 2>&1
|
|
if grep -q "$PKG" "$WORK/hist.out"; then
|
|
report mutate-history PASS "transaction recorded"
|
|
else
|
|
report mutate-history FAIL "install/removal missing from history"
|
|
fi
|
|
fi
|
|
|
|
echo "===SYS-ACC-END=== SUMMARY PASS=$PASS FAIL=$FAIL"
|
|
[[ "$FAIL" -eq 0 ]]
|