diff --git a/installer/tests/vm/README.md b/installer/tests/vm/README.md index 7d16c75..0dc17b9 100644 --- a/installer/tests/vm/README.md +++ b/installer/tests/vm/README.md @@ -7,7 +7,10 @@ touching a physical disk. The target is always a qcow2 file. - `run-vm.sh` interactive QEMU launcher (manual testing). - `install.sh` guest-side, non-interactive installation (run in the VM). -- `serial-driver.py` host-side pexpect driver (`install` and `boot` modes). +- `serial-driver.py` host-side pexpect driver (`install`, `boot` and + `rollback` modes). +- `rollback-guest.sh`, `rollback-verify.sh` guest-side rollback acceptance + (fetch `sys`, snapshot, break, undo, verify). - `test.conf.example` installer configuration used by the driver. ## Host prerequisites @@ -62,6 +65,33 @@ To patch a console into an existing entry for observation, the BLS entry in `/efi/loader/entries/*.conf` takes an `options` line; the installer can also embed a console via `KERNEL_CONSOLE` (`GENTOO_KERNEL_CONSOLE` equivalent). +## Rollback acceptance (`serial-driver.py rollback`) + +`rollback` drives the real rollback path end to end on an +**already-installed** disposable disk: + +1. boot the installed disk, unlock LUKS and log in; +2. fetch `sys` and `rollback-guest.sh` from the host (the target does not yet + install `sys`; see `docs/HANDOFF.md`), record the `/var/db/pkg` fingerprint + and a `/home` marker, create a recovery point with `sys snapshot`, install a + small package (a controlled, reversible update) and run `sys undo`; +3. power off, boot the disk detached, and run `rollback-verify.sh`, comparing + the package fingerprint and the `/home` marker, checking kernel modules, and + reading the snapshot recovery entry and `sys rollback status`. + +Serve the fixtures as well as the installer: + +```sh +cp installer/tests/vm/rollback-guest.sh installer/tests/vm/rollback-verify.sh /path/to/serve/ +cp installer-rs/target/x86_64-unknown-linux-gnu/release/sys /path/to/serve/ +DISK=/tmp/phoebus/installed-clone.qcow2 OVMF_CODE=... OVMF_VARS=... \ +python3 installer/tests/vm/serial-driver.py rollback +``` + +Point `DISK` at a **disposable clone** of the evidence image so the original is +preserved. Booting the recovery entry itself with the network removed remains a +manual gate; the harness records the entry but does not select it. + ## Safety (enforced, not just documented) `vmsafety.py` validates every writable drive before QEMU starts: diff --git a/installer/tests/vm/rollback-guest.sh b/installer/tests/vm/rollback-guest.sh new file mode 100755 index 0000000..5d91dfd --- /dev/null +++ b/installer/tests/vm/rollback-guest.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Guest-side rollback acceptance for the disposable VM. +# +# Fetched and run by `serial-driver.py rollback` inside an already-installed +# Phoebus system. It establishes a known-good package/root state, creates a +# recovery point through the supported `sys` interface, introduces a +# controlled, reversible mutation, runs the real `sys undo` path and powers +# off. The host then boots the disk detached and verifies the restored state. +# +# `sys` is fetched from the host (it is not installed into the target yet; see +# docs/HANDOFF.md), so this exercises the real command path, not a mock. +set -uo pipefail + +SERVE="${SERVE_URL:-http://10.0.2.2:8000}" +SYS="${SYS_BIN:-/tmp/sys}" +USER_NAME="${USERNAME:-phoebus}" +WORK=/tmp/phoebus-rollback +mkdir -p "$WORK" + +pkg_fingerprint() { + # The package database, not a marker file: a rollback must restore it. + find /var/db/pkg -mindepth 2 -maxdepth 2 -type d 2>/dev/null \ + | sort | sha256sum | awk '{print $1}' +} + +# 1. Establish and record a known-good root/package state. The /home marker +# proves user data survives (it is not part of the snapshotted subvolumes). +KERNEL="$(uname -r)" +HOME_MARKER="rollback-marker-$$-$KERNEL" +printf '%s\n' "$HOME_MARKER" > "/home/$USER_NAME/rollback-marker.txt" +PKGS_BEFORE="$(pkg_fingerprint)" +echo "===ROLLBACK-KNOWN-GOOD pkgs=$PKGS_BEFORE kernel=$KERNEL home=$HOME_MARKER===" + +# 2. Create the recovery point through the supported interface. +"$SYS" -y snapshot "rollback acceptance" > "$WORK/snapshot.out" 2>&1 +SNAP_ID="$(sed -n 's/^protective snapshot: //p' "$WORK/snapshot.out" | tail -1)" +echo "===ROLLBACK-SNAPSHOT id=$SNAP_ID===" + +# 3. Controlled, reversible update failure. A small package install is a real +# update: it changes both the root subvolume and /var/db/pkg, and a snapshot +# can restore it. A root marker proves the root generation too. +printf 'broken by rollback acceptance\n' > /etc/phoebus/rollback-broken.txt 2>/dev/null || true +"$SYS" -y install app-editors/nano > "$WORK/break.out" 2>&1 +echo "===ROLLBACK-BROKEN pkgs=$(pkg_fingerprint)===" + +# 4. Invoke the real sys undo path. +"$SYS" -y undo > "$WORK/undo.out" 2>&1 +RC=$? +echo "===ROLLBACK-UNDO rc=$RC===" +"$SYS" rollback status > "$WORK/status.out" 2>&1 || true +sed -n '1,40p' "$WORK/status.out" + +echo "===ROLLBACK-DONE===" +sync +sudo -n poweroff -f || poweroff -f +echo "===POWEROFF===" diff --git a/installer/tests/vm/rollback-verify.sh b/installer/tests/vm/rollback-verify.sh new file mode 100755 index 0000000..611cd23 --- /dev/null +++ b/installer/tests/vm/rollback-verify.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Guest-side post-rollback verification for the disposable VM. +# +# Fetched and run by `serial-driver.py rollback` after the detached reboot. It +# records the restored root/var generation, the package-database fingerprint, +# kernel modules, the /home marker, the recovery BLS entry and `sys rollback +# status`. The host compares these against the pre-rollback markers; a marker +# alone would not prove package-state consistency. +set -uo pipefail + +SYS="${SYS_BIN:-/tmp/sys}" +USER_NAME="${USERNAME:-phoebus}" + +pkg_fingerprint() { + find /var/db/pkg -mindepth 2 -maxdepth 2 -type d 2>/dev/null \ + | sort | sha256sum | awk '{print $1}' +} + +HOME_MARKER="$(cat "/home/$USER_NAME/rollback-marker.txt" 2>/dev/null || echo missing)" +echo "===AFTER-KNOWN-GOOD pkgs=$(pkg_fingerprint) kernel=$(uname -r) home=$HOME_MARKER===" +if [ -e /etc/phoebus/rollback-broken.txt ]; then + echo "===AFTER-BROKEN-MARKER present===" +else + echo "===AFTER-BROKEN-MARKER absent===" +fi + +echo "===AFTER-ROOT===" +btrfs subvolume show / 2>&1 | sed -n '1,10p' +echo "===AFTER-VAR===" +btrfs subvolume show /var 2>&1 | sed -n '1,10p' + +if [ -d "/lib/modules/$(uname -r)" ]; then + echo "===AFTER-MODULES ok===" +else + echo "===AFTER-MODULES missing===" +fi + +echo "===AFTER-ENTRIES===" +for d in /efi/loader/entries /boot/efi/loader/entries; do + if [ -d "$d" ]; then + ls "$d" + break + fi +done + +echo "===AFTER-RECOVERY===" +cat /efi/loader/entries/phoebus-recovery.conf \ + /boot/efi/loader/entries/phoebus-recovery.conf 2>/dev/null || true + +echo "===AFTER-STATUS===" +"$SYS" rollback status 2>&1 || true + +echo "===AFTER-DONE===" diff --git a/installer/tests/vm/serial-driver.py b/installer/tests/vm/serial-driver.py index 40082e7..86d4f79 100644 --- a/installer/tests/vm/serial-driver.py +++ b/installer/tests/vm/serial-driver.py @@ -1,12 +1,17 @@ #!/usr/bin/env python3 """Headless serial driver for the disposable UEFI VM. -Two modes: +Modes: install : boot the live ISO kernel/initrd, run a guest script that installs Phoebus non-interactively, then power off. boot : boot the installed disk alone (no ISO, no -kernel), unlock LUKS, log in and run verification commands. + rollback: boot an installed disk, record a known-good state, create a + recovery point through `sys snapshot`, introduce a reversible + update, run the real `sys undo`, power off and boot detached to + verify the restored root/var and package state. Destructive: point + DISK at a disposable clone, never a physical device. Everything is parameterised through environment variables so the script is not tied to one host. See tests/vm/README.md for the variables. @@ -44,6 +49,13 @@ BLS_APPEND = ( "rd.live.squashimg=image.squashfs cdroot console=ttyS0,115200" ) +# Rollback acceptance fixtures, served from the host over QEMU user networking. +ROLLBACK_URL = os.environ.get("ROLLBACK_URL", "http://10.0.2.2:8000/rollback-guest.sh") +ROLLBACK_VERIFY_URL = os.environ.get( + "ROLLBACK_VERIFY_URL", "http://10.0.2.2:8000/rollback-verify.sh" +) +SYS_URL = os.environ.get("SYS_URL", "http://10.0.2.2:8000/sys") + def disposable_dir(): import pathlib @@ -112,6 +124,147 @@ def do_install(results, paths, disks): log.close() +def _installed_args(paths, disks): + env, args = base_args(str(paths.ovmf_vars)) + for disk in disks: + args += ["-drive", f"file={disk},if=virtio,format=qcow2"] + args += ["-boot", "order=c"] + return env, args + + +def _unlock_and_login(child, results, tag): + child.expect([r"[Pp]assphrase", r"login:"], timeout=300) + if "assphrase" in (child.after or ""): + override = os.environ.get("BOOT_PASSPHRASE") + candidates = [override] if override else [USER_PW, LUKS_PW] + child.sendline(candidates[0]) + for candidate in candidates[1:]: + try: + child.expect(r"[Pp]assphrase", timeout=15) + child.sendline(candidate) + except (pexpect.TIMEOUT, pexpect.EOF): + break + logged_in = False + for _ in range(5): + child.expect(r"phoebus login:", timeout=180) + child.sendline(USER) + idx = child.expect([r"[Pp]assword:", r"phoebus login:", r"Login incorrect"], + timeout=30) + if idx == 0: + child.sendline(USER_PW) + if child.expect([r"[$#] $", r"Login incorrect", r"phoebus login:"], + timeout=60) == 0: + logged_in = True + break + results.assert_true(f"{tag}-login", logged_in, f"{tag}: could not log in") + + +def _fetch_sys(child): + child.sendline( + f"curl -fsSL {SYS_URL} -o /tmp/sys && chmod 755 /tmp/sys " + "&& test -x /tmp/sys && echo SYS-OK-$(printf READY)" + ) + child.expect("SYS-OK-READY", timeout=120) + + +def _parse_marker(text, key): + import re + + match = re.search(r"===" + re.escape(key) + r"(.+?)===", text) + if not match: + return None + out = {} + for token in match.group(1).split(): + if "=" in token: + name, value = token.split("=", 1) + out[name] = value + return out + + +def do_rollback(results, paths, disks): + """Snapshot -> reversible update -> sys undo -> detached verify.""" + # Phase 1: known-good -> snapshot -> break -> undo, then power off. + env, args = _installed_args(paths, disks) + log = open(paths.serial_log, "w") + child = spawn(args, env, log) + phase1 = "" + try: + _unlock_and_login(child, results, "rollback-before") + _fetch_sys(child) + child.sendline(f"curl -fsSL {ROLLBACK_URL} -o /tmp/rb.sh && bash /tmp/rb.sh") + child.expect( + ["===ROLLBACK-DONE===", "curl:", pexpect.TIMEOUT, pexpect.EOF], + timeout=int(os.environ.get("ROLLBACK_TIMEOUT", "7200")), + ) + if "ROLLBACK-DONE" not in (child.after or ""): + results.fail("rollback-sequence", f"the guest rollback did not finish: {child.after!r}") + else: + results.pass_("rollback-sequence") + child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600) + phase1 = paths.serial_log.read_text(errors="replace") + except pexpect.TIMEOUT as exc: + results.fail("rollback-before", f"timeout: {exc}") + except pexpect.EOF: + results.fail("rollback-before", "unexpected EOF") + finally: + child.close(force=True) + log.close() + + before = _parse_marker(phase1, "ROLLBACK-KNOWN-GOOD") + undo = _parse_marker(phase1, "ROLLBACK-UNDO") + if before is None: + results.fail("rollback-known-good", "the guest did not record the known-good state") + if undo is None or undo.get("rc") != "0": + results.fail("rollback-undo", f"sys undo did not succeed: {undo}") + + # Phase 2: boot the disk detached and verify the restored generation. + env, args = _installed_args(paths, disks) + log = open(paths.serial_log, "w") + child = spawn(args, env, log) + phase2 = "" + try: + _unlock_and_login(child, results, "rollback-after") + _fetch_sys(child) + child.sendline(f"curl -fsSL {ROLLBACK_VERIFY_URL} -o /tmp/rbv.sh && bash /tmp/rbv.sh") + child.expect("===AFTER-DONE===", + timeout=int(os.environ.get("ROLLBACK_VERIFY_TIMEOUT", "900"))) + phase2 = paths.serial_log.read_text(errors="replace") + results.assert_true("rollback-systemd-running", "running" in phase2, + "systemd did not report running after rollback") + after = _parse_marker(phase2, "AFTER-KNOWN-GOOD") + if before is None or after is None: + results.fail("rollback-state", "could not read the before/after state") + else: + results.assert_true( + "rollback-package-state-restored", + before.get("pkgs") == after.get("pkgs"), + f"package db changed across rollback: before={before.get('pkgs')} after={after.get('pkgs')}", + ) + results.assert_true( + "rollback-home-survives", + before.get("home") == after.get("home"), + f"/home did not survive: before={before.get('home')} after={after.get('home')}", + ) + results.assert_true("rollback-modules-present", "===AFTER-MODULES ok===" in phase2, + "no kernel modules for the running kernel") + results.assert_true( + "rollback-recovery-entry", + "systemd.unit=rescue.target" in phase2 and "@snapshots/" in phase2, + "the snapshot recovery entry is missing or not snapshot-based", + ) + status = phase2.split("===AFTER-STATUS===", 1) + status_text = status[1].split("===AFTER-DONE===", 1)[0].strip() if len(status) > 1 else "" + results.assert_true("rollback-status-recorded", bool(status_text), + "sys rollback status produced no evidence") + except pexpect.TIMEOUT as exc: + results.fail("rollback-after", f"timeout: {exc}") + except pexpect.EOF: + results.fail("rollback-after", "unexpected EOF") + finally: + child.close(force=True) + log.close() + + def do_boot(results, paths, disks): env, args = base_args(str(paths.ovmf_vars)) for disk in disks: @@ -162,7 +315,7 @@ def do_boot(results, paths, disks): def main() -> int: mode = sys.argv[1] if len(sys.argv) > 1 else "" - if mode not in ("install", "boot"): + if mode not in ("install", "boot", "rollback"): print(__doc__) return 2 results = Results(f"serial-{mode}") @@ -181,6 +334,8 @@ def main() -> int: try: if mode == "install": do_install(results, paths, disks) + elif mode == "rollback": + do_rollback(results, paths, disks) else: do_boot(results, paths, disks) finally: diff --git a/installer/tests/vm/test_vmsafety.py b/installer/tests/vm/test_vmsafety.py index 6c8b8c4..a9c719f 100644 --- a/installer/tests/vm/test_vmsafety.py +++ b/installer/tests/vm/test_vmsafety.py @@ -223,5 +223,31 @@ class QmpClientTests(unittest.TestCase): self.assertTrue(any(cmd.startswith("screendump") for cmd in server.commands)) +class HarnessScriptsTest(unittest.TestCase): + """Guest scripts must be valid shell before any VM is allowed to run them.""" + + VM_DIR = Path(__file__).resolve().parent + + def test_guest_scripts_are_syntactically_valid(self): + scripts = sorted(self.VM_DIR.glob("*.sh")) + self.assertTrue(scripts, "no guest scripts found") + for script in scripts: + proc = subprocess.run( + ["bash", "-n", str(script)], capture_output=True, text=True + ) + self.assertEqual( + proc.returncode, + 0, + f"bash -n failed for {script.name}: {proc.stderr}", + ) + + def test_rollback_fixtures_and_mode_are_present(self): + for name in ("rollback-guest.sh", "rollback-verify.sh"): + self.assertTrue((self.VM_DIR / name).is_file(), f"missing {name}") + driver = (self.VM_DIR / "serial-driver.py").read_text() + self.assertIn('"rollback"', driver) + self.assertIn("do_rollback", driver) + + if __name__ == "__main__": unittest.main(verbosity=2)