phoebus_os/installer/tests/vm/serial-driver.py
Phoebus c759477c7f test(vm): automate rollback acceptance on a disposable clone
Add a serial-driver rollback mode and guest scripts that exercise the real
rollback path: record the /var/db/pkg fingerprint and a /home marker,
create a recovery point with sys snapshot, install a small package as a
controlled reversible update, run sys undo, power off, boot detached and
compare the restored package state and /home against the pre-rollback
values. Checks kernel modules, the snapshot recovery entry and sys
rollback status. sys is fetched from the host, matching the existing
harness evidence procedure; booting the recovery entry offline remains a
documented manual gate. bash -n guards for every guest script are added
to the no-VM unit tests.
2026-09-23 06:19:17 -07:00

348 lines
14 KiB
Bash

#!/usr/bin/env python3
"""Headless serial driver for the disposable UEFI VM.
Modes:
install : boot the live ISO kernel/initrd, run a guest script that installs
Phoebus non-interactively, then power off.
boot : boot the installed disk alone (no ISO, no -kernel), unlock LUKS,
log in and run verification commands.
rollback: boot an installed disk, record a known-good state, create a
recovery point through `sys snapshot`, introduce a reversible
update, run the real `sys undo`, power off and boot detached to
verify the restored root/var and package state. Destructive: point
DISK at a disposable clone, never a physical device.
Everything is parameterised through environment variables so the script is not
tied to one host. See tests/vm/README.md for the variables.
Safety: `DISK`/`DISKS` must be qcow2 files inside the disposable directory named
by `PHOEBUS_VM_DISKS` (default `$PHOEBUS_VM_STATE/disks`); block devices,
`/dev/*`, escaping symlinks, raw images and outside backing chains are refused.
Results are structured and the exit status is nonzero on failure.
"""
import os
import sys
import pexpect
import vmsafety
from vmsafety import Results
QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64")
LIBDIR = os.environ.get("QEMU_LIBDIR", "")
DATADIR = os.environ.get("QEMU_DATADIR", "")
OVMF_CODE = os.environ["OVMF_CODE"]
OVMF_VARS_TEMPLATE = os.environ["OVMF_VARS"]
DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "")
ISO = os.environ.get("ISO", "")
KERNEL = os.environ.get("KERNEL", "")
INITRD = os.environ.get("INITRD", "")
LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test")
USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test")
USER = os.environ.get("USERNAME", "phoebus")
MEM = os.environ.get("VM_MEM", "8192")
SMP = os.environ.get("VM_SMP", "8")
BLS_APPEND = (
"root=live:CDLABEL=gentoo-amd64-livegui rd.live.dir=/ "
"rd.live.squashimg=image.squashfs cdroot console=ttyS0,115200"
)
# Rollback acceptance fixtures, served from the host over QEMU user networking.
ROLLBACK_URL = os.environ.get("ROLLBACK_URL", "http://10.0.2.2:8000/rollback-guest.sh")
ROLLBACK_VERIFY_URL = os.environ.get(
"ROLLBACK_VERIFY_URL", "http://10.0.2.2:8000/rollback-verify.sh"
)
SYS_URL = os.environ.get("SYS_URL", "http://10.0.2.2:8000/sys")
def disposable_dir():
import pathlib
base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")
return pathlib.Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks"))
def base_args(ovmf_vars):
env = dict(os.environ)
if LIBDIR:
env["LD_LIBRARY_PATH"] = LIBDIR
args = ["-machine", "q35,accel=kvm"]
if DATADIR:
args += ["-L", DATADIR]
args += [
"-cpu", "host", "-smp", SMP, "-m", MEM,
"-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}",
"-drive", f"if=pflash,format=raw,file={ovmf_vars}",
"-netdev", "user,id=n0", "-device", "virtio-net-pci,netdev=n0,romfile=",
"-device", "virtio-rng-pci", "-vga", "std", "-display", "none",
"-serial", "stdio", "-monitor", "none", "-no-reboot",
]
return env, args
def spawn(args, env, log):
child = pexpect.spawn(QEMU, args, env=env, timeout=600,
encoding="utf-8", codec_errors="replace",
maxread=65536)
child.delaybeforesend = 0.3
child.logfile = log
return child
def do_install(results, paths, disks):
env, args = base_args(str(paths.ovmf_vars))
for disk in disks:
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
args += ["-drive", f"file={ISO},media=cdrom,readonly=on",
"-kernel", KERNEL, "-initrd", INITRD, "-append", BLS_APPEND]
log = open(paths.serial_log, "w")
child = spawn(args, env, log)
try:
child.expect(r"root@livecd[^\n]*# ", timeout=600)
results.pass_("live-shell")
url = os.environ.get("INSTALL_URL", "http://10.0.2.2:8000/install.sh")
child.sendline(f"curl -fsSL {url} -o /tmp/run.sh && bash /tmp/run.sh")
child.expect([r"===INSTALL-RC=([0-9]+)===", "===FETCH-FAIL===",
"Kernel panic", pexpect.EOF, pexpect.TIMEOUT], timeout=9000)
if "INSTALL-RC" in (child.after or ""):
rc = child.match.group(1)
results.assert_true("install-rc-zero", rc == "0", f"INSTALL-RC={rc}")
else:
results.fail("install-rc-zero", "install did not report a status")
serial = paths.serial_log.read_text(errors="replace")
results.assert_true("no-secret-leak", "SECRET_LEAK_FOUND" not in serial,
"the guest reported a secret leak")
child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600)
except pexpect.TIMEOUT as exc:
results.fail("install", f"timeout: {exc}")
except pexpect.EOF:
results.fail("install", "unexpected EOF")
finally:
child.close(force=True)
log.close()
def _installed_args(paths, disks):
env, args = base_args(str(paths.ovmf_vars))
for disk in disks:
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
args += ["-boot", "order=c"]
return env, args
def _unlock_and_login(child, results, tag):
child.expect([r"[Pp]assphrase", r"login:"], timeout=300)
if "assphrase" in (child.after or ""):
override = os.environ.get("BOOT_PASSPHRASE")
candidates = [override] if override else [USER_PW, LUKS_PW]
child.sendline(candidates[0])
for candidate in candidates[1:]:
try:
child.expect(r"[Pp]assphrase", timeout=15)
child.sendline(candidate)
except (pexpect.TIMEOUT, pexpect.EOF):
break
logged_in = False
for _ in range(5):
child.expect(r"phoebus login:", timeout=180)
child.sendline(USER)
idx = child.expect([r"[Pp]assword:", r"phoebus login:", r"Login incorrect"],
timeout=30)
if idx == 0:
child.sendline(USER_PW)
if child.expect([r"[$#] $", r"Login incorrect", r"phoebus login:"],
timeout=60) == 0:
logged_in = True
break
results.assert_true(f"{tag}-login", logged_in, f"{tag}: could not log in")
def _fetch_sys(child):
child.sendline(
f"curl -fsSL {SYS_URL} -o /tmp/sys && chmod 755 /tmp/sys "
"&& test -x /tmp/sys && echo SYS-OK-$(printf READY)"
)
child.expect("SYS-OK-READY", timeout=120)
def _parse_marker(text, key):
import re
match = re.search(r"===" + re.escape(key) + r"(.+?)===", text)
if not match:
return None
out = {}
for token in match.group(1).split():
if "=" in token:
name, value = token.split("=", 1)
out[name] = value
return out
def do_rollback(results, paths, disks):
"""Snapshot -> reversible update -> sys undo -> detached verify."""
# Phase 1: known-good -> snapshot -> break -> undo, then power off.
env, args = _installed_args(paths, disks)
log = open(paths.serial_log, "w")
child = spawn(args, env, log)
phase1 = ""
try:
_unlock_and_login(child, results, "rollback-before")
_fetch_sys(child)
child.sendline(f"curl -fsSL {ROLLBACK_URL} -o /tmp/rb.sh && bash /tmp/rb.sh")
child.expect(
["===ROLLBACK-DONE===", "curl:", pexpect.TIMEOUT, pexpect.EOF],
timeout=int(os.environ.get("ROLLBACK_TIMEOUT", "7200")),
)
if "ROLLBACK-DONE" not in (child.after or ""):
results.fail("rollback-sequence", f"the guest rollback did not finish: {child.after!r}")
else:
results.pass_("rollback-sequence")
child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600)
phase1 = paths.serial_log.read_text(errors="replace")
except pexpect.TIMEOUT as exc:
results.fail("rollback-before", f"timeout: {exc}")
except pexpect.EOF:
results.fail("rollback-before", "unexpected EOF")
finally:
child.close(force=True)
log.close()
before = _parse_marker(phase1, "ROLLBACK-KNOWN-GOOD")
undo = _parse_marker(phase1, "ROLLBACK-UNDO")
if before is None:
results.fail("rollback-known-good", "the guest did not record the known-good state")
if undo is None or undo.get("rc") != "0":
results.fail("rollback-undo", f"sys undo did not succeed: {undo}")
# Phase 2: boot the disk detached and verify the restored generation.
env, args = _installed_args(paths, disks)
log = open(paths.serial_log, "w")
child = spawn(args, env, log)
phase2 = ""
try:
_unlock_and_login(child, results, "rollback-after")
_fetch_sys(child)
child.sendline(f"curl -fsSL {ROLLBACK_VERIFY_URL} -o /tmp/rbv.sh && bash /tmp/rbv.sh")
child.expect("===AFTER-DONE===",
timeout=int(os.environ.get("ROLLBACK_VERIFY_TIMEOUT", "900")))
phase2 = paths.serial_log.read_text(errors="replace")
results.assert_true("rollback-systemd-running", "running" in phase2,
"systemd did not report running after rollback")
after = _parse_marker(phase2, "AFTER-KNOWN-GOOD")
if before is None or after is None:
results.fail("rollback-state", "could not read the before/after state")
else:
results.assert_true(
"rollback-package-state-restored",
before.get("pkgs") == after.get("pkgs"),
f"package db changed across rollback: before={before.get('pkgs')} after={after.get('pkgs')}",
)
results.assert_true(
"rollback-home-survives",
before.get("home") == after.get("home"),
f"/home did not survive: before={before.get('home')} after={after.get('home')}",
)
results.assert_true("rollback-modules-present", "===AFTER-MODULES ok===" in phase2,
"no kernel modules for the running kernel")
results.assert_true(
"rollback-recovery-entry",
"systemd.unit=rescue.target" in phase2 and "@snapshots/" in phase2,
"the snapshot recovery entry is missing or not snapshot-based",
)
status = phase2.split("===AFTER-STATUS===", 1)
status_text = status[1].split("===AFTER-DONE===", 1)[0].strip() if len(status) > 1 else ""
results.assert_true("rollback-status-recorded", bool(status_text),
"sys rollback status produced no evidence")
except pexpect.TIMEOUT as exc:
results.fail("rollback-after", f"timeout: {exc}")
except pexpect.EOF:
results.fail("rollback-after", "unexpected EOF")
finally:
child.close(force=True)
log.close()
def do_boot(results, paths, disks):
env, args = base_args(str(paths.ovmf_vars))
for disk in disks:
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
args += ["-boot", "order=c"]
# The end marker is built with a command substitution so the shell's echo of
# the typed line does not itself contain it; otherwise `expect` would match
# the echo and close the console before the checks ran.
checks = os.environ.get(
"BOOT_CHECKS",
"echo ===CHECKS-$(printf START)===; "
"cat /proc/1/comm; systemctl is-system-running; "
"systemctl --failed --no-pager; ip -brief addr; "
"lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS; echo ===CHECKS-$(printf END)===",
)
log = open(paths.serial_log, "w")
child = spawn(args, env, log)
try:
child.expect([r"[Pp]assphrase", r"login:"], timeout=300)
if "assphrase" in (child.after or ""):
child.sendline(LUKS_PW)
logged_in = False
for _ in range(5):
child.expect(r"phoebus login:", timeout=180)
child.sendline(USER)
idx = child.expect([r"[Pp]assword:", r"phoebus login:",
r"Login incorrect"], timeout=30)
if idx == 0:
child.sendline(USER_PW)
if child.expect([r"[$#] $", r"Login incorrect",
r"phoebus login:"], timeout=60) == 0:
logged_in = True
break
results.assert_true("detached-boot-login", logged_in, "could not log in")
child.sendline(checks)
child.expect("===CHECKS-END===", timeout=int(os.environ.get("CHECKS_TIMEOUT", "180")))
serial = paths.serial_log.read_text(errors="replace")
results.assert_true("boot-systemd-running", "running" in serial,
"systemd did not report running")
except pexpect.TIMEOUT as exc:
results.fail("boot", f"timeout: {exc}")
except pexpect.EOF:
results.fail("boot", "unexpected EOF")
finally:
child.close(force=True)
log.close()
def main() -> int:
mode = sys.argv[1] if len(sys.argv) > 1 else ""
if mode not in ("install", "boot", "rollback"):
print(__doc__)
return 2
results = Results(f"serial-{mode}")
paths = vmsafety.make_run_paths(
f"serial-{mode}", base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"),
ovmf_vars_template=OVMF_VARS_TEMPLATE,
)
try:
disks = vmsafety.validate_writable_drives(DISKS.split(","), disposable_dir())
except vmsafety.UnsafeDiskError as exc:
results.blocked("disposable-disk", str(exc))
results.write(paths.results_json)
print(f"harness error: {exc}", file=sys.stderr)
return 2
results.pass_("disposable-disk", ", ".join(str(d) for d in disks))
try:
if mode == "install":
do_install(results, paths, disks)
elif mode == "rollback":
do_rollback(results, paths, disks)
else:
do_boot(results, paths, disks)
finally:
results.write(paths.results_json)
print(results.summary(), flush=True)
return results.exit_code()
if __name__ == "__main__":
sys.exit(main())