#!/usr/bin/env python3 """Headless serial driver for the disposable UEFI VM. Modes: install : boot the live ISO kernel/initrd, run a guest script that installs Phoebus non-interactively, then power off. boot : boot the installed disk alone (no ISO, no -kernel), unlock LUKS, log in and run verification commands. rollback: boot an installed disk, record a known-good state, create a recovery point through `sys snapshot`, introduce a reversible update, run the real `sys undo`, power off and boot detached to verify the restored root/var and package state. Destructive: point DISK at a disposable clone, never a physical device. Everything is parameterised through environment variables so the script is not tied to one host. See tests/vm/README.md for the variables. Safety: `DISK`/`DISKS` must be qcow2 files inside the disposable directory named by `PHOEBUS_VM_DISKS` (default `$PHOEBUS_VM_STATE/disks`); block devices, `/dev/*`, escaping symlinks, raw images and outside backing chains are refused. Results are structured and the exit status is nonzero on failure. """ import os import re import sys import pexpect import vmsafety from vmsafety import Results QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64") LIBDIR = os.environ.get("QEMU_LIBDIR", "") DATADIR = os.environ.get("QEMU_DATADIR", "") OVMF_CODE = os.environ["OVMF_CODE"] OVMF_VARS_TEMPLATE = os.environ["OVMF_VARS"] DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "") ISO = os.environ.get("ISO", "") KERNEL = os.environ.get("KERNEL", "") INITRD = os.environ.get("INITRD", "") LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test") USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test") USER = os.environ.get("USERNAME", "phoebus") MEM = os.environ.get("VM_MEM", "8192") SMP = os.environ.get("VM_SMP", "8") # Shell-prompt alternative for an installed Phoebus login. The default login # shell is zsh with a powerline-style prompt ending in the `❯` glyph; color # escapes sit between the glyph and the trailing space, so the bare glyph # (not `❯ `) is matched. Bash/root prompts keep matching `[$#] `. SHELL_PROMPT = r"[$#] |❯" BLS_APPEND = ( "root=live:CDLABEL=gentoo-amd64-livegui rd.live.dir=/ " "rd.live.squashimg=image.squashfs cdroot console=ttyS0,115200" ) # Rollback acceptance fixtures, served from the host over QEMU user networking. ROLLBACK_URL = os.environ.get("ROLLBACK_URL", "http://10.0.2.2:8000/rollback-guest.sh") ROLLBACK_VERIFY_URL = os.environ.get( "ROLLBACK_VERIFY_URL", "http://10.0.2.2:8000/rollback-verify.sh" ) SYS_URL = os.environ.get("SYS_URL", "http://10.0.2.2:8000/sys") def disposable_dir(): import pathlib base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm") return pathlib.Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks")) def base_args(ovmf_vars): env = dict(os.environ) if LIBDIR: env["LD_LIBRARY_PATH"] = LIBDIR args = ["-machine", "q35,accel=kvm"] if DATADIR: args += ["-L", DATADIR] args += [ "-cpu", "host", "-smp", SMP, "-m", MEM, "-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}", "-drive", f"if=pflash,format=raw,file={ovmf_vars}", "-netdev", "user,id=n0", "-device", "virtio-net-pci,netdev=n0,romfile=", "-device", "virtio-rng-pci", "-vga", "std", "-display", "none", "-serial", "stdio", "-monitor", "none", "-no-reboot", ] return env, args def spawn(args, env, log): child = pexpect.spawn(QEMU, args, env=env, timeout=600, encoding="utf-8", codec_errors="replace", maxread=65536) child.delaybeforesend = 0.3 # Log only guest output. The general logfile also records bytes sent by # pexpect, including LUKS, login and sudo passwords. child.logfile_read = log return child def close_quietly(child): """Best-effort teardown. After the guest powers off, pexpect can raise `ExceptionPexpect: Could not terminate the child`; that must not turn a successful run into a nonzero driver exit.""" try: child.close(force=True) except Exception: # noqa: BLE001 - cleanup must never mask the result pass def do_install(results, paths, disks): env, args = base_args(str(paths.ovmf_vars)) for disk in disks: args += ["-drive", f"file={disk},if=virtio,format=qcow2"] args += ["-drive", f"file={ISO},media=cdrom,readonly=on", "-kernel", KERNEL, "-initrd", INITRD, "-append", BLS_APPEND] log = open(paths.serial_log, "w") child = spawn(args, env, log) try: child.expect(r"root@livecd[^\n]*# ", timeout=600) results.pass_("live-shell") url = os.environ.get("INSTALL_URL", "http://10.0.2.2:8000/install.sh") child.sendline(f"curl -fsSL {url} -o /tmp/run.sh && bash /tmp/run.sh") child.expect([r"===INSTALL-RC=([0-9]+)===", "===FETCH-FAIL===", "Kernel panic", pexpect.EOF, pexpect.TIMEOUT], timeout=19800) # `child.after` is the matched pattern class (not a string) when EOF or # TIMEOUT matched; handle that instead of raising a TypeError. matched = child.after if isinstance(child.after, str) else "" if "INSTALL-RC" in matched: rc = child.match.group(1) results.assert_true("install-rc-zero", rc == "0", f"INSTALL-RC={rc}") else: results.fail( "install-rc-zero", f"install did not report a status (matched {matched!r})", ) serial = paths.serial_log.read_text(errors="replace") results.assert_true("no-secret-leak", "SECRET_LEAK_FOUND" not in serial, "the guest reported a secret leak") child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600) except pexpect.TIMEOUT as exc: results.fail("install", f"timeout: {exc}") except pexpect.EOF: results.fail("install", "unexpected EOF") finally: close_quietly(child) log.close() def _installed_args(paths, disks): env, args = base_args(str(paths.ovmf_vars)) for disk in disks: args += ["-drive", f"file={disk},if=virtio,format=qcow2"] args += ["-boot", "order=c"] return env, args def _unlock_and_login(child, results, tag): child.expect([r"[Pp]assphrase", r"login:"], timeout=300) if "assphrase" in (child.after or ""): override = os.environ.get("BOOT_PASSPHRASE") candidates = [override] if override else [USER_PW, LUKS_PW] child.sendline(candidates[0]) for candidate in candidates[1:]: try: child.expect(r"[Pp]assphrase", timeout=15) child.sendline(candidate) except (pexpect.TIMEOUT, pexpect.EOF): break logged_in = False for _ in range(5): child.expect(r"phoebus login:", timeout=180) child.sendline(USER) idx = child.expect([r"[Pp]assword:", r"phoebus login:", r"Login incorrect"], timeout=30) if idx == 0: child.sendline(USER_PW) if child.expect([SHELL_PROMPT, r"Login incorrect", r"phoebus login:"], timeout=60) == 0: logged_in = True break results.assert_true(f"{tag}-login", logged_in, f"{tag}: could not log in") def _sudo_auth(child, results, tag): """Authenticate sudo for the rollback acceptance guest scripts. The snapshot/break/undo/verify steps mutate the system and must run as root. The password is sent only at sudo's own prompt (terminal echo disabled, like the login password), so it never lands in the serial transcript the way a command-line credential would. Returns True when the timestamp stuck; the caller must abort the phase otherwise, or every later `sudo` would block on a prompt nobody answers. """ try: # Unique markers, never a bare `[$#] ` alternative: the buffer still # holds the previous shell prompt when this runs, and a generic # prompt pattern would match that stale text immediately — skipping # the password while sudo is still starting (the next typed line # would then be consumed as the password and rejected). child.sendline("sudo -v; echo SUDO-V-$(printf DONE)") idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", "SUDO-V-DONE"], timeout=60) if idx == 0: child.sendline(USER_PW) child.expect("SUDO-V-DONE", timeout=60) # Prove the timestamp stuck: without it every later `sudo` would block # on a password prompt nobody answers, hanging the run until timeout. child.sendline("sudo -n true && echo SUDO-OK-$(printf READY)") idx = child.expect(["SUDO-OK-READY", r"[Pp]assword"], timeout=30) except (pexpect.TIMEOUT, pexpect.EOF): idx = -1 if idx != 0: results.fail(f"{tag}-sudo", "sudo authentication did not stick") return False return True def _fetch_sys(child): # The installed system should already carry /usr/local/bin/sys; only fall # back to fetching the host build when it is absent. child.sendline( "if [ -x /usr/local/bin/sys ]; then SYS=/usr/local/bin/sys; " "elif [ -x /tmp/sys ]; then SYS=/tmp/sys; " f"else curl -fsSL {SYS_URL} -o /tmp/sys && chmod 755 /tmp/sys && SYS=/tmp/sys; fi; " "test -x \"$SYS\" && echo SYS-OK-$(printf READY)" ) child.expect("SYS-OK-READY", timeout=120) def _parse_marker(text, key): import re match = re.search(r"===" + re.escape(key) + r"(.+?)===", text) if not match: return None out = {} for token in match.group(1).split(): if "=" in token: name, value = token.split("=", 1) out[name] = value return out def do_rollback(results, paths, disks): """Snapshot -> reversible update -> sys undo -> detached verify.""" # Phase 1: known-good -> snapshot -> break -> undo, then power off. env, args = _installed_args(paths, disks) log = open(paths.serial_log, "w") child = spawn(args, env, log) phase1 = "" try: _unlock_and_login(child, results, "rollback-before") _fetch_sys(child) if not _sudo_auth(child, results, "rollback-before"): return child.sendline(f"curl -fsSL {ROLLBACK_URL} -o /tmp/rb.sh && sudo bash /tmp/rb.sh") child.expect( ["===ROLLBACK-DONE===", "curl:", pexpect.TIMEOUT, pexpect.EOF], timeout=int(os.environ.get("ROLLBACK_TIMEOUT", "7200")), ) if "ROLLBACK-DONE" not in (child.after or ""): results.fail("rollback-sequence", f"the guest rollback did not finish: {child.after!r}") else: results.pass_("rollback-sequence") child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600) phase1 = paths.serial_log.read_text(errors="replace") except pexpect.TIMEOUT as exc: results.fail("rollback-before", f"timeout: {exc}") except pexpect.EOF: results.fail("rollback-before", "unexpected EOF") finally: close_quietly(child) log.close() before = _parse_marker(phase1, "ROLLBACK-KNOWN-GOOD") broken = _parse_marker(phase1, "ROLLBACK-BROKEN") undo = _parse_marker(phase1, "ROLLBACK-UNDO") if before is None: results.fail("rollback-known-good", "the guest did not record the known-good state") if broken is None or broken.get("pkgs") == (before or {}).get("pkgs"): results.fail("rollback-break-broke", f"the controlled break did not change the package database: {broken}") return if undo is None or undo.get("rc") != "0": results.fail("rollback-undo", f"sys undo did not succeed: {undo}") # No rollback happened: a detached "verify" would compare identical # states and report vacuous passes. Stop instead of pretending. return # Phase 2: boot the disk detached and verify the restored generation. # The phase-1 transcript is preserved under its own name first: reopening # the same log would destroy the evidence for why phase 1 passed/failed. phase1_log = paths.serial_log.with_name("serial-phase1.log") paths.serial_log.rename(phase1_log) env, args = _installed_args(paths, disks) log = open(paths.serial_log, "w") child = spawn(args, env, log) phase2 = "" try: _unlock_and_login(child, results, "rollback-after") _fetch_sys(child) if not _sudo_auth(child, results, "rollback-after"): return child.sendline(f"curl -fsSL {ROLLBACK_VERIFY_URL} -o /tmp/rbv.sh && sudo bash /tmp/rbv.sh") child.expect("===AFTER-DONE===", timeout=int(os.environ.get("ROLLBACK_VERIFY_TIMEOUT", "900"))) phase2 = paths.serial_log.read_text(errors="replace") results.assert_true("rollback-systemd-running", "===AFTER-SYSTEMD running===" in phase2, "systemd did not report running after rollback") after = _parse_marker(phase2, "AFTER-KNOWN-GOOD") if before is None or after is None: results.fail("rollback-state", "could not read the before/after state") else: results.assert_true( "rollback-package-state-restored", before.get("pkgs") == after.get("pkgs"), f"package db changed across rollback: before={before.get('pkgs')} after={after.get('pkgs')}", ) results.assert_true( "rollback-home-survives", before.get("home") == after.get("home"), f"/home did not survive: before={before.get('home')} after={after.get('home')}", ) results.assert_true("rollback-modules-present", "===AFTER-MODULES ok===" in phase2, "no kernel modules for the running kernel") # The break wrote this marker after our snapshot; the targeted undo # restores our snapshot exactly, so the marker must be gone. Its # survival would prove the wrong generation was restored. results.assert_true("rollback-root-restored", "===AFTER-BROKEN-MARKER absent===" in phase2, "the root break marker survived the rollback") results.assert_true( "rollback-recovery-entry", "systemd.unit=rescue.target" in phase2 and "@snapshots/" in phase2, "the snapshot recovery entry is missing or not snapshot-based", ) status = phase2.split("===AFTER-STATUS===", 1) status_text = status[1].split("===AFTER-DONE===", 1)[0].strip() if len(status) > 1 else "" results.assert_true("rollback-status-recorded", bool(status_text), "sys rollback status produced no evidence") except pexpect.TIMEOUT as exc: results.fail("rollback-after", f"timeout: {exc}") except pexpect.EOF: results.fail("rollback-after", "unexpected EOF") finally: close_quietly(child) log.close() def do_boot(results, paths, disks): env, args = base_args(str(paths.ovmf_vars)) for disk in disks: args += ["-drive", f"file={disk},if=virtio,format=qcow2"] args += ["-boot", "order=c"] # The end marker is built with a command substitution so the shell's echo of # the typed line does not itself contain it; otherwise `expect` would match # the echo and close the console before the checks ran. checks = os.environ.get( "BOOT_CHECKS", "echo ===CHECKS-$(printf START)===; " "echo COMM_CHECK:$(cat /proc/1/comm 2>&1); " "echo SYSTEMD_STATUS:$(systemctl is-system-running 2>&1); " "systemctl --failed --no-pager; ip -brief addr; " "lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS; echo ===CHECKS-$(printf END)===", ) log = open(paths.serial_log, "w") child = spawn(args, env, log) try: first_prompt = child.expect([r"[Pp]assphrase", r"login:"], timeout=300) login_ready = first_prompt == 1 if first_prompt == 0: child.sendline(LUKS_PW) logged_in = False for _ in range(5): if not login_ready: child.expect(r"phoebus login:", timeout=180) login_ready = False child.sendline(USER) idx = child.expect([r"[Pp]assword:", r"phoebus login:", r"Login incorrect"], timeout=30) if idx == 0: child.sendline(USER_PW) login_result = child.expect([SHELL_PROMPT, r"Login incorrect", r"phoebus login:"], timeout=60) if login_result == 0: logged_in = True break login_ready = login_result == 2 else: login_ready = idx == 1 results.assert_true("detached-boot-login", logged_in, "could not log in") if os.environ.get("BOOT_AUTHENTICATE") == "1": child.sendline("sudo -v") idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", SHELL_PROMPT], timeout=30) if idx == 0: child.sendline(USER_PW) child.expect(SHELL_PROMPT, timeout=30) child.sendline(checks) child.expect("===CHECKS-END===", timeout=int(os.environ.get("CHECKS_TIMEOUT", "180"))) serial = paths.serial_log.read_text(errors="replace") checks_match = re.search(r"===CHECKS-START===(.*?)===CHECKS-END===", serial, re.DOTALL) checks_text = checks_match.group(1) if checks_match else serial m = re.search(r"SYSTEMD_STATUS:([a-z\-]+)", checks_text) status = m.group(1) if m else "unknown" results.assert_true( "boot-systemd-running", status in ("running", "degraded"), f"systemd did not report running/degraded (observed: {status})", ) for case in os.environ.get("BOOT_REQUIRE_CASES", "").split(","): case = case.strip() if case: results.assert_true( f"boot-case-{case}", f"CASE {case} PASS" in checks_text, f"{case} did not report PASS", ) if os.environ.get("BOOT_POWEROFF") == "1": child.sendline("sudo systemctl poweroff") idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", pexpect.EOF], timeout=60) if idx == 0: child.sendline(USER_PW) child.expect(pexpect.EOF, timeout=120) results.pass_("boot-clean-poweroff") except pexpect.TIMEOUT as exc: results.fail("boot", f"timeout: {exc}") except pexpect.EOF: results.fail("boot", "unexpected EOF") finally: close_quietly(child) log.close() def main() -> int: mode = sys.argv[1] if len(sys.argv) > 1 else "" if mode not in ("install", "boot", "rollback"): print(__doc__) return 2 results = Results(f"serial-{mode}") paths = vmsafety.make_run_paths( f"serial-{mode}", base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"), ovmf_vars_template=OVMF_VARS_TEMPLATE, ) try: disks = vmsafety.validate_writable_drives(DISKS.split(","), disposable_dir()) except vmsafety.UnsafeDiskError as exc: results.blocked("disposable-disk", str(exc)) results.write(paths.results_json) print(f"harness error: {exc}", file=sys.stderr) return 2 results.pass_("disposable-disk", ", ".join(str(d) for d in disks)) try: if mode == "install": do_install(results, paths, disks) elif mode == "rollback": do_rollback(results, paths, disks) else: do_boot(results, paths, disks) finally: results.write(paths.results_json) print(results.summary(), flush=True) return results.exit_code() if __name__ == "__main__": sys.exit(main())