phoebus_os/installer/tests/vm/serial-driver.py
Phoebus b8152054b6
Some checks are pending
ci / test (push) Waiting to run
fix(installer): harden boot recovery and desktop reliability
2026-10-01 07:03:42 -07:00

866 lines
40 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Headless serial driver for the disposable UEFI VM.
Modes:
install : boot the live ISO kernel/initrd, run a guest script that installs
Phoebus non-interactively, then power off.
boot : boot the installed disk alone (no ISO, no -kernel), unlock LUKS,
log in and run verification commands.
boot-assessed: detached boot plus the boot-complete health gate and mark-good.
rollback: boot an installed disk, record a known-good state, create a
recovery point through `sys snapshot`, introduce a reversible
update, run the real `sys undo`, power off and boot detached to
verify the restored root/var and package state. Destructive: point
DISK at a disposable clone, never a physical device.
recovery: select the snapshot recovery BLS entry for the next boot, then
boot it with no network and verify the mounted snapshot and sys.
fallback: on a qcow2 overlay, force three incomplete normal boots and
verify automatic selection of the recovery snapshot on attempt 4.
interrupted-update: kill a real sys/emerge transaction on an overlay, undo
its pre-update snapshot, and verify package state after reboot.
install-repeat: independent install to DISK_SECOND from the same ISO.
Everything is parameterised through environment variables so the script is not
tied to one host. See tests/vm/README.md for the variables.
Safety: `DISK`/`DISKS` must be qcow2 files inside the disposable directory named
by `PHOEBUS_VM_DISKS` (default `$PHOEBUS_VM_STATE/disks`); block devices,
`/dev/*`, escaping symlinks, raw images and outside backing chains are refused.
Results are structured and the exit status is nonzero on failure.
"""
import os
import re
import shlex
import subprocess
import sys
import time
import pexpect
import vmsafety
from vmsafety import Results
QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64")
LIBDIR = os.environ.get("QEMU_LIBDIR", "")
DATADIR = os.environ.get("QEMU_DATADIR", "")
OVMF_CODE = os.environ["OVMF_CODE"]
OVMF_VARS_TEMPLATE = os.environ["OVMF_VARS"]
DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "")
ISO = os.environ.get("ISO", "")
KERNEL = os.environ.get("KERNEL", "")
INITRD = os.environ.get("INITRD", "")
LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test")
USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test")
USER = os.environ.get("USERNAME", "phoebus")
LIVE_USER = os.environ.get("LIVE_USER", "phoebus")
LIVE_PW = os.environ.get("LIVE_PASSWORD", "phoebus")
MEM = os.environ.get("VM_MEM", "8192")
SMP = os.environ.get("VM_SMP", "8")
# Shell-prompt alternative for an installed Phoebus login. The default login
# shell is zsh with a powerline-style prompt ending in the `❯` glyph; color
# escapes sit between the glyph and the trailing space, so the bare glyph
# (not `❯ `) is matched. Bash/root prompts keep matching `[$#] `.
SHELL_PROMPT = r"[$#] |❯"
# The ISO is a Debian live medium. Its GRUB entry uses live-boot, so the
# direct-kernel serial path must pass the same root-discovery arguments.
BLS_APPEND = "boot=live components console=ttyS0,115200 noautologin"
# Rollback acceptance fixtures, served from the host over QEMU user networking.
ROLLBACK_URL = os.environ.get("ROLLBACK_URL", "http://10.0.2.2:8000/rollback-guest.sh")
ROLLBACK_VERIFY_URL = os.environ.get(
"ROLLBACK_VERIFY_URL", "http://10.0.2.2:8000/rollback-verify.sh"
)
INTERRUPT_URL = os.environ.get(
"INTERRUPT_URL", "http://10.0.2.2:8000/interrupted-update.sh"
)
SYS_URL = os.environ.get("SYS_URL", "http://10.0.2.2:8000/sys")
def disposable_dir():
import pathlib
base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")
return pathlib.Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks"))
def base_args(ovmf_vars, network=True):
env = dict(os.environ)
if LIBDIR:
env["LD_LIBRARY_PATH"] = LIBDIR
args = ["-machine", "q35,accel=kvm"]
if DATADIR:
args += ["-L", DATADIR]
args += [
"-cpu", "host", "-smp", SMP, "-m", MEM,
"-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}",
"-drive", f"if=pflash,format=raw,file={ovmf_vars}",
"-device", "virtio-rng-pci", "-vga", "std", "-display", "none",
"-serial", "stdio", "-monitor", "none", "-no-reboot",
]
if network:
args += ["-netdev", "user,id=n0", "-device", "virtio-net-pci,netdev=n0,romfile="]
return env, args
def spawn(args, env, log):
child = pexpect.spawn(QEMU, args, env=env, timeout=600,
encoding="utf-8", codec_errors="replace",
maxread=65536)
child.delaybeforesend = 0.3
# Log only guest output, and redact credentials if a login race makes a
# password reach an echoing shell instead of a no-echo password prompt.
child.logfile_read = vmsafety.RedactingTextLog(
log,
[LUKS_PW, USER_PW, LIVE_PW,
os.environ.get("OPENAI_API_KEY"), os.environ.get("DEEPSEEK_API_KEY")],
)
return child
def close_quietly(child):
"""Best-effort teardown. After the guest powers off, pexpect can raise
`ExceptionPexpect: Could not terminate the child`; that must not turn a
successful run into a nonzero driver exit."""
redactor = child.logfile_read
try:
child.close(force=True)
except Exception: # noqa: BLE001 - cleanup must never mask the result
pass
if isinstance(redactor, vmsafety.RedactingTextLog):
redactor.finish()
def do_install(results, paths, disks):
config_name = os.environ.get("VM_INSTALL_CONFIG", "test-mango.conf")
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", config_name):
results.blocked("install-config", "VM_INSTALL_CONFIG must be a safe file name")
return
env, args = base_args(str(paths.ovmf_vars))
for disk in disks:
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
args += ["-drive", f"file={ISO},media=cdrom,readonly=on",
"-kernel", KERNEL, "-initrd", INITRD, "-append", BLS_APPEND]
log = open(paths.serial_log, "w")
child = spawn(args, env, log)
try:
idx = child.expect([r"root@livecd[^\n]*# ", r"(?m)^[^\r\n]* login:"], timeout=600)
if idx == 1:
child.sendline(LIVE_USER)
child.expect(r"[Pp]assword:", timeout=90)
child.sendline(LIVE_PW)
child.expect(r"[$#] ", timeout=180)
child.sendline("sudo -v; echo SUDO-$(printf READY)")
choice = child.expect([r"[Pp]assword(?: for [^:]+)?:", "SUDO-READY"], timeout=60)
if choice == 0:
child.sendline(LIVE_PW)
child.expect("SUDO-READY", timeout=60)
child.sendline("sudo -n true && echo ROOT-$(printf READY)")
child.expect("ROOT-READY", timeout=30)
results.pass_("live-shell", "authenticated live user with sudo")
else:
results.pass_("live-shell", "root live shell")
url = os.environ.get("INSTALL_URL", "http://10.0.2.2:8000/install.sh")
command = f"curl -fsSL {shlex.quote(url)} -o /tmp/run.sh && "
run = f"env PHOEBUS_CONFIG={shlex.quote(config_name)} bash /tmp/run.sh"
command += f"sudo -n {run}" if idx == 1 else run
child.sendline(command)
child.expect([r"===INSTALL-RC=([0-9]+)===", "===FETCH-FAIL===",
"Kernel panic", pexpect.EOF, pexpect.TIMEOUT], timeout=19800)
# `child.after` is the matched pattern class (not a string) when EOF or
# TIMEOUT matched; handle that instead of raising a TypeError.
matched = child.after if isinstance(child.after, str) else ""
if "INSTALL-RC" in matched:
rc = child.match.group(1)
results.assert_true("install-rc-zero", rc == "0", f"INSTALL-RC={rc}")
else:
results.fail(
"install-rc-zero",
f"install did not report a status (matched {matched!r})",
)
serial = paths.serial_log.read_text(errors="replace")
results.assert_true("no-secret-leak", "SECRET_LEAK_FOUND" not in serial,
"the guest reported a secret leak")
shutdown = child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600)
if shutdown == 0:
child.expect(pexpect.EOF, timeout=180)
results.assert_true("guest-poweroff", shutdown != 2,
"installer returned but the live guest did not power off")
except pexpect.TIMEOUT as exc:
results.fail("install", f"timeout: {exc}")
except pexpect.EOF:
results.fail("install", "unexpected EOF")
finally:
close_quietly(child)
log.close()
def _installed_args(paths, disks, network=True):
env, args = base_args(str(paths.ovmf_vars), network=network)
for disk in disks:
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
args += ["-boot", "order=c"]
return env, args
def _unlock_and_login(child, results, tag, continue_locked_root=False):
child.expect([r"[Pp]assphrase", r"login:"], timeout=300)
if "assphrase" in (child.after or ""):
override = os.environ.get("BOOT_PASSPHRASE")
candidates = [override] if override else [USER_PW, LUKS_PW]
child.sendline(candidates[0])
for candidate in candidates[1:]:
try:
child.expect(r"[Pp]assphrase", timeout=15)
child.sendline(candidate)
except (pexpect.TIMEOUT, pexpect.EOF):
break
login_ready = False
if continue_locked_root:
# The default root account is locked. rescue.target's sulogin prints
# this instruction; Enter continues to the ordinary user getty on
# the read-only recovery snapshot.
outcome = child.expect([r"Press Enter to continue", r"phoebus login:"], timeout=300)
if outcome == 0:
child.sendline("")
# rescue.target hands control back to default.target. The serial
# getty can restart a few times during that transition; wait for
# the final login prompt before sending credentials.
time.sleep(5)
else:
login_ready = True
logged_in = False
for _ in range(5):
if not login_ready:
child.expect(r"phoebus login:", timeout=180)
login_ready = False
# During boot and the rescue-to-default transition, systemd may
# restart serial-getty several times. Credentials sent to an early
# prompt vanish when that agetty exits; let the final prompt settle.
time.sleep(5)
child.sendline(USER)
idx = child.expect([r"[Pp]assword:", r"phoebus login:", r"Login incorrect"],
timeout=30)
if idx == 0:
child.sendline(USER_PW)
if child.expect([SHELL_PROMPT, r"Login incorrect", r"phoebus login:"],
timeout=60) == 0:
logged_in = True
break
results.assert_true(f"{tag}-login", logged_in, f"{tag}: could not log in")
def _sudo_auth(child, results, tag):
"""Authenticate sudo for the rollback acceptance guest scripts.
The snapshot/break/undo/verify steps mutate the system and must run as
root. The password is sent only at sudo's own prompt (terminal echo
disabled, like the login password), so it never lands in the serial
transcript the way a command-line credential would. Returns True when the
timestamp stuck; the caller must abort the phase otherwise, or every
later `sudo` would block on a prompt nobody answers.
"""
try:
# Unique markers, never a bare `[$#] ` alternative: the buffer still
# holds the previous shell prompt when this runs, and a generic
# prompt pattern would match that stale text immediately — skipping
# the password while sudo is still starting (the next typed line
# would then be consumed as the password and rejected).
child.sendline("sudo -v; echo SUDO-V-$(printf DONE)")
idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", "SUDO-V-DONE"], timeout=60)
if idx == 0:
child.sendline(USER_PW)
child.expect("SUDO-V-DONE", timeout=60)
# Prove the timestamp stuck: without it every later `sudo` would block
# on a password prompt nobody answers, hanging the run until timeout.
child.sendline("sudo -n true && echo SUDO-OK-$(printf READY)")
idx = child.expect(["SUDO-OK-READY", r"[Pp]assword"], timeout=30)
except (pexpect.TIMEOUT, pexpect.EOF):
idx = -1
if idx != 0:
results.fail(f"{tag}-sudo", "sudo authentication did not stick")
return False
return True
def _fetch_sys(child):
# The installed system should already carry /usr/local/bin/sys; only fall
# back to fetching the host build when it is absent.
child.sendline(
"if [ -x /usr/local/bin/sys ]; then SYS=/usr/local/bin/sys; "
"elif [ -x /tmp/sys ]; then SYS=/tmp/sys; "
f"else curl -fsSL {SYS_URL} -o /tmp/sys && chmod 755 /tmp/sys && SYS=/tmp/sys; fi; "
"test -x \"$SYS\" && echo SYS-OK-$(printf READY)"
)
child.expect("SYS-OK-READY", timeout=120)
def _parse_marker(text, key):
import re
match = re.search(r"===" + re.escape(key) + r"(.+?)===", text)
if not match:
return None
out = {}
for token in match.group(1).split():
if "=" in token:
name, value = token.split("=", 1)
out[name] = value
return out
def do_rollback(results, paths, disks):
"""Snapshot -> reversible update -> sys undo -> detached verify."""
# Phase 1: known-good -> snapshot -> break -> undo, then power off.
env, args = _installed_args(paths, disks)
log = open(paths.serial_log, "w")
child = spawn(args, env, log)
phase1 = ""
try:
_unlock_and_login(child, results, "rollback-before")
_fetch_sys(child)
if not _sudo_auth(child, results, "rollback-before"):
return
child.sendline(f"curl -fsSL {ROLLBACK_URL} -o /tmp/rb.sh && sudo bash /tmp/rb.sh")
child.expect(
["===ROLLBACK-DONE===", "curl:", pexpect.TIMEOUT, pexpect.EOF],
timeout=int(os.environ.get("ROLLBACK_TIMEOUT", "7200")),
)
if "ROLLBACK-DONE" not in (child.after or ""):
results.fail("rollback-sequence", f"the guest rollback did not finish: {child.after!r}")
else:
results.pass_("rollback-sequence")
child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600)
phase1 = paths.serial_log.read_text(errors="replace")
except pexpect.TIMEOUT as exc:
results.fail("rollback-before", f"timeout: {exc}")
except pexpect.EOF:
results.fail("rollback-before", "unexpected EOF")
finally:
close_quietly(child)
log.close()
before = _parse_marker(phase1, "ROLLBACK-KNOWN-GOOD")
broken = _parse_marker(phase1, "ROLLBACK-BROKEN")
undo = _parse_marker(phase1, "ROLLBACK-UNDO")
if before is None:
results.fail("rollback-known-good", "the guest did not record the known-good state")
if broken is None or broken.get("pkgs") == (before or {}).get("pkgs"):
results.fail("rollback-break-broke", f"the controlled break did not change the package database: {broken}")
return
if undo is None or undo.get("rc") != "0":
results.fail("rollback-undo", f"sys undo did not succeed: {undo}")
# No rollback happened: a detached "verify" would compare identical
# states and report vacuous passes. Stop instead of pretending.
return
# Phase 2: boot the disk detached and verify the restored generation.
# The phase-1 transcript is preserved under its own name first: reopening
# the same log would destroy the evidence for why phase 1 passed/failed.
phase1_log = paths.serial_log.with_name("serial-phase1.log")
paths.serial_log.rename(phase1_log)
env, args = _installed_args(paths, disks)
log = open(paths.serial_log, "w")
child = spawn(args, env, log)
phase2 = ""
try:
_unlock_and_login(child, results, "rollback-after")
_fetch_sys(child)
if not _sudo_auth(child, results, "rollback-after"):
return
child.sendline(f"curl -fsSL {ROLLBACK_VERIFY_URL} -o /tmp/rbv.sh && sudo bash /tmp/rbv.sh")
child.expect("===AFTER-DONE===",
timeout=int(os.environ.get("ROLLBACK_VERIFY_TIMEOUT", "900")))
phase2 = paths.serial_log.read_text(errors="replace")
results.assert_true("rollback-systemd-running", "===AFTER-SYSTEMD running===" in phase2,
"systemd did not report running after rollback")
after = _parse_marker(phase2, "AFTER-KNOWN-GOOD")
if before is None or after is None:
results.fail("rollback-state", "could not read the before/after state")
else:
results.assert_true(
"rollback-package-state-restored",
before.get("pkgs") == after.get("pkgs"),
f"package db changed across rollback: before={before.get('pkgs')} after={after.get('pkgs')}",
)
results.assert_true(
"rollback-home-survives",
before.get("home") == after.get("home"),
f"/home did not survive: before={before.get('home')} after={after.get('home')}",
)
results.assert_true("rollback-modules-present", "===AFTER-MODULES ok===" in phase2,
"no kernel modules for the running kernel")
# The break wrote this marker after our snapshot; the targeted undo
# restores our snapshot exactly, so the marker must be gone. Its
# survival would prove the wrong generation was restored.
results.assert_true("rollback-root-restored", "===AFTER-BROKEN-MARKER absent===" in phase2,
"the root break marker survived the rollback")
results.assert_true(
"rollback-recovery-entry",
"systemd.unit=rescue.target" in phase2 and "@snapshots/" in phase2,
"the snapshot recovery entry is missing or not snapshot-based",
)
status = phase2.split("===AFTER-STATUS===", 1)
status_text = status[1].split("===AFTER-DONE===", 1)[0].strip() if len(status) > 1 else ""
results.assert_true("rollback-status-recorded", bool(status_text),
"sys rollback status produced no evidence")
except pexpect.TIMEOUT as exc:
results.fail("rollback-after", f"timeout: {exc}")
except pexpect.EOF:
results.fail("rollback-after", "unexpected EOF")
finally:
close_quietly(child)
log.close()
def do_recovery(results, paths, disks):
"""Select the actual boot menu entry, then prove its snapshot offline."""
env, args = _installed_args(paths, disks, network=False)
args += ["-qmp", f"unix:{paths.monitor_sock},server,nowait"]
log = open(paths.serial_log, "w")
child = spawn(args, env, log)
try:
child.expect(r"Phoebus Recovery \(known-good snapshot\)", timeout=60)
# The first key freezes systemd-boot's short timeout; the second moves
# from the normal entry to recovery. QMP drives the video keyboard,
# while the LUKS passphrase still goes to the serial console.
with vmsafety.QmpClient(paths.monitor_sock) as qmp:
qmp.sendkey("spc")
qmp.sendkey("down")
qmp.sendkey("ret")
results.pass_("recovery-entry-selected")
_unlock_and_login(child, results, "recovery-offline", continue_locked_root=True)
child.sendline(
"setopt NO_EQUALS 2>/dev/null; "
"echo ===RECOVERY-$(printf START)===; "
"echo CMDLINE:$(cat /proc/cmdline); "
"echo ROOT_SOURCE:$(findmnt -no SOURCE /); "
"echo ROOT_OPTIONS:$(findmnt -no OPTIONS /); "
"echo ROOT_PROPERTY:$(btrfs property get / ro 2>&1); "
"sys history --legacy >/tmp/phoebus-recovery-history 2>&1; "
"echo SYS_HISTORY_RC:$?; "
"sys rollback status --legacy >/tmp/phoebus-recovery-status 2>&1; "
"echo SYS_ROLLBACK_RC:$?; "
"echo ===RECOVERY-$(printf END)==="
)
child.expect("===RECOVERY-END===", timeout=180)
observed_output = child.before + "===RECOVERY-END==="
match = re.search(r"===RECOVERY-START===(.*?)===RECOVERY-END===",
observed_output, re.DOTALL)
observed = match.group(1) if match else ""
results.assert_true(
"recovery-snapshot-selected",
"rootflags=subvol=@snapshots/" in observed
and "systemd.unit=rescue.target" in observed,
"recovery kernel command line did not select the snapshot",
)
results.assert_true(
"recovery-snapshot-mounted",
"ROOT_SOURCE:/dev/mapper/cryptroot[/@snapshots/" in observed
and "ro=true" in observed,
"root is not the read-only recovery subvolume",
)
results.assert_true(
"recovery-sys-offline",
"SYS_HISTORY_RC:0" in observed and "SYS_ROLLBACK_RC:0" in observed,
"deterministic sys commands did not work offline",
)
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
results.fail("recovery-offline", str(exc))
finally:
close_quietly(child)
log.close()
def _gate_overlay(results, paths, disks, tag):
"""Create a writable disposable overlay without changing the install disk."""
if len(disks) != 1:
results.blocked(f"{tag}-disk", f"{tag} needs exactly one installed qcow2")
return None
overlay = disposable_dir() / f"{tag}-{paths.run_id}.qcow2"
create = subprocess.run(
["qemu-img", "create", "-f", "qcow2", "-F", "qcow2", "-b",
str(disks[0].resolve()), str(overlay)],
capture_output=True, text=True, check=False,
)
if create.returncode != 0:
results.fail(f"{tag}-overlay", create.stderr.strip() or "qemu-img create failed")
return None
try:
test_disks = vmsafety.validate_writable_drives([str(overlay)], disposable_dir())
except vmsafety.UnsafeDiskError as exc:
results.fail(f"{tag}-overlay", str(exc))
return None
results.pass_(f"{tag}-overlay", str(overlay))
return test_disks
def do_fallback(results, paths, disks):
"""Prove that exhausted normal boot attempts select recovery by default."""
test_disks = _gate_overlay(results, paths, disks, "fallback")
if test_disks is None:
return
# Prepare the overlay from a healthy boot. The next three boots stop at
# rescue.target's locked-root prompt and QEMU is terminated there. Pressing
# Enter would continue to default.target, which can bless the boot and
# clear the counter; a hard stop models the failed boot we need to test.
env, args = _installed_args(paths, test_disks, network=False)
with open(paths.directory / "serial-prepare.log", "w") as log:
child = spawn(args, env, log)
try:
_unlock_and_login(child, results, "fallback-prepare")
if not _sudo_auth(child, results, "fallback-prepare"):
return
child.sendline(
"entry=$(sudo -n find /efi/loader/entries -maxdepth 1 -type f -name '*.conf' "
"! -name '*recovery*' | sort | head -1); "
"test -n \"$entry\" && "
"test \"$(sudo -n awk '$1 == \"default\" {print $2}' /efi/loader/loader.conf)\" = '*' "
"&& sudo -n grep -Eq '^sort-key[[:space:]]+phoebus$' \"$entry\" && "
"sudo -n sed -i "
"'/^options[[:space:]]/ s/$/ systemd.unit=rescue.target/' \"$entry\" && "
"base=${entry%.conf}; base=${base%%+*}; "
"sudo -n mv \"$entry\" \"${base}+3.conf\" && "
"echo FALLBACK-$(printf PREPARED)"
)
child.expect("FALLBACK-PREPARED", timeout=60)
child.sendline("sudo -n systemctl poweroff")
child.expect(pexpect.EOF, timeout=120)
results.pass_("fallback-prepared")
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
results.fail("fallback-prepare", str(exc))
return
finally:
close_quietly(child)
for attempt in range(1, 4):
env, args = _installed_args(paths, test_disks, network=False)
log_path = paths.directory / f"serial-attempt-{attempt}.log"
with open(log_path, "w") as log:
child = spawn(args, env, log)
try:
child.expect(r"[Pp]assphrase", timeout=300)
child.sendline(LUKS_PW)
child.expect("Press Enter to continue", timeout=300)
results.pass_(f"fallback-attempt-{attempt}-failed-boot")
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
results.fail(f"fallback-attempt-{attempt}", str(exc))
return
finally:
close_quietly(child)
# QEMU can release the qcow2 write lock shortly after pexpect closes
# its PTY. Give the next boot a brief teardown window.
time.sleep(3)
env, args = _installed_args(paths, test_disks, network=False)
with open(paths.serial_log, "w") as log:
child = spawn(args, env, log)
try:
_unlock_and_login(child, results, "fallback-recovery",
continue_locked_root=True)
if not _sudo_auth(child, results, "fallback-recovery"):
return
child.sendline(
"setopt NO_EQUALS 2>/dev/null; echo ===FALLBACK-$(printf START)===; "
"cat /proc/cmdline; sudo -n ls -1 /efi/loader/entries; "
"echo ===FALLBACK-$(printf END)==="
)
child.expect("===FALLBACK-END===", timeout=60)
observed_output = child.before + "===FALLBACK-END==="
match = re.search(r"===FALLBACK-START===(.*?)===FALLBACK-END===",
observed_output, re.DOTALL)
observed = match.group(1) if match else ""
correct = ("rootflags=subvol=@snapshots/" in observed
and "systemd.unit=rescue.target" in observed
and re.search(r"\+0-3\.conf", observed) is not None)
results.assert_true("fallback-boot-4", correct,
f"recovery was not selected after three failures: {observed.strip()}")
if not correct:
return
child.sendline("sudo -n systemctl poweroff")
child.expect(pexpect.EOF, timeout=120)
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
results.fail("fallback-recovery", str(exc))
finally:
close_quietly(child)
def do_interrupted_update(results, paths, disks):
"""Kill an in-flight package install, undo its snapshot, and reboot."""
test_disks = _gate_overlay(results, paths, disks, "interrupted-update")
if test_disks is None:
return
env, args = _installed_args(paths, test_disks)
phase1_log = paths.directory / "serial-interrupted-phase1.log"
with open(phase1_log, "w") as log:
child = spawn(args, env, log)
try:
_unlock_and_login(child, results, "interrupted-before")
if not _sudo_auth(child, results, "interrupted-before"):
return
child.sendline(
f"curl -fsSL {INTERRUPT_URL} -o /tmp/interrupted-update.sh "
"&& sudo -n bash /tmp/interrupted-update.sh"
)
outcome = child.expect(
["===INTERRUPT-POWEROFF===", "===INTERRUPT-NEEDS-ROOT===",
"===INTERRUPT-PRECONDITION-FAILED===", "===INTERRUPT-SNAPSHOT-FAILED===",
"===INTERRUPT-NO-ACTIVE-UPDATE===", "curl:", pexpect.EOF],
timeout=1800,
)
if outcome != 0:
results.fail("interrupted-update-phase1",
f"guest stopped before recovery: {child.after!r}")
return
child.expect(pexpect.EOF, timeout=120)
log.flush()
phase1 = phase1_log.read_text(errors="replace")
before = _parse_marker(phase1, "INTERRUPT-BEFORE")
recovery = _parse_marker(phase1, "INTERRUPT-RECOVERY")
results.assert_true("interrupted-update-killed",
"===INTERRUPT-SIGKILL-DURING-EMERGE===" in phase1,
"the update process was not killed during emerge")
results.assert_true("interrupted-update-undone",
recovery is not None and recovery.get("history_rc") == "0"
and recovery.get("undo_rc") == "0",
"sys did not recover its lock and undo the snapshot")
if before is None or recovery is None or recovery.get("undo_rc") != "0":
return
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
results.fail("interrupted-update-phase1", str(exc))
return
finally:
close_quietly(child)
env, args = _installed_args(paths, test_disks, network=False)
with open(paths.serial_log, "w") as log:
child = spawn(args, env, log)
try:
_unlock_and_login(child, results, "interrupted-after")
child.sendline(
"setopt NO_EQUALS 2>/dev/null; echo ===INTERRUPT-$(printf VERIFY-START)===; "
"echo SYSTEMD:$(systemctl is-system-running); "
"echo PKGS:$(find /var/db/pkg -mindepth 2 -maxdepth 2 -type d 2>/dev/null "
"| sort | sha256sum | awk '{print $1}'); "
"echo HOME:$(cat /home/phoebus/interrupted-update-marker.txt); "
"echo HELLO:$(find /var/db/pkg/app-misc -maxdepth 1 -name 'hello-*' "
"-print -quit | grep -q . && echo yes || echo no); "
"sys history --legacy >/tmp/interrupted-history 2>&1; echo SYS_RC:$?; "
"echo ===INTERRUPT-$(printf VERIFY-END)==="
)
child.expect("===INTERRUPT-VERIFY-END===", timeout=120)
log.flush()
transcript = paths.serial_log.read_text(errors="replace")
match = re.search(r"===INTERRUPT-VERIFY-START===(.*?)===INTERRUPT-VERIFY-END===",
transcript, re.DOTALL)
observed = match.group(1) if match else ""
results.assert_true("interrupted-update-boot",
"SYSTEMD:running" in observed or "SYSTEMD:degraded" in observed,
"the restored system did not boot")
results.assert_true("interrupted-update-package-state",
f"PKGS:{before['pkgs']}" in observed and "HELLO:no" in observed,
"package database did not match the pre-update snapshot")
results.assert_true("interrupted-update-home",
f"HOME:{before['home']}" in observed,
"the user home marker was not preserved")
results.assert_true("interrupted-update-sys", "SYS_RC:0" in observed,
"deterministic sys failed after the interrupted update")
if _sudo_auth(child, results, "interrupted-after"):
child.sendline("sudo -n systemctl poweroff")
child.expect(pexpect.EOF, timeout=120)
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
results.fail("interrupted-update-phase2", str(exc))
finally:
close_quietly(child)
def do_boot(results, paths, disks, assessment=False):
env, args = base_args(str(paths.ovmf_vars))
for disk in disks:
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
args += ["-boot", "order=c"]
# The end marker is built with a command substitution so the shell's echo of
# the typed line does not itself contain it; otherwise `expect` would match
# the echo and close the console before the checks ran.
checks = os.environ.get(
"BOOT_CHECKS",
# `setopt NO_EQUALS` first: the installed default shell is zsh, where
# a leading `=` triggers command-path expansion and the `===MARKER===`
# convention would abort the whole line as "not found". The redirect
# swallows the error on shells without `setopt` (bash live envs).
"setopt NO_EQUALS 2>/dev/null; "
"echo ===CHECKS-$(printf START)===; "
"echo COMM_CHECK:$(cat /proc/1/comm 2>&1); "
"echo SYSTEMD_STATUS:$(systemctl is-system-running 2>&1); "
"systemctl --failed --no-pager; ip -brief addr; "
"lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS; echo ===CHECKS-$(printf END)===",
)
log = open(paths.serial_log, "w")
child = spawn(args, env, log)
try:
first_prompt = child.expect([r"[Pp]assphrase", r"login:"], timeout=300)
login_ready = first_prompt == 1
if first_prompt == 0:
child.sendline(LUKS_PW)
next_prompt = child.expect(
[r"phoebus login:", r"Press Enter to continue"], timeout=300
)
if next_prompt == 0:
login_ready = True
else:
child.sendline("")
time.sleep(5)
logged_in = False
for _ in range(5):
if not login_ready:
child.expect(r"phoebus login:", timeout=180)
login_ready = False
time.sleep(5)
child.sendline(USER)
idx = child.expect([r"[Pp]assword:", r"phoebus login:",
r"Login incorrect"], timeout=30)
if idx == 0:
child.sendline(USER_PW)
login_result = child.expect([SHELL_PROMPT, r"Login incorrect",
r"phoebus login:"], timeout=60)
if login_result == 0:
logged_in = True
break
login_ready = login_result == 2
else:
login_ready = idx == 1
results.assert_true("detached-boot-login", logged_in, "could not log in")
if os.environ.get("BOOT_AUTHENTICATE") == "1":
if not _sudo_auth(child, results, "boot"):
return
child.sendline(checks)
child.expect("===CHECKS-END===", timeout=int(os.environ.get("CHECKS_TIMEOUT", "180")))
# pexpect can match the end marker before the redacting log sink has
# received its trailing newline. Parse the just-consumed output.
observed_checks = child.before + "===CHECKS-END==="
checks_match = re.search(r"===CHECKS-START===(.*?)===CHECKS-END===",
observed_checks, re.DOTALL)
checks_text = checks_match.group(1) if checks_match else observed_checks
m = re.search(r"SYSTEMD_STATUS:([a-z\-]+)", checks_text)
status = m.group(1) if m else "unknown"
results.assert_true(
"boot-systemd-running",
status in ("running", "degraded"),
f"systemd did not report running/degraded (observed: {status})",
)
if assessment:
if not _sudo_auth(child, results, "assessment"):
return
child.sendline(
"setopt NO_EQUALS 2>/dev/null; "
"for i in $(seq 1 120); do "
"systemctl is-active --quiet systemd-bless-boot.service && break; "
"sleep 1; done; "
"echo ===ASSESS-$(printf START)===; "
"echo BLESS:$(systemctl is-active systemd-bless-boot.service); "
"echo CHECK:$(systemctl is-active systemd-boot-check-no-failures.service); "
"echo COMPLETE:$(systemctl is-active boot-complete.target); "
"echo COUNTED:$(sudo -n find /efi/loader/entries -maxdepth 1 "
"-name '*+*.conf' | wc -l); "
"sys history --legacy >/tmp/phoebus-assessed-history 2>&1; echo SYS_RC:$?; "
"echo ===ASSESS-$(printf END)==="
)
child.expect("===ASSESS-END===", timeout=180)
observed_assessment = child.before + "===ASSESS-END==="
match = re.search(r"===ASSESS-START===(.*?)===ASSESS-END===",
observed_assessment, re.DOTALL)
observed = match.group(1) if match else ""
results.assert_true(
"boot-assessment-confirmed",
all(marker in observed for marker in
("BLESS:active", "CHECK:active", "COMPLETE:active",
"COUNTED:0", "SYS_RC:0")),
f"boot health was not confirmed: {observed.strip()}",
)
for case in os.environ.get("BOOT_REQUIRE_CASES", "").split(","):
case = case.strip()
if case:
results.assert_true(
f"boot-case-{case}",
f"CASE {case} PASS" in checks_text,
f"{case} did not report PASS",
)
if assessment or os.environ.get("BOOT_POWEROFF") == "1":
child.sendline("sudo systemctl poweroff")
idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", pexpect.EOF], timeout=60)
if idx == 0:
child.sendline(USER_PW)
child.expect(pexpect.EOF, timeout=120)
results.pass_("boot-clean-poweroff")
except pexpect.TIMEOUT as exc:
results.fail("boot", f"timeout: {exc}")
except pexpect.EOF:
results.fail("boot", "unexpected EOF")
finally:
close_quietly(child)
log.close()
def main() -> int:
mode = sys.argv[1] if len(sys.argv) > 1 else ""
if mode not in ("install", "install-repeat", "boot", "boot-assessed", "rollback", "recovery",
"fallback", "interrupted-update"):
print(__doc__)
return 2
paths = vmsafety.make_run_paths(
f"serial-{mode}", base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"),
ovmf_vars_template=OVMF_VARS_TEMPLATE,
)
results = Results(
f"serial-{mode}", run_id=paths.run_id,
secrets=[LUKS_PW, USER_PW, LIVE_PW,
os.environ.get("OPENAI_API_KEY"), os.environ.get("DEEPSEEK_API_KEY")],
)
selected_disks = DISKS
if mode == "install-repeat":
selected_disks = os.environ.get("DISK_SECOND", "")
if not selected_disks or selected_disks == DISKS:
results.blocked("independent-install-disk", "DISK_SECOND must name a separate disposable qcow2")
results.write(paths.results_json)
return 2
try:
if mode == "install-repeat":
disks = vmsafety.validate_independent_disk_sets(
DISKS.split(","), selected_disks.split(","), disposable_dir()
)
else:
disks = vmsafety.validate_writable_drives(selected_disks.split(","), disposable_dir())
except vmsafety.UnsafeDiskError as exc:
results.blocked("disposable-disk", str(exc))
results.write(paths.results_json)
print(f"harness error: {exc}", file=sys.stderr)
return 2
results.pass_("disposable-disk", ", ".join(str(d) for d in disks))
try:
if mode in ("install", "install-repeat"):
do_install(results, paths, disks)
elif mode == "rollback":
do_rollback(results, paths, disks)
elif mode == "recovery":
do_recovery(results, paths, disks)
elif mode == "fallback":
do_fallback(results, paths, disks)
elif mode == "interrupted-update":
do_interrupted_update(results, paths, disks)
elif mode == "boot-assessed":
do_boot(results, paths, disks, assessment=True)
else:
do_boot(results, paths, disks)
finally:
results.write(paths.results_json)
print(results.summary(), flush=True)
return results.exit_code()
if __name__ == "__main__":
sys.exit(main())