866 lines
40 KiB
Bash
866 lines
40 KiB
Bash
#!/usr/bin/env python3
|
||
"""Headless serial driver for the disposable UEFI VM.
|
||
|
||
Modes:
|
||
|
||
install : boot the live ISO kernel/initrd, run a guest script that installs
|
||
Phoebus non-interactively, then power off.
|
||
boot : boot the installed disk alone (no ISO, no -kernel), unlock LUKS,
|
||
log in and run verification commands.
|
||
boot-assessed: detached boot plus the boot-complete health gate and mark-good.
|
||
rollback: boot an installed disk, record a known-good state, create a
|
||
recovery point through `sys snapshot`, introduce a reversible
|
||
update, run the real `sys undo`, power off and boot detached to
|
||
verify the restored root/var and package state. Destructive: point
|
||
DISK at a disposable clone, never a physical device.
|
||
recovery: select the snapshot recovery BLS entry for the next boot, then
|
||
boot it with no network and verify the mounted snapshot and sys.
|
||
fallback: on a qcow2 overlay, force three incomplete normal boots and
|
||
verify automatic selection of the recovery snapshot on attempt 4.
|
||
interrupted-update: kill a real sys/emerge transaction on an overlay, undo
|
||
its pre-update snapshot, and verify package state after reboot.
|
||
install-repeat: independent install to DISK_SECOND from the same ISO.
|
||
|
||
Everything is parameterised through environment variables so the script is not
|
||
tied to one host. See tests/vm/README.md for the variables.
|
||
|
||
Safety: `DISK`/`DISKS` must be qcow2 files inside the disposable directory named
|
||
by `PHOEBUS_VM_DISKS` (default `$PHOEBUS_VM_STATE/disks`); block devices,
|
||
`/dev/*`, escaping symlinks, raw images and outside backing chains are refused.
|
||
Results are structured and the exit status is nonzero on failure.
|
||
"""
|
||
import os
|
||
import re
|
||
import shlex
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
import pexpect
|
||
|
||
import vmsafety
|
||
from vmsafety import Results
|
||
|
||
QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64")
|
||
LIBDIR = os.environ.get("QEMU_LIBDIR", "")
|
||
DATADIR = os.environ.get("QEMU_DATADIR", "")
|
||
OVMF_CODE = os.environ["OVMF_CODE"]
|
||
OVMF_VARS_TEMPLATE = os.environ["OVMF_VARS"]
|
||
DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "")
|
||
ISO = os.environ.get("ISO", "")
|
||
KERNEL = os.environ.get("KERNEL", "")
|
||
INITRD = os.environ.get("INITRD", "")
|
||
LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test")
|
||
USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test")
|
||
USER = os.environ.get("USERNAME", "phoebus")
|
||
LIVE_USER = os.environ.get("LIVE_USER", "phoebus")
|
||
LIVE_PW = os.environ.get("LIVE_PASSWORD", "phoebus")
|
||
MEM = os.environ.get("VM_MEM", "8192")
|
||
SMP = os.environ.get("VM_SMP", "8")
|
||
|
||
# Shell-prompt alternative for an installed Phoebus login. The default login
|
||
# shell is zsh with a powerline-style prompt ending in the `❯` glyph; color
|
||
# escapes sit between the glyph and the trailing space, so the bare glyph
|
||
# (not `❯ `) is matched. Bash/root prompts keep matching `[$#] `.
|
||
SHELL_PROMPT = r"[$#] |❯"
|
||
|
||
# The ISO is a Debian live medium. Its GRUB entry uses live-boot, so the
|
||
# direct-kernel serial path must pass the same root-discovery arguments.
|
||
BLS_APPEND = "boot=live components console=ttyS0,115200 noautologin"
|
||
|
||
# Rollback acceptance fixtures, served from the host over QEMU user networking.
|
||
ROLLBACK_URL = os.environ.get("ROLLBACK_URL", "http://10.0.2.2:8000/rollback-guest.sh")
|
||
ROLLBACK_VERIFY_URL = os.environ.get(
|
||
"ROLLBACK_VERIFY_URL", "http://10.0.2.2:8000/rollback-verify.sh"
|
||
)
|
||
INTERRUPT_URL = os.environ.get(
|
||
"INTERRUPT_URL", "http://10.0.2.2:8000/interrupted-update.sh"
|
||
)
|
||
SYS_URL = os.environ.get("SYS_URL", "http://10.0.2.2:8000/sys")
|
||
|
||
|
||
def disposable_dir():
|
||
import pathlib
|
||
|
||
base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")
|
||
return pathlib.Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks"))
|
||
|
||
|
||
def base_args(ovmf_vars, network=True):
|
||
env = dict(os.environ)
|
||
if LIBDIR:
|
||
env["LD_LIBRARY_PATH"] = LIBDIR
|
||
args = ["-machine", "q35,accel=kvm"]
|
||
if DATADIR:
|
||
args += ["-L", DATADIR]
|
||
args += [
|
||
"-cpu", "host", "-smp", SMP, "-m", MEM,
|
||
"-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}",
|
||
"-drive", f"if=pflash,format=raw,file={ovmf_vars}",
|
||
"-device", "virtio-rng-pci", "-vga", "std", "-display", "none",
|
||
"-serial", "stdio", "-monitor", "none", "-no-reboot",
|
||
]
|
||
if network:
|
||
args += ["-netdev", "user,id=n0", "-device", "virtio-net-pci,netdev=n0,romfile="]
|
||
return env, args
|
||
|
||
|
||
def spawn(args, env, log):
|
||
child = pexpect.spawn(QEMU, args, env=env, timeout=600,
|
||
encoding="utf-8", codec_errors="replace",
|
||
maxread=65536)
|
||
child.delaybeforesend = 0.3
|
||
# Log only guest output, and redact credentials if a login race makes a
|
||
# password reach an echoing shell instead of a no-echo password prompt.
|
||
child.logfile_read = vmsafety.RedactingTextLog(
|
||
log,
|
||
[LUKS_PW, USER_PW, LIVE_PW,
|
||
os.environ.get("OPENAI_API_KEY"), os.environ.get("DEEPSEEK_API_KEY")],
|
||
)
|
||
return child
|
||
|
||
|
||
def close_quietly(child):
|
||
"""Best-effort teardown. After the guest powers off, pexpect can raise
|
||
`ExceptionPexpect: Could not terminate the child`; that must not turn a
|
||
successful run into a nonzero driver exit."""
|
||
redactor = child.logfile_read
|
||
try:
|
||
child.close(force=True)
|
||
except Exception: # noqa: BLE001 - cleanup must never mask the result
|
||
pass
|
||
if isinstance(redactor, vmsafety.RedactingTextLog):
|
||
redactor.finish()
|
||
|
||
|
||
def do_install(results, paths, disks):
|
||
config_name = os.environ.get("VM_INSTALL_CONFIG", "test-mango.conf")
|
||
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", config_name):
|
||
results.blocked("install-config", "VM_INSTALL_CONFIG must be a safe file name")
|
||
return
|
||
env, args = base_args(str(paths.ovmf_vars))
|
||
for disk in disks:
|
||
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
||
args += ["-drive", f"file={ISO},media=cdrom,readonly=on",
|
||
"-kernel", KERNEL, "-initrd", INITRD, "-append", BLS_APPEND]
|
||
log = open(paths.serial_log, "w")
|
||
child = spawn(args, env, log)
|
||
try:
|
||
idx = child.expect([r"root@livecd[^\n]*# ", r"(?m)^[^\r\n]* login:"], timeout=600)
|
||
if idx == 1:
|
||
child.sendline(LIVE_USER)
|
||
child.expect(r"[Pp]assword:", timeout=90)
|
||
child.sendline(LIVE_PW)
|
||
child.expect(r"[$#] ", timeout=180)
|
||
child.sendline("sudo -v; echo SUDO-$(printf READY)")
|
||
choice = child.expect([r"[Pp]assword(?: for [^:]+)?:", "SUDO-READY"], timeout=60)
|
||
if choice == 0:
|
||
child.sendline(LIVE_PW)
|
||
child.expect("SUDO-READY", timeout=60)
|
||
child.sendline("sudo -n true && echo ROOT-$(printf READY)")
|
||
child.expect("ROOT-READY", timeout=30)
|
||
results.pass_("live-shell", "authenticated live user with sudo")
|
||
else:
|
||
results.pass_("live-shell", "root live shell")
|
||
url = os.environ.get("INSTALL_URL", "http://10.0.2.2:8000/install.sh")
|
||
command = f"curl -fsSL {shlex.quote(url)} -o /tmp/run.sh && "
|
||
run = f"env PHOEBUS_CONFIG={shlex.quote(config_name)} bash /tmp/run.sh"
|
||
command += f"sudo -n {run}" if idx == 1 else run
|
||
child.sendline(command)
|
||
child.expect([r"===INSTALL-RC=([0-9]+)===", "===FETCH-FAIL===",
|
||
"Kernel panic", pexpect.EOF, pexpect.TIMEOUT], timeout=19800)
|
||
# `child.after` is the matched pattern class (not a string) when EOF or
|
||
# TIMEOUT matched; handle that instead of raising a TypeError.
|
||
matched = child.after if isinstance(child.after, str) else ""
|
||
if "INSTALL-RC" in matched:
|
||
rc = child.match.group(1)
|
||
results.assert_true("install-rc-zero", rc == "0", f"INSTALL-RC={rc}")
|
||
else:
|
||
results.fail(
|
||
"install-rc-zero",
|
||
f"install did not report a status (matched {matched!r})",
|
||
)
|
||
serial = paths.serial_log.read_text(errors="replace")
|
||
results.assert_true("no-secret-leak", "SECRET_LEAK_FOUND" not in serial,
|
||
"the guest reported a secret leak")
|
||
shutdown = child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600)
|
||
if shutdown == 0:
|
||
child.expect(pexpect.EOF, timeout=180)
|
||
results.assert_true("guest-poweroff", shutdown != 2,
|
||
"installer returned but the live guest did not power off")
|
||
except pexpect.TIMEOUT as exc:
|
||
results.fail("install", f"timeout: {exc}")
|
||
except pexpect.EOF:
|
||
results.fail("install", "unexpected EOF")
|
||
finally:
|
||
close_quietly(child)
|
||
log.close()
|
||
|
||
|
||
def _installed_args(paths, disks, network=True):
|
||
env, args = base_args(str(paths.ovmf_vars), network=network)
|
||
for disk in disks:
|
||
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
||
args += ["-boot", "order=c"]
|
||
return env, args
|
||
|
||
|
||
def _unlock_and_login(child, results, tag, continue_locked_root=False):
|
||
child.expect([r"[Pp]assphrase", r"login:"], timeout=300)
|
||
if "assphrase" in (child.after or ""):
|
||
override = os.environ.get("BOOT_PASSPHRASE")
|
||
candidates = [override] if override else [USER_PW, LUKS_PW]
|
||
child.sendline(candidates[0])
|
||
for candidate in candidates[1:]:
|
||
try:
|
||
child.expect(r"[Pp]assphrase", timeout=15)
|
||
child.sendline(candidate)
|
||
except (pexpect.TIMEOUT, pexpect.EOF):
|
||
break
|
||
login_ready = False
|
||
if continue_locked_root:
|
||
# The default root account is locked. rescue.target's sulogin prints
|
||
# this instruction; Enter continues to the ordinary user getty on
|
||
# the read-only recovery snapshot.
|
||
outcome = child.expect([r"Press Enter to continue", r"phoebus login:"], timeout=300)
|
||
if outcome == 0:
|
||
child.sendline("")
|
||
# rescue.target hands control back to default.target. The serial
|
||
# getty can restart a few times during that transition; wait for
|
||
# the final login prompt before sending credentials.
|
||
time.sleep(5)
|
||
else:
|
||
login_ready = True
|
||
logged_in = False
|
||
for _ in range(5):
|
||
if not login_ready:
|
||
child.expect(r"phoebus login:", timeout=180)
|
||
login_ready = False
|
||
# During boot and the rescue-to-default transition, systemd may
|
||
# restart serial-getty several times. Credentials sent to an early
|
||
# prompt vanish when that agetty exits; let the final prompt settle.
|
||
time.sleep(5)
|
||
child.sendline(USER)
|
||
idx = child.expect([r"[Pp]assword:", r"phoebus login:", r"Login incorrect"],
|
||
timeout=30)
|
||
if idx == 0:
|
||
child.sendline(USER_PW)
|
||
if child.expect([SHELL_PROMPT, r"Login incorrect", r"phoebus login:"],
|
||
timeout=60) == 0:
|
||
logged_in = True
|
||
break
|
||
results.assert_true(f"{tag}-login", logged_in, f"{tag}: could not log in")
|
||
|
||
|
||
def _sudo_auth(child, results, tag):
|
||
"""Authenticate sudo for the rollback acceptance guest scripts.
|
||
|
||
The snapshot/break/undo/verify steps mutate the system and must run as
|
||
root. The password is sent only at sudo's own prompt (terminal echo
|
||
disabled, like the login password), so it never lands in the serial
|
||
transcript the way a command-line credential would. Returns True when the
|
||
timestamp stuck; the caller must abort the phase otherwise, or every
|
||
later `sudo` would block on a prompt nobody answers.
|
||
"""
|
||
try:
|
||
# Unique markers, never a bare `[$#] ` alternative: the buffer still
|
||
# holds the previous shell prompt when this runs, and a generic
|
||
# prompt pattern would match that stale text immediately — skipping
|
||
# the password while sudo is still starting (the next typed line
|
||
# would then be consumed as the password and rejected).
|
||
child.sendline("sudo -v; echo SUDO-V-$(printf DONE)")
|
||
idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", "SUDO-V-DONE"], timeout=60)
|
||
if idx == 0:
|
||
child.sendline(USER_PW)
|
||
child.expect("SUDO-V-DONE", timeout=60)
|
||
# Prove the timestamp stuck: without it every later `sudo` would block
|
||
# on a password prompt nobody answers, hanging the run until timeout.
|
||
child.sendline("sudo -n true && echo SUDO-OK-$(printf READY)")
|
||
idx = child.expect(["SUDO-OK-READY", r"[Pp]assword"], timeout=30)
|
||
except (pexpect.TIMEOUT, pexpect.EOF):
|
||
idx = -1
|
||
if idx != 0:
|
||
results.fail(f"{tag}-sudo", "sudo authentication did not stick")
|
||
return False
|
||
return True
|
||
|
||
|
||
def _fetch_sys(child):
|
||
# The installed system should already carry /usr/local/bin/sys; only fall
|
||
# back to fetching the host build when it is absent.
|
||
child.sendline(
|
||
"if [ -x /usr/local/bin/sys ]; then SYS=/usr/local/bin/sys; "
|
||
"elif [ -x /tmp/sys ]; then SYS=/tmp/sys; "
|
||
f"else curl -fsSL {SYS_URL} -o /tmp/sys && chmod 755 /tmp/sys && SYS=/tmp/sys; fi; "
|
||
"test -x \"$SYS\" && echo SYS-OK-$(printf READY)"
|
||
)
|
||
child.expect("SYS-OK-READY", timeout=120)
|
||
|
||
|
||
def _parse_marker(text, key):
|
||
import re
|
||
|
||
match = re.search(r"===" + re.escape(key) + r"(.+?)===", text)
|
||
if not match:
|
||
return None
|
||
out = {}
|
||
for token in match.group(1).split():
|
||
if "=" in token:
|
||
name, value = token.split("=", 1)
|
||
out[name] = value
|
||
return out
|
||
|
||
|
||
def do_rollback(results, paths, disks):
|
||
"""Snapshot -> reversible update -> sys undo -> detached verify."""
|
||
# Phase 1: known-good -> snapshot -> break -> undo, then power off.
|
||
env, args = _installed_args(paths, disks)
|
||
log = open(paths.serial_log, "w")
|
||
child = spawn(args, env, log)
|
||
phase1 = ""
|
||
try:
|
||
_unlock_and_login(child, results, "rollback-before")
|
||
_fetch_sys(child)
|
||
if not _sudo_auth(child, results, "rollback-before"):
|
||
return
|
||
child.sendline(f"curl -fsSL {ROLLBACK_URL} -o /tmp/rb.sh && sudo bash /tmp/rb.sh")
|
||
child.expect(
|
||
["===ROLLBACK-DONE===", "curl:", pexpect.TIMEOUT, pexpect.EOF],
|
||
timeout=int(os.environ.get("ROLLBACK_TIMEOUT", "7200")),
|
||
)
|
||
if "ROLLBACK-DONE" not in (child.after or ""):
|
||
results.fail("rollback-sequence", f"the guest rollback did not finish: {child.after!r}")
|
||
else:
|
||
results.pass_("rollback-sequence")
|
||
child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600)
|
||
phase1 = paths.serial_log.read_text(errors="replace")
|
||
except pexpect.TIMEOUT as exc:
|
||
results.fail("rollback-before", f"timeout: {exc}")
|
||
except pexpect.EOF:
|
||
results.fail("rollback-before", "unexpected EOF")
|
||
finally:
|
||
close_quietly(child)
|
||
log.close()
|
||
|
||
|
||
before = _parse_marker(phase1, "ROLLBACK-KNOWN-GOOD")
|
||
broken = _parse_marker(phase1, "ROLLBACK-BROKEN")
|
||
undo = _parse_marker(phase1, "ROLLBACK-UNDO")
|
||
if before is None:
|
||
results.fail("rollback-known-good", "the guest did not record the known-good state")
|
||
if broken is None or broken.get("pkgs") == (before or {}).get("pkgs"):
|
||
results.fail("rollback-break-broke", f"the controlled break did not change the package database: {broken}")
|
||
return
|
||
if undo is None or undo.get("rc") != "0":
|
||
results.fail("rollback-undo", f"sys undo did not succeed: {undo}")
|
||
# No rollback happened: a detached "verify" would compare identical
|
||
# states and report vacuous passes. Stop instead of pretending.
|
||
return
|
||
|
||
# Phase 2: boot the disk detached and verify the restored generation.
|
||
# The phase-1 transcript is preserved under its own name first: reopening
|
||
# the same log would destroy the evidence for why phase 1 passed/failed.
|
||
phase1_log = paths.serial_log.with_name("serial-phase1.log")
|
||
paths.serial_log.rename(phase1_log)
|
||
env, args = _installed_args(paths, disks)
|
||
log = open(paths.serial_log, "w")
|
||
child = spawn(args, env, log)
|
||
phase2 = ""
|
||
try:
|
||
_unlock_and_login(child, results, "rollback-after")
|
||
_fetch_sys(child)
|
||
if not _sudo_auth(child, results, "rollback-after"):
|
||
return
|
||
child.sendline(f"curl -fsSL {ROLLBACK_VERIFY_URL} -o /tmp/rbv.sh && sudo bash /tmp/rbv.sh")
|
||
child.expect("===AFTER-DONE===",
|
||
timeout=int(os.environ.get("ROLLBACK_VERIFY_TIMEOUT", "900")))
|
||
phase2 = paths.serial_log.read_text(errors="replace")
|
||
results.assert_true("rollback-systemd-running", "===AFTER-SYSTEMD running===" in phase2,
|
||
"systemd did not report running after rollback")
|
||
after = _parse_marker(phase2, "AFTER-KNOWN-GOOD")
|
||
if before is None or after is None:
|
||
results.fail("rollback-state", "could not read the before/after state")
|
||
else:
|
||
results.assert_true(
|
||
"rollback-package-state-restored",
|
||
before.get("pkgs") == after.get("pkgs"),
|
||
f"package db changed across rollback: before={before.get('pkgs')} after={after.get('pkgs')}",
|
||
)
|
||
results.assert_true(
|
||
"rollback-home-survives",
|
||
before.get("home") == after.get("home"),
|
||
f"/home did not survive: before={before.get('home')} after={after.get('home')}",
|
||
)
|
||
results.assert_true("rollback-modules-present", "===AFTER-MODULES ok===" in phase2,
|
||
"no kernel modules for the running kernel")
|
||
# The break wrote this marker after our snapshot; the targeted undo
|
||
# restores our snapshot exactly, so the marker must be gone. Its
|
||
# survival would prove the wrong generation was restored.
|
||
results.assert_true("rollback-root-restored", "===AFTER-BROKEN-MARKER absent===" in phase2,
|
||
"the root break marker survived the rollback")
|
||
results.assert_true(
|
||
"rollback-recovery-entry",
|
||
"systemd.unit=rescue.target" in phase2 and "@snapshots/" in phase2,
|
||
"the snapshot recovery entry is missing or not snapshot-based",
|
||
)
|
||
status = phase2.split("===AFTER-STATUS===", 1)
|
||
status_text = status[1].split("===AFTER-DONE===", 1)[0].strip() if len(status) > 1 else ""
|
||
results.assert_true("rollback-status-recorded", bool(status_text),
|
||
"sys rollback status produced no evidence")
|
||
except pexpect.TIMEOUT as exc:
|
||
results.fail("rollback-after", f"timeout: {exc}")
|
||
except pexpect.EOF:
|
||
results.fail("rollback-after", "unexpected EOF")
|
||
finally:
|
||
close_quietly(child)
|
||
log.close()
|
||
|
||
|
||
def do_recovery(results, paths, disks):
|
||
"""Select the actual boot menu entry, then prove its snapshot offline."""
|
||
env, args = _installed_args(paths, disks, network=False)
|
||
args += ["-qmp", f"unix:{paths.monitor_sock},server,nowait"]
|
||
log = open(paths.serial_log, "w")
|
||
child = spawn(args, env, log)
|
||
try:
|
||
child.expect(r"Phoebus Recovery \(known-good snapshot\)", timeout=60)
|
||
# The first key freezes systemd-boot's short timeout; the second moves
|
||
# from the normal entry to recovery. QMP drives the video keyboard,
|
||
# while the LUKS passphrase still goes to the serial console.
|
||
with vmsafety.QmpClient(paths.monitor_sock) as qmp:
|
||
qmp.sendkey("spc")
|
||
qmp.sendkey("down")
|
||
qmp.sendkey("ret")
|
||
results.pass_("recovery-entry-selected")
|
||
_unlock_and_login(child, results, "recovery-offline", continue_locked_root=True)
|
||
child.sendline(
|
||
"setopt NO_EQUALS 2>/dev/null; "
|
||
"echo ===RECOVERY-$(printf START)===; "
|
||
"echo CMDLINE:$(cat /proc/cmdline); "
|
||
"echo ROOT_SOURCE:$(findmnt -no SOURCE /); "
|
||
"echo ROOT_OPTIONS:$(findmnt -no OPTIONS /); "
|
||
"echo ROOT_PROPERTY:$(btrfs property get / ro 2>&1); "
|
||
"sys history --legacy >/tmp/phoebus-recovery-history 2>&1; "
|
||
"echo SYS_HISTORY_RC:$?; "
|
||
"sys rollback status --legacy >/tmp/phoebus-recovery-status 2>&1; "
|
||
"echo SYS_ROLLBACK_RC:$?; "
|
||
"echo ===RECOVERY-$(printf END)==="
|
||
)
|
||
child.expect("===RECOVERY-END===", timeout=180)
|
||
observed_output = child.before + "===RECOVERY-END==="
|
||
match = re.search(r"===RECOVERY-START===(.*?)===RECOVERY-END===",
|
||
observed_output, re.DOTALL)
|
||
observed = match.group(1) if match else ""
|
||
results.assert_true(
|
||
"recovery-snapshot-selected",
|
||
"rootflags=subvol=@snapshots/" in observed
|
||
and "systemd.unit=rescue.target" in observed,
|
||
"recovery kernel command line did not select the snapshot",
|
||
)
|
||
results.assert_true(
|
||
"recovery-snapshot-mounted",
|
||
"ROOT_SOURCE:/dev/mapper/cryptroot[/@snapshots/" in observed
|
||
and "ro=true" in observed,
|
||
"root is not the read-only recovery subvolume",
|
||
)
|
||
results.assert_true(
|
||
"recovery-sys-offline",
|
||
"SYS_HISTORY_RC:0" in observed and "SYS_ROLLBACK_RC:0" in observed,
|
||
"deterministic sys commands did not work offline",
|
||
)
|
||
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
|
||
results.fail("recovery-offline", str(exc))
|
||
finally:
|
||
close_quietly(child)
|
||
log.close()
|
||
|
||
|
||
def _gate_overlay(results, paths, disks, tag):
|
||
"""Create a writable disposable overlay without changing the install disk."""
|
||
if len(disks) != 1:
|
||
results.blocked(f"{tag}-disk", f"{tag} needs exactly one installed qcow2")
|
||
return None
|
||
overlay = disposable_dir() / f"{tag}-{paths.run_id}.qcow2"
|
||
create = subprocess.run(
|
||
["qemu-img", "create", "-f", "qcow2", "-F", "qcow2", "-b",
|
||
str(disks[0].resolve()), str(overlay)],
|
||
capture_output=True, text=True, check=False,
|
||
)
|
||
if create.returncode != 0:
|
||
results.fail(f"{tag}-overlay", create.stderr.strip() or "qemu-img create failed")
|
||
return None
|
||
try:
|
||
test_disks = vmsafety.validate_writable_drives([str(overlay)], disposable_dir())
|
||
except vmsafety.UnsafeDiskError as exc:
|
||
results.fail(f"{tag}-overlay", str(exc))
|
||
return None
|
||
results.pass_(f"{tag}-overlay", str(overlay))
|
||
return test_disks
|
||
|
||
|
||
def do_fallback(results, paths, disks):
|
||
"""Prove that exhausted normal boot attempts select recovery by default."""
|
||
test_disks = _gate_overlay(results, paths, disks, "fallback")
|
||
if test_disks is None:
|
||
return
|
||
|
||
# Prepare the overlay from a healthy boot. The next three boots stop at
|
||
# rescue.target's locked-root prompt and QEMU is terminated there. Pressing
|
||
# Enter would continue to default.target, which can bless the boot and
|
||
# clear the counter; a hard stop models the failed boot we need to test.
|
||
env, args = _installed_args(paths, test_disks, network=False)
|
||
with open(paths.directory / "serial-prepare.log", "w") as log:
|
||
child = spawn(args, env, log)
|
||
try:
|
||
_unlock_and_login(child, results, "fallback-prepare")
|
||
if not _sudo_auth(child, results, "fallback-prepare"):
|
||
return
|
||
child.sendline(
|
||
"entry=$(sudo -n find /efi/loader/entries -maxdepth 1 -type f -name '*.conf' "
|
||
"! -name '*recovery*' | sort | head -1); "
|
||
"test -n \"$entry\" && "
|
||
"test \"$(sudo -n awk '$1 == \"default\" {print $2}' /efi/loader/loader.conf)\" = '*' "
|
||
"&& sudo -n grep -Eq '^sort-key[[:space:]]+phoebus$' \"$entry\" && "
|
||
"sudo -n sed -i "
|
||
"'/^options[[:space:]]/ s/$/ systemd.unit=rescue.target/' \"$entry\" && "
|
||
"base=${entry%.conf}; base=${base%%+*}; "
|
||
"sudo -n mv \"$entry\" \"${base}+3.conf\" && "
|
||
"echo FALLBACK-$(printf PREPARED)"
|
||
)
|
||
child.expect("FALLBACK-PREPARED", timeout=60)
|
||
child.sendline("sudo -n systemctl poweroff")
|
||
child.expect(pexpect.EOF, timeout=120)
|
||
results.pass_("fallback-prepared")
|
||
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
|
||
results.fail("fallback-prepare", str(exc))
|
||
return
|
||
finally:
|
||
close_quietly(child)
|
||
|
||
for attempt in range(1, 4):
|
||
env, args = _installed_args(paths, test_disks, network=False)
|
||
log_path = paths.directory / f"serial-attempt-{attempt}.log"
|
||
with open(log_path, "w") as log:
|
||
child = spawn(args, env, log)
|
||
try:
|
||
child.expect(r"[Pp]assphrase", timeout=300)
|
||
child.sendline(LUKS_PW)
|
||
child.expect("Press Enter to continue", timeout=300)
|
||
results.pass_(f"fallback-attempt-{attempt}-failed-boot")
|
||
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
|
||
results.fail(f"fallback-attempt-{attempt}", str(exc))
|
||
return
|
||
finally:
|
||
close_quietly(child)
|
||
# QEMU can release the qcow2 write lock shortly after pexpect closes
|
||
# its PTY. Give the next boot a brief teardown window.
|
||
time.sleep(3)
|
||
|
||
env, args = _installed_args(paths, test_disks, network=False)
|
||
with open(paths.serial_log, "w") as log:
|
||
child = spawn(args, env, log)
|
||
try:
|
||
_unlock_and_login(child, results, "fallback-recovery",
|
||
continue_locked_root=True)
|
||
if not _sudo_auth(child, results, "fallback-recovery"):
|
||
return
|
||
child.sendline(
|
||
"setopt NO_EQUALS 2>/dev/null; echo ===FALLBACK-$(printf START)===; "
|
||
"cat /proc/cmdline; sudo -n ls -1 /efi/loader/entries; "
|
||
"echo ===FALLBACK-$(printf END)==="
|
||
)
|
||
child.expect("===FALLBACK-END===", timeout=60)
|
||
observed_output = child.before + "===FALLBACK-END==="
|
||
match = re.search(r"===FALLBACK-START===(.*?)===FALLBACK-END===",
|
||
observed_output, re.DOTALL)
|
||
observed = match.group(1) if match else ""
|
||
correct = ("rootflags=subvol=@snapshots/" in observed
|
||
and "systemd.unit=rescue.target" in observed
|
||
and re.search(r"\+0-3\.conf", observed) is not None)
|
||
results.assert_true("fallback-boot-4", correct,
|
||
f"recovery was not selected after three failures: {observed.strip()}")
|
||
if not correct:
|
||
return
|
||
child.sendline("sudo -n systemctl poweroff")
|
||
child.expect(pexpect.EOF, timeout=120)
|
||
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
|
||
results.fail("fallback-recovery", str(exc))
|
||
finally:
|
||
close_quietly(child)
|
||
|
||
|
||
def do_interrupted_update(results, paths, disks):
|
||
"""Kill an in-flight package install, undo its snapshot, and reboot."""
|
||
test_disks = _gate_overlay(results, paths, disks, "interrupted-update")
|
||
if test_disks is None:
|
||
return
|
||
env, args = _installed_args(paths, test_disks)
|
||
phase1_log = paths.directory / "serial-interrupted-phase1.log"
|
||
with open(phase1_log, "w") as log:
|
||
child = spawn(args, env, log)
|
||
try:
|
||
_unlock_and_login(child, results, "interrupted-before")
|
||
if not _sudo_auth(child, results, "interrupted-before"):
|
||
return
|
||
child.sendline(
|
||
f"curl -fsSL {INTERRUPT_URL} -o /tmp/interrupted-update.sh "
|
||
"&& sudo -n bash /tmp/interrupted-update.sh"
|
||
)
|
||
outcome = child.expect(
|
||
["===INTERRUPT-POWEROFF===", "===INTERRUPT-NEEDS-ROOT===",
|
||
"===INTERRUPT-PRECONDITION-FAILED===", "===INTERRUPT-SNAPSHOT-FAILED===",
|
||
"===INTERRUPT-NO-ACTIVE-UPDATE===", "curl:", pexpect.EOF],
|
||
timeout=1800,
|
||
)
|
||
if outcome != 0:
|
||
results.fail("interrupted-update-phase1",
|
||
f"guest stopped before recovery: {child.after!r}")
|
||
return
|
||
child.expect(pexpect.EOF, timeout=120)
|
||
log.flush()
|
||
phase1 = phase1_log.read_text(errors="replace")
|
||
before = _parse_marker(phase1, "INTERRUPT-BEFORE")
|
||
recovery = _parse_marker(phase1, "INTERRUPT-RECOVERY")
|
||
results.assert_true("interrupted-update-killed",
|
||
"===INTERRUPT-SIGKILL-DURING-EMERGE===" in phase1,
|
||
"the update process was not killed during emerge")
|
||
results.assert_true("interrupted-update-undone",
|
||
recovery is not None and recovery.get("history_rc") == "0"
|
||
and recovery.get("undo_rc") == "0",
|
||
"sys did not recover its lock and undo the snapshot")
|
||
if before is None or recovery is None or recovery.get("undo_rc") != "0":
|
||
return
|
||
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
|
||
results.fail("interrupted-update-phase1", str(exc))
|
||
return
|
||
finally:
|
||
close_quietly(child)
|
||
|
||
env, args = _installed_args(paths, test_disks, network=False)
|
||
with open(paths.serial_log, "w") as log:
|
||
child = spawn(args, env, log)
|
||
try:
|
||
_unlock_and_login(child, results, "interrupted-after")
|
||
child.sendline(
|
||
"setopt NO_EQUALS 2>/dev/null; echo ===INTERRUPT-$(printf VERIFY-START)===; "
|
||
"echo SYSTEMD:$(systemctl is-system-running); "
|
||
"echo PKGS:$(find /var/db/pkg -mindepth 2 -maxdepth 2 -type d 2>/dev/null "
|
||
"| sort | sha256sum | awk '{print $1}'); "
|
||
"echo HOME:$(cat /home/phoebus/interrupted-update-marker.txt); "
|
||
"echo HELLO:$(find /var/db/pkg/app-misc -maxdepth 1 -name 'hello-*' "
|
||
"-print -quit | grep -q . && echo yes || echo no); "
|
||
"sys history --legacy >/tmp/interrupted-history 2>&1; echo SYS_RC:$?; "
|
||
"echo ===INTERRUPT-$(printf VERIFY-END)==="
|
||
)
|
||
child.expect("===INTERRUPT-VERIFY-END===", timeout=120)
|
||
log.flush()
|
||
transcript = paths.serial_log.read_text(errors="replace")
|
||
match = re.search(r"===INTERRUPT-VERIFY-START===(.*?)===INTERRUPT-VERIFY-END===",
|
||
transcript, re.DOTALL)
|
||
observed = match.group(1) if match else ""
|
||
results.assert_true("interrupted-update-boot",
|
||
"SYSTEMD:running" in observed or "SYSTEMD:degraded" in observed,
|
||
"the restored system did not boot")
|
||
results.assert_true("interrupted-update-package-state",
|
||
f"PKGS:{before['pkgs']}" in observed and "HELLO:no" in observed,
|
||
"package database did not match the pre-update snapshot")
|
||
results.assert_true("interrupted-update-home",
|
||
f"HOME:{before['home']}" in observed,
|
||
"the user home marker was not preserved")
|
||
results.assert_true("interrupted-update-sys", "SYS_RC:0" in observed,
|
||
"deterministic sys failed after the interrupted update")
|
||
if _sudo_auth(child, results, "interrupted-after"):
|
||
child.sendline("sudo -n systemctl poweroff")
|
||
child.expect(pexpect.EOF, timeout=120)
|
||
except (pexpect.TIMEOUT, pexpect.EOF) as exc:
|
||
results.fail("interrupted-update-phase2", str(exc))
|
||
finally:
|
||
close_quietly(child)
|
||
|
||
|
||
def do_boot(results, paths, disks, assessment=False):
|
||
env, args = base_args(str(paths.ovmf_vars))
|
||
for disk in disks:
|
||
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
||
args += ["-boot", "order=c"]
|
||
# The end marker is built with a command substitution so the shell's echo of
|
||
# the typed line does not itself contain it; otherwise `expect` would match
|
||
# the echo and close the console before the checks ran.
|
||
checks = os.environ.get(
|
||
"BOOT_CHECKS",
|
||
# `setopt NO_EQUALS` first: the installed default shell is zsh, where
|
||
# a leading `=` triggers command-path expansion and the `===MARKER===`
|
||
# convention would abort the whole line as "not found". The redirect
|
||
# swallows the error on shells without `setopt` (bash live envs).
|
||
"setopt NO_EQUALS 2>/dev/null; "
|
||
"echo ===CHECKS-$(printf START)===; "
|
||
"echo COMM_CHECK:$(cat /proc/1/comm 2>&1); "
|
||
"echo SYSTEMD_STATUS:$(systemctl is-system-running 2>&1); "
|
||
"systemctl --failed --no-pager; ip -brief addr; "
|
||
"lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS; echo ===CHECKS-$(printf END)===",
|
||
)
|
||
log = open(paths.serial_log, "w")
|
||
child = spawn(args, env, log)
|
||
try:
|
||
first_prompt = child.expect([r"[Pp]assphrase", r"login:"], timeout=300)
|
||
login_ready = first_prompt == 1
|
||
if first_prompt == 0:
|
||
child.sendline(LUKS_PW)
|
||
next_prompt = child.expect(
|
||
[r"phoebus login:", r"Press Enter to continue"], timeout=300
|
||
)
|
||
if next_prompt == 0:
|
||
login_ready = True
|
||
else:
|
||
child.sendline("")
|
||
time.sleep(5)
|
||
logged_in = False
|
||
for _ in range(5):
|
||
if not login_ready:
|
||
child.expect(r"phoebus login:", timeout=180)
|
||
login_ready = False
|
||
time.sleep(5)
|
||
child.sendline(USER)
|
||
idx = child.expect([r"[Pp]assword:", r"phoebus login:",
|
||
r"Login incorrect"], timeout=30)
|
||
if idx == 0:
|
||
child.sendline(USER_PW)
|
||
login_result = child.expect([SHELL_PROMPT, r"Login incorrect",
|
||
r"phoebus login:"], timeout=60)
|
||
if login_result == 0:
|
||
logged_in = True
|
||
break
|
||
login_ready = login_result == 2
|
||
else:
|
||
login_ready = idx == 1
|
||
results.assert_true("detached-boot-login", logged_in, "could not log in")
|
||
if os.environ.get("BOOT_AUTHENTICATE") == "1":
|
||
if not _sudo_auth(child, results, "boot"):
|
||
return
|
||
child.sendline(checks)
|
||
child.expect("===CHECKS-END===", timeout=int(os.environ.get("CHECKS_TIMEOUT", "180")))
|
||
# pexpect can match the end marker before the redacting log sink has
|
||
# received its trailing newline. Parse the just-consumed output.
|
||
observed_checks = child.before + "===CHECKS-END==="
|
||
checks_match = re.search(r"===CHECKS-START===(.*?)===CHECKS-END===",
|
||
observed_checks, re.DOTALL)
|
||
checks_text = checks_match.group(1) if checks_match else observed_checks
|
||
|
||
m = re.search(r"SYSTEMD_STATUS:([a-z\-]+)", checks_text)
|
||
status = m.group(1) if m else "unknown"
|
||
results.assert_true(
|
||
"boot-systemd-running",
|
||
status in ("running", "degraded"),
|
||
f"systemd did not report running/degraded (observed: {status})",
|
||
)
|
||
if assessment:
|
||
if not _sudo_auth(child, results, "assessment"):
|
||
return
|
||
child.sendline(
|
||
"setopt NO_EQUALS 2>/dev/null; "
|
||
"for i in $(seq 1 120); do "
|
||
"systemctl is-active --quiet systemd-bless-boot.service && break; "
|
||
"sleep 1; done; "
|
||
"echo ===ASSESS-$(printf START)===; "
|
||
"echo BLESS:$(systemctl is-active systemd-bless-boot.service); "
|
||
"echo CHECK:$(systemctl is-active systemd-boot-check-no-failures.service); "
|
||
"echo COMPLETE:$(systemctl is-active boot-complete.target); "
|
||
"echo COUNTED:$(sudo -n find /efi/loader/entries -maxdepth 1 "
|
||
"-name '*+*.conf' | wc -l); "
|
||
"sys history --legacy >/tmp/phoebus-assessed-history 2>&1; echo SYS_RC:$?; "
|
||
"echo ===ASSESS-$(printf END)==="
|
||
)
|
||
child.expect("===ASSESS-END===", timeout=180)
|
||
observed_assessment = child.before + "===ASSESS-END==="
|
||
match = re.search(r"===ASSESS-START===(.*?)===ASSESS-END===",
|
||
observed_assessment, re.DOTALL)
|
||
observed = match.group(1) if match else ""
|
||
results.assert_true(
|
||
"boot-assessment-confirmed",
|
||
all(marker in observed for marker in
|
||
("BLESS:active", "CHECK:active", "COMPLETE:active",
|
||
"COUNTED:0", "SYS_RC:0")),
|
||
f"boot health was not confirmed: {observed.strip()}",
|
||
)
|
||
for case in os.environ.get("BOOT_REQUIRE_CASES", "").split(","):
|
||
case = case.strip()
|
||
if case:
|
||
results.assert_true(
|
||
f"boot-case-{case}",
|
||
f"CASE {case} PASS" in checks_text,
|
||
f"{case} did not report PASS",
|
||
)
|
||
if assessment or os.environ.get("BOOT_POWEROFF") == "1":
|
||
child.sendline("sudo systemctl poweroff")
|
||
idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", pexpect.EOF], timeout=60)
|
||
if idx == 0:
|
||
child.sendline(USER_PW)
|
||
child.expect(pexpect.EOF, timeout=120)
|
||
results.pass_("boot-clean-poweroff")
|
||
except pexpect.TIMEOUT as exc:
|
||
results.fail("boot", f"timeout: {exc}")
|
||
except pexpect.EOF:
|
||
results.fail("boot", "unexpected EOF")
|
||
finally:
|
||
close_quietly(child)
|
||
log.close()
|
||
|
||
|
||
def main() -> int:
|
||
mode = sys.argv[1] if len(sys.argv) > 1 else ""
|
||
if mode not in ("install", "install-repeat", "boot", "boot-assessed", "rollback", "recovery",
|
||
"fallback", "interrupted-update"):
|
||
print(__doc__)
|
||
return 2
|
||
paths = vmsafety.make_run_paths(
|
||
f"serial-{mode}", base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"),
|
||
ovmf_vars_template=OVMF_VARS_TEMPLATE,
|
||
)
|
||
results = Results(
|
||
f"serial-{mode}", run_id=paths.run_id,
|
||
secrets=[LUKS_PW, USER_PW, LIVE_PW,
|
||
os.environ.get("OPENAI_API_KEY"), os.environ.get("DEEPSEEK_API_KEY")],
|
||
)
|
||
selected_disks = DISKS
|
||
if mode == "install-repeat":
|
||
selected_disks = os.environ.get("DISK_SECOND", "")
|
||
if not selected_disks or selected_disks == DISKS:
|
||
results.blocked("independent-install-disk", "DISK_SECOND must name a separate disposable qcow2")
|
||
results.write(paths.results_json)
|
||
return 2
|
||
try:
|
||
if mode == "install-repeat":
|
||
disks = vmsafety.validate_independent_disk_sets(
|
||
DISKS.split(","), selected_disks.split(","), disposable_dir()
|
||
)
|
||
else:
|
||
disks = vmsafety.validate_writable_drives(selected_disks.split(","), disposable_dir())
|
||
except vmsafety.UnsafeDiskError as exc:
|
||
results.blocked("disposable-disk", str(exc))
|
||
results.write(paths.results_json)
|
||
print(f"harness error: {exc}", file=sys.stderr)
|
||
return 2
|
||
results.pass_("disposable-disk", ", ".join(str(d) for d in disks))
|
||
try:
|
||
if mode in ("install", "install-repeat"):
|
||
do_install(results, paths, disks)
|
||
elif mode == "rollback":
|
||
do_rollback(results, paths, disks)
|
||
elif mode == "recovery":
|
||
do_recovery(results, paths, disks)
|
||
elif mode == "fallback":
|
||
do_fallback(results, paths, disks)
|
||
elif mode == "interrupted-update":
|
||
do_interrupted_update(results, paths, disks)
|
||
elif mode == "boot-assessed":
|
||
do_boot(results, paths, disks, assessment=True)
|
||
else:
|
||
do_boot(results, paths, disks)
|
||
finally:
|
||
results.write(paths.results_json)
|
||
print(results.summary(), flush=True)
|
||
return results.exit_code()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|