472 lines
20 KiB
Bash
472 lines
20 KiB
Bash
#!/usr/bin/env python3
|
||
"""Headless serial driver for the disposable UEFI VM.
|
||
|
||
Modes:
|
||
|
||
install : boot the live ISO kernel/initrd, run a guest script that installs
|
||
Phoebus non-interactively, then power off.
|
||
boot : boot the installed disk alone (no ISO, no -kernel), unlock LUKS,
|
||
log in and run verification commands.
|
||
rollback: boot an installed disk, record a known-good state, create a
|
||
recovery point through `sys snapshot`, introduce a reversible
|
||
update, run the real `sys undo`, power off and boot detached to
|
||
verify the restored root/var and package state. Destructive: point
|
||
DISK at a disposable clone, never a physical device.
|
||
|
||
Everything is parameterised through environment variables so the script is not
|
||
tied to one host. See tests/vm/README.md for the variables.
|
||
|
||
Safety: `DISK`/`DISKS` must be qcow2 files inside the disposable directory named
|
||
by `PHOEBUS_VM_DISKS` (default `$PHOEBUS_VM_STATE/disks`); block devices,
|
||
`/dev/*`, escaping symlinks, raw images and outside backing chains are refused.
|
||
Results are structured and the exit status is nonzero on failure.
|
||
"""
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
import pexpect
|
||
|
||
import vmsafety
|
||
from vmsafety import Results
|
||
|
||
QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64")
|
||
LIBDIR = os.environ.get("QEMU_LIBDIR", "")
|
||
DATADIR = os.environ.get("QEMU_DATADIR", "")
|
||
OVMF_CODE = os.environ["OVMF_CODE"]
|
||
OVMF_VARS_TEMPLATE = os.environ["OVMF_VARS"]
|
||
DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "")
|
||
ISO = os.environ.get("ISO", "")
|
||
KERNEL = os.environ.get("KERNEL", "")
|
||
INITRD = os.environ.get("INITRD", "")
|
||
LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test")
|
||
USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test")
|
||
USER = os.environ.get("USERNAME", "phoebus")
|
||
MEM = os.environ.get("VM_MEM", "8192")
|
||
SMP = os.environ.get("VM_SMP", "8")
|
||
|
||
# Shell-prompt alternative for an installed Phoebus login. The default login
|
||
# shell is zsh with a powerline-style prompt ending in the `❯` glyph; color
|
||
# escapes sit between the glyph and the trailing space, so the bare glyph
|
||
# (not `❯ `) is matched. Bash/root prompts keep matching `[$#] `.
|
||
SHELL_PROMPT = r"[$#] |❯"
|
||
|
||
BLS_APPEND = (
|
||
"root=live:CDLABEL=gentoo-amd64-livegui rd.live.dir=/ "
|
||
"rd.live.squashimg=image.squashfs cdroot console=ttyS0,115200"
|
||
)
|
||
|
||
# Rollback acceptance fixtures, served from the host over QEMU user networking.
|
||
ROLLBACK_URL = os.environ.get("ROLLBACK_URL", "http://10.0.2.2:8000/rollback-guest.sh")
|
||
ROLLBACK_VERIFY_URL = os.environ.get(
|
||
"ROLLBACK_VERIFY_URL", "http://10.0.2.2:8000/rollback-verify.sh"
|
||
)
|
||
SYS_URL = os.environ.get("SYS_URL", "http://10.0.2.2:8000/sys")
|
||
|
||
|
||
def disposable_dir():
|
||
import pathlib
|
||
|
||
base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")
|
||
return pathlib.Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks"))
|
||
|
||
|
||
def base_args(ovmf_vars):
|
||
env = dict(os.environ)
|
||
if LIBDIR:
|
||
env["LD_LIBRARY_PATH"] = LIBDIR
|
||
args = ["-machine", "q35,accel=kvm"]
|
||
if DATADIR:
|
||
args += ["-L", DATADIR]
|
||
args += [
|
||
"-cpu", "host", "-smp", SMP, "-m", MEM,
|
||
"-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}",
|
||
"-drive", f"if=pflash,format=raw,file={ovmf_vars}",
|
||
"-netdev", "user,id=n0", "-device", "virtio-net-pci,netdev=n0,romfile=",
|
||
"-device", "virtio-rng-pci", "-vga", "std", "-display", "none",
|
||
"-serial", "stdio", "-monitor", "none", "-no-reboot",
|
||
]
|
||
return env, args
|
||
|
||
|
||
def spawn(args, env, log):
|
||
child = pexpect.spawn(QEMU, args, env=env, timeout=600,
|
||
encoding="utf-8", codec_errors="replace",
|
||
maxread=65536)
|
||
child.delaybeforesend = 0.3
|
||
# Log only guest output. The general logfile also records bytes sent by
|
||
# pexpect, including LUKS, login and sudo passwords.
|
||
child.logfile_read = log
|
||
return child
|
||
|
||
|
||
def close_quietly(child):
|
||
"""Best-effort teardown. After the guest powers off, pexpect can raise
|
||
`ExceptionPexpect: Could not terminate the child`; that must not turn a
|
||
successful run into a nonzero driver exit."""
|
||
try:
|
||
child.close(force=True)
|
||
except Exception: # noqa: BLE001 - cleanup must never mask the result
|
||
pass
|
||
|
||
|
||
def do_install(results, paths, disks):
|
||
env, args = base_args(str(paths.ovmf_vars))
|
||
for disk in disks:
|
||
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
||
args += ["-drive", f"file={ISO},media=cdrom,readonly=on",
|
||
"-kernel", KERNEL, "-initrd", INITRD, "-append", BLS_APPEND]
|
||
log = open(paths.serial_log, "w")
|
||
child = spawn(args, env, log)
|
||
try:
|
||
child.expect(r"root@livecd[^\n]*# ", timeout=600)
|
||
results.pass_("live-shell")
|
||
url = os.environ.get("INSTALL_URL", "http://10.0.2.2:8000/install.sh")
|
||
child.sendline(f"curl -fsSL {url} -o /tmp/run.sh && bash /tmp/run.sh")
|
||
child.expect([r"===INSTALL-RC=([0-9]+)===", "===FETCH-FAIL===",
|
||
"Kernel panic", pexpect.EOF, pexpect.TIMEOUT], timeout=19800)
|
||
# `child.after` is the matched pattern class (not a string) when EOF or
|
||
# TIMEOUT matched; handle that instead of raising a TypeError.
|
||
matched = child.after if isinstance(child.after, str) else ""
|
||
if "INSTALL-RC" in matched:
|
||
rc = child.match.group(1)
|
||
results.assert_true("install-rc-zero", rc == "0", f"INSTALL-RC={rc}")
|
||
else:
|
||
results.fail(
|
||
"install-rc-zero",
|
||
f"install did not report a status (matched {matched!r})",
|
||
)
|
||
serial = paths.serial_log.read_text(errors="replace")
|
||
results.assert_true("no-secret-leak", "SECRET_LEAK_FOUND" not in serial,
|
||
"the guest reported a secret leak")
|
||
child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600)
|
||
except pexpect.TIMEOUT as exc:
|
||
results.fail("install", f"timeout: {exc}")
|
||
except pexpect.EOF:
|
||
results.fail("install", "unexpected EOF")
|
||
finally:
|
||
close_quietly(child)
|
||
log.close()
|
||
|
||
|
||
def _installed_args(paths, disks):
|
||
env, args = base_args(str(paths.ovmf_vars))
|
||
for disk in disks:
|
||
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
||
args += ["-boot", "order=c"]
|
||
return env, args
|
||
|
||
|
||
def _unlock_and_login(child, results, tag):
|
||
child.expect([r"[Pp]assphrase", r"login:"], timeout=300)
|
||
if "assphrase" in (child.after or ""):
|
||
override = os.environ.get("BOOT_PASSPHRASE")
|
||
candidates = [override] if override else [USER_PW, LUKS_PW]
|
||
child.sendline(candidates[0])
|
||
for candidate in candidates[1:]:
|
||
try:
|
||
child.expect(r"[Pp]assphrase", timeout=15)
|
||
child.sendline(candidate)
|
||
except (pexpect.TIMEOUT, pexpect.EOF):
|
||
break
|
||
logged_in = False
|
||
for _ in range(5):
|
||
child.expect(r"phoebus login:", timeout=180)
|
||
child.sendline(USER)
|
||
idx = child.expect([r"[Pp]assword:", r"phoebus login:", r"Login incorrect"],
|
||
timeout=30)
|
||
if idx == 0:
|
||
child.sendline(USER_PW)
|
||
if child.expect([SHELL_PROMPT, r"Login incorrect", r"phoebus login:"],
|
||
timeout=60) == 0:
|
||
logged_in = True
|
||
break
|
||
results.assert_true(f"{tag}-login", logged_in, f"{tag}: could not log in")
|
||
|
||
|
||
def _sudo_auth(child, results, tag):
|
||
"""Authenticate sudo for the rollback acceptance guest scripts.
|
||
|
||
The snapshot/break/undo/verify steps mutate the system and must run as
|
||
root. The password is sent only at sudo's own prompt (terminal echo
|
||
disabled, like the login password), so it never lands in the serial
|
||
transcript the way a command-line credential would. Returns True when the
|
||
timestamp stuck; the caller must abort the phase otherwise, or every
|
||
later `sudo` would block on a prompt nobody answers.
|
||
"""
|
||
try:
|
||
# Unique markers, never a bare `[$#] ` alternative: the buffer still
|
||
# holds the previous shell prompt when this runs, and a generic
|
||
# prompt pattern would match that stale text immediately — skipping
|
||
# the password while sudo is still starting (the next typed line
|
||
# would then be consumed as the password and rejected).
|
||
child.sendline("sudo -v; echo SUDO-V-$(printf DONE)")
|
||
idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", "SUDO-V-DONE"], timeout=60)
|
||
if idx == 0:
|
||
child.sendline(USER_PW)
|
||
child.expect("SUDO-V-DONE", timeout=60)
|
||
# Prove the timestamp stuck: without it every later `sudo` would block
|
||
# on a password prompt nobody answers, hanging the run until timeout.
|
||
child.sendline("sudo -n true && echo SUDO-OK-$(printf READY)")
|
||
idx = child.expect(["SUDO-OK-READY", r"[Pp]assword"], timeout=30)
|
||
except (pexpect.TIMEOUT, pexpect.EOF):
|
||
idx = -1
|
||
if idx != 0:
|
||
results.fail(f"{tag}-sudo", "sudo authentication did not stick")
|
||
return False
|
||
return True
|
||
|
||
|
||
def _fetch_sys(child):
|
||
# The installed system should already carry /usr/local/bin/sys; only fall
|
||
# back to fetching the host build when it is absent.
|
||
child.sendline(
|
||
"if [ -x /usr/local/bin/sys ]; then SYS=/usr/local/bin/sys; "
|
||
"elif [ -x /tmp/sys ]; then SYS=/tmp/sys; "
|
||
f"else curl -fsSL {SYS_URL} -o /tmp/sys && chmod 755 /tmp/sys && SYS=/tmp/sys; fi; "
|
||
"test -x \"$SYS\" && echo SYS-OK-$(printf READY)"
|
||
)
|
||
child.expect("SYS-OK-READY", timeout=120)
|
||
|
||
|
||
def _parse_marker(text, key):
|
||
import re
|
||
|
||
match = re.search(r"===" + re.escape(key) + r"(.+?)===", text)
|
||
if not match:
|
||
return None
|
||
out = {}
|
||
for token in match.group(1).split():
|
||
if "=" in token:
|
||
name, value = token.split("=", 1)
|
||
out[name] = value
|
||
return out
|
||
|
||
|
||
def do_rollback(results, paths, disks):
|
||
"""Snapshot -> reversible update -> sys undo -> detached verify."""
|
||
# Phase 1: known-good -> snapshot -> break -> undo, then power off.
|
||
env, args = _installed_args(paths, disks)
|
||
log = open(paths.serial_log, "w")
|
||
child = spawn(args, env, log)
|
||
phase1 = ""
|
||
try:
|
||
_unlock_and_login(child, results, "rollback-before")
|
||
_fetch_sys(child)
|
||
if not _sudo_auth(child, results, "rollback-before"):
|
||
return
|
||
child.sendline(f"curl -fsSL {ROLLBACK_URL} -o /tmp/rb.sh && sudo bash /tmp/rb.sh")
|
||
child.expect(
|
||
["===ROLLBACK-DONE===", "curl:", pexpect.TIMEOUT, pexpect.EOF],
|
||
timeout=int(os.environ.get("ROLLBACK_TIMEOUT", "7200")),
|
||
)
|
||
if "ROLLBACK-DONE" not in (child.after or ""):
|
||
results.fail("rollback-sequence", f"the guest rollback did not finish: {child.after!r}")
|
||
else:
|
||
results.pass_("rollback-sequence")
|
||
child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600)
|
||
phase1 = paths.serial_log.read_text(errors="replace")
|
||
except pexpect.TIMEOUT as exc:
|
||
results.fail("rollback-before", f"timeout: {exc}")
|
||
except pexpect.EOF:
|
||
results.fail("rollback-before", "unexpected EOF")
|
||
finally:
|
||
close_quietly(child)
|
||
log.close()
|
||
|
||
before = _parse_marker(phase1, "ROLLBACK-KNOWN-GOOD")
|
||
broken = _parse_marker(phase1, "ROLLBACK-BROKEN")
|
||
undo = _parse_marker(phase1, "ROLLBACK-UNDO")
|
||
if before is None:
|
||
results.fail("rollback-known-good", "the guest did not record the known-good state")
|
||
if broken is None or broken.get("pkgs") == (before or {}).get("pkgs"):
|
||
results.fail("rollback-break-broke", f"the controlled break did not change the package database: {broken}")
|
||
return
|
||
if undo is None or undo.get("rc") != "0":
|
||
results.fail("rollback-undo", f"sys undo did not succeed: {undo}")
|
||
# No rollback happened: a detached "verify" would compare identical
|
||
# states and report vacuous passes. Stop instead of pretending.
|
||
return
|
||
|
||
# Phase 2: boot the disk detached and verify the restored generation.
|
||
# The phase-1 transcript is preserved under its own name first: reopening
|
||
# the same log would destroy the evidence for why phase 1 passed/failed.
|
||
phase1_log = paths.serial_log.with_name("serial-phase1.log")
|
||
paths.serial_log.rename(phase1_log)
|
||
env, args = _installed_args(paths, disks)
|
||
log = open(paths.serial_log, "w")
|
||
child = spawn(args, env, log)
|
||
phase2 = ""
|
||
try:
|
||
_unlock_and_login(child, results, "rollback-after")
|
||
_fetch_sys(child)
|
||
if not _sudo_auth(child, results, "rollback-after"):
|
||
return
|
||
child.sendline(f"curl -fsSL {ROLLBACK_VERIFY_URL} -o /tmp/rbv.sh && sudo bash /tmp/rbv.sh")
|
||
child.expect("===AFTER-DONE===",
|
||
timeout=int(os.environ.get("ROLLBACK_VERIFY_TIMEOUT", "900")))
|
||
phase2 = paths.serial_log.read_text(errors="replace")
|
||
results.assert_true("rollback-systemd-running", "===AFTER-SYSTEMD running===" in phase2,
|
||
"systemd did not report running after rollback")
|
||
after = _parse_marker(phase2, "AFTER-KNOWN-GOOD")
|
||
if before is None or after is None:
|
||
results.fail("rollback-state", "could not read the before/after state")
|
||
else:
|
||
results.assert_true(
|
||
"rollback-package-state-restored",
|
||
before.get("pkgs") == after.get("pkgs"),
|
||
f"package db changed across rollback: before={before.get('pkgs')} after={after.get('pkgs')}",
|
||
)
|
||
results.assert_true(
|
||
"rollback-home-survives",
|
||
before.get("home") == after.get("home"),
|
||
f"/home did not survive: before={before.get('home')} after={after.get('home')}",
|
||
)
|
||
results.assert_true("rollback-modules-present", "===AFTER-MODULES ok===" in phase2,
|
||
"no kernel modules for the running kernel")
|
||
# The break wrote this marker after our snapshot; the targeted undo
|
||
# restores our snapshot exactly, so the marker must be gone. Its
|
||
# survival would prove the wrong generation was restored.
|
||
results.assert_true("rollback-root-restored", "===AFTER-BROKEN-MARKER absent===" in phase2,
|
||
"the root break marker survived the rollback")
|
||
results.assert_true(
|
||
"rollback-recovery-entry",
|
||
"systemd.unit=rescue.target" in phase2 and "@snapshots/" in phase2,
|
||
"the snapshot recovery entry is missing or not snapshot-based",
|
||
)
|
||
status = phase2.split("===AFTER-STATUS===", 1)
|
||
status_text = status[1].split("===AFTER-DONE===", 1)[0].strip() if len(status) > 1 else ""
|
||
results.assert_true("rollback-status-recorded", bool(status_text),
|
||
"sys rollback status produced no evidence")
|
||
except pexpect.TIMEOUT as exc:
|
||
results.fail("rollback-after", f"timeout: {exc}")
|
||
except pexpect.EOF:
|
||
results.fail("rollback-after", "unexpected EOF")
|
||
finally:
|
||
close_quietly(child)
|
||
log.close()
|
||
|
||
|
||
def do_boot(results, paths, disks):
|
||
env, args = base_args(str(paths.ovmf_vars))
|
||
for disk in disks:
|
||
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
||
args += ["-boot", "order=c"]
|
||
# The end marker is built with a command substitution so the shell's echo of
|
||
# the typed line does not itself contain it; otherwise `expect` would match
|
||
# the echo and close the console before the checks ran.
|
||
checks = os.environ.get(
|
||
"BOOT_CHECKS",
|
||
# `setopt NO_EQUALS` first: the installed default shell is zsh, where
|
||
# a leading `=` triggers command-path expansion and the `===MARKER===`
|
||
# convention would abort the whole line as "not found". The redirect
|
||
# swallows the error on shells without `setopt` (bash live envs).
|
||
"setopt NO_EQUALS 2>/dev/null; "
|
||
"echo ===CHECKS-$(printf START)===; "
|
||
"echo COMM_CHECK:$(cat /proc/1/comm 2>&1); "
|
||
"echo SYSTEMD_STATUS:$(systemctl is-system-running 2>&1); "
|
||
"systemctl --failed --no-pager; ip -brief addr; "
|
||
"lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS; echo ===CHECKS-$(printf END)===",
|
||
)
|
||
log = open(paths.serial_log, "w")
|
||
child = spawn(args, env, log)
|
||
try:
|
||
first_prompt = child.expect([r"[Pp]assphrase", r"login:"], timeout=300)
|
||
login_ready = first_prompt == 1
|
||
if first_prompt == 0:
|
||
child.sendline(LUKS_PW)
|
||
logged_in = False
|
||
for _ in range(5):
|
||
if not login_ready:
|
||
child.expect(r"phoebus login:", timeout=180)
|
||
login_ready = False
|
||
child.sendline(USER)
|
||
idx = child.expect([r"[Pp]assword:", r"phoebus login:",
|
||
r"Login incorrect"], timeout=30)
|
||
if idx == 0:
|
||
child.sendline(USER_PW)
|
||
login_result = child.expect([SHELL_PROMPT, r"Login incorrect",
|
||
r"phoebus login:"], timeout=60)
|
||
if login_result == 0:
|
||
logged_in = True
|
||
break
|
||
login_ready = login_result == 2
|
||
else:
|
||
login_ready = idx == 1
|
||
results.assert_true("detached-boot-login", logged_in, "could not log in")
|
||
if os.environ.get("BOOT_AUTHENTICATE") == "1":
|
||
child.sendline("sudo -v")
|
||
idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", SHELL_PROMPT], timeout=30)
|
||
if idx == 0:
|
||
child.sendline(USER_PW)
|
||
child.expect(SHELL_PROMPT, timeout=30)
|
||
child.sendline(checks)
|
||
child.expect("===CHECKS-END===", timeout=int(os.environ.get("CHECKS_TIMEOUT", "180")))
|
||
serial = paths.serial_log.read_text(errors="replace")
|
||
checks_match = re.search(r"===CHECKS-START===(.*?)===CHECKS-END===", serial, re.DOTALL)
|
||
checks_text = checks_match.group(1) if checks_match else serial
|
||
|
||
m = re.search(r"SYSTEMD_STATUS:([a-z\-]+)", checks_text)
|
||
status = m.group(1) if m else "unknown"
|
||
results.assert_true(
|
||
"boot-systemd-running",
|
||
status in ("running", "degraded"),
|
||
f"systemd did not report running/degraded (observed: {status})",
|
||
)
|
||
for case in os.environ.get("BOOT_REQUIRE_CASES", "").split(","):
|
||
case = case.strip()
|
||
if case:
|
||
results.assert_true(
|
||
f"boot-case-{case}",
|
||
f"CASE {case} PASS" in checks_text,
|
||
f"{case} did not report PASS",
|
||
)
|
||
if os.environ.get("BOOT_POWEROFF") == "1":
|
||
child.sendline("sudo systemctl poweroff")
|
||
idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", pexpect.EOF], timeout=60)
|
||
if idx == 0:
|
||
child.sendline(USER_PW)
|
||
child.expect(pexpect.EOF, timeout=120)
|
||
results.pass_("boot-clean-poweroff")
|
||
except pexpect.TIMEOUT as exc:
|
||
results.fail("boot", f"timeout: {exc}")
|
||
except pexpect.EOF:
|
||
results.fail("boot", "unexpected EOF")
|
||
finally:
|
||
close_quietly(child)
|
||
log.close()
|
||
|
||
|
||
def main() -> int:
|
||
mode = sys.argv[1] if len(sys.argv) > 1 else ""
|
||
if mode not in ("install", "boot", "rollback"):
|
||
print(__doc__)
|
||
return 2
|
||
results = Results(f"serial-{mode}")
|
||
paths = vmsafety.make_run_paths(
|
||
f"serial-{mode}", base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"),
|
||
ovmf_vars_template=OVMF_VARS_TEMPLATE,
|
||
)
|
||
try:
|
||
disks = vmsafety.validate_writable_drives(DISKS.split(","), disposable_dir())
|
||
except vmsafety.UnsafeDiskError as exc:
|
||
results.blocked("disposable-disk", str(exc))
|
||
results.write(paths.results_json)
|
||
print(f"harness error: {exc}", file=sys.stderr)
|
||
return 2
|
||
results.pass_("disposable-disk", ", ".join(str(d) for d in disks))
|
||
try:
|
||
if mode == "install":
|
||
do_install(results, paths, disks)
|
||
elif mode == "rollback":
|
||
do_rollback(results, paths, disks)
|
||
else:
|
||
do_boot(results, paths, disks)
|
||
finally:
|
||
results.write(paths.results_json)
|
||
print(results.summary(), flush=True)
|
||
return results.exit_code()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|