408 lines
17 KiB
Bash
408 lines
17 KiB
Bash
#!/usr/bin/env python3
|
|
"""Headless serial driver for the disposable UEFI VM.
|
|
|
|
Modes:
|
|
|
|
install : boot the live ISO kernel/initrd, run a guest script that installs
|
|
Phoebus non-interactively, then power off.
|
|
boot : boot the installed disk alone (no ISO, no -kernel), unlock LUKS,
|
|
log in and run verification commands.
|
|
rollback: boot an installed disk, record a known-good state, create a
|
|
recovery point through `sys snapshot`, introduce a reversible
|
|
update, run the real `sys undo`, power off and boot detached to
|
|
verify the restored root/var and package state. Destructive: point
|
|
DISK at a disposable clone, never a physical device.
|
|
|
|
Everything is parameterised through environment variables so the script is not
|
|
tied to one host. See tests/vm/README.md for the variables.
|
|
|
|
Safety: `DISK`/`DISKS` must be qcow2 files inside the disposable directory named
|
|
by `PHOEBUS_VM_DISKS` (default `$PHOEBUS_VM_STATE/disks`); block devices,
|
|
`/dev/*`, escaping symlinks, raw images and outside backing chains are refused.
|
|
Results are structured and the exit status is nonzero on failure.
|
|
"""
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
import pexpect
|
|
|
|
import vmsafety
|
|
from vmsafety import Results
|
|
|
|
QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64")
|
|
LIBDIR = os.environ.get("QEMU_LIBDIR", "")
|
|
DATADIR = os.environ.get("QEMU_DATADIR", "")
|
|
OVMF_CODE = os.environ["OVMF_CODE"]
|
|
OVMF_VARS_TEMPLATE = os.environ["OVMF_VARS"]
|
|
DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "")
|
|
ISO = os.environ.get("ISO", "")
|
|
KERNEL = os.environ.get("KERNEL", "")
|
|
INITRD = os.environ.get("INITRD", "")
|
|
LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test")
|
|
USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test")
|
|
USER = os.environ.get("USERNAME", "phoebus")
|
|
MEM = os.environ.get("VM_MEM", "8192")
|
|
SMP = os.environ.get("VM_SMP", "8")
|
|
|
|
BLS_APPEND = (
|
|
"root=live:CDLABEL=gentoo-amd64-livegui rd.live.dir=/ "
|
|
"rd.live.squashimg=image.squashfs cdroot console=ttyS0,115200"
|
|
)
|
|
|
|
# Rollback acceptance fixtures, served from the host over QEMU user networking.
|
|
ROLLBACK_URL = os.environ.get("ROLLBACK_URL", "http://10.0.2.2:8000/rollback-guest.sh")
|
|
ROLLBACK_VERIFY_URL = os.environ.get(
|
|
"ROLLBACK_VERIFY_URL", "http://10.0.2.2:8000/rollback-verify.sh"
|
|
)
|
|
SYS_URL = os.environ.get("SYS_URL", "http://10.0.2.2:8000/sys")
|
|
|
|
|
|
def disposable_dir():
|
|
import pathlib
|
|
|
|
base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")
|
|
return pathlib.Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks"))
|
|
|
|
|
|
def base_args(ovmf_vars):
|
|
env = dict(os.environ)
|
|
if LIBDIR:
|
|
env["LD_LIBRARY_PATH"] = LIBDIR
|
|
args = ["-machine", "q35,accel=kvm"]
|
|
if DATADIR:
|
|
args += ["-L", DATADIR]
|
|
args += [
|
|
"-cpu", "host", "-smp", SMP, "-m", MEM,
|
|
"-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}",
|
|
"-drive", f"if=pflash,format=raw,file={ovmf_vars}",
|
|
"-netdev", "user,id=n0", "-device", "virtio-net-pci,netdev=n0,romfile=",
|
|
"-device", "virtio-rng-pci", "-vga", "std", "-display", "none",
|
|
"-serial", "stdio", "-monitor", "none", "-no-reboot",
|
|
]
|
|
return env, args
|
|
|
|
|
|
def spawn(args, env, log):
|
|
child = pexpect.spawn(QEMU, args, env=env, timeout=600,
|
|
encoding="utf-8", codec_errors="replace",
|
|
maxread=65536)
|
|
child.delaybeforesend = 0.3
|
|
# Log only guest output. The general logfile also records bytes sent by
|
|
# pexpect, including LUKS, login and sudo passwords.
|
|
child.logfile_read = log
|
|
return child
|
|
|
|
|
|
def close_quietly(child):
|
|
"""Best-effort teardown. After the guest powers off, pexpect can raise
|
|
`ExceptionPexpect: Could not terminate the child`; that must not turn a
|
|
successful run into a nonzero driver exit."""
|
|
try:
|
|
child.close(force=True)
|
|
except Exception: # noqa: BLE001 - cleanup must never mask the result
|
|
pass
|
|
|
|
|
|
def do_install(results, paths, disks):
|
|
env, args = base_args(str(paths.ovmf_vars))
|
|
for disk in disks:
|
|
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
|
args += ["-drive", f"file={ISO},media=cdrom,readonly=on",
|
|
"-kernel", KERNEL, "-initrd", INITRD, "-append", BLS_APPEND]
|
|
log = open(paths.serial_log, "w")
|
|
child = spawn(args, env, log)
|
|
try:
|
|
child.expect(r"root@livecd[^\n]*# ", timeout=600)
|
|
results.pass_("live-shell")
|
|
url = os.environ.get("INSTALL_URL", "http://10.0.2.2:8000/install.sh")
|
|
child.sendline(f"curl -fsSL {url} -o /tmp/run.sh && bash /tmp/run.sh")
|
|
child.expect([r"===INSTALL-RC=([0-9]+)===", "===FETCH-FAIL===",
|
|
"Kernel panic", pexpect.EOF, pexpect.TIMEOUT], timeout=9000)
|
|
# `child.after` is the matched pattern class (not a string) when EOF or
|
|
# TIMEOUT matched; handle that instead of raising a TypeError.
|
|
matched = child.after if isinstance(child.after, str) else ""
|
|
if "INSTALL-RC" in matched:
|
|
rc = child.match.group(1)
|
|
results.assert_true("install-rc-zero", rc == "0", f"INSTALL-RC={rc}")
|
|
else:
|
|
results.fail(
|
|
"install-rc-zero",
|
|
f"install did not report a status (matched {matched!r})",
|
|
)
|
|
serial = paths.serial_log.read_text(errors="replace")
|
|
results.assert_true("no-secret-leak", "SECRET_LEAK_FOUND" not in serial,
|
|
"the guest reported a secret leak")
|
|
child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600)
|
|
except pexpect.TIMEOUT as exc:
|
|
results.fail("install", f"timeout: {exc}")
|
|
except pexpect.EOF:
|
|
results.fail("install", "unexpected EOF")
|
|
finally:
|
|
close_quietly(child)
|
|
log.close()
|
|
|
|
|
|
def _installed_args(paths, disks):
|
|
env, args = base_args(str(paths.ovmf_vars))
|
|
for disk in disks:
|
|
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
|
args += ["-boot", "order=c"]
|
|
return env, args
|
|
|
|
|
|
def _unlock_and_login(child, results, tag):
|
|
child.expect([r"[Pp]assphrase", r"login:"], timeout=300)
|
|
if "assphrase" in (child.after or ""):
|
|
override = os.environ.get("BOOT_PASSPHRASE")
|
|
candidates = [override] if override else [USER_PW, LUKS_PW]
|
|
child.sendline(candidates[0])
|
|
for candidate in candidates[1:]:
|
|
try:
|
|
child.expect(r"[Pp]assphrase", timeout=15)
|
|
child.sendline(candidate)
|
|
except (pexpect.TIMEOUT, pexpect.EOF):
|
|
break
|
|
logged_in = False
|
|
for _ in range(5):
|
|
child.expect(r"phoebus login:", timeout=180)
|
|
child.sendline(USER)
|
|
idx = child.expect([r"[Pp]assword:", r"phoebus login:", r"Login incorrect"],
|
|
timeout=30)
|
|
if idx == 0:
|
|
child.sendline(USER_PW)
|
|
if child.expect([r"[$#] $", r"Login incorrect", r"phoebus login:"],
|
|
timeout=60) == 0:
|
|
logged_in = True
|
|
break
|
|
results.assert_true(f"{tag}-login", logged_in, f"{tag}: could not log in")
|
|
|
|
|
|
def _fetch_sys(child):
|
|
# The installed system should already carry /usr/local/bin/sys; only fall
|
|
# back to fetching the host build when it is absent.
|
|
child.sendline(
|
|
"if [ -x /usr/local/bin/sys ]; then SYS=/usr/local/bin/sys; "
|
|
"elif [ -x /tmp/sys ]; then SYS=/tmp/sys; "
|
|
f"else curl -fsSL {SYS_URL} -o /tmp/sys && chmod 755 /tmp/sys && SYS=/tmp/sys; fi; "
|
|
"test -x \"$SYS\" && echo SYS-OK-$(printf READY)"
|
|
)
|
|
child.expect("SYS-OK-READY", timeout=120)
|
|
|
|
|
|
def _parse_marker(text, key):
|
|
import re
|
|
|
|
match = re.search(r"===" + re.escape(key) + r"(.+?)===", text)
|
|
if not match:
|
|
return None
|
|
out = {}
|
|
for token in match.group(1).split():
|
|
if "=" in token:
|
|
name, value = token.split("=", 1)
|
|
out[name] = value
|
|
return out
|
|
|
|
|
|
def do_rollback(results, paths, disks):
|
|
"""Snapshot -> reversible update -> sys undo -> detached verify."""
|
|
# Phase 1: known-good -> snapshot -> break -> undo, then power off.
|
|
env, args = _installed_args(paths, disks)
|
|
log = open(paths.serial_log, "w")
|
|
child = spawn(args, env, log)
|
|
phase1 = ""
|
|
try:
|
|
_unlock_and_login(child, results, "rollback-before")
|
|
_fetch_sys(child)
|
|
child.sendline(f"curl -fsSL {ROLLBACK_URL} -o /tmp/rb.sh && bash /tmp/rb.sh")
|
|
child.expect(
|
|
["===ROLLBACK-DONE===", "curl:", pexpect.TIMEOUT, pexpect.EOF],
|
|
timeout=int(os.environ.get("ROLLBACK_TIMEOUT", "7200")),
|
|
)
|
|
if "ROLLBACK-DONE" not in (child.after or ""):
|
|
results.fail("rollback-sequence", f"the guest rollback did not finish: {child.after!r}")
|
|
else:
|
|
results.pass_("rollback-sequence")
|
|
child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600)
|
|
phase1 = paths.serial_log.read_text(errors="replace")
|
|
except pexpect.TIMEOUT as exc:
|
|
results.fail("rollback-before", f"timeout: {exc}")
|
|
except pexpect.EOF:
|
|
results.fail("rollback-before", "unexpected EOF")
|
|
finally:
|
|
close_quietly(child)
|
|
log.close()
|
|
|
|
before = _parse_marker(phase1, "ROLLBACK-KNOWN-GOOD")
|
|
undo = _parse_marker(phase1, "ROLLBACK-UNDO")
|
|
if before is None:
|
|
results.fail("rollback-known-good", "the guest did not record the known-good state")
|
|
if undo is None or undo.get("rc") != "0":
|
|
results.fail("rollback-undo", f"sys undo did not succeed: {undo}")
|
|
|
|
# Phase 2: boot the disk detached and verify the restored generation.
|
|
env, args = _installed_args(paths, disks)
|
|
log = open(paths.serial_log, "w")
|
|
child = spawn(args, env, log)
|
|
phase2 = ""
|
|
try:
|
|
_unlock_and_login(child, results, "rollback-after")
|
|
_fetch_sys(child)
|
|
child.sendline(f"curl -fsSL {ROLLBACK_VERIFY_URL} -o /tmp/rbv.sh && bash /tmp/rbv.sh")
|
|
child.expect("===AFTER-DONE===",
|
|
timeout=int(os.environ.get("ROLLBACK_VERIFY_TIMEOUT", "900")))
|
|
phase2 = paths.serial_log.read_text(errors="replace")
|
|
results.assert_true("rollback-systemd-running", "running" in phase2,
|
|
"systemd did not report running after rollback")
|
|
after = _parse_marker(phase2, "AFTER-KNOWN-GOOD")
|
|
if before is None or after is None:
|
|
results.fail("rollback-state", "could not read the before/after state")
|
|
else:
|
|
results.assert_true(
|
|
"rollback-package-state-restored",
|
|
before.get("pkgs") == after.get("pkgs"),
|
|
f"package db changed across rollback: before={before.get('pkgs')} after={after.get('pkgs')}",
|
|
)
|
|
results.assert_true(
|
|
"rollback-home-survives",
|
|
before.get("home") == after.get("home"),
|
|
f"/home did not survive: before={before.get('home')} after={after.get('home')}",
|
|
)
|
|
results.assert_true("rollback-modules-present", "===AFTER-MODULES ok===" in phase2,
|
|
"no kernel modules for the running kernel")
|
|
results.assert_true(
|
|
"rollback-recovery-entry",
|
|
"systemd.unit=rescue.target" in phase2 and "@snapshots/" in phase2,
|
|
"the snapshot recovery entry is missing or not snapshot-based",
|
|
)
|
|
status = phase2.split("===AFTER-STATUS===", 1)
|
|
status_text = status[1].split("===AFTER-DONE===", 1)[0].strip() if len(status) > 1 else ""
|
|
results.assert_true("rollback-status-recorded", bool(status_text),
|
|
"sys rollback status produced no evidence")
|
|
except pexpect.TIMEOUT as exc:
|
|
results.fail("rollback-after", f"timeout: {exc}")
|
|
except pexpect.EOF:
|
|
results.fail("rollback-after", "unexpected EOF")
|
|
finally:
|
|
close_quietly(child)
|
|
log.close()
|
|
|
|
|
|
def do_boot(results, paths, disks):
|
|
env, args = base_args(str(paths.ovmf_vars))
|
|
for disk in disks:
|
|
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
|
args += ["-boot", "order=c"]
|
|
# The end marker is built with a command substitution so the shell's echo of
|
|
# the typed line does not itself contain it; otherwise `expect` would match
|
|
# the echo and close the console before the checks ran.
|
|
checks = os.environ.get(
|
|
"BOOT_CHECKS",
|
|
"echo ===CHECKS-$(printf START)===; "
|
|
"echo COMM_CHECK:$(cat /proc/1/comm 2>&1); "
|
|
"echo SYSTEMD_STATUS:$(systemctl is-system-running 2>&1); "
|
|
"systemctl --failed --no-pager; ip -brief addr; "
|
|
"lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS; echo ===CHECKS-$(printf END)===",
|
|
)
|
|
log = open(paths.serial_log, "w")
|
|
child = spawn(args, env, log)
|
|
try:
|
|
first_prompt = child.expect([r"[Pp]assphrase", r"login:"], timeout=300)
|
|
login_ready = first_prompt == 1
|
|
if first_prompt == 0:
|
|
child.sendline(LUKS_PW)
|
|
logged_in = False
|
|
for _ in range(5):
|
|
if not login_ready:
|
|
child.expect(r"phoebus login:", timeout=180)
|
|
login_ready = False
|
|
child.sendline(USER)
|
|
idx = child.expect([r"[Pp]assword:", r"phoebus login:",
|
|
r"Login incorrect"], timeout=30)
|
|
if idx == 0:
|
|
child.sendline(USER_PW)
|
|
login_result = child.expect([r"[$#] $", r"Login incorrect",
|
|
r"phoebus login:"], timeout=60)
|
|
if login_result == 0:
|
|
logged_in = True
|
|
break
|
|
login_ready = login_result == 2
|
|
else:
|
|
login_ready = idx == 1
|
|
results.assert_true("detached-boot-login", logged_in, "could not log in")
|
|
if os.environ.get("BOOT_AUTHENTICATE") == "1":
|
|
child.sendline("sudo -v")
|
|
idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", r"[$#] $"], timeout=30)
|
|
if idx == 0:
|
|
child.sendline(USER_PW)
|
|
child.expect(r"[$#] $", timeout=30)
|
|
child.sendline(checks)
|
|
child.expect("===CHECKS-END===", timeout=int(os.environ.get("CHECKS_TIMEOUT", "180")))
|
|
serial = paths.serial_log.read_text(errors="replace")
|
|
checks_match = re.search(r"===CHECKS-START===(.*?)===CHECKS-END===", serial, re.DOTALL)
|
|
checks_text = checks_match.group(1) if checks_match else serial
|
|
|
|
m = re.search(r"SYSTEMD_STATUS:([a-z\-]+)", checks_text)
|
|
status = m.group(1) if m else "unknown"
|
|
results.assert_true(
|
|
"boot-systemd-running",
|
|
status in ("running", "degraded"),
|
|
f"systemd did not report running/degraded (observed: {status})",
|
|
)
|
|
for case in os.environ.get("BOOT_REQUIRE_CASES", "").split(","):
|
|
case = case.strip()
|
|
if case:
|
|
results.assert_true(
|
|
f"boot-case-{case}",
|
|
f"CASE {case} PASS" in checks_text,
|
|
f"{case} did not report PASS",
|
|
)
|
|
if os.environ.get("BOOT_POWEROFF") == "1":
|
|
child.sendline("sudo systemctl poweroff")
|
|
idx = child.expect([r"[Pp]assword(?: for [^:]+)?:", pexpect.EOF], timeout=60)
|
|
if idx == 0:
|
|
child.sendline(USER_PW)
|
|
child.expect(pexpect.EOF, timeout=120)
|
|
results.pass_("boot-clean-poweroff")
|
|
except pexpect.TIMEOUT as exc:
|
|
results.fail("boot", f"timeout: {exc}")
|
|
except pexpect.EOF:
|
|
results.fail("boot", "unexpected EOF")
|
|
finally:
|
|
close_quietly(child)
|
|
log.close()
|
|
|
|
|
|
def main() -> int:
|
|
mode = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
if mode not in ("install", "boot", "rollback"):
|
|
print(__doc__)
|
|
return 2
|
|
results = Results(f"serial-{mode}")
|
|
paths = vmsafety.make_run_paths(
|
|
f"serial-{mode}", base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"),
|
|
ovmf_vars_template=OVMF_VARS_TEMPLATE,
|
|
)
|
|
try:
|
|
disks = vmsafety.validate_writable_drives(DISKS.split(","), disposable_dir())
|
|
except vmsafety.UnsafeDiskError as exc:
|
|
results.blocked("disposable-disk", str(exc))
|
|
results.write(paths.results_json)
|
|
print(f"harness error: {exc}", file=sys.stderr)
|
|
return 2
|
|
results.pass_("disposable-disk", ", ".join(str(d) for d in disks))
|
|
try:
|
|
if mode == "install":
|
|
do_install(results, paths, disks)
|
|
elif mode == "rollback":
|
|
do_rollback(results, paths, disks)
|
|
else:
|
|
do_boot(results, paths, disks)
|
|
finally:
|
|
results.write(paths.results_json)
|
|
print(results.summary(), flush=True)
|
|
return results.exit_code()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|