phoebus_os/installer/tests/vm/rollback-driver.py

419 lines
15 KiB
Bash

#!/usr/bin/env python3
"""Disposable-VM acceptance driver for rollback and no-model recovery.
This is the *live* counterpart to ``rollback_acceptance.py``: it boots an already
installed qcow2 disk (the installer ISO is **never** attached), drives the
serial console, and runs the ordered journey in
:mod:`rollback_acceptance`. Its ``results.json`` feeds the
``updates-and-rollback`` and ``no-model-recovery`` release gates.
Modes (the entry point is explicit; nothing runs by accident):
plan print the steps, the pinned input identities and the QEMU safety
checks, then exit 0 without starting a VM. Safe on any host.
run the full journey: snapshot, package change, breakage, `sys undo`,
detached reboot, restoration checks, then a recovery boot with
networking disabled, and write both scenarios' evidence.
recovery re-verify only the no-model recovery boot. Boots normally once to
arm the recovery entry, then reboots with no network. Writes the
``rollback-recovery`` evidence.
Safety: every writable disk is validated by :mod:`vmsafety` before QEMU starts.
It must be a qcow2 file inside ``PHOEBUS_VM_DISKS``; ``/dev/*`` paths, block and
character devices, raw images, escaping symlinks and outside backing chains are
refused. The installer ISO is deliberately not a boot input here, and the
recovery boot uses ``-net none``.
Environment (all paths must be local; no secrets on the command line):
OVMF_CODE, OVMF_VARS UEFI firmware and variable template (required)
DISK (or DISKS) disposable qcow2 installed disk(s)
PHOEBUS_VM_DISKS directory the disks must live in
ISO installer ISO whose digest is pinned (required)
PHOEBUS_VM_SYS host path to the `sys` binary under test
PHOEBUS_VM_SYS_SHA256 optional expected digest (otherwise computed)
PHOEBUS_VM_ARTIFACT[_SHA256] release tool candidate binding (optional here)
LUKS_PASSPHRASE, USER_PASSWORD, USERNAME
SYS_BIN guest `sys` path (default /usr/local/bin/sys)
ROLLBACK_PACKAGE controlled package change (default app-misc/hello)
QEMU_BIN, VM_MEM, VM_SMP, PHOEBUS_VM_STATE, PHOEBUS_VM_STRICT
Example (see docs/ROLLBACK-ACCEPTANCE.md):
PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \\
DISK=/var/tmp/phoebus-vm/disks/rollback.qcow2 \\
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \\
OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \\
ISO=/path/phoebus-installer-amd64.iso \\
PHOEBUS_VM_SYS=/path/sys \\
python3 installer/tests/vm/rollback-driver.py run
"""
from __future__ import annotations
import hashlib
import os
import re
import sys
from pathlib import Path
import pexpect
import vmsafety
from rollback_acceptance import (
EXPECTED_STEP_NAMES,
REQUIRED_INPUTS,
RC_SENTINEL,
SCENARIO_RECOVERY,
SCENARIO_ROLLBACK,
CommandResult,
RollbackJourney,
validate_acceptance_disks,
)
from vmsafety import INPUT_ISO, INPUT_SYS, Results
QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64")
LIBDIR = os.environ.get("QEMU_LIBDIR", "")
DATADIR = os.environ.get("QEMU_DATADIR", "")
OVMF_CODE = os.environ.get("OVMF_CODE", "")
OVMF_VARS_TEMPLATE = os.environ.get("OVMF_VARS", "")
DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "")
ISO = os.environ.get("ISO", "")
SYS_HOST = os.environ.get("PHOEBUS_VM_SYS", "")
SYS_BIN = os.environ.get("SYS_BIN", "/usr/local/bin/sys")
PACKAGE = os.environ.get("ROLLBACK_PACKAGE", "app-misc/hello")
HOME = os.environ.get("ROLLBACK_HOME", "/home/phoebus")
LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test")
USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test")
USER = os.environ.get("USERNAME", "phoebus")
MEM = os.environ.get("VM_MEM", "8192")
SMP = os.environ.get("VM_SMP", "8")
BOOT_TIMEOUT = int(os.environ.get("BOOT_TIMEOUT", "600"))
LOGIN_TIMEOUT = int(os.environ.get("LOGIN_TIMEOUT", "300"))
PROMPT = os.environ.get("GUEST_PROMPT", r"[$#] $")
def disposable_dir() -> Path:
base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")
return Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks"))
def sha256_file(path: str | os.PathLike) -> str:
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def strip_echo(text: str, command: str) -> str:
"""Drop the console echo of a typed command and the rc sentinel line."""
lines = text.splitlines()
if lines and command and command[:40] in lines[0]:
lines = lines[1:]
return "\n".join(line for line in lines if RC_SENTINEL not in line)
class PexpectGuest:
"""A :class:`rollback_acceptance.Guest` backed by a serial console."""
def __init__(self, child, log, user_pw: str):
self.child = child
self.log = log
self.user_pw = user_pw
def _ensure_sudo(self) -> None:
"""Refresh the sudo timestamp, feeding the password off the log.
``sudo -n true`` succeeds while the timestamp is valid; otherwise
``sudo -v`` prompts and the password is sent with the pexpect logfile
detached, so it never reaches the serial transcript.
"""
self.child.sendline("sudo -n true 2>/dev/null || sudo -v")
idx = self.child.expect(
[r"\[sudo\] password for [^:]+:", PROMPT, pexpect.TIMEOUT], timeout=30
)
if idx == 0:
saved = self.child.logfile
self.child.logfile = None
try:
self.child.sendline(self.user_pw)
self.child.expect(PROMPT, timeout=30)
finally:
self.child.logfile = saved
elif idx == 1:
return
else: # pragma: no cover - console fault
raise RuntimeError("sudo timestamp could not be refreshed")
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
try:
if "sudo" in command:
self._ensure_sudo()
end = f"echo {RC_SENTINEL}=$(printf '%s' $?)"
self.child.sendline(f"{command}; {end}")
idx = self.child.expect(
[rf"{RC_SENTINEL}=([0-9]+)", pexpect.TIMEOUT, pexpect.EOF],
timeout=timeout,
)
except (pexpect.TIMEOUT, pexpect.EOF):
return CommandResult(command=command, disconnected=True, stdout="")
if idx == 0:
return CommandResult(
command=command,
stdout=strip_echo(self.child.before or "", command),
exit_code=int(self.child.match.group(1)),
)
if idx == 1:
return CommandResult(
command=command, timed_out=True, stdout=self.child.before or ""
)
return CommandResult(
command=command, disconnected=True, stdout=self.child.before or ""
)
class PexpectTransport:
"""Boots the installed disk; the installer ISO is never attached."""
def __init__(self, disks, run_paths):
self.disks = disks
self.paths = run_paths
self.child = None
self.log = None
self._boot = 0
def _args(self, *, network: bool) -> list[str]:
env = dict(os.environ)
if LIBDIR:
env["LD_LIBRARY_PATH"] = LIBDIR
args = ["-machine", "q35,accel=kvm", "-cpu", "host", "-smp", SMP, "-m", MEM]
if DATADIR:
args += ["-L", DATADIR]
args += [
"-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}",
"-drive", f"if=pflash,format=raw,file={self.paths.ovmf_vars}",
]
for disk in self.disks:
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
if network:
args += ["-netdev", "user,id=n0",
"-device", "virtio-net-pci,netdev=n0,romfile="]
else:
# Recovery must work with networking disabled.
args += ["-net", "none"]
args += [
"-device", "virtio-rng-pci", "-vga", "std", "-display", "none",
"-serial", "stdio", "-monitor", "none", "-no-reboot",
]
return args
def _spawn(self, *, network: bool) -> None:
self._boot += 1
log_path = self.paths.directory / f"serial-boot{self._boot}.log"
self.log = open(log_path, "w")
env = dict(os.environ)
if LIBDIR:
env["LD_LIBRARY_PATH"] = LIBDIR
self.child = pexpect.spawn(
QEMU,
self._args(network=network),
env=env,
timeout=BOOT_TIMEOUT,
encoding="utf-8",
codec_errors="replace",
maxread=65536,
)
self.child.delaybeforesend = 0.4
self.child.logfile = self.log
def _unlock_and_login(self) -> None:
assert self.child is not None
self.child.expect([r"[Pp]assphrase", r"login:"], timeout=BOOT_TIMEOUT)
if "assphrase" in (self.child.after or "").lower():
self.child.sendline(LUKS_PW)
logged_in = False
for _ in range(5):
self.child.expect(fr"{re.escape(USER)} login:", timeout=LOGIN_TIMEOUT)
self.child.sendline(USER)
idx = self.child.expect(
[r"[Pp]assword:", fr"{re.escape(USER)} login:", r"Login incorrect"],
timeout=60,
)
if idx == 0:
self.child.sendline(USER_PW)
if self.child.expect([PROMPT, r"Login incorrect", fr"{re.escape(USER)} login:"],
timeout=120) == 0:
logged_in = True
break
if not logged_in:
raise RuntimeError("could not log in on the installed system")
def boot_installed(self):
self._spawn(network=True)
self._unlock_and_login()
return PexpectGuest(self.child, self.log, USER_PW)
def boot_recovery(self):
self._spawn(network=False)
self._unlock_and_login()
return PexpectGuest(self.child, self.log, USER_PW)
def poweroff(self) -> None:
if self.child is None:
return
try:
PexpectGuest(self.child, self.log, USER_PW).run("sudo poweroff -f", timeout=60)
except Exception:
pass
finally:
try:
self.child.close(force=True)
finally:
if self.log:
self.log.close()
self.child = None
self.log = None
def pin_inputs(results: Results, *, iso: str, sys_host: str) -> dict:
"""Bind the ISO and `sys` identities the run exercised."""
bound = {}
if iso:
results.bind_input_file(INPUT_ISO, iso)
bound[INPUT_ISO] = results.inputs[INPUT_ISO]
if sys_host:
expected = os.environ.get("PHOEBUS_VM_SYS_SHA256")
digest = expected or sha256_file(sys_host)
results.bind_input(INPUT_SYS, digest)
bound[INPUT_SYS] = results.inputs[INPUT_SYS]
results.bind_artifact_from_env()
return bound
def plan_text() -> str:
lines = [
"rollback acceptance journey (no VM started)",
"scenario: " + SCENARIO_ROLLBACK,
"recovery: " + SCENARIO_RECOVERY,
"steps:",
]
lines += [f" {i + 1:2d}. {name}" for i, name in enumerate(EXPECTED_STEP_NAMES)]
for scenario, required in REQUIRED_INPUTS.items():
lines.append(f"required inputs for {scenario}: {', '.join(required)}")
lines.append("safety: every DISK must be a qcow2 inside PHOEBUS_VM_DISKS;")
lines.append(" the installer ISO is never attached; recovery uses -net none.")
return "\n".join(lines)
def build_results(scenario: str, mocked: bool = False) -> Results:
results = Results(scenario)
if mocked:
results.mark_mocked("fake transport")
return results
def run_full(disks, run_paths, sys_digest) -> int:
rollback = build_results(SCENARIO_ROLLBACK)
recovery = build_results(SCENARIO_RECOVERY)
pin_inputs(rollback, iso=ISO, sys_host=SYS_HOST)
pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST)
transport = PexpectTransport(disks, run_paths)
journey = RollbackJourney(
transport,
results=rollback,
recovery_results=recovery,
home=HOME,
sys_bin=SYS_BIN,
package=PACKAGE,
iso_sha256=os.environ.get("PHOEBUS_VM_ARTIFACT_SHA256") or (rollback.inputs.get(INPUT_ISO) or {}).get("sha256"),
sys_sha256=sys_digest,
)
try:
journey.run()
finally:
transport.poweroff()
rollback.write(run_paths.results_json)
# The release tool imports any results.json under a directory tree, so
# the recovery scenario lives beside the rollback run.
recovery_dir = run_paths.directory / "recovery"
recovery_dir.mkdir(parents=True, exist_ok=True)
recovery.write(recovery_dir / "results.json")
print(rollback.summary())
print(recovery.summary())
return 1 if (rollback.exit_code() or recovery.exit_code()) else 0
def run_recovery_only(disks, run_paths, sys_digest) -> int:
recovery = build_results(SCENARIO_RECOVERY)
pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST)
# Arm the one-shot recovery entry on a normal boot first.
transport = PexpectTransport(disks, run_paths)
try:
guest = transport.boot_installed()
guest.run("sudo bootctl set-oneshot phoebus-recovery", timeout=120)
finally:
transport.poweroff()
transport = PexpectTransport(disks, run_paths)
journey = RollbackJourney(
transport,
results=build_results(SCENARIO_ROLLBACK),
recovery_results=recovery,
home=HOME,
sys_bin=SYS_BIN,
package=PACKAGE,
sys_sha256=sys_digest,
)
try:
journey.run_recovery()
finally:
transport.poweroff()
recovery.write(run_paths.results_json)
print(recovery.summary())
return recovery.exit_code()
def main() -> int:
mode = sys.argv[1] if len(sys.argv) > 1 else ""
if mode == "plan":
print(plan_text())
return 0
if mode not in ("run", "recovery"):
print(__doc__)
return 2
if not OVMF_CODE or not OVMF_VARS_TEMPLATE:
print("harness error: OVMF_CODE and OVMF_VARS are required", file=sys.stderr)
return 2
if not ISO:
print("harness error: ISO is required to pin the candidate identity", file=sys.stderr)
return 2
if not SYS_HOST or not Path(SYS_HOST).is_file():
print(
"harness error: PHOEBUS_VM_SYS must point at the `sys` binary under test",
file=sys.stderr,
)
return 2
scenario = SCENARIO_ROLLBACK if mode == "run" else SCENARIO_RECOVERY
run_paths = vmsafety.make_run_paths(
f"rollback-{mode}",
base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"),
ovmf_vars_template=OVMF_VARS_TEMPLATE,
)
try:
disks = validate_acceptance_disks(DISKS.split(","), disposable_dir())
except vmsafety.UnsafeDiskError as exc:
results = build_results(scenario)
results.blocked("disposable-disk", str(exc))
results.write(run_paths.results_json)
print(f"harness error: {exc}", file=sys.stderr)
return 2
sys_digest = os.environ.get("PHOEBUS_VM_SYS_SHA256") or sha256_file(SYS_HOST)
if mode == "run":
return run_full(disks, run_paths, sys_digest)
return run_recovery_only(disks, run_paths, sys_digest)
if __name__ == "__main__":
sys.exit(main())