419 lines
15 KiB
Bash
419 lines
15 KiB
Bash
#!/usr/bin/env python3
|
|
"""Disposable-VM acceptance driver for rollback and no-model recovery.
|
|
|
|
This is the *live* counterpart to ``rollback_acceptance.py``: it boots an already
|
|
installed qcow2 disk (the installer ISO is **never** attached), drives the
|
|
serial console, and runs the ordered journey in
|
|
:mod:`rollback_acceptance`. Its ``results.json`` feeds the
|
|
``updates-and-rollback`` and ``no-model-recovery`` release gates.
|
|
|
|
Modes (the entry point is explicit; nothing runs by accident):
|
|
|
|
plan print the steps, the pinned input identities and the QEMU safety
|
|
checks, then exit 0 without starting a VM. Safe on any host.
|
|
run the full journey: snapshot, package change, breakage, `sys undo`,
|
|
detached reboot, restoration checks, then a recovery boot with
|
|
networking disabled, and write both scenarios' evidence.
|
|
recovery re-verify only the no-model recovery boot. Boots normally once to
|
|
arm the recovery entry, then reboots with no network. Writes the
|
|
``rollback-recovery`` evidence.
|
|
|
|
Safety: every writable disk is validated by :mod:`vmsafety` before QEMU starts.
|
|
It must be a qcow2 file inside ``PHOEBUS_VM_DISKS``; ``/dev/*`` paths, block and
|
|
character devices, raw images, escaping symlinks and outside backing chains are
|
|
refused. The installer ISO is deliberately not a boot input here, and the
|
|
recovery boot uses ``-net none``.
|
|
|
|
Environment (all paths must be local; no secrets on the command line):
|
|
|
|
OVMF_CODE, OVMF_VARS UEFI firmware and variable template (required)
|
|
DISK (or DISKS) disposable qcow2 installed disk(s)
|
|
PHOEBUS_VM_DISKS directory the disks must live in
|
|
ISO installer ISO whose digest is pinned (required)
|
|
PHOEBUS_VM_SYS host path to the `sys` binary under test
|
|
PHOEBUS_VM_SYS_SHA256 optional expected digest (otherwise computed)
|
|
PHOEBUS_VM_ARTIFACT[_SHA256] release tool candidate binding (optional here)
|
|
LUKS_PASSPHRASE, USER_PASSWORD, USERNAME
|
|
SYS_BIN guest `sys` path (default /usr/local/bin/sys)
|
|
ROLLBACK_PACKAGE controlled package change (default app-misc/hello)
|
|
QEMU_BIN, VM_MEM, VM_SMP, PHOEBUS_VM_STATE, PHOEBUS_VM_STRICT
|
|
|
|
Example (see docs/ROLLBACK-ACCEPTANCE.md):
|
|
|
|
PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \\
|
|
DISK=/var/tmp/phoebus-vm/disks/rollback.qcow2 \\
|
|
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \\
|
|
OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \\
|
|
ISO=/path/phoebus-installer-amd64.iso \\
|
|
PHOEBUS_VM_SYS=/path/sys \\
|
|
python3 installer/tests/vm/rollback-driver.py run
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pexpect
|
|
|
|
import vmsafety
|
|
from rollback_acceptance import (
|
|
EXPECTED_STEP_NAMES,
|
|
REQUIRED_INPUTS,
|
|
RC_SENTINEL,
|
|
SCENARIO_RECOVERY,
|
|
SCENARIO_ROLLBACK,
|
|
CommandResult,
|
|
RollbackJourney,
|
|
validate_acceptance_disks,
|
|
)
|
|
from vmsafety import INPUT_ISO, INPUT_SYS, Results
|
|
|
|
QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64")
|
|
LIBDIR = os.environ.get("QEMU_LIBDIR", "")
|
|
DATADIR = os.environ.get("QEMU_DATADIR", "")
|
|
OVMF_CODE = os.environ.get("OVMF_CODE", "")
|
|
OVMF_VARS_TEMPLATE = os.environ.get("OVMF_VARS", "")
|
|
DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "")
|
|
ISO = os.environ.get("ISO", "")
|
|
SYS_HOST = os.environ.get("PHOEBUS_VM_SYS", "")
|
|
SYS_BIN = os.environ.get("SYS_BIN", "/usr/local/bin/sys")
|
|
PACKAGE = os.environ.get("ROLLBACK_PACKAGE", "app-misc/hello")
|
|
HOME = os.environ.get("ROLLBACK_HOME", "/home/phoebus")
|
|
LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test")
|
|
USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test")
|
|
USER = os.environ.get("USERNAME", "phoebus")
|
|
MEM = os.environ.get("VM_MEM", "8192")
|
|
SMP = os.environ.get("VM_SMP", "8")
|
|
BOOT_TIMEOUT = int(os.environ.get("BOOT_TIMEOUT", "600"))
|
|
LOGIN_TIMEOUT = int(os.environ.get("LOGIN_TIMEOUT", "300"))
|
|
PROMPT = os.environ.get("GUEST_PROMPT", r"[$#] $")
|
|
|
|
|
|
def disposable_dir() -> Path:
|
|
base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")
|
|
return Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks"))
|
|
|
|
|
|
def sha256_file(path: str | os.PathLike) -> str:
|
|
digest = hashlib.sha256()
|
|
with open(path, "rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def strip_echo(text: str, command: str) -> str:
|
|
"""Drop the console echo of a typed command and the rc sentinel line."""
|
|
lines = text.splitlines()
|
|
if lines and command and command[:40] in lines[0]:
|
|
lines = lines[1:]
|
|
return "\n".join(line for line in lines if RC_SENTINEL not in line)
|
|
|
|
|
|
class PexpectGuest:
|
|
"""A :class:`rollback_acceptance.Guest` backed by a serial console."""
|
|
|
|
def __init__(self, child, log, user_pw: str):
|
|
self.child = child
|
|
self.log = log
|
|
self.user_pw = user_pw
|
|
|
|
def _ensure_sudo(self) -> None:
|
|
"""Refresh the sudo timestamp, feeding the password off the log.
|
|
|
|
``sudo -n true`` succeeds while the timestamp is valid; otherwise
|
|
``sudo -v`` prompts and the password is sent with the pexpect logfile
|
|
detached, so it never reaches the serial transcript.
|
|
"""
|
|
self.child.sendline("sudo -n true 2>/dev/null || sudo -v")
|
|
idx = self.child.expect(
|
|
[r"\[sudo\] password for [^:]+:", PROMPT, pexpect.TIMEOUT], timeout=30
|
|
)
|
|
if idx == 0:
|
|
saved = self.child.logfile
|
|
self.child.logfile = None
|
|
try:
|
|
self.child.sendline(self.user_pw)
|
|
self.child.expect(PROMPT, timeout=30)
|
|
finally:
|
|
self.child.logfile = saved
|
|
elif idx == 1:
|
|
return
|
|
else: # pragma: no cover - console fault
|
|
raise RuntimeError("sudo timestamp could not be refreshed")
|
|
|
|
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
|
try:
|
|
if "sudo" in command:
|
|
self._ensure_sudo()
|
|
end = f"echo {RC_SENTINEL}=$(printf '%s' $?)"
|
|
self.child.sendline(f"{command}; {end}")
|
|
idx = self.child.expect(
|
|
[rf"{RC_SENTINEL}=([0-9]+)", pexpect.TIMEOUT, pexpect.EOF],
|
|
timeout=timeout,
|
|
)
|
|
except (pexpect.TIMEOUT, pexpect.EOF):
|
|
return CommandResult(command=command, disconnected=True, stdout="")
|
|
if idx == 0:
|
|
return CommandResult(
|
|
command=command,
|
|
stdout=strip_echo(self.child.before or "", command),
|
|
exit_code=int(self.child.match.group(1)),
|
|
)
|
|
if idx == 1:
|
|
return CommandResult(
|
|
command=command, timed_out=True, stdout=self.child.before or ""
|
|
)
|
|
return CommandResult(
|
|
command=command, disconnected=True, stdout=self.child.before or ""
|
|
)
|
|
|
|
|
|
class PexpectTransport:
|
|
"""Boots the installed disk; the installer ISO is never attached."""
|
|
|
|
def __init__(self, disks, run_paths):
|
|
self.disks = disks
|
|
self.paths = run_paths
|
|
self.child = None
|
|
self.log = None
|
|
self._boot = 0
|
|
|
|
def _args(self, *, network: bool) -> list[str]:
|
|
env = dict(os.environ)
|
|
if LIBDIR:
|
|
env["LD_LIBRARY_PATH"] = LIBDIR
|
|
args = ["-machine", "q35,accel=kvm", "-cpu", "host", "-smp", SMP, "-m", MEM]
|
|
if DATADIR:
|
|
args += ["-L", DATADIR]
|
|
args += [
|
|
"-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}",
|
|
"-drive", f"if=pflash,format=raw,file={self.paths.ovmf_vars}",
|
|
]
|
|
for disk in self.disks:
|
|
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
|
if network:
|
|
args += ["-netdev", "user,id=n0",
|
|
"-device", "virtio-net-pci,netdev=n0,romfile="]
|
|
else:
|
|
# Recovery must work with networking disabled.
|
|
args += ["-net", "none"]
|
|
args += [
|
|
"-device", "virtio-rng-pci", "-vga", "std", "-display", "none",
|
|
"-serial", "stdio", "-monitor", "none", "-no-reboot",
|
|
]
|
|
return args
|
|
|
|
def _spawn(self, *, network: bool) -> None:
|
|
self._boot += 1
|
|
log_path = self.paths.directory / f"serial-boot{self._boot}.log"
|
|
self.log = open(log_path, "w")
|
|
env = dict(os.environ)
|
|
if LIBDIR:
|
|
env["LD_LIBRARY_PATH"] = LIBDIR
|
|
self.child = pexpect.spawn(
|
|
QEMU,
|
|
self._args(network=network),
|
|
env=env,
|
|
timeout=BOOT_TIMEOUT,
|
|
encoding="utf-8",
|
|
codec_errors="replace",
|
|
maxread=65536,
|
|
)
|
|
self.child.delaybeforesend = 0.4
|
|
self.child.logfile = self.log
|
|
|
|
def _unlock_and_login(self) -> None:
|
|
assert self.child is not None
|
|
self.child.expect([r"[Pp]assphrase", r"login:"], timeout=BOOT_TIMEOUT)
|
|
if "assphrase" in (self.child.after or "").lower():
|
|
self.child.sendline(LUKS_PW)
|
|
logged_in = False
|
|
for _ in range(5):
|
|
self.child.expect(fr"{re.escape(USER)} login:", timeout=LOGIN_TIMEOUT)
|
|
self.child.sendline(USER)
|
|
idx = self.child.expect(
|
|
[r"[Pp]assword:", fr"{re.escape(USER)} login:", r"Login incorrect"],
|
|
timeout=60,
|
|
)
|
|
if idx == 0:
|
|
self.child.sendline(USER_PW)
|
|
if self.child.expect([PROMPT, r"Login incorrect", fr"{re.escape(USER)} login:"],
|
|
timeout=120) == 0:
|
|
logged_in = True
|
|
break
|
|
if not logged_in:
|
|
raise RuntimeError("could not log in on the installed system")
|
|
|
|
def boot_installed(self):
|
|
self._spawn(network=True)
|
|
self._unlock_and_login()
|
|
return PexpectGuest(self.child, self.log, USER_PW)
|
|
|
|
def boot_recovery(self):
|
|
self._spawn(network=False)
|
|
self._unlock_and_login()
|
|
return PexpectGuest(self.child, self.log, USER_PW)
|
|
|
|
def poweroff(self) -> None:
|
|
if self.child is None:
|
|
return
|
|
try:
|
|
PexpectGuest(self.child, self.log, USER_PW).run("sudo poweroff -f", timeout=60)
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
try:
|
|
self.child.close(force=True)
|
|
finally:
|
|
if self.log:
|
|
self.log.close()
|
|
self.child = None
|
|
self.log = None
|
|
|
|
|
|
def pin_inputs(results: Results, *, iso: str, sys_host: str) -> dict:
|
|
"""Bind the ISO and `sys` identities the run exercised."""
|
|
bound = {}
|
|
if iso:
|
|
results.bind_input_file(INPUT_ISO, iso)
|
|
bound[INPUT_ISO] = results.inputs[INPUT_ISO]
|
|
if sys_host:
|
|
expected = os.environ.get("PHOEBUS_VM_SYS_SHA256")
|
|
digest = expected or sha256_file(sys_host)
|
|
results.bind_input(INPUT_SYS, digest)
|
|
bound[INPUT_SYS] = results.inputs[INPUT_SYS]
|
|
results.bind_artifact_from_env()
|
|
return bound
|
|
|
|
|
|
def plan_text() -> str:
|
|
lines = [
|
|
"rollback acceptance journey (no VM started)",
|
|
"scenario: " + SCENARIO_ROLLBACK,
|
|
"recovery: " + SCENARIO_RECOVERY,
|
|
"steps:",
|
|
]
|
|
lines += [f" {i + 1:2d}. {name}" for i, name in enumerate(EXPECTED_STEP_NAMES)]
|
|
for scenario, required in REQUIRED_INPUTS.items():
|
|
lines.append(f"required inputs for {scenario}: {', '.join(required)}")
|
|
lines.append("safety: every DISK must be a qcow2 inside PHOEBUS_VM_DISKS;")
|
|
lines.append(" the installer ISO is never attached; recovery uses -net none.")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def build_results(scenario: str, mocked: bool = False) -> Results:
|
|
results = Results(scenario)
|
|
if mocked:
|
|
results.mark_mocked("fake transport")
|
|
return results
|
|
|
|
|
|
def run_full(disks, run_paths, sys_digest) -> int:
|
|
rollback = build_results(SCENARIO_ROLLBACK)
|
|
recovery = build_results(SCENARIO_RECOVERY)
|
|
pin_inputs(rollback, iso=ISO, sys_host=SYS_HOST)
|
|
pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST)
|
|
transport = PexpectTransport(disks, run_paths)
|
|
journey = RollbackJourney(
|
|
transport,
|
|
results=rollback,
|
|
recovery_results=recovery,
|
|
home=HOME,
|
|
sys_bin=SYS_BIN,
|
|
package=PACKAGE,
|
|
iso_sha256=os.environ.get("PHOEBUS_VM_ARTIFACT_SHA256") or (rollback.inputs.get(INPUT_ISO) or {}).get("sha256"),
|
|
sys_sha256=sys_digest,
|
|
)
|
|
try:
|
|
journey.run()
|
|
finally:
|
|
transport.poweroff()
|
|
rollback.write(run_paths.results_json)
|
|
# The release tool imports any results.json under a directory tree, so
|
|
# the recovery scenario lives beside the rollback run.
|
|
recovery_dir = run_paths.directory / "recovery"
|
|
recovery_dir.mkdir(parents=True, exist_ok=True)
|
|
recovery.write(recovery_dir / "results.json")
|
|
print(rollback.summary())
|
|
print(recovery.summary())
|
|
return 1 if (rollback.exit_code() or recovery.exit_code()) else 0
|
|
|
|
|
|
def run_recovery_only(disks, run_paths, sys_digest) -> int:
|
|
recovery = build_results(SCENARIO_RECOVERY)
|
|
pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST)
|
|
# Arm the one-shot recovery entry on a normal boot first.
|
|
transport = PexpectTransport(disks, run_paths)
|
|
try:
|
|
guest = transport.boot_installed()
|
|
guest.run("sudo bootctl set-oneshot phoebus-recovery", timeout=120)
|
|
finally:
|
|
transport.poweroff()
|
|
transport = PexpectTransport(disks, run_paths)
|
|
journey = RollbackJourney(
|
|
transport,
|
|
results=build_results(SCENARIO_ROLLBACK),
|
|
recovery_results=recovery,
|
|
home=HOME,
|
|
sys_bin=SYS_BIN,
|
|
package=PACKAGE,
|
|
sys_sha256=sys_digest,
|
|
)
|
|
try:
|
|
journey.run_recovery()
|
|
finally:
|
|
transport.poweroff()
|
|
recovery.write(run_paths.results_json)
|
|
print(recovery.summary())
|
|
return recovery.exit_code()
|
|
|
|
|
|
def main() -> int:
|
|
mode = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
if mode == "plan":
|
|
print(plan_text())
|
|
return 0
|
|
if mode not in ("run", "recovery"):
|
|
print(__doc__)
|
|
return 2
|
|
if not OVMF_CODE or not OVMF_VARS_TEMPLATE:
|
|
print("harness error: OVMF_CODE and OVMF_VARS are required", file=sys.stderr)
|
|
return 2
|
|
if not ISO:
|
|
print("harness error: ISO is required to pin the candidate identity", file=sys.stderr)
|
|
return 2
|
|
if not SYS_HOST or not Path(SYS_HOST).is_file():
|
|
print(
|
|
"harness error: PHOEBUS_VM_SYS must point at the `sys` binary under test",
|
|
file=sys.stderr,
|
|
)
|
|
return 2
|
|
|
|
scenario = SCENARIO_ROLLBACK if mode == "run" else SCENARIO_RECOVERY
|
|
run_paths = vmsafety.make_run_paths(
|
|
f"rollback-{mode}",
|
|
base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"),
|
|
ovmf_vars_template=OVMF_VARS_TEMPLATE,
|
|
)
|
|
try:
|
|
disks = validate_acceptance_disks(DISKS.split(","), disposable_dir())
|
|
except vmsafety.UnsafeDiskError as exc:
|
|
results = build_results(scenario)
|
|
results.blocked("disposable-disk", str(exc))
|
|
results.write(run_paths.results_json)
|
|
print(f"harness error: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
sys_digest = os.environ.get("PHOEBUS_VM_SYS_SHA256") or sha256_file(SYS_HOST)
|
|
if mode == "run":
|
|
return run_full(disks, run_paths, sys_digest)
|
|
return run_recovery_only(disks, run_paths, sys_digest)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|