phoebus_os/installer/tests/vm/vmsafety.py
Phoebus b8152054b6
Some checks are pending
ci / test (push) Waiting to run
fix(installer): harden boot recovery and desktop reliability
2026-10-01 07:03:42 -07:00

559 lines
19 KiB
Bash

#!/usr/bin/env python3
"""Safety, structured-result and QEMU-control helpers for the VM harnesses.
The VM drivers are test equipment: their own safety rules must be enforced, not
merely documented. This module centralises those rules so every driver shares
one implementation:
* every writable disk must be a regular **qcow2** file inside an explicitly
designated disposable directory, with a backing chain that is fully inside
that directory. Block devices, ``/dev/*`` paths, character devices, unsafe
symlinks and writable storage outside the test area are rejected;
* each run gets its own writable OVMF variable file and monitor socket;
* screenshots and monitor commands are issued without a shell, with argument
arrays, over structured QMP where possible;
* results are structured (PASS/FAIL/BLOCKED/SKIPPED/NOT RUN) and a failed,
timed-out or incomplete run exits nonzero.
Nothing here depends on pexpect, so the safety logic is unit-testable on any
host (see ``test_vmsafety.py``).
"""
from __future__ import annotations
import dataclasses
import json
import os
import shutil
import socket
import stat
import subprocess
import sys
import time
import uuid
from pathlib import Path
# ---------------------------------------------------------------------------
# Statuses
# ---------------------------------------------------------------------------
PASS = "PASS"
FAIL = "FAIL"
BLOCKED = "BLOCKED"
SKIPPED = "SKIPPED"
NOT_RUN = "NOT RUN"
_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED)
class UnsafeDiskError(Exception):
"""A writable target violated the disposable-disk rules."""
class HarnessError(Exception):
"""A harness precondition failed (bad environment, unsafe path, ...)."""
class RedactingTextLog:
"""Write guest output without recording registered credentials.
pexpect can split one echoed credential across reads, so redact complete
terminal lines rather than each write call. The final fragment is flushed
by ``finish`` when QEMU closes. Extremely long unterminated lines are
discarded instead of risking an unbounded buffer or a partial secret.
"""
def __init__(self, sink, secrets):
self.sink = sink
self.secrets = tuple(sorted({s for s in secrets if s}, key=len, reverse=True))
self.pending = ""
self.discarding = False
def _redact(self, value):
for secret in self.secrets:
value = value.replace(secret, "[REDACTED]")
return value
def write(self, value):
if not isinstance(value, str):
value = str(value)
self.pending += value
while True:
newline = self.pending.find("\n")
carriage = self.pending.find("\r")
endings = [at for at in (newline, carriage) if at >= 0]
if not endings:
break
end = min(endings) + 1
line, self.pending = self.pending[:end], self.pending[end:]
if self.discarding:
self.discarding = False
else:
self.sink.write(self._redact(line))
if len(self.pending) > 1_000_000:
self.sink.write("[REDACTED overlong serial line]\n")
self.pending = ""
self.discarding = True
def flush(self):
self.sink.flush()
def finish(self):
if self.pending and not self.discarding:
self.sink.write(self._redact(self.pending))
self.pending = ""
self.discarding = False
self.sink.flush()
# ---------------------------------------------------------------------------
# Structured results
# ---------------------------------------------------------------------------
@dataclasses.dataclass
class Check:
name: str
status: str
detail: str = ""
def to_dict(self) -> dict:
return {"name": self.name, "status": self.status, "detail": self.detail}
class Results:
"""A run's assertions and their outcome.
A check may be recorded once. ``assert_true`` records PASS or FAIL; the
helpers ``blocked``/``skip`` record those explicit states. Any FAIL (and,
when strict, any BLOCKED or NOT RUN) makes :meth:`exit_code` nonzero.
"""
def __init__(self, scenario: str, run_id: str | None = None,
strict: bool | None = None, secrets=()):
self.scenario = scenario
self.run_id = run_id or new_run_id(scenario)
self.checks: list[Check] = []
self.secrets = tuple(sorted({s for s in secrets if s}, key=len, reverse=True))
if strict is None:
strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true")
self.strict = bool(strict)
# -- recording ---------------------------------------------------------
def _add(self, name: str, status: str, detail: str) -> Check:
if status not in _TERMINAL:
raise ValueError(f"invalid result status: {status}")
for secret in self.secrets:
detail = detail.replace(secret, "[REDACTED]")
check = Check(name=name, status=status, detail=detail)
self.checks.append(check)
stream = sys.stderr if status == FAIL else sys.stdout
print(f"[{status:<7}] {name}" + (f": {detail}" if detail else ""), file=stream, flush=True)
return check
def assert_true(self, name: str, condition: bool, detail: str = "") -> bool:
ok = bool(condition)
# `detail` describes the failure; never print it on a pass, which would
# make a passing check read as if it had failed.
self._add(name, PASS if ok else FAIL, "" if ok else detail)
return ok
def fail(self, name: str, detail: str = "") -> Check:
return self._add(name, FAIL, detail)
def pass_(self, name: str, detail: str = "") -> Check:
return self._add(name, PASS, detail)
def blocked(self, name: str, detail: str = "") -> Check:
return self._add(name, BLOCKED, detail)
def skip(self, name: str, detail: str = "") -> Check:
return self._add(name, SKIPPED, detail)
# -- outcome -----------------------------------------------------------
@property
def failed(self) -> bool:
return any(c.status == FAIL for c in self.checks)
@property
def has_blockers(self) -> bool:
return any(c.status == BLOCKED for c in self.checks)
@property
def not_run(self) -> bool:
return not self.checks or any(c.status == NOT_RUN for c in self.checks)
def counts(self) -> dict:
out = {s: 0 for s in (PASS, FAIL, BLOCKED, SKIPPED, NOT_RUN)}
for c in self.checks:
out[c.status] += 1
return out
def exit_code(self) -> int:
if self.failed:
return 1
if self.strict and (self.has_blockers or self.not_run):
return 2
return 0
def summary(self) -> str:
counts = self.counts()
parts = [f"{k}={v}" for k, v in counts.items() if v]
return f"{self.scenario} [{self.run_id}]: " + (", ".join(parts) if parts else "no checks")
def write(self, path: str | Path) -> Path:
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"scenario": self.scenario,
"run_id": self.run_id,
"strict": self.strict,
"exit_code": self.exit_code(),
"counts": self.counts(),
"checks": [c.to_dict() for c in self.checks],
}
path.write_text(json.dumps(payload, indent=2) + "\n")
return path
def new_run_id(scenario: str) -> str:
stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
return f"{scenario}-{stamp}-{uuid.uuid4().hex[:8]}"
# ---------------------------------------------------------------------------
# Disposable disk rules
# ---------------------------------------------------------------------------
def _require_regular_and_safe(path: Path, test_dir: Path) -> os.stat_result:
text = str(path)
if text.startswith("/dev/") or text == "/dev":
raise UnsafeDiskError(f"refusing a /dev path: {text}")
# Symlink chains must stay inside the disposable directory when resolved.
real = Path(os.path.realpath(path))
if not _is_within(real, test_dir):
raise UnsafeDiskError(
f"writable disk resolves outside the disposable test directory "
f"({real} not under {test_dir})"
)
st = os.stat(path) # follows symlinks; realpath already checked containment
if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode):
raise UnsafeDiskError(f"refusing a block/character device: {text}")
if not stat.S_ISREG(st.st_mode):
raise UnsafeDiskError(f"refusing a non-regular file: {text}")
return st
def _is_within(child: Path, parent: Path) -> bool:
try:
child.relative_to(parent)
return True
except ValueError:
return False
def test_dir_is_safe(test_dir: Path) -> Path:
real = Path(os.path.realpath(test_dir))
if real == Path("/"):
raise UnsafeDiskError("the disposable test directory may not be /")
for bad in ("/dev", "/proc", "/sys"):
if real == Path(bad) or _is_within(real, Path(bad)):
raise UnsafeDiskError(f"the disposable test directory may not be under {bad}")
return real
def qcow2_magic(path: Path) -> bool:
try:
with open(path, "rb") as handle:
return handle.read(4) == b"QFI\xfb"
except OSError:
return False
def _qemu_img_backing_chain(path: Path) -> list[str]:
"""Return the backing-file paths reported by qemu-img (empty if none)."""
qemu_img = shutil.which("qemu-img")
if not qemu_img:
return []
proc = subprocess.run(
[qemu_img, "info", "--backing-chain", "--output=json", str(path)],
capture_output=True,
text=True,
)
if proc.returncode != 0:
return []
try:
data = json.loads(proc.stdout)
except json.JSONDecodeError:
return []
entries = data if isinstance(data, list) else [data]
chain: list[str] = []
for entry in entries:
for key in ("backing-filename", "full-backing-filename"):
value = entry.get(key)
if value:
chain.append(value)
return chain
def validate_disposable_disk(
path: str | os.PathLike,
test_dir: str | os.PathLike,
*,
require_qcow2: bool = True,
allow_missing: bool = False,
qemu_img: bool = True,
) -> Path:
"""Validate that ``path`` is a disposable qcow2 inside ``test_dir``.
Raises :class:`UnsafeDiskError` on any violation. On success returns the
resolved path.
"""
if path is None or str(path).strip() == "":
raise UnsafeDiskError("no writable disk path was provided")
test_real = test_dir_is_safe(Path(test_dir))
disk = Path(path)
if not disk.exists():
if allow_missing:
if not _is_within(Path(os.path.realpath(disk.parent)), test_real):
raise UnsafeDiskError(
f"new disk would be created outside the test directory: {disk}"
)
return disk
raise UnsafeDiskError(f"writable disk does not exist: {disk}")
st = _require_regular_and_safe(disk, test_real)
if st.st_size == 0:
raise UnsafeDiskError(f"writable disk is empty: {disk}")
if require_qcow2 and not qcow2_magic(disk):
raise UnsafeDiskError(
f"writable disk is not a qcow2 image (bad magic): {disk}; "
"refusing to hand a raw image to QEMU as writable"
)
if qemu_img:
for backing in _qemu_img_backing_chain(disk):
backing_real = Path(os.path.realpath(backing))
if not _is_within(backing_real, test_real):
raise UnsafeDiskError(
f"disk backing file is outside the test directory: {backing}"
)
if not os.access(disk, os.W_OK):
raise UnsafeDiskError(f"writable disk is not writable: {disk}")
return disk
def validate_writable_drives(paths, test_dir, *, qemu_img=True) -> list[Path]:
"""Validate every writable drive, not just one variable named DISK."""
checked = []
for path in paths:
if path is None or str(path).strip() == "":
continue
checked.append(validate_disposable_disk(path, test_dir, qemu_img=qemu_img))
if not checked:
raise UnsafeDiskError("no writable drives were provided to validate")
return checked
def validate_independent_disk_sets(first, second, test_dir) -> list[Path]:
"""Return validated second-install disks only when no image is reused."""
first_checked = validate_writable_drives(first, test_dir)
second_checked = validate_writable_drives(second, test_dir)
if {path.resolve() for path in first_checked} & {
path.resolve() for path in second_checked
}:
raise UnsafeDiskError("second install resolves to a disk used by the first install")
return second_checked
# ---------------------------------------------------------------------------
# Run-scoped state
# ---------------------------------------------------------------------------
@dataclasses.dataclass
class RunPaths:
run_id: str
directory: Path
serial_log: Path
monitor_sock: Path
shot_dir: Path
ovmf_vars: Path
results_json: Path
def ensure(self) -> "RunPaths":
self.directory.mkdir(parents=True, exist_ok=True)
os.chmod(self.directory, 0o700)
self.shot_dir.mkdir(parents=True, exist_ok=True)
os.chmod(self.shot_dir, 0o700)
return self
def make_run_paths(
scenario: str,
*,
base: str | os.PathLike | None = None,
ovmf_vars_template: str | os.PathLike | None = None,
run_id: str | None = None,
) -> RunPaths:
"""Create a private per-run directory and a fresh writable OVMF var file.
``base`` defaults to ``$PHOEBUS_VM_STATE`` or ``/var/tmp/phoebus-vm``.
"""
base_dir = Path(base or os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"))
base_dir.mkdir(parents=True, exist_ok=True)
os.chmod(base_dir, 0o700)
run = new_run_id(scenario) if run_id is None else run_id
directory = base_dir / run
paths = RunPaths(
run_id=run,
directory=directory,
serial_log=directory / "serial.log",
monitor_sock=directory / "monitor.sock",
shot_dir=directory / "shots",
ovmf_vars=directory / "OVMF_VARS.fd",
results_json=directory / "results.json",
).ensure()
if ovmf_vars_template:
shutil.copyfile(ovmf_vars_template, paths.ovmf_vars)
os.chmod(paths.ovmf_vars, 0o600)
return paths
# ---------------------------------------------------------------------------
# Screenshots (no shell)
# ---------------------------------------------------------------------------
def _reject_control_chars(value: str) -> str:
if "\n" in value or "\r" in value or "\x00" in value:
raise HarnessError("path contains a control character")
return value
def convert_ppm(ppm: str | os.PathLike, png: str | os.PathLike) -> Path | None:
"""Convert a PPM to PNG with an argument array (never a shell)."""
png = Path(_reject_control_chars(str(png)))
for program in ("magick", "convert"):
binary = shutil.which(program)
if not binary:
continue
proc = subprocess.run([binary, str(ppm), str(png)], capture_output=True)
if proc.returncode == 0 and png.exists():
return png
return None
def screenshot_name(seq: int, slug: str) -> str:
safe = "".join(c if (c.isalnum() or c in "-_") else "-" for c in slug).strip("-")
return f"{seq:02d}-{safe or 'shot'}"
# ---------------------------------------------------------------------------
# QMP client
# ---------------------------------------------------------------------------
class QmpError(Exception):
pass
class QmpClient:
"""A minimal structured QMP client over a UNIX socket.
Uses JSON framing (the monitor protocol deletes shell/quoting concerns) and
issues HMP commands through ``human-monitor-command`` for ``screendump`` and
``sendkey``.
"""
def __init__(self, sock_path: str | os.PathLike, timeout: float = 20.0):
self.sock_path = str(sock_path)
self.timeout = timeout
def __enter__(self) -> "QmpClient":
if not os.path.exists(self.sock_path):
raise QmpError(f"monitor socket does not exist: {self.sock_path}")
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sock.settimeout(self.timeout)
self.sock.connect(self.sock_path)
self._file = self.sock.makefile("rwb")
greeting = self._read_json()
if "QMP" not in greeting:
raise QmpError("server did not send a QMP greeting")
self._command({"execute": "qmp_capabilities"})
return self
def __exit__(self, *_exc) -> None:
try:
self._file.close()
finally:
self.sock.close()
def _read_json(self) -> dict:
line = self._file.readline()
if not line:
raise QmpError("unexpected EOF from the QMP monitor")
try:
return json.loads(line)
except json.JSONDecodeError as exc:
raise QmpError(f"invalid QMP message: {line[:200]!r}") from exc
def _command(self, command: dict) -> dict:
self._file.write(json.dumps(command).encode() + b"\n")
self._file.flush()
while True:
message = self._read_json()
if "event" in message:
continue
if "error" in message:
raise QmpError(str(message["error"]))
return message
def hmp(self, command: str) -> str:
reply = self._command({"execute": "human-monitor-command", "arguments": {"command-line": command}})
return reply.get("return", "")
def screendump(self, target: str | os.PathLike) -> Path:
target = Path(_reject_control_chars(str(target)))
target.parent.mkdir(parents=True, exist_ok=True)
# QEMU's HMP screendump takes a single filename argument, so a path with
# spaces is quoted for QEMU's own tiny parser, not for a shell.
quoted = '"' + str(target).replace("\\", "\\\\").replace('"', '\\"') + '"'
self.hmp(f"screendump {quoted}")
deadline = time.monotonic() + self.timeout
while time.monotonic() < deadline:
if target.exists() and target.stat().st_size > 0:
return target
time.sleep(0.2)
raise QmpError(f"screendump did not produce {target}")
def sendkey(self, key: str) -> None:
self.hmp(f"sendkey {key}")
def screendump(monitor_sock, target, *, convert: bool = True) -> Path | None:
"""Capture the framebuffer and optionally convert it to PNG.
Returns the PPM path, or ``None`` if capture failed. Never raises for a
missing PNG converter (the PPM is still evidence).
"""
target = Path(_reject_control_chars(str(target)))
try:
with QmpClient(monitor_sock) as client:
ppm = client.screendump(target)
except (QmpError, OSError):
return None
if convert:
convert_ppm(ppm, ppm.with_suffix(".png"))
return ppm
def wait_for_file(path: str | os.PathLike, timeout: float, interval: float = 0.5) -> bool:
deadline = time.monotonic() + timeout
p = Path(path)
while time.monotonic() < deadline:
if p.exists():
return True
time.sleep(interval)
return False