phoebus_os/installer/tests/vm/vmsafety.py
Phoebus cc23df69de test(vm): enforce harness disk safety and structured results
Validate every writable drive (qcow2, inside PHOEBUS_VM_DISKS, no /dev/*,
block/char devices, raw images, escaping symlinks or outside backing
chains), give each run its own OVMF variable file, QMP socket and state
directory, and use QMP plus argument arrays instead of shell interpolation.
Return nonzero on any failed assertion, timeout, EOF or missing result and
record PASS/FAIL/BLOCKED/SKIPPED/NOT RUN. Add 16 unit tests, a report
generator and a findings register; make test runs the harness suite.
2026-09-21 20:01:50 -07:00

492 lines
17 KiB
Bash

#!/usr/bin/env python3
"""Safety, structured-result and QEMU-control helpers for the VM harnesses.
The VM drivers are test equipment: their own safety rules must be enforced, not
merely documented. This module centralises those rules so every driver shares
one implementation:
* every writable disk must be a regular **qcow2** file inside an explicitly
designated disposable directory, with a backing chain that is fully inside
that directory. Block devices, ``/dev/*`` paths, character devices, unsafe
symlinks and writable storage outside the test area are rejected;
* each run gets its own writable OVMF variable file and monitor socket;
* screenshots and monitor commands are issued without a shell, with argument
arrays, over structured QMP where possible;
* results are structured (PASS/FAIL/BLOCKED/SKIPPED/NOT RUN) and a failed,
timed-out or incomplete run exits nonzero.
Nothing here depends on pexpect, so the safety logic is unit-testable on any
host (see ``test_vmsafety.py``).
"""
from __future__ import annotations
import dataclasses
import json
import os
import shutil
import socket
import stat
import subprocess
import sys
import time
import uuid
from pathlib import Path
# ---------------------------------------------------------------------------
# Statuses
# ---------------------------------------------------------------------------
PASS = "PASS"
FAIL = "FAIL"
BLOCKED = "BLOCKED"
SKIPPED = "SKIPPED"
NOT_RUN = "NOT RUN"
_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED)
class UnsafeDiskError(Exception):
"""A writable target violated the disposable-disk rules."""
class HarnessError(Exception):
"""A harness precondition failed (bad environment, unsafe path, ...)."""
# ---------------------------------------------------------------------------
# Structured results
# ---------------------------------------------------------------------------
@dataclasses.dataclass
class Check:
name: str
status: str
detail: str = ""
def to_dict(self) -> dict:
return {"name": self.name, "status": self.status, "detail": self.detail}
class Results:
"""A run's assertions and their outcome.
A check may be recorded once. ``assert_true`` records PASS or FAIL; the
helpers ``blocked``/``skip`` record those explicit states. Any FAIL (and,
when strict, any BLOCKED or NOT RUN) makes :meth:`exit_code` nonzero.
"""
def __init__(self, scenario: str, run_id: str | None = None, strict: bool | None = None):
self.scenario = scenario
self.run_id = run_id or new_run_id(scenario)
self.checks: list[Check] = []
if strict is None:
strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true")
self.strict = bool(strict)
# -- recording ---------------------------------------------------------
def _add(self, name: str, status: str, detail: str) -> Check:
if status not in _TERMINAL:
raise ValueError(f"invalid result status: {status}")
check = Check(name=name, status=status, detail=detail)
self.checks.append(check)
stream = sys.stderr if status == FAIL else sys.stdout
print(f"[{status:<7}] {name}" + (f": {detail}" if detail else ""), file=stream, flush=True)
return check
def assert_true(self, name: str, condition: bool, detail: str = "") -> bool:
ok = bool(condition)
# `detail` describes the failure; never print it on a pass, which would
# make a passing check read as if it had failed.
self._add(name, PASS if ok else FAIL, "" if ok else detail)
return ok
def fail(self, name: str, detail: str = "") -> Check:
return self._add(name, FAIL, detail)
def pass_(self, name: str, detail: str = "") -> Check:
return self._add(name, PASS, detail)
def blocked(self, name: str, detail: str = "") -> Check:
return self._add(name, BLOCKED, detail)
def skip(self, name: str, detail: str = "") -> Check:
return self._add(name, SKIPPED, detail)
# -- outcome -----------------------------------------------------------
@property
def failed(self) -> bool:
return any(c.status == FAIL for c in self.checks)
@property
def has_blockers(self) -> bool:
return any(c.status == BLOCKED for c in self.checks)
@property
def not_run(self) -> bool:
return not self.checks or any(c.status == NOT_RUN for c in self.checks)
def counts(self) -> dict:
out = {s: 0 for s in (PASS, FAIL, BLOCKED, SKIPPED, NOT_RUN)}
for c in self.checks:
out[c.status] += 1
return out
def exit_code(self) -> int:
if self.failed:
return 1
if self.strict and (self.has_blockers or self.not_run):
return 2
return 0
def summary(self) -> str:
counts = self.counts()
parts = [f"{k}={v}" for k, v in counts.items() if v]
return f"{self.scenario} [{self.run_id}]: " + (", ".join(parts) if parts else "no checks")
def write(self, path: str | Path) -> Path:
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"scenario": self.scenario,
"run_id": self.run_id,
"strict": self.strict,
"exit_code": self.exit_code(),
"counts": self.counts(),
"checks": [c.to_dict() for c in self.checks],
}
path.write_text(json.dumps(payload, indent=2) + "\n")
return path
def new_run_id(scenario: str) -> str:
stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
return f"{scenario}-{stamp}-{uuid.uuid4().hex[:8]}"
# ---------------------------------------------------------------------------
# Disposable disk rules
# ---------------------------------------------------------------------------
def _require_regular_and_safe(path: Path, test_dir: Path) -> os.stat_result:
text = str(path)
if text.startswith("/dev/") or text == "/dev":
raise UnsafeDiskError(f"refusing a /dev path: {text}")
# Symlink chains must stay inside the disposable directory when resolved.
real = Path(os.path.realpath(path))
if not _is_within(real, test_dir):
raise UnsafeDiskError(
f"writable disk resolves outside the disposable test directory "
f"({real} not under {test_dir})"
)
st = os.stat(path) # follows symlinks; realpath already checked containment
if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode):
raise UnsafeDiskError(f"refusing a block/character device: {text}")
if not stat.S_ISREG(st.st_mode):
raise UnsafeDiskError(f"refusing a non-regular file: {text}")
return st
def _is_within(child: Path, parent: Path) -> bool:
try:
child.relative_to(parent)
return True
except ValueError:
return False
def test_dir_is_safe(test_dir: Path) -> Path:
real = Path(os.path.realpath(test_dir))
if real == Path("/"):
raise UnsafeDiskError("the disposable test directory may not be /")
for bad in ("/dev", "/proc", "/sys"):
if real == Path(bad) or _is_within(real, Path(bad)):
raise UnsafeDiskError(f"the disposable test directory may not be under {bad}")
return real
def qcow2_magic(path: Path) -> bool:
try:
with open(path, "rb") as handle:
return handle.read(4) == b"QFI\xfb"
except OSError:
return False
def _qemu_img_backing_chain(path: Path) -> list[str]:
"""Return the backing-file paths reported by qemu-img (empty if none)."""
qemu_img = shutil.which("qemu-img")
if not qemu_img:
return []
proc = subprocess.run(
[qemu_img, "info", "--backing-chain", "--output=json", str(path)],
capture_output=True,
text=True,
)
if proc.returncode != 0:
return []
try:
data = json.loads(proc.stdout)
except json.JSONDecodeError:
return []
entries = data if isinstance(data, list) else [data]
chain: list[str] = []
for entry in entries:
for key in ("backing-filename", "full-backing-filename"):
value = entry.get(key)
if value:
chain.append(value)
return chain
def validate_disposable_disk(
path: str | os.PathLike,
test_dir: str | os.PathLike,
*,
require_qcow2: bool = True,
allow_missing: bool = False,
qemu_img: bool = True,
) -> Path:
"""Validate that ``path`` is a disposable qcow2 inside ``test_dir``.
Raises :class:`UnsafeDiskError` on any violation. On success returns the
resolved path.
"""
if path is None or str(path).strip() == "":
raise UnsafeDiskError("no writable disk path was provided")
test_real = test_dir_is_safe(Path(test_dir))
disk = Path(path)
if not disk.exists():
if allow_missing:
if not _is_within(Path(os.path.realpath(disk.parent)), test_real):
raise UnsafeDiskError(
f"new disk would be created outside the test directory: {disk}"
)
return disk
raise UnsafeDiskError(f"writable disk does not exist: {disk}")
st = _require_regular_and_safe(disk, test_real)
if st.st_size == 0:
raise UnsafeDiskError(f"writable disk is empty: {disk}")
if require_qcow2 and not qcow2_magic(disk):
raise UnsafeDiskError(
f"writable disk is not a qcow2 image (bad magic): {disk}; "
"refusing to hand a raw image to QEMU as writable"
)
if qemu_img:
for backing in _qemu_img_backing_chain(disk):
backing_real = Path(os.path.realpath(backing))
if not _is_within(backing_real, test_real):
raise UnsafeDiskError(
f"disk backing file is outside the test directory: {backing}"
)
if not os.access(disk, os.W_OK):
raise UnsafeDiskError(f"writable disk is not writable: {disk}")
return disk
def validate_writable_drives(paths, test_dir, *, qemu_img=True) -> list[Path]:
"""Validate every writable drive, not just one variable named DISK."""
checked = []
for path in paths:
if path is None or str(path).strip() == "":
continue
checked.append(validate_disposable_disk(path, test_dir, qemu_img=qemu_img))
if not checked:
raise UnsafeDiskError("no writable drives were provided to validate")
return checked
# ---------------------------------------------------------------------------
# Run-scoped state
# ---------------------------------------------------------------------------
@dataclasses.dataclass
class RunPaths:
run_id: str
directory: Path
serial_log: Path
monitor_sock: Path
shot_dir: Path
ovmf_vars: Path
results_json: Path
def ensure(self) -> "RunPaths":
self.directory.mkdir(parents=True, exist_ok=True)
os.chmod(self.directory, 0o700)
self.shot_dir.mkdir(parents=True, exist_ok=True)
os.chmod(self.shot_dir, 0o700)
return self
def make_run_paths(
scenario: str,
*,
base: str | os.PathLike | None = None,
ovmf_vars_template: str | os.PathLike | None = None,
run_id: str | None = None,
) -> RunPaths:
"""Create a private per-run directory and a fresh writable OVMF var file.
``base`` defaults to ``$PHOEBUS_VM_STATE`` or ``/var/tmp/phoebus-vm``.
"""
base_dir = Path(base or os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"))
base_dir.mkdir(parents=True, exist_ok=True)
os.chmod(base_dir, 0o700)
run = new_run_id(scenario) if run_id is None else run_id
directory = base_dir / run
paths = RunPaths(
run_id=run,
directory=directory,
serial_log=directory / "serial.log",
monitor_sock=directory / "monitor.sock",
shot_dir=directory / "shots",
ovmf_vars=directory / "OVMF_VARS.fd",
results_json=directory / "results.json",
).ensure()
if ovmf_vars_template:
shutil.copyfile(ovmf_vars_template, paths.ovmf_vars)
os.chmod(paths.ovmf_vars, 0o600)
return paths
# ---------------------------------------------------------------------------
# Screenshots (no shell)
# ---------------------------------------------------------------------------
def _reject_control_chars(value: str) -> str:
if "\n" in value or "\r" in value or "\x00" in value:
raise HarnessError("path contains a control character")
return value
def convert_ppm(ppm: str | os.PathLike, png: str | os.PathLike) -> Path | None:
"""Convert a PPM to PNG with an argument array (never a shell)."""
png = Path(_reject_control_chars(str(png)))
for program in ("magick", "convert"):
binary = shutil.which(program)
if not binary:
continue
proc = subprocess.run([binary, str(ppm), str(png)], capture_output=True)
if proc.returncode == 0 and png.exists():
return png
return None
def screenshot_name(seq: int, slug: str) -> str:
safe = "".join(c if (c.isalnum() or c in "-_") else "-" for c in slug).strip("-")
return f"{seq:02d}-{safe or 'shot'}"
# ---------------------------------------------------------------------------
# QMP client
# ---------------------------------------------------------------------------
class QmpError(Exception):
pass
class QmpClient:
"""A minimal structured QMP client over a UNIX socket.
Uses JSON framing (the monitor protocol deletes shell/quoting concerns) and
issues HMP commands through ``human-monitor-command`` for ``screendump`` and
``sendkey``.
"""
def __init__(self, sock_path: str | os.PathLike, timeout: float = 20.0):
self.sock_path = str(sock_path)
self.timeout = timeout
def __enter__(self) -> "QmpClient":
if not os.path.exists(self.sock_path):
raise QmpError(f"monitor socket does not exist: {self.sock_path}")
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sock.settimeout(self.timeout)
self.sock.connect(self.sock_path)
self._file = self.sock.makefile("rwb")
greeting = self._read_json()
if "QMP" not in greeting:
raise QmpError("server did not send a QMP greeting")
self._command({"execute": "qmp_capabilities"})
return self
def __exit__(self, *_exc) -> None:
try:
self._file.close()
finally:
self.sock.close()
def _read_json(self) -> dict:
line = self._file.readline()
if not line:
raise QmpError("unexpected EOF from the QMP monitor")
try:
return json.loads(line)
except json.JSONDecodeError as exc:
raise QmpError(f"invalid QMP message: {line[:200]!r}") from exc
def _command(self, command: dict) -> dict:
self._file.write(json.dumps(command).encode() + b"\n")
self._file.flush()
while True:
message = self._read_json()
if "event" in message:
continue
if "error" in message:
raise QmpError(str(message["error"]))
return message
def hmp(self, command: str) -> str:
reply = self._command({"execute": "human-monitor-command", "arguments": {"command-line": command}})
return reply.get("return", "")
def screendump(self, target: str | os.PathLike) -> Path:
target = Path(_reject_control_chars(str(target)))
target.parent.mkdir(parents=True, exist_ok=True)
# QEMU's HMP screendump takes a single filename argument, so a path with
# spaces is quoted for QEMU's own tiny parser, not for a shell.
quoted = '"' + str(target).replace("\\", "\\\\").replace('"', '\\"') + '"'
self.hmp(f"screendump {quoted}")
deadline = time.monotonic() + self.timeout
while time.monotonic() < deadline:
if target.exists() and target.stat().st_size > 0:
return target
time.sleep(0.2)
raise QmpError(f"screendump did not produce {target}")
def sendkey(self, key: str) -> None:
self.hmp(f"sendkey {key}")
def screendump(monitor_sock, target, *, convert: bool = True) -> Path | None:
"""Capture the framebuffer and optionally convert it to PNG.
Returns the PPM path, or ``None`` if capture failed. Never raises for a
missing PNG converter (the PPM is still evidence).
"""
target = Path(_reject_control_chars(str(target)))
try:
with QmpClient(monitor_sock) as client:
ppm = client.screendump(target)
except (QmpError, OSError):
return None
if convert:
convert_ppm(ppm, ppm.with_suffix(".png"))
return ppm
def wait_for_file(path: str | os.PathLike, timeout: float, interval: float = 0.5) -> bool:
deadline = time.monotonic() + timeout
p = Path(path)
while time.monotonic() < deadline:
if p.exists():
return True
time.sleep(interval)
return False