Validate every writable drive (qcow2, inside PHOEBUS_VM_DISKS, no /dev/*, block/char devices, raw images, escaping symlinks or outside backing chains), give each run its own OVMF variable file, QMP socket and state directory, and use QMP plus argument arrays instead of shell interpolation. Return nonzero on any failed assertion, timeout, EOF or missing result and record PASS/FAIL/BLOCKED/SKIPPED/NOT RUN. Add 16 unit tests, a report generator and a findings register; make test runs the harness suite.
492 lines
17 KiB
Bash
492 lines
17 KiB
Bash
#!/usr/bin/env python3
|
|
"""Safety, structured-result and QEMU-control helpers for the VM harnesses.
|
|
|
|
The VM drivers are test equipment: their own safety rules must be enforced, not
|
|
merely documented. This module centralises those rules so every driver shares
|
|
one implementation:
|
|
|
|
* every writable disk must be a regular **qcow2** file inside an explicitly
|
|
designated disposable directory, with a backing chain that is fully inside
|
|
that directory. Block devices, ``/dev/*`` paths, character devices, unsafe
|
|
symlinks and writable storage outside the test area are rejected;
|
|
* each run gets its own writable OVMF variable file and monitor socket;
|
|
* screenshots and monitor commands are issued without a shell, with argument
|
|
arrays, over structured QMP where possible;
|
|
* results are structured (PASS/FAIL/BLOCKED/SKIPPED/NOT RUN) and a failed,
|
|
timed-out or incomplete run exits nonzero.
|
|
|
|
Nothing here depends on pexpect, so the safety logic is unit-testable on any
|
|
host (see ``test_vmsafety.py``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import json
|
|
import os
|
|
import shutil
|
|
import socket
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Statuses
|
|
# ---------------------------------------------------------------------------
|
|
|
|
PASS = "PASS"
|
|
FAIL = "FAIL"
|
|
BLOCKED = "BLOCKED"
|
|
SKIPPED = "SKIPPED"
|
|
NOT_RUN = "NOT RUN"
|
|
|
|
_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED)
|
|
|
|
|
|
class UnsafeDiskError(Exception):
|
|
"""A writable target violated the disposable-disk rules."""
|
|
|
|
|
|
class HarnessError(Exception):
|
|
"""A harness precondition failed (bad environment, unsafe path, ...)."""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Structured results
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Check:
|
|
name: str
|
|
status: str
|
|
detail: str = ""
|
|
|
|
def to_dict(self) -> dict:
|
|
return {"name": self.name, "status": self.status, "detail": self.detail}
|
|
|
|
|
|
class Results:
|
|
"""A run's assertions and their outcome.
|
|
|
|
A check may be recorded once. ``assert_true`` records PASS or FAIL; the
|
|
helpers ``blocked``/``skip`` record those explicit states. Any FAIL (and,
|
|
when strict, any BLOCKED or NOT RUN) makes :meth:`exit_code` nonzero.
|
|
"""
|
|
|
|
def __init__(self, scenario: str, run_id: str | None = None, strict: bool | None = None):
|
|
self.scenario = scenario
|
|
self.run_id = run_id or new_run_id(scenario)
|
|
self.checks: list[Check] = []
|
|
if strict is None:
|
|
strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true")
|
|
self.strict = bool(strict)
|
|
|
|
# -- recording ---------------------------------------------------------
|
|
def _add(self, name: str, status: str, detail: str) -> Check:
|
|
if status not in _TERMINAL:
|
|
raise ValueError(f"invalid result status: {status}")
|
|
check = Check(name=name, status=status, detail=detail)
|
|
self.checks.append(check)
|
|
stream = sys.stderr if status == FAIL else sys.stdout
|
|
print(f"[{status:<7}] {name}" + (f": {detail}" if detail else ""), file=stream, flush=True)
|
|
return check
|
|
|
|
def assert_true(self, name: str, condition: bool, detail: str = "") -> bool:
|
|
ok = bool(condition)
|
|
# `detail` describes the failure; never print it on a pass, which would
|
|
# make a passing check read as if it had failed.
|
|
self._add(name, PASS if ok else FAIL, "" if ok else detail)
|
|
return ok
|
|
|
|
def fail(self, name: str, detail: str = "") -> Check:
|
|
return self._add(name, FAIL, detail)
|
|
|
|
def pass_(self, name: str, detail: str = "") -> Check:
|
|
return self._add(name, PASS, detail)
|
|
|
|
def blocked(self, name: str, detail: str = "") -> Check:
|
|
return self._add(name, BLOCKED, detail)
|
|
|
|
def skip(self, name: str, detail: str = "") -> Check:
|
|
return self._add(name, SKIPPED, detail)
|
|
|
|
# -- outcome -----------------------------------------------------------
|
|
@property
|
|
def failed(self) -> bool:
|
|
return any(c.status == FAIL for c in self.checks)
|
|
|
|
@property
|
|
def has_blockers(self) -> bool:
|
|
return any(c.status == BLOCKED for c in self.checks)
|
|
|
|
@property
|
|
def not_run(self) -> bool:
|
|
return not self.checks or any(c.status == NOT_RUN for c in self.checks)
|
|
|
|
def counts(self) -> dict:
|
|
out = {s: 0 for s in (PASS, FAIL, BLOCKED, SKIPPED, NOT_RUN)}
|
|
for c in self.checks:
|
|
out[c.status] += 1
|
|
return out
|
|
|
|
def exit_code(self) -> int:
|
|
if self.failed:
|
|
return 1
|
|
if self.strict and (self.has_blockers or self.not_run):
|
|
return 2
|
|
return 0
|
|
|
|
def summary(self) -> str:
|
|
counts = self.counts()
|
|
parts = [f"{k}={v}" for k, v in counts.items() if v]
|
|
return f"{self.scenario} [{self.run_id}]: " + (", ".join(parts) if parts else "no checks")
|
|
|
|
def write(self, path: str | Path) -> Path:
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
payload = {
|
|
"scenario": self.scenario,
|
|
"run_id": self.run_id,
|
|
"strict": self.strict,
|
|
"exit_code": self.exit_code(),
|
|
"counts": self.counts(),
|
|
"checks": [c.to_dict() for c in self.checks],
|
|
}
|
|
path.write_text(json.dumps(payload, indent=2) + "\n")
|
|
return path
|
|
|
|
|
|
def new_run_id(scenario: str) -> str:
|
|
stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
|
|
return f"{scenario}-{stamp}-{uuid.uuid4().hex[:8]}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Disposable disk rules
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _require_regular_and_safe(path: Path, test_dir: Path) -> os.stat_result:
|
|
text = str(path)
|
|
if text.startswith("/dev/") or text == "/dev":
|
|
raise UnsafeDiskError(f"refusing a /dev path: {text}")
|
|
# Symlink chains must stay inside the disposable directory when resolved.
|
|
real = Path(os.path.realpath(path))
|
|
if not _is_within(real, test_dir):
|
|
raise UnsafeDiskError(
|
|
f"writable disk resolves outside the disposable test directory "
|
|
f"({real} not under {test_dir})"
|
|
)
|
|
st = os.stat(path) # follows symlinks; realpath already checked containment
|
|
if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode):
|
|
raise UnsafeDiskError(f"refusing a block/character device: {text}")
|
|
if not stat.S_ISREG(st.st_mode):
|
|
raise UnsafeDiskError(f"refusing a non-regular file: {text}")
|
|
return st
|
|
|
|
|
|
def _is_within(child: Path, parent: Path) -> bool:
|
|
try:
|
|
child.relative_to(parent)
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def test_dir_is_safe(test_dir: Path) -> Path:
|
|
real = Path(os.path.realpath(test_dir))
|
|
if real == Path("/"):
|
|
raise UnsafeDiskError("the disposable test directory may not be /")
|
|
for bad in ("/dev", "/proc", "/sys"):
|
|
if real == Path(bad) or _is_within(real, Path(bad)):
|
|
raise UnsafeDiskError(f"the disposable test directory may not be under {bad}")
|
|
return real
|
|
|
|
|
|
def qcow2_magic(path: Path) -> bool:
|
|
try:
|
|
with open(path, "rb") as handle:
|
|
return handle.read(4) == b"QFI\xfb"
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _qemu_img_backing_chain(path: Path) -> list[str]:
|
|
"""Return the backing-file paths reported by qemu-img (empty if none)."""
|
|
qemu_img = shutil.which("qemu-img")
|
|
if not qemu_img:
|
|
return []
|
|
proc = subprocess.run(
|
|
[qemu_img, "info", "--backing-chain", "--output=json", str(path)],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if proc.returncode != 0:
|
|
return []
|
|
try:
|
|
data = json.loads(proc.stdout)
|
|
except json.JSONDecodeError:
|
|
return []
|
|
entries = data if isinstance(data, list) else [data]
|
|
chain: list[str] = []
|
|
for entry in entries:
|
|
for key in ("backing-filename", "full-backing-filename"):
|
|
value = entry.get(key)
|
|
if value:
|
|
chain.append(value)
|
|
return chain
|
|
|
|
|
|
def validate_disposable_disk(
|
|
path: str | os.PathLike,
|
|
test_dir: str | os.PathLike,
|
|
*,
|
|
require_qcow2: bool = True,
|
|
allow_missing: bool = False,
|
|
qemu_img: bool = True,
|
|
) -> Path:
|
|
"""Validate that ``path`` is a disposable qcow2 inside ``test_dir``.
|
|
|
|
Raises :class:`UnsafeDiskError` on any violation. On success returns the
|
|
resolved path.
|
|
"""
|
|
if path is None or str(path).strip() == "":
|
|
raise UnsafeDiskError("no writable disk path was provided")
|
|
test_real = test_dir_is_safe(Path(test_dir))
|
|
disk = Path(path)
|
|
|
|
if not disk.exists():
|
|
if allow_missing:
|
|
if not _is_within(Path(os.path.realpath(disk.parent)), test_real):
|
|
raise UnsafeDiskError(
|
|
f"new disk would be created outside the test directory: {disk}"
|
|
)
|
|
return disk
|
|
raise UnsafeDiskError(f"writable disk does not exist: {disk}")
|
|
|
|
st = _require_regular_and_safe(disk, test_real)
|
|
if st.st_size == 0:
|
|
raise UnsafeDiskError(f"writable disk is empty: {disk}")
|
|
if require_qcow2 and not qcow2_magic(disk):
|
|
raise UnsafeDiskError(
|
|
f"writable disk is not a qcow2 image (bad magic): {disk}; "
|
|
"refusing to hand a raw image to QEMU as writable"
|
|
)
|
|
if qemu_img:
|
|
for backing in _qemu_img_backing_chain(disk):
|
|
backing_real = Path(os.path.realpath(backing))
|
|
if not _is_within(backing_real, test_real):
|
|
raise UnsafeDiskError(
|
|
f"disk backing file is outside the test directory: {backing}"
|
|
)
|
|
if not os.access(disk, os.W_OK):
|
|
raise UnsafeDiskError(f"writable disk is not writable: {disk}")
|
|
return disk
|
|
|
|
|
|
def validate_writable_drives(paths, test_dir, *, qemu_img=True) -> list[Path]:
|
|
"""Validate every writable drive, not just one variable named DISK."""
|
|
checked = []
|
|
for path in paths:
|
|
if path is None or str(path).strip() == "":
|
|
continue
|
|
checked.append(validate_disposable_disk(path, test_dir, qemu_img=qemu_img))
|
|
if not checked:
|
|
raise UnsafeDiskError("no writable drives were provided to validate")
|
|
return checked
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Run-scoped state
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class RunPaths:
|
|
run_id: str
|
|
directory: Path
|
|
serial_log: Path
|
|
monitor_sock: Path
|
|
shot_dir: Path
|
|
ovmf_vars: Path
|
|
results_json: Path
|
|
|
|
def ensure(self) -> "RunPaths":
|
|
self.directory.mkdir(parents=True, exist_ok=True)
|
|
os.chmod(self.directory, 0o700)
|
|
self.shot_dir.mkdir(parents=True, exist_ok=True)
|
|
os.chmod(self.shot_dir, 0o700)
|
|
return self
|
|
|
|
|
|
def make_run_paths(
|
|
scenario: str,
|
|
*,
|
|
base: str | os.PathLike | None = None,
|
|
ovmf_vars_template: str | os.PathLike | None = None,
|
|
run_id: str | None = None,
|
|
) -> RunPaths:
|
|
"""Create a private per-run directory and a fresh writable OVMF var file.
|
|
|
|
``base`` defaults to ``$PHOEBUS_VM_STATE`` or ``/var/tmp/phoebus-vm``.
|
|
"""
|
|
base_dir = Path(base or os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"))
|
|
base_dir.mkdir(parents=True, exist_ok=True)
|
|
os.chmod(base_dir, 0o700)
|
|
run = new_run_id(scenario) if run_id is None else run_id
|
|
directory = base_dir / run
|
|
paths = RunPaths(
|
|
run_id=run,
|
|
directory=directory,
|
|
serial_log=directory / "serial.log",
|
|
monitor_sock=directory / "monitor.sock",
|
|
shot_dir=directory / "shots",
|
|
ovmf_vars=directory / "OVMF_VARS.fd",
|
|
results_json=directory / "results.json",
|
|
).ensure()
|
|
if ovmf_vars_template:
|
|
shutil.copyfile(ovmf_vars_template, paths.ovmf_vars)
|
|
os.chmod(paths.ovmf_vars, 0o600)
|
|
return paths
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Screenshots (no shell)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _reject_control_chars(value: str) -> str:
|
|
if "\n" in value or "\r" in value or "\x00" in value:
|
|
raise HarnessError("path contains a control character")
|
|
return value
|
|
|
|
|
|
def convert_ppm(ppm: str | os.PathLike, png: str | os.PathLike) -> Path | None:
|
|
"""Convert a PPM to PNG with an argument array (never a shell)."""
|
|
png = Path(_reject_control_chars(str(png)))
|
|
for program in ("magick", "convert"):
|
|
binary = shutil.which(program)
|
|
if not binary:
|
|
continue
|
|
proc = subprocess.run([binary, str(ppm), str(png)], capture_output=True)
|
|
if proc.returncode == 0 and png.exists():
|
|
return png
|
|
return None
|
|
|
|
|
|
def screenshot_name(seq: int, slug: str) -> str:
|
|
safe = "".join(c if (c.isalnum() or c in "-_") else "-" for c in slug).strip("-")
|
|
return f"{seq:02d}-{safe or 'shot'}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# QMP client
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class QmpError(Exception):
|
|
pass
|
|
|
|
|
|
class QmpClient:
|
|
"""A minimal structured QMP client over a UNIX socket.
|
|
|
|
Uses JSON framing (the monitor protocol deletes shell/quoting concerns) and
|
|
issues HMP commands through ``human-monitor-command`` for ``screendump`` and
|
|
``sendkey``.
|
|
"""
|
|
|
|
def __init__(self, sock_path: str | os.PathLike, timeout: float = 20.0):
|
|
self.sock_path = str(sock_path)
|
|
self.timeout = timeout
|
|
|
|
def __enter__(self) -> "QmpClient":
|
|
if not os.path.exists(self.sock_path):
|
|
raise QmpError(f"monitor socket does not exist: {self.sock_path}")
|
|
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
self.sock.settimeout(self.timeout)
|
|
self.sock.connect(self.sock_path)
|
|
self._file = self.sock.makefile("rwb")
|
|
greeting = self._read_json()
|
|
if "QMP" not in greeting:
|
|
raise QmpError("server did not send a QMP greeting")
|
|
self._command({"execute": "qmp_capabilities"})
|
|
return self
|
|
|
|
def __exit__(self, *_exc) -> None:
|
|
try:
|
|
self._file.close()
|
|
finally:
|
|
self.sock.close()
|
|
|
|
def _read_json(self) -> dict:
|
|
line = self._file.readline()
|
|
if not line:
|
|
raise QmpError("unexpected EOF from the QMP monitor")
|
|
try:
|
|
return json.loads(line)
|
|
except json.JSONDecodeError as exc:
|
|
raise QmpError(f"invalid QMP message: {line[:200]!r}") from exc
|
|
|
|
def _command(self, command: dict) -> dict:
|
|
self._file.write(json.dumps(command).encode() + b"\n")
|
|
self._file.flush()
|
|
while True:
|
|
message = self._read_json()
|
|
if "event" in message:
|
|
continue
|
|
if "error" in message:
|
|
raise QmpError(str(message["error"]))
|
|
return message
|
|
|
|
def hmp(self, command: str) -> str:
|
|
reply = self._command({"execute": "human-monitor-command", "arguments": {"command-line": command}})
|
|
return reply.get("return", "")
|
|
|
|
def screendump(self, target: str | os.PathLike) -> Path:
|
|
target = Path(_reject_control_chars(str(target)))
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
# QEMU's HMP screendump takes a single filename argument, so a path with
|
|
# spaces is quoted for QEMU's own tiny parser, not for a shell.
|
|
quoted = '"' + str(target).replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
self.hmp(f"screendump {quoted}")
|
|
deadline = time.monotonic() + self.timeout
|
|
while time.monotonic() < deadline:
|
|
if target.exists() and target.stat().st_size > 0:
|
|
return target
|
|
time.sleep(0.2)
|
|
raise QmpError(f"screendump did not produce {target}")
|
|
|
|
def sendkey(self, key: str) -> None:
|
|
self.hmp(f"sendkey {key}")
|
|
|
|
|
|
def screendump(monitor_sock, target, *, convert: bool = True) -> Path | None:
|
|
"""Capture the framebuffer and optionally convert it to PNG.
|
|
|
|
Returns the PPM path, or ``None`` if capture failed. Never raises for a
|
|
missing PNG converter (the PPM is still evidence).
|
|
"""
|
|
target = Path(_reject_control_chars(str(target)))
|
|
try:
|
|
with QmpClient(monitor_sock) as client:
|
|
ppm = client.screendump(target)
|
|
except (QmpError, OSError):
|
|
return None
|
|
if convert:
|
|
convert_ppm(ppm, ppm.with_suffix(".png"))
|
|
return ppm
|
|
|
|
|
|
def wait_for_file(path: str | os.PathLike, timeout: float, interval: float = 0.5) -> bool:
|
|
deadline = time.monotonic() + timeout
|
|
p = Path(path)
|
|
while time.monotonic() < deadline:
|
|
if p.exists():
|
|
return True
|
|
time.sleep(interval)
|
|
return False
|