phoebus_os/installer/tests/vm/vmsafety.py
Phoebus 1d37fbc1b8 ci(release): fail closed and bind gates to the candidate artifact
Make a passing CI/release result mean the checks actually ran.

- .forgejo/workflows/ci.yml is fail-closed: it provisions the pinned
  toolchain or fails, then runs one 'make ci' (fmt-check, clippy, workspace
  tests, VM-harness safety tests, release build, static installer build).
- Pin stable Rust 1.93.1 in installer-rs/rust-toolchain.toml; run cargo gates
  with --locked for reproducible dependencies.
- phoebus-release gates now records passed/failed/blocked/skipped/not_run
  (only passed is a pass), executes real host gate commands, and imports the
  existing VM-harness results.json instead of a competing harness. Gates with
  no automation are always blocked.
- Bind gate reports and harness evidence to a candidate name+SHA-256
  (--candidate; PHOEBUS_VM_ARTIFACT[_SHA256]); stale or unbound evidence is
  failed, never passed.
- publish refuses unless the report is candidate-bound and all-passed, the
  manifest verifies the artifact and a detached signature verifies; atomic
  publish preserves and restores the previous release on failure.
- vmsafety.Results records the verified artifact digest; 3 new Python tests.
- 20 new Rust regression tests; ADR 0012 and docs updated.

Verified locally: make ci (364 Rust tests, 19 harness tests, release and
static builds) exit 0; static binary is static-pie linked. CI runner, VM and
model gates remain unexecuted here.
2026-09-22 07:34:27 -07:00

521 lines
18 KiB
Bash

#!/usr/bin/env python3
"""Safety, structured-result and QEMU-control helpers for the VM harnesses.
The VM drivers are test equipment: their own safety rules must be enforced, not
merely documented. This module centralises those rules so every driver shares
one implementation:
* every writable disk must be a regular **qcow2** file inside an explicitly
designated disposable directory, with a backing chain that is fully inside
that directory. Block devices, ``/dev/*`` paths, character devices, unsafe
symlinks and writable storage outside the test area are rejected;
* each run gets its own writable OVMF variable file and monitor socket;
* screenshots and monitor commands are issued without a shell, with argument
arrays, over structured QMP where possible;
* results are structured (PASS/FAIL/BLOCKED/SKIPPED/NOT RUN) and a failed,
timed-out or incomplete run exits nonzero.
Nothing here depends on pexpect, so the safety logic is unit-testable on any
host (see ``test_vmsafety.py``).
"""
from __future__ import annotations
import dataclasses
import json
import os
import shutil
import socket
import stat
import subprocess
import sys
import time
import uuid
from pathlib import Path
# ---------------------------------------------------------------------------
# Statuses
# ---------------------------------------------------------------------------
PASS = "PASS"
FAIL = "FAIL"
BLOCKED = "BLOCKED"
SKIPPED = "SKIPPED"
NOT_RUN = "NOT RUN"
_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED)
# The candidate artifact a harness run is executed against. The release tool
# binds a gate report to a specific digest; recording it here is what lets a
# later verification step refuse stale evidence rather than trust any pass.
ARTIFACT_ENV_NAME = "PHOEBUS_VM_ARTIFACT"
ARTIFACT_ENV_SHA256 = "PHOEBUS_VM_ARTIFACT_SHA256"
class UnsafeDiskError(Exception):
"""A writable target violated the disposable-disk rules."""
class HarnessError(Exception):
"""A harness precondition failed (bad environment, unsafe path, ...)."""
# ---------------------------------------------------------------------------
# Structured results
# ---------------------------------------------------------------------------
@dataclasses.dataclass
class Check:
name: str
status: str
detail: str = ""
def to_dict(self) -> dict:
return {"name": self.name, "status": self.status, "detail": self.detail}
class Results:
"""A run's assertions and their outcome.
A check may be recorded once. ``assert_true`` records PASS or FAIL; the
helpers ``blocked``/``skip`` record those explicit states. Any FAIL (and,
when strict, any BLOCKED or NOT RUN) makes :meth:`exit_code` nonzero.
"""
def __init__(self, scenario: str, run_id: str | None = None, strict: bool | None = None):
self.scenario = scenario
self.run_id = run_id or new_run_id(scenario)
self.checks: list[Check] = []
# The candidate artifact this run verifies (name + sha256), if known.
self.artifact: dict | None = None
if strict is None:
strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true")
self.strict = bool(strict)
# -- candidate artifact binding ---------------------------------------
def bind_artifact(self, name: str, sha256: str) -> dict:
"""Record the candidate artifact (name + digest) this run verified."""
self.artifact = {"name": name, "sha256": sha256}
return self.artifact
def bind_artifact_from_env(self) -> dict | None:
"""Bind from ``PHOEBUS_VM_ARTIFACT[_SHA256]`` when both are present.
The drivers call this through :meth:`write`, so a run always records the
digest of the candidate it exercised (when the launcher supplied it).
"""
name = os.environ.get(ARTIFACT_ENV_NAME)
sha256 = os.environ.get(ARTIFACT_ENV_SHA256)
if name and sha256:
return self.bind_artifact(name, sha256)
return self.artifact
# -- recording ---------------------------------------------------------
def _add(self, name: str, status: str, detail: str) -> Check:
if status not in _TERMINAL:
raise ValueError(f"invalid result status: {status}")
check = Check(name=name, status=status, detail=detail)
self.checks.append(check)
stream = sys.stderr if status == FAIL else sys.stdout
print(f"[{status:<7}] {name}" + (f": {detail}" if detail else ""), file=stream, flush=True)
return check
def assert_true(self, name: str, condition: bool, detail: str = "") -> bool:
ok = bool(condition)
# `detail` describes the failure; never print it on a pass, which would
# make a passing check read as if it had failed.
self._add(name, PASS if ok else FAIL, "" if ok else detail)
return ok
def fail(self, name: str, detail: str = "") -> Check:
return self._add(name, FAIL, detail)
def pass_(self, name: str, detail: str = "") -> Check:
return self._add(name, PASS, detail)
def blocked(self, name: str, detail: str = "") -> Check:
return self._add(name, BLOCKED, detail)
def skip(self, name: str, detail: str = "") -> Check:
return self._add(name, SKIPPED, detail)
# -- outcome -----------------------------------------------------------
@property
def failed(self) -> bool:
return any(c.status == FAIL for c in self.checks)
@property
def has_blockers(self) -> bool:
return any(c.status == BLOCKED for c in self.checks)
@property
def not_run(self) -> bool:
return not self.checks or any(c.status == NOT_RUN for c in self.checks)
def counts(self) -> dict:
out = {s: 0 for s in (PASS, FAIL, BLOCKED, SKIPPED, NOT_RUN)}
for c in self.checks:
out[c.status] += 1
return out
def exit_code(self) -> int:
if self.failed:
return 1
if self.strict and (self.has_blockers or self.not_run):
return 2
return 0
def summary(self) -> str:
counts = self.counts()
parts = [f"{k}={v}" for k, v in counts.items() if v]
return f"{self.scenario} [{self.run_id}]: " + (", ".join(parts) if parts else "no checks")
def write(self, path: str | Path) -> Path:
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
if self.artifact is None:
self.bind_artifact_from_env()
payload = {
"scenario": self.scenario,
"run_id": self.run_id,
"strict": self.strict,
"exit_code": self.exit_code(),
"counts": self.counts(),
"artifact": self.artifact,
"checks": [c.to_dict() for c in self.checks],
}
path.write_text(json.dumps(payload, indent=2) + "\n")
return path
def new_run_id(scenario: str) -> str:
stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
return f"{scenario}-{stamp}-{uuid.uuid4().hex[:8]}"
# ---------------------------------------------------------------------------
# Disposable disk rules
# ---------------------------------------------------------------------------
def _require_regular_and_safe(path: Path, test_dir: Path) -> os.stat_result:
text = str(path)
if text.startswith("/dev/") or text == "/dev":
raise UnsafeDiskError(f"refusing a /dev path: {text}")
# Symlink chains must stay inside the disposable directory when resolved.
real = Path(os.path.realpath(path))
if not _is_within(real, test_dir):
raise UnsafeDiskError(
f"writable disk resolves outside the disposable test directory "
f"({real} not under {test_dir})"
)
st = os.stat(path) # follows symlinks; realpath already checked containment
if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode):
raise UnsafeDiskError(f"refusing a block/character device: {text}")
if not stat.S_ISREG(st.st_mode):
raise UnsafeDiskError(f"refusing a non-regular file: {text}")
return st
def _is_within(child: Path, parent: Path) -> bool:
try:
child.relative_to(parent)
return True
except ValueError:
return False
def test_dir_is_safe(test_dir: Path) -> Path:
real = Path(os.path.realpath(test_dir))
if real == Path("/"):
raise UnsafeDiskError("the disposable test directory may not be /")
for bad in ("/dev", "/proc", "/sys"):
if real == Path(bad) or _is_within(real, Path(bad)):
raise UnsafeDiskError(f"the disposable test directory may not be under {bad}")
return real
def qcow2_magic(path: Path) -> bool:
try:
with open(path, "rb") as handle:
return handle.read(4) == b"QFI\xfb"
except OSError:
return False
def _qemu_img_backing_chain(path: Path) -> list[str]:
"""Return the backing-file paths reported by qemu-img (empty if none)."""
qemu_img = shutil.which("qemu-img")
if not qemu_img:
return []
proc = subprocess.run(
[qemu_img, "info", "--backing-chain", "--output=json", str(path)],
capture_output=True,
text=True,
)
if proc.returncode != 0:
return []
try:
data = json.loads(proc.stdout)
except json.JSONDecodeError:
return []
entries = data if isinstance(data, list) else [data]
chain: list[str] = []
for entry in entries:
for key in ("backing-filename", "full-backing-filename"):
value = entry.get(key)
if value:
chain.append(value)
return chain
def validate_disposable_disk(
path: str | os.PathLike,
test_dir: str | os.PathLike,
*,
require_qcow2: bool = True,
allow_missing: bool = False,
qemu_img: bool = True,
) -> Path:
"""Validate that ``path`` is a disposable qcow2 inside ``test_dir``.
Raises :class:`UnsafeDiskError` on any violation. On success returns the
resolved path.
"""
if path is None or str(path).strip() == "":
raise UnsafeDiskError("no writable disk path was provided")
test_real = test_dir_is_safe(Path(test_dir))
disk = Path(path)
if not disk.exists():
if allow_missing:
if not _is_within(Path(os.path.realpath(disk.parent)), test_real):
raise UnsafeDiskError(
f"new disk would be created outside the test directory: {disk}"
)
return disk
raise UnsafeDiskError(f"writable disk does not exist: {disk}")
st = _require_regular_and_safe(disk, test_real)
if st.st_size == 0:
raise UnsafeDiskError(f"writable disk is empty: {disk}")
if require_qcow2 and not qcow2_magic(disk):
raise UnsafeDiskError(
f"writable disk is not a qcow2 image (bad magic): {disk}; "
"refusing to hand a raw image to QEMU as writable"
)
if qemu_img:
for backing in _qemu_img_backing_chain(disk):
backing_real = Path(os.path.realpath(backing))
if not _is_within(backing_real, test_real):
raise UnsafeDiskError(
f"disk backing file is outside the test directory: {backing}"
)
if not os.access(disk, os.W_OK):
raise UnsafeDiskError(f"writable disk is not writable: {disk}")
return disk
def validate_writable_drives(paths, test_dir, *, qemu_img=True) -> list[Path]:
"""Validate every writable drive, not just one variable named DISK."""
checked = []
for path in paths:
if path is None or str(path).strip() == "":
continue
checked.append(validate_disposable_disk(path, test_dir, qemu_img=qemu_img))
if not checked:
raise UnsafeDiskError("no writable drives were provided to validate")
return checked
# ---------------------------------------------------------------------------
# Run-scoped state
# ---------------------------------------------------------------------------
@dataclasses.dataclass
class RunPaths:
run_id: str
directory: Path
serial_log: Path
monitor_sock: Path
shot_dir: Path
ovmf_vars: Path
results_json: Path
def ensure(self) -> "RunPaths":
self.directory.mkdir(parents=True, exist_ok=True)
os.chmod(self.directory, 0o700)
self.shot_dir.mkdir(parents=True, exist_ok=True)
os.chmod(self.shot_dir, 0o700)
return self
def make_run_paths(
scenario: str,
*,
base: str | os.PathLike | None = None,
ovmf_vars_template: str | os.PathLike | None = None,
run_id: str | None = None,
) -> RunPaths:
"""Create a private per-run directory and a fresh writable OVMF var file.
``base`` defaults to ``$PHOEBUS_VM_STATE`` or ``/var/tmp/phoebus-vm``.
"""
base_dir = Path(base or os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"))
base_dir.mkdir(parents=True, exist_ok=True)
os.chmod(base_dir, 0o700)
run = new_run_id(scenario) if run_id is None else run_id
directory = base_dir / run
paths = RunPaths(
run_id=run,
directory=directory,
serial_log=directory / "serial.log",
monitor_sock=directory / "monitor.sock",
shot_dir=directory / "shots",
ovmf_vars=directory / "OVMF_VARS.fd",
results_json=directory / "results.json",
).ensure()
if ovmf_vars_template:
shutil.copyfile(ovmf_vars_template, paths.ovmf_vars)
os.chmod(paths.ovmf_vars, 0o600)
return paths
# ---------------------------------------------------------------------------
# Screenshots (no shell)
# ---------------------------------------------------------------------------
def _reject_control_chars(value: str) -> str:
if "\n" in value or "\r" in value or "\x00" in value:
raise HarnessError("path contains a control character")
return value
def convert_ppm(ppm: str | os.PathLike, png: str | os.PathLike) -> Path | None:
"""Convert a PPM to PNG with an argument array (never a shell)."""
png = Path(_reject_control_chars(str(png)))
for program in ("magick", "convert"):
binary = shutil.which(program)
if not binary:
continue
proc = subprocess.run([binary, str(ppm), str(png)], capture_output=True)
if proc.returncode == 0 and png.exists():
return png
return None
def screenshot_name(seq: int, slug: str) -> str:
safe = "".join(c if (c.isalnum() or c in "-_") else "-" for c in slug).strip("-")
return f"{seq:02d}-{safe or 'shot'}"
# ---------------------------------------------------------------------------
# QMP client
# ---------------------------------------------------------------------------
class QmpError(Exception):
pass
class QmpClient:
"""A minimal structured QMP client over a UNIX socket.
Uses JSON framing (the monitor protocol deletes shell/quoting concerns) and
issues HMP commands through ``human-monitor-command`` for ``screendump`` and
``sendkey``.
"""
def __init__(self, sock_path: str | os.PathLike, timeout: float = 20.0):
self.sock_path = str(sock_path)
self.timeout = timeout
def __enter__(self) -> "QmpClient":
if not os.path.exists(self.sock_path):
raise QmpError(f"monitor socket does not exist: {self.sock_path}")
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sock.settimeout(self.timeout)
self.sock.connect(self.sock_path)
self._file = self.sock.makefile("rwb")
greeting = self._read_json()
if "QMP" not in greeting:
raise QmpError("server did not send a QMP greeting")
self._command({"execute": "qmp_capabilities"})
return self
def __exit__(self, *_exc) -> None:
try:
self._file.close()
finally:
self.sock.close()
def _read_json(self) -> dict:
line = self._file.readline()
if not line:
raise QmpError("unexpected EOF from the QMP monitor")
try:
return json.loads(line)
except json.JSONDecodeError as exc:
raise QmpError(f"invalid QMP message: {line[:200]!r}") from exc
def _command(self, command: dict) -> dict:
self._file.write(json.dumps(command).encode() + b"\n")
self._file.flush()
while True:
message = self._read_json()
if "event" in message:
continue
if "error" in message:
raise QmpError(str(message["error"]))
return message
def hmp(self, command: str) -> str:
reply = self._command({"execute": "human-monitor-command", "arguments": {"command-line": command}})
return reply.get("return", "")
def screendump(self, target: str | os.PathLike) -> Path:
target = Path(_reject_control_chars(str(target)))
target.parent.mkdir(parents=True, exist_ok=True)
# QEMU's HMP screendump takes a single filename argument, so a path with
# spaces is quoted for QEMU's own tiny parser, not for a shell.
quoted = '"' + str(target).replace("\\", "\\\\").replace('"', '\\"') + '"'
self.hmp(f"screendump {quoted}")
deadline = time.monotonic() + self.timeout
while time.monotonic() < deadline:
if target.exists() and target.stat().st_size > 0:
return target
time.sleep(0.2)
raise QmpError(f"screendump did not produce {target}")
def sendkey(self, key: str) -> None:
self.hmp(f"sendkey {key}")
def screendump(monitor_sock, target, *, convert: bool = True) -> Path | None:
"""Capture the framebuffer and optionally convert it to PNG.
Returns the PPM path, or ``None`` if capture failed. Never raises for a
missing PNG converter (the PPM is still evidence).
"""
target = Path(_reject_control_chars(str(target)))
try:
with QmpClient(monitor_sock) as client:
ppm = client.screendump(target)
except (QmpError, OSError):
return None
if convert:
convert_ppm(ppm, ppm.with_suffix(".png"))
return ppm
def wait_for_file(path: str | os.PathLike, timeout: float, interval: float = 0.5) -> bool:
deadline = time.monotonic() + timeout
p = Path(path)
while time.monotonic() < deadline:
if p.exists():
return True
time.sleep(interval)
return False