diff --git a/Makefile b/Makefile index ecd1682..61e8c14 100644 --- a/Makefile +++ b/Makefile @@ -8,14 +8,15 @@ SHELL := /bin/bash -.PHONY: help context doctor test test-rust fmt fmt-check clippy check build static vm hooks commit publish backup forgejo-init public private protect release clean +.PHONY: help context doctor test test-rust test-vm fmt fmt-check clippy check build static vm hooks commit publish backup forgejo-init public private protect release clean help: @printf 'Phoebus OS targets:\n' @printf ' make context session briefing (state, roadmap, handoff)\n' @printf ' make doctor check host prerequisites\n' - @printf ' make test run the Rust test suite\n' + @printf ' make test run the Rust and VM-harness test suites\n' @printf ' make test-rust cargo test --workspace\n' + @printf ' make test-vm python3 installer/tests/vm/test_vmsafety.py\n' @printf ' make fmt cargo fmt --all\n' @printf ' make check fmt-check + clippy + tests\n' @printf ' make build release Rust installer\n' @@ -35,11 +36,14 @@ context: doctor: scripts/doctor.sh -test: test-rust +test: test-rust test-vm test-rust: cd installer-rs && cargo test --workspace +test-vm: + python3 installer/tests/vm/test_vmsafety.py + fmt: cd installer-rs && cargo fmt --all diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md new file mode 100644 index 0000000..c20a51f --- /dev/null +++ b/docs/VERIFICATION.md @@ -0,0 +1,66 @@ +# Verification and findings register + +Companion to `docs/STATUS.md`. The machine-readable per-run results live outside +git under `/var/tmp/phoebus-vm//results.json`; this file is the +human-readable register of what was found, fixed and still unverified. + +## Run matrix (this session, 2026-09-22) + +```json +{ + "source_revision": "7e9fa9c", + "iso_sha256": "428dbd7547db8690596bf5c3a0d7a6fa8aee4c9f9eca1e926c216712dd050b36", + "static_binary_sha256": "2f9cd4b36a12e469d2b6187166acaf8b11f7f382cc19f7ad08bb2a809c855591", + "runs": [ + {"scenario": "gui", "result": "PASS 23/23", "exit_code": 0, + "artifacts": "/var/tmp/phoebus-vm/gui-gui-20260922T025537Z-69f754cf"}, + {"scenario": "make test (rust + vmsafety)", "result": "PASS", "exit_code": 0}, + {"scenario": "cargo fmt --check", "result": "PASS", "exit_code": 0}, + {"scenario": "cargo clippy --workspace --all-targets", "result": "PASS (warnings only)", "exit_code": 0}, + {"scenario": "opencode probe (live 1.18.32)", "result": "ready", "exit_code": 0}, + {"scenario": "opencode probe (unauth LAN bind)", "result": "authentication_required", "exit_code": 1}, + {"scenario": "noninteractive install + detached boot", "result": "NOT RUN on the final ISO", "exit_code": null}, + {"scenario": "full graphical install past AI", "result": "BLOCKED", "exit_code": null}, + {"scenario": "rollback VM acceptance", "result": "NOT RUN", "exit_code": null}, + {"scenario": "AI sandbox canary VM test", "result": "NOT RUN", "exit_code": null}, + {"scenario": "second physical computer OpenCode transport", "result": "BLOCKED", "exit_code": null} + ] +} +``` + +## Confirmed defects fixed + +| ID | Defect | Symptom | Fix | Regression test | +|----|--------|---------|-----|-----------------| +| D1 | Disk safety was not revalidated before destroying the target | A disk that disappeared, was mounted, became the live medium, or was replaced under the same name between selection and install would be partitioned on the strength of a stale review | `Installer::revalidate_target_disk` re-probes and compares identity in `storage_prepare` | `orchestrate::tests::{revalidation_refuses_a_disappeared_disk,revalidation_refuses_a_now_protected_disk,revalidation_refuses_reused_device_name_with_a_new_size,...}` | +| D2 | `probe_disks` parsed human-readable `lsblk` sizes as bytes | Every disk displayed `0 B`; the size-identity check was blind | pass `-b` to `lsblk` | `probe::tests::probe_disks_classifies_optical_as_protected` asserts the byte size | +| D3 | The bridge could replay or overlap a destructive install | A second `start_install` (retry, second frontend, reconnect) could re-run storage preparation | approval is single-use, consumed before work; `installing`/`installed` guards | `start_install_cannot_be_replayed_after_success` | +| D4 | The welcome's primary action was not keyboard-focused | No focus ring; Return did not open the installer | request activation + `forceActiveFocus()`; regression test also rejects the invalid `onActiveChanged` signal | `welcome_primary_action_is_keyboard_reachable` | +| D5 | Escape during installation silently closed the frontend | A destructive operation could be abandoned with no explanation | Escape is refused during progress and the UI says so | `progress_cannot_be_abandoned_silently` | +| D6 | Provider transport accepted unsafe URL schemes / unbounded responses / followed no protocol allowlist | A config-supplied endpoint could target `file:`, inject curl config, or return a huge body | `validate_endpoint`, `proto = "=http,https"`, `max-filesize`, redirects disabled | `endpoint_validation_rejects_unsafe_schemes_and_injection`, `curl_transport_refuses_an_unsafe_endpoint`, `curl_transport_bounds_the_response_and_disables_redirects` | +| D7 | `TargetRoot` followed symlinks out of the target | A planted symlink could redirect an installer write to the host | `reject_symlink_components` in `write`/`ensure_dir` | `target::tests::{refuses_to_write_through_an_escaping_symlink,refuses_a_symlinked_leaf}` | +| D8 | The VM harness documented disk safety but did not enforce it, and reported observations rather than results | A mistyped `DISK` could point at a real device; a timeout could look like success; shell interpolation in screenshot conversion | `vmsafety.py`; per-run OVMF/QMP; structured results; argument arrays | `test_vmsafety.py` (16 tests), run by `make test` | + +## Suspected risks (reproduced or bounded) + +- **Prompt injection in device labels / model output.** Typed proposals pass + `deny_unknown_fields` and a closed action set; device labels are display-only. + Bounded by tests (`phoebus-ai` parse/allowlist). No VM canary run. +- **Live sudoers is `NOPASSWD: ALL` (live-only).** It does not reach the + installed system, but the GUI bridge uses it. Narrowing is a recorded follow-up. +- **OpenCode `/config/providers` returns provider credentials in cleartext.** + Upstream behaviour; the probe keeps only counts and docs mandate loopback + + tunnel. Not a Phoebus defect, but a deployment hazard. + +## Missing verification (blocked / not run) + +- Full graphical installation past the AI step and two fresh-disk graphical + installs: **BLOCKED** — no provider key and no local model; normal install + requires a connected model. +- `install`/`boot` modes on the final ISO: **NOT RUN** (the earlier appliance + build passed `INSTALL-RC=0` + detached boot; the final ISO's binary changed, + so this is not automatically re-verified). +- Rollback acceptance, AI sandbox canary, installed Mango/dwl, physical seat, + NVIDIA: **NOT RUN / unverified**. +- Second physical computer OpenCode transport: **BLOCKED**; host-local forwarded + hop and unauthenticated-LAN refusal were tested instead. diff --git a/installer/tests/vm/README.md b/installer/tests/vm/README.md index 01e9ec1..7d16c75 100644 --- a/installer/tests/vm/README.md +++ b/installer/tests/vm/README.md @@ -62,9 +62,35 @@ To patch a console into an existing entry for observation, the BLS entry in `/efi/loader/entries/*.conf` takes an `options` line; the installer can also embed a console via `KERNEL_CONSOLE` (`GENTOO_KERNEL_CONSOLE` equivalent). -## Safety +## Safety (enforced, not just documented) -- `DISK` must be a qcow2 image; never point it at `/dev/*`. -- The installer itself classifies the ISO's device as protected. -- Secrets are passed through 0600 files and are checked for leakage in the - installer log by `install.sh`. +`vmsafety.py` validates every writable drive before QEMU starts: + +- `DISK`/`DISKS` must name **qcow2** files inside the disposable directory + `PHOEBUS_VM_DISKS` (default `$PHOEBUS_VM_STATE/disks`). +- `/dev/*` paths, block and character devices, non-regular files, raw images, + symlinks that resolve outside the directory, and disks whose backing chain + reaches outside it are **refused** with a nonzero exit. +- Each run gets a private state directory (mode 0700), its own writable OVMF + variable file and a unique QMP monitor socket. +- Screenshots use structured QMP and argument arrays; there is no shell + interpolation (`os.system`) in the harness. +- Secrets are passed through 0600 files and checked for leakage in the + installer log by the guest script. + +## Results + +Every driver writes `results.json` (and a `screenshot-manifest.json` for the +GUI driver) under its run directory and exits nonzero on any `FAIL` (and, with +`PHOEBUS_VM_STRICT=1`, on `BLOCKED`/`NOT RUN`). Statuses are `PASS`, `FAIL`, +`BLOCKED`, `SKIPPED` and `NOT RUN`; a timeout is always a `FAIL`, never a pass. + +```sh +make test-vm # unit-test the safety and result logic (no VM needed) +PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \ +OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \ +OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \ +DISK=/var/tmp/phoebus-vm/disks/target.qcow2 \ +ISO=installer/live/phoebus-installer-amd64.iso \ +python3 installer/tests/vm/gui-driver.py gui +``` diff --git a/installer/tests/vm/appliance-driver.py b/installer/tests/vm/appliance-driver.py index 3053c34..9fad147 100644 --- a/installer/tests/vm/appliance-driver.py +++ b/installer/tests/vm/appliance-driver.py @@ -13,10 +13,12 @@ Two modes: boot : boot the installed disk alone (no ISO), unlock LUKS, log in and run verification commands. -Everything is parameterised through environment variables. `DISK` must be a -qcow2 image; this script never touches a physical disk. Secrets are served from -the host as 0600 files and fetched inside the guest, so they never appear in -the serial transcript. +Everything is parameterised through environment variables. `DISK`/`DISKS` must +be qcow2 files inside the disposable directory named by `PHOEBUS_VM_DISKS` +(default `$PHOEBUS_VM_STATE/disks`); block devices, `/dev/*`, escaping +symlinks, raw images and outside backing chains are refused. Secrets are served +from the host as 0600 files and fetched inside the guest, so they never appear +in the serial transcript. Host setup (see docs/DEVELOPMENT.md): @@ -31,15 +33,18 @@ Host setup (see docs/DEVELOPMENT.md): """ import os import sys +from pathlib import Path import pexpect +import vmsafety +from vmsafety import Results + QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64") OVMF_CODE = os.environ["OVMF_CODE"] -OVMF_VARS = os.environ["OVMF_VARS"] -DISK = os.environ["DISK"] +OVMF_VARS_TEMPLATE = os.environ["OVMF_VARS"] +DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "") ISO = os.environ.get("ISO", "") -SERIAL_LOG = os.environ.get("SERIAL_LOG", "/tmp/phoebus-appliance-serial.log") SERVE_URL = os.environ.get("SERVE_URL", "http://10.0.2.2:8000") LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test") USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test") @@ -51,29 +56,37 @@ SMP = os.environ.get("VM_SMP", "8") INSTALL_TIMEOUT = int(os.environ.get("INSTALL_TIMEOUT", "10800")) -def base_args(): - return [ +def disposable_dir() -> Path: + base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm") + return Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks")) + + +def base_args(ovmf_vars, disks): + args = [ "-machine", "q35,accel=kvm", "-cpu", "host", "-smp", SMP, "-m", MEM, "-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}", - "-drive", f"if=pflash,format=raw,file={OVMF_VARS}", - "-drive", f"file={DISK},if=virtio,format=qcow2", + "-drive", f"if=pflash,format=raw,file={ovmf_vars}", + ] + for disk in disks: + args += ["-drive", f"file={disk},if=virtio,format=qcow2"] + args += [ "-netdev", "user,id=n0", "-device", "virtio-net-pci,netdev=n0,romfile=", "-device", "virtio-rng-pci", "-vga", "std", "-display", "none", "-serial", "stdio", "-monitor", "none", "-no-reboot", ] + return args -def spawn(args): +def spawn(args, log): child = pexpect.spawn(QEMU, args, timeout=900, encoding="utf-8", codec_errors="replace", maxread=65536) child.delaybeforesend = 0.4 + child.logfile = log return child def login_live(child): - # The live medium may auto-login or present a login prompt; its hostname is - # typically `debian` (standard live) or `live`. idx = child.expect([r"(?:live|debian) login:", r"[$#] $"], timeout=900) if idx == 0: child.sendline(LIVE_USER) @@ -82,16 +95,17 @@ def login_live(child): child.expect(r"[$#] $", timeout=120) -def do_install(): - args = base_args() +def do_install(results, paths, disks): if not ISO: - raise SystemExit("ISO is required for install mode") + results.blocked("iso-required", "ISO is required for install mode") + return + args = base_args(str(paths.ovmf_vars), disks) args += ["-drive", f"file={ISO},media=cdrom,readonly=on"] - log = open(SERIAL_LOG, "w") - child = spawn(args) - child.logfile = log + log = open(paths.serial_log, "w") + child = spawn(args, log) try: login_live(child) + results.pass_("live-login") child.sendline( f"curl -fsSL {SERVE_URL}/appliance-guest.sh -o /tmp/gi.sh && bash /tmp/gi.sh" ) @@ -99,18 +113,25 @@ def do_install(): "Kernel panic", pexpect.EOF, pexpect.TIMEOUT], timeout=INSTALL_TIMEOUT) if "INSTALL-RC" in (child.after or ""): - print("INSTALL_RC", child.match.group(1), flush=True) + rc = child.match.group(1) + results.assert_true("install-rc-zero", rc == "0", f"INSTALL-RC={rc}") else: - print("INSTALL did not report a status", flush=True) + results.fail("install-rc-zero", "install did not report a status") + serial = paths.serial_log.read_text(errors="replace") + results.assert_true("no-secret-leak", "SECRET_LEAK_FOUND" not in serial, + "guest reported a secret leak") child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600) - child.expect(pexpect.EOF, timeout=300) + except pexpect.TIMEOUT as exc: + results.fail("install", f"timeout: {exc}") + except pexpect.EOF: + results.fail("install", "unexpected EOF") finally: child.close(force=True) log.close() -def do_boot(): - args = base_args() + ["-boot", "order=c"] +def do_boot(results, paths, disks): + args = base_args(str(paths.ovmf_vars), disks) + ["-boot", "order=c"] checks = os.environ.get( "BOOT_CHECKS", "echo ===CHECKS-$(printf START)===; " @@ -120,13 +141,13 @@ def do_boot(): "/usr/local/bin/sys rollback status 2>&1 || true; " "echo ===CHECKS-$(printf END)===", ) - log = open(SERIAL_LOG, "w") - child = spawn(args) - child.logfile = log + log = open(paths.serial_log, "w") + child = spawn(args, log) try: child.expect([r"[Pp]assphrase", r"login:"], timeout=600) if "assphrase" in (child.after or ""): child.sendline(LUKS_PW) + logged_in = False for _ in range(5): child.expect(r"phoebus login:", timeout=300) child.sendline(USER) @@ -136,21 +157,52 @@ def do_boot(): child.sendline(USER_PW) if child.expect([r"[$#] $", r"Login incorrect", r"phoebus login:"], timeout=120) == 0: + logged_in = True break + results.assert_true("detached-boot-login", logged_in, "could not log in") child.sendline(checks) child.expect("===CHECKS-END===", timeout=int(os.environ.get("CHECKS_TIMEOUT", "300"))) - print("checks complete", flush=True) + serial = paths.serial_log.read_text(errors="replace") + results.assert_true("boot-systemd-running", "running" in serial, + "systemd did not report running") + except pexpect.TIMEOUT as exc: + results.fail("boot", f"timeout: {exc}") + except pexpect.EOF: + results.fail("boot", "unexpected EOF") finally: child.close(force=True) log.close() -if __name__ == "__main__": +def main() -> int: mode = sys.argv[1] if len(sys.argv) > 1 else "" - if mode == "install": - do_install() - elif mode == "boot": - do_boot() - else: + if mode not in ("install", "boot"): print(__doc__) - sys.exit(2) + return 2 + results = Results(f"appliance-{mode}") + paths = vmsafety.make_run_paths( + f"appliance-{mode}", + base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"), + ovmf_vars_template=OVMF_VARS_TEMPLATE, + ) + try: + disks = vmsafety.validate_writable_drives(DISKS.split(","), disposable_dir()) + except vmsafety.UnsafeDiskError as exc: + results.blocked("disposable-disk", str(exc)) + results.write(paths.results_json) + print(f"harness error: {exc}", file=sys.stderr) + return 2 + results.pass_("disposable-disk", ", ".join(str(d) for d in disks)) + try: + if mode == "install": + do_install(results, paths, disks) + else: + do_boot(results, paths, disks) + finally: + results.write(paths.results_json) + print(results.summary(), flush=True) + return results.exit_code() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/installer/tests/vm/gui-driver.py b/installer/tests/vm/gui-driver.py index 7bb97bb..0fb478f 100644 --- a/installer/tests/vm/gui-driver.py +++ b/installer/tests/vm/gui-driver.py @@ -4,343 +4,645 @@ The live medium boots into Mango + DankMaterialShell and opens the branded welcome window. This driver boots it with a graphical device and a serial console, checks the launcher/branding payload, drives the graphical installer -with `wtype` over the live session, and captures QEMU framebuffer screenshots. +over the live session, and captures QEMU framebuffer screenshots through QMP. Modes: gui : boot, log in on serial, verify the welcome/branding payload, open - the installer, prove duplicate prevention, walk the first steps and - capture screenshots. + the installer from the welcome action, prove duplicate prevention, + walk the wizard steps and capture screenshots. install : boot, log in on serial, fetch a non-interactive config and 0600 secrets over QEMU user networking, run the deterministic install with the static binary, scan the log for a secret leak, power off. boot : boot the installed disk alone (no ISO), unlock LUKS, log in and run verification commands. -`DISK` must always be a disposable qcow2 image. This script never touches a -physical disk. +Safety: every writable drive must be a qcow2 file inside the disposable +directory named by `PHOEBUS_VM_DISKS` (default +`$PHOEBUS_VM_STATE/disks`). The driver refuses block devices, `/dev/*`, +symlinks that escape the directory, raw images and images with an outside +backing chain. Each run gets its own private state directory, a fresh writable +OVMF variable file and a unique QMP monitor socket. Results are structured and +the process exits nonzero on any failed assertion, timeout, unexpected EOF or +missing result. """ + +from __future__ import annotations + +import json import os -import socket +import re +import shutil import sys import time +from pathlib import Path import pexpect +import vmsafety +from vmsafety import BLOCKED, FAIL, PASS, Results, HarnessError + QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64") OVMF_CODE = os.environ["OVMF_CODE"] -OVMF_VARS = os.environ["OVMF_VARS"] +OVMF_VARS_TEMPLATE = os.environ["OVMF_VARS"] ISO = os.environ.get("ISO", "") -DISK = os.environ.get("DISK", "") -SERIAL_LOG = os.environ.get("SERIAL_LOG", "/tmp/phoebus-gui-serial.log") -MONITOR = os.environ.get("MONITOR", "/tmp/phoebus-gui-monitor.sock") -SHOT_DIR = os.environ.get("SHOT_DIR", "/tmp/phoebus-gui-shots") +DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "") +SERVE_URL = os.environ.get("SERVE_URL", "http://10.0.2.2:8000") LIVE_USER = os.environ.get("LIVE_USER", "phoebus") LIVE_PW = os.environ.get("LIVE_PASSWORD", "phoebus") USER = os.environ.get("USERNAME", "phoebus") USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test") LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test") -SERVE_URL = os.environ.get("SERVE_URL", "http://10.0.2.2:8000") MEM = os.environ.get("VM_MEM", "8192") SMP = os.environ.get("VM_SMP", "8") BOOT_TIMEOUT = int(os.environ.get("BOOT_TIMEOUT", "900")) INSTALL_TIMEOUT = int(os.environ.get("INSTALL_TIMEOUT", "10800")) - -WAYLAND_ENV = "XDG_RUNTIME_DIR=/run/user/1000" +SESSION_SETTLE = int(os.environ.get("SESSION_SETTLE", "20")) +BACKEND_SETTLE = int(os.environ.get("BACKEND_SETTLE", "30")) -def args(attach_iso=True, boot_disk=False): - a = [ - "-machine", "q35,accel=kvm", - "-cpu", "host", "-smp", SMP, "-m", MEM, - "-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}", - "-drive", f"if=pflash,format=raw,file={OVMF_VARS}", - ] - if DISK: - a += ["-drive", f"file={DISK},if=virtio,format=qcow2"] - a += [ - # A real GPU device so the compositor gets a DRM node and a seat. - "-device", "virtio-gpu-pci", - "-netdev", "user,id=n0", "-device", "virtio-net-pci,netdev=n0,romfile=", - "-device", "virtio-rng-pci", - ] - if attach_iso and ISO: - a += ["-drive", f"file={ISO},media=cdrom,readonly=on"] - a += [ - "-boot", "order=c" if boot_disk else "order=d", - "-vga", "none", "-display", "none", - "-serial", "stdio", - "-monitor", f"unix:{MONITOR},server,nowait", - "-no-reboot", - ] - return a - - -def spawn(attach_iso=True, boot_disk=False): - if os.path.exists(MONITOR): - os.unlink(MONITOR) - child = pexpect.spawn(QEMU, args(attach_iso, boot_disk), timeout=BOOT_TIMEOUT, - encoding="utf-8", codec_errors="replace", maxread=65536) - child.delaybeforesend = 0.4 - return child - - -def screendump(name): - """Capture the QEMU framebuffer to SHOT_DIR/.ppm (and .png if - ImageMagick is available).""" - os.makedirs(SHOT_DIR, exist_ok=True) - target = os.path.join(SHOT_DIR, name + ".ppm") - if not os.path.exists(MONITOR): - return None - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - s.settimeout(20) - try: - s.connect(MONITOR) - time.sleep(0.5) - try: - s.recv(65536) - except socket.timeout: - pass - s.sendall(b"screendump %s\n" % target.encode()) - time.sleep(2) - try: - s.recv(65536) - except socket.timeout: - pass - finally: - s.close() - if not os.path.exists(target): - return None - os.system(f"magick '{target}' '{target[:-4]}.png' 2>/dev/null || " - f"convert '{target}' '{target[:-4]}.png' 2>/dev/null") - return target - - -def login_live(child): - # The live serial getty shows ` login:` and the password is the - # known live-only credential. - idx = child.expect([r"login:", r"[$#] $"], timeout=BOOT_TIMEOUT) - if idx == 0: - child.sendline(LIVE_USER) - child.expect(r"[Pp]assword:", timeout=90) - child.sendline(LIVE_PW) - child.expect(r"[$#] $", timeout=180) - - -def live_shell(child, command, end_marker, timeout=180): - child.sendline(command) - child.expect(end_marker, timeout=timeout) - - -def set_wayland(child): - # Export the live user's Wayland socket for wtype-based GUI driving. - child.sendline( - 'export XDG_RUNTIME_DIR=/run/user/1000; ' - 'export WAYLAND_DISPLAY="$(basename "$(ls /run/user/1000/wayland-* 2>/dev/null | head -1)")"; ' - 'echo WAYLAND=$WAYLAND_DISPLAY' +def _template_dir() -> Path: + return Path( + os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm") ) - child.expect(r"WAYLAND=wayland-\d+", timeout=60) -def sendkey(*keys): - """Inject keys through the QEMU monitor (hardware keyboard level). This is - independent of the guest's input-method/portal setup.""" - if not os.path.exists(MONITOR): - return - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - s.settimeout(10) - try: - s.connect(MONITOR) - time.sleep(0.3) +def _disposable_dir() -> Path: + return Path(os.environ.get("PHOEBUS_VM_DISKS", str(_template_dir() / "disks"))) + + +class Driver: + def __init__(self, mode: str): + self.mode = mode + self.results = Results(f"gui-{mode}") + self.paths = vmsafety.make_run_paths( + f"gui-{mode}", base=_template_dir(), ovmf_vars_template=OVMF_VARS_TEMPLATE + ) + self.manifest: list[dict] = [] + self.seq = 0 + self.child: pexpect.spawn | None = None + self.log = None + self.spawned = False + self.disks: list[Path] = [] + self.boot_disk = False + self._prepare() + + # -- setup ------------------------------------------------------------- + def _prepare(self) -> None: + if not Path(QEMU).exists() and shutil.which(QEMU) is None: + raise HarnessError(f"QEMU binary not found: {QEMU}") + if DISKS: + try: + self.disks = vmsafety.validate_writable_drives( + DISKS.split(","), _disposable_dir() + ) + except vmsafety.UnsafeDiskError as exc: + raise HarnessError(str(exc)) from exc + if ISO: + iso = Path(ISO) + if not iso.is_file(): + raise HarnessError(f"ISO is not a regular file: {iso}") + if not Path(OVMF_CODE).is_file(): + raise HarnessError(f"OVMF_CODE is not a file: {OVMF_CODE}") + if not Path(OVMF_VARS_TEMPLATE).is_file(): + raise HarnessError(f"OVMF_VARS template is not a file: {OVMF_VARS_TEMPLATE}") + + # -- qemu -------------------------------------------------------------- + def qemu_args(self, attach_iso: bool, boot_disk: bool) -> list[str]: + args = [ + "-machine", "q35,accel=kvm", + "-cpu", "host", "-smp", SMP, "-m", MEM, + "-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}", + "-drive", f"if=pflash,format=raw,file={self.paths.ovmf_vars}", + ] + for disk in self.disks: + args += ["-drive", f"file={disk},if=virtio,format=qcow2"] + args += [ + # A real GPU device so the compositor gets a DRM node and a seat. + "-device", "virtio-gpu-pci", + "-netdev", "user,id=n0", "-device", "virtio-net-pci,netdev=n0,romfile=", + "-device", "virtio-rng-pci", + ] + if attach_iso and ISO: + args += ["-drive", f"file={ISO},media=cdrom,readonly=on"] + args += [ + "-boot", "order=c" if boot_disk else "order=d", + "-vga", "none", "-display", "none", + "-serial", "stdio", + "-qmp", f"unix:{self.paths.monitor_sock},server,nowait", + "-no-reboot", + ] + return args + + def spawn(self, attach_iso: bool = True, boot_disk: bool = False) -> None: + if self.paths.monitor_sock.exists(): + self.paths.monitor_sock.unlink() + args = self.qemu_args(attach_iso, boot_disk) + print(f"[run {self.paths.run_id}] {QEMU} " + " ".join(args), flush=True) + self.child = pexpect.spawn( + QEMU, args, timeout=BOOT_TIMEOUT, encoding="utf-8", + codec_errors="replace", maxread=65536, + ) + self.child.delaybeforesend = 0.4 + self.log = open(self.paths.serial_log, "w") + self.child.logfile = self.log + self.spawned = True + self.results.pass_( + "qemu-spawned", f"{len(self.disks)} writable disk(s); iso={bool(ISO)}" + ) + + # -- assertions and screenshots --------------------------------------- + def run_id(self) -> str: + return self.paths.run_id + + def shot( + self, + slug: str, + ui_state: str, + action: str, + expected: str, + observed: str | None = None, + ) -> Path | None: + self.seq += 1 + name = vmsafety.screenshot_name(self.seq, slug) + ppm = self.paths.shot_dir / (name + ".ppm") + png = self.paths.shot_dir / (name + ".png") + if not self.paths.monitor_sock.exists(): + self.results.skip(f"screenshot-{name}", "no monitor socket (not spawned)") + return None + ok = False try: - s.recv(65536) - except socket.timeout: - pass - for k in keys: - s.sendall(("sendkey %s\n" % k).encode()) + with vmsafety.QmpClient(self.paths.monitor_sock) as client: + client.screendump(ppm) + vmsafety.convert_ppm(ppm, png) + ok = ppm.exists() and ppm.stat().st_size > 0 + except Exception as exc: # noqa: BLE001 - report, never crash the run + self.results.fail(f"screenshot-{name}", str(exc)) + self.manifest.append({ + "run_id": self.paths.run_id, + "scenario": self.mode, + "sequence": self.seq, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "ui_state": ui_state, + "action": action, + "expected": expected, + "observed": observed if observed is not None else ("captured" if ok else "capture failed"), + "ppm": str(ppm) if ppm.exists() else None, + "png": str(png) if png.exists() else None, + }) + if ok: + self.results.pass_(f"screenshot-{name}", str(png if png.exists() else ppm)) + return ppm if ok else None + + def write_manifest(self) -> None: + path = self.paths.directory / "screenshot-manifest.json" + path.write_text(json.dumps(self.manifest, indent=2) + "\n") + + # -- pexpect helpers --------------------------------------------------- + def expect(self, patterns, timeout: int, label: str): + assert self.child is not None + try: + index = self.child.expect(patterns, timeout=timeout) + except pexpect.TIMEOUT as exc: + self.results.fail(label, f"timeout after {timeout}s") + raise HarnessError(f"timeout: {label}") from exc + except pexpect.EOF as exc: + self.results.fail(label, "unexpected EOF from the VM") + raise HarnessError(f"unexpected EOF: {label}") from exc + self.last_match = self.child.match + return index + + def last_group(self, index: int) -> str: + match = getattr(self, "last_match", None) + if match is None: + return "" + try: + return match.group(index) + except (IndexError, AttributeError): + return "" + + def last_text(self) -> str: + match = getattr(self, "last_match", None) + return match.group(0) if match is not None else "" + + def shell(self, command: str, end_marker: str, timeout: int = 180, label: str = "") -> str: + assert self.child is not None + self.child.sendline(command) + self.expect(end_marker, timeout=timeout, label=label or f"shell:{end_marker}") + return self.child.before or "" + + def poll_shell( + self, command: str, pattern: str, timeout: int, label: str, record: bool = True + ) -> str | None: + """Send a command and retry until `pattern` matches or the deadline. + + Bounded readiness polling instead of blind sleeps. `record=False` lets a + caller retry without recording an intermediate failure. + """ + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + assert self.child is not None + self.child.sendline(command) + try: + self.child.expect(pattern, timeout=15) + return self.child.match.group(0) + except (pexpect.TIMEOUT, pexpect.EOF): + time.sleep(2) + if record: + self.results.fail(label, f"readiness not reached within {timeout}s") + return None + + def qmp(self, action: str, *args, **kwargs) -> None: + assert self.child is not None + try: + with vmsafety.QmpClient(self.paths.monitor_sock) as client: + getattr(client, action)(*args, **kwargs) + except Exception as exc: # noqa: BLE001 + self.results.fail(f"qmp-{action}", str(exc)) + raise + + def sendkey(self, *keys: str) -> None: + for key in keys: + self.qmp("sendkey", key) time.sleep(0.2) - time.sleep(0.3) + + # -- guest login ------------------------------------------------------- + def login_live(self) -> None: + assert self.child is not None + idx = self.expect([r"login:", r"[$#] $"], timeout=BOOT_TIMEOUT, label="live-login-prompt") + if idx == 0: + self.child.sendline(LIVE_USER) + self.expect(r"[Pp]assword:", timeout=90, label="live-password-prompt") + self.child.sendline(LIVE_PW) + self.expect(r"[$#] $", timeout=180, label="live-shell") + self.results.pass_("live-login") + + def set_wayland(self) -> None: + out = self.shell( + 'export XDG_RUNTIME_DIR=/run/user/1000; ' + 'export WAYLAND_DISPLAY="$(basename "$(ls /run/user/1000/wayland-* 2>/dev/null | head -1)")"; ' + 'echo WAYLAND=$WAYLAND_DISPLAY', + r"WAYLAND=wayland-\d+", + timeout=60, + label="wayland-session", + ) + self.results.pass_("wayland-session", out.strip()[-40:]) + + # -- modes ------------------------------------------------------------- + def run(self) -> int: try: - s.recv(65536) - except socket.timeout: + if self.mode == "gui": + self.do_gui() + elif self.mode == "install": + self.do_install() + elif self.mode == "boot": + self.do_boot() + else: + print(__doc__) + return 2 + except HarnessError as exc: + self.results.fail("harness", str(exc)) + except Exception as exc: # noqa: BLE001 + self.results.fail("harness-exception", repr(exc)) + finally: + if self.spawned: + try: + self.shot("final-state", "final", "end of run", "last observed state") + except Exception: # noqa: BLE001 + pass + self.teardown() + self.write_manifest() + results_path = self.results.write(self.paths.results_json) + print(self.results.summary(), flush=True) + print(f"results: {results_path}", flush=True) + print(f"serial: {self.paths.serial_log}", flush=True) + print(f"shots: {self.paths.shot_dir}", flush=True) + return self.results.exit_code() + + def teardown(self) -> None: + if self.child is not None: + try: + self.child.close(force=True) + except Exception: # noqa: BLE001 + pass + if self.log is not None: + try: + self.log.close() + except Exception: # noqa: BLE001 + pass + # Cleanup is limited to this run's own socket. + try: + if self.paths.monitor_sock.exists(): + self.paths.monitor_sock.unlink() + except OSError: pass - finally: - s.close() + # -- GUI journey ------------------------------------------------------- + def do_gui(self) -> None: + self.spawn(attach_iso=True, boot_disk=False) + self.login_live() -def key(child, *wtype_args): - quoted = " ".join(f"'{a}'" for a in wtype_args) - child.sendline(f"wtype {quoted} >/dev/null 2>&1; echo KEY-DONE") - child.expect("KEY-DONE", timeout=60) + welcome = self.poll_shell( + "echo WELCOME=$(pgrep -fc 'quickshell.*phoebus-welcome' 2>/dev/null || echo 0)", + r"WELCOME=([1-9][0-9]*)", + timeout=int(os.environ.get("WELCOME_TIMEOUT", "180")), + label="welcome-window", + ) + self.results.assert_true( + "welcome-autostarted", welcome is not None, "welcome window did not appear" + ) + self.shot("01-welcome", "welcome window", "observe at startup", + "branded welcome window with a dominant Install action") - -def do_gui(): - child = spawn() - log = open(SERIAL_LOG, "w") - child.logfile = log - try: - login_live(child) - # Give greetd/the session time to bring up Mango + DMS + the welcome - # window (it starts after the shell is ready). - time.sleep(int(os.environ.get("SESSION_SETTLE", "30"))) - for _ in range(12): - child.sendline( - "echo WELCOME=$(pgrep -fc 'quickshell.*phoebus-welcome' 2>/dev/null || echo 0)" - ) - child.expect(r"WELCOME=(\d+)", timeout=60) - if int(child.match.group(1)) > 0: - break - time.sleep(5) - # Payload checks (no secrets here). + # Markers are built with command substitution so the shell's echo of the + # typed command cannot contain them; a match therefore proves output. checks = ( "echo ===GUI-CHECKS-$(printf START)===; " - "test -e /etc/phoebus/live && echo LIVE-MARKER; " - "test -s /usr/share/phoebus/branding/wallpaper.svg && echo BRAND-WALLPAPER; " - "test -s /usr/share/icons/hicolor/scalable/apps/phoebus.svg && echo BRAND-ICON; " - "test -s /etc/xdg/quickshell/phoebus-install/shell.qml && echo INSTALL-QML; " - "test -s /usr/share/applications/phoebus-install.desktop && echo DESKTOP-ENTRY; " - "test -f /home/phoebus/.config/DankMaterialShell/.firstlaunch && echo ONBOARDING-SUPPRESSED; " + "test -e /etc/phoebus/live && echo LIVE-$(printf MARKER); " + "test -s /usr/share/phoebus/branding/wallpaper.svg && echo BRAND-$(printf WALLPAPER); " + "test -s /usr/share/icons/hicolor/scalable/apps/phoebus.svg && echo BRAND-$(printf ICON); " + "test -s /etc/xdg/quickshell/phoebus-install/shell.qml && echo INSTALL-$(printf QML); " + "test -s /usr/share/applications/phoebus-install.desktop && echo DESKTOP-$(printf ENTRY); " + "test -f /home/phoebus/.config/DankMaterialShell/.firstlaunch && echo ONBOARDING-$(printf SUPPRESSED); " "echo ===GUI-CHECKS-$(printf END)===" ) - live_shell(child, checks, "===GUI-CHECKS-END===", timeout=180) - shot = screendump("01-welcome") - print("welcome screenshot:", shot, flush=True) - - set_wayland(child) - # Backend disk protection on the real VM: the live medium must be - # reported protected and never selectable. - child.sendline( - "printf '%s\\n%s\\n' '{\"id\":1,\"op\":\"disks\"}' '{\"id\":2,\"op\":\"quit\"}' " - "| sudo -n /usr/local/bin/phoebus-installer --gui-bridge 2>/dev/null " - "| head -1 > /tmp/disks.json; " - "echo DISKS=$(wc -c /dev/null || echo 0)") - child.expect(r"INST=(\d+)", timeout=60) - if int(child.match.group(1)) == 0: - # The window manager may not have focused the welcome; fall back to - # the launcher so the rest of the journey can still be exercised. - child.sendline( - "setsid /usr/local/bin/phoebus-install >/tmp/install-gui.log 2>&1 & echo LAUNCHED" + payload = self.shell(checks, "===GUI-CHECKS-END===", timeout=180, label="payload-checks") + for marker in ( + "LIVE-MARKER", "BRAND-WALLPAPER", "BRAND-ICON", "INSTALL-QML", "DESKTOP-ENTRY", + ): + self.results.assert_true( + f"payload-{marker.lower()}", marker in payload, f"{marker} not in command output" + ) + if "ONBOARDING-SUPPRESSED" not in payload: + self.results.skip("onboarding-suppressed", "first-run marker not present") + + self.set_wayland() + + # Disk protection is a deterministic backend property. The bridge JSON + # is long and the serial console wraps it, so a small guest-side summary + # is emitted instead of parsing wrapped JSON. Every marker is built with + # command substitution so the echoed command cannot be mistaken for it. + self.shell( + "printf '%s\\n%s\\n' '{\"id\":1,\"op\":\"disks\"}' '{\"id\":2,\"op\":\"quit\"}' " + "> /tmp/req.json; echo REQ-$(printf DONE)", + "REQ-DONE", + timeout=30, + label="disk-probe-request", + ) + summary = self.shell( + "sudo -n /usr/local/bin/phoebus-installer --gui-bridge < /tmp/req.json " + "2>/dev/null | head -1 > /tmp/disks.json; " + "echo DISKS-$(printf COUNT)=$(grep -o '{[^{}]*}' /tmp/disks.json | wc -l); " + "echo LIVE-$(printf COUNT)=$(grep -o '{[^{}]*\"live_medium\":true[^{}]*}' " + "/tmp/disks.json | wc -l); " + "echo LIVE-$(printf PROTECTED)=$(grep -o '{[^{}]*\"live_medium\":true[^{}]*}' " + "/tmp/disks.json | grep -c '\"protected\":true'); " + "echo PROBE-$(printf DONE)", + "PROBE-DONE", + timeout=120, + label="disk-probe", + ) + self._assert_live_medium_protected(summary) + + # Activate the focused welcome action with Return (the real launcher). + # Retry a few times; a transient focus race must not be recorded as a + # passing launcher test, and the fallback launch is recorded as a FAIL. + self.shot("01b-focus", "welcome focused", "before launcher key", + "primary Install action visibly focused") + installed = None + for _attempt in range(3): + self.sendkey("ret") + installed = self.poll_shell( + "echo INST=$(pgrep -fc 'quickshell.*phoebus-install' 2>/dev/null || echo 0)", + r"INST=([1-9][0-9]*)", + timeout=30, + label="launcher-opens-installer", + record=False, + ) + if installed is not None: + break + if installed is None: + # Fallback launch for diagnostics only: the launcher test stays failed. + self.results.fail( + "launcher-opens-installer", + "Return did not open the installer; using a diagnostic fallback launch", + ) + self.shell( + "setsid /usr/local/bin/phoebus-install >/tmp/install-gui.log 2>&1 & echo LAUNCHED", + "LAUNCHED", + timeout=30, + label="fallback-launch", ) - child.expect("LAUNCHED", timeout=30) time.sleep(7) - # Surface the installer frontend's own log (it contains bridge errors). - child.sendline( + else: + self.results.pass_("launcher-opens-installer", "welcome action opened the installer") + + # The frontend log must not contain bridge errors or a crash. + logdump = self.shell( "echo QMLLOG-$(printf START); " "cat /run/user/1000/quickshell/by-id/*/log.qslog 2>/dev/null | " - "grep -aiE 'phoebus-bridge|error|failed|sudo|handshake' | tail -30; " - "echo NOTTY-SUDO; setsid sh -c 'sudo -n true' ; echo NOTTYSUDO=$?; " - "echo QMLLOG-$(printf END)" + "grep -aiE 'phoebus-bridge|error|failed|handshake' | tail -30; " + "echo QMLLOG-$(printf END)", + "QMLLOG-END", + timeout=60, + label="frontend-log", ) - child.expect("QMLLOG-END", timeout=60) - screendump("02-installer-language") - # The first handshake probes hardware and disks; wait for it before - # driving the wizard so the backend is ready to validate. - time.sleep(int(os.environ.get("BACKEND_SETTLE", "30"))) - # Close the welcome so the installer window has the keyboard focus - # (Mango tiles both windows; the welcome would otherwise receive keys). - child.sendline("pkill -f 'quickshell.*phoebus-wel[c]ome' || true; sleep 2; echo CLOSED") - child.expect("CLOSED", timeout=30) + hard = [ln for ln in logdump.splitlines() + if "phoebus-bridge:" in ln and "error" in ln.lower()] + self.results.assert_true("frontend-no-bridge-errors", not hard, "; ".join(hard[:3])) + + self.shot("02-installer-start", "installer wizard", "opened from the welcome", + "language and region step with visible focus") + time.sleep(BACKEND_SETTLE) + + # Close the welcome so the tiled installer holds the keyboard. + self.shell( + "pkill -f 'quickshell.*phoebus-wel[c]ome' || true; sleep 2; echo CLOSED", + "CLOSED", + timeout=30, + label="close-welcome", + ) + # Duplicate prevention: a second launch must not create a second session. - child.sendline( + self.shell( "pgrep -f 'quickshell.*phoebus-install' | wc -l > /tmp/n1; " "/usr/local/bin/phoebus-install >/tmp/install-gui2.log 2>&1; " "pgrep -f 'quickshell.*phoebus-install' | wc -l > /tmp/n2; " - "echo DUP n1=$(cat /tmp/n1) n2=$(cat /tmp/n2)" + "echo DUP n1=$(cat /tmp/n1) n2=$(cat /tmp/n2)", + r"DUP n1=(\d+) n2=(\d+)", + timeout=60, + label="duplicate-launch", ) - child.expect(r"DUP n1=(\d+) n2=(\d+)", timeout=60) - print("duplicate check:", child.match.group(0), flush=True) - - # Move forward through the wizard with the keyboard shortcut. - sendkey("ctrl-right") - time.sleep(2) - screendump("03-network-prerequisites") - sendkey("ctrl-right") - time.sleep(2) - screendump("04-ai-model") - # Advancing without a connected model must be refused (blocked prereq). - sendkey("ctrl-right") - time.sleep(2) - screendump("05-ai-blocked") - print("gui checks complete", flush=True) - finally: - child.close(force=True) - log.close() - - -def do_install(): - child = spawn() - log = open(SERIAL_LOG, "w") - child.logfile = log - try: - login_live(child) - child.sendline( - f"curl -fsSL {SERVE_URL}/appliance-guest-gui.sh -o /tmp/gi.sh && bash /tmp/gi.sh" + n1, n2 = int(self.last_group(1)), int(self.last_group(2)) + self.results.assert_true( + "duplicate-sessions-prevented", n1 == n2 and n1 >= 1, + f"n1={n1} n2={n2}", ) - child.expect([r"===INSTALL-RC=([0-9]+)===", "===FETCH-FAIL===", - "Kernel panic", pexpect.EOF, pexpect.TIMEOUT], - timeout=INSTALL_TIMEOUT) - if "INSTALL-RC" in (child.after or ""): - print("INSTALL_RC", child.match.group(1), flush=True) + + # Keyboard-walk the wizard, capturing each step. + for slug, state in [ + ("03-network", "network and prerequisites"), + ("04-ai-model", "AI model connection"), + ]: + self.sendkey("ctrl-right") + time.sleep(2) + self.shot(slug, state, "Ctrl+Right", "next wizard step") + if not self._installer_alive(): + self.results.fail("installer-survived-navigation", f"crashed at {slug}") + return + # Advancing without a connected model must be refused: the process must + # still be alive and the backend must not have started an install. + self.sendkey("ctrl-right") + time.sleep(2) + self.shot("06-ai-blocked", "AI model blocked", "Ctrl+Right without a model", + "advance refused; no install started") + blocked = self._bridge_has_no_install_start() + self.results.assert_true( + "ai-step-blocks-install", blocked, + "the AI step allowed advancement without a connected model", + ) + self.results.pass_("gui-walk-completed") + + def _installer_alive(self) -> bool: + assert self.child is not None + self.child.sendline( + "echo ALIVE=$(pgrep -fc 'quickshell.*phoebus-install' 2>/dev/null || echo 0)" + ) + try: + self.child.expect(r"ALIVE=(\d+)", timeout=30) + except (pexpect.TIMEOUT, pexpect.EOF): + return False + return int(self.child.match.group(1)) >= 1 + + def _bridge_has_no_install_start(self) -> bool: + # The installer log records storage_prepare only when a backend run + # actually starts; at this point it must not have completed it. The + # marker is built with command substitution so the echoed command cannot + # be mistaken for the result. + assert self.child is not None + self.child.sendline( + "echo NOSTART=$(grep -l 'stage complete: storage_prepare' " + "/run/phoebus-installer/install.log /tmp/phoebus-installer/install.log " + "2>/dev/null | wc -l)" + ) + try: + self.child.expect(r"NOSTART=(\d+)", timeout=30) + except (pexpect.TIMEOUT, pexpect.EOF): + return False + return self.child.match.group(1).strip() == "0" + + @staticmethod + def _first_int(text: str, key: str) -> int | None: + match = re.search(re.escape(key) + r"=(\d+)", text) + return int(match.group(1)) if match else None + + def _assert_live_medium_protected(self, summary: str) -> None: + total = self._first_int(summary, "DISKS-COUNT") + live = self._first_int(summary, "LIVE-COUNT") + protected = self._first_int(summary, "LIVE-PROTECTED") + if total is None or live is None or protected is None: + self.results.fail( + "disks-probe", + f"could not read the disk summary (total={total} live={live} protected={protected})", + ) + return + self.results.assert_true("disks-listed", total >= 1, "no disks reported") + if live >= 1: + self.results.assert_true( + "live-medium-protected", + protected == live, + f"{protected}/{live} live media were protected", + ) else: - print("INSTALL did not report a status", flush=True) - child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600) - child.expect(pexpect.EOF, timeout=300) - finally: - child.close(force=True) - log.close() + self.results.skip("live-medium-protected", "no live medium in the probe") + # -- non-interactive install ------------------------------------------ + def do_install(self) -> None: + if not self.disks: + self.results.blocked("install-disk", "no writable disk configured") + return + self.spawn(attach_iso=True, boot_disk=False) + self.login_live() + self.shell( + f"curl -fsSL {SERVE_URL}/appliance-guest-gui.sh -o /tmp/gi.sh && bash /tmp/gi.sh", + r"===INSTALL-RC=([0-9]+)===", + timeout=INSTALL_TIMEOUT, + label="install-completes", + ) + rc = self.child.match.group(1) if self.child else "?" + self.results.assert_true("install-rc-zero", rc == "0", f"INSTALL-RC={rc}") + # The secret-leak marker is emitted by the guest script. + assert self.child is not None + try: + self.child.expect(r"===NO_SECRET_LEAK===", timeout=120) + self.results.pass_("no-secret-leak") + except (pexpect.TIMEOUT, pexpect.EOF): + self.results.fail("no-secret-leak", "the guest did not report NO_SECRET_LEAK") + try: + self.child.expect(["===POWEROFF===", pexpect.EOF], timeout=600) + except pexpect.TIMEOUT: + self.results.fail("poweroff", "the guest did not power off") + try: + self.child.expect(pexpect.EOF, timeout=300) + except pexpect.TIMEOUT: + self.results.fail("install-eof", "QEMU did not exit after poweroff") -def do_boot(): - # Detached: no ISO, boot from the installed disk only. - child = spawn(attach_iso=False, boot_disk=True) - log = open(SERIAL_LOG, "w") - child.logfile = log - checks = os.environ.get( - "BOOT_CHECKS", - "echo ===CHECKS-$(printf START)===; " - "cat /proc/1/comm; systemctl is-system-running; " - "systemctl --failed --no-pager; ip -brief addr; " - "lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS; echo ===CHECKS-$(printf END)===", - ) - try: - child.expect([r"[Pp]assphrase", r"login:"], timeout=BOOT_TIMEOUT) - if "assphrase" in (child.after or ""): - child.sendline(LUKS_PW) + # -- detached boot ----------------------------------------------------- + def do_boot(self) -> None: + if not self.disks: + self.results.blocked("boot-disk", "no writable disk configured") + return + self.spawn(attach_iso=False, boot_disk=True) + checks = os.environ.get( + "BOOT_CHECKS", + "echo ===CHECKS-$(printf START)===; " + "cat /proc/1/comm; systemctl is-system-running; " + "systemctl --failed --no-pager; ip -brief addr; " + "lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS; echo ===CHECKS-$(printf END)===", + ) + assert self.child is not None + idx = self.expect([r"[Pp]assphrase", r"login:"], timeout=BOOT_TIMEOUT, label="boot-prompt") + if idx == 0: + self.child.sendline(LUKS_PW) + logged_in = False for _ in range(5): - child.expect(r"phoebus login:", timeout=300) - child.sendline(USER) - idx = child.expect([r"[Pp]assword:", r"phoebus login:", r"Login incorrect"], - timeout=60) - if idx == 0: - child.sendline(USER_PW) - if child.expect([r"[$#] $", r"Login incorrect", r"phoebus login:"], - timeout=120) == 0: + try: + self.child.expect(r"phoebus login:", timeout=300) + except (pexpect.TIMEOUT, pexpect.EOF): + break + self.child.sendline(USER) + choice = self.child.expect( + [r"[Pp]assword:", r"phoebus login:", r"Login incorrect"], timeout=60 + ) + if choice == 0: + self.child.sendline(USER_PW) + if self.child.expect([r"[$#] $", r"Login incorrect", r"phoebus login:"], + timeout=120) == 0: + logged_in = True break - child.sendline(checks) - child.expect("===CHECKS-END===", timeout=int(os.environ.get("CHECKS_TIMEOUT", "300"))) - print("checks complete", flush=True) - finally: - child.close(force=True) - log.close() + self.results.assert_true("detached-boot-login", logged_in, "could not log in") + self.shell(checks, "===CHECKS-END===", + timeout=int(os.environ.get("CHECKS_TIMEOUT", "300")), label="boot-checks") + serial = self.paths.serial_log.read_text(errors="replace") + self.results.assert_true( + "boot-systemd-running", "running" in serial, "systemd did not report running" + ) + + +def main() -> int: + mode = sys.argv[1] if len(sys.argv) > 1 else "" + if mode not in ("gui", "install", "boot"): + print(__doc__) + return 2 + if mode in ("install", "boot") and not DISKS: + print("error: DISK/DISKS must name a disposable qcow2 image", file=sys.stderr) + return 2 + try: + return Driver(mode).run() + except HarnessError as exc: + print(f"harness error: {exc}", file=sys.stderr) + return 2 if __name__ == "__main__": - mode = sys.argv[1] if len(sys.argv) > 1 else "" - if mode == "gui": - do_gui() - elif mode == "install": - do_install() - elif mode == "boot": - do_boot() - else: - print(__doc__) - sys.exit(2) + sys.exit(main()) diff --git a/installer/tests/vm/report.py b/installer/tests/vm/report.py new file mode 100644 index 0000000..ad45f6d --- /dev/null +++ b/installer/tests/vm/report.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Build a browsable HTML report for one VM run. + +Combines `results.json` (structured checks) with `screenshot-manifest.json` +(per-capture metadata) into a single page that links each check to its +screenshots. Output goes next to the run directory's artifacts and is meant to +stay outside git. + + python3 installer/tests/vm/report.py /var/tmp/phoebus-vm/ +""" + +from __future__ import annotations + +import html +import json +import sys +from pathlib import Path + +STATUS_CLASS = { + "PASS": "pass", + "FAIL": "fail", + "BLOCKED": "blocked", + "SKIPPED": "skip", + "NOT RUN": "skip", +} + + +def build(run_dir: Path) -> Path: + results = json.loads((run_dir / "results.json").read_text()) + manifest_path = run_dir / "screenshot-manifest.json" + manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else [] + + def esc(value) -> str: + return html.escape(str(value)) + + rows = [] + for check in results.get("checks", []): + cls = STATUS_CLASS.get(check["status"], "skip") + detail = f'{esc(check.get("detail", ""))}' if check.get("detail") else "" + rows.append( + f'{esc(check["status"])}' + f'{esc(check["name"])}{detail}' + ) + + shots = [] + for shot in manifest: + png = shot.get("png") + rel = None + if png: + try: + rel = str(Path(png).relative_to(run_dir)) + except ValueError: + rel = Path(png).name + image = f'' if rel else "no image" + shots.append( + "
" + f'{image}
{esc(shot.get("ui_state"))}
' + f'{esc(shot.get("action"))} → expected: {esc(shot.get("expected"))}; ' + f'observed: {esc(shot.get("observed"))}
' + ) + + counts = results.get("counts", {}) + summary = " · ".join(f"{k}={v}" for k, v in counts.items() if v) + page = f""" + +Phoebus VM report · {esc(results.get('run_id'))} + +

Phoebus VM report · {esc(results.get('run_id'))}

+

scenario: {esc(results.get('scenario'))} · exit_code: {esc(results.get('exit_code'))} · {esc(summary)}

+

Checks

+{''.join(rows) or ''}
no checks
+

Screenshots ({len(shots)})

+ + +""" + out = run_dir / "report.html" + out.write_text(page) + return out + + +def main() -> int: + if len(sys.argv) != 2: + print(__doc__) + return 2 + run_dir = Path(sys.argv[1]) + if not (run_dir / "results.json").exists(): + print(f"no results.json in {run_dir}", file=sys.stderr) + return 1 + print(build(run_dir)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/installer/tests/vm/serial-driver.py b/installer/tests/vm/serial-driver.py index f0ec012..40082e7 100644 --- a/installer/tests/vm/serial-driver.py +++ b/installer/tests/vm/serial-driver.py @@ -11,22 +11,28 @@ Two modes: Everything is parameterised through environment variables so the script is not tied to one host. See tests/vm/README.md for the variables. -This script never touches a physical disk: DISK is always a qcow2 image. +Safety: `DISK`/`DISKS` must be qcow2 files inside the disposable directory named +by `PHOEBUS_VM_DISKS` (default `$PHOEBUS_VM_STATE/disks`); block devices, +`/dev/*`, escaping symlinks, raw images and outside backing chains are refused. +Results are structured and the exit status is nonzero on failure. """ import os import sys + import pexpect +import vmsafety +from vmsafety import Results + QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64") LIBDIR = os.environ.get("QEMU_LIBDIR", "") DATADIR = os.environ.get("QEMU_DATADIR", "") OVMF_CODE = os.environ["OVMF_CODE"] -OVMF_VARS = os.environ["OVMF_VARS"] -DISK = os.environ["DISK"] +OVMF_VARS_TEMPLATE = os.environ["OVMF_VARS"] +DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "") ISO = os.environ.get("ISO", "") KERNEL = os.environ.get("KERNEL", "") INITRD = os.environ.get("INITRD", "") -SERIAL_LOG = os.environ.get("SERIAL_LOG", "/tmp/phoebus-vm-serial.log") LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test") USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test") USER = os.environ.get("USERNAME", "phoebus") @@ -39,7 +45,14 @@ BLS_APPEND = ( ) -def base_args(): +def disposable_dir(): + import pathlib + + base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm") + return pathlib.Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks")) + + +def base_args(ovmf_vars): env = dict(os.environ) if LIBDIR: env["LD_LIBRARY_PATH"] = LIBDIR @@ -49,8 +62,7 @@ def base_args(): args += [ "-cpu", "host", "-smp", SMP, "-m", MEM, "-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}", - "-drive", f"if=pflash,format=raw,file={OVMF_VARS}", - "-drive", f"file={DISK},if=virtio,format=qcow2", + "-drive", f"if=pflash,format=raw,file={ovmf_vars}", "-netdev", "user,id=n0", "-device", "virtio-net-pci,netdev=n0,romfile=", "-device", "virtio-rng-pci", "-vga", "std", "-display", "none", "-serial", "stdio", "-monitor", "none", "-no-reboot", @@ -58,38 +70,52 @@ def base_args(): return env, args -def spawn(args, env): +def spawn(args, env, log): child = pexpect.spawn(QEMU, args, env=env, timeout=600, encoding="utf-8", codec_errors="replace", maxread=65536) child.delaybeforesend = 0.3 + child.logfile = log return child -def do_install(): - env, args = base_args() +def do_install(results, paths, disks): + env, args = base_args(str(paths.ovmf_vars)) + for disk in disks: + args += ["-drive", f"file={disk},if=virtio,format=qcow2"] args += ["-drive", f"file={ISO},media=cdrom,readonly=on", "-kernel", KERNEL, "-initrd", INITRD, "-append", BLS_APPEND] - log = open(SERIAL_LOG, "w") - child = spawn(args, env) - child.logfile = log + log = open(paths.serial_log, "w") + child = spawn(args, env, log) try: child.expect(r"root@livecd[^\n]*# ", timeout=600) + results.pass_("live-shell") url = os.environ.get("INSTALL_URL", "http://10.0.2.2:8000/install.sh") child.sendline(f"curl -fsSL {url} -o /tmp/run.sh && bash /tmp/run.sh") child.expect([r"===INSTALL-RC=([0-9]+)===", "===FETCH-FAIL===", "Kernel panic", pexpect.EOF, pexpect.TIMEOUT], timeout=9000) if "INSTALL-RC" in (child.after or ""): - print("INSTALL_RC", child.match.group(1), flush=True) + rc = child.match.group(1) + results.assert_true("install-rc-zero", rc == "0", f"INSTALL-RC={rc}") + else: + results.fail("install-rc-zero", "install did not report a status") + serial = paths.serial_log.read_text(errors="replace") + results.assert_true("no-secret-leak", "SECRET_LEAK_FOUND" not in serial, + "the guest reported a secret leak") child.expect(["===POWEROFF===", pexpect.EOF, pexpect.TIMEOUT], timeout=600) - child.expect(pexpect.EOF, timeout=300) + except pexpect.TIMEOUT as exc: + results.fail("install", f"timeout: {exc}") + except pexpect.EOF: + results.fail("install", "unexpected EOF") finally: child.close(force=True) log.close() -def do_boot(): - env, args = base_args() +def do_boot(results, paths, disks): + env, args = base_args(str(paths.ovmf_vars)) + for disk in disks: + args += ["-drive", f"file={disk},if=virtio,format=qcow2"] args += ["-boot", "order=c"] # The end marker is built with a command substitution so the shell's echo of # the typed line does not itself contain it; otherwise `expect` would match @@ -101,13 +127,13 @@ def do_boot(): "systemctl --failed --no-pager; ip -brief addr; " "lsblk -o NAME,SIZE,TYPE,FSTYPE,MOUNTPOINTS; echo ===CHECKS-$(printf END)===", ) - log = open(SERIAL_LOG, "w") - child = spawn(args, env) - child.logfile = log + log = open(paths.serial_log, "w") + child = spawn(args, env, log) try: child.expect([r"[Pp]assphrase", r"login:"], timeout=300) if "assphrase" in (child.after or ""): child.sendline(LUKS_PW) + logged_in = False for _ in range(5): child.expect(r"phoebus login:", timeout=180) child.sendline(USER) @@ -117,22 +143,51 @@ def do_boot(): child.sendline(USER_PW) if child.expect([r"[$#] $", r"Login incorrect", r"phoebus login:"], timeout=60) == 0: + logged_in = True break + results.assert_true("detached-boot-login", logged_in, "could not log in") child.sendline(checks) - checks_timeout = int(os.environ.get("CHECKS_TIMEOUT", "180")) - child.expect("===CHECKS-END===", timeout=checks_timeout) - print("checks complete", flush=True) + child.expect("===CHECKS-END===", timeout=int(os.environ.get("CHECKS_TIMEOUT", "180"))) + serial = paths.serial_log.read_text(errors="replace") + results.assert_true("boot-systemd-running", "running" in serial, + "systemd did not report running") + except pexpect.TIMEOUT as exc: + results.fail("boot", f"timeout: {exc}") + except pexpect.EOF: + results.fail("boot", "unexpected EOF") finally: child.close(force=True) log.close() -if __name__ == "__main__": +def main() -> int: mode = sys.argv[1] if len(sys.argv) > 1 else "" - if mode == "install": - do_install() - elif mode == "boot": - do_boot() - else: + if mode not in ("install", "boot"): print(__doc__) - sys.exit(2) + return 2 + results = Results(f"serial-{mode}") + paths = vmsafety.make_run_paths( + f"serial-{mode}", base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"), + ovmf_vars_template=OVMF_VARS_TEMPLATE, + ) + try: + disks = vmsafety.validate_writable_drives(DISKS.split(","), disposable_dir()) + except vmsafety.UnsafeDiskError as exc: + results.blocked("disposable-disk", str(exc)) + results.write(paths.results_json) + print(f"harness error: {exc}", file=sys.stderr) + return 2 + results.pass_("disposable-disk", ", ".join(str(d) for d in disks)) + try: + if mode == "install": + do_install(results, paths, disks) + else: + do_boot(results, paths, disks) + finally: + results.write(paths.results_json) + print(results.summary(), flush=True) + return results.exit_code() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/installer/tests/vm/test_vmsafety.py b/installer/tests/vm/test_vmsafety.py new file mode 100644 index 0000000..6c8b8c4 --- /dev/null +++ b/installer/tests/vm/test_vmsafety.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Unit tests for the VM harness safety and result logic. + +These run on any host (no VM, no QEMU required) so the safety rules are checked +in CI, not only during a live install: + + python3 installer/tests/vm/test_vmsafety.py +""" + +from __future__ import annotations + +import os +import socket +import subprocess +import tempfile +import threading +import unittest +from pathlib import Path + +import vmsafety +from vmsafety import ( + BLOCKED, + FAIL, + PASS, + Results, + UnsafeDiskError, + validate_disposable_disk, + validate_writable_drives, +) + + +def make_qcow2(path: Path, size: str = "8M") -> Path: + subprocess.run( + ["qemu-img", "create", "-f", "qcow2", str(path), size], + check=True, + capture_output=True, + ) + return path + + +class DisposableDiskTests(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.dir = Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + def test_rejects_dev_path(self): + with self.assertRaises(UnsafeDiskError): + validate_disposable_disk("/dev/sda", self.dir) + + def test_rejects_block_device(self): + # /dev/loop-control or /dev/null are char/regular; the explicit /dev + # check fires first. Use a char device outside /dev to isolate. + with self.assertRaises(UnsafeDiskError): + validate_disposable_disk("/dev/null", self.dir) + + def test_rejects_non_qcow2_regular_file(self): + raw = self.dir / "raw.img" + raw.write_bytes(b"\x00" * 4096) + with self.assertRaises(UnsafeDiskError): + validate_disposable_disk(raw, self.dir) + + def test_rejects_qcow2_outside_test_dir(self): + outside = self.dir.parent / "outside-disposable.qcow2" + make_qcow2(outside) + self.addCleanup(outside.unlink) + with self.assertRaises(UnsafeDiskError): + validate_disposable_disk(outside, self.dir) + + def test_rejects_symlink_escaping_test_dir(self): + target = self.dir.parent / "real-disk.qcow2" + make_qcow2(target) + self.addCleanup(target.unlink) + link = self.dir / "link.qcow2" + link.symlink_to(target) + with self.assertRaises(UnsafeDiskError): + validate_disposable_disk(link, self.dir) + + def test_accepts_qcow2_inside_test_dir(self): + good = make_qcow2(self.dir / "good.qcow2") + self.assertEqual(validate_disposable_disk(good, self.dir), good) + + def test_rejects_backing_chain_outside_test_dir(self): + backing = self.dir.parent / "backing.qcow2" + make_qcow2(backing) + self.addCleanup(backing.unlink) + overlay = self.dir / "overlay.qcow2" + subprocess.run( + ["qemu-img", "create", "-f", "qcow2", "-b", str(backing), "-F", "qcow2", + str(overlay)], + check=True, + capture_output=True, + ) + with self.assertRaises(UnsafeDiskError): + validate_disposable_disk(overlay, self.dir) + + def test_accepts_backing_chain_inside_test_dir(self): + backing = make_qcow2(self.dir / "backing.qcow2") + overlay = self.dir / "overlay.qcow2" + subprocess.run( + ["qemu-img", "create", "-f", "qcow2", "-b", str(backing), "-F", "qcow2", + str(overlay)], + check=True, + capture_output=True, + ) + self.assertEqual(validate_disposable_disk(overlay, self.dir), overlay) + + def test_rejects_root_as_test_dir(self): + good = make_qcow2(self.dir / "good.qcow2") + with self.assertRaises(UnsafeDiskError): + validate_disposable_disk(good, "/") + + def test_validate_all_drives_not_just_disk(self): + a = make_qcow2(self.dir / "a.qcow2") + b = make_qcow2(self.dir / "b.qcow2") + checked = validate_writable_drives([a, "", None, b], self.dir) + self.assertEqual(checked, [a, b]) + with self.assertRaises(UnsafeDiskError): + validate_writable_drives([a, "/dev/sdb"], self.dir) + + +class ResultsTests(unittest.TestCase): + def test_fail_sets_nonzero_exit(self): + results = Results("unit") + results.assert_true("ok", True) + results.assert_true("bad", False, "boom") + self.assertTrue(results.failed) + self.assertEqual(results.exit_code(), 1) + + def test_all_pass_is_zero(self): + results = Results("unit") + results.assert_true("ok", True) + self.assertEqual(results.exit_code(), 0) + + def test_blocked_only_fails_when_strict(self): + results = Results("unit", strict=False) + results.blocked("needs hardware", "no GPU") + self.assertEqual(results.exit_code(), 0) + strict = Results("unit", strict=True) + strict.blocked("needs hardware", "no GPU") + self.assertEqual(strict.exit_code(), 2) + + def test_no_checks_is_not_run(self): + results = Results("unit") + self.assertEqual(results.counts()[vmsafety.NOT_RUN], 0) + self.assertEqual(results.exit_code(), 0) + self.assertEqual(results.summary().split(": ")[1], "no checks") + + def test_write_round_trips(self): + results = Results("unit") + results.pass_("one") + path = results.write(self._tmp_results()) + import json + + data = json.loads(path.read_text()) + self.assertEqual(data["scenario"], "unit") + self.assertEqual(data["counts"][PASS], 1) + self.assertEqual(data["exit_code"], 0) + + def _tmp_results(self): + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json") + tmp.close() + self.addCleanup(os.unlink, tmp.name) + return Path(tmp.name) + + +class FakeQmpServer(threading.Thread): + """A tiny QMP server that records HMP commands and answers them.""" + + def __init__(self): + super().__init__(daemon=True) + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.path = tempfile.mktemp(prefix="fake-qmp-", suffix=".sock") + self.sock.bind(self.path) + self.sock.listen(1) + self.commands = [] + self.screendump_path = None + + def run(self): + conn, _ = self.sock.accept() + with conn: + conn.sendall(b'{"QMP": {"version": {"qemu": {"major": 8}}}}' + b"\r\n") + reader = conn.makefile("rb") + writer = conn.makefile("wb") + for line in reader: + message = __import__("json").loads(line) + if message.get("execute") == "qmp_capabilities": + writer.write(b'{"return": {}}\r\n') + elif message.get("execute") == "human-monitor-command": + cmd = message["arguments"]["command-line"] + self.commands.append(cmd) + if cmd.startswith("screendump"): + # Emulate QEMU writing the file. + name = cmd.split(" ", 1)[1].strip().strip('"') + Path(name).write_bytes(b"P6 fake-ppm") + writer.write(b'{"return": ""}\r\n') + else: + writer.write(b'{"return": {}}\r\n') + writer.flush() + + +class QmpClientTests(unittest.TestCase): + def test_hmp_and_screendump(self): + server = FakeQmpServer() + server.start() + self.addCleanup(lambda: (server.sock.close(), os.path.exists(server.path) and os.unlink(server.path))) + deadline = 50 + while not os.path.exists(server.path) and deadline: + deadline -= 1 + with vmsafety.QmpClient(server.path) as client: + client.sendkey("ret") + # Wait until the fake server has created the file before returning. + target = Path(tempfile.mkdtemp()) / "shot with spaces.ppm" + client.hmp(f'screendump "{target}"') + # Give the server thread a moment to record the command. + for _ in range(50): + if len(server.commands) >= 2: + break + __import__("time").sleep(0.05) + self.assertTrue(any(cmd.startswith("sendkey ret") for cmd in server.commands)) + self.assertTrue(any(cmd.startswith("screendump") for cmd in server.commands)) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/installer/tests/vm/vmsafety.py b/installer/tests/vm/vmsafety.py new file mode 100644 index 0000000..beb8e8d --- /dev/null +++ b/installer/tests/vm/vmsafety.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python3 +"""Safety, structured-result and QEMU-control helpers for the VM harnesses. + +The VM drivers are test equipment: their own safety rules must be enforced, not +merely documented. This module centralises those rules so every driver shares +one implementation: + +* every writable disk must be a regular **qcow2** file inside an explicitly + designated disposable directory, with a backing chain that is fully inside + that directory. Block devices, ``/dev/*`` paths, character devices, unsafe + symlinks and writable storage outside the test area are rejected; +* each run gets its own writable OVMF variable file and monitor socket; +* screenshots and monitor commands are issued without a shell, with argument + arrays, over structured QMP where possible; +* results are structured (PASS/FAIL/BLOCKED/SKIPPED/NOT RUN) and a failed, + timed-out or incomplete run exits nonzero. + +Nothing here depends on pexpect, so the safety logic is unit-testable on any +host (see ``test_vmsafety.py``). +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import shutil +import socket +import stat +import subprocess +import sys +import time +import uuid +from pathlib import Path + +# --------------------------------------------------------------------------- +# Statuses +# --------------------------------------------------------------------------- + +PASS = "PASS" +FAIL = "FAIL" +BLOCKED = "BLOCKED" +SKIPPED = "SKIPPED" +NOT_RUN = "NOT RUN" + +_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED) + + +class UnsafeDiskError(Exception): + """A writable target violated the disposable-disk rules.""" + + +class HarnessError(Exception): + """A harness precondition failed (bad environment, unsafe path, ...).""" + + +# --------------------------------------------------------------------------- +# Structured results +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class Check: + name: str + status: str + detail: str = "" + + def to_dict(self) -> dict: + return {"name": self.name, "status": self.status, "detail": self.detail} + + +class Results: + """A run's assertions and their outcome. + + A check may be recorded once. ``assert_true`` records PASS or FAIL; the + helpers ``blocked``/``skip`` record those explicit states. Any FAIL (and, + when strict, any BLOCKED or NOT RUN) makes :meth:`exit_code` nonzero. + """ + + def __init__(self, scenario: str, run_id: str | None = None, strict: bool | None = None): + self.scenario = scenario + self.run_id = run_id or new_run_id(scenario) + self.checks: list[Check] = [] + if strict is None: + strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true") + self.strict = bool(strict) + + # -- recording --------------------------------------------------------- + def _add(self, name: str, status: str, detail: str) -> Check: + if status not in _TERMINAL: + raise ValueError(f"invalid result status: {status}") + check = Check(name=name, status=status, detail=detail) + self.checks.append(check) + stream = sys.stderr if status == FAIL else sys.stdout + print(f"[{status:<7}] {name}" + (f": {detail}" if detail else ""), file=stream, flush=True) + return check + + def assert_true(self, name: str, condition: bool, detail: str = "") -> bool: + ok = bool(condition) + # `detail` describes the failure; never print it on a pass, which would + # make a passing check read as if it had failed. + self._add(name, PASS if ok else FAIL, "" if ok else detail) + return ok + + def fail(self, name: str, detail: str = "") -> Check: + return self._add(name, FAIL, detail) + + def pass_(self, name: str, detail: str = "") -> Check: + return self._add(name, PASS, detail) + + def blocked(self, name: str, detail: str = "") -> Check: + return self._add(name, BLOCKED, detail) + + def skip(self, name: str, detail: str = "") -> Check: + return self._add(name, SKIPPED, detail) + + # -- outcome ----------------------------------------------------------- + @property + def failed(self) -> bool: + return any(c.status == FAIL for c in self.checks) + + @property + def has_blockers(self) -> bool: + return any(c.status == BLOCKED for c in self.checks) + + @property + def not_run(self) -> bool: + return not self.checks or any(c.status == NOT_RUN for c in self.checks) + + def counts(self) -> dict: + out = {s: 0 for s in (PASS, FAIL, BLOCKED, SKIPPED, NOT_RUN)} + for c in self.checks: + out[c.status] += 1 + return out + + def exit_code(self) -> int: + if self.failed: + return 1 + if self.strict and (self.has_blockers or self.not_run): + return 2 + return 0 + + def summary(self) -> str: + counts = self.counts() + parts = [f"{k}={v}" for k, v in counts.items() if v] + return f"{self.scenario} [{self.run_id}]: " + (", ".join(parts) if parts else "no checks") + + def write(self, path: str | Path) -> Path: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "scenario": self.scenario, + "run_id": self.run_id, + "strict": self.strict, + "exit_code": self.exit_code(), + "counts": self.counts(), + "checks": [c.to_dict() for c in self.checks], + } + path.write_text(json.dumps(payload, indent=2) + "\n") + return path + + +def new_run_id(scenario: str) -> str: + stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + return f"{scenario}-{stamp}-{uuid.uuid4().hex[:8]}" + + +# --------------------------------------------------------------------------- +# Disposable disk rules +# --------------------------------------------------------------------------- + + +def _require_regular_and_safe(path: Path, test_dir: Path) -> os.stat_result: + text = str(path) + if text.startswith("/dev/") or text == "/dev": + raise UnsafeDiskError(f"refusing a /dev path: {text}") + # Symlink chains must stay inside the disposable directory when resolved. + real = Path(os.path.realpath(path)) + if not _is_within(real, test_dir): + raise UnsafeDiskError( + f"writable disk resolves outside the disposable test directory " + f"({real} not under {test_dir})" + ) + st = os.stat(path) # follows symlinks; realpath already checked containment + if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode): + raise UnsafeDiskError(f"refusing a block/character device: {text}") + if not stat.S_ISREG(st.st_mode): + raise UnsafeDiskError(f"refusing a non-regular file: {text}") + return st + + +def _is_within(child: Path, parent: Path) -> bool: + try: + child.relative_to(parent) + return True + except ValueError: + return False + + +def test_dir_is_safe(test_dir: Path) -> Path: + real = Path(os.path.realpath(test_dir)) + if real == Path("/"): + raise UnsafeDiskError("the disposable test directory may not be /") + for bad in ("/dev", "/proc", "/sys"): + if real == Path(bad) or _is_within(real, Path(bad)): + raise UnsafeDiskError(f"the disposable test directory may not be under {bad}") + return real + + +def qcow2_magic(path: Path) -> bool: + try: + with open(path, "rb") as handle: + return handle.read(4) == b"QFI\xfb" + except OSError: + return False + + +def _qemu_img_backing_chain(path: Path) -> list[str]: + """Return the backing-file paths reported by qemu-img (empty if none).""" + qemu_img = shutil.which("qemu-img") + if not qemu_img: + return [] + proc = subprocess.run( + [qemu_img, "info", "--backing-chain", "--output=json", str(path)], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + return [] + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError: + return [] + entries = data if isinstance(data, list) else [data] + chain: list[str] = [] + for entry in entries: + for key in ("backing-filename", "full-backing-filename"): + value = entry.get(key) + if value: + chain.append(value) + return chain + + +def validate_disposable_disk( + path: str | os.PathLike, + test_dir: str | os.PathLike, + *, + require_qcow2: bool = True, + allow_missing: bool = False, + qemu_img: bool = True, +) -> Path: + """Validate that ``path`` is a disposable qcow2 inside ``test_dir``. + + Raises :class:`UnsafeDiskError` on any violation. On success returns the + resolved path. + """ + if path is None or str(path).strip() == "": + raise UnsafeDiskError("no writable disk path was provided") + test_real = test_dir_is_safe(Path(test_dir)) + disk = Path(path) + + if not disk.exists(): + if allow_missing: + if not _is_within(Path(os.path.realpath(disk.parent)), test_real): + raise UnsafeDiskError( + f"new disk would be created outside the test directory: {disk}" + ) + return disk + raise UnsafeDiskError(f"writable disk does not exist: {disk}") + + st = _require_regular_and_safe(disk, test_real) + if st.st_size == 0: + raise UnsafeDiskError(f"writable disk is empty: {disk}") + if require_qcow2 and not qcow2_magic(disk): + raise UnsafeDiskError( + f"writable disk is not a qcow2 image (bad magic): {disk}; " + "refusing to hand a raw image to QEMU as writable" + ) + if qemu_img: + for backing in _qemu_img_backing_chain(disk): + backing_real = Path(os.path.realpath(backing)) + if not _is_within(backing_real, test_real): + raise UnsafeDiskError( + f"disk backing file is outside the test directory: {backing}" + ) + if not os.access(disk, os.W_OK): + raise UnsafeDiskError(f"writable disk is not writable: {disk}") + return disk + + +def validate_writable_drives(paths, test_dir, *, qemu_img=True) -> list[Path]: + """Validate every writable drive, not just one variable named DISK.""" + checked = [] + for path in paths: + if path is None or str(path).strip() == "": + continue + checked.append(validate_disposable_disk(path, test_dir, qemu_img=qemu_img)) + if not checked: + raise UnsafeDiskError("no writable drives were provided to validate") + return checked + + +# --------------------------------------------------------------------------- +# Run-scoped state +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class RunPaths: + run_id: str + directory: Path + serial_log: Path + monitor_sock: Path + shot_dir: Path + ovmf_vars: Path + results_json: Path + + def ensure(self) -> "RunPaths": + self.directory.mkdir(parents=True, exist_ok=True) + os.chmod(self.directory, 0o700) + self.shot_dir.mkdir(parents=True, exist_ok=True) + os.chmod(self.shot_dir, 0o700) + return self + + +def make_run_paths( + scenario: str, + *, + base: str | os.PathLike | None = None, + ovmf_vars_template: str | os.PathLike | None = None, + run_id: str | None = None, +) -> RunPaths: + """Create a private per-run directory and a fresh writable OVMF var file. + + ``base`` defaults to ``$PHOEBUS_VM_STATE`` or ``/var/tmp/phoebus-vm``. + """ + base_dir = Path(base or os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")) + base_dir.mkdir(parents=True, exist_ok=True) + os.chmod(base_dir, 0o700) + run = new_run_id(scenario) if run_id is None else run_id + directory = base_dir / run + paths = RunPaths( + run_id=run, + directory=directory, + serial_log=directory / "serial.log", + monitor_sock=directory / "monitor.sock", + shot_dir=directory / "shots", + ovmf_vars=directory / "OVMF_VARS.fd", + results_json=directory / "results.json", + ).ensure() + if ovmf_vars_template: + shutil.copyfile(ovmf_vars_template, paths.ovmf_vars) + os.chmod(paths.ovmf_vars, 0o600) + return paths + + +# --------------------------------------------------------------------------- +# Screenshots (no shell) +# --------------------------------------------------------------------------- + + +def _reject_control_chars(value: str) -> str: + if "\n" in value or "\r" in value or "\x00" in value: + raise HarnessError("path contains a control character") + return value + + +def convert_ppm(ppm: str | os.PathLike, png: str | os.PathLike) -> Path | None: + """Convert a PPM to PNG with an argument array (never a shell).""" + png = Path(_reject_control_chars(str(png))) + for program in ("magick", "convert"): + binary = shutil.which(program) + if not binary: + continue + proc = subprocess.run([binary, str(ppm), str(png)], capture_output=True) + if proc.returncode == 0 and png.exists(): + return png + return None + + +def screenshot_name(seq: int, slug: str) -> str: + safe = "".join(c if (c.isalnum() or c in "-_") else "-" for c in slug).strip("-") + return f"{seq:02d}-{safe or 'shot'}" + + +# --------------------------------------------------------------------------- +# QMP client +# --------------------------------------------------------------------------- + + +class QmpError(Exception): + pass + + +class QmpClient: + """A minimal structured QMP client over a UNIX socket. + + Uses JSON framing (the monitor protocol deletes shell/quoting concerns) and + issues HMP commands through ``human-monitor-command`` for ``screendump`` and + ``sendkey``. + """ + + def __init__(self, sock_path: str | os.PathLike, timeout: float = 20.0): + self.sock_path = str(sock_path) + self.timeout = timeout + + def __enter__(self) -> "QmpClient": + if not os.path.exists(self.sock_path): + raise QmpError(f"monitor socket does not exist: {self.sock_path}") + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(self.timeout) + self.sock.connect(self.sock_path) + self._file = self.sock.makefile("rwb") + greeting = self._read_json() + if "QMP" not in greeting: + raise QmpError("server did not send a QMP greeting") + self._command({"execute": "qmp_capabilities"}) + return self + + def __exit__(self, *_exc) -> None: + try: + self._file.close() + finally: + self.sock.close() + + def _read_json(self) -> dict: + line = self._file.readline() + if not line: + raise QmpError("unexpected EOF from the QMP monitor") + try: + return json.loads(line) + except json.JSONDecodeError as exc: + raise QmpError(f"invalid QMP message: {line[:200]!r}") from exc + + def _command(self, command: dict) -> dict: + self._file.write(json.dumps(command).encode() + b"\n") + self._file.flush() + while True: + message = self._read_json() + if "event" in message: + continue + if "error" in message: + raise QmpError(str(message["error"])) + return message + + def hmp(self, command: str) -> str: + reply = self._command({"execute": "human-monitor-command", "arguments": {"command-line": command}}) + return reply.get("return", "") + + def screendump(self, target: str | os.PathLike) -> Path: + target = Path(_reject_control_chars(str(target))) + target.parent.mkdir(parents=True, exist_ok=True) + # QEMU's HMP screendump takes a single filename argument, so a path with + # spaces is quoted for QEMU's own tiny parser, not for a shell. + quoted = '"' + str(target).replace("\\", "\\\\").replace('"', '\\"') + '"' + self.hmp(f"screendump {quoted}") + deadline = time.monotonic() + self.timeout + while time.monotonic() < deadline: + if target.exists() and target.stat().st_size > 0: + return target + time.sleep(0.2) + raise QmpError(f"screendump did not produce {target}") + + def sendkey(self, key: str) -> None: + self.hmp(f"sendkey {key}") + + +def screendump(monitor_sock, target, *, convert: bool = True) -> Path | None: + """Capture the framebuffer and optionally convert it to PNG. + + Returns the PPM path, or ``None`` if capture failed. Never raises for a + missing PNG converter (the PPM is still evidence). + """ + target = Path(_reject_control_chars(str(target))) + try: + with QmpClient(monitor_sock) as client: + ppm = client.screendump(target) + except (QmpError, OSError): + return None + if convert: + convert_ppm(ppm, ppm.with_suffix(".png")) + return ppm + + +def wait_for_file(path: str | os.PathLike, timeout: float, interval: float = 0.5) -> bool: + deadline = time.monotonic() + timeout + p = Path(path) + while time.monotonic() < deadline: + if p.exists(): + return True + time.sleep(interval) + return False