phoebus_os/installer/tests/vm/report.py
Phoebus cc23df69de test(vm): enforce harness disk safety and structured results
Validate every writable drive (qcow2, inside PHOEBUS_VM_DISKS, no /dev/*,
block/char devices, raw images, escaping symlinks or outside backing
chains), give each run its own OVMF variable file, QMP socket and state
directory, and use QMP plus argument arrays instead of shell interpolation.
Return nonzero on any failed assertion, timeout, EOF or missing result and
record PASS/FAIL/BLOCKED/SKIPPED/NOT RUN. Add 16 unit tests, a report
generator and a findings register; make test runs the harness suite.
2026-09-21 20:01:50 -07:00

108 lines
3.8 KiB
Bash

#!/usr/bin/env python3
"""Build a browsable HTML report for one VM run.
Combines `results.json` (structured checks) with `screenshot-manifest.json`
(per-capture metadata) into a single page that links each check to its
screenshots. Output goes next to the run directory's artifacts and is meant to
stay outside git.
python3 installer/tests/vm/report.py /var/tmp/phoebus-vm/<run-id>
"""
from __future__ import annotations
import html
import json
import sys
from pathlib import Path
STATUS_CLASS = {
"PASS": "pass",
"FAIL": "fail",
"BLOCKED": "blocked",
"SKIPPED": "skip",
"NOT RUN": "skip",
}
def build(run_dir: Path) -> Path:
results = json.loads((run_dir / "results.json").read_text())
manifest_path = run_dir / "screenshot-manifest.json"
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else []
def esc(value) -> str:
return html.escape(str(value))
rows = []
for check in results.get("checks", []):
cls = STATUS_CLASS.get(check["status"], "skip")
detail = f'<span class="detail">{esc(check.get("detail", ""))}</span>' if check.get("detail") else ""
rows.append(
f'<tr class="{cls}"><td class="status">{esc(check["status"])}</td>'
f'<td>{esc(check["name"])}</td><td>{detail}</td></tr>'
)
shots = []
for shot in manifest:
png = shot.get("png")
rel = None
if png:
try:
rel = str(Path(png).relative_to(run_dir))
except ValueError:
rel = Path(png).name
image = f'<a href="{esc(rel)}"><img src="{esc(rel)}" loading="lazy"></a>' if rel else "<em>no image</em>"
shots.append(
"<figure>"
f'{image}<figcaption><b>{esc(shot.get("ui_state"))}</b><br>'
f'<small>{esc(shot.get("action"))} → expected: {esc(shot.get("expected"))}; '
f'observed: {esc(shot.get("observed"))}</small></figcaption></figure>'
)
counts = results.get("counts", {})
summary = " · ".join(f"{k}={v}" for k, v in counts.items() if v)
page = f"""<!doctype html>
<html lang="en"><head><meta charset="utf-8">
<title>Phoebus VM report · {esc(results.get('run_id'))}</title>
<style>
:root {{ color-scheme: dark; }}
body {{ background:#17171b; color:#f4efe6; font:14px/1.5 system-ui,sans-serif; margin:0; padding:24px; }}
h1 {{ font-size:20px; }} h2 {{ font-size:16px; margin-top:28px; color:#e8a33d; }}
table {{ border-collapse:collapse; width:100%; margin-top:8px; }}
td {{ padding:6px 10px; border-bottom:1px solid #3a3a42; vertical-align:top; }}
.status {{ font-weight:700; }}
tr.pass .status {{ color:#7fb069; }} tr.fail .status {{ color:#d2564b; }}
tr.blocked .status {{ color:#e0a24a; }} tr.skip .status {{ color:#a8a29a; }}
.detail {{ color:#a8a29a; }} .meta {{ color:#a8a29a; }}
.gallery {{ display:flex; flex-wrap:wrap; gap:16px; margin-top:12px; }}
figure {{ margin:0; width:420px; background:#232329; border:1px solid #3a3a42; border-radius:10px; overflow:hidden; }}
figure img {{ width:100%; display:block; }}
figcaption {{ padding:8px 10px; }}
</style></head><body>
<h1>Phoebus VM report · {esc(results.get('run_id'))}</h1>
<p class="meta">scenario: {esc(results.get('scenario'))} · exit_code: {esc(results.get('exit_code'))} · {esc(summary)}</p>
<h2>Checks</h2>
<table>{''.join(rows) or '<tr><td>no checks</td></tr>'}</table>
<h2>Screenshots ({len(shots)})</h2>
<div class="gallery">{''.join(shots)}</div>
</body></html>
"""
out = run_dir / "report.html"
out.write_text(page)
return out
def main() -> int:
if len(sys.argv) != 2:
print(__doc__)
return 2
run_dir = Path(sys.argv[1])
if not (run_dir / "results.json").exists():
print(f"no results.json in {run_dir}", file=sys.stderr)
return 1
print(build(run_dir))
return 0
if __name__ == "__main__":
sys.exit(main())