Validate every writable drive (qcow2, inside PHOEBUS_VM_DISKS, no /dev/*, block/char devices, raw images, escaping symlinks or outside backing chains), give each run its own OVMF variable file, QMP socket and state directory, and use QMP plus argument arrays instead of shell interpolation. Return nonzero on any failed assertion, timeout, EOF or missing result and record PASS/FAIL/BLOCKED/SKIPPED/NOT RUN. Add 16 unit tests, a report generator and a findings register; make test runs the harness suite.
108 lines
3.8 KiB
Bash
108 lines
3.8 KiB
Bash
#!/usr/bin/env python3
|
|
"""Build a browsable HTML report for one VM run.
|
|
|
|
Combines `results.json` (structured checks) with `screenshot-manifest.json`
|
|
(per-capture metadata) into a single page that links each check to its
|
|
screenshots. Output goes next to the run directory's artifacts and is meant to
|
|
stay outside git.
|
|
|
|
python3 installer/tests/vm/report.py /var/tmp/phoebus-vm/<run-id>
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
STATUS_CLASS = {
|
|
"PASS": "pass",
|
|
"FAIL": "fail",
|
|
"BLOCKED": "blocked",
|
|
"SKIPPED": "skip",
|
|
"NOT RUN": "skip",
|
|
}
|
|
|
|
|
|
def build(run_dir: Path) -> Path:
|
|
results = json.loads((run_dir / "results.json").read_text())
|
|
manifest_path = run_dir / "screenshot-manifest.json"
|
|
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else []
|
|
|
|
def esc(value) -> str:
|
|
return html.escape(str(value))
|
|
|
|
rows = []
|
|
for check in results.get("checks", []):
|
|
cls = STATUS_CLASS.get(check["status"], "skip")
|
|
detail = f'<span class="detail">{esc(check.get("detail", ""))}</span>' if check.get("detail") else ""
|
|
rows.append(
|
|
f'<tr class="{cls}"><td class="status">{esc(check["status"])}</td>'
|
|
f'<td>{esc(check["name"])}</td><td>{detail}</td></tr>'
|
|
)
|
|
|
|
shots = []
|
|
for shot in manifest:
|
|
png = shot.get("png")
|
|
rel = None
|
|
if png:
|
|
try:
|
|
rel = str(Path(png).relative_to(run_dir))
|
|
except ValueError:
|
|
rel = Path(png).name
|
|
image = f'<a href="{esc(rel)}"><img src="{esc(rel)}" loading="lazy"></a>' if rel else "<em>no image</em>"
|
|
shots.append(
|
|
"<figure>"
|
|
f'{image}<figcaption><b>{esc(shot.get("ui_state"))}</b><br>'
|
|
f'<small>{esc(shot.get("action"))} → expected: {esc(shot.get("expected"))}; '
|
|
f'observed: {esc(shot.get("observed"))}</small></figcaption></figure>'
|
|
)
|
|
|
|
counts = results.get("counts", {})
|
|
summary = " · ".join(f"{k}={v}" for k, v in counts.items() if v)
|
|
page = f"""<!doctype html>
|
|
<html lang="en"><head><meta charset="utf-8">
|
|
<title>Phoebus VM report · {esc(results.get('run_id'))}</title>
|
|
<style>
|
|
:root {{ color-scheme: dark; }}
|
|
body {{ background:#17171b; color:#f4efe6; font:14px/1.5 system-ui,sans-serif; margin:0; padding:24px; }}
|
|
h1 {{ font-size:20px; }} h2 {{ font-size:16px; margin-top:28px; color:#e8a33d; }}
|
|
table {{ border-collapse:collapse; width:100%; margin-top:8px; }}
|
|
td {{ padding:6px 10px; border-bottom:1px solid #3a3a42; vertical-align:top; }}
|
|
.status {{ font-weight:700; }}
|
|
tr.pass .status {{ color:#7fb069; }} tr.fail .status {{ color:#d2564b; }}
|
|
tr.blocked .status {{ color:#e0a24a; }} tr.skip .status {{ color:#a8a29a; }}
|
|
.detail {{ color:#a8a29a; }} .meta {{ color:#a8a29a; }}
|
|
.gallery {{ display:flex; flex-wrap:wrap; gap:16px; margin-top:12px; }}
|
|
figure {{ margin:0; width:420px; background:#232329; border:1px solid #3a3a42; border-radius:10px; overflow:hidden; }}
|
|
figure img {{ width:100%; display:block; }}
|
|
figcaption {{ padding:8px 10px; }}
|
|
</style></head><body>
|
|
<h1>Phoebus VM report · {esc(results.get('run_id'))}</h1>
|
|
<p class="meta">scenario: {esc(results.get('scenario'))} · exit_code: {esc(results.get('exit_code'))} · {esc(summary)}</p>
|
|
<h2>Checks</h2>
|
|
<table>{''.join(rows) or '<tr><td>no checks</td></tr>'}</table>
|
|
<h2>Screenshots ({len(shots)})</h2>
|
|
<div class="gallery">{''.join(shots)}</div>
|
|
</body></html>
|
|
"""
|
|
out = run_dir / "report.html"
|
|
out.write_text(page)
|
|
return out
|
|
|
|
|
|
def main() -> int:
|
|
if len(sys.argv) != 2:
|
|
print(__doc__)
|
|
return 2
|
|
run_dir = Path(sys.argv[1])
|
|
if not (run_dir / "results.json").exists():
|
|
print(f"no results.json in {run_dir}", file=sys.stderr)
|
|
return 1
|
|
print(build(run_dir))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|