Make a passing CI/release result mean the checks actually ran. - .forgejo/workflows/ci.yml is fail-closed: it provisions the pinned toolchain or fails, then runs one 'make ci' (fmt-check, clippy, workspace tests, VM-harness safety tests, release build, static installer build). - Pin stable Rust 1.93.1 in installer-rs/rust-toolchain.toml; run cargo gates with --locked for reproducible dependencies. - phoebus-release gates now records passed/failed/blocked/skipped/not_run (only passed is a pass), executes real host gate commands, and imports the existing VM-harness results.json instead of a competing harness. Gates with no automation are always blocked. - Bind gate reports and harness evidence to a candidate name+SHA-256 (--candidate; PHOEBUS_VM_ARTIFACT[_SHA256]); stale or unbound evidence is failed, never passed. - publish refuses unless the report is candidate-bound and all-passed, the manifest verifies the artifact and a detached signature verifies; atomic publish preserves and restores the previous release on failure. - vmsafety.Results records the verified artifact digest; 3 new Python tests. - 20 new Rust regression tests; ADR 0012 and docs updated. Verified locally: make ci (364 Rust tests, 19 harness tests, release and static builds) exit 0; static binary is static-pie linked. CI runner, VM and model gates remain unexecuted here.
278 lines
10 KiB
Bash
278 lines
10 KiB
Bash
#!/usr/bin/env python3
|
|
"""Unit tests for the VM harness safety and result logic.
|
|
|
|
These run on any host (no VM, no QEMU required) so the safety rules are checked
|
|
in CI, not only during a live install:
|
|
|
|
python3 installer/tests/vm/test_vmsafety.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
import vmsafety
|
|
from vmsafety import (
|
|
ARTIFACT_ENV_NAME,
|
|
ARTIFACT_ENV_SHA256,
|
|
BLOCKED,
|
|
FAIL,
|
|
PASS,
|
|
Results,
|
|
UnsafeDiskError,
|
|
validate_disposable_disk,
|
|
validate_writable_drives,
|
|
)
|
|
|
|
|
|
def make_qcow2(path: Path, size: str = "8M") -> Path:
|
|
subprocess.run(
|
|
["qemu-img", "create", "-f", "qcow2", str(path), size],
|
|
check=True,
|
|
capture_output=True,
|
|
)
|
|
return path
|
|
|
|
|
|
class DisposableDiskTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.dir = Path(self._tmp.name)
|
|
|
|
def tearDown(self):
|
|
self._tmp.cleanup()
|
|
|
|
def test_rejects_dev_path(self):
|
|
with self.assertRaises(UnsafeDiskError):
|
|
validate_disposable_disk("/dev/sda", self.dir)
|
|
|
|
def test_rejects_block_device(self):
|
|
# /dev/loop-control or /dev/null are char/regular; the explicit /dev
|
|
# check fires first. Use a char device outside /dev to isolate.
|
|
with self.assertRaises(UnsafeDiskError):
|
|
validate_disposable_disk("/dev/null", self.dir)
|
|
|
|
def test_rejects_non_qcow2_regular_file(self):
|
|
raw = self.dir / "raw.img"
|
|
raw.write_bytes(b"\x00" * 4096)
|
|
with self.assertRaises(UnsafeDiskError):
|
|
validate_disposable_disk(raw, self.dir)
|
|
|
|
def test_rejects_qcow2_outside_test_dir(self):
|
|
outside = self.dir.parent / "outside-disposable.qcow2"
|
|
make_qcow2(outside)
|
|
self.addCleanup(outside.unlink)
|
|
with self.assertRaises(UnsafeDiskError):
|
|
validate_disposable_disk(outside, self.dir)
|
|
|
|
def test_rejects_symlink_escaping_test_dir(self):
|
|
target = self.dir.parent / "real-disk.qcow2"
|
|
make_qcow2(target)
|
|
self.addCleanup(target.unlink)
|
|
link = self.dir / "link.qcow2"
|
|
link.symlink_to(target)
|
|
with self.assertRaises(UnsafeDiskError):
|
|
validate_disposable_disk(link, self.dir)
|
|
|
|
def test_accepts_qcow2_inside_test_dir(self):
|
|
good = make_qcow2(self.dir / "good.qcow2")
|
|
self.assertEqual(validate_disposable_disk(good, self.dir), good)
|
|
|
|
def test_rejects_backing_chain_outside_test_dir(self):
|
|
backing = self.dir.parent / "backing.qcow2"
|
|
make_qcow2(backing)
|
|
self.addCleanup(backing.unlink)
|
|
overlay = self.dir / "overlay.qcow2"
|
|
subprocess.run(
|
|
["qemu-img", "create", "-f", "qcow2", "-b", str(backing), "-F", "qcow2",
|
|
str(overlay)],
|
|
check=True,
|
|
capture_output=True,
|
|
)
|
|
with self.assertRaises(UnsafeDiskError):
|
|
validate_disposable_disk(overlay, self.dir)
|
|
|
|
def test_accepts_backing_chain_inside_test_dir(self):
|
|
backing = make_qcow2(self.dir / "backing.qcow2")
|
|
overlay = self.dir / "overlay.qcow2"
|
|
subprocess.run(
|
|
["qemu-img", "create", "-f", "qcow2", "-b", str(backing), "-F", "qcow2",
|
|
str(overlay)],
|
|
check=True,
|
|
capture_output=True,
|
|
)
|
|
self.assertEqual(validate_disposable_disk(overlay, self.dir), overlay)
|
|
|
|
def test_rejects_root_as_test_dir(self):
|
|
good = make_qcow2(self.dir / "good.qcow2")
|
|
with self.assertRaises(UnsafeDiskError):
|
|
validate_disposable_disk(good, "/")
|
|
|
|
def test_validate_all_drives_not_just_disk(self):
|
|
a = make_qcow2(self.dir / "a.qcow2")
|
|
b = make_qcow2(self.dir / "b.qcow2")
|
|
checked = validate_writable_drives([a, "", None, b], self.dir)
|
|
self.assertEqual(checked, [a, b])
|
|
with self.assertRaises(UnsafeDiskError):
|
|
validate_writable_drives([a, "/dev/sdb"], self.dir)
|
|
|
|
|
|
class ResultsTests(unittest.TestCase):
|
|
def test_fail_sets_nonzero_exit(self):
|
|
results = Results("unit")
|
|
results.assert_true("ok", True)
|
|
results.assert_true("bad", False, "boom")
|
|
self.assertTrue(results.failed)
|
|
self.assertEqual(results.exit_code(), 1)
|
|
|
|
def test_all_pass_is_zero(self):
|
|
results = Results("unit")
|
|
results.assert_true("ok", True)
|
|
self.assertEqual(results.exit_code(), 0)
|
|
|
|
def test_blocked_only_fails_when_strict(self):
|
|
results = Results("unit", strict=False)
|
|
results.blocked("needs hardware", "no GPU")
|
|
self.assertEqual(results.exit_code(), 0)
|
|
strict = Results("unit", strict=True)
|
|
strict.blocked("needs hardware", "no GPU")
|
|
self.assertEqual(strict.exit_code(), 2)
|
|
|
|
def test_no_checks_is_not_run(self):
|
|
results = Results("unit")
|
|
self.assertEqual(results.counts()[vmsafety.NOT_RUN], 0)
|
|
self.assertEqual(results.exit_code(), 0)
|
|
self.assertEqual(results.summary().split(": ")[1], "no checks")
|
|
|
|
def test_write_round_trips(self):
|
|
results = Results("unit")
|
|
results.pass_("one")
|
|
path = results.write(self._tmp_results())
|
|
import json
|
|
|
|
data = json.loads(path.read_text())
|
|
self.assertEqual(data["scenario"], "unit")
|
|
self.assertEqual(data["counts"][PASS], 1)
|
|
self.assertEqual(data["exit_code"], 0)
|
|
|
|
def _tmp_results(self):
|
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
|
|
tmp.close()
|
|
self.addCleanup(os.unlink, tmp.name)
|
|
return Path(tmp.name)
|
|
|
|
|
|
class ArtifactBindingTests(unittest.TestCase):
|
|
"""The release tool binds evidence to a candidate digest; a harness run
|
|
must record the artifact it verified, or a stale pass could be replayed."""
|
|
|
|
def _tmp_results(self):
|
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
|
|
tmp.close()
|
|
self.addCleanup(os.unlink, tmp.name)
|
|
return Path(tmp.name)
|
|
|
|
def test_artifact_from_env_is_written(self):
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{ARTIFACT_ENV_NAME: "phoebus.iso", ARTIFACT_ENV_SHA256: "abc123"},
|
|
):
|
|
results = Results("unit")
|
|
results.pass_("one")
|
|
# write() is what records the binding, so exercise it directly.
|
|
path = results.write(self._tmp_results())
|
|
data = json.loads(path.read_text())
|
|
self.assertEqual(data["artifact"], {"name": "phoebus.iso", "sha256": "abc123"})
|
|
|
|
def test_artifact_absent_is_null(self):
|
|
clean = {
|
|
k: v
|
|
for k, v in os.environ.items()
|
|
if k not in (ARTIFACT_ENV_NAME, ARTIFACT_ENV_SHA256)
|
|
}
|
|
with mock.patch.dict(os.environ, clean, clear=True):
|
|
results = Results("unit")
|
|
results.pass_("one")
|
|
data = json.loads(results.write(self._tmp_results()).read_text())
|
|
self.assertIsNone(data["artifact"])
|
|
|
|
def test_explicit_binding_is_not_overridden_by_env(self):
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{ARTIFACT_ENV_NAME: "env.iso", ARTIFACT_ENV_SHA256: "env-digest"},
|
|
):
|
|
results = Results("unit")
|
|
results.bind_artifact("explicit.iso", "explicit-digest")
|
|
data = json.loads(results.write(self._tmp_results()).read_text())
|
|
self.assertEqual(
|
|
data["artifact"], {"name": "explicit.iso", "sha256": "explicit-digest"}
|
|
)
|
|
|
|
|
|
class FakeQmpServer(threading.Thread):
|
|
"""A tiny QMP server that records HMP commands and answers them."""
|
|
|
|
def __init__(self):
|
|
super().__init__(daemon=True)
|
|
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
self.path = tempfile.mktemp(prefix="fake-qmp-", suffix=".sock")
|
|
self.sock.bind(self.path)
|
|
self.sock.listen(1)
|
|
self.commands = []
|
|
self.screendump_path = None
|
|
|
|
def run(self):
|
|
conn, _ = self.sock.accept()
|
|
with conn:
|
|
conn.sendall(b'{"QMP": {"version": {"qemu": {"major": 8}}}}' + b"\r\n")
|
|
reader = conn.makefile("rb")
|
|
writer = conn.makefile("wb")
|
|
for line in reader:
|
|
message = __import__("json").loads(line)
|
|
if message.get("execute") == "qmp_capabilities":
|
|
writer.write(b'{"return": {}}\r\n')
|
|
elif message.get("execute") == "human-monitor-command":
|
|
cmd = message["arguments"]["command-line"]
|
|
self.commands.append(cmd)
|
|
if cmd.startswith("screendump"):
|
|
# Emulate QEMU writing the file.
|
|
name = cmd.split(" ", 1)[1].strip().strip('"')
|
|
Path(name).write_bytes(b"P6 fake-ppm")
|
|
writer.write(b'{"return": ""}\r\n')
|
|
else:
|
|
writer.write(b'{"return": {}}\r\n')
|
|
writer.flush()
|
|
|
|
|
|
class QmpClientTests(unittest.TestCase):
|
|
def test_hmp_and_screendump(self):
|
|
server = FakeQmpServer()
|
|
server.start()
|
|
self.addCleanup(lambda: (server.sock.close(), os.path.exists(server.path) and os.unlink(server.path)))
|
|
deadline = 50
|
|
while not os.path.exists(server.path) and deadline:
|
|
deadline -= 1
|
|
with vmsafety.QmpClient(server.path) as client:
|
|
client.sendkey("ret")
|
|
# Wait until the fake server has created the file before returning.
|
|
target = Path(tempfile.mkdtemp()) / "shot with spaces.ppm"
|
|
client.hmp(f'screendump "{target}"')
|
|
# Give the server thread a moment to record the command.
|
|
for _ in range(50):
|
|
if len(server.commands) >= 2:
|
|
break
|
|
__import__("time").sleep(0.05)
|
|
self.assertTrue(any(cmd.startswith("sendkey ret") for cmd in server.commands))
|
|
self.assertTrue(any(cmd.startswith("screendump") for cmd in server.commands))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|