phoebus_os/installer/tests/vm/test_rollback_acceptance.py

497 lines
20 KiB
Bash

#!/usr/bin/env python3
"""Mocked-transport tests for the rollback acceptance orchestration.
These run on any host (no QEMU, no disk), so the journey's ordering, safety and
failure semantics are checked in CI rather than only during a live run. Every
transport here is a fake: the results are marked ``mocked`` and are explicitly
**not** release evidence (the release tool refuses them).
python3 installer/tests/vm/test_rollback_acceptance.py
"""
from __future__ import annotations
import json
import os
import re
import tempfile
import unittest
from pathlib import Path
import rollback_acceptance as ra
from rollback_acceptance import (
SCENARIO_RECOVERY,
SCENARIO_ROLLBACK,
CommandResult,
RollbackJourney,
validate_acceptance_disks,
)
from vmsafety import FAIL, MOCKED, NOT_RUN, Results, UnsafeDiskError
BASE_PKGS = [
"/var/db/pkg/sys-apps/busybox-1.36.1",
"/var/db/pkg/app-shells/bash-5.2_p26",
]
ADDED_PKG = "/var/db/pkg/app-misc/hello-2.12"
HISTORY = """\
[[records]]
id = "snap-initial"
created_unix = 1000
operation = "initial known-good"
source_subvol = "/"
snapshot_subvol = "@-20260101T000000-initial-known-good"
var_snapshot = "@var-20260101T000000-initial-known-good"
status = "committed"
"""
ROLLBACK_STATE = """\
status = "activated"
generation = 1001
target_snapshot = "@-20260101T000000-initial-known-good"
previous_root = "@"
root_subvol = "@restore-20260101T000000"
var_subvol = "@var-restore-20260101T000000"
started_unix = 1001
finished_unix = 1001
note = "restore from snap-initial"
"""
class FakeGuest:
"""A scripted guest. First matching handler wins."""
def __init__(self, name: str):
self.name = name
self.commands: list[str] = []
self.handlers: list = []
def on(self, needle: str, handler):
# A re-registered needle overrides the earlier one, so a test can tweak
# one behaviour without disturbing the ordering of distinct needles.
self.handlers = [(n, h) for (n, h) in self.handlers if n != needle]
self.handlers.append((needle, handler))
return self
def reply(self, needle: str, stdout: str = "", exit_code: int = 0):
return self.on(needle, lambda cmd: CommandResult(cmd, stdout=stdout, exit_code=exit_code))
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
self.commands.append(command)
for needle, handler in self.handlers:
if needle in command:
return handler(command)
return CommandResult(command=command, exit_code=0)
class FakeTransport:
def __init__(self, installed, recovery):
# installed is a list of guests returned by successive boot_installed()
# calls; the first is the change/undo boot, the second the restored boot.
self._installed = list(installed)
self._recovery = recovery
self.boots: list[str] = []
self.powered_off = 0
def boot_installed(self):
self.boots.append("installed")
return self._installed.pop(0)
def boot_recovery(self):
self.boots.append("recovery")
return self._recovery
def poweroff(self):
self.powered_off += 1
def make_phase1(state: dict) -> FakeGuest:
g = FakeGuest("phase1")
def write_marker(cmd):
match = re.search(r"printf '%s' '([^']*)' >", cmd)
if match:
state["token"] = match.group(1)
return CommandResult(cmd, exit_code=0)
def read_marker(cmd):
return CommandResult(cmd, stdout=state.get("token", ""))
def break_hostname(cmd):
state["broken"] = True
return CommandResult(cmd, exit_code=0)
def read_hostname(cmd):
return CommandResult(
cmd,
stdout=ra.HOSTNAME_SENTINEL + "\n" if state.get("broken") else "phoebus\n",
)
def install(cmd):
state["installed"] = True
return CommandResult(cmd, exit_code=0)
def pkg_dirs(cmd):
dirs = list(BASE_PKGS)
if state.get("installed"):
dirs.append(ADDED_PKG)
return CommandResult(cmd, stdout="\n".join(dirs) + "\n")
g.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=HISTORY))
g.on("sha256sum", lambda c: CommandResult(c, exit_code=0, stdout="b" * 64 + " /usr/local/bin/sys\n"))
g.on("> /home/phoebus/rollback-acceptance/marker", write_marker)
g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker)
g.on("tee /etc/hostname", break_hostname)
g.on("cat /etc/hostname", read_hostname)
g.on("sys install", install)
g.on("find /var/db/pkg", pkg_dirs)
g.on("cat " + ADDED_PKG + "/CONTENTS", lambda c: CommandResult(
c, stdout="obj /usr/bin/hello abcdef\nobj /usr/share/hello/README def\n"
))
g.on("sys undo", lambda c: CommandResult(c, exit_code=0, stdout="rollback activated"))
g.on("cat /var/lib/phoebus/rollback-state.toml", lambda c: CommandResult(c, stdout=ROLLBACK_STATE))
return g
def make_phase2(state: dict) -> FakeGuest:
g = FakeGuest("phase2")
def read_marker(cmd):
return CommandResult(cmd, stdout=state.get("token", ""))
fstab = (
"/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n"
"/dev/mapper/luks-root /var btrfs subvol=/@var-restore-20260101T000000 0 0\n"
"/dev/mapper/luks-root /home btrfs subvol=/@home 0 0\n"
)
g.on("cat /etc/hostname", lambda c: CommandResult(c, stdout="phoebus\n"))
g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker)
g.on("find /var/db/pkg", lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS) + "\n"))
g.on("test -e", lambda c: CommandResult(c, exit_code=1))
g.on("cat /proc/cmdline", lambda c: CommandResult(
c, stdout="rootflags=subvol=@restore-20260101T000000 quiet console=ttyS0\n"
))
g.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab))
g.on("uname -r", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n"))
g.on("cat /lib/modules", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n"))
g.on("bootctl set-oneshot", lambda c: CommandResult(c, exit_code=0))
return g
def make_recovery() -> FakeGuest:
g = FakeGuest("recovery")
g.on("cat /proc/cmdline", lambda c: CommandResult(
c, stdout="rootflags=subvol=@-20260101T000000-initial-known-good "
"systemd.unit=rescue.target no-network\n"
))
g.on("ip -brief addr", lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8 ::1/128\n"))
g.on("sys undo status", lambda c: CommandResult(c, exit_code=0, stdout="no active rollback\n"))
g.on("sys ask", lambda c: CommandResult(
c,
exit_code=1,
stderr="No model is connected. The deterministic interface still works:\n",
))
return g
def build_journey(*, state=None, transport=None, **kwargs):
state = state if state is not None else {}
transport = transport or FakeTransport([make_phase1(state), make_phase2(state)], make_recovery())
journey = RollbackJourney(
transport,
iso_sha256="a" * 64,
sys_sha256="b" * 64,
mocked=True,
**kwargs,
)
state["token"] = journey.marker_token
return journey
def status_of(results: Results, name: str) -> str | None:
for check in results.checks:
if check.name == name:
return check.status
return None
class HappyPathTests(unittest.TestCase):
def test_full_journey_passes_in_expected_order(self):
journey = build_journey()
self.assertTrue(journey.run(), "happy path should pass")
self.assertEqual(list(journey.executed), list(ra.EXPECTED_STEP_NAMES))
self.assertEqual(journey.boots, ["installed", "installed", "recovery"])
self.assertEqual(journey.results.exit_code(), 0)
self.assertEqual(journey.recovery_results.exit_code(), 0)
def test_happy_path_has_no_not_run_steps(self):
journey = build_journey()
journey.run()
for stream in (journey.results, journey.recovery_results):
self.assertFalse(
any(c.status == NOT_RUN for c in stream.checks),
"a complete run must not contain NOT RUN steps",
)
def test_mocked_run_is_marked_and_never_live(self):
journey = build_journey()
journey.run()
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
tmp.close()
self.addCleanup(os.unlink, tmp.name)
data = json.loads(journey.results.write(tmp.name).read_text())
self.assertEqual(data["evidence"], MOCKED)
self.assertEqual(data["inputs"]["iso"]["sha256"], "a" * 64)
self.assertEqual(data["inputs"]["sys"]["sha256"], "b" * 64)
self.assertEqual(data["scenario"], SCENARIO_ROLLBACK)
def test_recovery_scenario_is_written_separately(self):
journey = build_journey()
journey.run()
self.assertEqual(journey.recovery_results.scenario, SCENARIO_RECOVERY)
self.assertEqual(journey.recovery_results.exit_code(), 0)
class StopOnFailureTests(unittest.TestCase):
def test_undo_failure_stops_and_marks_rest_unrun(self):
state = {}
phase1 = make_phase1(state)
# Override undo to fail.
phase1.on("sys undo", lambda c: CommandResult(c, exit_code=1, stderr="boom"))
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
state["token"] = journey.marker_token
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "sys-undo-activates"), FAIL)
self.assertEqual(status_of(journey.results, "journey-complete"), FAIL)
self.assertEqual(status_of(journey.results, "restored-root-var-generations"), NOT_RUN)
self.assertEqual(status_of(journey.recovery_results, "recovery-boot-no-network-no-model"), NOT_RUN)
# The second installed boot and the recovery boot must not happen.
self.assertEqual(transport.boots, ["installed"])
self.assertEqual(journey.results.exit_code(), 1)
def test_timeout_is_a_failure_not_a_pass(self):
state = {}
phase1 = make_phase1(state)
phase1.on("sys install", lambda c: CommandResult(c, timed_out=True))
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
state["token"] = journey.marker_token
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "controlled-package-change"), FAIL)
self.assertIn("timeout", "".join(c.detail for c in journey.results.checks))
self.assertEqual(transport.boots, ["installed"])
def test_disconnect_is_a_failure(self):
state = {}
phase1 = make_phase1(state)
phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, disconnected=True))
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
state["token"] = journey.marker_token
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL)
self.assertEqual(journey.results.exit_code(), 1)
def test_missing_evidence_is_refused_not_treated_as_empty(self):
state = {}
phase1 = make_phase1(state)
phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=""))
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
state["token"] = journey.marker_token
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL)
def test_missing_input_identities_block_the_run(self):
# No iso/sys digests: the run must not boot anything.
transport = FakeTransport([make_phase1({}), make_phase2({})], make_recovery())
journey = RollbackJourney(transport, mocked=True)
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "input-identities"), FAIL)
self.assertEqual(status_of(journey.results, "journey-complete"), FAIL)
self.assertEqual(transport.boots, [])
class RestorationMismatchTests(unittest.TestCase):
def _journey_with_phase2(self, configure):
state = {}
phase1 = make_phase1(state)
phase2 = make_phase2(state)
configure(phase2)
transport = FakeTransport([phase1, phase2], make_recovery())
journey = RollbackJourney(
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
)
state["token"] = journey.marker_token
return journey
def test_wrong_restored_generation_fails(self):
def configure(phase2):
phase2.on("cat /proc/cmdline", lambda c: CommandResult(c, stdout="rootflags=subvol=@\n"))
journey = self._journey_with_phase2(configure)
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "restored-root-generation"), FAIL)
self.assertEqual(journey.results.exit_code(), 1)
def test_wrong_var_generation_fails(self):
def configure(phase2):
fstab = "/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n"
phase2.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab))
journey = self._journey_with_phase2(configure)
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "restored-var-generation"), FAIL)
def test_package_db_mismatch_fails(self):
def configure(phase2):
phase2.on(
"find /var/db/pkg",
lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS + [ADDED_PKG]) + "\n"),
)
journey = self._journey_with_phase2(configure)
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "package-db-restored"), FAIL)
def test_surviving_package_file_fails(self):
def configure(phase2):
phase2.on("test -e", lambda c: CommandResult(c, exit_code=0))
journey = self._journey_with_phase2(configure)
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "package-files-restored"), FAIL)
def test_kernel_without_modules_fails(self):
def configure(phase2):
phase2.on("cat /lib/modules", lambda c: CommandResult(c, stdout="5.0.0-other\n"))
journey = self._journey_with_phase2(configure)
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "kernel-modules-compatible"), FAIL)
def test_lost_home_marker_fails(self):
def configure(phase2):
phase2.on("cat /home/phoebus/rollback-acceptance/marker", lambda c: CommandResult(c, stdout=""))
journey = self._journey_with_phase2(configure)
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.results, "home-data-survives"), FAIL)
def test_recovery_with_network_fails(self):
state = {}
recovery = make_recovery()
recovery.on(
"ip -brief addr",
lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"),
)
transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery)
journey = RollbackJourney(
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
)
state["token"] = journey.marker_token
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.recovery_results, "recovery-no-network"), FAIL)
def test_recovery_without_no_model_fallback_fails(self):
state = {}
recovery = make_recovery()
recovery.on("sys ask", lambda c: CommandResult(c, exit_code=0, stdout="here is a plan"))
transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery)
journey = RollbackJourney(
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
)
state["token"] = journey.marker_token
self.assertFalse(journey.run())
self.assertEqual(status_of(journey.recovery_results, "recovery-no-model-fallback"), FAIL)
class RecoveryOnlyTests(unittest.TestCase):
def test_recovery_only_passes(self):
transport = FakeTransport([], make_recovery())
journey = RollbackJourney(
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
)
self.assertTrue(journey.run_recovery())
self.assertEqual(journey.recovery_results.exit_code(), 0)
self.assertEqual(transport.boots, ["recovery"])
def test_recovery_only_without_inputs_does_not_boot(self):
transport = FakeTransport([], make_recovery())
journey = RollbackJourney(transport, mocked=True)
self.assertFalse(journey.run_recovery())
self.assertEqual(transport.boots, [])
self.assertEqual(
status_of(journey.recovery_results, "recovery-boot-no-network-no-model"),
NOT_RUN,
)
class SafetyTests(unittest.TestCase):
def test_rejects_dev_disk(self):
with self.assertRaises(UnsafeDiskError):
validate_acceptance_disks(["/dev/sda"], "/var/tmp/phoebus-vm/disks")
def test_rejects_disk_outside_test_dir(self):
with tempfile.TemporaryDirectory() as tmp:
outside = Path(tmp).parent / "rollback-outside.qcow2"
outside.write_bytes(b"QFI\xfb" + b"\x00" * 64)
self.addCleanup(outside.unlink)
with self.assertRaises(UnsafeDiskError):
validate_acceptance_disks([outside], tmp)
class ParserTests(unittest.TestCase):
def test_newest_committed_coordinated_skips_unpaired(self):
records = ra.parse_snapshot_history(
HISTORY
+ """
[[records]]
id = "snap-varless"
created_unix = 2000
operation = "risk"
source_subvol = "/"
snapshot_subvol = "@-20260102T000000-risk"
status = "committed"
"""
)
chosen = ra.newest_committed_coordinated(records)
self.assertIsNotNone(chosen)
self.assertEqual(chosen.id, "snap-initial")
def test_parse_rootflags_subvol(self):
self.assertEqual(
ra.parse_rootflags_subvol("rootflags=subvol=@restore-x quiet"), "@restore-x"
)
self.assertEqual(ra.parse_rootflags_subvol("quiet"), None)
def test_parse_fstab_subvol(self):
text = "/dev/x / btrfs subvol=/@restore-y 0 0\n"
self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y")
self.assertIsNone(ra.parse_fstab_subvol(text, "/var"))
def test_parse_fstab_subvol_without_leading_slash(self):
text = "/dev/x / btrfs subvol=@restore-y 0 0\n"
self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y")
def test_parse_contents_paths(self):
paths = ra.parse_contents_files("obj /usr/bin/hello abc\ndir /usr/share/hello def\n")
self.assertEqual(paths, ["/usr/bin/hello", "/usr/share/hello"])
def test_parse_network_addresses(self):
self.assertEqual(ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\n"), ["lo"])
self.assertEqual(
ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"),
["lo", "enp0s2"],
)
def test_unreadable_history_raises(self):
with self.assertRaises(ValueError):
ra.parse_snapshot_history("not = = toml [[")
if __name__ == "__main__":
unittest.main(verbosity=2)