497 lines
20 KiB
Bash
497 lines
20 KiB
Bash
#!/usr/bin/env python3
|
|
"""Mocked-transport tests for the rollback acceptance orchestration.
|
|
|
|
These run on any host (no QEMU, no disk), so the journey's ordering, safety and
|
|
failure semantics are checked in CI rather than only during a live run. Every
|
|
transport here is a fake: the results are marked ``mocked`` and are explicitly
|
|
**not** release evidence (the release tool refuses them).
|
|
|
|
python3 installer/tests/vm/test_rollback_acceptance.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import rollback_acceptance as ra
|
|
from rollback_acceptance import (
|
|
SCENARIO_RECOVERY,
|
|
SCENARIO_ROLLBACK,
|
|
CommandResult,
|
|
RollbackJourney,
|
|
validate_acceptance_disks,
|
|
)
|
|
from vmsafety import FAIL, MOCKED, NOT_RUN, Results, UnsafeDiskError
|
|
|
|
BASE_PKGS = [
|
|
"/var/db/pkg/sys-apps/busybox-1.36.1",
|
|
"/var/db/pkg/app-shells/bash-5.2_p26",
|
|
]
|
|
ADDED_PKG = "/var/db/pkg/app-misc/hello-2.12"
|
|
HISTORY = """\
|
|
[[records]]
|
|
id = "snap-initial"
|
|
created_unix = 1000
|
|
operation = "initial known-good"
|
|
source_subvol = "/"
|
|
snapshot_subvol = "@-20260101T000000-initial-known-good"
|
|
var_snapshot = "@var-20260101T000000-initial-known-good"
|
|
status = "committed"
|
|
"""
|
|
ROLLBACK_STATE = """\
|
|
status = "activated"
|
|
generation = 1001
|
|
target_snapshot = "@-20260101T000000-initial-known-good"
|
|
previous_root = "@"
|
|
root_subvol = "@restore-20260101T000000"
|
|
var_subvol = "@var-restore-20260101T000000"
|
|
started_unix = 1001
|
|
finished_unix = 1001
|
|
note = "restore from snap-initial"
|
|
"""
|
|
|
|
|
|
class FakeGuest:
|
|
"""A scripted guest. First matching handler wins."""
|
|
|
|
def __init__(self, name: str):
|
|
self.name = name
|
|
self.commands: list[str] = []
|
|
self.handlers: list = []
|
|
|
|
def on(self, needle: str, handler):
|
|
# A re-registered needle overrides the earlier one, so a test can tweak
|
|
# one behaviour without disturbing the ordering of distinct needles.
|
|
self.handlers = [(n, h) for (n, h) in self.handlers if n != needle]
|
|
self.handlers.append((needle, handler))
|
|
return self
|
|
|
|
def reply(self, needle: str, stdout: str = "", exit_code: int = 0):
|
|
return self.on(needle, lambda cmd: CommandResult(cmd, stdout=stdout, exit_code=exit_code))
|
|
|
|
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
|
self.commands.append(command)
|
|
for needle, handler in self.handlers:
|
|
if needle in command:
|
|
return handler(command)
|
|
return CommandResult(command=command, exit_code=0)
|
|
|
|
|
|
class FakeTransport:
|
|
def __init__(self, installed, recovery):
|
|
# installed is a list of guests returned by successive boot_installed()
|
|
# calls; the first is the change/undo boot, the second the restored boot.
|
|
self._installed = list(installed)
|
|
self._recovery = recovery
|
|
self.boots: list[str] = []
|
|
self.powered_off = 0
|
|
|
|
def boot_installed(self):
|
|
self.boots.append("installed")
|
|
return self._installed.pop(0)
|
|
|
|
def boot_recovery(self):
|
|
self.boots.append("recovery")
|
|
return self._recovery
|
|
|
|
def poweroff(self):
|
|
self.powered_off += 1
|
|
|
|
|
|
def make_phase1(state: dict) -> FakeGuest:
|
|
g = FakeGuest("phase1")
|
|
|
|
def write_marker(cmd):
|
|
match = re.search(r"printf '%s' '([^']*)' >", cmd)
|
|
if match:
|
|
state["token"] = match.group(1)
|
|
return CommandResult(cmd, exit_code=0)
|
|
|
|
def read_marker(cmd):
|
|
return CommandResult(cmd, stdout=state.get("token", ""))
|
|
|
|
def break_hostname(cmd):
|
|
state["broken"] = True
|
|
return CommandResult(cmd, exit_code=0)
|
|
|
|
def read_hostname(cmd):
|
|
return CommandResult(
|
|
cmd,
|
|
stdout=ra.HOSTNAME_SENTINEL + "\n" if state.get("broken") else "phoebus\n",
|
|
)
|
|
|
|
def install(cmd):
|
|
state["installed"] = True
|
|
return CommandResult(cmd, exit_code=0)
|
|
|
|
def pkg_dirs(cmd):
|
|
dirs = list(BASE_PKGS)
|
|
if state.get("installed"):
|
|
dirs.append(ADDED_PKG)
|
|
return CommandResult(cmd, stdout="\n".join(dirs) + "\n")
|
|
|
|
g.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=HISTORY))
|
|
g.on("sha256sum", lambda c: CommandResult(c, exit_code=0, stdout="b" * 64 + " /usr/local/bin/sys\n"))
|
|
g.on("> /home/phoebus/rollback-acceptance/marker", write_marker)
|
|
g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker)
|
|
g.on("tee /etc/hostname", break_hostname)
|
|
g.on("cat /etc/hostname", read_hostname)
|
|
g.on("sys install", install)
|
|
g.on("find /var/db/pkg", pkg_dirs)
|
|
g.on("cat " + ADDED_PKG + "/CONTENTS", lambda c: CommandResult(
|
|
c, stdout="obj /usr/bin/hello abcdef\nobj /usr/share/hello/README def\n"
|
|
))
|
|
g.on("sys undo", lambda c: CommandResult(c, exit_code=0, stdout="rollback activated"))
|
|
g.on("cat /var/lib/phoebus/rollback-state.toml", lambda c: CommandResult(c, stdout=ROLLBACK_STATE))
|
|
return g
|
|
|
|
|
|
def make_phase2(state: dict) -> FakeGuest:
|
|
g = FakeGuest("phase2")
|
|
|
|
def read_marker(cmd):
|
|
return CommandResult(cmd, stdout=state.get("token", ""))
|
|
|
|
fstab = (
|
|
"/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n"
|
|
"/dev/mapper/luks-root /var btrfs subvol=/@var-restore-20260101T000000 0 0\n"
|
|
"/dev/mapper/luks-root /home btrfs subvol=/@home 0 0\n"
|
|
)
|
|
g.on("cat /etc/hostname", lambda c: CommandResult(c, stdout="phoebus\n"))
|
|
g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker)
|
|
g.on("find /var/db/pkg", lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS) + "\n"))
|
|
g.on("test -e", lambda c: CommandResult(c, exit_code=1))
|
|
g.on("cat /proc/cmdline", lambda c: CommandResult(
|
|
c, stdout="rootflags=subvol=@restore-20260101T000000 quiet console=ttyS0\n"
|
|
))
|
|
g.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab))
|
|
g.on("uname -r", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n"))
|
|
g.on("cat /lib/modules", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n"))
|
|
g.on("bootctl set-oneshot", lambda c: CommandResult(c, exit_code=0))
|
|
return g
|
|
|
|
|
|
def make_recovery() -> FakeGuest:
|
|
g = FakeGuest("recovery")
|
|
g.on("cat /proc/cmdline", lambda c: CommandResult(
|
|
c, stdout="rootflags=subvol=@-20260101T000000-initial-known-good "
|
|
"systemd.unit=rescue.target no-network\n"
|
|
))
|
|
g.on("ip -brief addr", lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8 ::1/128\n"))
|
|
g.on("sys undo status", lambda c: CommandResult(c, exit_code=0, stdout="no active rollback\n"))
|
|
g.on("sys ask", lambda c: CommandResult(
|
|
c,
|
|
exit_code=1,
|
|
stderr="No model is connected. The deterministic interface still works:\n",
|
|
))
|
|
return g
|
|
|
|
|
|
def build_journey(*, state=None, transport=None, **kwargs):
|
|
state = state if state is not None else {}
|
|
transport = transport or FakeTransport([make_phase1(state), make_phase2(state)], make_recovery())
|
|
journey = RollbackJourney(
|
|
transport,
|
|
iso_sha256="a" * 64,
|
|
sys_sha256="b" * 64,
|
|
mocked=True,
|
|
**kwargs,
|
|
)
|
|
state["token"] = journey.marker_token
|
|
return journey
|
|
|
|
|
|
def status_of(results: Results, name: str) -> str | None:
|
|
for check in results.checks:
|
|
if check.name == name:
|
|
return check.status
|
|
return None
|
|
|
|
|
|
class HappyPathTests(unittest.TestCase):
|
|
def test_full_journey_passes_in_expected_order(self):
|
|
journey = build_journey()
|
|
self.assertTrue(journey.run(), "happy path should pass")
|
|
self.assertEqual(list(journey.executed), list(ra.EXPECTED_STEP_NAMES))
|
|
self.assertEqual(journey.boots, ["installed", "installed", "recovery"])
|
|
self.assertEqual(journey.results.exit_code(), 0)
|
|
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
|
|
|
def test_happy_path_has_no_not_run_steps(self):
|
|
journey = build_journey()
|
|
journey.run()
|
|
for stream in (journey.results, journey.recovery_results):
|
|
self.assertFalse(
|
|
any(c.status == NOT_RUN for c in stream.checks),
|
|
"a complete run must not contain NOT RUN steps",
|
|
)
|
|
|
|
def test_mocked_run_is_marked_and_never_live(self):
|
|
journey = build_journey()
|
|
journey.run()
|
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
|
|
tmp.close()
|
|
self.addCleanup(os.unlink, tmp.name)
|
|
data = json.loads(journey.results.write(tmp.name).read_text())
|
|
self.assertEqual(data["evidence"], MOCKED)
|
|
self.assertEqual(data["inputs"]["iso"]["sha256"], "a" * 64)
|
|
self.assertEqual(data["inputs"]["sys"]["sha256"], "b" * 64)
|
|
self.assertEqual(data["scenario"], SCENARIO_ROLLBACK)
|
|
|
|
def test_recovery_scenario_is_written_separately(self):
|
|
journey = build_journey()
|
|
journey.run()
|
|
self.assertEqual(journey.recovery_results.scenario, SCENARIO_RECOVERY)
|
|
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
|
|
|
|
|
class StopOnFailureTests(unittest.TestCase):
|
|
def test_undo_failure_stops_and_marks_rest_unrun(self):
|
|
state = {}
|
|
phase1 = make_phase1(state)
|
|
# Override undo to fail.
|
|
phase1.on("sys undo", lambda c: CommandResult(c, exit_code=1, stderr="boom"))
|
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
|
state["token"] = journey.marker_token
|
|
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "sys-undo-activates"), FAIL)
|
|
self.assertEqual(status_of(journey.results, "journey-complete"), FAIL)
|
|
self.assertEqual(status_of(journey.results, "restored-root-var-generations"), NOT_RUN)
|
|
self.assertEqual(status_of(journey.recovery_results, "recovery-boot-no-network-no-model"), NOT_RUN)
|
|
# The second installed boot and the recovery boot must not happen.
|
|
self.assertEqual(transport.boots, ["installed"])
|
|
self.assertEqual(journey.results.exit_code(), 1)
|
|
|
|
def test_timeout_is_a_failure_not_a_pass(self):
|
|
state = {}
|
|
phase1 = make_phase1(state)
|
|
phase1.on("sys install", lambda c: CommandResult(c, timed_out=True))
|
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
|
state["token"] = journey.marker_token
|
|
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "controlled-package-change"), FAIL)
|
|
self.assertIn("timeout", "".join(c.detail for c in journey.results.checks))
|
|
self.assertEqual(transport.boots, ["installed"])
|
|
|
|
def test_disconnect_is_a_failure(self):
|
|
state = {}
|
|
phase1 = make_phase1(state)
|
|
phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, disconnected=True))
|
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
|
state["token"] = journey.marker_token
|
|
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL)
|
|
self.assertEqual(journey.results.exit_code(), 1)
|
|
|
|
def test_missing_evidence_is_refused_not_treated_as_empty(self):
|
|
state = {}
|
|
phase1 = make_phase1(state)
|
|
phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=""))
|
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
|
state["token"] = journey.marker_token
|
|
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL)
|
|
|
|
def test_missing_input_identities_block_the_run(self):
|
|
# No iso/sys digests: the run must not boot anything.
|
|
transport = FakeTransport([make_phase1({}), make_phase2({})], make_recovery())
|
|
journey = RollbackJourney(transport, mocked=True)
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "input-identities"), FAIL)
|
|
self.assertEqual(status_of(journey.results, "journey-complete"), FAIL)
|
|
self.assertEqual(transport.boots, [])
|
|
|
|
|
|
class RestorationMismatchTests(unittest.TestCase):
|
|
def _journey_with_phase2(self, configure):
|
|
state = {}
|
|
phase1 = make_phase1(state)
|
|
phase2 = make_phase2(state)
|
|
configure(phase2)
|
|
transport = FakeTransport([phase1, phase2], make_recovery())
|
|
journey = RollbackJourney(
|
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
|
)
|
|
state["token"] = journey.marker_token
|
|
return journey
|
|
|
|
def test_wrong_restored_generation_fails(self):
|
|
def configure(phase2):
|
|
phase2.on("cat /proc/cmdline", lambda c: CommandResult(c, stdout="rootflags=subvol=@\n"))
|
|
|
|
journey = self._journey_with_phase2(configure)
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "restored-root-generation"), FAIL)
|
|
self.assertEqual(journey.results.exit_code(), 1)
|
|
|
|
def test_wrong_var_generation_fails(self):
|
|
def configure(phase2):
|
|
fstab = "/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n"
|
|
phase2.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab))
|
|
|
|
journey = self._journey_with_phase2(configure)
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "restored-var-generation"), FAIL)
|
|
|
|
def test_package_db_mismatch_fails(self):
|
|
def configure(phase2):
|
|
phase2.on(
|
|
"find /var/db/pkg",
|
|
lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS + [ADDED_PKG]) + "\n"),
|
|
)
|
|
|
|
journey = self._journey_with_phase2(configure)
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "package-db-restored"), FAIL)
|
|
|
|
def test_surviving_package_file_fails(self):
|
|
def configure(phase2):
|
|
phase2.on("test -e", lambda c: CommandResult(c, exit_code=0))
|
|
|
|
journey = self._journey_with_phase2(configure)
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "package-files-restored"), FAIL)
|
|
|
|
def test_kernel_without_modules_fails(self):
|
|
def configure(phase2):
|
|
phase2.on("cat /lib/modules", lambda c: CommandResult(c, stdout="5.0.0-other\n"))
|
|
|
|
journey = self._journey_with_phase2(configure)
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "kernel-modules-compatible"), FAIL)
|
|
|
|
def test_lost_home_marker_fails(self):
|
|
def configure(phase2):
|
|
phase2.on("cat /home/phoebus/rollback-acceptance/marker", lambda c: CommandResult(c, stdout=""))
|
|
|
|
journey = self._journey_with_phase2(configure)
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.results, "home-data-survives"), FAIL)
|
|
|
|
def test_recovery_with_network_fails(self):
|
|
state = {}
|
|
recovery = make_recovery()
|
|
recovery.on(
|
|
"ip -brief addr",
|
|
lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"),
|
|
)
|
|
transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery)
|
|
journey = RollbackJourney(
|
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
|
)
|
|
state["token"] = journey.marker_token
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.recovery_results, "recovery-no-network"), FAIL)
|
|
|
|
def test_recovery_without_no_model_fallback_fails(self):
|
|
state = {}
|
|
recovery = make_recovery()
|
|
recovery.on("sys ask", lambda c: CommandResult(c, exit_code=0, stdout="here is a plan"))
|
|
transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery)
|
|
journey = RollbackJourney(
|
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
|
)
|
|
state["token"] = journey.marker_token
|
|
self.assertFalse(journey.run())
|
|
self.assertEqual(status_of(journey.recovery_results, "recovery-no-model-fallback"), FAIL)
|
|
|
|
|
|
class RecoveryOnlyTests(unittest.TestCase):
|
|
def test_recovery_only_passes(self):
|
|
transport = FakeTransport([], make_recovery())
|
|
journey = RollbackJourney(
|
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
|
)
|
|
self.assertTrue(journey.run_recovery())
|
|
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
|
self.assertEqual(transport.boots, ["recovery"])
|
|
|
|
def test_recovery_only_without_inputs_does_not_boot(self):
|
|
transport = FakeTransport([], make_recovery())
|
|
journey = RollbackJourney(transport, mocked=True)
|
|
self.assertFalse(journey.run_recovery())
|
|
self.assertEqual(transport.boots, [])
|
|
self.assertEqual(
|
|
status_of(journey.recovery_results, "recovery-boot-no-network-no-model"),
|
|
NOT_RUN,
|
|
)
|
|
|
|
|
|
class SafetyTests(unittest.TestCase):
|
|
def test_rejects_dev_disk(self):
|
|
with self.assertRaises(UnsafeDiskError):
|
|
validate_acceptance_disks(["/dev/sda"], "/var/tmp/phoebus-vm/disks")
|
|
|
|
def test_rejects_disk_outside_test_dir(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
outside = Path(tmp).parent / "rollback-outside.qcow2"
|
|
outside.write_bytes(b"QFI\xfb" + b"\x00" * 64)
|
|
self.addCleanup(outside.unlink)
|
|
with self.assertRaises(UnsafeDiskError):
|
|
validate_acceptance_disks([outside], tmp)
|
|
|
|
|
|
class ParserTests(unittest.TestCase):
|
|
def test_newest_committed_coordinated_skips_unpaired(self):
|
|
records = ra.parse_snapshot_history(
|
|
HISTORY
|
|
+ """
|
|
[[records]]
|
|
id = "snap-varless"
|
|
created_unix = 2000
|
|
operation = "risk"
|
|
source_subvol = "/"
|
|
snapshot_subvol = "@-20260102T000000-risk"
|
|
status = "committed"
|
|
"""
|
|
)
|
|
chosen = ra.newest_committed_coordinated(records)
|
|
self.assertIsNotNone(chosen)
|
|
self.assertEqual(chosen.id, "snap-initial")
|
|
|
|
def test_parse_rootflags_subvol(self):
|
|
self.assertEqual(
|
|
ra.parse_rootflags_subvol("rootflags=subvol=@restore-x quiet"), "@restore-x"
|
|
)
|
|
self.assertEqual(ra.parse_rootflags_subvol("quiet"), None)
|
|
|
|
def test_parse_fstab_subvol(self):
|
|
text = "/dev/x / btrfs subvol=/@restore-y 0 0\n"
|
|
self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y")
|
|
self.assertIsNone(ra.parse_fstab_subvol(text, "/var"))
|
|
|
|
def test_parse_fstab_subvol_without_leading_slash(self):
|
|
text = "/dev/x / btrfs subvol=@restore-y 0 0\n"
|
|
self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y")
|
|
|
|
def test_parse_contents_paths(self):
|
|
paths = ra.parse_contents_files("obj /usr/bin/hello abc\ndir /usr/share/hello def\n")
|
|
self.assertEqual(paths, ["/usr/bin/hello", "/usr/share/hello"])
|
|
|
|
def test_parse_network_addresses(self):
|
|
self.assertEqual(ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\n"), ["lo"])
|
|
self.assertEqual(
|
|
ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"),
|
|
["lo", "enp0s2"],
|
|
)
|
|
|
|
def test_unreadable_history_raises(self):
|
|
with self.assertRaises(ValueError):
|
|
ra.parse_snapshot_history("not = = toml [[")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|