852 lines
32 KiB
Bash
852 lines
32 KiB
Bash
#!/usr/bin/env python3
|
|
"""Offline orchestration for the rollback + no-model-recovery acceptance run.
|
|
|
|
This module holds the *logic* of the journey: the ordered steps, the guest
|
|
commands they issue, and the assertions that decide PASS/FAIL. It deliberately
|
|
imports neither pexpect nor QEMU, so the whole journey can be exercised on a
|
|
plain host against a fake transport (see ``test_rollback_acceptance.py``).
|
|
|
|
The real VM driver (``rollback-driver.py``) supplies a pexpect-backed transport
|
|
implementing :class:`BootTransport` and a :class:`Guest` that runs commands over
|
|
the serial console. The release gates import the ``results.json`` these runs
|
|
write; a run made with a fake transport is marked ``mocked`` and can never be a
|
|
gate pass.
|
|
|
|
Journey shape (the installer already created a coordinated ``@`` + ``@var``
|
|
known-good snapshot at install time):
|
|
|
|
boot installed (ISO detached)
|
|
1. identify the newest committed coordinated snapshot
|
|
2. write a marker into /home (a subvolume rollback must not touch)
|
|
3. apply a controlled package change (touches / and /var/db/pkg)
|
|
4. introduce deterministic guest-only breakage in /
|
|
5. run the real `sys undo` (restores root + var as the next boot root)
|
|
reboot, boot installed (ISO detached)
|
|
6. restored root/var generations match the rollback state
|
|
7. the changed package and its files are gone
|
|
8. /var/db/pkg matches / (consistent package database)
|
|
9. the booted kernel has matching modules in the restored root
|
|
10. /home data survived
|
|
arm the recovery entry, reboot, boot recovery (no network, no model)
|
|
11. recovery entry boots, no network, and `sys` still works with no model
|
|
|
|
Nothing here decides that a *live* run passed: only a real transport on a
|
|
disposable qcow2 disk can produce release evidence. The mocked run exists to
|
|
prove the orchestration itself.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import re
|
|
import tomllib
|
|
from typing import Callable, Protocol, runtime_checkable
|
|
|
|
import vmsafety
|
|
from vmsafety import INPUT_ISO, INPUT_SYS, Results
|
|
|
|
# Scenario names. Each is a release-gate evidence key.
|
|
SCENARIO_ROLLBACK = "rollback-acceptance"
|
|
SCENARIO_RECOVERY = "rollback-recovery"
|
|
|
|
# The input identities a gate must see before it may pass. The rollback
|
|
# semantics live in `sys`, so naming only the ISO is incomplete evidence.
|
|
REQUIRED_INPUTS: dict[str, tuple[str, ...]] = {
|
|
SCENARIO_ROLLBACK: (INPUT_ISO, INPUT_SYS),
|
|
SCENARIO_RECOVERY: (INPUT_ISO, INPUT_SYS),
|
|
}
|
|
|
|
# Stable guest paths and sentinels.
|
|
SNAPSHOT_HISTORY = "/var/lib/phoebus/snapshot-history.toml"
|
|
ROLLBACK_STATE = "/var/lib/phoebus/rollback-state.toml"
|
|
PKG_DB = "/var/db/pkg"
|
|
HOSTNAME_FILE = "/etc/hostname"
|
|
HOME_MARKER_REL = "rollback-acceptance/marker"
|
|
HOME_MARKER_TOKEN = "phoebus-rollback-home-marker"
|
|
HOSTNAME_SENTINEL = "phoebus-rollback-broken-hostname"
|
|
DEFAULT_SYS = "/usr/local/bin/sys"
|
|
DEFAULT_PACKAGE = "app-misc/hello"
|
|
|
|
# File framing so a read is not confused with console noise.
|
|
FILE_FRAME = "__PHOEBUS_RB_FILE__"
|
|
# The driver appends this and parses the exit status from the console.
|
|
RC_SENTINEL = "__PHOEBUS_RB_RC"
|
|
|
|
MAX_OUTPUT_BYTES = 64 * 1024
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Small, testable value types
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class CommandResult:
|
|
"""The outcome of one guest command.
|
|
|
|
``timed_out`` and ``disconnected`` are distinct from a nonzero exit so a
|
|
broken connection can never be mistaken for a clean failure that a later
|
|
step might tolerate.
|
|
"""
|
|
|
|
command: str
|
|
stdout: str = ""
|
|
stderr: str = ""
|
|
exit_code: int | None = None
|
|
timed_out: bool = False
|
|
disconnected: bool = False
|
|
|
|
def bounded(self) -> "CommandResult":
|
|
combined = self.stdout[-MAX_OUTPUT_BYTES:]
|
|
self.stdout = combined
|
|
self.stderr = self.stderr[-MAX_OUTPUT_BYTES:]
|
|
return self
|
|
|
|
@property
|
|
def ok(self) -> bool:
|
|
return (
|
|
not self.timed_out
|
|
and not self.disconnected
|
|
and self.exit_code == 0
|
|
)
|
|
|
|
@property
|
|
def failure_reason(self) -> str:
|
|
if self.timed_out:
|
|
return f"timeout running `{self.command}`"
|
|
if self.disconnected:
|
|
return f"guest disconnected running `{self.command}`"
|
|
if self.exit_code is None:
|
|
return f"no exit status for `{self.command}`"
|
|
if self.exit_code != 0:
|
|
detail = (self.stderr or self.stdout).strip().splitlines()
|
|
tail = detail[-1] if detail else ""
|
|
return f"`{self.command}` exited {self.exit_code}: {tail}"
|
|
return ""
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class SnapshotRecord:
|
|
id: str
|
|
operation: str
|
|
snapshot_subvol: str
|
|
var_snapshot: str | None
|
|
status: str
|
|
|
|
|
|
@runtime_checkable
|
|
class Guest(Protocol):
|
|
"""A shell on a booted installed system."""
|
|
|
|
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
|
...
|
|
|
|
|
|
@runtime_checkable
|
|
class BootTransport(Protocol):
|
|
"""Boots the installed disk in the requested mode.
|
|
|
|
* ``boot_installed`` boots the normal entry with the installer ISO detached.
|
|
* ``boot_recovery`` boots the recovery entry with networking disabled.
|
|
"""
|
|
|
|
def boot_installed(self) -> Guest:
|
|
...
|
|
|
|
def boot_recovery(self) -> Guest:
|
|
...
|
|
|
|
def poweroff(self) -> None:
|
|
...
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Parsers (pure functions, unit-tested directly)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_between(text: str, start: str, end: str) -> str:
|
|
"""Return the text between the first ``start`` and the next ``end``.
|
|
|
|
If the markers are absent the whole text is returned unchanged, so a fake
|
|
transport may return clean content directly.
|
|
"""
|
|
if start not in text or end not in text:
|
|
return text
|
|
body = text.split(start, 1)[1]
|
|
return body.split(end, 1)[0]
|
|
|
|
|
|
def parse_snapshot_history(text: str) -> list[SnapshotRecord]:
|
|
"""Parse the snapshot history TOML into records.
|
|
|
|
Raises :class:`ValueError` when the document is unreadable; a caller must
|
|
never treat "could not parse" as "no snapshots".
|
|
"""
|
|
text = text.strip()
|
|
if not text:
|
|
raise ValueError("empty snapshot history")
|
|
try:
|
|
data = tomllib.loads(text)
|
|
except tomllib.TOMLDecodeError as exc: # pragma: no cover - message detail
|
|
raise ValueError(f"invalid snapshot history: {exc}") from exc
|
|
records = data.get("records", [])
|
|
if not isinstance(records, list):
|
|
raise ValueError("snapshot history has no `records` list")
|
|
out: list[SnapshotRecord] = []
|
|
for entry in records:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
out.append(
|
|
SnapshotRecord(
|
|
id=str(entry.get("id", "")),
|
|
operation=str(entry.get("operation", "")),
|
|
snapshot_subvol=str(entry.get("snapshot_subvol", "")),
|
|
var_snapshot=entry.get("var_snapshot"),
|
|
status=str(entry.get("status", "")),
|
|
)
|
|
)
|
|
return out
|
|
|
|
|
|
def newest_committed_coordinated(
|
|
records: list[SnapshotRecord],
|
|
) -> SnapshotRecord | None:
|
|
"""The newest committed snapshot that also captured ``/var``.
|
|
|
|
A root snapshot with no paired ``@var`` snapshot cannot restore a
|
|
consistent package database, so it is not a valid rollback target.
|
|
"""
|
|
candidates = [
|
|
r
|
|
for r in records
|
|
if r.status == "committed" and r.snapshot_subvol and r.var_snapshot
|
|
]
|
|
if not candidates:
|
|
return None
|
|
return candidates[-1]
|
|
|
|
|
|
def parse_rollback_state(text: str) -> dict:
|
|
"""Parse ``/var/lib/phoebus/rollback-state.toml`` into a dict."""
|
|
text = text.strip()
|
|
if not text:
|
|
raise ValueError("empty rollback state")
|
|
try:
|
|
return tomllib.loads(text)
|
|
except tomllib.TOMLDecodeError as exc: # pragma: no cover - message detail
|
|
raise ValueError(f"invalid rollback state: {exc}") from exc
|
|
|
|
|
|
def normalize_subvol(value: str | None) -> str | None:
|
|
"""Canonicalise a subvolume path for comparison.
|
|
|
|
Btrfs reports the same subvolume as ``@restore-x`` in ``rootflags`` and
|
|
``/@restore-x`` in a fstab mount option; comparison must ignore that.
|
|
"""
|
|
if value is None:
|
|
return None
|
|
return value.lstrip("/")
|
|
|
|
|
|
def parse_rootflags_subvol(cmdline: str) -> str | None:
|
|
"""Extract the ``subvol=`` value from the kernel ``rootflags=`` option."""
|
|
match = re.search(r"rootflags=[^ ]*?\bsubvol=([^,\s]+)", cmdline)
|
|
if match:
|
|
return normalize_subvol(match.group(1))
|
|
# Some kernels present rootflags only as the bare option.
|
|
match = re.search(r"\bsubvol=([^,\s]+)", cmdline)
|
|
return normalize_subvol(match.group(1)) if match else None
|
|
|
|
|
|
def parse_fstab_subvol(text: str, mountpoint: str) -> str | None:
|
|
"""The ``subvol=`` value for a mountpoint in ``/etc/fstab``."""
|
|
for line in text.splitlines():
|
|
line = line.split("#", 1)[0].strip()
|
|
if not line:
|
|
continue
|
|
fields = line.split()
|
|
if len(fields) < 4:
|
|
continue
|
|
if fields[1] != mountpoint:
|
|
continue
|
|
match = re.search(r"(?:^|,)subvol=([^,\s]+)", fields[3])
|
|
if match:
|
|
return normalize_subvol(match.group(1))
|
|
return None
|
|
|
|
|
|
def parse_package_dirs(text: str) -> list[str]:
|
|
"""Absolute ``/var/db/pkg/<cat>/<pkg>`` directories from a listing."""
|
|
return sorted({line.strip() for line in text.splitlines() if line.strip()})
|
|
|
|
|
|
def parse_contents_files(text: str) -> list[str]:
|
|
"""Installed file paths from a Portage ``CONTENTS`` file.
|
|
|
|
The format is ``<obj> <path> <hash...>`` per line.
|
|
"""
|
|
paths: list[str] = []
|
|
for line in text.splitlines():
|
|
fields = line.split()
|
|
if len(fields) >= 2 and fields[0] in ("obj", "dir", "sym"):
|
|
paths.append(fields[1])
|
|
return paths
|
|
|
|
|
|
def parse_uname(text: str) -> str:
|
|
return text.strip().splitlines()[-1].strip() if text.strip() else ""
|
|
|
|
|
|
def parse_network_addresses(text: str) -> list[str]:
|
|
"""Interfaces that have an address, from ``ip -brief addr`` output."""
|
|
out: list[str] = []
|
|
for line in text.splitlines():
|
|
fields = line.split()
|
|
if not fields:
|
|
continue
|
|
name = fields[0]
|
|
rest = fields[1:]
|
|
if any("." in f or ":" in f for f in rest):
|
|
out.append(name)
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Journey
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class _Step:
|
|
name: str
|
|
method_name: str
|
|
boot: str | None = None
|
|
stream: str = "rollback"
|
|
|
|
|
|
# Order is the contract. A change here changes the release gate; keep every
|
|
# expected step in the list so a stopped run can mark the rest NOT RUN.
|
|
_STEPS: tuple[_Step, ...] = (
|
|
_Step("sys-binary-identity", "_step_sys_identity", boot="installed"),
|
|
_Step("known-good-coordinated-snapshot", "_step_known_good"),
|
|
_Step("home-data-marker", "_step_mark_home"),
|
|
_Step("controlled-package-change", "_step_package_change"),
|
|
_Step("deterministic-guest-breakage", "_step_break"),
|
|
_Step("sys-undo-activates", "_step_undo"),
|
|
_Step("reboot-detached", "_step_reboot", boot="installed"),
|
|
_Step("restored-root-var-generations", "_step_generations"),
|
|
_Step("package-files-restored", "_step_package_files"),
|
|
_Step("portage-database-consistent", "_step_portage_db"),
|
|
_Step("kernel-modules-compatible", "_step_kernel_modules"),
|
|
_Step("home-data-survives", "_step_home_survives"),
|
|
_Step("recovery-entry-armed", "_step_arm_recovery"),
|
|
_Step(
|
|
"recovery-boot-no-network-no-model",
|
|
"_step_recovery",
|
|
boot="recovery",
|
|
stream="recovery",
|
|
),
|
|
)
|
|
|
|
EXPECTED_STEP_NAMES: tuple[str, ...] = tuple(step.name for step in _STEPS)
|
|
|
|
|
|
class RollbackJourney:
|
|
"""Run the ordered acceptance steps against a :class:`BootTransport`.
|
|
|
|
The class records every expected step, so a run that stops early reports the
|
|
remainder as NOT RUN and fails an explicit ``journey-complete`` check rather
|
|
than silently looking like a pass.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
transport: BootTransport,
|
|
*,
|
|
results: Results | None = None,
|
|
recovery_results: Results | None = None,
|
|
home: str = "/home/phoebus",
|
|
sys_bin: str = DEFAULT_SYS,
|
|
package: str = DEFAULT_PACKAGE,
|
|
iso_sha256: str | None = None,
|
|
sys_sha256: str | None = None,
|
|
mocked: bool = False,
|
|
):
|
|
self.transport = transport
|
|
self.results = results or Results(SCENARIO_ROLLBACK)
|
|
self.recovery_results = recovery_results or Results(SCENARIO_RECOVERY)
|
|
for stream in (self.results, self.recovery_results):
|
|
if iso_sha256:
|
|
stream.bind_input(INPUT_ISO, iso_sha256)
|
|
if sys_sha256:
|
|
stream.bind_input(INPUT_SYS, sys_sha256)
|
|
if mocked:
|
|
stream.mark_mocked("fake transport: not release evidence")
|
|
self.home = home
|
|
self.sys_bin = sys_bin
|
|
self.package = package
|
|
self.expected_sys_sha256 = sys_sha256
|
|
self.marker_path = f"{home.rstrip('/')}/{HOME_MARKER_REL}"
|
|
self.marker_token = f"{HOME_MARKER_TOKEN}:{self.results.run_id}"
|
|
self.boots: list[str] = []
|
|
self.executed: list[str] = []
|
|
|
|
# Findings carried between steps.
|
|
self.snapshot: SnapshotRecord | None = None
|
|
self.original_hostname: str | None = None
|
|
self.package_dirs_before: list[str] = []
|
|
self.added_package_dirs: list[str] = []
|
|
self.added_package_files: list[str] = []
|
|
self.rollback_state: dict = {}
|
|
|
|
self._active = self.results
|
|
|
|
# -- helpers -----------------------------------------------------------
|
|
def _check(self, name: str, condition: bool, detail: str = "") -> bool:
|
|
return self._active.assert_true(name, condition, detail)
|
|
|
|
def _run(
|
|
self, guest: Guest, script: str, timeout: float = 240.0
|
|
) -> CommandResult:
|
|
try:
|
|
return guest.run(script, timeout=timeout).bounded()
|
|
except Exception as exc: # a transport fault is a failure, not a crash
|
|
return CommandResult(
|
|
command=script, disconnected=True, stderr=str(exc)
|
|
)
|
|
|
|
def _read_file(self, guest: Guest, path: str, timeout: float = 60.0) -> str:
|
|
result = self._run(
|
|
guest,
|
|
f"echo {FILE_FRAME}; cat {path} 2>/dev/null; echo {FILE_FRAME}",
|
|
timeout=timeout,
|
|
)
|
|
return extract_between(result.stdout, FILE_FRAME, FILE_FRAME)
|
|
|
|
# -- public API --------------------------------------------------------
|
|
def required_inputs(self) -> tuple[str, ...]:
|
|
return REQUIRED_INPUTS[self.results.scenario]
|
|
|
|
def run(self) -> bool:
|
|
"""Execute the journey. Returns True only if every step passed."""
|
|
if not self._check_inputs():
|
|
for step in _STEPS:
|
|
stream = (
|
|
self.results
|
|
if step.stream == "rollback"
|
|
else self.recovery_results
|
|
)
|
|
stream.mark_not_run(step.name, "input identities were not pinned")
|
|
self.results.fail("journey-complete", "missing input identities")
|
|
self.recovery_results.fail("journey-complete", "missing input identities")
|
|
return False
|
|
|
|
guest: Guest | None = None
|
|
for index, step in enumerate(_STEPS):
|
|
self._active = (
|
|
self.results if step.stream == "rollback" else self.recovery_results
|
|
)
|
|
if step.boot is not None:
|
|
if guest is not None:
|
|
self.transport.poweroff()
|
|
guest = (
|
|
self.transport.boot_recovery()
|
|
if step.boot == "recovery"
|
|
else self.transport.boot_installed()
|
|
)
|
|
self.boots.append(step.boot)
|
|
method: Callable[[Guest], bool] = getattr(self, step.method_name)
|
|
self.executed.append(step.name)
|
|
if not method(guest):
|
|
self._finish_incomplete(index, step.name)
|
|
return False
|
|
|
|
self.results.assert_true("journey-complete", True)
|
|
self.recovery_results.assert_true("journey-complete", True)
|
|
return True
|
|
|
|
def run_recovery(self) -> bool:
|
|
"""Verify only the no-model recovery boot (used by the `recovery` mode)."""
|
|
if not self._check_inputs():
|
|
self.recovery_results.mark_not_run(
|
|
"recovery-boot-no-network-no-model", "input identities were not pinned"
|
|
)
|
|
self.recovery_results.fail("journey-complete", "missing input identities")
|
|
return False
|
|
self._active = self.recovery_results
|
|
ok = self._step_recovery(self.transport.boot_recovery())
|
|
if ok:
|
|
self.recovery_results.assert_true("journey-complete", True)
|
|
else:
|
|
self.recovery_results.fail(
|
|
"journey-complete", "recovery verification failed"
|
|
)
|
|
return ok
|
|
|
|
# -- input identity gate ----------------------------------------------
|
|
def _check_inputs(self) -> bool:
|
|
ok = True
|
|
for stream, scenario in (
|
|
(self.results, SCENARIO_ROLLBACK),
|
|
(self.recovery_results, SCENARIO_RECOVERY),
|
|
):
|
|
required = REQUIRED_INPUTS[scenario]
|
|
missing = stream.missing_inputs(required)
|
|
ok &= stream.assert_true(
|
|
"input-identities",
|
|
not missing,
|
|
f"missing pinned inputs for {scenario}: {', '.join(missing)}",
|
|
)
|
|
return bool(ok)
|
|
|
|
def _finish_incomplete(self, failed_index: int, failed_step: str) -> None:
|
|
for step in _STEPS[failed_index + 1:]:
|
|
stream = (
|
|
self.results if step.stream == "rollback" else self.recovery_results
|
|
)
|
|
stream.mark_not_run(step.name, "journey stopped before this step")
|
|
# The failed step's own stream already has a FAIL; make the incompleteness
|
|
# explicit on both so neither gate can read a partial run as a pass.
|
|
self.results.fail(
|
|
"journey-complete", f"stopped after `{failed_step}`"
|
|
)
|
|
self.recovery_results.fail(
|
|
"journey-complete", f"stopped after `{failed_step}`"
|
|
)
|
|
|
|
# -- steps -------------------------------------------------------------
|
|
def _step_sys_identity(self, guest: Guest) -> bool:
|
|
# The rollback semantics live in `sys`; pin the exact binary under test.
|
|
self._active = self.results
|
|
expected = self.expected_sys_sha256 or ""
|
|
result = self._run(guest, f"sha256sum {shlex_quote(self.sys_bin)}", timeout=60.0)
|
|
if not result.ok:
|
|
return self._check("sys-binary-identity", False, result.failure_reason)
|
|
digest = result.stdout.split()[0] if result.stdout.split() else ""
|
|
return self._check(
|
|
"sys-binary-identity",
|
|
bool(expected) and digest == expected,
|
|
f"guest sys digest {digest!r} != pinned {expected!r}",
|
|
)
|
|
|
|
def _step_known_good(self, guest: Guest) -> bool:
|
|
self._active = self.results
|
|
text = self._read_file(guest, SNAPSHOT_HISTORY)
|
|
try:
|
|
records = parse_snapshot_history(text)
|
|
except ValueError as exc:
|
|
return self._check("known-good-coordinated-snapshot", False, str(exc))
|
|
snapshot = newest_committed_coordinated(records)
|
|
if snapshot is None:
|
|
return self._check(
|
|
"known-good-coordinated-snapshot",
|
|
False,
|
|
"no committed snapshot with a paired @var snapshot",
|
|
)
|
|
self.snapshot = snapshot
|
|
self._check("known-good-root-snapshot", bool(snapshot.snapshot_subvol), "empty root snapshot path")
|
|
self._check("known-good-var-snapshot", bool(snapshot.var_snapshot), "empty @var snapshot path")
|
|
self._check(
|
|
"known-good-coordinated-snapshot",
|
|
True,
|
|
f"{snapshot.id} ({snapshot.operation})",
|
|
)
|
|
return True
|
|
|
|
def _step_mark_home(self, guest: Guest) -> bool:
|
|
self._active = self.results
|
|
result = self._run(
|
|
guest,
|
|
f"mkdir -p {self.home.rstrip('/')}/{HOME_MARKER_REL.rsplit('/', 1)[0]} && "
|
|
f"printf '%s' '{self.marker_token}' > {self.marker_path}",
|
|
)
|
|
if not result.ok:
|
|
return self._check("home-data-marker", False, result.failure_reason)
|
|
readback = self._read_file(guest, self.marker_path)
|
|
if readback.strip() != self.marker_token:
|
|
return self._check(
|
|
"home-data-marker", False, "home marker did not read back"
|
|
)
|
|
self.original_hostname = self._read_file(guest, HOSTNAME_FILE).strip()
|
|
self.package_dirs_before = parse_package_dirs(
|
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
|
)
|
|
self._check("home-data-marker", True, self.marker_path)
|
|
self._check(
|
|
"hostname-recorded", bool(self.original_hostname), "could not read /etc/hostname"
|
|
)
|
|
return bool(self.original_hostname)
|
|
|
|
def _step_package_change(self, guest: Guest) -> bool:
|
|
self._active = self.results
|
|
result = self._run(
|
|
guest, f"sudo {self.sys_bin} install {self.package} -y", timeout=1800.0
|
|
)
|
|
if not result.ok:
|
|
return self._check("controlled-package-change", False, result.failure_reason)
|
|
after = parse_package_dirs(
|
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
|
)
|
|
added = sorted(set(after) - set(self.package_dirs_before))
|
|
if not added:
|
|
return self._check(
|
|
"controlled-package-change",
|
|
False,
|
|
f"`{self.package}` produced no new /var/db/pkg entry",
|
|
)
|
|
self.added_package_dirs = added
|
|
contents = self._read_file(guest, f"{added[0]}/CONTENTS")
|
|
self.added_package_files = parse_contents_files(contents)
|
|
self._check(
|
|
"controlled-package-change", True, f"added {', '.join(added)}"
|
|
)
|
|
self._check(
|
|
"package-db-recorded",
|
|
bool(self.added_package_files),
|
|
"changed package has no recorded files",
|
|
)
|
|
return True
|
|
|
|
def _step_break(self, guest: Guest) -> bool:
|
|
self._active = self.results
|
|
result = self._run(
|
|
guest,
|
|
f"printf '%s\\n' '{HOSTNAME_SENTINEL}' | sudo tee {HOSTNAME_FILE} >/dev/null",
|
|
)
|
|
if not result.ok:
|
|
return self._check("deterministic-guest-breakage", False, result.failure_reason)
|
|
broken = self._read_file(guest, HOSTNAME_FILE).strip()
|
|
if broken != HOSTNAME_SENTINEL:
|
|
return self._check(
|
|
"deterministic-guest-breakage",
|
|
False,
|
|
"breakage did not take effect",
|
|
)
|
|
self._check(
|
|
"deterministic-guest-breakage",
|
|
True,
|
|
f"{HOSTNAME_FILE} -> {HOSTNAME_SENTINEL}",
|
|
)
|
|
return True
|
|
|
|
def _step_undo(self, guest: Guest) -> bool:
|
|
self._active = self.results
|
|
if self.snapshot is None:
|
|
return self._check("sys-undo-activates", False, "no snapshot selected")
|
|
result = self._run(
|
|
guest, f"sudo {self.sys_bin} undo {self.snapshot.id} -y", timeout=900.0
|
|
)
|
|
if not result.ok:
|
|
return self._check("sys-undo-activates", False, result.failure_reason)
|
|
state = self._read_file(guest, ROLLBACK_STATE)
|
|
try:
|
|
self.rollback_state = parse_rollback_state(state)
|
|
except ValueError as exc:
|
|
return self._check("sys-undo-activates", False, str(exc))
|
|
if self.rollback_state.get("status") != "activated":
|
|
return self._check(
|
|
"sys-undo-activates",
|
|
False,
|
|
f"rollback state is {self.rollback_state.get('status')!r}, not 'activated'",
|
|
)
|
|
self._check(
|
|
"sys-undo-activates",
|
|
True,
|
|
f"next boot uses {self.rollback_state.get('root_subvol')}",
|
|
)
|
|
return True
|
|
|
|
def _step_reboot(self, guest: Guest) -> bool:
|
|
# The boot itself is performed by the transport; the driver asserts the
|
|
# ISO is detached before this returns. Here we only record the boundary.
|
|
self._active = self.results
|
|
self._check("reboot-detached", True, "booted installed disk with ISO detached")
|
|
return True
|
|
|
|
def _step_generations(self, guest: Guest) -> bool:
|
|
self._active = self.results
|
|
expected_root = str(self.rollback_state.get("root_subvol", ""))
|
|
expected_var = str(self.rollback_state.get("var_subvol", "") or "")
|
|
cmdline = self._read_file(guest, "/proc/cmdline")
|
|
booted_subvol = parse_rootflags_subvol(cmdline)
|
|
fstab = self._read_file(guest, "/etc/fstab")
|
|
fstab_root = parse_fstab_subvol(fstab, "/")
|
|
fstab_var = parse_fstab_subvol(fstab, "/var")
|
|
ok = True
|
|
ok &= self._check(
|
|
"restored-root-generation",
|
|
bool(expected_root) and booted_subvol == expected_root,
|
|
f"booted subvol {booted_subvol!r} != expected {expected_root!r}",
|
|
)
|
|
ok &= self._check(
|
|
"restored-fstab-root",
|
|
bool(expected_root) and fstab_root == expected_root,
|
|
f"fstab / subvol {fstab_root!r} != {expected_root!r}",
|
|
)
|
|
ok &= self._check(
|
|
"restored-var-generation",
|
|
bool(expected_var) and fstab_var == expected_var,
|
|
f"fstab /var subvol {fstab_var!r} != {expected_var!r}",
|
|
)
|
|
# The running system must not still be the pre-rollback generation.
|
|
ok &= self._check(
|
|
"generation-advanced",
|
|
booted_subvol.startswith("@restore-") if booted_subvol else False,
|
|
f"booted subvol {booted_subvol!r} is not a restored generation",
|
|
)
|
|
return ok
|
|
|
|
def _step_package_files(self, guest: Guest) -> bool:
|
|
self._active = self.results
|
|
after = parse_package_dirs(
|
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
|
)
|
|
still_present = [d for d in self.added_package_dirs if d in after]
|
|
ok = self._check(
|
|
"package-db-restored",
|
|
not still_present,
|
|
f"package DB still lists {still_present}",
|
|
)
|
|
missing_files = []
|
|
for path in self.added_package_files:
|
|
result = self._run(guest, f"test -e {shlex_quote(path)}")
|
|
if result.exit_code == 0:
|
|
missing_files.append(path)
|
|
ok &= self._check(
|
|
"package-files-restored",
|
|
not missing_files,
|
|
f"package files survived the rollback: {missing_files[:5]}",
|
|
)
|
|
hostname = self._read_file(guest, HOSTNAME_FILE).strip()
|
|
ok &= self._check(
|
|
"breakage-reverted",
|
|
hostname == (self.original_hostname or ""),
|
|
f"{HOSTNAME_FILE} is {hostname!r}, expected {self.original_hostname!r}",
|
|
)
|
|
return ok
|
|
|
|
def _step_portage_db(self, guest: Guest) -> bool:
|
|
self._active = self.results
|
|
listing = parse_package_dirs(
|
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
|
)
|
|
ok = self._check(
|
|
"portage-db-generation-count",
|
|
listing == self.package_dirs_before,
|
|
f"/var/db/pkg has {len(listing)} entries, expected {len(self.package_dirs_before)}",
|
|
)
|
|
# Every entry must carry a CONTENTS file; an inconsistent database is
|
|
# how a mismatched root/var pairing shows up.
|
|
probe = self._run(
|
|
guest,
|
|
f"for d in {PKG_DB}/*/*; do [ -f \"$d/CONTENTS\" ] || echo \"$d\"; done",
|
|
)
|
|
missing_contents = [
|
|
line.strip() for line in probe.stdout.splitlines() if line.strip()
|
|
]
|
|
ok &= self._check(
|
|
"portage-db-contents",
|
|
not missing_contents,
|
|
f"package entries without CONTENTS: {missing_contents[:5]}",
|
|
)
|
|
return ok
|
|
|
|
def _step_kernel_modules(self, guest: Guest) -> bool:
|
|
self._active = self.results
|
|
kernel = parse_uname(self._run(guest, "uname -r").stdout)
|
|
modules = self._read_file(guest, "/lib/modules")
|
|
module_dirs = [line.strip() for line in modules.splitlines() if line.strip()]
|
|
return self._check(
|
|
"kernel-modules-compatible",
|
|
bool(kernel) and kernel in module_dirs,
|
|
f"no /lib/modules/{kernel} in the restored root ({module_dirs})",
|
|
)
|
|
|
|
def _step_home_survives(self, guest: Guest) -> bool:
|
|
self._active = self.results
|
|
readback = self._read_file(guest, self.marker_path).strip()
|
|
return self._check(
|
|
"home-data-survives",
|
|
readback == self.marker_token,
|
|
"the /home marker did not survive the rollback",
|
|
)
|
|
|
|
def _step_arm_recovery(self, guest: Guest) -> bool:
|
|
# Boot the recovery entry exactly once, then fall back to the restored
|
|
# root. This is a real systemd-boot interface, not a QEMU shortcut.
|
|
self._active = self.results
|
|
result = self._run(
|
|
guest, "sudo bootctl set-oneshot phoebus-recovery", timeout=120.0
|
|
)
|
|
return self._check("recovery-entry-armed", result.ok, result.failure_reason)
|
|
|
|
def _step_recovery(self, guest: Guest) -> bool:
|
|
self._active = self.recovery_results
|
|
cmdline = self._read_file(guest, "/proc/cmdline")
|
|
ok = self._check(
|
|
"recovery-entry-booted",
|
|
"systemd.unit=rescue.target" in cmdline,
|
|
"the recovery entry did not boot into rescue.target",
|
|
)
|
|
addresses = parse_network_addresses(self._run(guest, "ip -brief addr").stdout)
|
|
non_loopback = [a for a in addresses if a != "lo"]
|
|
ok &= self._check(
|
|
"recovery-no-network",
|
|
not non_loopback,
|
|
f"network interfaces were up: {non_loopback}",
|
|
)
|
|
# `sys` itself must work with no model: a status query is deterministic
|
|
# and offline.
|
|
status = self._run(guest, f"{self.sys_bin} undo status")
|
|
ok &= self._check(
|
|
"recovery-sys-offline",
|
|
status.ok,
|
|
status.failure_reason or "sys status failed offline",
|
|
)
|
|
ask = self._run(guest, f'{self.sys_bin} ask "show system status"', timeout=120.0)
|
|
combined = (ask.stdout + ask.stderr).lower()
|
|
ok &= self._check(
|
|
"recovery-no-model-fallback",
|
|
(not ask.ok) and "no model" in combined,
|
|
"sys ask did not report the deterministic no-model fallback",
|
|
)
|
|
return ok
|
|
|
|
|
|
def shlex_quote(value: str) -> str:
|
|
"""Minimal single-quote shell escaping for a known-safe path."""
|
|
return "'" + value.replace("'", "'\\''") + "'"
|
|
|
|
|
|
def validate_acceptance_disks(disks, test_dir) -> list:
|
|
"""Refuse anything but disposable qcow2 disks inside the test directory.
|
|
|
|
Kept here so the journey's safety entry point is unit-testable without QEMU.
|
|
"""
|
|
return vmsafety.validate_writable_drives(disks, test_dir)
|
|
|
|
|
|
__all__ = [
|
|
"SCENARIO_ROLLBACK",
|
|
"SCENARIO_RECOVERY",
|
|
"REQUIRED_INPUTS",
|
|
"EXPECTED_STEP_NAMES",
|
|
"CommandResult",
|
|
"SnapshotRecord",
|
|
"Guest",
|
|
"BootTransport",
|
|
"RollbackJourney",
|
|
"validate_acceptance_disks",
|
|
"extract_between",
|
|
"normalize_subvol",
|
|
"parse_snapshot_history",
|
|
"newest_committed_coordinated",
|
|
"parse_rollback_state",
|
|
"parse_rootflags_subvol",
|
|
"parse_fstab_subvol",
|
|
"parse_package_dirs",
|
|
"parse_contents_files",
|
|
"parse_uname",
|
|
"parse_network_addresses",
|
|
]
|