From fc36f637b449af56d3b7566e6cf0358b0d032ec8 Mon Sep 17 00:00:00 2001 From: Phoebus Date: Tue, 22 Sep 2026 07:56:11 -0700 Subject: [PATCH] test(rollback): add rollback and no-model recovery acceptance harness --- Makefile | 1 + docs/ROLLBACK-ACCEPTANCE.md | 156 ++++ .../crates/phoebus-installer/src/release.rs | 304 ++++++- installer/tests/vm/README.md | 4 + installer/tests/vm/rollback-driver.py | 419 +++++++++ installer/tests/vm/rollback_acceptance.py | 852 ++++++++++++++++++ .../tests/vm/test_rollback_acceptance.py | 497 ++++++++++ installer/tests/vm/vmsafety.py | 63 +- 8 files changed, 2253 insertions(+), 43 deletions(-) create mode 100644 docs/ROLLBACK-ACCEPTANCE.md create mode 100644 installer/tests/vm/rollback-driver.py create mode 100644 installer/tests/vm/rollback_acceptance.py create mode 100644 installer/tests/vm/test_rollback_acceptance.py diff --git a/Makefile b/Makefile index 871bcdf..65d39d5 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,7 @@ test-rust: test-vm: python3 installer/tests/vm/test_vmsafety.py + python3 installer/tests/vm/test_rollback_acceptance.py # The complete set of required CI gates. Every target here must actually run; # the workflow fails the job if a prerequisite is missing rather than skipping. diff --git a/docs/ROLLBACK-ACCEPTANCE.md b/docs/ROLLBACK-ACCEPTANCE.md new file mode 100644 index 0000000..f9bbbd6 --- /dev/null +++ b/docs/ROLLBACK-ACCEPTANCE.md @@ -0,0 +1,156 @@ +# Rollback and no-model-recovery acceptance + +This is the execution guide for the **rollback** (`updates-and-rollback`) and +**no-model recovery** (`no-model-recovery`) release gates. It describes how to +run the acceptance journey in the disposable VM harness, what it asserts, where +the evidence lands, and how a failure is diagnosed. + +> **Status: NOT YET RUN.** The harness and its offline tests exist and pass on a +> plain host, but the live journey has **not** been executed. Until a real run +> succeeds and its `results.json` is imported, both release gates stay +> `blocked`. A run made with a fake transport is marked `mocked` and is +> **rejected** by the release tool. + +## What the journey proves + +The installer writes a coordinated `@` + `@var` known-good snapshot at install +time. The journey then deliberately changes the disk and asks the real +`sys undo` to put it back. The ordered steps (from +`installer/tests/vm/rollback_acceptance.py`) are: + +1. `sys-binary-identity` — the installed `sys` matches the pinned digest. +2. `known-good-coordinated-snapshot` — a committed root snapshot with a paired + `@var` snapshot exists. +3. `home-data-marker` — a marker is written into `/home` (a separate subvolume). +4. `controlled-package-change` — a small package is installed, adding a + `/var/db/pkg` entry and files under `/`. +5. `deterministic-guest-breakage` — `/etc/hostname` is overwritten with a + sentinel. +6. `sys-undo-activates` — `sys undo ` writes an `activated` rollback state + for the next boot. +7. `reboot-detached` — the installed disk boots with the installer ISO absent. +8. `restored-root-var-generations` — the booted `rootflags`, `/etc/fstab` `/` + and `/var` subvolumes match the rollback state. +9. `package-files-restored` — the changed package's DB entry and files are gone, + and the sentinel hostname is reverted. +10. `portage-database-consistent` — `/var/db/pkg` equals its pre-change listing + and every entry has a `CONTENTS` file. +11. `kernel-modules-compatible` — the running kernel has a module tree in the + restored root. +12. `home-data-survives` — the `/home` marker is byte-identical. +13. `recovery-entry-armed` — `bootctl set-oneshot phoebus-recovery` succeeds. +14. `recovery-boot-no-network-no-model` — the recovery entry boots + `rescue.target`, no non-loopback interface is up, `sys` status works + offline, and `sys ask` reports the deterministic no-model fallback. + +The package changed in step 4 is controlled by `ROLLBACK_PACKAGE` (default +`app-misc/hello`); pick a small, signed binary package for a fast run. + +## Prerequisites + +- A **disposable qcow2** already installed by an existing harness run + (`serial-driver.py install` or `appliance-driver.py install`). This driver + never installs and never attaches the installer ISO. +- `/var/tmp/phoebus-vm/disks` (or `PHOEBUS_VM_DISKS`) has the disk. Never point + a writable disk at `/dev/*`; the harness refuses it. +- `qemu-system-x86_64`, `qemu-img`, OVMF firmware, Python `pexpect`. +- The candidate ISO and the `sys` binary under test, as files on the host, so + their digests can be pinned. +- Network access during the change phase (the package install); the recovery + phase uses `-net none`. + +## Running it + +Print the plan without starting a VM (safe on any host): + +```sh +python3 installer/tests/vm/rollback-driver.py plan +``` + +Run the full journey: + +```sh +PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \ +DISK=/var/tmp/phoebus-vm/disks/rollback.qcow2 \ +OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \ +OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \ +ISO=/var/tmp/phoebus-vm/phoebus-installer-amd64.iso \ +PHOEBUS_VM_SYS=/var/tmp/phoebus-vm/sys \ +PHOEBUS_VM_ARTIFACT=phoebus-installer-amd64.iso \ +PHOEBUS_VM_ARTIFACT_SHA256="$(sha256sum /var/tmp/phoebus-vm/phoebus-installer-amd64.iso | cut -d' ' -f1)" \ +PHOEBUS_VM_STRICT=1 \ +LUKS_PASSPHRASE=phoebus-luks-test USER_PASSWORD=phoebus-user-test \ +python3 installer/tests/vm/rollback-driver.py run +``` + +Re-verify only the recovery boot (boots normally once to arm the one-shot, then +reboots with no network): + +```sh +# same environment, then: +python3 installer/tests/vm/rollback-driver.py recovery +``` + +`PHOEBUS_VM_ARTIFACT[_SHA256]` is what binds the evidence to a candidate; the +release tool refuses unbound or stale evidence. `PHOEBUS_VM_SYS` is hashed and +pinned as the `sys` identity (override with `PHOEBUS_VM_SYS_SHA256`). The guest +must report the same digest in step 1. + +## Artifacts + +Under `$PHOEBUS_VM_STATE//` (default `/var/tmp/phoebus-vm/`): + +- `results.json` — the `rollback-acceptance` scenario. +- `recovery/results.json` — the `rollback-recovery` scenario (written by `run`). +- `serial-boot1.log`, `serial-boot2.log`, `serial-boot3.log` — per-boot serial + transcripts. +- `OVMF_VARS.fd` — the run's own writable firmware variables. + +`results.json` records `scenario`, `run_id`, `evidence` (`live`), `exit_code`, +`counts`, `artifact` (name + ISO digest), `inputs` (`iso` and `sys` digests) and +the per-check `checks` list. + +## Feeding the release gates + +Import the run directory when producing a gate report: + +```sh +installer-rs/target/release/phoebus-release gates --execute \ + --candidate /path/phoebus-installer-amd64.iso \ + --harness-results /var/tmp/phoebus-vm/ \ + --output gate-report.json +``` + +The `updates-and-rollback` gate resolves `rollback-acceptance`; the +`no-model-recovery` gate resolves `rollback-recovery`. A gate **fails** +(never passes) when the evidence is: + +- `mocked` (a fake-transport unit run), +- missing the `iso` or `sys` pinned input, +- bound to a different candidate digest (stale), +- unbound while a candidate is supplied, or +- present but with `exit_code != 0`. + +## Failure diagnosis + +| Symptom | Likely cause | +|---|---| +| `sys-binary-identity` fails | the installed `sys` is not the build under test; rebuild the ISO from the merged branch and re-install | +| `known-good-coordinated-snapshot` fails | the install did not record a paired `@var` snapshot, or `snapshot-history.toml` is unreadable (it is never treated as empty) | +| `controlled-package-change` fails/times out | no network, no signed binhost, or the atom needs a long compile; choose another `ROLLBACK_PACKAGE` | +| `sys-undo-activates` fails | `sys undo` refused (no committed snapshot, unreadable history) or the broker exited nonzero | +| `restored-root-var-generations` fails | `rootflags`/fstab were not rewritten to the restored generation | +| `portage-database-consistent` fails | root and `/var` came from different generations | +| `kernel-modules-compatible` fails | the selected boot entry does not match the restored `lib/modules` | +| `recovery-no-network` fails | QEMU was still given a network device; check `-net none` | +| `recovery-no-model-fallback` fails | a provider is configured, or `sys ask` did not fail closed offline | +| harness exits 2 with `disposable-disk` | the disk is not a disposable qcow2 inside `PHOEBUS_VM_DISKS` | + +## Integration note + +The branch this work is built on (`codex/rollback-acceptance`) is based on the +CI-hardening commit `1d37fbc` and does **not** include the sibling `sys` +hardening commit `4679f93`. The installed `sys` used in a live run must come +from a build that contains the sys hardening; otherwise step 1 +(`sys-binary-identity`) will correctly fail. Build the acceptance ISO after the +two branches are integrated. diff --git a/installer-rs/crates/phoebus-installer/src/release.rs b/installer-rs/crates/phoebus-installer/src/release.rs index 438e408..f036c5d 100644 --- a/installer-rs/crates/phoebus-installer/src/release.rs +++ b/installer-rs/crates/phoebus-installer/src/release.rs @@ -335,6 +335,18 @@ pub struct HarnessEvidence { pub counts: BTreeMap, #[serde(default)] pub artifact: Option, + /// How the run was produced: `live` for a real VM run, `mocked` for a + /// fake-transport unit run. A mocked run is never release evidence. + #[serde(default = "default_evidence_class")] + pub evidence: String, + /// Additional pinned inputs, keyed by name (e.g. `iso`, `sys`). A gate can + /// require these so evidence that names only the candidate is refused. + #[serde(default)] + pub inputs: BTreeMap, +} + +fn default_evidence_class() -> String { + "live".into() } impl HarnessEvidence { @@ -345,6 +357,28 @@ impl HarnessEvidence { .filter(|s| !s.is_empty()) } + /// The evidence class, treating a missing field as `live` (older fixture + /// JSON predates the field; every current driver writes it explicitly). + pub fn evidence_class(&self) -> &str { + if self.evidence.is_empty() { + "live" + } else { + &self.evidence + } + } + + /// Whether this is a real run that may be used as release evidence. + pub fn is_live(&self) -> bool { + self.evidence_class() == "live" + } + + pub fn input_sha256(&self, name: &str) -> Option<&str> { + self.inputs + .get(name) + .and_then(|a| a.sha256.as_deref()) + .filter(|s| !s.is_empty()) + } + pub fn from_json_str(text: &str) -> std::result::Result { let value: serde_json::Value = serde_json::from_str(text).map_err(|e| format!("invalid harness JSON: {e}"))?; @@ -374,12 +408,25 @@ impl HarnessEvidence { let artifact = value .get("artifact") .and_then(|a| serde_json::from_value::(a.clone()).ok()); + let evidence = value + .get("evidence") + .and_then(|v| v.as_str()) + .unwrap_or("live") + .to_string(); + let inputs = value + .get("inputs") + .and_then(|v| { + serde_json::from_value::>(v.clone()).ok() + }) + .unwrap_or_default(); Ok(Self { scenario, run_id, exit_code, counts, artifact, + evidence, + inputs, }) } @@ -801,6 +848,17 @@ fn candidate_env(candidate: Option<&CandidateRef>) -> Vec<(String, String)> { } } +/// Input identities a scenario's evidence must pin before it may pass. +/// +/// The rollback journey records both the booted ISO and the exact `sys` binary +/// it exercised; evidence that names only the candidate is incomplete. +fn required_inputs_for(scenario: &str) -> &'static [&'static str] { + match scenario { + "rollback-acceptance" | "rollback-recovery" => &["iso", "sys"], + _ => &[], + } +} + fn evidence_result_for( gate: &Gate, scenario: &str, @@ -814,42 +872,75 @@ fn evidence_result_for( detail: format!("no harness evidence found for scenario '{scenario}'"), log: None, }, - Some(found) => match (candidate_sha, found.artifact_sha256()) { - (Some(bound), Some(ev)) if bound != ev => GateResult { - name: gate.name.clone(), - status: GateStatus::Failed, - detail: format!( - "stale evidence: scenario '{scenario}' ran against {} but the candidate is {}", - short_digest(ev), - short_digest(bound) - ), - log: None, - }, - (Some(_), None) => GateResult { - name: gate.name.clone(), - status: GateStatus::Failed, - detail: format!( - "harness evidence for scenario '{scenario}' is not bound to an artifact; \ - re-run the harness with PHOEBUS_VM_ARTIFACT[_SHA256] set" - ), - log: None, - }, - _ if found.exit_code == 0 => GateResult { - name: gate.name.clone(), - status: GateStatus::Passed, - detail: format!("harness scenario '{scenario}' passed ({})", found.run_id), - log: None, - }, - _ => GateResult { - name: gate.name.clone(), - status: GateStatus::Failed, - detail: format!( - "harness scenario '{scenario}' failed (exit {})", - found.exit_code - ), - log: None, - }, - }, + Some(found) => { + // A mocked run proves the orchestration, never the system: it can + // never authorize a release. + if !found.is_live() { + return GateResult { + name: gate.name.clone(), + status: GateStatus::Failed, + detail: format!( + "mocked harness evidence for scenario '{scenario}' \ + (evidence={}) is not release evidence; run it on a real VM", + found.evidence_class() + ), + log: None, + }; + } + let missing: Vec<&str> = required_inputs_for(scenario) + .iter() + .copied() + .filter(|name| found.input_sha256(name).is_none()) + .collect(); + if !missing.is_empty() { + return GateResult { + name: gate.name.clone(), + status: GateStatus::Failed, + detail: format!( + "incomplete evidence: scenario '{scenario}' is missing pinned \ + input(s): {}", + missing.join(", ") + ), + log: None, + }; + } + match (candidate_sha, found.artifact_sha256()) { + (Some(bound), Some(ev)) if bound != ev => GateResult { + name: gate.name.clone(), + status: GateStatus::Failed, + detail: format!( + "stale evidence: scenario '{scenario}' ran against {} but the candidate is {}", + short_digest(ev), + short_digest(bound) + ), + log: None, + }, + (Some(_), None) => GateResult { + name: gate.name.clone(), + status: GateStatus::Failed, + detail: format!( + "harness evidence for scenario '{scenario}' is not bound to an artifact; \ + re-run the harness with PHOEBUS_VM_ARTIFACT[_SHA256] set" + ), + log: None, + }, + _ if found.exit_code == 0 => GateResult { + name: gate.name.clone(), + status: GateStatus::Passed, + detail: format!("harness scenario '{scenario}' passed ({})", found.run_id), + log: None, + }, + _ => GateResult { + name: gate.name.clone(), + status: GateStatus::Failed, + detail: format!( + "harness scenario '{scenario}' failed (exit {})", + found.exit_code + ), + log: None, + }, + } + } } } @@ -862,6 +953,7 @@ fn evidence_result_for( pub fn candidate_gates() -> Vec { let serial = "installer/tests/vm/serial-driver.py"; let gui = "installer/tests/vm/gui-driver.py"; + let rollback = "installer/tests/vm/rollback-driver.py"; vec![ Gate { name: "rust-fmt".into(), @@ -933,21 +1025,23 @@ pub fn candidate_gates() -> Vec { command: vec!["python3".into(), gui.into(), "opencode".into()], scenario: Some("gui-opencode".into()), }, - // No automated harness exists for these yet. They stay explicitly - // blocked until one does; an empty command can never be a pass. + // Rollback and no-model recovery are produced by the dedicated + // rollback driver. The gates stay `Blocked` until a real VM run records + // evidence that pins both the ISO and the `sys` binary under test; + // mocked unit runs are refused outright. Gate { name: "no-model-recovery".into(), description: "operate and recover with no model connected".into(), requires: GateRequirement::Vm, - command: Vec::new(), - scenario: None, + command: vec!["python3".into(), rollback.into(), "recovery".into()], + scenario: Some("rollback-recovery".into()), }, Gate { name: "updates-and-rollback".into(), description: "prepare, apply, roll back and reboot detached".into(), requires: GateRequirement::Vm, - command: Vec::new(), - scenario: None, + command: vec!["python3".into(), rollback.into(), "run".into()], + scenario: Some("rollback-acceptance".into()), }, Gate { name: "desktop-session".into(), @@ -1513,6 +1607,8 @@ mod tests { run_id: "r1".into(), exit_code: 0, counts: BTreeMap::new(), + evidence: "live".into(), + inputs: BTreeMap::new(), artifact: Some(HarnessArtifact { name: Some("phoebus.iso".into()), sha256: Some(candidate.sha256.clone()), @@ -1537,6 +1633,8 @@ mod tests { run_id: "old-run".into(), exit_code: 0, counts: BTreeMap::new(), + evidence: "live".into(), + inputs: BTreeMap::new(), artifact: Some(HarnessArtifact { name: Some("phoebus.iso".into()), sha256: Some( @@ -1568,6 +1666,8 @@ mod tests { run_id: "r1".into(), exit_code: 0, counts: BTreeMap::new(), + evidence: "live".into(), + inputs: BTreeMap::new(), artifact: None, }); let mut opts = execution_opts(); @@ -1593,6 +1693,8 @@ mod tests { run_id: "r1".into(), exit_code: 1, counts: BTreeMap::new(), + evidence: "live".into(), + inputs: BTreeMap::new(), artifact: Some(HarnessArtifact { name: Some("phoebus.iso".into()), sha256: Some(candidate.sha256.clone()), @@ -1605,6 +1707,124 @@ mod tests { assert_eq!(result(&report, "detached-boot").status, GateStatus::Failed); } + fn rollback_evidence( + candidate: &CandidateRef, + evidence: &str, + inputs: &[(&str, &str)], + ) -> HarnessEvidence { + HarnessEvidence { + scenario: "rollback-acceptance".into(), + run_id: "rollback-run".into(), + exit_code: 0, + counts: BTreeMap::new(), + artifact: Some(HarnessArtifact { + name: Some("phoebus.iso".into()), + sha256: Some(candidate.sha256.clone()), + }), + evidence: evidence.into(), + inputs: inputs + .iter() + .map(|(name, digest)| { + ( + (*name).to_string(), + HarnessArtifact { + name: None, + sha256: Some((*digest).to_string()), + }, + ) + }) + .collect(), + } + } + + #[test] + fn rollback_gates_name_the_dedicated_driver() { + let gates = candidate_gates(); + let rollback = gates + .iter() + .find(|g| g.name == "updates-and-rollback") + .expect("updates-and-rollback gate"); + assert_eq!(rollback.scenario.as_deref(), Some("rollback-acceptance")); + assert!( + rollback + .command + .iter() + .any(|a| a.contains("rollback-driver.py")), + "command: {:?}", + rollback.command + ); + let recovery = gates + .iter() + .find(|g| g.name == "no-model-recovery") + .expect("no-model-recovery gate"); + assert_eq!(recovery.scenario.as_deref(), Some("rollback-recovery")); + assert!(recovery.command.iter().any(|a| a == "recovery")); + } + + #[test] + fn mocked_rollback_evidence_is_never_a_pass() { + let dir = tempfile::tempdir().unwrap(); + let (_, candidate) = bind(dir.path(), "phoebus.iso", b"candidate"); + let mut set = HarnessEvidenceSet::new(); + set.push(rollback_evidence( + &candidate, + "mocked", + &[("iso", candidate.sha256.as_str()), ("sys", "bbbb")], + )); + let mut opts = execution_opts(); + opts.evidence = Some(&set); + opts.candidate = Some(candidate); + let report = run_gates(&FakeExecutor::new(), &opts); + let gate = result(&report, "updates-and-rollback"); + assert_eq!(gate.status, GateStatus::Failed); + assert!(gate.detail.contains("mocked"), "detail: {}", gate.detail); + } + + #[test] + fn incomplete_rollback_evidence_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let (_, candidate) = bind(dir.path(), "phoebus.iso", b"candidate"); + let mut set = HarnessEvidenceSet::new(); + // The ISO is pinned but the `sys` binary under test is not. + set.push(rollback_evidence( + &candidate, + "live", + &[("iso", candidate.sha256.as_str())], + )); + let mut opts = execution_opts(); + opts.evidence = Some(&set); + opts.candidate = Some(candidate); + let report = run_gates(&FakeExecutor::new(), &opts); + let gate = result(&report, "updates-and-rollback"); + assert_eq!(gate.status, GateStatus::Failed); + assert!( + gate.detail.contains("incomplete"), + "detail: {}", + gate.detail + ); + assert!(gate.detail.contains("sys"), "detail: {}", gate.detail); + } + + #[test] + fn complete_live_rollback_evidence_passes() { + let dir = tempfile::tempdir().unwrap(); + let (_, candidate) = bind(dir.path(), "phoebus.iso", b"candidate"); + let mut set = HarnessEvidenceSet::new(); + set.push(rollback_evidence( + &candidate, + "live", + &[("iso", candidate.sha256.as_str()), ("sys", "bbbb")], + )); + let mut opts = execution_opts(); + opts.evidence = Some(&set); + opts.candidate = Some(candidate); + let report = run_gates(&FakeExecutor::new(), &opts); + assert_eq!( + result(&report, "updates-and-rollback").status, + GateStatus::Passed + ); + } + fn publish_fixture( reverse: bool, ) -> ( diff --git a/installer/tests/vm/README.md b/installer/tests/vm/README.md index 7d16c75..6610913 100644 --- a/installer/tests/vm/README.md +++ b/installer/tests/vm/README.md @@ -8,6 +8,10 @@ touching a physical disk. The target is always a qcow2 file. - `run-vm.sh` interactive QEMU launcher (manual testing). - `install.sh` guest-side, non-interactive installation (run in the VM). - `serial-driver.py` host-side pexpect driver (`install` and `boot` modes). +- `rollback-driver.py` host-side rollback + no-model-recovery driver (`plan`, + `run` and `recovery` modes); see `docs/ROLLBACK-ACCEPTANCE.md`. +- `rollback_acceptance.py` the ordered journey logic, testable with fakes. +- `vmsafety.py` shared disk-safety, structured-result and QMP helpers. - `test.conf.example` installer configuration used by the driver. ## Host prerequisites diff --git a/installer/tests/vm/rollback-driver.py b/installer/tests/vm/rollback-driver.py new file mode 100644 index 0000000..979ebe2 --- /dev/null +++ b/installer/tests/vm/rollback-driver.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +"""Disposable-VM acceptance driver for rollback and no-model recovery. + +This is the *live* counterpart to ``rollback_acceptance.py``: it boots an already +installed qcow2 disk (the installer ISO is **never** attached), drives the +serial console, and runs the ordered journey in +:mod:`rollback_acceptance`. Its ``results.json`` feeds the +``updates-and-rollback`` and ``no-model-recovery`` release gates. + +Modes (the entry point is explicit; nothing runs by accident): + + plan print the steps, the pinned input identities and the QEMU safety + checks, then exit 0 without starting a VM. Safe on any host. + run the full journey: snapshot, package change, breakage, `sys undo`, + detached reboot, restoration checks, then a recovery boot with + networking disabled, and write both scenarios' evidence. + recovery re-verify only the no-model recovery boot. Boots normally once to + arm the recovery entry, then reboots with no network. Writes the + ``rollback-recovery`` evidence. + +Safety: every writable disk is validated by :mod:`vmsafety` before QEMU starts. +It must be a qcow2 file inside ``PHOEBUS_VM_DISKS``; ``/dev/*`` paths, block and +character devices, raw images, escaping symlinks and outside backing chains are +refused. The installer ISO is deliberately not a boot input here, and the +recovery boot uses ``-net none``. + +Environment (all paths must be local; no secrets on the command line): + + OVMF_CODE, OVMF_VARS UEFI firmware and variable template (required) + DISK (or DISKS) disposable qcow2 installed disk(s) + PHOEBUS_VM_DISKS directory the disks must live in + ISO installer ISO whose digest is pinned (required) + PHOEBUS_VM_SYS host path to the `sys` binary under test + PHOEBUS_VM_SYS_SHA256 optional expected digest (otherwise computed) + PHOEBUS_VM_ARTIFACT[_SHA256] release tool candidate binding (optional here) + LUKS_PASSPHRASE, USER_PASSWORD, USERNAME + SYS_BIN guest `sys` path (default /usr/local/bin/sys) + ROLLBACK_PACKAGE controlled package change (default app-misc/hello) + QEMU_BIN, VM_MEM, VM_SMP, PHOEBUS_VM_STATE, PHOEBUS_VM_STRICT + +Example (see docs/ROLLBACK-ACCEPTANCE.md): + + PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \\ + DISK=/var/tmp/phoebus-vm/disks/rollback.qcow2 \\ + OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \\ + OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \\ + ISO=/path/phoebus-installer-amd64.iso \\ + PHOEBUS_VM_SYS=/path/sys \\ + python3 installer/tests/vm/rollback-driver.py run +""" + +from __future__ import annotations + +import hashlib +import os +import re +import sys +from pathlib import Path + +import pexpect + +import vmsafety +from rollback_acceptance import ( + EXPECTED_STEP_NAMES, + REQUIRED_INPUTS, + RC_SENTINEL, + SCENARIO_RECOVERY, + SCENARIO_ROLLBACK, + CommandResult, + RollbackJourney, + validate_acceptance_disks, +) +from vmsafety import INPUT_ISO, INPUT_SYS, Results + +QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64") +LIBDIR = os.environ.get("QEMU_LIBDIR", "") +DATADIR = os.environ.get("QEMU_DATADIR", "") +OVMF_CODE = os.environ.get("OVMF_CODE", "") +OVMF_VARS_TEMPLATE = os.environ.get("OVMF_VARS", "") +DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "") +ISO = os.environ.get("ISO", "") +SYS_HOST = os.environ.get("PHOEBUS_VM_SYS", "") +SYS_BIN = os.environ.get("SYS_BIN", "/usr/local/bin/sys") +PACKAGE = os.environ.get("ROLLBACK_PACKAGE", "app-misc/hello") +HOME = os.environ.get("ROLLBACK_HOME", "/home/phoebus") +LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test") +USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test") +USER = os.environ.get("USERNAME", "phoebus") +MEM = os.environ.get("VM_MEM", "8192") +SMP = os.environ.get("VM_SMP", "8") +BOOT_TIMEOUT = int(os.environ.get("BOOT_TIMEOUT", "600")) +LOGIN_TIMEOUT = int(os.environ.get("LOGIN_TIMEOUT", "300")) +PROMPT = os.environ.get("GUEST_PROMPT", r"[$#] $") + + +def disposable_dir() -> Path: + base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm") + return Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks")) + + +def sha256_file(path: str | os.PathLike) -> str: + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def strip_echo(text: str, command: str) -> str: + """Drop the console echo of a typed command and the rc sentinel line.""" + lines = text.splitlines() + if lines and command and command[:40] in lines[0]: + lines = lines[1:] + return "\n".join(line for line in lines if RC_SENTINEL not in line) + + +class PexpectGuest: + """A :class:`rollback_acceptance.Guest` backed by a serial console.""" + + def __init__(self, child, log, user_pw: str): + self.child = child + self.log = log + self.user_pw = user_pw + + def _ensure_sudo(self) -> None: + """Refresh the sudo timestamp, feeding the password off the log. + + ``sudo -n true`` succeeds while the timestamp is valid; otherwise + ``sudo -v`` prompts and the password is sent with the pexpect logfile + detached, so it never reaches the serial transcript. + """ + self.child.sendline("sudo -n true 2>/dev/null || sudo -v") + idx = self.child.expect( + [r"\[sudo\] password for [^:]+:", PROMPT, pexpect.TIMEOUT], timeout=30 + ) + if idx == 0: + saved = self.child.logfile + self.child.logfile = None + try: + self.child.sendline(self.user_pw) + self.child.expect(PROMPT, timeout=30) + finally: + self.child.logfile = saved + elif idx == 1: + return + else: # pragma: no cover - console fault + raise RuntimeError("sudo timestamp could not be refreshed") + + def run(self, command: str, timeout: float = 180.0) -> CommandResult: + try: + if "sudo" in command: + self._ensure_sudo() + end = f"echo {RC_SENTINEL}=$(printf '%s' $?)" + self.child.sendline(f"{command}; {end}") + idx = self.child.expect( + [rf"{RC_SENTINEL}=([0-9]+)", pexpect.TIMEOUT, pexpect.EOF], + timeout=timeout, + ) + except (pexpect.TIMEOUT, pexpect.EOF): + return CommandResult(command=command, disconnected=True, stdout="") + if idx == 0: + return CommandResult( + command=command, + stdout=strip_echo(self.child.before or "", command), + exit_code=int(self.child.match.group(1)), + ) + if idx == 1: + return CommandResult( + command=command, timed_out=True, stdout=self.child.before or "" + ) + return CommandResult( + command=command, disconnected=True, stdout=self.child.before or "" + ) + + +class PexpectTransport: + """Boots the installed disk; the installer ISO is never attached.""" + + def __init__(self, disks, run_paths): + self.disks = disks + self.paths = run_paths + self.child = None + self.log = None + self._boot = 0 + + def _args(self, *, network: bool) -> list[str]: + env = dict(os.environ) + if LIBDIR: + env["LD_LIBRARY_PATH"] = LIBDIR + args = ["-machine", "q35,accel=kvm", "-cpu", "host", "-smp", SMP, "-m", MEM] + if DATADIR: + args += ["-L", DATADIR] + args += [ + "-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}", + "-drive", f"if=pflash,format=raw,file={self.paths.ovmf_vars}", + ] + for disk in self.disks: + args += ["-drive", f"file={disk},if=virtio,format=qcow2"] + if network: + args += ["-netdev", "user,id=n0", + "-device", "virtio-net-pci,netdev=n0,romfile="] + else: + # Recovery must work with networking disabled. + args += ["-net", "none"] + args += [ + "-device", "virtio-rng-pci", "-vga", "std", "-display", "none", + "-serial", "stdio", "-monitor", "none", "-no-reboot", + ] + return args + + def _spawn(self, *, network: bool) -> None: + self._boot += 1 + log_path = self.paths.directory / f"serial-boot{self._boot}.log" + self.log = open(log_path, "w") + env = dict(os.environ) + if LIBDIR: + env["LD_LIBRARY_PATH"] = LIBDIR + self.child = pexpect.spawn( + QEMU, + self._args(network=network), + env=env, + timeout=BOOT_TIMEOUT, + encoding="utf-8", + codec_errors="replace", + maxread=65536, + ) + self.child.delaybeforesend = 0.4 + self.child.logfile = self.log + + def _unlock_and_login(self) -> None: + assert self.child is not None + self.child.expect([r"[Pp]assphrase", r"login:"], timeout=BOOT_TIMEOUT) + if "assphrase" in (self.child.after or "").lower(): + self.child.sendline(LUKS_PW) + logged_in = False + for _ in range(5): + self.child.expect(fr"{re.escape(USER)} login:", timeout=LOGIN_TIMEOUT) + self.child.sendline(USER) + idx = self.child.expect( + [r"[Pp]assword:", fr"{re.escape(USER)} login:", r"Login incorrect"], + timeout=60, + ) + if idx == 0: + self.child.sendline(USER_PW) + if self.child.expect([PROMPT, r"Login incorrect", fr"{re.escape(USER)} login:"], + timeout=120) == 0: + logged_in = True + break + if not logged_in: + raise RuntimeError("could not log in on the installed system") + + def boot_installed(self): + self._spawn(network=True) + self._unlock_and_login() + return PexpectGuest(self.child, self.log, USER_PW) + + def boot_recovery(self): + self._spawn(network=False) + self._unlock_and_login() + return PexpectGuest(self.child, self.log, USER_PW) + + def poweroff(self) -> None: + if self.child is None: + return + try: + PexpectGuest(self.child, self.log, USER_PW).run("sudo poweroff -f", timeout=60) + except Exception: + pass + finally: + try: + self.child.close(force=True) + finally: + if self.log: + self.log.close() + self.child = None + self.log = None + + +def pin_inputs(results: Results, *, iso: str, sys_host: str) -> dict: + """Bind the ISO and `sys` identities the run exercised.""" + bound = {} + if iso: + results.bind_input_file(INPUT_ISO, iso) + bound[INPUT_ISO] = results.inputs[INPUT_ISO] + if sys_host: + expected = os.environ.get("PHOEBUS_VM_SYS_SHA256") + digest = expected or sha256_file(sys_host) + results.bind_input(INPUT_SYS, digest) + bound[INPUT_SYS] = results.inputs[INPUT_SYS] + results.bind_artifact_from_env() + return bound + + +def plan_text() -> str: + lines = [ + "rollback acceptance journey (no VM started)", + "scenario: " + SCENARIO_ROLLBACK, + "recovery: " + SCENARIO_RECOVERY, + "steps:", + ] + lines += [f" {i + 1:2d}. {name}" for i, name in enumerate(EXPECTED_STEP_NAMES)] + for scenario, required in REQUIRED_INPUTS.items(): + lines.append(f"required inputs for {scenario}: {', '.join(required)}") + lines.append("safety: every DISK must be a qcow2 inside PHOEBUS_VM_DISKS;") + lines.append(" the installer ISO is never attached; recovery uses -net none.") + return "\n".join(lines) + + +def build_results(scenario: str, mocked: bool = False) -> Results: + results = Results(scenario) + if mocked: + results.mark_mocked("fake transport") + return results + + +def run_full(disks, run_paths, sys_digest) -> int: + rollback = build_results(SCENARIO_ROLLBACK) + recovery = build_results(SCENARIO_RECOVERY) + pin_inputs(rollback, iso=ISO, sys_host=SYS_HOST) + pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST) + transport = PexpectTransport(disks, run_paths) + journey = RollbackJourney( + transport, + results=rollback, + recovery_results=recovery, + home=HOME, + sys_bin=SYS_BIN, + package=PACKAGE, + iso_sha256=os.environ.get("PHOEBUS_VM_ARTIFACT_SHA256") or (rollback.inputs.get(INPUT_ISO) or {}).get("sha256"), + sys_sha256=sys_digest, + ) + try: + journey.run() + finally: + transport.poweroff() + rollback.write(run_paths.results_json) + # The release tool imports any results.json under a directory tree, so + # the recovery scenario lives beside the rollback run. + recovery_dir = run_paths.directory / "recovery" + recovery_dir.mkdir(parents=True, exist_ok=True) + recovery.write(recovery_dir / "results.json") + print(rollback.summary()) + print(recovery.summary()) + return 1 if (rollback.exit_code() or recovery.exit_code()) else 0 + + +def run_recovery_only(disks, run_paths, sys_digest) -> int: + recovery = build_results(SCENARIO_RECOVERY) + pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST) + # Arm the one-shot recovery entry on a normal boot first. + transport = PexpectTransport(disks, run_paths) + try: + guest = transport.boot_installed() + guest.run("sudo bootctl set-oneshot phoebus-recovery", timeout=120) + finally: + transport.poweroff() + transport = PexpectTransport(disks, run_paths) + journey = RollbackJourney( + transport, + results=build_results(SCENARIO_ROLLBACK), + recovery_results=recovery, + home=HOME, + sys_bin=SYS_BIN, + package=PACKAGE, + sys_sha256=sys_digest, + ) + try: + journey.run_recovery() + finally: + transport.poweroff() + recovery.write(run_paths.results_json) + print(recovery.summary()) + return recovery.exit_code() + + +def main() -> int: + mode = sys.argv[1] if len(sys.argv) > 1 else "" + if mode == "plan": + print(plan_text()) + return 0 + if mode not in ("run", "recovery"): + print(__doc__) + return 2 + if not OVMF_CODE or not OVMF_VARS_TEMPLATE: + print("harness error: OVMF_CODE and OVMF_VARS are required", file=sys.stderr) + return 2 + if not ISO: + print("harness error: ISO is required to pin the candidate identity", file=sys.stderr) + return 2 + if not SYS_HOST or not Path(SYS_HOST).is_file(): + print( + "harness error: PHOEBUS_VM_SYS must point at the `sys` binary under test", + file=sys.stderr, + ) + return 2 + + scenario = SCENARIO_ROLLBACK if mode == "run" else SCENARIO_RECOVERY + run_paths = vmsafety.make_run_paths( + f"rollback-{mode}", + base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"), + ovmf_vars_template=OVMF_VARS_TEMPLATE, + ) + try: + disks = validate_acceptance_disks(DISKS.split(","), disposable_dir()) + except vmsafety.UnsafeDiskError as exc: + results = build_results(scenario) + results.blocked("disposable-disk", str(exc)) + results.write(run_paths.results_json) + print(f"harness error: {exc}", file=sys.stderr) + return 2 + + sys_digest = os.environ.get("PHOEBUS_VM_SYS_SHA256") or sha256_file(SYS_HOST) + if mode == "run": + return run_full(disks, run_paths, sys_digest) + return run_recovery_only(disks, run_paths, sys_digest) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/installer/tests/vm/rollback_acceptance.py b/installer/tests/vm/rollback_acceptance.py new file mode 100644 index 0000000..7ede984 --- /dev/null +++ b/installer/tests/vm/rollback_acceptance.py @@ -0,0 +1,852 @@ +#!/usr/bin/env python3 +"""Offline orchestration for the rollback + no-model-recovery acceptance run. + +This module holds the *logic* of the journey: the ordered steps, the guest +commands they issue, and the assertions that decide PASS/FAIL. It deliberately +imports neither pexpect nor QEMU, so the whole journey can be exercised on a +plain host against a fake transport (see ``test_rollback_acceptance.py``). + +The real VM driver (``rollback-driver.py``) supplies a pexpect-backed transport +implementing :class:`BootTransport` and a :class:`Guest` that runs commands over +the serial console. The release gates import the ``results.json`` these runs +write; a run made with a fake transport is marked ``mocked`` and can never be a +gate pass. + +Journey shape (the installer already created a coordinated ``@`` + ``@var`` +known-good snapshot at install time): + + boot installed (ISO detached) + 1. identify the newest committed coordinated snapshot + 2. write a marker into /home (a subvolume rollback must not touch) + 3. apply a controlled package change (touches / and /var/db/pkg) + 4. introduce deterministic guest-only breakage in / + 5. run the real `sys undo` (restores root + var as the next boot root) + reboot, boot installed (ISO detached) + 6. restored root/var generations match the rollback state + 7. the changed package and its files are gone + 8. /var/db/pkg matches / (consistent package database) + 9. the booted kernel has matching modules in the restored root + 10. /home data survived + arm the recovery entry, reboot, boot recovery (no network, no model) + 11. recovery entry boots, no network, and `sys` still works with no model + +Nothing here decides that a *live* run passed: only a real transport on a +disposable qcow2 disk can produce release evidence. The mocked run exists to +prove the orchestration itself. +""" + +from __future__ import annotations + +import dataclasses +import re +import tomllib +from typing import Callable, Protocol, runtime_checkable + +import vmsafety +from vmsafety import INPUT_ISO, INPUT_SYS, Results + +# Scenario names. Each is a release-gate evidence key. +SCENARIO_ROLLBACK = "rollback-acceptance" +SCENARIO_RECOVERY = "rollback-recovery" + +# The input identities a gate must see before it may pass. The rollback +# semantics live in `sys`, so naming only the ISO is incomplete evidence. +REQUIRED_INPUTS: dict[str, tuple[str, ...]] = { + SCENARIO_ROLLBACK: (INPUT_ISO, INPUT_SYS), + SCENARIO_RECOVERY: (INPUT_ISO, INPUT_SYS), +} + +# Stable guest paths and sentinels. +SNAPSHOT_HISTORY = "/var/lib/phoebus/snapshot-history.toml" +ROLLBACK_STATE = "/var/lib/phoebus/rollback-state.toml" +PKG_DB = "/var/db/pkg" +HOSTNAME_FILE = "/etc/hostname" +HOME_MARKER_REL = "rollback-acceptance/marker" +HOME_MARKER_TOKEN = "phoebus-rollback-home-marker" +HOSTNAME_SENTINEL = "phoebus-rollback-broken-hostname" +DEFAULT_SYS = "/usr/local/bin/sys" +DEFAULT_PACKAGE = "app-misc/hello" + +# File framing so a read is not confused with console noise. +FILE_FRAME = "__PHOEBUS_RB_FILE__" +# The driver appends this and parses the exit status from the console. +RC_SENTINEL = "__PHOEBUS_RB_RC" + +MAX_OUTPUT_BYTES = 64 * 1024 + + +# --------------------------------------------------------------------------- +# Small, testable value types +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class CommandResult: + """The outcome of one guest command. + + ``timed_out`` and ``disconnected`` are distinct from a nonzero exit so a + broken connection can never be mistaken for a clean failure that a later + step might tolerate. + """ + + command: str + stdout: str = "" + stderr: str = "" + exit_code: int | None = None + timed_out: bool = False + disconnected: bool = False + + def bounded(self) -> "CommandResult": + combined = self.stdout[-MAX_OUTPUT_BYTES:] + self.stdout = combined + self.stderr = self.stderr[-MAX_OUTPUT_BYTES:] + return self + + @property + def ok(self) -> bool: + return ( + not self.timed_out + and not self.disconnected + and self.exit_code == 0 + ) + + @property + def failure_reason(self) -> str: + if self.timed_out: + return f"timeout running `{self.command}`" + if self.disconnected: + return f"guest disconnected running `{self.command}`" + if self.exit_code is None: + return f"no exit status for `{self.command}`" + if self.exit_code != 0: + detail = (self.stderr or self.stdout).strip().splitlines() + tail = detail[-1] if detail else "" + return f"`{self.command}` exited {self.exit_code}: {tail}" + return "" + + +@dataclasses.dataclass +class SnapshotRecord: + id: str + operation: str + snapshot_subvol: str + var_snapshot: str | None + status: str + + +@runtime_checkable +class Guest(Protocol): + """A shell on a booted installed system.""" + + def run(self, command: str, timeout: float = 180.0) -> CommandResult: + ... + + +@runtime_checkable +class BootTransport(Protocol): + """Boots the installed disk in the requested mode. + + * ``boot_installed`` boots the normal entry with the installer ISO detached. + * ``boot_recovery`` boots the recovery entry with networking disabled. + """ + + def boot_installed(self) -> Guest: + ... + + def boot_recovery(self) -> Guest: + ... + + def poweroff(self) -> None: + ... + + +# --------------------------------------------------------------------------- +# Parsers (pure functions, unit-tested directly) +# --------------------------------------------------------------------------- + + +def extract_between(text: str, start: str, end: str) -> str: + """Return the text between the first ``start`` and the next ``end``. + + If the markers are absent the whole text is returned unchanged, so a fake + transport may return clean content directly. + """ + if start not in text or end not in text: + return text + body = text.split(start, 1)[1] + return body.split(end, 1)[0] + + +def parse_snapshot_history(text: str) -> list[SnapshotRecord]: + """Parse the snapshot history TOML into records. + + Raises :class:`ValueError` when the document is unreadable; a caller must + never treat "could not parse" as "no snapshots". + """ + text = text.strip() + if not text: + raise ValueError("empty snapshot history") + try: + data = tomllib.loads(text) + except tomllib.TOMLDecodeError as exc: # pragma: no cover - message detail + raise ValueError(f"invalid snapshot history: {exc}") from exc + records = data.get("records", []) + if not isinstance(records, list): + raise ValueError("snapshot history has no `records` list") + out: list[SnapshotRecord] = [] + for entry in records: + if not isinstance(entry, dict): + continue + out.append( + SnapshotRecord( + id=str(entry.get("id", "")), + operation=str(entry.get("operation", "")), + snapshot_subvol=str(entry.get("snapshot_subvol", "")), + var_snapshot=entry.get("var_snapshot"), + status=str(entry.get("status", "")), + ) + ) + return out + + +def newest_committed_coordinated( + records: list[SnapshotRecord], +) -> SnapshotRecord | None: + """The newest committed snapshot that also captured ``/var``. + + A root snapshot with no paired ``@var`` snapshot cannot restore a + consistent package database, so it is not a valid rollback target. + """ + candidates = [ + r + for r in records + if r.status == "committed" and r.snapshot_subvol and r.var_snapshot + ] + if not candidates: + return None + return candidates[-1] + + +def parse_rollback_state(text: str) -> dict: + """Parse ``/var/lib/phoebus/rollback-state.toml`` into a dict.""" + text = text.strip() + if not text: + raise ValueError("empty rollback state") + try: + return tomllib.loads(text) + except tomllib.TOMLDecodeError as exc: # pragma: no cover - message detail + raise ValueError(f"invalid rollback state: {exc}") from exc + + +def normalize_subvol(value: str | None) -> str | None: + """Canonicalise a subvolume path for comparison. + + Btrfs reports the same subvolume as ``@restore-x`` in ``rootflags`` and + ``/@restore-x`` in a fstab mount option; comparison must ignore that. + """ + if value is None: + return None + return value.lstrip("/") + + +def parse_rootflags_subvol(cmdline: str) -> str | None: + """Extract the ``subvol=`` value from the kernel ``rootflags=`` option.""" + match = re.search(r"rootflags=[^ ]*?\bsubvol=([^,\s]+)", cmdline) + if match: + return normalize_subvol(match.group(1)) + # Some kernels present rootflags only as the bare option. + match = re.search(r"\bsubvol=([^,\s]+)", cmdline) + return normalize_subvol(match.group(1)) if match else None + + +def parse_fstab_subvol(text: str, mountpoint: str) -> str | None: + """The ``subvol=`` value for a mountpoint in ``/etc/fstab``.""" + for line in text.splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + fields = line.split() + if len(fields) < 4: + continue + if fields[1] != mountpoint: + continue + match = re.search(r"(?:^|,)subvol=([^,\s]+)", fields[3]) + if match: + return normalize_subvol(match.group(1)) + return None + + +def parse_package_dirs(text: str) -> list[str]: + """Absolute ``/var/db/pkg//`` directories from a listing.""" + return sorted({line.strip() for line in text.splitlines() if line.strip()}) + + +def parse_contents_files(text: str) -> list[str]: + """Installed file paths from a Portage ``CONTENTS`` file. + + The format is `` `` per line. + """ + paths: list[str] = [] + for line in text.splitlines(): + fields = line.split() + if len(fields) >= 2 and fields[0] in ("obj", "dir", "sym"): + paths.append(fields[1]) + return paths + + +def parse_uname(text: str) -> str: + return text.strip().splitlines()[-1].strip() if text.strip() else "" + + +def parse_network_addresses(text: str) -> list[str]: + """Interfaces that have an address, from ``ip -brief addr`` output.""" + out: list[str] = [] + for line in text.splitlines(): + fields = line.split() + if not fields: + continue + name = fields[0] + rest = fields[1:] + if any("." in f or ":" in f for f in rest): + out.append(name) + return out + + +# --------------------------------------------------------------------------- +# Journey +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class _Step: + name: str + method_name: str + boot: str | None = None + stream: str = "rollback" + + +# Order is the contract. A change here changes the release gate; keep every +# expected step in the list so a stopped run can mark the rest NOT RUN. +_STEPS: tuple[_Step, ...] = ( + _Step("sys-binary-identity", "_step_sys_identity", boot="installed"), + _Step("known-good-coordinated-snapshot", "_step_known_good"), + _Step("home-data-marker", "_step_mark_home"), + _Step("controlled-package-change", "_step_package_change"), + _Step("deterministic-guest-breakage", "_step_break"), + _Step("sys-undo-activates", "_step_undo"), + _Step("reboot-detached", "_step_reboot", boot="installed"), + _Step("restored-root-var-generations", "_step_generations"), + _Step("package-files-restored", "_step_package_files"), + _Step("portage-database-consistent", "_step_portage_db"), + _Step("kernel-modules-compatible", "_step_kernel_modules"), + _Step("home-data-survives", "_step_home_survives"), + _Step("recovery-entry-armed", "_step_arm_recovery"), + _Step( + "recovery-boot-no-network-no-model", + "_step_recovery", + boot="recovery", + stream="recovery", + ), +) + +EXPECTED_STEP_NAMES: tuple[str, ...] = tuple(step.name for step in _STEPS) + + +class RollbackJourney: + """Run the ordered acceptance steps against a :class:`BootTransport`. + + The class records every expected step, so a run that stops early reports the + remainder as NOT RUN and fails an explicit ``journey-complete`` check rather + than silently looking like a pass. + """ + + def __init__( + self, + transport: BootTransport, + *, + results: Results | None = None, + recovery_results: Results | None = None, + home: str = "/home/phoebus", + sys_bin: str = DEFAULT_SYS, + package: str = DEFAULT_PACKAGE, + iso_sha256: str | None = None, + sys_sha256: str | None = None, + mocked: bool = False, + ): + self.transport = transport + self.results = results or Results(SCENARIO_ROLLBACK) + self.recovery_results = recovery_results or Results(SCENARIO_RECOVERY) + for stream in (self.results, self.recovery_results): + if iso_sha256: + stream.bind_input(INPUT_ISO, iso_sha256) + if sys_sha256: + stream.bind_input(INPUT_SYS, sys_sha256) + if mocked: + stream.mark_mocked("fake transport: not release evidence") + self.home = home + self.sys_bin = sys_bin + self.package = package + self.expected_sys_sha256 = sys_sha256 + self.marker_path = f"{home.rstrip('/')}/{HOME_MARKER_REL}" + self.marker_token = f"{HOME_MARKER_TOKEN}:{self.results.run_id}" + self.boots: list[str] = [] + self.executed: list[str] = [] + + # Findings carried between steps. + self.snapshot: SnapshotRecord | None = None + self.original_hostname: str | None = None + self.package_dirs_before: list[str] = [] + self.added_package_dirs: list[str] = [] + self.added_package_files: list[str] = [] + self.rollback_state: dict = {} + + self._active = self.results + + # -- helpers ----------------------------------------------------------- + def _check(self, name: str, condition: bool, detail: str = "") -> bool: + return self._active.assert_true(name, condition, detail) + + def _run( + self, guest: Guest, script: str, timeout: float = 240.0 + ) -> CommandResult: + try: + return guest.run(script, timeout=timeout).bounded() + except Exception as exc: # a transport fault is a failure, not a crash + return CommandResult( + command=script, disconnected=True, stderr=str(exc) + ) + + def _read_file(self, guest: Guest, path: str, timeout: float = 60.0) -> str: + result = self._run( + guest, + f"echo {FILE_FRAME}; cat {path} 2>/dev/null; echo {FILE_FRAME}", + timeout=timeout, + ) + return extract_between(result.stdout, FILE_FRAME, FILE_FRAME) + + # -- public API -------------------------------------------------------- + def required_inputs(self) -> tuple[str, ...]: + return REQUIRED_INPUTS[self.results.scenario] + + def run(self) -> bool: + """Execute the journey. Returns True only if every step passed.""" + if not self._check_inputs(): + for step in _STEPS: + stream = ( + self.results + if step.stream == "rollback" + else self.recovery_results + ) + stream.mark_not_run(step.name, "input identities were not pinned") + self.results.fail("journey-complete", "missing input identities") + self.recovery_results.fail("journey-complete", "missing input identities") + return False + + guest: Guest | None = None + for index, step in enumerate(_STEPS): + self._active = ( + self.results if step.stream == "rollback" else self.recovery_results + ) + if step.boot is not None: + if guest is not None: + self.transport.poweroff() + guest = ( + self.transport.boot_recovery() + if step.boot == "recovery" + else self.transport.boot_installed() + ) + self.boots.append(step.boot) + method: Callable[[Guest], bool] = getattr(self, step.method_name) + self.executed.append(step.name) + if not method(guest): + self._finish_incomplete(index, step.name) + return False + + self.results.assert_true("journey-complete", True) + self.recovery_results.assert_true("journey-complete", True) + return True + + def run_recovery(self) -> bool: + """Verify only the no-model recovery boot (used by the `recovery` mode).""" + if not self._check_inputs(): + self.recovery_results.mark_not_run( + "recovery-boot-no-network-no-model", "input identities were not pinned" + ) + self.recovery_results.fail("journey-complete", "missing input identities") + return False + self._active = self.recovery_results + ok = self._step_recovery(self.transport.boot_recovery()) + if ok: + self.recovery_results.assert_true("journey-complete", True) + else: + self.recovery_results.fail( + "journey-complete", "recovery verification failed" + ) + return ok + + # -- input identity gate ---------------------------------------------- + def _check_inputs(self) -> bool: + ok = True + for stream, scenario in ( + (self.results, SCENARIO_ROLLBACK), + (self.recovery_results, SCENARIO_RECOVERY), + ): + required = REQUIRED_INPUTS[scenario] + missing = stream.missing_inputs(required) + ok &= stream.assert_true( + "input-identities", + not missing, + f"missing pinned inputs for {scenario}: {', '.join(missing)}", + ) + return bool(ok) + + def _finish_incomplete(self, failed_index: int, failed_step: str) -> None: + for step in _STEPS[failed_index + 1:]: + stream = ( + self.results if step.stream == "rollback" else self.recovery_results + ) + stream.mark_not_run(step.name, "journey stopped before this step") + # The failed step's own stream already has a FAIL; make the incompleteness + # explicit on both so neither gate can read a partial run as a pass. + self.results.fail( + "journey-complete", f"stopped after `{failed_step}`" + ) + self.recovery_results.fail( + "journey-complete", f"stopped after `{failed_step}`" + ) + + # -- steps ------------------------------------------------------------- + def _step_sys_identity(self, guest: Guest) -> bool: + # The rollback semantics live in `sys`; pin the exact binary under test. + self._active = self.results + expected = self.expected_sys_sha256 or "" + result = self._run(guest, f"sha256sum {shlex_quote(self.sys_bin)}", timeout=60.0) + if not result.ok: + return self._check("sys-binary-identity", False, result.failure_reason) + digest = result.stdout.split()[0] if result.stdout.split() else "" + return self._check( + "sys-binary-identity", + bool(expected) and digest == expected, + f"guest sys digest {digest!r} != pinned {expected!r}", + ) + + def _step_known_good(self, guest: Guest) -> bool: + self._active = self.results + text = self._read_file(guest, SNAPSHOT_HISTORY) + try: + records = parse_snapshot_history(text) + except ValueError as exc: + return self._check("known-good-coordinated-snapshot", False, str(exc)) + snapshot = newest_committed_coordinated(records) + if snapshot is None: + return self._check( + "known-good-coordinated-snapshot", + False, + "no committed snapshot with a paired @var snapshot", + ) + self.snapshot = snapshot + self._check("known-good-root-snapshot", bool(snapshot.snapshot_subvol), "empty root snapshot path") + self._check("known-good-var-snapshot", bool(snapshot.var_snapshot), "empty @var snapshot path") + self._check( + "known-good-coordinated-snapshot", + True, + f"{snapshot.id} ({snapshot.operation})", + ) + return True + + def _step_mark_home(self, guest: Guest) -> bool: + self._active = self.results + result = self._run( + guest, + f"mkdir -p {self.home.rstrip('/')}/{HOME_MARKER_REL.rsplit('/', 1)[0]} && " + f"printf '%s' '{self.marker_token}' > {self.marker_path}", + ) + if not result.ok: + return self._check("home-data-marker", False, result.failure_reason) + readback = self._read_file(guest, self.marker_path) + if readback.strip() != self.marker_token: + return self._check( + "home-data-marker", False, "home marker did not read back" + ) + self.original_hostname = self._read_file(guest, HOSTNAME_FILE).strip() + self.package_dirs_before = parse_package_dirs( + self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout + ) + self._check("home-data-marker", True, self.marker_path) + self._check( + "hostname-recorded", bool(self.original_hostname), "could not read /etc/hostname" + ) + return bool(self.original_hostname) + + def _step_package_change(self, guest: Guest) -> bool: + self._active = self.results + result = self._run( + guest, f"sudo {self.sys_bin} install {self.package} -y", timeout=1800.0 + ) + if not result.ok: + return self._check("controlled-package-change", False, result.failure_reason) + after = parse_package_dirs( + self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout + ) + added = sorted(set(after) - set(self.package_dirs_before)) + if not added: + return self._check( + "controlled-package-change", + False, + f"`{self.package}` produced no new /var/db/pkg entry", + ) + self.added_package_dirs = added + contents = self._read_file(guest, f"{added[0]}/CONTENTS") + self.added_package_files = parse_contents_files(contents) + self._check( + "controlled-package-change", True, f"added {', '.join(added)}" + ) + self._check( + "package-db-recorded", + bool(self.added_package_files), + "changed package has no recorded files", + ) + return True + + def _step_break(self, guest: Guest) -> bool: + self._active = self.results + result = self._run( + guest, + f"printf '%s\\n' '{HOSTNAME_SENTINEL}' | sudo tee {HOSTNAME_FILE} >/dev/null", + ) + if not result.ok: + return self._check("deterministic-guest-breakage", False, result.failure_reason) + broken = self._read_file(guest, HOSTNAME_FILE).strip() + if broken != HOSTNAME_SENTINEL: + return self._check( + "deterministic-guest-breakage", + False, + "breakage did not take effect", + ) + self._check( + "deterministic-guest-breakage", + True, + f"{HOSTNAME_FILE} -> {HOSTNAME_SENTINEL}", + ) + return True + + def _step_undo(self, guest: Guest) -> bool: + self._active = self.results + if self.snapshot is None: + return self._check("sys-undo-activates", False, "no snapshot selected") + result = self._run( + guest, f"sudo {self.sys_bin} undo {self.snapshot.id} -y", timeout=900.0 + ) + if not result.ok: + return self._check("sys-undo-activates", False, result.failure_reason) + state = self._read_file(guest, ROLLBACK_STATE) + try: + self.rollback_state = parse_rollback_state(state) + except ValueError as exc: + return self._check("sys-undo-activates", False, str(exc)) + if self.rollback_state.get("status") != "activated": + return self._check( + "sys-undo-activates", + False, + f"rollback state is {self.rollback_state.get('status')!r}, not 'activated'", + ) + self._check( + "sys-undo-activates", + True, + f"next boot uses {self.rollback_state.get('root_subvol')}", + ) + return True + + def _step_reboot(self, guest: Guest) -> bool: + # The boot itself is performed by the transport; the driver asserts the + # ISO is detached before this returns. Here we only record the boundary. + self._active = self.results + self._check("reboot-detached", True, "booted installed disk with ISO detached") + return True + + def _step_generations(self, guest: Guest) -> bool: + self._active = self.results + expected_root = str(self.rollback_state.get("root_subvol", "")) + expected_var = str(self.rollback_state.get("var_subvol", "") or "") + cmdline = self._read_file(guest, "/proc/cmdline") + booted_subvol = parse_rootflags_subvol(cmdline) + fstab = self._read_file(guest, "/etc/fstab") + fstab_root = parse_fstab_subvol(fstab, "/") + fstab_var = parse_fstab_subvol(fstab, "/var") + ok = True + ok &= self._check( + "restored-root-generation", + bool(expected_root) and booted_subvol == expected_root, + f"booted subvol {booted_subvol!r} != expected {expected_root!r}", + ) + ok &= self._check( + "restored-fstab-root", + bool(expected_root) and fstab_root == expected_root, + f"fstab / subvol {fstab_root!r} != {expected_root!r}", + ) + ok &= self._check( + "restored-var-generation", + bool(expected_var) and fstab_var == expected_var, + f"fstab /var subvol {fstab_var!r} != {expected_var!r}", + ) + # The running system must not still be the pre-rollback generation. + ok &= self._check( + "generation-advanced", + booted_subvol.startswith("@restore-") if booted_subvol else False, + f"booted subvol {booted_subvol!r} is not a restored generation", + ) + return ok + + def _step_package_files(self, guest: Guest) -> bool: + self._active = self.results + after = parse_package_dirs( + self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout + ) + still_present = [d for d in self.added_package_dirs if d in after] + ok = self._check( + "package-db-restored", + not still_present, + f"package DB still lists {still_present}", + ) + missing_files = [] + for path in self.added_package_files: + result = self._run(guest, f"test -e {shlex_quote(path)}") + if result.exit_code == 0: + missing_files.append(path) + ok &= self._check( + "package-files-restored", + not missing_files, + f"package files survived the rollback: {missing_files[:5]}", + ) + hostname = self._read_file(guest, HOSTNAME_FILE).strip() + ok &= self._check( + "breakage-reverted", + hostname == (self.original_hostname or ""), + f"{HOSTNAME_FILE} is {hostname!r}, expected {self.original_hostname!r}", + ) + return ok + + def _step_portage_db(self, guest: Guest) -> bool: + self._active = self.results + listing = parse_package_dirs( + self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout + ) + ok = self._check( + "portage-db-generation-count", + listing == self.package_dirs_before, + f"/var/db/pkg has {len(listing)} entries, expected {len(self.package_dirs_before)}", + ) + # Every entry must carry a CONTENTS file; an inconsistent database is + # how a mismatched root/var pairing shows up. + probe = self._run( + guest, + f"for d in {PKG_DB}/*/*; do [ -f \"$d/CONTENTS\" ] || echo \"$d\"; done", + ) + missing_contents = [ + line.strip() for line in probe.stdout.splitlines() if line.strip() + ] + ok &= self._check( + "portage-db-contents", + not missing_contents, + f"package entries without CONTENTS: {missing_contents[:5]}", + ) + return ok + + def _step_kernel_modules(self, guest: Guest) -> bool: + self._active = self.results + kernel = parse_uname(self._run(guest, "uname -r").stdout) + modules = self._read_file(guest, "/lib/modules") + module_dirs = [line.strip() for line in modules.splitlines() if line.strip()] + return self._check( + "kernel-modules-compatible", + bool(kernel) and kernel in module_dirs, + f"no /lib/modules/{kernel} in the restored root ({module_dirs})", + ) + + def _step_home_survives(self, guest: Guest) -> bool: + self._active = self.results + readback = self._read_file(guest, self.marker_path).strip() + return self._check( + "home-data-survives", + readback == self.marker_token, + "the /home marker did not survive the rollback", + ) + + def _step_arm_recovery(self, guest: Guest) -> bool: + # Boot the recovery entry exactly once, then fall back to the restored + # root. This is a real systemd-boot interface, not a QEMU shortcut. + self._active = self.results + result = self._run( + guest, "sudo bootctl set-oneshot phoebus-recovery", timeout=120.0 + ) + return self._check("recovery-entry-armed", result.ok, result.failure_reason) + + def _step_recovery(self, guest: Guest) -> bool: + self._active = self.recovery_results + cmdline = self._read_file(guest, "/proc/cmdline") + ok = self._check( + "recovery-entry-booted", + "systemd.unit=rescue.target" in cmdline, + "the recovery entry did not boot into rescue.target", + ) + addresses = parse_network_addresses(self._run(guest, "ip -brief addr").stdout) + non_loopback = [a for a in addresses if a != "lo"] + ok &= self._check( + "recovery-no-network", + not non_loopback, + f"network interfaces were up: {non_loopback}", + ) + # `sys` itself must work with no model: a status query is deterministic + # and offline. + status = self._run(guest, f"{self.sys_bin} undo status") + ok &= self._check( + "recovery-sys-offline", + status.ok, + status.failure_reason or "sys status failed offline", + ) + ask = self._run(guest, f'{self.sys_bin} ask "show system status"', timeout=120.0) + combined = (ask.stdout + ask.stderr).lower() + ok &= self._check( + "recovery-no-model-fallback", + (not ask.ok) and "no model" in combined, + "sys ask did not report the deterministic no-model fallback", + ) + return ok + + +def shlex_quote(value: str) -> str: + """Minimal single-quote shell escaping for a known-safe path.""" + return "'" + value.replace("'", "'\\''") + "'" + + +def validate_acceptance_disks(disks, test_dir) -> list: + """Refuse anything but disposable qcow2 disks inside the test directory. + + Kept here so the journey's safety entry point is unit-testable without QEMU. + """ + return vmsafety.validate_writable_drives(disks, test_dir) + + +__all__ = [ + "SCENARIO_ROLLBACK", + "SCENARIO_RECOVERY", + "REQUIRED_INPUTS", + "EXPECTED_STEP_NAMES", + "CommandResult", + "SnapshotRecord", + "Guest", + "BootTransport", + "RollbackJourney", + "validate_acceptance_disks", + "extract_between", + "normalize_subvol", + "parse_snapshot_history", + "newest_committed_coordinated", + "parse_rollback_state", + "parse_rootflags_subvol", + "parse_fstab_subvol", + "parse_package_dirs", + "parse_contents_files", + "parse_uname", + "parse_network_addresses", +] diff --git a/installer/tests/vm/test_rollback_acceptance.py b/installer/tests/vm/test_rollback_acceptance.py new file mode 100644 index 0000000..e22b525 --- /dev/null +++ b/installer/tests/vm/test_rollback_acceptance.py @@ -0,0 +1,497 @@ +#!/usr/bin/env python3 +"""Mocked-transport tests for the rollback acceptance orchestration. + +These run on any host (no QEMU, no disk), so the journey's ordering, safety and +failure semantics are checked in CI rather than only during a live run. Every +transport here is a fake: the results are marked ``mocked`` and are explicitly +**not** release evidence (the release tool refuses them). + + python3 installer/tests/vm/test_rollback_acceptance.py +""" + +from __future__ import annotations + +import json +import os +import re +import tempfile +import unittest +from pathlib import Path + +import rollback_acceptance as ra +from rollback_acceptance import ( + SCENARIO_RECOVERY, + SCENARIO_ROLLBACK, + CommandResult, + RollbackJourney, + validate_acceptance_disks, +) +from vmsafety import FAIL, MOCKED, NOT_RUN, Results, UnsafeDiskError + +BASE_PKGS = [ + "/var/db/pkg/sys-apps/busybox-1.36.1", + "/var/db/pkg/app-shells/bash-5.2_p26", +] +ADDED_PKG = "/var/db/pkg/app-misc/hello-2.12" +HISTORY = """\ +[[records]] +id = "snap-initial" +created_unix = 1000 +operation = "initial known-good" +source_subvol = "/" +snapshot_subvol = "@-20260101T000000-initial-known-good" +var_snapshot = "@var-20260101T000000-initial-known-good" +status = "committed" +""" +ROLLBACK_STATE = """\ +status = "activated" +generation = 1001 +target_snapshot = "@-20260101T000000-initial-known-good" +previous_root = "@" +root_subvol = "@restore-20260101T000000" +var_subvol = "@var-restore-20260101T000000" +started_unix = 1001 +finished_unix = 1001 +note = "restore from snap-initial" +""" + + +class FakeGuest: + """A scripted guest. First matching handler wins.""" + + def __init__(self, name: str): + self.name = name + self.commands: list[str] = [] + self.handlers: list = [] + + def on(self, needle: str, handler): + # A re-registered needle overrides the earlier one, so a test can tweak + # one behaviour without disturbing the ordering of distinct needles. + self.handlers = [(n, h) for (n, h) in self.handlers if n != needle] + self.handlers.append((needle, handler)) + return self + + def reply(self, needle: str, stdout: str = "", exit_code: int = 0): + return self.on(needle, lambda cmd: CommandResult(cmd, stdout=stdout, exit_code=exit_code)) + + def run(self, command: str, timeout: float = 180.0) -> CommandResult: + self.commands.append(command) + for needle, handler in self.handlers: + if needle in command: + return handler(command) + return CommandResult(command=command, exit_code=0) + + +class FakeTransport: + def __init__(self, installed, recovery): + # installed is a list of guests returned by successive boot_installed() + # calls; the first is the change/undo boot, the second the restored boot. + self._installed = list(installed) + self._recovery = recovery + self.boots: list[str] = [] + self.powered_off = 0 + + def boot_installed(self): + self.boots.append("installed") + return self._installed.pop(0) + + def boot_recovery(self): + self.boots.append("recovery") + return self._recovery + + def poweroff(self): + self.powered_off += 1 + + +def make_phase1(state: dict) -> FakeGuest: + g = FakeGuest("phase1") + + def write_marker(cmd): + match = re.search(r"printf '%s' '([^']*)' >", cmd) + if match: + state["token"] = match.group(1) + return CommandResult(cmd, exit_code=0) + + def read_marker(cmd): + return CommandResult(cmd, stdout=state.get("token", "")) + + def break_hostname(cmd): + state["broken"] = True + return CommandResult(cmd, exit_code=0) + + def read_hostname(cmd): + return CommandResult( + cmd, + stdout=ra.HOSTNAME_SENTINEL + "\n" if state.get("broken") else "phoebus\n", + ) + + def install(cmd): + state["installed"] = True + return CommandResult(cmd, exit_code=0) + + def pkg_dirs(cmd): + dirs = list(BASE_PKGS) + if state.get("installed"): + dirs.append(ADDED_PKG) + return CommandResult(cmd, stdout="\n".join(dirs) + "\n") + + g.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=HISTORY)) + g.on("sha256sum", lambda c: CommandResult(c, exit_code=0, stdout="b" * 64 + " /usr/local/bin/sys\n")) + g.on("> /home/phoebus/rollback-acceptance/marker", write_marker) + g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker) + g.on("tee /etc/hostname", break_hostname) + g.on("cat /etc/hostname", read_hostname) + g.on("sys install", install) + g.on("find /var/db/pkg", pkg_dirs) + g.on("cat " + ADDED_PKG + "/CONTENTS", lambda c: CommandResult( + c, stdout="obj /usr/bin/hello abcdef\nobj /usr/share/hello/README def\n" + )) + g.on("sys undo", lambda c: CommandResult(c, exit_code=0, stdout="rollback activated")) + g.on("cat /var/lib/phoebus/rollback-state.toml", lambda c: CommandResult(c, stdout=ROLLBACK_STATE)) + return g + + +def make_phase2(state: dict) -> FakeGuest: + g = FakeGuest("phase2") + + def read_marker(cmd): + return CommandResult(cmd, stdout=state.get("token", "")) + + fstab = ( + "/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n" + "/dev/mapper/luks-root /var btrfs subvol=/@var-restore-20260101T000000 0 0\n" + "/dev/mapper/luks-root /home btrfs subvol=/@home 0 0\n" + ) + g.on("cat /etc/hostname", lambda c: CommandResult(c, stdout="phoebus\n")) + g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker) + g.on("find /var/db/pkg", lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS) + "\n")) + g.on("test -e", lambda c: CommandResult(c, exit_code=1)) + g.on("cat /proc/cmdline", lambda c: CommandResult( + c, stdout="rootflags=subvol=@restore-20260101T000000 quiet console=ttyS0\n" + )) + g.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab)) + g.on("uname -r", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n")) + g.on("cat /lib/modules", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n")) + g.on("bootctl set-oneshot", lambda c: CommandResult(c, exit_code=0)) + return g + + +def make_recovery() -> FakeGuest: + g = FakeGuest("recovery") + g.on("cat /proc/cmdline", lambda c: CommandResult( + c, stdout="rootflags=subvol=@-20260101T000000-initial-known-good " + "systemd.unit=rescue.target no-network\n" + )) + g.on("ip -brief addr", lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8 ::1/128\n")) + g.on("sys undo status", lambda c: CommandResult(c, exit_code=0, stdout="no active rollback\n")) + g.on("sys ask", lambda c: CommandResult( + c, + exit_code=1, + stderr="No model is connected. The deterministic interface still works:\n", + )) + return g + + +def build_journey(*, state=None, transport=None, **kwargs): + state = state if state is not None else {} + transport = transport or FakeTransport([make_phase1(state), make_phase2(state)], make_recovery()) + journey = RollbackJourney( + transport, + iso_sha256="a" * 64, + sys_sha256="b" * 64, + mocked=True, + **kwargs, + ) + state["token"] = journey.marker_token + return journey + + +def status_of(results: Results, name: str) -> str | None: + for check in results.checks: + if check.name == name: + return check.status + return None + + +class HappyPathTests(unittest.TestCase): + def test_full_journey_passes_in_expected_order(self): + journey = build_journey() + self.assertTrue(journey.run(), "happy path should pass") + self.assertEqual(list(journey.executed), list(ra.EXPECTED_STEP_NAMES)) + self.assertEqual(journey.boots, ["installed", "installed", "recovery"]) + self.assertEqual(journey.results.exit_code(), 0) + self.assertEqual(journey.recovery_results.exit_code(), 0) + + def test_happy_path_has_no_not_run_steps(self): + journey = build_journey() + journey.run() + for stream in (journey.results, journey.recovery_results): + self.assertFalse( + any(c.status == NOT_RUN for c in stream.checks), + "a complete run must not contain NOT RUN steps", + ) + + def test_mocked_run_is_marked_and_never_live(self): + journey = build_journey() + journey.run() + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json") + tmp.close() + self.addCleanup(os.unlink, tmp.name) + data = json.loads(journey.results.write(tmp.name).read_text()) + self.assertEqual(data["evidence"], MOCKED) + self.assertEqual(data["inputs"]["iso"]["sha256"], "a" * 64) + self.assertEqual(data["inputs"]["sys"]["sha256"], "b" * 64) + self.assertEqual(data["scenario"], SCENARIO_ROLLBACK) + + def test_recovery_scenario_is_written_separately(self): + journey = build_journey() + journey.run() + self.assertEqual(journey.recovery_results.scenario, SCENARIO_RECOVERY) + self.assertEqual(journey.recovery_results.exit_code(), 0) + + +class StopOnFailureTests(unittest.TestCase): + def test_undo_failure_stops_and_marks_rest_unrun(self): + state = {} + phase1 = make_phase1(state) + # Override undo to fail. + phase1.on("sys undo", lambda c: CommandResult(c, exit_code=1, stderr="boom")) + transport = FakeTransport([phase1, make_phase2(state)], make_recovery()) + journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True) + state["token"] = journey.marker_token + + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "sys-undo-activates"), FAIL) + self.assertEqual(status_of(journey.results, "journey-complete"), FAIL) + self.assertEqual(status_of(journey.results, "restored-root-var-generations"), NOT_RUN) + self.assertEqual(status_of(journey.recovery_results, "recovery-boot-no-network-no-model"), NOT_RUN) + # The second installed boot and the recovery boot must not happen. + self.assertEqual(transport.boots, ["installed"]) + self.assertEqual(journey.results.exit_code(), 1) + + def test_timeout_is_a_failure_not_a_pass(self): + state = {} + phase1 = make_phase1(state) + phase1.on("sys install", lambda c: CommandResult(c, timed_out=True)) + transport = FakeTransport([phase1, make_phase2(state)], make_recovery()) + journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True) + state["token"] = journey.marker_token + + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "controlled-package-change"), FAIL) + self.assertIn("timeout", "".join(c.detail for c in journey.results.checks)) + self.assertEqual(transport.boots, ["installed"]) + + def test_disconnect_is_a_failure(self): + state = {} + phase1 = make_phase1(state) + phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, disconnected=True)) + transport = FakeTransport([phase1, make_phase2(state)], make_recovery()) + journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True) + state["token"] = journey.marker_token + + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL) + self.assertEqual(journey.results.exit_code(), 1) + + def test_missing_evidence_is_refused_not_treated_as_empty(self): + state = {} + phase1 = make_phase1(state) + phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout="")) + transport = FakeTransport([phase1, make_phase2(state)], make_recovery()) + journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True) + state["token"] = journey.marker_token + + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL) + + def test_missing_input_identities_block_the_run(self): + # No iso/sys digests: the run must not boot anything. + transport = FakeTransport([make_phase1({}), make_phase2({})], make_recovery()) + journey = RollbackJourney(transport, mocked=True) + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "input-identities"), FAIL) + self.assertEqual(status_of(journey.results, "journey-complete"), FAIL) + self.assertEqual(transport.boots, []) + + +class RestorationMismatchTests(unittest.TestCase): + def _journey_with_phase2(self, configure): + state = {} + phase1 = make_phase1(state) + phase2 = make_phase2(state) + configure(phase2) + transport = FakeTransport([phase1, phase2], make_recovery()) + journey = RollbackJourney( + transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True + ) + state["token"] = journey.marker_token + return journey + + def test_wrong_restored_generation_fails(self): + def configure(phase2): + phase2.on("cat /proc/cmdline", lambda c: CommandResult(c, stdout="rootflags=subvol=@\n")) + + journey = self._journey_with_phase2(configure) + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "restored-root-generation"), FAIL) + self.assertEqual(journey.results.exit_code(), 1) + + def test_wrong_var_generation_fails(self): + def configure(phase2): + fstab = "/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n" + phase2.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab)) + + journey = self._journey_with_phase2(configure) + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "restored-var-generation"), FAIL) + + def test_package_db_mismatch_fails(self): + def configure(phase2): + phase2.on( + "find /var/db/pkg", + lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS + [ADDED_PKG]) + "\n"), + ) + + journey = self._journey_with_phase2(configure) + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "package-db-restored"), FAIL) + + def test_surviving_package_file_fails(self): + def configure(phase2): + phase2.on("test -e", lambda c: CommandResult(c, exit_code=0)) + + journey = self._journey_with_phase2(configure) + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "package-files-restored"), FAIL) + + def test_kernel_without_modules_fails(self): + def configure(phase2): + phase2.on("cat /lib/modules", lambda c: CommandResult(c, stdout="5.0.0-other\n")) + + journey = self._journey_with_phase2(configure) + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "kernel-modules-compatible"), FAIL) + + def test_lost_home_marker_fails(self): + def configure(phase2): + phase2.on("cat /home/phoebus/rollback-acceptance/marker", lambda c: CommandResult(c, stdout="")) + + journey = self._journey_with_phase2(configure) + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.results, "home-data-survives"), FAIL) + + def test_recovery_with_network_fails(self): + state = {} + recovery = make_recovery() + recovery.on( + "ip -brief addr", + lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"), + ) + transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery) + journey = RollbackJourney( + transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True + ) + state["token"] = journey.marker_token + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.recovery_results, "recovery-no-network"), FAIL) + + def test_recovery_without_no_model_fallback_fails(self): + state = {} + recovery = make_recovery() + recovery.on("sys ask", lambda c: CommandResult(c, exit_code=0, stdout="here is a plan")) + transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery) + journey = RollbackJourney( + transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True + ) + state["token"] = journey.marker_token + self.assertFalse(journey.run()) + self.assertEqual(status_of(journey.recovery_results, "recovery-no-model-fallback"), FAIL) + + +class RecoveryOnlyTests(unittest.TestCase): + def test_recovery_only_passes(self): + transport = FakeTransport([], make_recovery()) + journey = RollbackJourney( + transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True + ) + self.assertTrue(journey.run_recovery()) + self.assertEqual(journey.recovery_results.exit_code(), 0) + self.assertEqual(transport.boots, ["recovery"]) + + def test_recovery_only_without_inputs_does_not_boot(self): + transport = FakeTransport([], make_recovery()) + journey = RollbackJourney(transport, mocked=True) + self.assertFalse(journey.run_recovery()) + self.assertEqual(transport.boots, []) + self.assertEqual( + status_of(journey.recovery_results, "recovery-boot-no-network-no-model"), + NOT_RUN, + ) + + +class SafetyTests(unittest.TestCase): + def test_rejects_dev_disk(self): + with self.assertRaises(UnsafeDiskError): + validate_acceptance_disks(["/dev/sda"], "/var/tmp/phoebus-vm/disks") + + def test_rejects_disk_outside_test_dir(self): + with tempfile.TemporaryDirectory() as tmp: + outside = Path(tmp).parent / "rollback-outside.qcow2" + outside.write_bytes(b"QFI\xfb" + b"\x00" * 64) + self.addCleanup(outside.unlink) + with self.assertRaises(UnsafeDiskError): + validate_acceptance_disks([outside], tmp) + + +class ParserTests(unittest.TestCase): + def test_newest_committed_coordinated_skips_unpaired(self): + records = ra.parse_snapshot_history( + HISTORY + + """ +[[records]] +id = "snap-varless" +created_unix = 2000 +operation = "risk" +source_subvol = "/" +snapshot_subvol = "@-20260102T000000-risk" +status = "committed" +""" + ) + chosen = ra.newest_committed_coordinated(records) + self.assertIsNotNone(chosen) + self.assertEqual(chosen.id, "snap-initial") + + def test_parse_rootflags_subvol(self): + self.assertEqual( + ra.parse_rootflags_subvol("rootflags=subvol=@restore-x quiet"), "@restore-x" + ) + self.assertEqual(ra.parse_rootflags_subvol("quiet"), None) + + def test_parse_fstab_subvol(self): + text = "/dev/x / btrfs subvol=/@restore-y 0 0\n" + self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y") + self.assertIsNone(ra.parse_fstab_subvol(text, "/var")) + + def test_parse_fstab_subvol_without_leading_slash(self): + text = "/dev/x / btrfs subvol=@restore-y 0 0\n" + self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y") + + def test_parse_contents_paths(self): + paths = ra.parse_contents_files("obj /usr/bin/hello abc\ndir /usr/share/hello def\n") + self.assertEqual(paths, ["/usr/bin/hello", "/usr/share/hello"]) + + def test_parse_network_addresses(self): + self.assertEqual(ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\n"), ["lo"]) + self.assertEqual( + ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"), + ["lo", "enp0s2"], + ) + + def test_unreadable_history_raises(self): + with self.assertRaises(ValueError): + ra.parse_snapshot_history("not = = toml [[") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/installer/tests/vm/vmsafety.py b/installer/tests/vm/vmsafety.py index 32b56a9..83cd3dc 100644 --- a/installer/tests/vm/vmsafety.py +++ b/installer/tests/vm/vmsafety.py @@ -22,6 +22,7 @@ host (see ``test_vmsafety.py``). from __future__ import annotations import dataclasses +import hashlib import json import os import shutil @@ -43,7 +44,7 @@ BLOCKED = "BLOCKED" SKIPPED = "SKIPPED" NOT_RUN = "NOT RUN" -_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED) +_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED, NOT_RUN) # The candidate artifact a harness run is executed against. The release tool # binds a gate report to a specific digest; recording it here is what lets a @@ -51,6 +52,19 @@ _TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED) ARTIFACT_ENV_NAME = "PHOEBUS_VM_ARTIFACT" ARTIFACT_ENV_SHA256 = "PHOEBUS_VM_ARTIFACT_SHA256" +# How a result set was produced. Only a `live` run is real release evidence; a +# `mocked` run exercises the orchestration on fakes and must never be imported +# as a gate pass. The release tool checks this field and refuses anything else. +LIVE = "live" +MOCKED = "mocked" + +# The additional inputs a run may be bound to. The rollback acceptance journey +# pins both the bootable ISO and the `sys` binary it exercised, because the +# rollback semantics live in `sys`; a report that names only the ISO cannot say +# which broker was tested. +INPUT_ISO = "iso" +INPUT_SYS = "sys" + class UnsafeDiskError(Exception): """A writable target violated the disposable-disk rules.""" @@ -89,6 +103,13 @@ class Results: self.checks: list[Check] = [] # The candidate artifact this run verifies (name + sha256), if known. self.artifact: dict | None = None + # Additional pinned inputs (e.g. the tested ISO and `sys` binary), keyed + # by name. `artifact` remains the single release candidate for backwards + # compatibility; `inputs` carries the wider identity set. + self.inputs: dict[str, dict] = {} + # `live` for a real run; `mocked` for a fake-transport unit run. A + # mocked run is never valid release evidence. + self.evidence_class = LIVE if strict is None: strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true") self.strict = bool(strict) @@ -111,6 +132,36 @@ class Results: return self.bind_artifact(name, sha256) return self.artifact + # -- additional pinned inputs ----------------------------------------- + def mark_mocked(self, reason: str = "") -> None: + """Declare this run a mocked one, so it can never be release evidence.""" + self.evidence_class = MOCKED + if reason: + self.inputs.setdefault("mocked_reason", {})["reason"] = reason + + def bind_input( + self, name: str, sha256: str, size_bytes: int | None = None + ) -> dict: + """Pin an input by digest (e.g. the ISO or the `sys` binary).""" + entry: dict = {"sha256": sha256} + if size_bytes is not None: + entry["size_bytes"] = int(size_bytes) + self.inputs[name] = entry + return entry + + def bind_input_file(self, name: str, path: str | os.PathLike) -> dict: + """Hash a local file and pin it as an input.""" + path = Path(path) + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return self.bind_input(name, digest.hexdigest(), path.stat().st_size) + + def missing_inputs(self, names) -> list[str]: + """Input names that were required but not bound.""" + return [name for name in names if name not in self.inputs] + # -- recording --------------------------------------------------------- def _add(self, name: str, status: str, detail: str) -> Check: if status not in _TERMINAL: @@ -140,6 +191,14 @@ class Results: def skip(self, name: str, detail: str = "") -> Check: return self._add(name, SKIPPED, detail) + def mark_not_run(self, name: str, detail: str = "") -> Check: + """Record a step that was expected but never reached. + + A stopped journey must not look like a pass, so unexecuted steps are + visible rather than simply absent. + """ + return self._add(name, NOT_RUN, detail) + # -- outcome ----------------------------------------------------------- @property def failed(self) -> bool: @@ -180,9 +239,11 @@ class Results: "scenario": self.scenario, "run_id": self.run_id, "strict": self.strict, + "evidence": self.evidence_class, "exit_code": self.exit_code(), "counts": self.counts(), "artifact": self.artifact, + "inputs": self.inputs, "checks": [c.to_dict() for c in self.checks], } path.write_text(json.dumps(payload, indent=2) + "\n")