test(rollback): add rollback and no-model recovery acceptance harness
This commit is contained in:
parent
1d37fbc1b8
commit
fc36f637b4
8 changed files with 2253 additions and 43 deletions
1
Makefile
1
Makefile
|
|
@ -49,6 +49,7 @@ test-rust:
|
||||||
|
|
||||||
test-vm:
|
test-vm:
|
||||||
python3 installer/tests/vm/test_vmsafety.py
|
python3 installer/tests/vm/test_vmsafety.py
|
||||||
|
python3 installer/tests/vm/test_rollback_acceptance.py
|
||||||
|
|
||||||
# The complete set of required CI gates. Every target here must actually run;
|
# The complete set of required CI gates. Every target here must actually run;
|
||||||
# the workflow fails the job if a prerequisite is missing rather than skipping.
|
# the workflow fails the job if a prerequisite is missing rather than skipping.
|
||||||
|
|
|
||||||
156
docs/ROLLBACK-ACCEPTANCE.md
Normal file
156
docs/ROLLBACK-ACCEPTANCE.md
Normal file
|
|
@ -0,0 +1,156 @@
|
||||||
|
# Rollback and no-model-recovery acceptance
|
||||||
|
|
||||||
|
This is the execution guide for the **rollback** (`updates-and-rollback`) and
|
||||||
|
**no-model recovery** (`no-model-recovery`) release gates. It describes how to
|
||||||
|
run the acceptance journey in the disposable VM harness, what it asserts, where
|
||||||
|
the evidence lands, and how a failure is diagnosed.
|
||||||
|
|
||||||
|
> **Status: NOT YET RUN.** The harness and its offline tests exist and pass on a
|
||||||
|
> plain host, but the live journey has **not** been executed. Until a real run
|
||||||
|
> succeeds and its `results.json` is imported, both release gates stay
|
||||||
|
> `blocked`. A run made with a fake transport is marked `mocked` and is
|
||||||
|
> **rejected** by the release tool.
|
||||||
|
|
||||||
|
## What the journey proves
|
||||||
|
|
||||||
|
The installer writes a coordinated `@` + `@var` known-good snapshot at install
|
||||||
|
time. The journey then deliberately changes the disk and asks the real
|
||||||
|
`sys undo` to put it back. The ordered steps (from
|
||||||
|
`installer/tests/vm/rollback_acceptance.py`) are:
|
||||||
|
|
||||||
|
1. `sys-binary-identity` — the installed `sys` matches the pinned digest.
|
||||||
|
2. `known-good-coordinated-snapshot` — a committed root snapshot with a paired
|
||||||
|
`@var` snapshot exists.
|
||||||
|
3. `home-data-marker` — a marker is written into `/home` (a separate subvolume).
|
||||||
|
4. `controlled-package-change` — a small package is installed, adding a
|
||||||
|
`/var/db/pkg` entry and files under `/`.
|
||||||
|
5. `deterministic-guest-breakage` — `/etc/hostname` is overwritten with a
|
||||||
|
sentinel.
|
||||||
|
6. `sys-undo-activates` — `sys undo <id>` writes an `activated` rollback state
|
||||||
|
for the next boot.
|
||||||
|
7. `reboot-detached` — the installed disk boots with the installer ISO absent.
|
||||||
|
8. `restored-root-var-generations` — the booted `rootflags`, `/etc/fstab` `/`
|
||||||
|
and `/var` subvolumes match the rollback state.
|
||||||
|
9. `package-files-restored` — the changed package's DB entry and files are gone,
|
||||||
|
and the sentinel hostname is reverted.
|
||||||
|
10. `portage-database-consistent` — `/var/db/pkg` equals its pre-change listing
|
||||||
|
and every entry has a `CONTENTS` file.
|
||||||
|
11. `kernel-modules-compatible` — the running kernel has a module tree in the
|
||||||
|
restored root.
|
||||||
|
12. `home-data-survives` — the `/home` marker is byte-identical.
|
||||||
|
13. `recovery-entry-armed` — `bootctl set-oneshot phoebus-recovery` succeeds.
|
||||||
|
14. `recovery-boot-no-network-no-model` — the recovery entry boots
|
||||||
|
`rescue.target`, no non-loopback interface is up, `sys` status works
|
||||||
|
offline, and `sys ask` reports the deterministic no-model fallback.
|
||||||
|
|
||||||
|
The package changed in step 4 is controlled by `ROLLBACK_PACKAGE` (default
|
||||||
|
`app-misc/hello`); pick a small, signed binary package for a fast run.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
- A **disposable qcow2** already installed by an existing harness run
|
||||||
|
(`serial-driver.py install` or `appliance-driver.py install`). This driver
|
||||||
|
never installs and never attaches the installer ISO.
|
||||||
|
- `/var/tmp/phoebus-vm/disks` (or `PHOEBUS_VM_DISKS`) has the disk. Never point
|
||||||
|
a writable disk at `/dev/*`; the harness refuses it.
|
||||||
|
- `qemu-system-x86_64`, `qemu-img`, OVMF firmware, Python `pexpect`.
|
||||||
|
- The candidate ISO and the `sys` binary under test, as files on the host, so
|
||||||
|
their digests can be pinned.
|
||||||
|
- Network access during the change phase (the package install); the recovery
|
||||||
|
phase uses `-net none`.
|
||||||
|
|
||||||
|
## Running it
|
||||||
|
|
||||||
|
Print the plan without starting a VM (safe on any host):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 installer/tests/vm/rollback-driver.py plan
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the full journey:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \
|
||||||
|
DISK=/var/tmp/phoebus-vm/disks/rollback.qcow2 \
|
||||||
|
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \
|
||||||
|
OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \
|
||||||
|
ISO=/var/tmp/phoebus-vm/phoebus-installer-amd64.iso \
|
||||||
|
PHOEBUS_VM_SYS=/var/tmp/phoebus-vm/sys \
|
||||||
|
PHOEBUS_VM_ARTIFACT=phoebus-installer-amd64.iso \
|
||||||
|
PHOEBUS_VM_ARTIFACT_SHA256="$(sha256sum /var/tmp/phoebus-vm/phoebus-installer-amd64.iso | cut -d' ' -f1)" \
|
||||||
|
PHOEBUS_VM_STRICT=1 \
|
||||||
|
LUKS_PASSPHRASE=phoebus-luks-test USER_PASSWORD=phoebus-user-test \
|
||||||
|
python3 installer/tests/vm/rollback-driver.py run
|
||||||
|
```
|
||||||
|
|
||||||
|
Re-verify only the recovery boot (boots normally once to arm the one-shot, then
|
||||||
|
reboots with no network):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# same environment, then:
|
||||||
|
python3 installer/tests/vm/rollback-driver.py recovery
|
||||||
|
```
|
||||||
|
|
||||||
|
`PHOEBUS_VM_ARTIFACT[_SHA256]` is what binds the evidence to a candidate; the
|
||||||
|
release tool refuses unbound or stale evidence. `PHOEBUS_VM_SYS` is hashed and
|
||||||
|
pinned as the `sys` identity (override with `PHOEBUS_VM_SYS_SHA256`). The guest
|
||||||
|
must report the same digest in step 1.
|
||||||
|
|
||||||
|
## Artifacts
|
||||||
|
|
||||||
|
Under `$PHOEBUS_VM_STATE/<run-id>/` (default `/var/tmp/phoebus-vm/`):
|
||||||
|
|
||||||
|
- `results.json` — the `rollback-acceptance` scenario.
|
||||||
|
- `recovery/results.json` — the `rollback-recovery` scenario (written by `run`).
|
||||||
|
- `serial-boot1.log`, `serial-boot2.log`, `serial-boot3.log` — per-boot serial
|
||||||
|
transcripts.
|
||||||
|
- `OVMF_VARS.fd` — the run's own writable firmware variables.
|
||||||
|
|
||||||
|
`results.json` records `scenario`, `run_id`, `evidence` (`live`), `exit_code`,
|
||||||
|
`counts`, `artifact` (name + ISO digest), `inputs` (`iso` and `sys` digests) and
|
||||||
|
the per-check `checks` list.
|
||||||
|
|
||||||
|
## Feeding the release gates
|
||||||
|
|
||||||
|
Import the run directory when producing a gate report:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
installer-rs/target/release/phoebus-release gates --execute \
|
||||||
|
--candidate /path/phoebus-installer-amd64.iso \
|
||||||
|
--harness-results /var/tmp/phoebus-vm/<run-id> \
|
||||||
|
--output gate-report.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The `updates-and-rollback` gate resolves `rollback-acceptance`; the
|
||||||
|
`no-model-recovery` gate resolves `rollback-recovery`. A gate **fails**
|
||||||
|
(never passes) when the evidence is:
|
||||||
|
|
||||||
|
- `mocked` (a fake-transport unit run),
|
||||||
|
- missing the `iso` or `sys` pinned input,
|
||||||
|
- bound to a different candidate digest (stale),
|
||||||
|
- unbound while a candidate is supplied, or
|
||||||
|
- present but with `exit_code != 0`.
|
||||||
|
|
||||||
|
## Failure diagnosis
|
||||||
|
|
||||||
|
| Symptom | Likely cause |
|
||||||
|
|---|---|
|
||||||
|
| `sys-binary-identity` fails | the installed `sys` is not the build under test; rebuild the ISO from the merged branch and re-install |
|
||||||
|
| `known-good-coordinated-snapshot` fails | the install did not record a paired `@var` snapshot, or `snapshot-history.toml` is unreadable (it is never treated as empty) |
|
||||||
|
| `controlled-package-change` fails/times out | no network, no signed binhost, or the atom needs a long compile; choose another `ROLLBACK_PACKAGE` |
|
||||||
|
| `sys-undo-activates` fails | `sys undo` refused (no committed snapshot, unreadable history) or the broker exited nonzero |
|
||||||
|
| `restored-root-var-generations` fails | `rootflags`/fstab were not rewritten to the restored generation |
|
||||||
|
| `portage-database-consistent` fails | root and `/var` came from different generations |
|
||||||
|
| `kernel-modules-compatible` fails | the selected boot entry does not match the restored `lib/modules` |
|
||||||
|
| `recovery-no-network` fails | QEMU was still given a network device; check `-net none` |
|
||||||
|
| `recovery-no-model-fallback` fails | a provider is configured, or `sys ask` did not fail closed offline |
|
||||||
|
| harness exits 2 with `disposable-disk` | the disk is not a disposable qcow2 inside `PHOEBUS_VM_DISKS` |
|
||||||
|
|
||||||
|
## Integration note
|
||||||
|
|
||||||
|
The branch this work is built on (`codex/rollback-acceptance`) is based on the
|
||||||
|
CI-hardening commit `1d37fbc` and does **not** include the sibling `sys`
|
||||||
|
hardening commit `4679f93`. The installed `sys` used in a live run must come
|
||||||
|
from a build that contains the sys hardening; otherwise step 1
|
||||||
|
(`sys-binary-identity`) will correctly fail. Build the acceptance ISO after the
|
||||||
|
two branches are integrated.
|
||||||
|
|
@ -335,6 +335,18 @@ pub struct HarnessEvidence {
|
||||||
pub counts: BTreeMap<String, u64>,
|
pub counts: BTreeMap<String, u64>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub artifact: Option<HarnessArtifact>,
|
pub artifact: Option<HarnessArtifact>,
|
||||||
|
/// How the run was produced: `live` for a real VM run, `mocked` for a
|
||||||
|
/// fake-transport unit run. A mocked run is never release evidence.
|
||||||
|
#[serde(default = "default_evidence_class")]
|
||||||
|
pub evidence: String,
|
||||||
|
/// Additional pinned inputs, keyed by name (e.g. `iso`, `sys`). A gate can
|
||||||
|
/// require these so evidence that names only the candidate is refused.
|
||||||
|
#[serde(default)]
|
||||||
|
pub inputs: BTreeMap<String, HarnessArtifact>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_evidence_class() -> String {
|
||||||
|
"live".into()
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HarnessEvidence {
|
impl HarnessEvidence {
|
||||||
|
|
@ -345,6 +357,28 @@ impl HarnessEvidence {
|
||||||
.filter(|s| !s.is_empty())
|
.filter(|s| !s.is_empty())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The evidence class, treating a missing field as `live` (older fixture
|
||||||
|
/// JSON predates the field; every current driver writes it explicitly).
|
||||||
|
pub fn evidence_class(&self) -> &str {
|
||||||
|
if self.evidence.is_empty() {
|
||||||
|
"live"
|
||||||
|
} else {
|
||||||
|
&self.evidence
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether this is a real run that may be used as release evidence.
|
||||||
|
pub fn is_live(&self) -> bool {
|
||||||
|
self.evidence_class() == "live"
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn input_sha256(&self, name: &str) -> Option<&str> {
|
||||||
|
self.inputs
|
||||||
|
.get(name)
|
||||||
|
.and_then(|a| a.sha256.as_deref())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn from_json_str(text: &str) -> std::result::Result<Self, String> {
|
pub fn from_json_str(text: &str) -> std::result::Result<Self, String> {
|
||||||
let value: serde_json::Value =
|
let value: serde_json::Value =
|
||||||
serde_json::from_str(text).map_err(|e| format!("invalid harness JSON: {e}"))?;
|
serde_json::from_str(text).map_err(|e| format!("invalid harness JSON: {e}"))?;
|
||||||
|
|
@ -374,12 +408,25 @@ impl HarnessEvidence {
|
||||||
let artifact = value
|
let artifact = value
|
||||||
.get("artifact")
|
.get("artifact")
|
||||||
.and_then(|a| serde_json::from_value::<HarnessArtifact>(a.clone()).ok());
|
.and_then(|a| serde_json::from_value::<HarnessArtifact>(a.clone()).ok());
|
||||||
|
let evidence = value
|
||||||
|
.get("evidence")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("live")
|
||||||
|
.to_string();
|
||||||
|
let inputs = value
|
||||||
|
.get("inputs")
|
||||||
|
.and_then(|v| {
|
||||||
|
serde_json::from_value::<BTreeMap<String, HarnessArtifact>>(v.clone()).ok()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
scenario,
|
scenario,
|
||||||
run_id,
|
run_id,
|
||||||
exit_code,
|
exit_code,
|
||||||
counts,
|
counts,
|
||||||
artifact,
|
artifact,
|
||||||
|
evidence,
|
||||||
|
inputs,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -801,6 +848,17 @@ fn candidate_env(candidate: Option<&CandidateRef>) -> Vec<(String, String)> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Input identities a scenario's evidence must pin before it may pass.
|
||||||
|
///
|
||||||
|
/// The rollback journey records both the booted ISO and the exact `sys` binary
|
||||||
|
/// it exercised; evidence that names only the candidate is incomplete.
|
||||||
|
fn required_inputs_for(scenario: &str) -> &'static [&'static str] {
|
||||||
|
match scenario {
|
||||||
|
"rollback-acceptance" | "rollback-recovery" => &["iso", "sys"],
|
||||||
|
_ => &[],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn evidence_result_for(
|
fn evidence_result_for(
|
||||||
gate: &Gate,
|
gate: &Gate,
|
||||||
scenario: &str,
|
scenario: &str,
|
||||||
|
|
@ -814,7 +872,39 @@ fn evidence_result_for(
|
||||||
detail: format!("no harness evidence found for scenario '{scenario}'"),
|
detail: format!("no harness evidence found for scenario '{scenario}'"),
|
||||||
log: None,
|
log: None,
|
||||||
},
|
},
|
||||||
Some(found) => match (candidate_sha, found.artifact_sha256()) {
|
Some(found) => {
|
||||||
|
// A mocked run proves the orchestration, never the system: it can
|
||||||
|
// never authorize a release.
|
||||||
|
if !found.is_live() {
|
||||||
|
return GateResult {
|
||||||
|
name: gate.name.clone(),
|
||||||
|
status: GateStatus::Failed,
|
||||||
|
detail: format!(
|
||||||
|
"mocked harness evidence for scenario '{scenario}' \
|
||||||
|
(evidence={}) is not release evidence; run it on a real VM",
|
||||||
|
found.evidence_class()
|
||||||
|
),
|
||||||
|
log: None,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let missing: Vec<&str> = required_inputs_for(scenario)
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.filter(|name| found.input_sha256(name).is_none())
|
||||||
|
.collect();
|
||||||
|
if !missing.is_empty() {
|
||||||
|
return GateResult {
|
||||||
|
name: gate.name.clone(),
|
||||||
|
status: GateStatus::Failed,
|
||||||
|
detail: format!(
|
||||||
|
"incomplete evidence: scenario '{scenario}' is missing pinned \
|
||||||
|
input(s): {}",
|
||||||
|
missing.join(", ")
|
||||||
|
),
|
||||||
|
log: None,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
match (candidate_sha, found.artifact_sha256()) {
|
||||||
(Some(bound), Some(ev)) if bound != ev => GateResult {
|
(Some(bound), Some(ev)) if bound != ev => GateResult {
|
||||||
name: gate.name.clone(),
|
name: gate.name.clone(),
|
||||||
status: GateStatus::Failed,
|
status: GateStatus::Failed,
|
||||||
|
|
@ -849,7 +939,8 @@ fn evidence_result_for(
|
||||||
),
|
),
|
||||||
log: None,
|
log: None,
|
||||||
},
|
},
|
||||||
},
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -862,6 +953,7 @@ fn evidence_result_for(
|
||||||
pub fn candidate_gates() -> Vec<Gate> {
|
pub fn candidate_gates() -> Vec<Gate> {
|
||||||
let serial = "installer/tests/vm/serial-driver.py";
|
let serial = "installer/tests/vm/serial-driver.py";
|
||||||
let gui = "installer/tests/vm/gui-driver.py";
|
let gui = "installer/tests/vm/gui-driver.py";
|
||||||
|
let rollback = "installer/tests/vm/rollback-driver.py";
|
||||||
vec![
|
vec![
|
||||||
Gate {
|
Gate {
|
||||||
name: "rust-fmt".into(),
|
name: "rust-fmt".into(),
|
||||||
|
|
@ -933,21 +1025,23 @@ pub fn candidate_gates() -> Vec<Gate> {
|
||||||
command: vec!["python3".into(), gui.into(), "opencode".into()],
|
command: vec!["python3".into(), gui.into(), "opencode".into()],
|
||||||
scenario: Some("gui-opencode".into()),
|
scenario: Some("gui-opencode".into()),
|
||||||
},
|
},
|
||||||
// No automated harness exists for these yet. They stay explicitly
|
// Rollback and no-model recovery are produced by the dedicated
|
||||||
// blocked until one does; an empty command can never be a pass.
|
// rollback driver. The gates stay `Blocked` until a real VM run records
|
||||||
|
// evidence that pins both the ISO and the `sys` binary under test;
|
||||||
|
// mocked unit runs are refused outright.
|
||||||
Gate {
|
Gate {
|
||||||
name: "no-model-recovery".into(),
|
name: "no-model-recovery".into(),
|
||||||
description: "operate and recover with no model connected".into(),
|
description: "operate and recover with no model connected".into(),
|
||||||
requires: GateRequirement::Vm,
|
requires: GateRequirement::Vm,
|
||||||
command: Vec::new(),
|
command: vec!["python3".into(), rollback.into(), "recovery".into()],
|
||||||
scenario: None,
|
scenario: Some("rollback-recovery".into()),
|
||||||
},
|
},
|
||||||
Gate {
|
Gate {
|
||||||
name: "updates-and-rollback".into(),
|
name: "updates-and-rollback".into(),
|
||||||
description: "prepare, apply, roll back and reboot detached".into(),
|
description: "prepare, apply, roll back and reboot detached".into(),
|
||||||
requires: GateRequirement::Vm,
|
requires: GateRequirement::Vm,
|
||||||
command: Vec::new(),
|
command: vec!["python3".into(), rollback.into(), "run".into()],
|
||||||
scenario: None,
|
scenario: Some("rollback-acceptance".into()),
|
||||||
},
|
},
|
||||||
Gate {
|
Gate {
|
||||||
name: "desktop-session".into(),
|
name: "desktop-session".into(),
|
||||||
|
|
@ -1513,6 +1607,8 @@ mod tests {
|
||||||
run_id: "r1".into(),
|
run_id: "r1".into(),
|
||||||
exit_code: 0,
|
exit_code: 0,
|
||||||
counts: BTreeMap::new(),
|
counts: BTreeMap::new(),
|
||||||
|
evidence: "live".into(),
|
||||||
|
inputs: BTreeMap::new(),
|
||||||
artifact: Some(HarnessArtifact {
|
artifact: Some(HarnessArtifact {
|
||||||
name: Some("phoebus.iso".into()),
|
name: Some("phoebus.iso".into()),
|
||||||
sha256: Some(candidate.sha256.clone()),
|
sha256: Some(candidate.sha256.clone()),
|
||||||
|
|
@ -1537,6 +1633,8 @@ mod tests {
|
||||||
run_id: "old-run".into(),
|
run_id: "old-run".into(),
|
||||||
exit_code: 0,
|
exit_code: 0,
|
||||||
counts: BTreeMap::new(),
|
counts: BTreeMap::new(),
|
||||||
|
evidence: "live".into(),
|
||||||
|
inputs: BTreeMap::new(),
|
||||||
artifact: Some(HarnessArtifact {
|
artifact: Some(HarnessArtifact {
|
||||||
name: Some("phoebus.iso".into()),
|
name: Some("phoebus.iso".into()),
|
||||||
sha256: Some(
|
sha256: Some(
|
||||||
|
|
@ -1568,6 +1666,8 @@ mod tests {
|
||||||
run_id: "r1".into(),
|
run_id: "r1".into(),
|
||||||
exit_code: 0,
|
exit_code: 0,
|
||||||
counts: BTreeMap::new(),
|
counts: BTreeMap::new(),
|
||||||
|
evidence: "live".into(),
|
||||||
|
inputs: BTreeMap::new(),
|
||||||
artifact: None,
|
artifact: None,
|
||||||
});
|
});
|
||||||
let mut opts = execution_opts();
|
let mut opts = execution_opts();
|
||||||
|
|
@ -1593,6 +1693,8 @@ mod tests {
|
||||||
run_id: "r1".into(),
|
run_id: "r1".into(),
|
||||||
exit_code: 1,
|
exit_code: 1,
|
||||||
counts: BTreeMap::new(),
|
counts: BTreeMap::new(),
|
||||||
|
evidence: "live".into(),
|
||||||
|
inputs: BTreeMap::new(),
|
||||||
artifact: Some(HarnessArtifact {
|
artifact: Some(HarnessArtifact {
|
||||||
name: Some("phoebus.iso".into()),
|
name: Some("phoebus.iso".into()),
|
||||||
sha256: Some(candidate.sha256.clone()),
|
sha256: Some(candidate.sha256.clone()),
|
||||||
|
|
@ -1605,6 +1707,124 @@ mod tests {
|
||||||
assert_eq!(result(&report, "detached-boot").status, GateStatus::Failed);
|
assert_eq!(result(&report, "detached-boot").status, GateStatus::Failed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn rollback_evidence(
|
||||||
|
candidate: &CandidateRef,
|
||||||
|
evidence: &str,
|
||||||
|
inputs: &[(&str, &str)],
|
||||||
|
) -> HarnessEvidence {
|
||||||
|
HarnessEvidence {
|
||||||
|
scenario: "rollback-acceptance".into(),
|
||||||
|
run_id: "rollback-run".into(),
|
||||||
|
exit_code: 0,
|
||||||
|
counts: BTreeMap::new(),
|
||||||
|
artifact: Some(HarnessArtifact {
|
||||||
|
name: Some("phoebus.iso".into()),
|
||||||
|
sha256: Some(candidate.sha256.clone()),
|
||||||
|
}),
|
||||||
|
evidence: evidence.into(),
|
||||||
|
inputs: inputs
|
||||||
|
.iter()
|
||||||
|
.map(|(name, digest)| {
|
||||||
|
(
|
||||||
|
(*name).to_string(),
|
||||||
|
HarnessArtifact {
|
||||||
|
name: None,
|
||||||
|
sha256: Some((*digest).to_string()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rollback_gates_name_the_dedicated_driver() {
|
||||||
|
let gates = candidate_gates();
|
||||||
|
let rollback = gates
|
||||||
|
.iter()
|
||||||
|
.find(|g| g.name == "updates-and-rollback")
|
||||||
|
.expect("updates-and-rollback gate");
|
||||||
|
assert_eq!(rollback.scenario.as_deref(), Some("rollback-acceptance"));
|
||||||
|
assert!(
|
||||||
|
rollback
|
||||||
|
.command
|
||||||
|
.iter()
|
||||||
|
.any(|a| a.contains("rollback-driver.py")),
|
||||||
|
"command: {:?}",
|
||||||
|
rollback.command
|
||||||
|
);
|
||||||
|
let recovery = gates
|
||||||
|
.iter()
|
||||||
|
.find(|g| g.name == "no-model-recovery")
|
||||||
|
.expect("no-model-recovery gate");
|
||||||
|
assert_eq!(recovery.scenario.as_deref(), Some("rollback-recovery"));
|
||||||
|
assert!(recovery.command.iter().any(|a| a == "recovery"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mocked_rollback_evidence_is_never_a_pass() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (_, candidate) = bind(dir.path(), "phoebus.iso", b"candidate");
|
||||||
|
let mut set = HarnessEvidenceSet::new();
|
||||||
|
set.push(rollback_evidence(
|
||||||
|
&candidate,
|
||||||
|
"mocked",
|
||||||
|
&[("iso", candidate.sha256.as_str()), ("sys", "bbbb")],
|
||||||
|
));
|
||||||
|
let mut opts = execution_opts();
|
||||||
|
opts.evidence = Some(&set);
|
||||||
|
opts.candidate = Some(candidate);
|
||||||
|
let report = run_gates(&FakeExecutor::new(), &opts);
|
||||||
|
let gate = result(&report, "updates-and-rollback");
|
||||||
|
assert_eq!(gate.status, GateStatus::Failed);
|
||||||
|
assert!(gate.detail.contains("mocked"), "detail: {}", gate.detail);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn incomplete_rollback_evidence_is_refused() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (_, candidate) = bind(dir.path(), "phoebus.iso", b"candidate");
|
||||||
|
let mut set = HarnessEvidenceSet::new();
|
||||||
|
// The ISO is pinned but the `sys` binary under test is not.
|
||||||
|
set.push(rollback_evidence(
|
||||||
|
&candidate,
|
||||||
|
"live",
|
||||||
|
&[("iso", candidate.sha256.as_str())],
|
||||||
|
));
|
||||||
|
let mut opts = execution_opts();
|
||||||
|
opts.evidence = Some(&set);
|
||||||
|
opts.candidate = Some(candidate);
|
||||||
|
let report = run_gates(&FakeExecutor::new(), &opts);
|
||||||
|
let gate = result(&report, "updates-and-rollback");
|
||||||
|
assert_eq!(gate.status, GateStatus::Failed);
|
||||||
|
assert!(
|
||||||
|
gate.detail.contains("incomplete"),
|
||||||
|
"detail: {}",
|
||||||
|
gate.detail
|
||||||
|
);
|
||||||
|
assert!(gate.detail.contains("sys"), "detail: {}", gate.detail);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn complete_live_rollback_evidence_passes() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (_, candidate) = bind(dir.path(), "phoebus.iso", b"candidate");
|
||||||
|
let mut set = HarnessEvidenceSet::new();
|
||||||
|
set.push(rollback_evidence(
|
||||||
|
&candidate,
|
||||||
|
"live",
|
||||||
|
&[("iso", candidate.sha256.as_str()), ("sys", "bbbb")],
|
||||||
|
));
|
||||||
|
let mut opts = execution_opts();
|
||||||
|
opts.evidence = Some(&set);
|
||||||
|
opts.candidate = Some(candidate);
|
||||||
|
let report = run_gates(&FakeExecutor::new(), &opts);
|
||||||
|
assert_eq!(
|
||||||
|
result(&report, "updates-and-rollback").status,
|
||||||
|
GateStatus::Passed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn publish_fixture(
|
fn publish_fixture(
|
||||||
reverse: bool,
|
reverse: bool,
|
||||||
) -> (
|
) -> (
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,10 @@ touching a physical disk. The target is always a qcow2 file.
|
||||||
- `run-vm.sh` interactive QEMU launcher (manual testing).
|
- `run-vm.sh` interactive QEMU launcher (manual testing).
|
||||||
- `install.sh` guest-side, non-interactive installation (run in the VM).
|
- `install.sh` guest-side, non-interactive installation (run in the VM).
|
||||||
- `serial-driver.py` host-side pexpect driver (`install` and `boot` modes).
|
- `serial-driver.py` host-side pexpect driver (`install` and `boot` modes).
|
||||||
|
- `rollback-driver.py` host-side rollback + no-model-recovery driver (`plan`,
|
||||||
|
`run` and `recovery` modes); see `docs/ROLLBACK-ACCEPTANCE.md`.
|
||||||
|
- `rollback_acceptance.py` the ordered journey logic, testable with fakes.
|
||||||
|
- `vmsafety.py` shared disk-safety, structured-result and QMP helpers.
|
||||||
- `test.conf.example` installer configuration used by the driver.
|
- `test.conf.example` installer configuration used by the driver.
|
||||||
|
|
||||||
## Host prerequisites
|
## Host prerequisites
|
||||||
|
|
|
||||||
419
installer/tests/vm/rollback-driver.py
Normal file
419
installer/tests/vm/rollback-driver.py
Normal file
|
|
@ -0,0 +1,419 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Disposable-VM acceptance driver for rollback and no-model recovery.
|
||||||
|
|
||||||
|
This is the *live* counterpart to ``rollback_acceptance.py``: it boots an already
|
||||||
|
installed qcow2 disk (the installer ISO is **never** attached), drives the
|
||||||
|
serial console, and runs the ordered journey in
|
||||||
|
:mod:`rollback_acceptance`. Its ``results.json`` feeds the
|
||||||
|
``updates-and-rollback`` and ``no-model-recovery`` release gates.
|
||||||
|
|
||||||
|
Modes (the entry point is explicit; nothing runs by accident):
|
||||||
|
|
||||||
|
plan print the steps, the pinned input identities and the QEMU safety
|
||||||
|
checks, then exit 0 without starting a VM. Safe on any host.
|
||||||
|
run the full journey: snapshot, package change, breakage, `sys undo`,
|
||||||
|
detached reboot, restoration checks, then a recovery boot with
|
||||||
|
networking disabled, and write both scenarios' evidence.
|
||||||
|
recovery re-verify only the no-model recovery boot. Boots normally once to
|
||||||
|
arm the recovery entry, then reboots with no network. Writes the
|
||||||
|
``rollback-recovery`` evidence.
|
||||||
|
|
||||||
|
Safety: every writable disk is validated by :mod:`vmsafety` before QEMU starts.
|
||||||
|
It must be a qcow2 file inside ``PHOEBUS_VM_DISKS``; ``/dev/*`` paths, block and
|
||||||
|
character devices, raw images, escaping symlinks and outside backing chains are
|
||||||
|
refused. The installer ISO is deliberately not a boot input here, and the
|
||||||
|
recovery boot uses ``-net none``.
|
||||||
|
|
||||||
|
Environment (all paths must be local; no secrets on the command line):
|
||||||
|
|
||||||
|
OVMF_CODE, OVMF_VARS UEFI firmware and variable template (required)
|
||||||
|
DISK (or DISKS) disposable qcow2 installed disk(s)
|
||||||
|
PHOEBUS_VM_DISKS directory the disks must live in
|
||||||
|
ISO installer ISO whose digest is pinned (required)
|
||||||
|
PHOEBUS_VM_SYS host path to the `sys` binary under test
|
||||||
|
PHOEBUS_VM_SYS_SHA256 optional expected digest (otherwise computed)
|
||||||
|
PHOEBUS_VM_ARTIFACT[_SHA256] release tool candidate binding (optional here)
|
||||||
|
LUKS_PASSPHRASE, USER_PASSWORD, USERNAME
|
||||||
|
SYS_BIN guest `sys` path (default /usr/local/bin/sys)
|
||||||
|
ROLLBACK_PACKAGE controlled package change (default app-misc/hello)
|
||||||
|
QEMU_BIN, VM_MEM, VM_SMP, PHOEBUS_VM_STATE, PHOEBUS_VM_STRICT
|
||||||
|
|
||||||
|
Example (see docs/ROLLBACK-ACCEPTANCE.md):
|
||||||
|
|
||||||
|
PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \\
|
||||||
|
DISK=/var/tmp/phoebus-vm/disks/rollback.qcow2 \\
|
||||||
|
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \\
|
||||||
|
OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \\
|
||||||
|
ISO=/path/phoebus-installer-amd64.iso \\
|
||||||
|
PHOEBUS_VM_SYS=/path/sys \\
|
||||||
|
python3 installer/tests/vm/rollback-driver.py run
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pexpect
|
||||||
|
|
||||||
|
import vmsafety
|
||||||
|
from rollback_acceptance import (
|
||||||
|
EXPECTED_STEP_NAMES,
|
||||||
|
REQUIRED_INPUTS,
|
||||||
|
RC_SENTINEL,
|
||||||
|
SCENARIO_RECOVERY,
|
||||||
|
SCENARIO_ROLLBACK,
|
||||||
|
CommandResult,
|
||||||
|
RollbackJourney,
|
||||||
|
validate_acceptance_disks,
|
||||||
|
)
|
||||||
|
from vmsafety import INPUT_ISO, INPUT_SYS, Results
|
||||||
|
|
||||||
|
QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64")
|
||||||
|
LIBDIR = os.environ.get("QEMU_LIBDIR", "")
|
||||||
|
DATADIR = os.environ.get("QEMU_DATADIR", "")
|
||||||
|
OVMF_CODE = os.environ.get("OVMF_CODE", "")
|
||||||
|
OVMF_VARS_TEMPLATE = os.environ.get("OVMF_VARS", "")
|
||||||
|
DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "")
|
||||||
|
ISO = os.environ.get("ISO", "")
|
||||||
|
SYS_HOST = os.environ.get("PHOEBUS_VM_SYS", "")
|
||||||
|
SYS_BIN = os.environ.get("SYS_BIN", "/usr/local/bin/sys")
|
||||||
|
PACKAGE = os.environ.get("ROLLBACK_PACKAGE", "app-misc/hello")
|
||||||
|
HOME = os.environ.get("ROLLBACK_HOME", "/home/phoebus")
|
||||||
|
LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test")
|
||||||
|
USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test")
|
||||||
|
USER = os.environ.get("USERNAME", "phoebus")
|
||||||
|
MEM = os.environ.get("VM_MEM", "8192")
|
||||||
|
SMP = os.environ.get("VM_SMP", "8")
|
||||||
|
BOOT_TIMEOUT = int(os.environ.get("BOOT_TIMEOUT", "600"))
|
||||||
|
LOGIN_TIMEOUT = int(os.environ.get("LOGIN_TIMEOUT", "300"))
|
||||||
|
PROMPT = os.environ.get("GUEST_PROMPT", r"[$#] $")
|
||||||
|
|
||||||
|
|
||||||
|
def disposable_dir() -> Path:
|
||||||
|
base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")
|
||||||
|
return Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks"))
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: str | os.PathLike) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with open(path, "rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def strip_echo(text: str, command: str) -> str:
|
||||||
|
"""Drop the console echo of a typed command and the rc sentinel line."""
|
||||||
|
lines = text.splitlines()
|
||||||
|
if lines and command and command[:40] in lines[0]:
|
||||||
|
lines = lines[1:]
|
||||||
|
return "\n".join(line for line in lines if RC_SENTINEL not in line)
|
||||||
|
|
||||||
|
|
||||||
|
class PexpectGuest:
|
||||||
|
"""A :class:`rollback_acceptance.Guest` backed by a serial console."""
|
||||||
|
|
||||||
|
def __init__(self, child, log, user_pw: str):
|
||||||
|
self.child = child
|
||||||
|
self.log = log
|
||||||
|
self.user_pw = user_pw
|
||||||
|
|
||||||
|
def _ensure_sudo(self) -> None:
|
||||||
|
"""Refresh the sudo timestamp, feeding the password off the log.
|
||||||
|
|
||||||
|
``sudo -n true`` succeeds while the timestamp is valid; otherwise
|
||||||
|
``sudo -v`` prompts and the password is sent with the pexpect logfile
|
||||||
|
detached, so it never reaches the serial transcript.
|
||||||
|
"""
|
||||||
|
self.child.sendline("sudo -n true 2>/dev/null || sudo -v")
|
||||||
|
idx = self.child.expect(
|
||||||
|
[r"\[sudo\] password for [^:]+:", PROMPT, pexpect.TIMEOUT], timeout=30
|
||||||
|
)
|
||||||
|
if idx == 0:
|
||||||
|
saved = self.child.logfile
|
||||||
|
self.child.logfile = None
|
||||||
|
try:
|
||||||
|
self.child.sendline(self.user_pw)
|
||||||
|
self.child.expect(PROMPT, timeout=30)
|
||||||
|
finally:
|
||||||
|
self.child.logfile = saved
|
||||||
|
elif idx == 1:
|
||||||
|
return
|
||||||
|
else: # pragma: no cover - console fault
|
||||||
|
raise RuntimeError("sudo timestamp could not be refreshed")
|
||||||
|
|
||||||
|
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
||||||
|
try:
|
||||||
|
if "sudo" in command:
|
||||||
|
self._ensure_sudo()
|
||||||
|
end = f"echo {RC_SENTINEL}=$(printf '%s' $?)"
|
||||||
|
self.child.sendline(f"{command}; {end}")
|
||||||
|
idx = self.child.expect(
|
||||||
|
[rf"{RC_SENTINEL}=([0-9]+)", pexpect.TIMEOUT, pexpect.EOF],
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
except (pexpect.TIMEOUT, pexpect.EOF):
|
||||||
|
return CommandResult(command=command, disconnected=True, stdout="")
|
||||||
|
if idx == 0:
|
||||||
|
return CommandResult(
|
||||||
|
command=command,
|
||||||
|
stdout=strip_echo(self.child.before or "", command),
|
||||||
|
exit_code=int(self.child.match.group(1)),
|
||||||
|
)
|
||||||
|
if idx == 1:
|
||||||
|
return CommandResult(
|
||||||
|
command=command, timed_out=True, stdout=self.child.before or ""
|
||||||
|
)
|
||||||
|
return CommandResult(
|
||||||
|
command=command, disconnected=True, stdout=self.child.before or ""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PexpectTransport:
|
||||||
|
"""Boots the installed disk; the installer ISO is never attached."""
|
||||||
|
|
||||||
|
def __init__(self, disks, run_paths):
|
||||||
|
self.disks = disks
|
||||||
|
self.paths = run_paths
|
||||||
|
self.child = None
|
||||||
|
self.log = None
|
||||||
|
self._boot = 0
|
||||||
|
|
||||||
|
def _args(self, *, network: bool) -> list[str]:
|
||||||
|
env = dict(os.environ)
|
||||||
|
if LIBDIR:
|
||||||
|
env["LD_LIBRARY_PATH"] = LIBDIR
|
||||||
|
args = ["-machine", "q35,accel=kvm", "-cpu", "host", "-smp", SMP, "-m", MEM]
|
||||||
|
if DATADIR:
|
||||||
|
args += ["-L", DATADIR]
|
||||||
|
args += [
|
||||||
|
"-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}",
|
||||||
|
"-drive", f"if=pflash,format=raw,file={self.paths.ovmf_vars}",
|
||||||
|
]
|
||||||
|
for disk in self.disks:
|
||||||
|
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
||||||
|
if network:
|
||||||
|
args += ["-netdev", "user,id=n0",
|
||||||
|
"-device", "virtio-net-pci,netdev=n0,romfile="]
|
||||||
|
else:
|
||||||
|
# Recovery must work with networking disabled.
|
||||||
|
args += ["-net", "none"]
|
||||||
|
args += [
|
||||||
|
"-device", "virtio-rng-pci", "-vga", "std", "-display", "none",
|
||||||
|
"-serial", "stdio", "-monitor", "none", "-no-reboot",
|
||||||
|
]
|
||||||
|
return args
|
||||||
|
|
||||||
|
def _spawn(self, *, network: bool) -> None:
|
||||||
|
self._boot += 1
|
||||||
|
log_path = self.paths.directory / f"serial-boot{self._boot}.log"
|
||||||
|
self.log = open(log_path, "w")
|
||||||
|
env = dict(os.environ)
|
||||||
|
if LIBDIR:
|
||||||
|
env["LD_LIBRARY_PATH"] = LIBDIR
|
||||||
|
self.child = pexpect.spawn(
|
||||||
|
QEMU,
|
||||||
|
self._args(network=network),
|
||||||
|
env=env,
|
||||||
|
timeout=BOOT_TIMEOUT,
|
||||||
|
encoding="utf-8",
|
||||||
|
codec_errors="replace",
|
||||||
|
maxread=65536,
|
||||||
|
)
|
||||||
|
self.child.delaybeforesend = 0.4
|
||||||
|
self.child.logfile = self.log
|
||||||
|
|
||||||
|
def _unlock_and_login(self) -> None:
|
||||||
|
assert self.child is not None
|
||||||
|
self.child.expect([r"[Pp]assphrase", r"login:"], timeout=BOOT_TIMEOUT)
|
||||||
|
if "assphrase" in (self.child.after or "").lower():
|
||||||
|
self.child.sendline(LUKS_PW)
|
||||||
|
logged_in = False
|
||||||
|
for _ in range(5):
|
||||||
|
self.child.expect(fr"{re.escape(USER)} login:", timeout=LOGIN_TIMEOUT)
|
||||||
|
self.child.sendline(USER)
|
||||||
|
idx = self.child.expect(
|
||||||
|
[r"[Pp]assword:", fr"{re.escape(USER)} login:", r"Login incorrect"],
|
||||||
|
timeout=60,
|
||||||
|
)
|
||||||
|
if idx == 0:
|
||||||
|
self.child.sendline(USER_PW)
|
||||||
|
if self.child.expect([PROMPT, r"Login incorrect", fr"{re.escape(USER)} login:"],
|
||||||
|
timeout=120) == 0:
|
||||||
|
logged_in = True
|
||||||
|
break
|
||||||
|
if not logged_in:
|
||||||
|
raise RuntimeError("could not log in on the installed system")
|
||||||
|
|
||||||
|
def boot_installed(self):
|
||||||
|
self._spawn(network=True)
|
||||||
|
self._unlock_and_login()
|
||||||
|
return PexpectGuest(self.child, self.log, USER_PW)
|
||||||
|
|
||||||
|
def boot_recovery(self):
|
||||||
|
self._spawn(network=False)
|
||||||
|
self._unlock_and_login()
|
||||||
|
return PexpectGuest(self.child, self.log, USER_PW)
|
||||||
|
|
||||||
|
def poweroff(self) -> None:
|
||||||
|
if self.child is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
PexpectGuest(self.child, self.log, USER_PW).run("sudo poweroff -f", timeout=60)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
self.child.close(force=True)
|
||||||
|
finally:
|
||||||
|
if self.log:
|
||||||
|
self.log.close()
|
||||||
|
self.child = None
|
||||||
|
self.log = None
|
||||||
|
|
||||||
|
|
||||||
|
def pin_inputs(results: Results, *, iso: str, sys_host: str) -> dict:
|
||||||
|
"""Bind the ISO and `sys` identities the run exercised."""
|
||||||
|
bound = {}
|
||||||
|
if iso:
|
||||||
|
results.bind_input_file(INPUT_ISO, iso)
|
||||||
|
bound[INPUT_ISO] = results.inputs[INPUT_ISO]
|
||||||
|
if sys_host:
|
||||||
|
expected = os.environ.get("PHOEBUS_VM_SYS_SHA256")
|
||||||
|
digest = expected or sha256_file(sys_host)
|
||||||
|
results.bind_input(INPUT_SYS, digest)
|
||||||
|
bound[INPUT_SYS] = results.inputs[INPUT_SYS]
|
||||||
|
results.bind_artifact_from_env()
|
||||||
|
return bound
|
||||||
|
|
||||||
|
|
||||||
|
def plan_text() -> str:
|
||||||
|
lines = [
|
||||||
|
"rollback acceptance journey (no VM started)",
|
||||||
|
"scenario: " + SCENARIO_ROLLBACK,
|
||||||
|
"recovery: " + SCENARIO_RECOVERY,
|
||||||
|
"steps:",
|
||||||
|
]
|
||||||
|
lines += [f" {i + 1:2d}. {name}" for i, name in enumerate(EXPECTED_STEP_NAMES)]
|
||||||
|
for scenario, required in REQUIRED_INPUTS.items():
|
||||||
|
lines.append(f"required inputs for {scenario}: {', '.join(required)}")
|
||||||
|
lines.append("safety: every DISK must be a qcow2 inside PHOEBUS_VM_DISKS;")
|
||||||
|
lines.append(" the installer ISO is never attached; recovery uses -net none.")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def build_results(scenario: str, mocked: bool = False) -> Results:
|
||||||
|
results = Results(scenario)
|
||||||
|
if mocked:
|
||||||
|
results.mark_mocked("fake transport")
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def run_full(disks, run_paths, sys_digest) -> int:
|
||||||
|
rollback = build_results(SCENARIO_ROLLBACK)
|
||||||
|
recovery = build_results(SCENARIO_RECOVERY)
|
||||||
|
pin_inputs(rollback, iso=ISO, sys_host=SYS_HOST)
|
||||||
|
pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST)
|
||||||
|
transport = PexpectTransport(disks, run_paths)
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport,
|
||||||
|
results=rollback,
|
||||||
|
recovery_results=recovery,
|
||||||
|
home=HOME,
|
||||||
|
sys_bin=SYS_BIN,
|
||||||
|
package=PACKAGE,
|
||||||
|
iso_sha256=os.environ.get("PHOEBUS_VM_ARTIFACT_SHA256") or (rollback.inputs.get(INPUT_ISO) or {}).get("sha256"),
|
||||||
|
sys_sha256=sys_digest,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
journey.run()
|
||||||
|
finally:
|
||||||
|
transport.poweroff()
|
||||||
|
rollback.write(run_paths.results_json)
|
||||||
|
# The release tool imports any results.json under a directory tree, so
|
||||||
|
# the recovery scenario lives beside the rollback run.
|
||||||
|
recovery_dir = run_paths.directory / "recovery"
|
||||||
|
recovery_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
recovery.write(recovery_dir / "results.json")
|
||||||
|
print(rollback.summary())
|
||||||
|
print(recovery.summary())
|
||||||
|
return 1 if (rollback.exit_code() or recovery.exit_code()) else 0
|
||||||
|
|
||||||
|
|
||||||
|
def run_recovery_only(disks, run_paths, sys_digest) -> int:
|
||||||
|
recovery = build_results(SCENARIO_RECOVERY)
|
||||||
|
pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST)
|
||||||
|
# Arm the one-shot recovery entry on a normal boot first.
|
||||||
|
transport = PexpectTransport(disks, run_paths)
|
||||||
|
try:
|
||||||
|
guest = transport.boot_installed()
|
||||||
|
guest.run("sudo bootctl set-oneshot phoebus-recovery", timeout=120)
|
||||||
|
finally:
|
||||||
|
transport.poweroff()
|
||||||
|
transport = PexpectTransport(disks, run_paths)
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport,
|
||||||
|
results=build_results(SCENARIO_ROLLBACK),
|
||||||
|
recovery_results=recovery,
|
||||||
|
home=HOME,
|
||||||
|
sys_bin=SYS_BIN,
|
||||||
|
package=PACKAGE,
|
||||||
|
sys_sha256=sys_digest,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
journey.run_recovery()
|
||||||
|
finally:
|
||||||
|
transport.poweroff()
|
||||||
|
recovery.write(run_paths.results_json)
|
||||||
|
print(recovery.summary())
|
||||||
|
return recovery.exit_code()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
mode = sys.argv[1] if len(sys.argv) > 1 else ""
|
||||||
|
if mode == "plan":
|
||||||
|
print(plan_text())
|
||||||
|
return 0
|
||||||
|
if mode not in ("run", "recovery"):
|
||||||
|
print(__doc__)
|
||||||
|
return 2
|
||||||
|
if not OVMF_CODE or not OVMF_VARS_TEMPLATE:
|
||||||
|
print("harness error: OVMF_CODE and OVMF_VARS are required", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
if not ISO:
|
||||||
|
print("harness error: ISO is required to pin the candidate identity", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
if not SYS_HOST or not Path(SYS_HOST).is_file():
|
||||||
|
print(
|
||||||
|
"harness error: PHOEBUS_VM_SYS must point at the `sys` binary under test",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
scenario = SCENARIO_ROLLBACK if mode == "run" else SCENARIO_RECOVERY
|
||||||
|
run_paths = vmsafety.make_run_paths(
|
||||||
|
f"rollback-{mode}",
|
||||||
|
base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"),
|
||||||
|
ovmf_vars_template=OVMF_VARS_TEMPLATE,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
disks = validate_acceptance_disks(DISKS.split(","), disposable_dir())
|
||||||
|
except vmsafety.UnsafeDiskError as exc:
|
||||||
|
results = build_results(scenario)
|
||||||
|
results.blocked("disposable-disk", str(exc))
|
||||||
|
results.write(run_paths.results_json)
|
||||||
|
print(f"harness error: {exc}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
sys_digest = os.environ.get("PHOEBUS_VM_SYS_SHA256") or sha256_file(SYS_HOST)
|
||||||
|
if mode == "run":
|
||||||
|
return run_full(disks, run_paths, sys_digest)
|
||||||
|
return run_recovery_only(disks, run_paths, sys_digest)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
852
installer/tests/vm/rollback_acceptance.py
Normal file
852
installer/tests/vm/rollback_acceptance.py
Normal file
|
|
@ -0,0 +1,852 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Offline orchestration for the rollback + no-model-recovery acceptance run.
|
||||||
|
|
||||||
|
This module holds the *logic* of the journey: the ordered steps, the guest
|
||||||
|
commands they issue, and the assertions that decide PASS/FAIL. It deliberately
|
||||||
|
imports neither pexpect nor QEMU, so the whole journey can be exercised on a
|
||||||
|
plain host against a fake transport (see ``test_rollback_acceptance.py``).
|
||||||
|
|
||||||
|
The real VM driver (``rollback-driver.py``) supplies a pexpect-backed transport
|
||||||
|
implementing :class:`BootTransport` and a :class:`Guest` that runs commands over
|
||||||
|
the serial console. The release gates import the ``results.json`` these runs
|
||||||
|
write; a run made with a fake transport is marked ``mocked`` and can never be a
|
||||||
|
gate pass.
|
||||||
|
|
||||||
|
Journey shape (the installer already created a coordinated ``@`` + ``@var``
|
||||||
|
known-good snapshot at install time):
|
||||||
|
|
||||||
|
boot installed (ISO detached)
|
||||||
|
1. identify the newest committed coordinated snapshot
|
||||||
|
2. write a marker into /home (a subvolume rollback must not touch)
|
||||||
|
3. apply a controlled package change (touches / and /var/db/pkg)
|
||||||
|
4. introduce deterministic guest-only breakage in /
|
||||||
|
5. run the real `sys undo` (restores root + var as the next boot root)
|
||||||
|
reboot, boot installed (ISO detached)
|
||||||
|
6. restored root/var generations match the rollback state
|
||||||
|
7. the changed package and its files are gone
|
||||||
|
8. /var/db/pkg matches / (consistent package database)
|
||||||
|
9. the booted kernel has matching modules in the restored root
|
||||||
|
10. /home data survived
|
||||||
|
arm the recovery entry, reboot, boot recovery (no network, no model)
|
||||||
|
11. recovery entry boots, no network, and `sys` still works with no model
|
||||||
|
|
||||||
|
Nothing here decides that a *live* run passed: only a real transport on a
|
||||||
|
disposable qcow2 disk can produce release evidence. The mocked run exists to
|
||||||
|
prove the orchestration itself.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import dataclasses
|
||||||
|
import re
|
||||||
|
import tomllib
|
||||||
|
from typing import Callable, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
import vmsafety
|
||||||
|
from vmsafety import INPUT_ISO, INPUT_SYS, Results
|
||||||
|
|
||||||
|
# Scenario names. Each is a release-gate evidence key.
|
||||||
|
SCENARIO_ROLLBACK = "rollback-acceptance"
|
||||||
|
SCENARIO_RECOVERY = "rollback-recovery"
|
||||||
|
|
||||||
|
# The input identities a gate must see before it may pass. The rollback
|
||||||
|
# semantics live in `sys`, so naming only the ISO is incomplete evidence.
|
||||||
|
REQUIRED_INPUTS: dict[str, tuple[str, ...]] = {
|
||||||
|
SCENARIO_ROLLBACK: (INPUT_ISO, INPUT_SYS),
|
||||||
|
SCENARIO_RECOVERY: (INPUT_ISO, INPUT_SYS),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Stable guest paths and sentinels.
|
||||||
|
SNAPSHOT_HISTORY = "/var/lib/phoebus/snapshot-history.toml"
|
||||||
|
ROLLBACK_STATE = "/var/lib/phoebus/rollback-state.toml"
|
||||||
|
PKG_DB = "/var/db/pkg"
|
||||||
|
HOSTNAME_FILE = "/etc/hostname"
|
||||||
|
HOME_MARKER_REL = "rollback-acceptance/marker"
|
||||||
|
HOME_MARKER_TOKEN = "phoebus-rollback-home-marker"
|
||||||
|
HOSTNAME_SENTINEL = "phoebus-rollback-broken-hostname"
|
||||||
|
DEFAULT_SYS = "/usr/local/bin/sys"
|
||||||
|
DEFAULT_PACKAGE = "app-misc/hello"
|
||||||
|
|
||||||
|
# File framing so a read is not confused with console noise.
|
||||||
|
FILE_FRAME = "__PHOEBUS_RB_FILE__"
|
||||||
|
# The driver appends this and parses the exit status from the console.
|
||||||
|
RC_SENTINEL = "__PHOEBUS_RB_RC"
|
||||||
|
|
||||||
|
MAX_OUTPUT_BYTES = 64 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Small, testable value types
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class CommandResult:
|
||||||
|
"""The outcome of one guest command.
|
||||||
|
|
||||||
|
``timed_out`` and ``disconnected`` are distinct from a nonzero exit so a
|
||||||
|
broken connection can never be mistaken for a clean failure that a later
|
||||||
|
step might tolerate.
|
||||||
|
"""
|
||||||
|
|
||||||
|
command: str
|
||||||
|
stdout: str = ""
|
||||||
|
stderr: str = ""
|
||||||
|
exit_code: int | None = None
|
||||||
|
timed_out: bool = False
|
||||||
|
disconnected: bool = False
|
||||||
|
|
||||||
|
def bounded(self) -> "CommandResult":
|
||||||
|
combined = self.stdout[-MAX_OUTPUT_BYTES:]
|
||||||
|
self.stdout = combined
|
||||||
|
self.stderr = self.stderr[-MAX_OUTPUT_BYTES:]
|
||||||
|
return self
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ok(self) -> bool:
|
||||||
|
return (
|
||||||
|
not self.timed_out
|
||||||
|
and not self.disconnected
|
||||||
|
and self.exit_code == 0
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def failure_reason(self) -> str:
|
||||||
|
if self.timed_out:
|
||||||
|
return f"timeout running `{self.command}`"
|
||||||
|
if self.disconnected:
|
||||||
|
return f"guest disconnected running `{self.command}`"
|
||||||
|
if self.exit_code is None:
|
||||||
|
return f"no exit status for `{self.command}`"
|
||||||
|
if self.exit_code != 0:
|
||||||
|
detail = (self.stderr or self.stdout).strip().splitlines()
|
||||||
|
tail = detail[-1] if detail else ""
|
||||||
|
return f"`{self.command}` exited {self.exit_code}: {tail}"
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class SnapshotRecord:
|
||||||
|
id: str
|
||||||
|
operation: str
|
||||||
|
snapshot_subvol: str
|
||||||
|
var_snapshot: str | None
|
||||||
|
status: str
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class Guest(Protocol):
|
||||||
|
"""A shell on a booted installed system."""
|
||||||
|
|
||||||
|
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class BootTransport(Protocol):
|
||||||
|
"""Boots the installed disk in the requested mode.
|
||||||
|
|
||||||
|
* ``boot_installed`` boots the normal entry with the installer ISO detached.
|
||||||
|
* ``boot_recovery`` boots the recovery entry with networking disabled.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def boot_installed(self) -> Guest:
|
||||||
|
...
|
||||||
|
|
||||||
|
def boot_recovery(self) -> Guest:
|
||||||
|
...
|
||||||
|
|
||||||
|
def poweroff(self) -> None:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Parsers (pure functions, unit-tested directly)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def extract_between(text: str, start: str, end: str) -> str:
|
||||||
|
"""Return the text between the first ``start`` and the next ``end``.
|
||||||
|
|
||||||
|
If the markers are absent the whole text is returned unchanged, so a fake
|
||||||
|
transport may return clean content directly.
|
||||||
|
"""
|
||||||
|
if start not in text or end not in text:
|
||||||
|
return text
|
||||||
|
body = text.split(start, 1)[1]
|
||||||
|
return body.split(end, 1)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_snapshot_history(text: str) -> list[SnapshotRecord]:
|
||||||
|
"""Parse the snapshot history TOML into records.
|
||||||
|
|
||||||
|
Raises :class:`ValueError` when the document is unreadable; a caller must
|
||||||
|
never treat "could not parse" as "no snapshots".
|
||||||
|
"""
|
||||||
|
text = text.strip()
|
||||||
|
if not text:
|
||||||
|
raise ValueError("empty snapshot history")
|
||||||
|
try:
|
||||||
|
data = tomllib.loads(text)
|
||||||
|
except tomllib.TOMLDecodeError as exc: # pragma: no cover - message detail
|
||||||
|
raise ValueError(f"invalid snapshot history: {exc}") from exc
|
||||||
|
records = data.get("records", [])
|
||||||
|
if not isinstance(records, list):
|
||||||
|
raise ValueError("snapshot history has no `records` list")
|
||||||
|
out: list[SnapshotRecord] = []
|
||||||
|
for entry in records:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
out.append(
|
||||||
|
SnapshotRecord(
|
||||||
|
id=str(entry.get("id", "")),
|
||||||
|
operation=str(entry.get("operation", "")),
|
||||||
|
snapshot_subvol=str(entry.get("snapshot_subvol", "")),
|
||||||
|
var_snapshot=entry.get("var_snapshot"),
|
||||||
|
status=str(entry.get("status", "")),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def newest_committed_coordinated(
|
||||||
|
records: list[SnapshotRecord],
|
||||||
|
) -> SnapshotRecord | None:
|
||||||
|
"""The newest committed snapshot that also captured ``/var``.
|
||||||
|
|
||||||
|
A root snapshot with no paired ``@var`` snapshot cannot restore a
|
||||||
|
consistent package database, so it is not a valid rollback target.
|
||||||
|
"""
|
||||||
|
candidates = [
|
||||||
|
r
|
||||||
|
for r in records
|
||||||
|
if r.status == "committed" and r.snapshot_subvol and r.var_snapshot
|
||||||
|
]
|
||||||
|
if not candidates:
|
||||||
|
return None
|
||||||
|
return candidates[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_rollback_state(text: str) -> dict:
|
||||||
|
"""Parse ``/var/lib/phoebus/rollback-state.toml`` into a dict."""
|
||||||
|
text = text.strip()
|
||||||
|
if not text:
|
||||||
|
raise ValueError("empty rollback state")
|
||||||
|
try:
|
||||||
|
return tomllib.loads(text)
|
||||||
|
except tomllib.TOMLDecodeError as exc: # pragma: no cover - message detail
|
||||||
|
raise ValueError(f"invalid rollback state: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_subvol(value: str | None) -> str | None:
|
||||||
|
"""Canonicalise a subvolume path for comparison.
|
||||||
|
|
||||||
|
Btrfs reports the same subvolume as ``@restore-x`` in ``rootflags`` and
|
||||||
|
``/@restore-x`` in a fstab mount option; comparison must ignore that.
|
||||||
|
"""
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
return value.lstrip("/")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_rootflags_subvol(cmdline: str) -> str | None:
|
||||||
|
"""Extract the ``subvol=`` value from the kernel ``rootflags=`` option."""
|
||||||
|
match = re.search(r"rootflags=[^ ]*?\bsubvol=([^,\s]+)", cmdline)
|
||||||
|
if match:
|
||||||
|
return normalize_subvol(match.group(1))
|
||||||
|
# Some kernels present rootflags only as the bare option.
|
||||||
|
match = re.search(r"\bsubvol=([^,\s]+)", cmdline)
|
||||||
|
return normalize_subvol(match.group(1)) if match else None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_fstab_subvol(text: str, mountpoint: str) -> str | None:
|
||||||
|
"""The ``subvol=`` value for a mountpoint in ``/etc/fstab``."""
|
||||||
|
for line in text.splitlines():
|
||||||
|
line = line.split("#", 1)[0].strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
fields = line.split()
|
||||||
|
if len(fields) < 4:
|
||||||
|
continue
|
||||||
|
if fields[1] != mountpoint:
|
||||||
|
continue
|
||||||
|
match = re.search(r"(?:^|,)subvol=([^,\s]+)", fields[3])
|
||||||
|
if match:
|
||||||
|
return normalize_subvol(match.group(1))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_package_dirs(text: str) -> list[str]:
|
||||||
|
"""Absolute ``/var/db/pkg/<cat>/<pkg>`` directories from a listing."""
|
||||||
|
return sorted({line.strip() for line in text.splitlines() if line.strip()})
|
||||||
|
|
||||||
|
|
||||||
|
def parse_contents_files(text: str) -> list[str]:
|
||||||
|
"""Installed file paths from a Portage ``CONTENTS`` file.
|
||||||
|
|
||||||
|
The format is ``<obj> <path> <hash...>`` per line.
|
||||||
|
"""
|
||||||
|
paths: list[str] = []
|
||||||
|
for line in text.splitlines():
|
||||||
|
fields = line.split()
|
||||||
|
if len(fields) >= 2 and fields[0] in ("obj", "dir", "sym"):
|
||||||
|
paths.append(fields[1])
|
||||||
|
return paths
|
||||||
|
|
||||||
|
|
||||||
|
def parse_uname(text: str) -> str:
|
||||||
|
return text.strip().splitlines()[-1].strip() if text.strip() else ""
|
||||||
|
|
||||||
|
|
||||||
|
def parse_network_addresses(text: str) -> list[str]:
|
||||||
|
"""Interfaces that have an address, from ``ip -brief addr`` output."""
|
||||||
|
out: list[str] = []
|
||||||
|
for line in text.splitlines():
|
||||||
|
fields = line.split()
|
||||||
|
if not fields:
|
||||||
|
continue
|
||||||
|
name = fields[0]
|
||||||
|
rest = fields[1:]
|
||||||
|
if any("." in f or ":" in f for f in rest):
|
||||||
|
out.append(name)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Journey
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class _Step:
|
||||||
|
name: str
|
||||||
|
method_name: str
|
||||||
|
boot: str | None = None
|
||||||
|
stream: str = "rollback"
|
||||||
|
|
||||||
|
|
||||||
|
# Order is the contract. A change here changes the release gate; keep every
|
||||||
|
# expected step in the list so a stopped run can mark the rest NOT RUN.
|
||||||
|
_STEPS: tuple[_Step, ...] = (
|
||||||
|
_Step("sys-binary-identity", "_step_sys_identity", boot="installed"),
|
||||||
|
_Step("known-good-coordinated-snapshot", "_step_known_good"),
|
||||||
|
_Step("home-data-marker", "_step_mark_home"),
|
||||||
|
_Step("controlled-package-change", "_step_package_change"),
|
||||||
|
_Step("deterministic-guest-breakage", "_step_break"),
|
||||||
|
_Step("sys-undo-activates", "_step_undo"),
|
||||||
|
_Step("reboot-detached", "_step_reboot", boot="installed"),
|
||||||
|
_Step("restored-root-var-generations", "_step_generations"),
|
||||||
|
_Step("package-files-restored", "_step_package_files"),
|
||||||
|
_Step("portage-database-consistent", "_step_portage_db"),
|
||||||
|
_Step("kernel-modules-compatible", "_step_kernel_modules"),
|
||||||
|
_Step("home-data-survives", "_step_home_survives"),
|
||||||
|
_Step("recovery-entry-armed", "_step_arm_recovery"),
|
||||||
|
_Step(
|
||||||
|
"recovery-boot-no-network-no-model",
|
||||||
|
"_step_recovery",
|
||||||
|
boot="recovery",
|
||||||
|
stream="recovery",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
EXPECTED_STEP_NAMES: tuple[str, ...] = tuple(step.name for step in _STEPS)
|
||||||
|
|
||||||
|
|
||||||
|
class RollbackJourney:
|
||||||
|
"""Run the ordered acceptance steps against a :class:`BootTransport`.
|
||||||
|
|
||||||
|
The class records every expected step, so a run that stops early reports the
|
||||||
|
remainder as NOT RUN and fails an explicit ``journey-complete`` check rather
|
||||||
|
than silently looking like a pass.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
transport: BootTransport,
|
||||||
|
*,
|
||||||
|
results: Results | None = None,
|
||||||
|
recovery_results: Results | None = None,
|
||||||
|
home: str = "/home/phoebus",
|
||||||
|
sys_bin: str = DEFAULT_SYS,
|
||||||
|
package: str = DEFAULT_PACKAGE,
|
||||||
|
iso_sha256: str | None = None,
|
||||||
|
sys_sha256: str | None = None,
|
||||||
|
mocked: bool = False,
|
||||||
|
):
|
||||||
|
self.transport = transport
|
||||||
|
self.results = results or Results(SCENARIO_ROLLBACK)
|
||||||
|
self.recovery_results = recovery_results or Results(SCENARIO_RECOVERY)
|
||||||
|
for stream in (self.results, self.recovery_results):
|
||||||
|
if iso_sha256:
|
||||||
|
stream.bind_input(INPUT_ISO, iso_sha256)
|
||||||
|
if sys_sha256:
|
||||||
|
stream.bind_input(INPUT_SYS, sys_sha256)
|
||||||
|
if mocked:
|
||||||
|
stream.mark_mocked("fake transport: not release evidence")
|
||||||
|
self.home = home
|
||||||
|
self.sys_bin = sys_bin
|
||||||
|
self.package = package
|
||||||
|
self.expected_sys_sha256 = sys_sha256
|
||||||
|
self.marker_path = f"{home.rstrip('/')}/{HOME_MARKER_REL}"
|
||||||
|
self.marker_token = f"{HOME_MARKER_TOKEN}:{self.results.run_id}"
|
||||||
|
self.boots: list[str] = []
|
||||||
|
self.executed: list[str] = []
|
||||||
|
|
||||||
|
# Findings carried between steps.
|
||||||
|
self.snapshot: SnapshotRecord | None = None
|
||||||
|
self.original_hostname: str | None = None
|
||||||
|
self.package_dirs_before: list[str] = []
|
||||||
|
self.added_package_dirs: list[str] = []
|
||||||
|
self.added_package_files: list[str] = []
|
||||||
|
self.rollback_state: dict = {}
|
||||||
|
|
||||||
|
self._active = self.results
|
||||||
|
|
||||||
|
# -- helpers -----------------------------------------------------------
|
||||||
|
def _check(self, name: str, condition: bool, detail: str = "") -> bool:
|
||||||
|
return self._active.assert_true(name, condition, detail)
|
||||||
|
|
||||||
|
def _run(
|
||||||
|
self, guest: Guest, script: str, timeout: float = 240.0
|
||||||
|
) -> CommandResult:
|
||||||
|
try:
|
||||||
|
return guest.run(script, timeout=timeout).bounded()
|
||||||
|
except Exception as exc: # a transport fault is a failure, not a crash
|
||||||
|
return CommandResult(
|
||||||
|
command=script, disconnected=True, stderr=str(exc)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _read_file(self, guest: Guest, path: str, timeout: float = 60.0) -> str:
|
||||||
|
result = self._run(
|
||||||
|
guest,
|
||||||
|
f"echo {FILE_FRAME}; cat {path} 2>/dev/null; echo {FILE_FRAME}",
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
return extract_between(result.stdout, FILE_FRAME, FILE_FRAME)
|
||||||
|
|
||||||
|
# -- public API --------------------------------------------------------
|
||||||
|
def required_inputs(self) -> tuple[str, ...]:
|
||||||
|
return REQUIRED_INPUTS[self.results.scenario]
|
||||||
|
|
||||||
|
def run(self) -> bool:
|
||||||
|
"""Execute the journey. Returns True only if every step passed."""
|
||||||
|
if not self._check_inputs():
|
||||||
|
for step in _STEPS:
|
||||||
|
stream = (
|
||||||
|
self.results
|
||||||
|
if step.stream == "rollback"
|
||||||
|
else self.recovery_results
|
||||||
|
)
|
||||||
|
stream.mark_not_run(step.name, "input identities were not pinned")
|
||||||
|
self.results.fail("journey-complete", "missing input identities")
|
||||||
|
self.recovery_results.fail("journey-complete", "missing input identities")
|
||||||
|
return False
|
||||||
|
|
||||||
|
guest: Guest | None = None
|
||||||
|
for index, step in enumerate(_STEPS):
|
||||||
|
self._active = (
|
||||||
|
self.results if step.stream == "rollback" else self.recovery_results
|
||||||
|
)
|
||||||
|
if step.boot is not None:
|
||||||
|
if guest is not None:
|
||||||
|
self.transport.poweroff()
|
||||||
|
guest = (
|
||||||
|
self.transport.boot_recovery()
|
||||||
|
if step.boot == "recovery"
|
||||||
|
else self.transport.boot_installed()
|
||||||
|
)
|
||||||
|
self.boots.append(step.boot)
|
||||||
|
method: Callable[[Guest], bool] = getattr(self, step.method_name)
|
||||||
|
self.executed.append(step.name)
|
||||||
|
if not method(guest):
|
||||||
|
self._finish_incomplete(index, step.name)
|
||||||
|
return False
|
||||||
|
|
||||||
|
self.results.assert_true("journey-complete", True)
|
||||||
|
self.recovery_results.assert_true("journey-complete", True)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def run_recovery(self) -> bool:
|
||||||
|
"""Verify only the no-model recovery boot (used by the `recovery` mode)."""
|
||||||
|
if not self._check_inputs():
|
||||||
|
self.recovery_results.mark_not_run(
|
||||||
|
"recovery-boot-no-network-no-model", "input identities were not pinned"
|
||||||
|
)
|
||||||
|
self.recovery_results.fail("journey-complete", "missing input identities")
|
||||||
|
return False
|
||||||
|
self._active = self.recovery_results
|
||||||
|
ok = self._step_recovery(self.transport.boot_recovery())
|
||||||
|
if ok:
|
||||||
|
self.recovery_results.assert_true("journey-complete", True)
|
||||||
|
else:
|
||||||
|
self.recovery_results.fail(
|
||||||
|
"journey-complete", "recovery verification failed"
|
||||||
|
)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
# -- input identity gate ----------------------------------------------
|
||||||
|
def _check_inputs(self) -> bool:
|
||||||
|
ok = True
|
||||||
|
for stream, scenario in (
|
||||||
|
(self.results, SCENARIO_ROLLBACK),
|
||||||
|
(self.recovery_results, SCENARIO_RECOVERY),
|
||||||
|
):
|
||||||
|
required = REQUIRED_INPUTS[scenario]
|
||||||
|
missing = stream.missing_inputs(required)
|
||||||
|
ok &= stream.assert_true(
|
||||||
|
"input-identities",
|
||||||
|
not missing,
|
||||||
|
f"missing pinned inputs for {scenario}: {', '.join(missing)}",
|
||||||
|
)
|
||||||
|
return bool(ok)
|
||||||
|
|
||||||
|
def _finish_incomplete(self, failed_index: int, failed_step: str) -> None:
|
||||||
|
for step in _STEPS[failed_index + 1:]:
|
||||||
|
stream = (
|
||||||
|
self.results if step.stream == "rollback" else self.recovery_results
|
||||||
|
)
|
||||||
|
stream.mark_not_run(step.name, "journey stopped before this step")
|
||||||
|
# The failed step's own stream already has a FAIL; make the incompleteness
|
||||||
|
# explicit on both so neither gate can read a partial run as a pass.
|
||||||
|
self.results.fail(
|
||||||
|
"journey-complete", f"stopped after `{failed_step}`"
|
||||||
|
)
|
||||||
|
self.recovery_results.fail(
|
||||||
|
"journey-complete", f"stopped after `{failed_step}`"
|
||||||
|
)
|
||||||
|
|
||||||
|
# -- steps -------------------------------------------------------------
|
||||||
|
def _step_sys_identity(self, guest: Guest) -> bool:
|
||||||
|
# The rollback semantics live in `sys`; pin the exact binary under test.
|
||||||
|
self._active = self.results
|
||||||
|
expected = self.expected_sys_sha256 or ""
|
||||||
|
result = self._run(guest, f"sha256sum {shlex_quote(self.sys_bin)}", timeout=60.0)
|
||||||
|
if not result.ok:
|
||||||
|
return self._check("sys-binary-identity", False, result.failure_reason)
|
||||||
|
digest = result.stdout.split()[0] if result.stdout.split() else ""
|
||||||
|
return self._check(
|
||||||
|
"sys-binary-identity",
|
||||||
|
bool(expected) and digest == expected,
|
||||||
|
f"guest sys digest {digest!r} != pinned {expected!r}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _step_known_good(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
text = self._read_file(guest, SNAPSHOT_HISTORY)
|
||||||
|
try:
|
||||||
|
records = parse_snapshot_history(text)
|
||||||
|
except ValueError as exc:
|
||||||
|
return self._check("known-good-coordinated-snapshot", False, str(exc))
|
||||||
|
snapshot = newest_committed_coordinated(records)
|
||||||
|
if snapshot is None:
|
||||||
|
return self._check(
|
||||||
|
"known-good-coordinated-snapshot",
|
||||||
|
False,
|
||||||
|
"no committed snapshot with a paired @var snapshot",
|
||||||
|
)
|
||||||
|
self.snapshot = snapshot
|
||||||
|
self._check("known-good-root-snapshot", bool(snapshot.snapshot_subvol), "empty root snapshot path")
|
||||||
|
self._check("known-good-var-snapshot", bool(snapshot.var_snapshot), "empty @var snapshot path")
|
||||||
|
self._check(
|
||||||
|
"known-good-coordinated-snapshot",
|
||||||
|
True,
|
||||||
|
f"{snapshot.id} ({snapshot.operation})",
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _step_mark_home(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
result = self._run(
|
||||||
|
guest,
|
||||||
|
f"mkdir -p {self.home.rstrip('/')}/{HOME_MARKER_REL.rsplit('/', 1)[0]} && "
|
||||||
|
f"printf '%s' '{self.marker_token}' > {self.marker_path}",
|
||||||
|
)
|
||||||
|
if not result.ok:
|
||||||
|
return self._check("home-data-marker", False, result.failure_reason)
|
||||||
|
readback = self._read_file(guest, self.marker_path)
|
||||||
|
if readback.strip() != self.marker_token:
|
||||||
|
return self._check(
|
||||||
|
"home-data-marker", False, "home marker did not read back"
|
||||||
|
)
|
||||||
|
self.original_hostname = self._read_file(guest, HOSTNAME_FILE).strip()
|
||||||
|
self.package_dirs_before = parse_package_dirs(
|
||||||
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||||
|
)
|
||||||
|
self._check("home-data-marker", True, self.marker_path)
|
||||||
|
self._check(
|
||||||
|
"hostname-recorded", bool(self.original_hostname), "could not read /etc/hostname"
|
||||||
|
)
|
||||||
|
return bool(self.original_hostname)
|
||||||
|
|
||||||
|
def _step_package_change(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
result = self._run(
|
||||||
|
guest, f"sudo {self.sys_bin} install {self.package} -y", timeout=1800.0
|
||||||
|
)
|
||||||
|
if not result.ok:
|
||||||
|
return self._check("controlled-package-change", False, result.failure_reason)
|
||||||
|
after = parse_package_dirs(
|
||||||
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||||
|
)
|
||||||
|
added = sorted(set(after) - set(self.package_dirs_before))
|
||||||
|
if not added:
|
||||||
|
return self._check(
|
||||||
|
"controlled-package-change",
|
||||||
|
False,
|
||||||
|
f"`{self.package}` produced no new /var/db/pkg entry",
|
||||||
|
)
|
||||||
|
self.added_package_dirs = added
|
||||||
|
contents = self._read_file(guest, f"{added[0]}/CONTENTS")
|
||||||
|
self.added_package_files = parse_contents_files(contents)
|
||||||
|
self._check(
|
||||||
|
"controlled-package-change", True, f"added {', '.join(added)}"
|
||||||
|
)
|
||||||
|
self._check(
|
||||||
|
"package-db-recorded",
|
||||||
|
bool(self.added_package_files),
|
||||||
|
"changed package has no recorded files",
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _step_break(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
result = self._run(
|
||||||
|
guest,
|
||||||
|
f"printf '%s\\n' '{HOSTNAME_SENTINEL}' | sudo tee {HOSTNAME_FILE} >/dev/null",
|
||||||
|
)
|
||||||
|
if not result.ok:
|
||||||
|
return self._check("deterministic-guest-breakage", False, result.failure_reason)
|
||||||
|
broken = self._read_file(guest, HOSTNAME_FILE).strip()
|
||||||
|
if broken != HOSTNAME_SENTINEL:
|
||||||
|
return self._check(
|
||||||
|
"deterministic-guest-breakage",
|
||||||
|
False,
|
||||||
|
"breakage did not take effect",
|
||||||
|
)
|
||||||
|
self._check(
|
||||||
|
"deterministic-guest-breakage",
|
||||||
|
True,
|
||||||
|
f"{HOSTNAME_FILE} -> {HOSTNAME_SENTINEL}",
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _step_undo(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
if self.snapshot is None:
|
||||||
|
return self._check("sys-undo-activates", False, "no snapshot selected")
|
||||||
|
result = self._run(
|
||||||
|
guest, f"sudo {self.sys_bin} undo {self.snapshot.id} -y", timeout=900.0
|
||||||
|
)
|
||||||
|
if not result.ok:
|
||||||
|
return self._check("sys-undo-activates", False, result.failure_reason)
|
||||||
|
state = self._read_file(guest, ROLLBACK_STATE)
|
||||||
|
try:
|
||||||
|
self.rollback_state = parse_rollback_state(state)
|
||||||
|
except ValueError as exc:
|
||||||
|
return self._check("sys-undo-activates", False, str(exc))
|
||||||
|
if self.rollback_state.get("status") != "activated":
|
||||||
|
return self._check(
|
||||||
|
"sys-undo-activates",
|
||||||
|
False,
|
||||||
|
f"rollback state is {self.rollback_state.get('status')!r}, not 'activated'",
|
||||||
|
)
|
||||||
|
self._check(
|
||||||
|
"sys-undo-activates",
|
||||||
|
True,
|
||||||
|
f"next boot uses {self.rollback_state.get('root_subvol')}",
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _step_reboot(self, guest: Guest) -> bool:
|
||||||
|
# The boot itself is performed by the transport; the driver asserts the
|
||||||
|
# ISO is detached before this returns. Here we only record the boundary.
|
||||||
|
self._active = self.results
|
||||||
|
self._check("reboot-detached", True, "booted installed disk with ISO detached")
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _step_generations(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
expected_root = str(self.rollback_state.get("root_subvol", ""))
|
||||||
|
expected_var = str(self.rollback_state.get("var_subvol", "") or "")
|
||||||
|
cmdline = self._read_file(guest, "/proc/cmdline")
|
||||||
|
booted_subvol = parse_rootflags_subvol(cmdline)
|
||||||
|
fstab = self._read_file(guest, "/etc/fstab")
|
||||||
|
fstab_root = parse_fstab_subvol(fstab, "/")
|
||||||
|
fstab_var = parse_fstab_subvol(fstab, "/var")
|
||||||
|
ok = True
|
||||||
|
ok &= self._check(
|
||||||
|
"restored-root-generation",
|
||||||
|
bool(expected_root) and booted_subvol == expected_root,
|
||||||
|
f"booted subvol {booted_subvol!r} != expected {expected_root!r}",
|
||||||
|
)
|
||||||
|
ok &= self._check(
|
||||||
|
"restored-fstab-root",
|
||||||
|
bool(expected_root) and fstab_root == expected_root,
|
||||||
|
f"fstab / subvol {fstab_root!r} != {expected_root!r}",
|
||||||
|
)
|
||||||
|
ok &= self._check(
|
||||||
|
"restored-var-generation",
|
||||||
|
bool(expected_var) and fstab_var == expected_var,
|
||||||
|
f"fstab /var subvol {fstab_var!r} != {expected_var!r}",
|
||||||
|
)
|
||||||
|
# The running system must not still be the pre-rollback generation.
|
||||||
|
ok &= self._check(
|
||||||
|
"generation-advanced",
|
||||||
|
booted_subvol.startswith("@restore-") if booted_subvol else False,
|
||||||
|
f"booted subvol {booted_subvol!r} is not a restored generation",
|
||||||
|
)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
def _step_package_files(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
after = parse_package_dirs(
|
||||||
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||||
|
)
|
||||||
|
still_present = [d for d in self.added_package_dirs if d in after]
|
||||||
|
ok = self._check(
|
||||||
|
"package-db-restored",
|
||||||
|
not still_present,
|
||||||
|
f"package DB still lists {still_present}",
|
||||||
|
)
|
||||||
|
missing_files = []
|
||||||
|
for path in self.added_package_files:
|
||||||
|
result = self._run(guest, f"test -e {shlex_quote(path)}")
|
||||||
|
if result.exit_code == 0:
|
||||||
|
missing_files.append(path)
|
||||||
|
ok &= self._check(
|
||||||
|
"package-files-restored",
|
||||||
|
not missing_files,
|
||||||
|
f"package files survived the rollback: {missing_files[:5]}",
|
||||||
|
)
|
||||||
|
hostname = self._read_file(guest, HOSTNAME_FILE).strip()
|
||||||
|
ok &= self._check(
|
||||||
|
"breakage-reverted",
|
||||||
|
hostname == (self.original_hostname or ""),
|
||||||
|
f"{HOSTNAME_FILE} is {hostname!r}, expected {self.original_hostname!r}",
|
||||||
|
)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
def _step_portage_db(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
listing = parse_package_dirs(
|
||||||
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||||
|
)
|
||||||
|
ok = self._check(
|
||||||
|
"portage-db-generation-count",
|
||||||
|
listing == self.package_dirs_before,
|
||||||
|
f"/var/db/pkg has {len(listing)} entries, expected {len(self.package_dirs_before)}",
|
||||||
|
)
|
||||||
|
# Every entry must carry a CONTENTS file; an inconsistent database is
|
||||||
|
# how a mismatched root/var pairing shows up.
|
||||||
|
probe = self._run(
|
||||||
|
guest,
|
||||||
|
f"for d in {PKG_DB}/*/*; do [ -f \"$d/CONTENTS\" ] || echo \"$d\"; done",
|
||||||
|
)
|
||||||
|
missing_contents = [
|
||||||
|
line.strip() for line in probe.stdout.splitlines() if line.strip()
|
||||||
|
]
|
||||||
|
ok &= self._check(
|
||||||
|
"portage-db-contents",
|
||||||
|
not missing_contents,
|
||||||
|
f"package entries without CONTENTS: {missing_contents[:5]}",
|
||||||
|
)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
def _step_kernel_modules(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
kernel = parse_uname(self._run(guest, "uname -r").stdout)
|
||||||
|
modules = self._read_file(guest, "/lib/modules")
|
||||||
|
module_dirs = [line.strip() for line in modules.splitlines() if line.strip()]
|
||||||
|
return self._check(
|
||||||
|
"kernel-modules-compatible",
|
||||||
|
bool(kernel) and kernel in module_dirs,
|
||||||
|
f"no /lib/modules/{kernel} in the restored root ({module_dirs})",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _step_home_survives(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
readback = self._read_file(guest, self.marker_path).strip()
|
||||||
|
return self._check(
|
||||||
|
"home-data-survives",
|
||||||
|
readback == self.marker_token,
|
||||||
|
"the /home marker did not survive the rollback",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _step_arm_recovery(self, guest: Guest) -> bool:
|
||||||
|
# Boot the recovery entry exactly once, then fall back to the restored
|
||||||
|
# root. This is a real systemd-boot interface, not a QEMU shortcut.
|
||||||
|
self._active = self.results
|
||||||
|
result = self._run(
|
||||||
|
guest, "sudo bootctl set-oneshot phoebus-recovery", timeout=120.0
|
||||||
|
)
|
||||||
|
return self._check("recovery-entry-armed", result.ok, result.failure_reason)
|
||||||
|
|
||||||
|
def _step_recovery(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.recovery_results
|
||||||
|
cmdline = self._read_file(guest, "/proc/cmdline")
|
||||||
|
ok = self._check(
|
||||||
|
"recovery-entry-booted",
|
||||||
|
"systemd.unit=rescue.target" in cmdline,
|
||||||
|
"the recovery entry did not boot into rescue.target",
|
||||||
|
)
|
||||||
|
addresses = parse_network_addresses(self._run(guest, "ip -brief addr").stdout)
|
||||||
|
non_loopback = [a for a in addresses if a != "lo"]
|
||||||
|
ok &= self._check(
|
||||||
|
"recovery-no-network",
|
||||||
|
not non_loopback,
|
||||||
|
f"network interfaces were up: {non_loopback}",
|
||||||
|
)
|
||||||
|
# `sys` itself must work with no model: a status query is deterministic
|
||||||
|
# and offline.
|
||||||
|
status = self._run(guest, f"{self.sys_bin} undo status")
|
||||||
|
ok &= self._check(
|
||||||
|
"recovery-sys-offline",
|
||||||
|
status.ok,
|
||||||
|
status.failure_reason or "sys status failed offline",
|
||||||
|
)
|
||||||
|
ask = self._run(guest, f'{self.sys_bin} ask "show system status"', timeout=120.0)
|
||||||
|
combined = (ask.stdout + ask.stderr).lower()
|
||||||
|
ok &= self._check(
|
||||||
|
"recovery-no-model-fallback",
|
||||||
|
(not ask.ok) and "no model" in combined,
|
||||||
|
"sys ask did not report the deterministic no-model fallback",
|
||||||
|
)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
|
||||||
|
def shlex_quote(value: str) -> str:
|
||||||
|
"""Minimal single-quote shell escaping for a known-safe path."""
|
||||||
|
return "'" + value.replace("'", "'\\''") + "'"
|
||||||
|
|
||||||
|
|
||||||
|
def validate_acceptance_disks(disks, test_dir) -> list:
|
||||||
|
"""Refuse anything but disposable qcow2 disks inside the test directory.
|
||||||
|
|
||||||
|
Kept here so the journey's safety entry point is unit-testable without QEMU.
|
||||||
|
"""
|
||||||
|
return vmsafety.validate_writable_drives(disks, test_dir)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"SCENARIO_ROLLBACK",
|
||||||
|
"SCENARIO_RECOVERY",
|
||||||
|
"REQUIRED_INPUTS",
|
||||||
|
"EXPECTED_STEP_NAMES",
|
||||||
|
"CommandResult",
|
||||||
|
"SnapshotRecord",
|
||||||
|
"Guest",
|
||||||
|
"BootTransport",
|
||||||
|
"RollbackJourney",
|
||||||
|
"validate_acceptance_disks",
|
||||||
|
"extract_between",
|
||||||
|
"normalize_subvol",
|
||||||
|
"parse_snapshot_history",
|
||||||
|
"newest_committed_coordinated",
|
||||||
|
"parse_rollback_state",
|
||||||
|
"parse_rootflags_subvol",
|
||||||
|
"parse_fstab_subvol",
|
||||||
|
"parse_package_dirs",
|
||||||
|
"parse_contents_files",
|
||||||
|
"parse_uname",
|
||||||
|
"parse_network_addresses",
|
||||||
|
]
|
||||||
497
installer/tests/vm/test_rollback_acceptance.py
Normal file
497
installer/tests/vm/test_rollback_acceptance.py
Normal file
|
|
@ -0,0 +1,497 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Mocked-transport tests for the rollback acceptance orchestration.
|
||||||
|
|
||||||
|
These run on any host (no QEMU, no disk), so the journey's ordering, safety and
|
||||||
|
failure semantics are checked in CI rather than only during a live run. Every
|
||||||
|
transport here is a fake: the results are marked ``mocked`` and are explicitly
|
||||||
|
**not** release evidence (the release tool refuses them).
|
||||||
|
|
||||||
|
python3 installer/tests/vm/test_rollback_acceptance.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import rollback_acceptance as ra
|
||||||
|
from rollback_acceptance import (
|
||||||
|
SCENARIO_RECOVERY,
|
||||||
|
SCENARIO_ROLLBACK,
|
||||||
|
CommandResult,
|
||||||
|
RollbackJourney,
|
||||||
|
validate_acceptance_disks,
|
||||||
|
)
|
||||||
|
from vmsafety import FAIL, MOCKED, NOT_RUN, Results, UnsafeDiskError
|
||||||
|
|
||||||
|
BASE_PKGS = [
|
||||||
|
"/var/db/pkg/sys-apps/busybox-1.36.1",
|
||||||
|
"/var/db/pkg/app-shells/bash-5.2_p26",
|
||||||
|
]
|
||||||
|
ADDED_PKG = "/var/db/pkg/app-misc/hello-2.12"
|
||||||
|
HISTORY = """\
|
||||||
|
[[records]]
|
||||||
|
id = "snap-initial"
|
||||||
|
created_unix = 1000
|
||||||
|
operation = "initial known-good"
|
||||||
|
source_subvol = "/"
|
||||||
|
snapshot_subvol = "@-20260101T000000-initial-known-good"
|
||||||
|
var_snapshot = "@var-20260101T000000-initial-known-good"
|
||||||
|
status = "committed"
|
||||||
|
"""
|
||||||
|
ROLLBACK_STATE = """\
|
||||||
|
status = "activated"
|
||||||
|
generation = 1001
|
||||||
|
target_snapshot = "@-20260101T000000-initial-known-good"
|
||||||
|
previous_root = "@"
|
||||||
|
root_subvol = "@restore-20260101T000000"
|
||||||
|
var_subvol = "@var-restore-20260101T000000"
|
||||||
|
started_unix = 1001
|
||||||
|
finished_unix = 1001
|
||||||
|
note = "restore from snap-initial"
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class FakeGuest:
|
||||||
|
"""A scripted guest. First matching handler wins."""
|
||||||
|
|
||||||
|
def __init__(self, name: str):
|
||||||
|
self.name = name
|
||||||
|
self.commands: list[str] = []
|
||||||
|
self.handlers: list = []
|
||||||
|
|
||||||
|
def on(self, needle: str, handler):
|
||||||
|
# A re-registered needle overrides the earlier one, so a test can tweak
|
||||||
|
# one behaviour without disturbing the ordering of distinct needles.
|
||||||
|
self.handlers = [(n, h) for (n, h) in self.handlers if n != needle]
|
||||||
|
self.handlers.append((needle, handler))
|
||||||
|
return self
|
||||||
|
|
||||||
|
def reply(self, needle: str, stdout: str = "", exit_code: int = 0):
|
||||||
|
return self.on(needle, lambda cmd: CommandResult(cmd, stdout=stdout, exit_code=exit_code))
|
||||||
|
|
||||||
|
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
||||||
|
self.commands.append(command)
|
||||||
|
for needle, handler in self.handlers:
|
||||||
|
if needle in command:
|
||||||
|
return handler(command)
|
||||||
|
return CommandResult(command=command, exit_code=0)
|
||||||
|
|
||||||
|
|
||||||
|
class FakeTransport:
|
||||||
|
def __init__(self, installed, recovery):
|
||||||
|
# installed is a list of guests returned by successive boot_installed()
|
||||||
|
# calls; the first is the change/undo boot, the second the restored boot.
|
||||||
|
self._installed = list(installed)
|
||||||
|
self._recovery = recovery
|
||||||
|
self.boots: list[str] = []
|
||||||
|
self.powered_off = 0
|
||||||
|
|
||||||
|
def boot_installed(self):
|
||||||
|
self.boots.append("installed")
|
||||||
|
return self._installed.pop(0)
|
||||||
|
|
||||||
|
def boot_recovery(self):
|
||||||
|
self.boots.append("recovery")
|
||||||
|
return self._recovery
|
||||||
|
|
||||||
|
def poweroff(self):
|
||||||
|
self.powered_off += 1
|
||||||
|
|
||||||
|
|
||||||
|
def make_phase1(state: dict) -> FakeGuest:
|
||||||
|
g = FakeGuest("phase1")
|
||||||
|
|
||||||
|
def write_marker(cmd):
|
||||||
|
match = re.search(r"printf '%s' '([^']*)' >", cmd)
|
||||||
|
if match:
|
||||||
|
state["token"] = match.group(1)
|
||||||
|
return CommandResult(cmd, exit_code=0)
|
||||||
|
|
||||||
|
def read_marker(cmd):
|
||||||
|
return CommandResult(cmd, stdout=state.get("token", ""))
|
||||||
|
|
||||||
|
def break_hostname(cmd):
|
||||||
|
state["broken"] = True
|
||||||
|
return CommandResult(cmd, exit_code=0)
|
||||||
|
|
||||||
|
def read_hostname(cmd):
|
||||||
|
return CommandResult(
|
||||||
|
cmd,
|
||||||
|
stdout=ra.HOSTNAME_SENTINEL + "\n" if state.get("broken") else "phoebus\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
def install(cmd):
|
||||||
|
state["installed"] = True
|
||||||
|
return CommandResult(cmd, exit_code=0)
|
||||||
|
|
||||||
|
def pkg_dirs(cmd):
|
||||||
|
dirs = list(BASE_PKGS)
|
||||||
|
if state.get("installed"):
|
||||||
|
dirs.append(ADDED_PKG)
|
||||||
|
return CommandResult(cmd, stdout="\n".join(dirs) + "\n")
|
||||||
|
|
||||||
|
g.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=HISTORY))
|
||||||
|
g.on("sha256sum", lambda c: CommandResult(c, exit_code=0, stdout="b" * 64 + " /usr/local/bin/sys\n"))
|
||||||
|
g.on("> /home/phoebus/rollback-acceptance/marker", write_marker)
|
||||||
|
g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker)
|
||||||
|
g.on("tee /etc/hostname", break_hostname)
|
||||||
|
g.on("cat /etc/hostname", read_hostname)
|
||||||
|
g.on("sys install", install)
|
||||||
|
g.on("find /var/db/pkg", pkg_dirs)
|
||||||
|
g.on("cat " + ADDED_PKG + "/CONTENTS", lambda c: CommandResult(
|
||||||
|
c, stdout="obj /usr/bin/hello abcdef\nobj /usr/share/hello/README def\n"
|
||||||
|
))
|
||||||
|
g.on("sys undo", lambda c: CommandResult(c, exit_code=0, stdout="rollback activated"))
|
||||||
|
g.on("cat /var/lib/phoebus/rollback-state.toml", lambda c: CommandResult(c, stdout=ROLLBACK_STATE))
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
def make_phase2(state: dict) -> FakeGuest:
|
||||||
|
g = FakeGuest("phase2")
|
||||||
|
|
||||||
|
def read_marker(cmd):
|
||||||
|
return CommandResult(cmd, stdout=state.get("token", ""))
|
||||||
|
|
||||||
|
fstab = (
|
||||||
|
"/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n"
|
||||||
|
"/dev/mapper/luks-root /var btrfs subvol=/@var-restore-20260101T000000 0 0\n"
|
||||||
|
"/dev/mapper/luks-root /home btrfs subvol=/@home 0 0\n"
|
||||||
|
)
|
||||||
|
g.on("cat /etc/hostname", lambda c: CommandResult(c, stdout="phoebus\n"))
|
||||||
|
g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker)
|
||||||
|
g.on("find /var/db/pkg", lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS) + "\n"))
|
||||||
|
g.on("test -e", lambda c: CommandResult(c, exit_code=1))
|
||||||
|
g.on("cat /proc/cmdline", lambda c: CommandResult(
|
||||||
|
c, stdout="rootflags=subvol=@restore-20260101T000000 quiet console=ttyS0\n"
|
||||||
|
))
|
||||||
|
g.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab))
|
||||||
|
g.on("uname -r", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n"))
|
||||||
|
g.on("cat /lib/modules", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n"))
|
||||||
|
g.on("bootctl set-oneshot", lambda c: CommandResult(c, exit_code=0))
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
def make_recovery() -> FakeGuest:
|
||||||
|
g = FakeGuest("recovery")
|
||||||
|
g.on("cat /proc/cmdline", lambda c: CommandResult(
|
||||||
|
c, stdout="rootflags=subvol=@-20260101T000000-initial-known-good "
|
||||||
|
"systemd.unit=rescue.target no-network\n"
|
||||||
|
))
|
||||||
|
g.on("ip -brief addr", lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8 ::1/128\n"))
|
||||||
|
g.on("sys undo status", lambda c: CommandResult(c, exit_code=0, stdout="no active rollback\n"))
|
||||||
|
g.on("sys ask", lambda c: CommandResult(
|
||||||
|
c,
|
||||||
|
exit_code=1,
|
||||||
|
stderr="No model is connected. The deterministic interface still works:\n",
|
||||||
|
))
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
def build_journey(*, state=None, transport=None, **kwargs):
|
||||||
|
state = state if state is not None else {}
|
||||||
|
transport = transport or FakeTransport([make_phase1(state), make_phase2(state)], make_recovery())
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport,
|
||||||
|
iso_sha256="a" * 64,
|
||||||
|
sys_sha256="b" * 64,
|
||||||
|
mocked=True,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
return journey
|
||||||
|
|
||||||
|
|
||||||
|
def status_of(results: Results, name: str) -> str | None:
|
||||||
|
for check in results.checks:
|
||||||
|
if check.name == name:
|
||||||
|
return check.status
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class HappyPathTests(unittest.TestCase):
|
||||||
|
def test_full_journey_passes_in_expected_order(self):
|
||||||
|
journey = build_journey()
|
||||||
|
self.assertTrue(journey.run(), "happy path should pass")
|
||||||
|
self.assertEqual(list(journey.executed), list(ra.EXPECTED_STEP_NAMES))
|
||||||
|
self.assertEqual(journey.boots, ["installed", "installed", "recovery"])
|
||||||
|
self.assertEqual(journey.results.exit_code(), 0)
|
||||||
|
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
||||||
|
|
||||||
|
def test_happy_path_has_no_not_run_steps(self):
|
||||||
|
journey = build_journey()
|
||||||
|
journey.run()
|
||||||
|
for stream in (journey.results, journey.recovery_results):
|
||||||
|
self.assertFalse(
|
||||||
|
any(c.status == NOT_RUN for c in stream.checks),
|
||||||
|
"a complete run must not contain NOT RUN steps",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_mocked_run_is_marked_and_never_live(self):
|
||||||
|
journey = build_journey()
|
||||||
|
journey.run()
|
||||||
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
|
||||||
|
tmp.close()
|
||||||
|
self.addCleanup(os.unlink, tmp.name)
|
||||||
|
data = json.loads(journey.results.write(tmp.name).read_text())
|
||||||
|
self.assertEqual(data["evidence"], MOCKED)
|
||||||
|
self.assertEqual(data["inputs"]["iso"]["sha256"], "a" * 64)
|
||||||
|
self.assertEqual(data["inputs"]["sys"]["sha256"], "b" * 64)
|
||||||
|
self.assertEqual(data["scenario"], SCENARIO_ROLLBACK)
|
||||||
|
|
||||||
|
def test_recovery_scenario_is_written_separately(self):
|
||||||
|
journey = build_journey()
|
||||||
|
journey.run()
|
||||||
|
self.assertEqual(journey.recovery_results.scenario, SCENARIO_RECOVERY)
|
||||||
|
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
||||||
|
|
||||||
|
|
||||||
|
class StopOnFailureTests(unittest.TestCase):
|
||||||
|
def test_undo_failure_stops_and_marks_rest_unrun(self):
|
||||||
|
state = {}
|
||||||
|
phase1 = make_phase1(state)
|
||||||
|
# Override undo to fail.
|
||||||
|
phase1.on("sys undo", lambda c: CommandResult(c, exit_code=1, stderr="boom"))
|
||||||
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "sys-undo-activates"), FAIL)
|
||||||
|
self.assertEqual(status_of(journey.results, "journey-complete"), FAIL)
|
||||||
|
self.assertEqual(status_of(journey.results, "restored-root-var-generations"), NOT_RUN)
|
||||||
|
self.assertEqual(status_of(journey.recovery_results, "recovery-boot-no-network-no-model"), NOT_RUN)
|
||||||
|
# The second installed boot and the recovery boot must not happen.
|
||||||
|
self.assertEqual(transport.boots, ["installed"])
|
||||||
|
self.assertEqual(journey.results.exit_code(), 1)
|
||||||
|
|
||||||
|
def test_timeout_is_a_failure_not_a_pass(self):
|
||||||
|
state = {}
|
||||||
|
phase1 = make_phase1(state)
|
||||||
|
phase1.on("sys install", lambda c: CommandResult(c, timed_out=True))
|
||||||
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "controlled-package-change"), FAIL)
|
||||||
|
self.assertIn("timeout", "".join(c.detail for c in journey.results.checks))
|
||||||
|
self.assertEqual(transport.boots, ["installed"])
|
||||||
|
|
||||||
|
def test_disconnect_is_a_failure(self):
|
||||||
|
state = {}
|
||||||
|
phase1 = make_phase1(state)
|
||||||
|
phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, disconnected=True))
|
||||||
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL)
|
||||||
|
self.assertEqual(journey.results.exit_code(), 1)
|
||||||
|
|
||||||
|
def test_missing_evidence_is_refused_not_treated_as_empty(self):
|
||||||
|
state = {}
|
||||||
|
phase1 = make_phase1(state)
|
||||||
|
phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=""))
|
||||||
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL)
|
||||||
|
|
||||||
|
def test_missing_input_identities_block_the_run(self):
|
||||||
|
# No iso/sys digests: the run must not boot anything.
|
||||||
|
transport = FakeTransport([make_phase1({}), make_phase2({})], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, mocked=True)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "input-identities"), FAIL)
|
||||||
|
self.assertEqual(status_of(journey.results, "journey-complete"), FAIL)
|
||||||
|
self.assertEqual(transport.boots, [])
|
||||||
|
|
||||||
|
|
||||||
|
class RestorationMismatchTests(unittest.TestCase):
|
||||||
|
def _journey_with_phase2(self, configure):
|
||||||
|
state = {}
|
||||||
|
phase1 = make_phase1(state)
|
||||||
|
phase2 = make_phase2(state)
|
||||||
|
configure(phase2)
|
||||||
|
transport = FakeTransport([phase1, phase2], make_recovery())
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||||
|
)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
return journey
|
||||||
|
|
||||||
|
def test_wrong_restored_generation_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
phase2.on("cat /proc/cmdline", lambda c: CommandResult(c, stdout="rootflags=subvol=@\n"))
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "restored-root-generation"), FAIL)
|
||||||
|
self.assertEqual(journey.results.exit_code(), 1)
|
||||||
|
|
||||||
|
def test_wrong_var_generation_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
fstab = "/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n"
|
||||||
|
phase2.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab))
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "restored-var-generation"), FAIL)
|
||||||
|
|
||||||
|
def test_package_db_mismatch_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
phase2.on(
|
||||||
|
"find /var/db/pkg",
|
||||||
|
lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS + [ADDED_PKG]) + "\n"),
|
||||||
|
)
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "package-db-restored"), FAIL)
|
||||||
|
|
||||||
|
def test_surviving_package_file_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
phase2.on("test -e", lambda c: CommandResult(c, exit_code=0))
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "package-files-restored"), FAIL)
|
||||||
|
|
||||||
|
def test_kernel_without_modules_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
phase2.on("cat /lib/modules", lambda c: CommandResult(c, stdout="5.0.0-other\n"))
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "kernel-modules-compatible"), FAIL)
|
||||||
|
|
||||||
|
def test_lost_home_marker_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
phase2.on("cat /home/phoebus/rollback-acceptance/marker", lambda c: CommandResult(c, stdout=""))
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "home-data-survives"), FAIL)
|
||||||
|
|
||||||
|
def test_recovery_with_network_fails(self):
|
||||||
|
state = {}
|
||||||
|
recovery = make_recovery()
|
||||||
|
recovery.on(
|
||||||
|
"ip -brief addr",
|
||||||
|
lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"),
|
||||||
|
)
|
||||||
|
transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery)
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||||
|
)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.recovery_results, "recovery-no-network"), FAIL)
|
||||||
|
|
||||||
|
def test_recovery_without_no_model_fallback_fails(self):
|
||||||
|
state = {}
|
||||||
|
recovery = make_recovery()
|
||||||
|
recovery.on("sys ask", lambda c: CommandResult(c, exit_code=0, stdout="here is a plan"))
|
||||||
|
transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery)
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||||
|
)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.recovery_results, "recovery-no-model-fallback"), FAIL)
|
||||||
|
|
||||||
|
|
||||||
|
class RecoveryOnlyTests(unittest.TestCase):
|
||||||
|
def test_recovery_only_passes(self):
|
||||||
|
transport = FakeTransport([], make_recovery())
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||||
|
)
|
||||||
|
self.assertTrue(journey.run_recovery())
|
||||||
|
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
||||||
|
self.assertEqual(transport.boots, ["recovery"])
|
||||||
|
|
||||||
|
def test_recovery_only_without_inputs_does_not_boot(self):
|
||||||
|
transport = FakeTransport([], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, mocked=True)
|
||||||
|
self.assertFalse(journey.run_recovery())
|
||||||
|
self.assertEqual(transport.boots, [])
|
||||||
|
self.assertEqual(
|
||||||
|
status_of(journey.recovery_results, "recovery-boot-no-network-no-model"),
|
||||||
|
NOT_RUN,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SafetyTests(unittest.TestCase):
|
||||||
|
def test_rejects_dev_disk(self):
|
||||||
|
with self.assertRaises(UnsafeDiskError):
|
||||||
|
validate_acceptance_disks(["/dev/sda"], "/var/tmp/phoebus-vm/disks")
|
||||||
|
|
||||||
|
def test_rejects_disk_outside_test_dir(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
outside = Path(tmp).parent / "rollback-outside.qcow2"
|
||||||
|
outside.write_bytes(b"QFI\xfb" + b"\x00" * 64)
|
||||||
|
self.addCleanup(outside.unlink)
|
||||||
|
with self.assertRaises(UnsafeDiskError):
|
||||||
|
validate_acceptance_disks([outside], tmp)
|
||||||
|
|
||||||
|
|
||||||
|
class ParserTests(unittest.TestCase):
|
||||||
|
def test_newest_committed_coordinated_skips_unpaired(self):
|
||||||
|
records = ra.parse_snapshot_history(
|
||||||
|
HISTORY
|
||||||
|
+ """
|
||||||
|
[[records]]
|
||||||
|
id = "snap-varless"
|
||||||
|
created_unix = 2000
|
||||||
|
operation = "risk"
|
||||||
|
source_subvol = "/"
|
||||||
|
snapshot_subvol = "@-20260102T000000-risk"
|
||||||
|
status = "committed"
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
chosen = ra.newest_committed_coordinated(records)
|
||||||
|
self.assertIsNotNone(chosen)
|
||||||
|
self.assertEqual(chosen.id, "snap-initial")
|
||||||
|
|
||||||
|
def test_parse_rootflags_subvol(self):
|
||||||
|
self.assertEqual(
|
||||||
|
ra.parse_rootflags_subvol("rootflags=subvol=@restore-x quiet"), "@restore-x"
|
||||||
|
)
|
||||||
|
self.assertEqual(ra.parse_rootflags_subvol("quiet"), None)
|
||||||
|
|
||||||
|
def test_parse_fstab_subvol(self):
|
||||||
|
text = "/dev/x / btrfs subvol=/@restore-y 0 0\n"
|
||||||
|
self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y")
|
||||||
|
self.assertIsNone(ra.parse_fstab_subvol(text, "/var"))
|
||||||
|
|
||||||
|
def test_parse_fstab_subvol_without_leading_slash(self):
|
||||||
|
text = "/dev/x / btrfs subvol=@restore-y 0 0\n"
|
||||||
|
self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y")
|
||||||
|
|
||||||
|
def test_parse_contents_paths(self):
|
||||||
|
paths = ra.parse_contents_files("obj /usr/bin/hello abc\ndir /usr/share/hello def\n")
|
||||||
|
self.assertEqual(paths, ["/usr/bin/hello", "/usr/share/hello"])
|
||||||
|
|
||||||
|
def test_parse_network_addresses(self):
|
||||||
|
self.assertEqual(ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\n"), ["lo"])
|
||||||
|
self.assertEqual(
|
||||||
|
ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"),
|
||||||
|
["lo", "enp0s2"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_unreadable_history_raises(self):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
ra.parse_snapshot_history("not = = toml [[")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
|
|
@ -22,6 +22,7 @@ host (see ``test_vmsafety.py``).
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import dataclasses
|
import dataclasses
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
|
@ -43,7 +44,7 @@ BLOCKED = "BLOCKED"
|
||||||
SKIPPED = "SKIPPED"
|
SKIPPED = "SKIPPED"
|
||||||
NOT_RUN = "NOT RUN"
|
NOT_RUN = "NOT RUN"
|
||||||
|
|
||||||
_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED)
|
_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED, NOT_RUN)
|
||||||
|
|
||||||
# The candidate artifact a harness run is executed against. The release tool
|
# The candidate artifact a harness run is executed against. The release tool
|
||||||
# binds a gate report to a specific digest; recording it here is what lets a
|
# binds a gate report to a specific digest; recording it here is what lets a
|
||||||
|
|
@ -51,6 +52,19 @@ _TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED)
|
||||||
ARTIFACT_ENV_NAME = "PHOEBUS_VM_ARTIFACT"
|
ARTIFACT_ENV_NAME = "PHOEBUS_VM_ARTIFACT"
|
||||||
ARTIFACT_ENV_SHA256 = "PHOEBUS_VM_ARTIFACT_SHA256"
|
ARTIFACT_ENV_SHA256 = "PHOEBUS_VM_ARTIFACT_SHA256"
|
||||||
|
|
||||||
|
# How a result set was produced. Only a `live` run is real release evidence; a
|
||||||
|
# `mocked` run exercises the orchestration on fakes and must never be imported
|
||||||
|
# as a gate pass. The release tool checks this field and refuses anything else.
|
||||||
|
LIVE = "live"
|
||||||
|
MOCKED = "mocked"
|
||||||
|
|
||||||
|
# The additional inputs a run may be bound to. The rollback acceptance journey
|
||||||
|
# pins both the bootable ISO and the `sys` binary it exercised, because the
|
||||||
|
# rollback semantics live in `sys`; a report that names only the ISO cannot say
|
||||||
|
# which broker was tested.
|
||||||
|
INPUT_ISO = "iso"
|
||||||
|
INPUT_SYS = "sys"
|
||||||
|
|
||||||
|
|
||||||
class UnsafeDiskError(Exception):
|
class UnsafeDiskError(Exception):
|
||||||
"""A writable target violated the disposable-disk rules."""
|
"""A writable target violated the disposable-disk rules."""
|
||||||
|
|
@ -89,6 +103,13 @@ class Results:
|
||||||
self.checks: list[Check] = []
|
self.checks: list[Check] = []
|
||||||
# The candidate artifact this run verifies (name + sha256), if known.
|
# The candidate artifact this run verifies (name + sha256), if known.
|
||||||
self.artifact: dict | None = None
|
self.artifact: dict | None = None
|
||||||
|
# Additional pinned inputs (e.g. the tested ISO and `sys` binary), keyed
|
||||||
|
# by name. `artifact` remains the single release candidate for backwards
|
||||||
|
# compatibility; `inputs` carries the wider identity set.
|
||||||
|
self.inputs: dict[str, dict] = {}
|
||||||
|
# `live` for a real run; `mocked` for a fake-transport unit run. A
|
||||||
|
# mocked run is never valid release evidence.
|
||||||
|
self.evidence_class = LIVE
|
||||||
if strict is None:
|
if strict is None:
|
||||||
strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true")
|
strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true")
|
||||||
self.strict = bool(strict)
|
self.strict = bool(strict)
|
||||||
|
|
@ -111,6 +132,36 @@ class Results:
|
||||||
return self.bind_artifact(name, sha256)
|
return self.bind_artifact(name, sha256)
|
||||||
return self.artifact
|
return self.artifact
|
||||||
|
|
||||||
|
# -- additional pinned inputs -----------------------------------------
|
||||||
|
def mark_mocked(self, reason: str = "") -> None:
|
||||||
|
"""Declare this run a mocked one, so it can never be release evidence."""
|
||||||
|
self.evidence_class = MOCKED
|
||||||
|
if reason:
|
||||||
|
self.inputs.setdefault("mocked_reason", {})["reason"] = reason
|
||||||
|
|
||||||
|
def bind_input(
|
||||||
|
self, name: str, sha256: str, size_bytes: int | None = None
|
||||||
|
) -> dict:
|
||||||
|
"""Pin an input by digest (e.g. the ISO or the `sys` binary)."""
|
||||||
|
entry: dict = {"sha256": sha256}
|
||||||
|
if size_bytes is not None:
|
||||||
|
entry["size_bytes"] = int(size_bytes)
|
||||||
|
self.inputs[name] = entry
|
||||||
|
return entry
|
||||||
|
|
||||||
|
def bind_input_file(self, name: str, path: str | os.PathLike) -> dict:
|
||||||
|
"""Hash a local file and pin it as an input."""
|
||||||
|
path = Path(path)
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with open(path, "rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return self.bind_input(name, digest.hexdigest(), path.stat().st_size)
|
||||||
|
|
||||||
|
def missing_inputs(self, names) -> list[str]:
|
||||||
|
"""Input names that were required but not bound."""
|
||||||
|
return [name for name in names if name not in self.inputs]
|
||||||
|
|
||||||
# -- recording ---------------------------------------------------------
|
# -- recording ---------------------------------------------------------
|
||||||
def _add(self, name: str, status: str, detail: str) -> Check:
|
def _add(self, name: str, status: str, detail: str) -> Check:
|
||||||
if status not in _TERMINAL:
|
if status not in _TERMINAL:
|
||||||
|
|
@ -140,6 +191,14 @@ class Results:
|
||||||
def skip(self, name: str, detail: str = "") -> Check:
|
def skip(self, name: str, detail: str = "") -> Check:
|
||||||
return self._add(name, SKIPPED, detail)
|
return self._add(name, SKIPPED, detail)
|
||||||
|
|
||||||
|
def mark_not_run(self, name: str, detail: str = "") -> Check:
|
||||||
|
"""Record a step that was expected but never reached.
|
||||||
|
|
||||||
|
A stopped journey must not look like a pass, so unexecuted steps are
|
||||||
|
visible rather than simply absent.
|
||||||
|
"""
|
||||||
|
return self._add(name, NOT_RUN, detail)
|
||||||
|
|
||||||
# -- outcome -----------------------------------------------------------
|
# -- outcome -----------------------------------------------------------
|
||||||
@property
|
@property
|
||||||
def failed(self) -> bool:
|
def failed(self) -> bool:
|
||||||
|
|
@ -180,9 +239,11 @@ class Results:
|
||||||
"scenario": self.scenario,
|
"scenario": self.scenario,
|
||||||
"run_id": self.run_id,
|
"run_id": self.run_id,
|
||||||
"strict": self.strict,
|
"strict": self.strict,
|
||||||
|
"evidence": self.evidence_class,
|
||||||
"exit_code": self.exit_code(),
|
"exit_code": self.exit_code(),
|
||||||
"counts": self.counts(),
|
"counts": self.counts(),
|
||||||
"artifact": self.artifact,
|
"artifact": self.artifact,
|
||||||
|
"inputs": self.inputs,
|
||||||
"checks": [c.to_dict() for c in self.checks],
|
"checks": [c.to_dict() for c in self.checks],
|
||||||
}
|
}
|
||||||
path.write_text(json.dumps(payload, indent=2) + "\n")
|
path.write_text(json.dumps(payload, indent=2) + "\n")
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue