Compare commits
2 commits
main
...
codex/roll
| Author | SHA1 | Date | |
|---|---|---|---|
| fc36f637b4 | |||
| 1d37fbc1b8 |
21 changed files with 4157 additions and 184 deletions
|
|
@ -1,5 +1,13 @@
|
|||
# Forgejo Actions CI. Runs only once an Actions runner is registered with this
|
||||
# label (see docs/DEVELOPMENT.md). It mirrors the local quality gates.
|
||||
# Forgejo Actions CI.
|
||||
#
|
||||
# This workflow mirrors the local quality gates exactly and is deliberately
|
||||
# fail-closed: a required check that cannot run because a tool is missing fails
|
||||
# the job rather than printing "skipping". A green job therefore means every
|
||||
# listed check actually executed.
|
||||
#
|
||||
# The Rust toolchain is pinned in `installer-rs/rust-toolchain.toml`. The runner
|
||||
# must provide either `rustup` (which installs the pin) or a cargo/rustfmt/clippy
|
||||
# toolchain; the preflight below fails clearly otherwise.
|
||||
#
|
||||
# If your runner uses different labels, change `runs-on` accordingly.
|
||||
name: ci
|
||||
|
|
@ -9,32 +17,35 @@ on:
|
|||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
quality-gates:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Rust format
|
||||
- name: Provision the pinned Rust toolchain
|
||||
run: |
|
||||
if command -v cargo >/dev/null 2>&1; then
|
||||
cd installer-rs && cargo fmt --all -- --check
|
||||
set -euo pipefail
|
||||
if command -v rustup >/dev/null 2>&1; then
|
||||
rustup show active-toolchain
|
||||
rustup component add rustfmt clippy
|
||||
rustup target add x86_64-unknown-linux-gnu
|
||||
elif command -v cargo >/dev/null 2>&1; then
|
||||
cargo --version
|
||||
cargo fmt --version
|
||||
cargo clippy --version
|
||||
else
|
||||
echo "cargo not installed on this runner; skipping"
|
||||
echo "ERROR: no Rust toolchain on this runner." >&2
|
||||
echo "Install rustup (recommended, honours the pinned toolchain) or" >&2
|
||||
echo "provide cargo with the rustfmt and clippy components, then retry." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Rust clippy
|
||||
run: |
|
||||
if command -v cargo >/dev/null 2>&1; then
|
||||
cd installer-rs && cargo clippy --workspace --all-targets
|
||||
else
|
||||
echo "cargo not installed on this runner; skipping"
|
||||
fi
|
||||
|
||||
- name: Rust test suite
|
||||
run: |
|
||||
if command -v cargo >/dev/null 2>&1; then
|
||||
cd installer-rs && cargo test --workspace
|
||||
else
|
||||
echo "cargo not installed on this runner; skipping"
|
||||
fi
|
||||
# One command for the whole gate set, identical to local development.
|
||||
- name: Quality gates (fmt, clippy, tests, harness tests, builds)
|
||||
run: make ci
|
||||
|
|
|
|||
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -38,3 +38,8 @@ credentials
|
|||
installer/live/build/
|
||||
installer/live/*.iso
|
||||
installer/live/*.iso.sha256
|
||||
|
||||
# --- Release verification reports and logs (regenerated; not source) --------
|
||||
gate-report*.json
|
||||
gate-logs/
|
||||
*.minisig
|
||||
|
|
|
|||
28
Makefile
28
Makefile
|
|
@ -8,7 +8,10 @@
|
|||
|
||||
SHELL := /bin/bash
|
||||
|
||||
.PHONY: help context doctor test test-rust test-vm opencode-acceptance fmt fmt-check clippy check build static vm hooks commit publish backup forgejo-init public private protect release clean
|
||||
# The release tool produced by `make build`, used by `make release-gates`.
|
||||
RELEASE ?= installer-rs/target/release/phoebus-release
|
||||
|
||||
.PHONY: help context doctor test test-rust test-vm ci opencode-acceptance fmt fmt-check clippy check build static release-gates vm hooks commit publish backup forgejo-init public private protect release clean
|
||||
|
||||
help:
|
||||
@printf 'Phoebus OS targets:\n'
|
||||
|
|
@ -17,6 +20,8 @@ help:
|
|||
@printf ' make test run the Rust and VM-harness test suites\n'
|
||||
@printf ' make test-rust cargo test --workspace\n'
|
||||
@printf ' make test-vm python3 installer/tests/vm/test_vmsafety.py\n'
|
||||
@printf ' make ci every CI quality gate (fmt, clippy, tests, builds)\n'
|
||||
@printf ' make release-gates run the executable release gates (evidence report)\n'
|
||||
@printf ' make opencode-acceptance live-model acceptance (needs PHOEBUS_OPENCODE_LIVE_*)\n'
|
||||
@printf ' make fmt cargo fmt --all\n'
|
||||
@printf ' make check fmt-check + clippy + tests\n'
|
||||
|
|
@ -40,10 +45,15 @@ doctor:
|
|||
test: test-rust test-vm
|
||||
|
||||
test-rust:
|
||||
cd installer-rs && cargo test --workspace
|
||||
cd installer-rs && cargo test --workspace --locked
|
||||
|
||||
test-vm:
|
||||
python3 installer/tests/vm/test_vmsafety.py
|
||||
python3 installer/tests/vm/test_rollback_acceptance.py
|
||||
|
||||
# The complete set of required CI gates. Every target here must actually run;
|
||||
# the workflow fails the job if a prerequisite is missing rather than skipping.
|
||||
ci: fmt-check clippy test-rust test-vm build static
|
||||
|
||||
# A real-model acceptance run. Fails (BLOCKED) when the live server is not
|
||||
# configured, so it can never pass by silently not running.
|
||||
|
|
@ -53,7 +63,7 @@ opencode-acceptance:
|
|||
exit 2; \
|
||||
fi
|
||||
cd installer-rs && PHOEBUS_OPENCODE_REQUIRE_LIVE=1 \
|
||||
cargo test -p phoebus-executor --test opencode_live -- --nocapture
|
||||
cargo test -p phoebus-executor --test opencode_live --locked -- --nocapture
|
||||
|
||||
fmt:
|
||||
cd installer-rs && cargo fmt --all
|
||||
|
|
@ -62,16 +72,22 @@ fmt-check:
|
|||
cd installer-rs && cargo fmt --all -- --check
|
||||
|
||||
clippy:
|
||||
cd installer-rs && cargo clippy --workspace --all-targets
|
||||
cd installer-rs && cargo clippy --workspace --all-targets --locked
|
||||
|
||||
check: fmt-check clippy test
|
||||
|
||||
build:
|
||||
cd installer-rs && cargo build --release -p phoebus-installer
|
||||
cd installer-rs && cargo build --release -p phoebus-installer --locked
|
||||
|
||||
static:
|
||||
cd installer-rs && RUSTFLAGS="-C target-feature=+crt-static" \
|
||||
cargo build --release -p phoebus-installer --target x86_64-unknown-linux-gnu
|
||||
cargo build --release -p phoebus-installer --target x86_64-unknown-linux-gnu --locked
|
||||
|
||||
# Run the executable release gates and write an evidence-bound report. Host
|
||||
# gates run here; VM/seat/model gates require their environment and are
|
||||
# recorded as blocked, never silently passed.
|
||||
release-gates: build
|
||||
$(RELEASE) gates --execute --output gate-report.json
|
||||
|
||||
hooks:
|
||||
scripts/setup-hooks.sh
|
||||
|
|
|
|||
|
|
@ -8,6 +8,27 @@ Format: Keep a Changelog. Dates are UTC.
|
|||
## Unreleased
|
||||
|
||||
### Added
|
||||
- **Fail-closed CI with a full gate set.** `.forgejo/workflows/ci.yml` now runs
|
||||
one command, `make ci` (fmt-check, clippy, workspace tests, VM-harness safety
|
||||
tests, release build, static installer build). It provisions the pinned
|
||||
toolchain or fails with a clear message; a missing Cargo/rustfmt/clippy is
|
||||
never reported as a pass. The toolchain is pinned in
|
||||
`installer-rs/rust-toolchain.toml` and dependencies are checked with
|
||||
`--locked`. See ADR 0012.
|
||||
- **Executable, evidence-bound release gates.** `phoebus-release gates` now
|
||||
records `passed`/`failed`/`blocked`/`skipped`/`not_run` (only `passed` is a
|
||||
pass), actually runs host gates, and imports evidence from the existing VM
|
||||
harness `results.json` (`--harness-results`) instead of a competing harness.
|
||||
`--candidate` binds the report to an artifact digest, and the harness records
|
||||
the digest it verified from `PHOEBUS_VM_ARTIFACT[_SHA256]`, so stale evidence
|
||||
is refused. Gates without automation (`no-model-recovery`,
|
||||
`updates-and-rollback`, `desktop-session`) are always `blocked`.
|
||||
- **Publication authorization.** `phoebus-release publish` refuses unless the
|
||||
report is bound to the exact candidate digest, every gate passed, the
|
||||
candidate matches its manifest, and a detached signature verifies. Check
|
||||
failures happen before the destination is touched, and `atomic_publish`
|
||||
restores the previous release if the final rename fails.
|
||||
|
||||
- **Real OpenCode agent-server adapter.** `phoebus-executor::opencode_client`
|
||||
implements the OpenCode session/message protocol over the project's curl
|
||||
transport and the `ModelTransport` trait, so the strict proposal parser,
|
||||
|
|
@ -34,6 +55,10 @@ Format: Keep a Changelog. Dates are UTC.
|
|||
selection (`ai.backend`) in canonical configuration.
|
||||
|
||||
### Security
|
||||
- **Release evidence cannot be replayed across builds.** A gate report and
|
||||
imported harness evidence are bound to a candidate name and SHA-256; a report
|
||||
for a different digest (or unbound evidence) fails rather than authorizing a
|
||||
publication.
|
||||
- **The mandatory-AI gate now requires a validated interaction.** Both the GUI
|
||||
bridge and the installer CLI refuse to install when only a provider name is
|
||||
configured; a successful, validated model interaction bound to the current
|
||||
|
|
|
|||
|
|
@ -230,10 +230,48 @@ prerequisites, and `make test`, `make check`, `make build`, `make static`,
|
|||
|
||||
## Continuous integration
|
||||
|
||||
`.forgejo/workflows/ci.yml` mirrors the local gates (`cargo fmt`, `cargo clippy`,
|
||||
`cargo test`). It runs only once a Forgejo Actions runner is registered for the
|
||||
repository; see Forgejo's documentation for `act_runner`. Until then, run
|
||||
`make test` before pushing.
|
||||
`.forgejo/workflows/ci.yml` runs the same checks as `make ci`:
|
||||
|
||||
```sh
|
||||
make ci # fmt-check, clippy, test-rust, test-vm, build, static
|
||||
```
|
||||
|
||||
The workflow is **fail-closed**: it provisions the pinned toolchain
|
||||
(`installer-rs/rust-toolchain.toml`) or fails with a clear message, so a missing
|
||||
Cargo/rustfmt/clippy is never reported as a pass. Dependencies are checked
|
||||
against the committed `Cargo.lock` (`--locked`).
|
||||
|
||||
It runs only once a Forgejo Actions runner is registered for the repository; see
|
||||
Forgejo's documentation for `act_runner`. Until then, run `make ci` before
|
||||
pushing. A runner that uses different labels needs its `runs-on` value updated.
|
||||
|
||||
## Release verification
|
||||
|
||||
`phoebus-release` writes an evidence-bound gate report and refuses to publish
|
||||
without one (ADR 0012):
|
||||
|
||||
```sh
|
||||
# Host gates run here; VM/seat/model gates are imported from existing harness
|
||||
# results and are `blocked` without evidence.
|
||||
make release-gates
|
||||
installer-rs/target/release/phoebus-release gates --execute \
|
||||
--candidate /path/phoebus-installer-amd64.iso \
|
||||
--harness-results /var/tmp/phoebus-vm
|
||||
|
||||
# Publication requires: a candidate-bound all-passed report, a matching
|
||||
# manifest, and a verified detached signature.
|
||||
installer-rs/target/release/phoebus-release publish \
|
||||
/path/phoebus-installer-amd64.iso /srv/releases/latest.iso \
|
||||
--gate-report gate-report.json --manifest /path/phoebus-release.json \
|
||||
--signature /path/phoebus-installer-amd64.iso.minisig
|
||||
```
|
||||
|
||||
Gate outcomes are `passed`, `failed`, `blocked`, `skipped` and `not_run`; only
|
||||
`passed` is a pass. A report for one artifact digest cannot authorize another
|
||||
(`--candidate` binds the report; the harness records the digest it verified from
|
||||
`PHOEBUS_VM_ARTIFACT[_SHA256]`). `no-model-recovery`, `updates-and-rollback` and
|
||||
`desktop-session` have no automated harness yet, so they stay explicitly
|
||||
`blocked` until one exists.
|
||||
|
||||
## Backups
|
||||
|
||||
|
|
@ -247,7 +285,6 @@ Restore with `git clone /path/to/phoebus-<stamp>.bundle phoebus_os`.
|
|||
|
||||
## Quality gates before commit
|
||||
|
||||
1. `make test` (`cd installer-rs && cargo test --workspace`)
|
||||
2. `cargo fmt --all -- --check` and `cargo clippy --workspace --all-targets`
|
||||
3. Update `docs/CHANGELOG.md` and `docs/STATUS.md` when behavior changes.
|
||||
4. Commit (see `docs/GIT-WORKFLOW.md`).
|
||||
1. `make ci` (the complete gate set the CI workflow runs)
|
||||
2. Update `docs/CHANGELOG.md` and `docs/STATUS.md` when behavior changes.
|
||||
3. Commit (see `docs/GIT-WORKFLOW.md`).
|
||||
|
|
|
|||
|
|
@ -117,11 +117,10 @@ a staged file is too large, then runs the fast tests.
|
|||
|
||||
1. scans staged files for secrets (`scripts/check-secrets.sh`);
|
||||
2. blocks staged `*.iso`, `*.qcow2` and files larger than 5 MiB;
|
||||
3. runs the fast Bash test suite.
|
||||
3. optionally runs the Rust suite (set `PHOEBUS_PRECOMMIT_RUST=1` to enable).
|
||||
|
||||
Bypass for a genuine emergency with `git commit --no-verify`, or set
|
||||
`SKIP_PHOEBUS_HOOKS=1`. Set `PHOEBUS_PRECOMMIT_RUST=1` to also run
|
||||
`cargo test --workspace` in the hook.
|
||||
`SKIP_PHOEBUS_HOOKS=1`.
|
||||
|
||||
## Repository settings and hardening
|
||||
|
||||
|
|
@ -136,13 +135,17 @@ Current state:
|
|||
`enable DEST` at a synced/remote folder for a genuinely off-box copy, since
|
||||
the only git remote is one VPS.
|
||||
|
||||
### CI runner (optional)
|
||||
### CI runner (recommended)
|
||||
|
||||
`.forgejo/workflows/ci.yml` mirrors the local gates but only runs once a
|
||||
Forgejo Actions runner is registered. Install `act_runner` on a machine,
|
||||
`.forgejo/workflows/ci.yml` runs `make ci` (fmt, clippy, workspace tests,
|
||||
VM-harness safety tests, release build, static installer build). It is
|
||||
fail-closed: if the runner lacks a Rust toolchain it fails the job rather than
|
||||
skipping checks, so a green result always means the checks ran. It runs once a
|
||||
Forgejo Actions runner is registered; install `act_runner` on a machine,
|
||||
register it (Site Administration -> Actions -> Runners, or
|
||||
`act_runner register --instance https://git.phoebus.club --token <TOKEN>`), and
|
||||
run it as a service. Until then, run `make test` before pushing.
|
||||
run it with a `rustup`-provisioned toolchain (or cargo + rustfmt + clippy).
|
||||
Until then, run `make ci` before pushing.
|
||||
|
||||
### Commit signing (optional)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,36 +3,43 @@
|
|||
Rolling note between work sessions. **Update this file before you finish**, then
|
||||
commit it. It should be short and current; use `git log` for detail.
|
||||
|
||||
- **Last updated:** this session (2026-09-22), OpenCode adapter.
|
||||
- **Current focus:** Implemented a real OpenCode agent-server adapter
|
||||
(`phoebus-executor::opencode_client`) that speaks the session/message protocol
|
||||
over curl and implements `ModelTransport`, wired into canonical config
|
||||
(`ai.backend = "opencode"`), the GUI ("OpenCode agent" + Test connection),
|
||||
the installer CLI (`--ai-backend opencode`, `--opencode-test`) and `sys ask`.
|
||||
The mandatory-AI gate now requires a validated interaction with invalidated
|
||||
state. The probe/transport were hardened (loopback/TLS policy, DNS pinning,
|
||||
status checks, auth), and the GUI harness gained exact-session and
|
||||
frontend-state assertions.
|
||||
- **Last updated:** this session (2026-09-22), CI and release verification
|
||||
hardening (branch `codex/harden-ci-release-verification`).
|
||||
- **Current focus:** Made CI and release verification mean what they claim.
|
||||
(1) `.forgejo/workflows/ci.yml` is fail-closed: it provisions the pinned
|
||||
toolchain or fails, and runs one `make ci` (fmt-check, clippy, workspace
|
||||
tests, VM-harness safety tests, release build, static installer build).
|
||||
`installer-rs/rust-toolchain.toml` pins stable 1.93.1; all cargo gates use
|
||||
`--locked`. (2) `phoebus-release` gate statuses are now
|
||||
`passed`/`failed`/`blocked`/`skipped`/`not_run` (only `passed` is a pass);
|
||||
host gates execute real commands; VM/seat/model gates import the existing VM
|
||||
harness `results.json` (`--harness-results`). (3) Gate reports and harness
|
||||
evidence are bound to a candidate name+SHA-256 (`--candidate`;
|
||||
`PHOEBUS_VM_ARTIFACT[_SHA256]`), so stale evidence cannot authorize another
|
||||
build. (4) `publish` refuses unless the report is candidate-bound and
|
||||
all-passed, the manifest verifies the artifact and a detached signature
|
||||
verifies; `atomic_publish` restores the previous release on failure.
|
||||
- **Last verified (this session):**
|
||||
- 336 Rust tests + 16 harness tests green; fmt/clippy clean.
|
||||
- Real inference through `opencode 1.18.32` with `deepseek-v4-flash`;
|
||||
`--opencode-test` → `{"phoebus":"ok"}`; the GUI bridge `ai_check` applied 8
|
||||
reviewed changes; `sys ask` proposed and ran a read-only action.
|
||||
- Auth: none → `authentication_required`, wrong → `authentication_rejected`,
|
||||
correct → `model_catalog` + successful inference. Plain http to a LAN host
|
||||
→ `insecure_transport`, exit 1, no request sent.
|
||||
- 7 real-HTTP integration tests (status, chunked multibyte, oversize,
|
||||
redirect, disconnect, auth) and 2 env-gated live tests.
|
||||
- GUI acceptance in a disposable VM: `gui-driver.py opencode` passed 17/17
|
||||
(selected OpenCode, entered the endpoint/model through the UI, real model
|
||||
response + validated proposal); `gui-driver.py gui` passed 25/25.
|
||||
ISO `sha256 4db98db3…`, static binary `sha256 72c2dcd0…`.
|
||||
- **Not yet verified this session:** the full graphical **installation** through
|
||||
OpenCode past the AI gate (disk/account/review/ERASE/install/detached boot),
|
||||
and a separate physical computer. The pinned manifest is `v1.18.31` while the
|
||||
tested binary is `1.18.32`. On this server use `deepseek-flash`; the model
|
||||
must match the server's catalog (the GUI default `deepseek-chat` does not
|
||||
exist on it).
|
||||
- 364 Rust tests + 19 harness tests green; `cargo fmt --all -- --check` and
|
||||
`cargo clippy --workspace --all-targets` clean (exit 0); `make ci` exit 0.
|
||||
- Release build and static installer build succeed; the static binary is
|
||||
`static-pie linked` (4.99 MB).
|
||||
- New regression tests (20 Rust + 3 Python) cover: absent prerequisites are
|
||||
`blocked`; a failed host gate is `failed` with diagnostics; explicit skips
|
||||
and `--only` are never passes; manual gates are `blocked`; stale or
|
||||
unbound harness evidence is `failed`; `publish` refuses on not-run gates,
|
||||
unbound/stale reports, missing manifest, manifest mismatch, missing or
|
||||
failing signature, and a failed publication leaves the previous release
|
||||
intact.
|
||||
- CLI smoke: `gates` (all `not_run`), `gates --execute --only rust-fmt`
|
||||
(executes and reports `passed`), harness-evidence import (`install`
|
||||
`passed`), and `publish` refusal on an unbound report.
|
||||
- **Not yet verified this session:** no Forgejo Actions runner is registered,
|
||||
so the workflow is unexecuted. The VM, graphical-seat and live-model gates
|
||||
are `blocked` here (no disposable VM/model in this environment).
|
||||
`no-model-recovery`, `updates-and-rollback` and `desktop-session` have no
|
||||
automated harness yet and are always `blocked`, so a full publication is not
|
||||
possible until they are automated (intentional: no false pass).
|
||||
- **Public repo:** <https://git.phoebus.club/phoebus/phoebus_os>
|
||||
(remote `origin`, SSH port 2222).
|
||||
|
||||
|
|
@ -55,11 +62,24 @@ commit it. It should be short and current; use `git log` for detail.
|
|||
5. **Physical-seat desktop**: run `phoebus-session` on a real VT/seat and check
|
||||
`phoebus-menu`, `phoebus-lock`, `phoebus-idle`, kitty, XWayland, the polkit
|
||||
agent, logout/reap and reboot (`docs/DEVELOPMENT.md`).
|
||||
6. **Release gates on a CI runner**: `phoebus-release gates --execute` and
|
||||
`publish` once a runner exists; no gate has executed.
|
||||
6. **Register a Forgejo Actions runner** and confirm `make ci` runs green
|
||||
there; no runner is registered, so `.forgejo/workflows/ci.yml` is
|
||||
unexecuted. `make ci` passes locally, but that is not CI verification.
|
||||
7. **Automate the three blocked release gates**: `no-model-recovery`,
|
||||
`updates-and-rollback` and `desktop-session` have no harness scenario yet,
|
||||
so they are always `blocked` and a full publication is impossible. Add
|
||||
harness modes (or reviewed manual evidence) and re-run
|
||||
`make release-gates`.
|
||||
|
||||
## Known blockers / risks
|
||||
|
||||
- **No Forgejo Actions runner is registered**, so `.forgejo/workflows/ci.yml`
|
||||
is unexecuted; `make ci` is green locally only. Registering an `act_runner`
|
||||
with a rustup toolchain is the external prerequisite (ADR 0012).
|
||||
- **A full release cannot be authorized yet**: `no-model-recovery`,
|
||||
`updates-and-rollback` and `desktop-session` have no automated harness, so
|
||||
the gate report is never all-passed. This is deliberate until they are
|
||||
automated.
|
||||
- No live DeepSeek/OpenAI key and no local model are available; the **full GUI
|
||||
install past the AI step** and two fresh-disk graphical installs are NOT
|
||||
verified (normal installation requires a connected model). The AI step is
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ more interesting. Status vocabulary: **Done**, **Partial**, **Next**, **Later**.
|
|||
| 11 | AI privilege and privacy architecture | Partial | typed actions + enforced privacy boundary + guided proposal review + OS-level sandbox (systemd-run hardening) + approval-bound resolved effects; live call/sandbox not VM-verified |
|
||||
| 12 | `sys` operations | Partial | history, enable/disable, inspect, install, prepare-updates VM-verified; `undo`, `undo status`, `ask` added (unit-tested, VM gate pending) |
|
||||
| 13 | Kernel optimization policy | Later | conservative fragments only; SAFE/OPTIMIZED/AGGRESSIVE selection exists |
|
||||
| 14 | ISO / release pipeline | Partial | manifest/verify/assemble-plan tooling; release gates, atomic publish and signing helpers added; no CI runner has executed a gate |
|
||||
| 14 | ISO / release pipeline | Partial | manifest/verify/assemble tooling; executable evidence-bound gates + publication authorization (ADR 0012); no CI runner has executed a gate |
|
||||
| 15 | Recovery environment | Later | recovery BLS entry generated; snapshot-based recovery entry written at install; full recovery image not built |
|
||||
| — | Rust TUI | Partial | ratatui UI with plan/ERASE and AI provider selection; not yet the default on all paths |
|
||||
| — | Graphical installer (welcome + wizard) | Partial | Quickshell/QML over the typed `--gui-bridge`; live graphical VM verified, including a detached install+boot; live AI call pending (ADR 0010) |
|
||||
|
|
|
|||
156
docs/ROLLBACK-ACCEPTANCE.md
Normal file
156
docs/ROLLBACK-ACCEPTANCE.md
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
# Rollback and no-model-recovery acceptance
|
||||
|
||||
This is the execution guide for the **rollback** (`updates-and-rollback`) and
|
||||
**no-model recovery** (`no-model-recovery`) release gates. It describes how to
|
||||
run the acceptance journey in the disposable VM harness, what it asserts, where
|
||||
the evidence lands, and how a failure is diagnosed.
|
||||
|
||||
> **Status: NOT YET RUN.** The harness and its offline tests exist and pass on a
|
||||
> plain host, but the live journey has **not** been executed. Until a real run
|
||||
> succeeds and its `results.json` is imported, both release gates stay
|
||||
> `blocked`. A run made with a fake transport is marked `mocked` and is
|
||||
> **rejected** by the release tool.
|
||||
|
||||
## What the journey proves
|
||||
|
||||
The installer writes a coordinated `@` + `@var` known-good snapshot at install
|
||||
time. The journey then deliberately changes the disk and asks the real
|
||||
`sys undo` to put it back. The ordered steps (from
|
||||
`installer/tests/vm/rollback_acceptance.py`) are:
|
||||
|
||||
1. `sys-binary-identity` — the installed `sys` matches the pinned digest.
|
||||
2. `known-good-coordinated-snapshot` — a committed root snapshot with a paired
|
||||
`@var` snapshot exists.
|
||||
3. `home-data-marker` — a marker is written into `/home` (a separate subvolume).
|
||||
4. `controlled-package-change` — a small package is installed, adding a
|
||||
`/var/db/pkg` entry and files under `/`.
|
||||
5. `deterministic-guest-breakage` — `/etc/hostname` is overwritten with a
|
||||
sentinel.
|
||||
6. `sys-undo-activates` — `sys undo <id>` writes an `activated` rollback state
|
||||
for the next boot.
|
||||
7. `reboot-detached` — the installed disk boots with the installer ISO absent.
|
||||
8. `restored-root-var-generations` — the booted `rootflags`, `/etc/fstab` `/`
|
||||
and `/var` subvolumes match the rollback state.
|
||||
9. `package-files-restored` — the changed package's DB entry and files are gone,
|
||||
and the sentinel hostname is reverted.
|
||||
10. `portage-database-consistent` — `/var/db/pkg` equals its pre-change listing
|
||||
and every entry has a `CONTENTS` file.
|
||||
11. `kernel-modules-compatible` — the running kernel has a module tree in the
|
||||
restored root.
|
||||
12. `home-data-survives` — the `/home` marker is byte-identical.
|
||||
13. `recovery-entry-armed` — `bootctl set-oneshot phoebus-recovery` succeeds.
|
||||
14. `recovery-boot-no-network-no-model` — the recovery entry boots
|
||||
`rescue.target`, no non-loopback interface is up, `sys` status works
|
||||
offline, and `sys ask` reports the deterministic no-model fallback.
|
||||
|
||||
The package changed in step 4 is controlled by `ROLLBACK_PACKAGE` (default
|
||||
`app-misc/hello`); pick a small, signed binary package for a fast run.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A **disposable qcow2** already installed by an existing harness run
|
||||
(`serial-driver.py install` or `appliance-driver.py install`). This driver
|
||||
never installs and never attaches the installer ISO.
|
||||
- `/var/tmp/phoebus-vm/disks` (or `PHOEBUS_VM_DISKS`) has the disk. Never point
|
||||
a writable disk at `/dev/*`; the harness refuses it.
|
||||
- `qemu-system-x86_64`, `qemu-img`, OVMF firmware, Python `pexpect`.
|
||||
- The candidate ISO and the `sys` binary under test, as files on the host, so
|
||||
their digests can be pinned.
|
||||
- Network access during the change phase (the package install); the recovery
|
||||
phase uses `-net none`.
|
||||
|
||||
## Running it
|
||||
|
||||
Print the plan without starting a VM (safe on any host):
|
||||
|
||||
```sh
|
||||
python3 installer/tests/vm/rollback-driver.py plan
|
||||
```
|
||||
|
||||
Run the full journey:
|
||||
|
||||
```sh
|
||||
PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \
|
||||
DISK=/var/tmp/phoebus-vm/disks/rollback.qcow2 \
|
||||
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \
|
||||
OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \
|
||||
ISO=/var/tmp/phoebus-vm/phoebus-installer-amd64.iso \
|
||||
PHOEBUS_VM_SYS=/var/tmp/phoebus-vm/sys \
|
||||
PHOEBUS_VM_ARTIFACT=phoebus-installer-amd64.iso \
|
||||
PHOEBUS_VM_ARTIFACT_SHA256="$(sha256sum /var/tmp/phoebus-vm/phoebus-installer-amd64.iso | cut -d' ' -f1)" \
|
||||
PHOEBUS_VM_STRICT=1 \
|
||||
LUKS_PASSPHRASE=phoebus-luks-test USER_PASSWORD=phoebus-user-test \
|
||||
python3 installer/tests/vm/rollback-driver.py run
|
||||
```
|
||||
|
||||
Re-verify only the recovery boot (boots normally once to arm the one-shot, then
|
||||
reboots with no network):
|
||||
|
||||
```sh
|
||||
# same environment, then:
|
||||
python3 installer/tests/vm/rollback-driver.py recovery
|
||||
```
|
||||
|
||||
`PHOEBUS_VM_ARTIFACT[_SHA256]` is what binds the evidence to a candidate; the
|
||||
release tool refuses unbound or stale evidence. `PHOEBUS_VM_SYS` is hashed and
|
||||
pinned as the `sys` identity (override with `PHOEBUS_VM_SYS_SHA256`). The guest
|
||||
must report the same digest in step 1.
|
||||
|
||||
## Artifacts
|
||||
|
||||
Under `$PHOEBUS_VM_STATE/<run-id>/` (default `/var/tmp/phoebus-vm/`):
|
||||
|
||||
- `results.json` — the `rollback-acceptance` scenario.
|
||||
- `recovery/results.json` — the `rollback-recovery` scenario (written by `run`).
|
||||
- `serial-boot1.log`, `serial-boot2.log`, `serial-boot3.log` — per-boot serial
|
||||
transcripts.
|
||||
- `OVMF_VARS.fd` — the run's own writable firmware variables.
|
||||
|
||||
`results.json` records `scenario`, `run_id`, `evidence` (`live`), `exit_code`,
|
||||
`counts`, `artifact` (name + ISO digest), `inputs` (`iso` and `sys` digests) and
|
||||
the per-check `checks` list.
|
||||
|
||||
## Feeding the release gates
|
||||
|
||||
Import the run directory when producing a gate report:
|
||||
|
||||
```sh
|
||||
installer-rs/target/release/phoebus-release gates --execute \
|
||||
--candidate /path/phoebus-installer-amd64.iso \
|
||||
--harness-results /var/tmp/phoebus-vm/<run-id> \
|
||||
--output gate-report.json
|
||||
```
|
||||
|
||||
The `updates-and-rollback` gate resolves `rollback-acceptance`; the
|
||||
`no-model-recovery` gate resolves `rollback-recovery`. A gate **fails**
|
||||
(never passes) when the evidence is:
|
||||
|
||||
- `mocked` (a fake-transport unit run),
|
||||
- missing the `iso` or `sys` pinned input,
|
||||
- bound to a different candidate digest (stale),
|
||||
- unbound while a candidate is supplied, or
|
||||
- present but with `exit_code != 0`.
|
||||
|
||||
## Failure diagnosis
|
||||
|
||||
| Symptom | Likely cause |
|
||||
|---|---|
|
||||
| `sys-binary-identity` fails | the installed `sys` is not the build under test; rebuild the ISO from the merged branch and re-install |
|
||||
| `known-good-coordinated-snapshot` fails | the install did not record a paired `@var` snapshot, or `snapshot-history.toml` is unreadable (it is never treated as empty) |
|
||||
| `controlled-package-change` fails/times out | no network, no signed binhost, or the atom needs a long compile; choose another `ROLLBACK_PACKAGE` |
|
||||
| `sys-undo-activates` fails | `sys undo` refused (no committed snapshot, unreadable history) or the broker exited nonzero |
|
||||
| `restored-root-var-generations` fails | `rootflags`/fstab were not rewritten to the restored generation |
|
||||
| `portage-database-consistent` fails | root and `/var` came from different generations |
|
||||
| `kernel-modules-compatible` fails | the selected boot entry does not match the restored `lib/modules` |
|
||||
| `recovery-no-network` fails | QEMU was still given a network device; check `-net none` |
|
||||
| `recovery-no-model-fallback` fails | a provider is configured, or `sys ask` did not fail closed offline |
|
||||
| harness exits 2 with `disposable-disk` | the disk is not a disposable qcow2 inside `PHOEBUS_VM_DISKS` |
|
||||
|
||||
## Integration note
|
||||
|
||||
The branch this work is built on (`codex/rollback-acceptance`) is based on the
|
||||
CI-hardening commit `1d37fbc` and does **not** include the sibling `sys`
|
||||
hardening commit `4679f93`. The installed `sys` used in a live run must come
|
||||
from a build that contains the sys hardening; otherwise step 1
|
||||
(`sys-binary-identity`) will correctly fail. Build the acceptance ISO after the
|
||||
two branches are integrated.
|
||||
|
|
@ -94,7 +94,7 @@ still not VM-verified.
|
|||
| Live desktop ISO (Mango + DMS) | yes | guards tested | graphical VM + GUI journey + detached boot | no |
|
||||
| Live welcome autostart + duplicate prevention | yes | guards tested | live graphical VM | no |
|
||||
| Welcome primary action keyboard-focus | yes | guards tested | live graphical VM | no |
|
||||
| VM harness disk safety + structured results | yes | yes (16 tests) | n/a | n/a |
|
||||
| VM harness disk safety + structured results + artifact binding | yes | yes (19 tests) | n/a | n/a |
|
||||
| Target disk revalidation before partition | yes | yes | no | n/a |
|
||||
| Bridge replay/concurrency + input bounds | yes | yes | no | n/a |
|
||||
| Provider transport endpoint/size hardening | yes | yes | no | n/a |
|
||||
|
|
@ -105,7 +105,10 @@ still not VM-verified.
|
|||
| OpenCode full graphical install | yes | yes | AI gate passed; install not run | n/a |
|
||||
| Mandatory AI requires a validated interaction | yes | yes | bridge + CLI | n/a |
|
||||
| GURU packaging (scoped keywords) | yes | yes | no (build/install unverified) | no |
|
||||
| Release gates + atomic publish | yes | yes | no | n/a |
|
||||
| Fail-closed CI (`make ci`, pinned toolchain) | yes | yes (`make ci` green locally) | no runner registered | n/a |
|
||||
| Executable release gates + evidence-bound publish | yes | yes | no (no VM/model here) | n/a |
|
||||
| Harness evidence import + stale-evidence refusal | yes | yes | no | n/a |
|
||||
| Atomic publish + previous-release preservation | yes | yes | no | n/a |
|
||||
| `sys` verbs | yes | yes | yes | no |
|
||||
| OpenCode pinned bootstrap | yes | yes | no | n/a |
|
||||
| Release manifest/verify | yes | yes | no | n/a |
|
||||
|
|
@ -160,9 +163,19 @@ still not VM-verified.
|
|||
the argv level) and fails closed without `systemd-run`; it has not been
|
||||
exercised in a VM. Only typed actions and the privacy boundary were previously
|
||||
enforced.
|
||||
- **Release automation** now has gates, a report, atomic publish and signing
|
||||
command helpers, but no gate has been executed by a configured CI runner; the
|
||||
report records unexecuted gates as `NotRun`.
|
||||
- **Release gates are executable and evidence-bound (ADR 0012).** `make ci`
|
||||
runs fmt/clippy/tests/harness-tests/builds and is fail-closed. `phoebus-release
|
||||
gates --execute` runs host gates and imports VM/seat/model evidence from the
|
||||
existing harness `results.json`; unexecuted, blocked or skipped gates are
|
||||
never passes. `publish` refuses unless the report is bound to the exact
|
||||
candidate digest, every gate passed, the manifest verifies the artifact and a
|
||||
detached signature verifies. **No CI runner is registered**, so the workflow
|
||||
itself is unexecuted; the VM, graphical-seat and live-model gates are
|
||||
**blocked** here (no disposable VM or model in this environment).
|
||||
`no-model-recovery`, `updates-and-rollback` and `desktop-session` have no
|
||||
automated harness yet and are always `blocked`, so a full release cannot be
|
||||
authorized until they are automated (or given reviewed manual evidence). That
|
||||
is intentional: no false pass.
|
||||
- The desktop transaction compiles heavily because desktop USE does not match
|
||||
the signed binhost (~26 min for the desktop set on 8 vCPUs in the test VM).
|
||||
The official Gentoo binhost is OpenRC/hardened, so a systemd system can only
|
||||
|
|
|
|||
78
docs/adr/0012-release-verification-and-publication.md
Normal file
78
docs/adr/0012-release-verification-and-publication.md
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
# ADR 0012: Release verification evidence and publication authorization
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-09-22
|
||||
|
||||
## Context
|
||||
|
||||
Two failure modes made a passing result meaningless:
|
||||
|
||||
1. **CI skipped required checks.** `.forgejo/workflows/ci.yml` wrapped every
|
||||
Rust step in `if command -v cargo; then ...; else echo "skipping"; fi`. A
|
||||
runner without Cargo reported green while running nothing.
|
||||
2. **Release gates did not execute and were not bound to an artifact.**
|
||||
`phoebus-release gates --execute` ran no command (there were no host gates,
|
||||
and every non-host gate was recorded `not_run`). `publish` accepted any
|
||||
report whose results were all `passed`, without checking that the report
|
||||
described the artifact being published, that the artifact matched its
|
||||
manifest, or that a signature had been verified. A stale report for an old
|
||||
build could therefore authorize a different one.
|
||||
|
||||
The project already has a disposable VM harness (`installer/tests/vm/*.py`)
|
||||
that writes structured `results.json`. A second, competing harness is not
|
||||
wanted.
|
||||
|
||||
## Decision
|
||||
|
||||
1. **CI is fail-closed.** The workflow provisions the pinned toolchain or fails
|
||||
with a clear message; it never treats a missing prerequisite as a pass. The
|
||||
whole required set is one command, `make ci`, so CI and local development run
|
||||
the same checks (fmt, clippy, workspace tests, VM-harness safety tests,
|
||||
release build, static installer build). The toolchain is pinned in
|
||||
`installer-rs/rust-toolchain.toml` and dependencies use the committed
|
||||
`Cargo.lock` (`--locked`).
|
||||
2. **Gate outcomes are distinct.** A gate is `passed`, `failed`, `blocked`,
|
||||
`skipped` or `not_run`; **only `passed` is a pass**. `blocked` means a
|
||||
prerequisite (VM, graphical seat, model, CI runner, or an automated command)
|
||||
is missing; `skipped` means an operator excluded it; `not_run` means
|
||||
execution was not requested.
|
||||
3. **Gates perform their advertised check.** Host gates run their real command
|
||||
through the command layer and record the exit status. VM/seat/model gates
|
||||
name the real harness scenario and are resolved from that harness's
|
||||
`results.json` (reusing the existing harness); with no evidence they are
|
||||
`blocked`. Gates whose automation does not exist yet have an empty command
|
||||
and can only ever be `blocked`.
|
||||
4. **Evidence is bound to the artifact.** `phoebus-release gates --candidate
|
||||
<file>` records the candidate's name and SHA-256 in the report; the harness
|
||||
records the digest it verified from `PHOEBUS_VM_ARTIFACT[_SHA256]`. Imported
|
||||
evidence whose digest differs from the candidate is reported as `failed`
|
||||
("stale evidence"), never passed.
|
||||
5. **Publication is authorized, not assumed.** `phoebus-release publish`
|
||||
refuses unless the report is bound to the exact candidate digest, every gate
|
||||
passed, the candidate matches its manifest entry, and a detached signature
|
||||
verifies. Check failures happen before the destination is touched.
|
||||
`atomic_publish` keeps `latest.iso.prev` and restores it if the final rename
|
||||
fails, so a failed or interrupted publication never removes the working
|
||||
release.
|
||||
|
||||
## Consequences
|
||||
|
||||
- A green CI job means the listed checks actually ran. There is no "skipping"
|
||||
path for a required check.
|
||||
- `make release-gates` writes a report that reflects reality. Because
|
||||
`no-model-recovery`, `updates-and-rollback` and `desktop-session` have no
|
||||
automated harness yet, they are always `blocked`, so a full release cannot be
|
||||
authorized until those checks are automated (or given reviewed manual
|
||||
evidence). This is deliberate: it is preferable to be unable to publish than
|
||||
to publish on a false pass.
|
||||
- The release report and per-gate logs are diagnostics, not source; they are
|
||||
gitignored, redacted with the environment redactor, and bounded in size.
|
||||
- The remaining external prerequisite is a registered Forgejo Actions runner;
|
||||
until one exists the workflow is unexecuted and is recorded as such.
|
||||
|
||||
## Status
|
||||
|
||||
Accepted and implemented. The gate engine, harness-evidence import, artifact
|
||||
binding and publication refusals are covered by unit tests; `make ci` passes
|
||||
locally. CI execution and VM/model gates remain unverified here (no runner, no
|
||||
disposable VM or model in this environment).
|
||||
|
|
@ -17,5 +17,6 @@ and link it here.
|
|||
| [0009](0009-mango-dankmaterialshell-desktop.md) | Mango + DankMaterialShell desktop option | Accepted |
|
||||
| [0010](0010-graphical-installer-frontend.md) | Graphical installer frontend and typed bridge | Accepted |
|
||||
| [0011](0011-opencode-connection-boundary.md) | OpenCode connection boundary | Accepted |
|
||||
| [0012](0012-release-verification-and-publication.md) | Release verification evidence and publication authorization | Accepted |
|
||||
|
||||
Format: Context / Decision / Consequences / Status.
|
||||
|
|
|
|||
|
|
@ -8,9 +8,11 @@ use std::process::ExitCode;
|
|||
|
||||
use clap::{Parser, Subcommand};
|
||||
use phoebus_core::VERSION;
|
||||
use phoebus_executor::{CommandSpec, ProcessRunner, RealRunner};
|
||||
use phoebus_installer::release::{
|
||||
artifact_for, atomic_publish, default_manifest_path, run_gates, sha256_file, sign_command,
|
||||
verify_signature_command, GateReport, ReleaseManifest,
|
||||
artifact_for, candidate_ref, default_manifest_path, publish_release, run_gates, sha256_file,
|
||||
sign_command, verify_signature_command, GateOptions, GateReport, HarnessEvidenceSet,
|
||||
MinisignVerifier, PublishRequest, ReleaseManifest, SystemGateExecutor,
|
||||
};
|
||||
|
||||
#[derive(Parser)]
|
||||
|
|
@ -51,22 +53,49 @@ enum Command {
|
|||
#[arg(long)]
|
||||
output: PathBuf,
|
||||
},
|
||||
/// Write the candidate gate report. `--execute` runs host gates; VM,
|
||||
/// graphical-seat and model gates need their environment and are recorded
|
||||
/// as not run otherwise.
|
||||
/// Write the candidate gate report.
|
||||
///
|
||||
/// Without `--execute` every gate is recorded as `not_run` and never as a
|
||||
/// pass. With `--execute`, host gates run their real commands; VM, seat and
|
||||
/// model gates are resolved from `--harness-results` when supplied, and are
|
||||
/// otherwise recorded as `blocked` (never silently passed).
|
||||
Gates {
|
||||
#[arg(long, default_value = "gate-report.json")]
|
||||
output: PathBuf,
|
||||
#[arg(long)]
|
||||
execute: bool,
|
||||
/// Bind the report to this candidate artifact's digest.
|
||||
#[arg(long)]
|
||||
candidate: Option<PathBuf>,
|
||||
/// Import VM-harness `results.json` files from this file or directory.
|
||||
#[arg(long)]
|
||||
harness_results: Option<PathBuf>,
|
||||
/// Record a gate as `skipped` (repeatable). A skipped gate is never a
|
||||
/// pass, so this cannot be used to authorize a publication.
|
||||
#[arg(long = "skip")]
|
||||
skip: Vec<String>,
|
||||
/// Run only these gates (repeatable); the rest are `skipped`.
|
||||
#[arg(long = "only")]
|
||||
only: Vec<String>,
|
||||
/// Directory for redacted per-gate failure logs.
|
||||
#[arg(long)]
|
||||
log_dir: Option<PathBuf>,
|
||||
},
|
||||
/// Publish a candidate atomically, refusing unless a passing gate report is
|
||||
/// supplied. The previous release is preserved.
|
||||
/// Publish a candidate atomically. Refuses unless the gate report is bound
|
||||
/// to the exact candidate digest, every gate passed, the manifest verifies
|
||||
/// the artifact and a detached signature verifies. The previous release is
|
||||
/// preserved.
|
||||
Publish {
|
||||
candidate: PathBuf,
|
||||
dest: PathBuf,
|
||||
#[arg(long)]
|
||||
gate_report: PathBuf,
|
||||
/// The manifest the candidate is verified against.
|
||||
#[arg(long)]
|
||||
manifest: Option<PathBuf>,
|
||||
/// The detached signature (`*.minisig`) for the candidate.
|
||||
#[arg(long)]
|
||||
signature: Option<PathBuf>,
|
||||
},
|
||||
/// Print the signing command for an artifact (key outside the repository).
|
||||
Sign { artifact: PathBuf, key: PathBuf },
|
||||
|
|
@ -86,6 +115,20 @@ fn main() -> ExitCode {
|
|||
}
|
||||
}
|
||||
|
||||
/// The short source commit the tool is running against, for evidence binding.
|
||||
fn git_commit() -> String {
|
||||
let spec = CommandSpec::new("release", "git-rev-parse", "git")
|
||||
.arg("rev-parse")
|
||||
.arg("--short")
|
||||
.arg("HEAD");
|
||||
RealRunner
|
||||
.execute(&spec)
|
||||
.ok()
|
||||
.filter(|output| output.success)
|
||||
.map(|output| output.stdout_lossy().trim().to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn run() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||
match Cli::parse().command {
|
||||
Command::Manifest {
|
||||
|
|
@ -163,24 +206,65 @@ fn run() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||
println!("{}", sha256_file(&file)?);
|
||||
Ok(())
|
||||
}
|
||||
Command::Gates { output, execute } => {
|
||||
let report = run_gates(execute);
|
||||
Command::Gates {
|
||||
output,
|
||||
execute,
|
||||
candidate,
|
||||
harness_results,
|
||||
skip,
|
||||
only,
|
||||
log_dir,
|
||||
} => {
|
||||
let bound = match &candidate {
|
||||
Some(path) => Some(candidate_ref(path)?),
|
||||
None => None,
|
||||
};
|
||||
let evidence = match &harness_results {
|
||||
Some(path) => Some(HarnessEvidenceSet::from_path(path)?),
|
||||
None => None,
|
||||
};
|
||||
let log_dir = log_dir.or_else(|| {
|
||||
Some(
|
||||
output
|
||||
.parent()
|
||||
.unwrap_or_else(|| std::path::Path::new("."))
|
||||
.join("gate-logs"),
|
||||
)
|
||||
});
|
||||
let options = GateOptions {
|
||||
execute,
|
||||
skip,
|
||||
only,
|
||||
candidate: bound,
|
||||
evidence: evidence.as_ref(),
|
||||
log_dir,
|
||||
git_commit: git_commit(),
|
||||
build_id: std::env::var("PHOEBUS_BUILD_ID").unwrap_or_default(),
|
||||
};
|
||||
let report = run_gates(&SystemGateExecutor, &options);
|
||||
report.save(&output)?;
|
||||
println!(
|
||||
"wrote {} ({} passed, {} not run)",
|
||||
output.display(),
|
||||
report
|
||||
.results
|
||||
.iter()
|
||||
.filter(|r| r.status == phoebus_installer::release::GateStatus::Passed)
|
||||
.count(),
|
||||
report.not_run()
|
||||
);
|
||||
println!("wrote {} ({})", output.display(), report.summary_line());
|
||||
for result in &report.results {
|
||||
println!(
|
||||
" {:<18} {:<8} {}",
|
||||
result.name,
|
||||
result.status.as_str(),
|
||||
result.detail
|
||||
);
|
||||
}
|
||||
if !execute {
|
||||
eprintln!(
|
||||
"note: gates were not executed; pass --execute on a configured runner \
|
||||
to run them"
|
||||
to run them. An unexecuted gate is never a pass."
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
if !report.all_passed() {
|
||||
return Err(format!(
|
||||
"candidate checks did not all pass ({})",
|
||||
report.summary_line()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -188,22 +272,22 @@ fn run() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||
candidate,
|
||||
dest,
|
||||
gate_report,
|
||||
manifest,
|
||||
signature,
|
||||
} => {
|
||||
let report = GateReport::load(&gate_report)?;
|
||||
if !report.all_passed() {
|
||||
return Err(format!(
|
||||
"refusing to publish: {} gate(s) passed, {} not run \
|
||||
(run the gates on a configured runner and supply a passing report)",
|
||||
report
|
||||
.results
|
||||
.iter()
|
||||
.filter(|r| r.status == phoebus_installer::release::GateStatus::Passed)
|
||||
.count(),
|
||||
report.not_run()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
atomic_publish(&candidate, &dest)?;
|
||||
let manifest = match &manifest {
|
||||
Some(path) => Some(ReleaseManifest::load(path)?),
|
||||
None => None,
|
||||
};
|
||||
let request = PublishRequest {
|
||||
candidate: &candidate,
|
||||
dest: &dest,
|
||||
report: &report,
|
||||
manifest: manifest.as_ref(),
|
||||
signature: signature.as_deref(),
|
||||
};
|
||||
publish_release(&request, &MinisignVerifier)?;
|
||||
println!("published {} -> {}", candidate.display(), dest.display());
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
10
installer-rs/rust-toolchain.toml
Normal file
10
installer-rs/rust-toolchain.toml
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# Pinned stable Rust toolchain for reproducible builds and CI.
|
||||
#
|
||||
# This is a stable release, not a nightly: the project's stability policy
|
||||
# forbids release candidates and unstable defaults. Rustup-managed hosts will
|
||||
# use this automatically; the distribution toolchain ignores the file.
|
||||
[toolchain]
|
||||
channel = "1.93.1"
|
||||
profile = "minimal"
|
||||
components = ["rustfmt", "clippy"]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
|
@ -8,6 +8,10 @@ touching a physical disk. The target is always a qcow2 file.
|
|||
- `run-vm.sh` interactive QEMU launcher (manual testing).
|
||||
- `install.sh` guest-side, non-interactive installation (run in the VM).
|
||||
- `serial-driver.py` host-side pexpect driver (`install` and `boot` modes).
|
||||
- `rollback-driver.py` host-side rollback + no-model-recovery driver (`plan`,
|
||||
`run` and `recovery` modes); see `docs/ROLLBACK-ACCEPTANCE.md`.
|
||||
- `rollback_acceptance.py` the ordered journey logic, testable with fakes.
|
||||
- `vmsafety.py` shared disk-safety, structured-result and QMP helpers.
|
||||
- `test.conf.example` installer configuration used by the driver.
|
||||
|
||||
## Host prerequisites
|
||||
|
|
|
|||
419
installer/tests/vm/rollback-driver.py
Normal file
419
installer/tests/vm/rollback-driver.py
Normal file
|
|
@ -0,0 +1,419 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Disposable-VM acceptance driver for rollback and no-model recovery.
|
||||
|
||||
This is the *live* counterpart to ``rollback_acceptance.py``: it boots an already
|
||||
installed qcow2 disk (the installer ISO is **never** attached), drives the
|
||||
serial console, and runs the ordered journey in
|
||||
:mod:`rollback_acceptance`. Its ``results.json`` feeds the
|
||||
``updates-and-rollback`` and ``no-model-recovery`` release gates.
|
||||
|
||||
Modes (the entry point is explicit; nothing runs by accident):
|
||||
|
||||
plan print the steps, the pinned input identities and the QEMU safety
|
||||
checks, then exit 0 without starting a VM. Safe on any host.
|
||||
run the full journey: snapshot, package change, breakage, `sys undo`,
|
||||
detached reboot, restoration checks, then a recovery boot with
|
||||
networking disabled, and write both scenarios' evidence.
|
||||
recovery re-verify only the no-model recovery boot. Boots normally once to
|
||||
arm the recovery entry, then reboots with no network. Writes the
|
||||
``rollback-recovery`` evidence.
|
||||
|
||||
Safety: every writable disk is validated by :mod:`vmsafety` before QEMU starts.
|
||||
It must be a qcow2 file inside ``PHOEBUS_VM_DISKS``; ``/dev/*`` paths, block and
|
||||
character devices, raw images, escaping symlinks and outside backing chains are
|
||||
refused. The installer ISO is deliberately not a boot input here, and the
|
||||
recovery boot uses ``-net none``.
|
||||
|
||||
Environment (all paths must be local; no secrets on the command line):
|
||||
|
||||
OVMF_CODE, OVMF_VARS UEFI firmware and variable template (required)
|
||||
DISK (or DISKS) disposable qcow2 installed disk(s)
|
||||
PHOEBUS_VM_DISKS directory the disks must live in
|
||||
ISO installer ISO whose digest is pinned (required)
|
||||
PHOEBUS_VM_SYS host path to the `sys` binary under test
|
||||
PHOEBUS_VM_SYS_SHA256 optional expected digest (otherwise computed)
|
||||
PHOEBUS_VM_ARTIFACT[_SHA256] release tool candidate binding (optional here)
|
||||
LUKS_PASSPHRASE, USER_PASSWORD, USERNAME
|
||||
SYS_BIN guest `sys` path (default /usr/local/bin/sys)
|
||||
ROLLBACK_PACKAGE controlled package change (default app-misc/hello)
|
||||
QEMU_BIN, VM_MEM, VM_SMP, PHOEBUS_VM_STATE, PHOEBUS_VM_STRICT
|
||||
|
||||
Example (see docs/ROLLBACK-ACCEPTANCE.md):
|
||||
|
||||
PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \\
|
||||
DISK=/var/tmp/phoebus-vm/disks/rollback.qcow2 \\
|
||||
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \\
|
||||
OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \\
|
||||
ISO=/path/phoebus-installer-amd64.iso \\
|
||||
PHOEBUS_VM_SYS=/path/sys \\
|
||||
python3 installer/tests/vm/rollback-driver.py run
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pexpect
|
||||
|
||||
import vmsafety
|
||||
from rollback_acceptance import (
|
||||
EXPECTED_STEP_NAMES,
|
||||
REQUIRED_INPUTS,
|
||||
RC_SENTINEL,
|
||||
SCENARIO_RECOVERY,
|
||||
SCENARIO_ROLLBACK,
|
||||
CommandResult,
|
||||
RollbackJourney,
|
||||
validate_acceptance_disks,
|
||||
)
|
||||
from vmsafety import INPUT_ISO, INPUT_SYS, Results
|
||||
|
||||
QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64")
|
||||
LIBDIR = os.environ.get("QEMU_LIBDIR", "")
|
||||
DATADIR = os.environ.get("QEMU_DATADIR", "")
|
||||
OVMF_CODE = os.environ.get("OVMF_CODE", "")
|
||||
OVMF_VARS_TEMPLATE = os.environ.get("OVMF_VARS", "")
|
||||
DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "")
|
||||
ISO = os.environ.get("ISO", "")
|
||||
SYS_HOST = os.environ.get("PHOEBUS_VM_SYS", "")
|
||||
SYS_BIN = os.environ.get("SYS_BIN", "/usr/local/bin/sys")
|
||||
PACKAGE = os.environ.get("ROLLBACK_PACKAGE", "app-misc/hello")
|
||||
HOME = os.environ.get("ROLLBACK_HOME", "/home/phoebus")
|
||||
LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test")
|
||||
USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test")
|
||||
USER = os.environ.get("USERNAME", "phoebus")
|
||||
MEM = os.environ.get("VM_MEM", "8192")
|
||||
SMP = os.environ.get("VM_SMP", "8")
|
||||
BOOT_TIMEOUT = int(os.environ.get("BOOT_TIMEOUT", "600"))
|
||||
LOGIN_TIMEOUT = int(os.environ.get("LOGIN_TIMEOUT", "300"))
|
||||
PROMPT = os.environ.get("GUEST_PROMPT", r"[$#] $")
|
||||
|
||||
|
||||
def disposable_dir() -> Path:
|
||||
base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")
|
||||
return Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks"))
|
||||
|
||||
|
||||
def sha256_file(path: str | os.PathLike) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def strip_echo(text: str, command: str) -> str:
|
||||
"""Drop the console echo of a typed command and the rc sentinel line."""
|
||||
lines = text.splitlines()
|
||||
if lines and command and command[:40] in lines[0]:
|
||||
lines = lines[1:]
|
||||
return "\n".join(line for line in lines if RC_SENTINEL not in line)
|
||||
|
||||
|
||||
class PexpectGuest:
|
||||
"""A :class:`rollback_acceptance.Guest` backed by a serial console."""
|
||||
|
||||
def __init__(self, child, log, user_pw: str):
|
||||
self.child = child
|
||||
self.log = log
|
||||
self.user_pw = user_pw
|
||||
|
||||
def _ensure_sudo(self) -> None:
|
||||
"""Refresh the sudo timestamp, feeding the password off the log.
|
||||
|
||||
``sudo -n true`` succeeds while the timestamp is valid; otherwise
|
||||
``sudo -v`` prompts and the password is sent with the pexpect logfile
|
||||
detached, so it never reaches the serial transcript.
|
||||
"""
|
||||
self.child.sendline("sudo -n true 2>/dev/null || sudo -v")
|
||||
idx = self.child.expect(
|
||||
[r"\[sudo\] password for [^:]+:", PROMPT, pexpect.TIMEOUT], timeout=30
|
||||
)
|
||||
if idx == 0:
|
||||
saved = self.child.logfile
|
||||
self.child.logfile = None
|
||||
try:
|
||||
self.child.sendline(self.user_pw)
|
||||
self.child.expect(PROMPT, timeout=30)
|
||||
finally:
|
||||
self.child.logfile = saved
|
||||
elif idx == 1:
|
||||
return
|
||||
else: # pragma: no cover - console fault
|
||||
raise RuntimeError("sudo timestamp could not be refreshed")
|
||||
|
||||
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
||||
try:
|
||||
if "sudo" in command:
|
||||
self._ensure_sudo()
|
||||
end = f"echo {RC_SENTINEL}=$(printf '%s' $?)"
|
||||
self.child.sendline(f"{command}; {end}")
|
||||
idx = self.child.expect(
|
||||
[rf"{RC_SENTINEL}=([0-9]+)", pexpect.TIMEOUT, pexpect.EOF],
|
||||
timeout=timeout,
|
||||
)
|
||||
except (pexpect.TIMEOUT, pexpect.EOF):
|
||||
return CommandResult(command=command, disconnected=True, stdout="")
|
||||
if idx == 0:
|
||||
return CommandResult(
|
||||
command=command,
|
||||
stdout=strip_echo(self.child.before or "", command),
|
||||
exit_code=int(self.child.match.group(1)),
|
||||
)
|
||||
if idx == 1:
|
||||
return CommandResult(
|
||||
command=command, timed_out=True, stdout=self.child.before or ""
|
||||
)
|
||||
return CommandResult(
|
||||
command=command, disconnected=True, stdout=self.child.before or ""
|
||||
)
|
||||
|
||||
|
||||
class PexpectTransport:
|
||||
"""Boots the installed disk; the installer ISO is never attached."""
|
||||
|
||||
def __init__(self, disks, run_paths):
|
||||
self.disks = disks
|
||||
self.paths = run_paths
|
||||
self.child = None
|
||||
self.log = None
|
||||
self._boot = 0
|
||||
|
||||
def _args(self, *, network: bool) -> list[str]:
|
||||
env = dict(os.environ)
|
||||
if LIBDIR:
|
||||
env["LD_LIBRARY_PATH"] = LIBDIR
|
||||
args = ["-machine", "q35,accel=kvm", "-cpu", "host", "-smp", SMP, "-m", MEM]
|
||||
if DATADIR:
|
||||
args += ["-L", DATADIR]
|
||||
args += [
|
||||
"-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}",
|
||||
"-drive", f"if=pflash,format=raw,file={self.paths.ovmf_vars}",
|
||||
]
|
||||
for disk in self.disks:
|
||||
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
||||
if network:
|
||||
args += ["-netdev", "user,id=n0",
|
||||
"-device", "virtio-net-pci,netdev=n0,romfile="]
|
||||
else:
|
||||
# Recovery must work with networking disabled.
|
||||
args += ["-net", "none"]
|
||||
args += [
|
||||
"-device", "virtio-rng-pci", "-vga", "std", "-display", "none",
|
||||
"-serial", "stdio", "-monitor", "none", "-no-reboot",
|
||||
]
|
||||
return args
|
||||
|
||||
def _spawn(self, *, network: bool) -> None:
|
||||
self._boot += 1
|
||||
log_path = self.paths.directory / f"serial-boot{self._boot}.log"
|
||||
self.log = open(log_path, "w")
|
||||
env = dict(os.environ)
|
||||
if LIBDIR:
|
||||
env["LD_LIBRARY_PATH"] = LIBDIR
|
||||
self.child = pexpect.spawn(
|
||||
QEMU,
|
||||
self._args(network=network),
|
||||
env=env,
|
||||
timeout=BOOT_TIMEOUT,
|
||||
encoding="utf-8",
|
||||
codec_errors="replace",
|
||||
maxread=65536,
|
||||
)
|
||||
self.child.delaybeforesend = 0.4
|
||||
self.child.logfile = self.log
|
||||
|
||||
def _unlock_and_login(self) -> None:
|
||||
assert self.child is not None
|
||||
self.child.expect([r"[Pp]assphrase", r"login:"], timeout=BOOT_TIMEOUT)
|
||||
if "assphrase" in (self.child.after or "").lower():
|
||||
self.child.sendline(LUKS_PW)
|
||||
logged_in = False
|
||||
for _ in range(5):
|
||||
self.child.expect(fr"{re.escape(USER)} login:", timeout=LOGIN_TIMEOUT)
|
||||
self.child.sendline(USER)
|
||||
idx = self.child.expect(
|
||||
[r"[Pp]assword:", fr"{re.escape(USER)} login:", r"Login incorrect"],
|
||||
timeout=60,
|
||||
)
|
||||
if idx == 0:
|
||||
self.child.sendline(USER_PW)
|
||||
if self.child.expect([PROMPT, r"Login incorrect", fr"{re.escape(USER)} login:"],
|
||||
timeout=120) == 0:
|
||||
logged_in = True
|
||||
break
|
||||
if not logged_in:
|
||||
raise RuntimeError("could not log in on the installed system")
|
||||
|
||||
def boot_installed(self):
|
||||
self._spawn(network=True)
|
||||
self._unlock_and_login()
|
||||
return PexpectGuest(self.child, self.log, USER_PW)
|
||||
|
||||
def boot_recovery(self):
|
||||
self._spawn(network=False)
|
||||
self._unlock_and_login()
|
||||
return PexpectGuest(self.child, self.log, USER_PW)
|
||||
|
||||
def poweroff(self) -> None:
|
||||
if self.child is None:
|
||||
return
|
||||
try:
|
||||
PexpectGuest(self.child, self.log, USER_PW).run("sudo poweroff -f", timeout=60)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
self.child.close(force=True)
|
||||
finally:
|
||||
if self.log:
|
||||
self.log.close()
|
||||
self.child = None
|
||||
self.log = None
|
||||
|
||||
|
||||
def pin_inputs(results: Results, *, iso: str, sys_host: str) -> dict:
|
||||
"""Bind the ISO and `sys` identities the run exercised."""
|
||||
bound = {}
|
||||
if iso:
|
||||
results.bind_input_file(INPUT_ISO, iso)
|
||||
bound[INPUT_ISO] = results.inputs[INPUT_ISO]
|
||||
if sys_host:
|
||||
expected = os.environ.get("PHOEBUS_VM_SYS_SHA256")
|
||||
digest = expected or sha256_file(sys_host)
|
||||
results.bind_input(INPUT_SYS, digest)
|
||||
bound[INPUT_SYS] = results.inputs[INPUT_SYS]
|
||||
results.bind_artifact_from_env()
|
||||
return bound
|
||||
|
||||
|
||||
def plan_text() -> str:
|
||||
lines = [
|
||||
"rollback acceptance journey (no VM started)",
|
||||
"scenario: " + SCENARIO_ROLLBACK,
|
||||
"recovery: " + SCENARIO_RECOVERY,
|
||||
"steps:",
|
||||
]
|
||||
lines += [f" {i + 1:2d}. {name}" for i, name in enumerate(EXPECTED_STEP_NAMES)]
|
||||
for scenario, required in REQUIRED_INPUTS.items():
|
||||
lines.append(f"required inputs for {scenario}: {', '.join(required)}")
|
||||
lines.append("safety: every DISK must be a qcow2 inside PHOEBUS_VM_DISKS;")
|
||||
lines.append(" the installer ISO is never attached; recovery uses -net none.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_results(scenario: str, mocked: bool = False) -> Results:
|
||||
results = Results(scenario)
|
||||
if mocked:
|
||||
results.mark_mocked("fake transport")
|
||||
return results
|
||||
|
||||
|
||||
def run_full(disks, run_paths, sys_digest) -> int:
|
||||
rollback = build_results(SCENARIO_ROLLBACK)
|
||||
recovery = build_results(SCENARIO_RECOVERY)
|
||||
pin_inputs(rollback, iso=ISO, sys_host=SYS_HOST)
|
||||
pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST)
|
||||
transport = PexpectTransport(disks, run_paths)
|
||||
journey = RollbackJourney(
|
||||
transport,
|
||||
results=rollback,
|
||||
recovery_results=recovery,
|
||||
home=HOME,
|
||||
sys_bin=SYS_BIN,
|
||||
package=PACKAGE,
|
||||
iso_sha256=os.environ.get("PHOEBUS_VM_ARTIFACT_SHA256") or (rollback.inputs.get(INPUT_ISO) or {}).get("sha256"),
|
||||
sys_sha256=sys_digest,
|
||||
)
|
||||
try:
|
||||
journey.run()
|
||||
finally:
|
||||
transport.poweroff()
|
||||
rollback.write(run_paths.results_json)
|
||||
# The release tool imports any results.json under a directory tree, so
|
||||
# the recovery scenario lives beside the rollback run.
|
||||
recovery_dir = run_paths.directory / "recovery"
|
||||
recovery_dir.mkdir(parents=True, exist_ok=True)
|
||||
recovery.write(recovery_dir / "results.json")
|
||||
print(rollback.summary())
|
||||
print(recovery.summary())
|
||||
return 1 if (rollback.exit_code() or recovery.exit_code()) else 0
|
||||
|
||||
|
||||
def run_recovery_only(disks, run_paths, sys_digest) -> int:
|
||||
recovery = build_results(SCENARIO_RECOVERY)
|
||||
pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST)
|
||||
# Arm the one-shot recovery entry on a normal boot first.
|
||||
transport = PexpectTransport(disks, run_paths)
|
||||
try:
|
||||
guest = transport.boot_installed()
|
||||
guest.run("sudo bootctl set-oneshot phoebus-recovery", timeout=120)
|
||||
finally:
|
||||
transport.poweroff()
|
||||
transport = PexpectTransport(disks, run_paths)
|
||||
journey = RollbackJourney(
|
||||
transport,
|
||||
results=build_results(SCENARIO_ROLLBACK),
|
||||
recovery_results=recovery,
|
||||
home=HOME,
|
||||
sys_bin=SYS_BIN,
|
||||
package=PACKAGE,
|
||||
sys_sha256=sys_digest,
|
||||
)
|
||||
try:
|
||||
journey.run_recovery()
|
||||
finally:
|
||||
transport.poweroff()
|
||||
recovery.write(run_paths.results_json)
|
||||
print(recovery.summary())
|
||||
return recovery.exit_code()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
mode = sys.argv[1] if len(sys.argv) > 1 else ""
|
||||
if mode == "plan":
|
||||
print(plan_text())
|
||||
return 0
|
||||
if mode not in ("run", "recovery"):
|
||||
print(__doc__)
|
||||
return 2
|
||||
if not OVMF_CODE or not OVMF_VARS_TEMPLATE:
|
||||
print("harness error: OVMF_CODE and OVMF_VARS are required", file=sys.stderr)
|
||||
return 2
|
||||
if not ISO:
|
||||
print("harness error: ISO is required to pin the candidate identity", file=sys.stderr)
|
||||
return 2
|
||||
if not SYS_HOST or not Path(SYS_HOST).is_file():
|
||||
print(
|
||||
"harness error: PHOEBUS_VM_SYS must point at the `sys` binary under test",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
scenario = SCENARIO_ROLLBACK if mode == "run" else SCENARIO_RECOVERY
|
||||
run_paths = vmsafety.make_run_paths(
|
||||
f"rollback-{mode}",
|
||||
base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"),
|
||||
ovmf_vars_template=OVMF_VARS_TEMPLATE,
|
||||
)
|
||||
try:
|
||||
disks = validate_acceptance_disks(DISKS.split(","), disposable_dir())
|
||||
except vmsafety.UnsafeDiskError as exc:
|
||||
results = build_results(scenario)
|
||||
results.blocked("disposable-disk", str(exc))
|
||||
results.write(run_paths.results_json)
|
||||
print(f"harness error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
sys_digest = os.environ.get("PHOEBUS_VM_SYS_SHA256") or sha256_file(SYS_HOST)
|
||||
if mode == "run":
|
||||
return run_full(disks, run_paths, sys_digest)
|
||||
return run_recovery_only(disks, run_paths, sys_digest)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
852
installer/tests/vm/rollback_acceptance.py
Normal file
852
installer/tests/vm/rollback_acceptance.py
Normal file
|
|
@ -0,0 +1,852 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Offline orchestration for the rollback + no-model-recovery acceptance run.
|
||||
|
||||
This module holds the *logic* of the journey: the ordered steps, the guest
|
||||
commands they issue, and the assertions that decide PASS/FAIL. It deliberately
|
||||
imports neither pexpect nor QEMU, so the whole journey can be exercised on a
|
||||
plain host against a fake transport (see ``test_rollback_acceptance.py``).
|
||||
|
||||
The real VM driver (``rollback-driver.py``) supplies a pexpect-backed transport
|
||||
implementing :class:`BootTransport` and a :class:`Guest` that runs commands over
|
||||
the serial console. The release gates import the ``results.json`` these runs
|
||||
write; a run made with a fake transport is marked ``mocked`` and can never be a
|
||||
gate pass.
|
||||
|
||||
Journey shape (the installer already created a coordinated ``@`` + ``@var``
|
||||
known-good snapshot at install time):
|
||||
|
||||
boot installed (ISO detached)
|
||||
1. identify the newest committed coordinated snapshot
|
||||
2. write a marker into /home (a subvolume rollback must not touch)
|
||||
3. apply a controlled package change (touches / and /var/db/pkg)
|
||||
4. introduce deterministic guest-only breakage in /
|
||||
5. run the real `sys undo` (restores root + var as the next boot root)
|
||||
reboot, boot installed (ISO detached)
|
||||
6. restored root/var generations match the rollback state
|
||||
7. the changed package and its files are gone
|
||||
8. /var/db/pkg matches / (consistent package database)
|
||||
9. the booted kernel has matching modules in the restored root
|
||||
10. /home data survived
|
||||
arm the recovery entry, reboot, boot recovery (no network, no model)
|
||||
11. recovery entry boots, no network, and `sys` still works with no model
|
||||
|
||||
Nothing here decides that a *live* run passed: only a real transport on a
|
||||
disposable qcow2 disk can produce release evidence. The mocked run exists to
|
||||
prove the orchestration itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import re
|
||||
import tomllib
|
||||
from typing import Callable, Protocol, runtime_checkable
|
||||
|
||||
import vmsafety
|
||||
from vmsafety import INPUT_ISO, INPUT_SYS, Results
|
||||
|
||||
# Scenario names. Each is a release-gate evidence key.
|
||||
SCENARIO_ROLLBACK = "rollback-acceptance"
|
||||
SCENARIO_RECOVERY = "rollback-recovery"
|
||||
|
||||
# The input identities a gate must see before it may pass. The rollback
|
||||
# semantics live in `sys`, so naming only the ISO is incomplete evidence.
|
||||
REQUIRED_INPUTS: dict[str, tuple[str, ...]] = {
|
||||
SCENARIO_ROLLBACK: (INPUT_ISO, INPUT_SYS),
|
||||
SCENARIO_RECOVERY: (INPUT_ISO, INPUT_SYS),
|
||||
}
|
||||
|
||||
# Stable guest paths and sentinels.
|
||||
SNAPSHOT_HISTORY = "/var/lib/phoebus/snapshot-history.toml"
|
||||
ROLLBACK_STATE = "/var/lib/phoebus/rollback-state.toml"
|
||||
PKG_DB = "/var/db/pkg"
|
||||
HOSTNAME_FILE = "/etc/hostname"
|
||||
HOME_MARKER_REL = "rollback-acceptance/marker"
|
||||
HOME_MARKER_TOKEN = "phoebus-rollback-home-marker"
|
||||
HOSTNAME_SENTINEL = "phoebus-rollback-broken-hostname"
|
||||
DEFAULT_SYS = "/usr/local/bin/sys"
|
||||
DEFAULT_PACKAGE = "app-misc/hello"
|
||||
|
||||
# File framing so a read is not confused with console noise.
|
||||
FILE_FRAME = "__PHOEBUS_RB_FILE__"
|
||||
# The driver appends this and parses the exit status from the console.
|
||||
RC_SENTINEL = "__PHOEBUS_RB_RC"
|
||||
|
||||
MAX_OUTPUT_BYTES = 64 * 1024
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Small, testable value types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class CommandResult:
|
||||
"""The outcome of one guest command.
|
||||
|
||||
``timed_out`` and ``disconnected`` are distinct from a nonzero exit so a
|
||||
broken connection can never be mistaken for a clean failure that a later
|
||||
step might tolerate.
|
||||
"""
|
||||
|
||||
command: str
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
exit_code: int | None = None
|
||||
timed_out: bool = False
|
||||
disconnected: bool = False
|
||||
|
||||
def bounded(self) -> "CommandResult":
|
||||
combined = self.stdout[-MAX_OUTPUT_BYTES:]
|
||||
self.stdout = combined
|
||||
self.stderr = self.stderr[-MAX_OUTPUT_BYTES:]
|
||||
return self
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return (
|
||||
not self.timed_out
|
||||
and not self.disconnected
|
||||
and self.exit_code == 0
|
||||
)
|
||||
|
||||
@property
|
||||
def failure_reason(self) -> str:
|
||||
if self.timed_out:
|
||||
return f"timeout running `{self.command}`"
|
||||
if self.disconnected:
|
||||
return f"guest disconnected running `{self.command}`"
|
||||
if self.exit_code is None:
|
||||
return f"no exit status for `{self.command}`"
|
||||
if self.exit_code != 0:
|
||||
detail = (self.stderr or self.stdout).strip().splitlines()
|
||||
tail = detail[-1] if detail else ""
|
||||
return f"`{self.command}` exited {self.exit_code}: {tail}"
|
||||
return ""
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class SnapshotRecord:
|
||||
id: str
|
||||
operation: str
|
||||
snapshot_subvol: str
|
||||
var_snapshot: str | None
|
||||
status: str
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Guest(Protocol):
|
||||
"""A shell on a booted installed system."""
|
||||
|
||||
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class BootTransport(Protocol):
|
||||
"""Boots the installed disk in the requested mode.
|
||||
|
||||
* ``boot_installed`` boots the normal entry with the installer ISO detached.
|
||||
* ``boot_recovery`` boots the recovery entry with networking disabled.
|
||||
"""
|
||||
|
||||
def boot_installed(self) -> Guest:
|
||||
...
|
||||
|
||||
def boot_recovery(self) -> Guest:
|
||||
...
|
||||
|
||||
def poweroff(self) -> None:
|
||||
...
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parsers (pure functions, unit-tested directly)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_between(text: str, start: str, end: str) -> str:
|
||||
"""Return the text between the first ``start`` and the next ``end``.
|
||||
|
||||
If the markers are absent the whole text is returned unchanged, so a fake
|
||||
transport may return clean content directly.
|
||||
"""
|
||||
if start not in text or end not in text:
|
||||
return text
|
||||
body = text.split(start, 1)[1]
|
||||
return body.split(end, 1)[0]
|
||||
|
||||
|
||||
def parse_snapshot_history(text: str) -> list[SnapshotRecord]:
|
||||
"""Parse the snapshot history TOML into records.
|
||||
|
||||
Raises :class:`ValueError` when the document is unreadable; a caller must
|
||||
never treat "could not parse" as "no snapshots".
|
||||
"""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
raise ValueError("empty snapshot history")
|
||||
try:
|
||||
data = tomllib.loads(text)
|
||||
except tomllib.TOMLDecodeError as exc: # pragma: no cover - message detail
|
||||
raise ValueError(f"invalid snapshot history: {exc}") from exc
|
||||
records = data.get("records", [])
|
||||
if not isinstance(records, list):
|
||||
raise ValueError("snapshot history has no `records` list")
|
||||
out: list[SnapshotRecord] = []
|
||||
for entry in records:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
out.append(
|
||||
SnapshotRecord(
|
||||
id=str(entry.get("id", "")),
|
||||
operation=str(entry.get("operation", "")),
|
||||
snapshot_subvol=str(entry.get("snapshot_subvol", "")),
|
||||
var_snapshot=entry.get("var_snapshot"),
|
||||
status=str(entry.get("status", "")),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def newest_committed_coordinated(
|
||||
records: list[SnapshotRecord],
|
||||
) -> SnapshotRecord | None:
|
||||
"""The newest committed snapshot that also captured ``/var``.
|
||||
|
||||
A root snapshot with no paired ``@var`` snapshot cannot restore a
|
||||
consistent package database, so it is not a valid rollback target.
|
||||
"""
|
||||
candidates = [
|
||||
r
|
||||
for r in records
|
||||
if r.status == "committed" and r.snapshot_subvol and r.var_snapshot
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
return candidates[-1]
|
||||
|
||||
|
||||
def parse_rollback_state(text: str) -> dict:
|
||||
"""Parse ``/var/lib/phoebus/rollback-state.toml`` into a dict."""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
raise ValueError("empty rollback state")
|
||||
try:
|
||||
return tomllib.loads(text)
|
||||
except tomllib.TOMLDecodeError as exc: # pragma: no cover - message detail
|
||||
raise ValueError(f"invalid rollback state: {exc}") from exc
|
||||
|
||||
|
||||
def normalize_subvol(value: str | None) -> str | None:
|
||||
"""Canonicalise a subvolume path for comparison.
|
||||
|
||||
Btrfs reports the same subvolume as ``@restore-x`` in ``rootflags`` and
|
||||
``/@restore-x`` in a fstab mount option; comparison must ignore that.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
return value.lstrip("/")
|
||||
|
||||
|
||||
def parse_rootflags_subvol(cmdline: str) -> str | None:
|
||||
"""Extract the ``subvol=`` value from the kernel ``rootflags=`` option."""
|
||||
match = re.search(r"rootflags=[^ ]*?\bsubvol=([^,\s]+)", cmdline)
|
||||
if match:
|
||||
return normalize_subvol(match.group(1))
|
||||
# Some kernels present rootflags only as the bare option.
|
||||
match = re.search(r"\bsubvol=([^,\s]+)", cmdline)
|
||||
return normalize_subvol(match.group(1)) if match else None
|
||||
|
||||
|
||||
def parse_fstab_subvol(text: str, mountpoint: str) -> str | None:
|
||||
"""The ``subvol=`` value for a mountpoint in ``/etc/fstab``."""
|
||||
for line in text.splitlines():
|
||||
line = line.split("#", 1)[0].strip()
|
||||
if not line:
|
||||
continue
|
||||
fields = line.split()
|
||||
if len(fields) < 4:
|
||||
continue
|
||||
if fields[1] != mountpoint:
|
||||
continue
|
||||
match = re.search(r"(?:^|,)subvol=([^,\s]+)", fields[3])
|
||||
if match:
|
||||
return normalize_subvol(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def parse_package_dirs(text: str) -> list[str]:
|
||||
"""Absolute ``/var/db/pkg/<cat>/<pkg>`` directories from a listing."""
|
||||
return sorted({line.strip() for line in text.splitlines() if line.strip()})
|
||||
|
||||
|
||||
def parse_contents_files(text: str) -> list[str]:
|
||||
"""Installed file paths from a Portage ``CONTENTS`` file.
|
||||
|
||||
The format is ``<obj> <path> <hash...>`` per line.
|
||||
"""
|
||||
paths: list[str] = []
|
||||
for line in text.splitlines():
|
||||
fields = line.split()
|
||||
if len(fields) >= 2 and fields[0] in ("obj", "dir", "sym"):
|
||||
paths.append(fields[1])
|
||||
return paths
|
||||
|
||||
|
||||
def parse_uname(text: str) -> str:
|
||||
return text.strip().splitlines()[-1].strip() if text.strip() else ""
|
||||
|
||||
|
||||
def parse_network_addresses(text: str) -> list[str]:
|
||||
"""Interfaces that have an address, from ``ip -brief addr`` output."""
|
||||
out: list[str] = []
|
||||
for line in text.splitlines():
|
||||
fields = line.split()
|
||||
if not fields:
|
||||
continue
|
||||
name = fields[0]
|
||||
rest = fields[1:]
|
||||
if any("." in f or ":" in f for f in rest):
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Journey
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class _Step:
|
||||
name: str
|
||||
method_name: str
|
||||
boot: str | None = None
|
||||
stream: str = "rollback"
|
||||
|
||||
|
||||
# Order is the contract. A change here changes the release gate; keep every
|
||||
# expected step in the list so a stopped run can mark the rest NOT RUN.
|
||||
_STEPS: tuple[_Step, ...] = (
|
||||
_Step("sys-binary-identity", "_step_sys_identity", boot="installed"),
|
||||
_Step("known-good-coordinated-snapshot", "_step_known_good"),
|
||||
_Step("home-data-marker", "_step_mark_home"),
|
||||
_Step("controlled-package-change", "_step_package_change"),
|
||||
_Step("deterministic-guest-breakage", "_step_break"),
|
||||
_Step("sys-undo-activates", "_step_undo"),
|
||||
_Step("reboot-detached", "_step_reboot", boot="installed"),
|
||||
_Step("restored-root-var-generations", "_step_generations"),
|
||||
_Step("package-files-restored", "_step_package_files"),
|
||||
_Step("portage-database-consistent", "_step_portage_db"),
|
||||
_Step("kernel-modules-compatible", "_step_kernel_modules"),
|
||||
_Step("home-data-survives", "_step_home_survives"),
|
||||
_Step("recovery-entry-armed", "_step_arm_recovery"),
|
||||
_Step(
|
||||
"recovery-boot-no-network-no-model",
|
||||
"_step_recovery",
|
||||
boot="recovery",
|
||||
stream="recovery",
|
||||
),
|
||||
)
|
||||
|
||||
EXPECTED_STEP_NAMES: tuple[str, ...] = tuple(step.name for step in _STEPS)
|
||||
|
||||
|
||||
class RollbackJourney:
|
||||
"""Run the ordered acceptance steps against a :class:`BootTransport`.
|
||||
|
||||
The class records every expected step, so a run that stops early reports the
|
||||
remainder as NOT RUN and fails an explicit ``journey-complete`` check rather
|
||||
than silently looking like a pass.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
transport: BootTransport,
|
||||
*,
|
||||
results: Results | None = None,
|
||||
recovery_results: Results | None = None,
|
||||
home: str = "/home/phoebus",
|
||||
sys_bin: str = DEFAULT_SYS,
|
||||
package: str = DEFAULT_PACKAGE,
|
||||
iso_sha256: str | None = None,
|
||||
sys_sha256: str | None = None,
|
||||
mocked: bool = False,
|
||||
):
|
||||
self.transport = transport
|
||||
self.results = results or Results(SCENARIO_ROLLBACK)
|
||||
self.recovery_results = recovery_results or Results(SCENARIO_RECOVERY)
|
||||
for stream in (self.results, self.recovery_results):
|
||||
if iso_sha256:
|
||||
stream.bind_input(INPUT_ISO, iso_sha256)
|
||||
if sys_sha256:
|
||||
stream.bind_input(INPUT_SYS, sys_sha256)
|
||||
if mocked:
|
||||
stream.mark_mocked("fake transport: not release evidence")
|
||||
self.home = home
|
||||
self.sys_bin = sys_bin
|
||||
self.package = package
|
||||
self.expected_sys_sha256 = sys_sha256
|
||||
self.marker_path = f"{home.rstrip('/')}/{HOME_MARKER_REL}"
|
||||
self.marker_token = f"{HOME_MARKER_TOKEN}:{self.results.run_id}"
|
||||
self.boots: list[str] = []
|
||||
self.executed: list[str] = []
|
||||
|
||||
# Findings carried between steps.
|
||||
self.snapshot: SnapshotRecord | None = None
|
||||
self.original_hostname: str | None = None
|
||||
self.package_dirs_before: list[str] = []
|
||||
self.added_package_dirs: list[str] = []
|
||||
self.added_package_files: list[str] = []
|
||||
self.rollback_state: dict = {}
|
||||
|
||||
self._active = self.results
|
||||
|
||||
# -- helpers -----------------------------------------------------------
|
||||
def _check(self, name: str, condition: bool, detail: str = "") -> bool:
|
||||
return self._active.assert_true(name, condition, detail)
|
||||
|
||||
def _run(
|
||||
self, guest: Guest, script: str, timeout: float = 240.0
|
||||
) -> CommandResult:
|
||||
try:
|
||||
return guest.run(script, timeout=timeout).bounded()
|
||||
except Exception as exc: # a transport fault is a failure, not a crash
|
||||
return CommandResult(
|
||||
command=script, disconnected=True, stderr=str(exc)
|
||||
)
|
||||
|
||||
def _read_file(self, guest: Guest, path: str, timeout: float = 60.0) -> str:
|
||||
result = self._run(
|
||||
guest,
|
||||
f"echo {FILE_FRAME}; cat {path} 2>/dev/null; echo {FILE_FRAME}",
|
||||
timeout=timeout,
|
||||
)
|
||||
return extract_between(result.stdout, FILE_FRAME, FILE_FRAME)
|
||||
|
||||
# -- public API --------------------------------------------------------
|
||||
def required_inputs(self) -> tuple[str, ...]:
|
||||
return REQUIRED_INPUTS[self.results.scenario]
|
||||
|
||||
def run(self) -> bool:
|
||||
"""Execute the journey. Returns True only if every step passed."""
|
||||
if not self._check_inputs():
|
||||
for step in _STEPS:
|
||||
stream = (
|
||||
self.results
|
||||
if step.stream == "rollback"
|
||||
else self.recovery_results
|
||||
)
|
||||
stream.mark_not_run(step.name, "input identities were not pinned")
|
||||
self.results.fail("journey-complete", "missing input identities")
|
||||
self.recovery_results.fail("journey-complete", "missing input identities")
|
||||
return False
|
||||
|
||||
guest: Guest | None = None
|
||||
for index, step in enumerate(_STEPS):
|
||||
self._active = (
|
||||
self.results if step.stream == "rollback" else self.recovery_results
|
||||
)
|
||||
if step.boot is not None:
|
||||
if guest is not None:
|
||||
self.transport.poweroff()
|
||||
guest = (
|
||||
self.transport.boot_recovery()
|
||||
if step.boot == "recovery"
|
||||
else self.transport.boot_installed()
|
||||
)
|
||||
self.boots.append(step.boot)
|
||||
method: Callable[[Guest], bool] = getattr(self, step.method_name)
|
||||
self.executed.append(step.name)
|
||||
if not method(guest):
|
||||
self._finish_incomplete(index, step.name)
|
||||
return False
|
||||
|
||||
self.results.assert_true("journey-complete", True)
|
||||
self.recovery_results.assert_true("journey-complete", True)
|
||||
return True
|
||||
|
||||
def run_recovery(self) -> bool:
|
||||
"""Verify only the no-model recovery boot (used by the `recovery` mode)."""
|
||||
if not self._check_inputs():
|
||||
self.recovery_results.mark_not_run(
|
||||
"recovery-boot-no-network-no-model", "input identities were not pinned"
|
||||
)
|
||||
self.recovery_results.fail("journey-complete", "missing input identities")
|
||||
return False
|
||||
self._active = self.recovery_results
|
||||
ok = self._step_recovery(self.transport.boot_recovery())
|
||||
if ok:
|
||||
self.recovery_results.assert_true("journey-complete", True)
|
||||
else:
|
||||
self.recovery_results.fail(
|
||||
"journey-complete", "recovery verification failed"
|
||||
)
|
||||
return ok
|
||||
|
||||
# -- input identity gate ----------------------------------------------
|
||||
def _check_inputs(self) -> bool:
|
||||
ok = True
|
||||
for stream, scenario in (
|
||||
(self.results, SCENARIO_ROLLBACK),
|
||||
(self.recovery_results, SCENARIO_RECOVERY),
|
||||
):
|
||||
required = REQUIRED_INPUTS[scenario]
|
||||
missing = stream.missing_inputs(required)
|
||||
ok &= stream.assert_true(
|
||||
"input-identities",
|
||||
not missing,
|
||||
f"missing pinned inputs for {scenario}: {', '.join(missing)}",
|
||||
)
|
||||
return bool(ok)
|
||||
|
||||
def _finish_incomplete(self, failed_index: int, failed_step: str) -> None:
|
||||
for step in _STEPS[failed_index + 1:]:
|
||||
stream = (
|
||||
self.results if step.stream == "rollback" else self.recovery_results
|
||||
)
|
||||
stream.mark_not_run(step.name, "journey stopped before this step")
|
||||
# The failed step's own stream already has a FAIL; make the incompleteness
|
||||
# explicit on both so neither gate can read a partial run as a pass.
|
||||
self.results.fail(
|
||||
"journey-complete", f"stopped after `{failed_step}`"
|
||||
)
|
||||
self.recovery_results.fail(
|
||||
"journey-complete", f"stopped after `{failed_step}`"
|
||||
)
|
||||
|
||||
# -- steps -------------------------------------------------------------
|
||||
def _step_sys_identity(self, guest: Guest) -> bool:
|
||||
# The rollback semantics live in `sys`; pin the exact binary under test.
|
||||
self._active = self.results
|
||||
expected = self.expected_sys_sha256 or ""
|
||||
result = self._run(guest, f"sha256sum {shlex_quote(self.sys_bin)}", timeout=60.0)
|
||||
if not result.ok:
|
||||
return self._check("sys-binary-identity", False, result.failure_reason)
|
||||
digest = result.stdout.split()[0] if result.stdout.split() else ""
|
||||
return self._check(
|
||||
"sys-binary-identity",
|
||||
bool(expected) and digest == expected,
|
||||
f"guest sys digest {digest!r} != pinned {expected!r}",
|
||||
)
|
||||
|
||||
def _step_known_good(self, guest: Guest) -> bool:
|
||||
self._active = self.results
|
||||
text = self._read_file(guest, SNAPSHOT_HISTORY)
|
||||
try:
|
||||
records = parse_snapshot_history(text)
|
||||
except ValueError as exc:
|
||||
return self._check("known-good-coordinated-snapshot", False, str(exc))
|
||||
snapshot = newest_committed_coordinated(records)
|
||||
if snapshot is None:
|
||||
return self._check(
|
||||
"known-good-coordinated-snapshot",
|
||||
False,
|
||||
"no committed snapshot with a paired @var snapshot",
|
||||
)
|
||||
self.snapshot = snapshot
|
||||
self._check("known-good-root-snapshot", bool(snapshot.snapshot_subvol), "empty root snapshot path")
|
||||
self._check("known-good-var-snapshot", bool(snapshot.var_snapshot), "empty @var snapshot path")
|
||||
self._check(
|
||||
"known-good-coordinated-snapshot",
|
||||
True,
|
||||
f"{snapshot.id} ({snapshot.operation})",
|
||||
)
|
||||
return True
|
||||
|
||||
def _step_mark_home(self, guest: Guest) -> bool:
|
||||
self._active = self.results
|
||||
result = self._run(
|
||||
guest,
|
||||
f"mkdir -p {self.home.rstrip('/')}/{HOME_MARKER_REL.rsplit('/', 1)[0]} && "
|
||||
f"printf '%s' '{self.marker_token}' > {self.marker_path}",
|
||||
)
|
||||
if not result.ok:
|
||||
return self._check("home-data-marker", False, result.failure_reason)
|
||||
readback = self._read_file(guest, self.marker_path)
|
||||
if readback.strip() != self.marker_token:
|
||||
return self._check(
|
||||
"home-data-marker", False, "home marker did not read back"
|
||||
)
|
||||
self.original_hostname = self._read_file(guest, HOSTNAME_FILE).strip()
|
||||
self.package_dirs_before = parse_package_dirs(
|
||||
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||
)
|
||||
self._check("home-data-marker", True, self.marker_path)
|
||||
self._check(
|
||||
"hostname-recorded", bool(self.original_hostname), "could not read /etc/hostname"
|
||||
)
|
||||
return bool(self.original_hostname)
|
||||
|
||||
def _step_package_change(self, guest: Guest) -> bool:
|
||||
self._active = self.results
|
||||
result = self._run(
|
||||
guest, f"sudo {self.sys_bin} install {self.package} -y", timeout=1800.0
|
||||
)
|
||||
if not result.ok:
|
||||
return self._check("controlled-package-change", False, result.failure_reason)
|
||||
after = parse_package_dirs(
|
||||
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||
)
|
||||
added = sorted(set(after) - set(self.package_dirs_before))
|
||||
if not added:
|
||||
return self._check(
|
||||
"controlled-package-change",
|
||||
False,
|
||||
f"`{self.package}` produced no new /var/db/pkg entry",
|
||||
)
|
||||
self.added_package_dirs = added
|
||||
contents = self._read_file(guest, f"{added[0]}/CONTENTS")
|
||||
self.added_package_files = parse_contents_files(contents)
|
||||
self._check(
|
||||
"controlled-package-change", True, f"added {', '.join(added)}"
|
||||
)
|
||||
self._check(
|
||||
"package-db-recorded",
|
||||
bool(self.added_package_files),
|
||||
"changed package has no recorded files",
|
||||
)
|
||||
return True
|
||||
|
||||
def _step_break(self, guest: Guest) -> bool:
|
||||
self._active = self.results
|
||||
result = self._run(
|
||||
guest,
|
||||
f"printf '%s\\n' '{HOSTNAME_SENTINEL}' | sudo tee {HOSTNAME_FILE} >/dev/null",
|
||||
)
|
||||
if not result.ok:
|
||||
return self._check("deterministic-guest-breakage", False, result.failure_reason)
|
||||
broken = self._read_file(guest, HOSTNAME_FILE).strip()
|
||||
if broken != HOSTNAME_SENTINEL:
|
||||
return self._check(
|
||||
"deterministic-guest-breakage",
|
||||
False,
|
||||
"breakage did not take effect",
|
||||
)
|
||||
self._check(
|
||||
"deterministic-guest-breakage",
|
||||
True,
|
||||
f"{HOSTNAME_FILE} -> {HOSTNAME_SENTINEL}",
|
||||
)
|
||||
return True
|
||||
|
||||
def _step_undo(self, guest: Guest) -> bool:
|
||||
self._active = self.results
|
||||
if self.snapshot is None:
|
||||
return self._check("sys-undo-activates", False, "no snapshot selected")
|
||||
result = self._run(
|
||||
guest, f"sudo {self.sys_bin} undo {self.snapshot.id} -y", timeout=900.0
|
||||
)
|
||||
if not result.ok:
|
||||
return self._check("sys-undo-activates", False, result.failure_reason)
|
||||
state = self._read_file(guest, ROLLBACK_STATE)
|
||||
try:
|
||||
self.rollback_state = parse_rollback_state(state)
|
||||
except ValueError as exc:
|
||||
return self._check("sys-undo-activates", False, str(exc))
|
||||
if self.rollback_state.get("status") != "activated":
|
||||
return self._check(
|
||||
"sys-undo-activates",
|
||||
False,
|
||||
f"rollback state is {self.rollback_state.get('status')!r}, not 'activated'",
|
||||
)
|
||||
self._check(
|
||||
"sys-undo-activates",
|
||||
True,
|
||||
f"next boot uses {self.rollback_state.get('root_subvol')}",
|
||||
)
|
||||
return True
|
||||
|
||||
def _step_reboot(self, guest: Guest) -> bool:
|
||||
# The boot itself is performed by the transport; the driver asserts the
|
||||
# ISO is detached before this returns. Here we only record the boundary.
|
||||
self._active = self.results
|
||||
self._check("reboot-detached", True, "booted installed disk with ISO detached")
|
||||
return True
|
||||
|
||||
def _step_generations(self, guest: Guest) -> bool:
|
||||
self._active = self.results
|
||||
expected_root = str(self.rollback_state.get("root_subvol", ""))
|
||||
expected_var = str(self.rollback_state.get("var_subvol", "") or "")
|
||||
cmdline = self._read_file(guest, "/proc/cmdline")
|
||||
booted_subvol = parse_rootflags_subvol(cmdline)
|
||||
fstab = self._read_file(guest, "/etc/fstab")
|
||||
fstab_root = parse_fstab_subvol(fstab, "/")
|
||||
fstab_var = parse_fstab_subvol(fstab, "/var")
|
||||
ok = True
|
||||
ok &= self._check(
|
||||
"restored-root-generation",
|
||||
bool(expected_root) and booted_subvol == expected_root,
|
||||
f"booted subvol {booted_subvol!r} != expected {expected_root!r}",
|
||||
)
|
||||
ok &= self._check(
|
||||
"restored-fstab-root",
|
||||
bool(expected_root) and fstab_root == expected_root,
|
||||
f"fstab / subvol {fstab_root!r} != {expected_root!r}",
|
||||
)
|
||||
ok &= self._check(
|
||||
"restored-var-generation",
|
||||
bool(expected_var) and fstab_var == expected_var,
|
||||
f"fstab /var subvol {fstab_var!r} != {expected_var!r}",
|
||||
)
|
||||
# The running system must not still be the pre-rollback generation.
|
||||
ok &= self._check(
|
||||
"generation-advanced",
|
||||
booted_subvol.startswith("@restore-") if booted_subvol else False,
|
||||
f"booted subvol {booted_subvol!r} is not a restored generation",
|
||||
)
|
||||
return ok
|
||||
|
||||
def _step_package_files(self, guest: Guest) -> bool:
|
||||
self._active = self.results
|
||||
after = parse_package_dirs(
|
||||
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||
)
|
||||
still_present = [d for d in self.added_package_dirs if d in after]
|
||||
ok = self._check(
|
||||
"package-db-restored",
|
||||
not still_present,
|
||||
f"package DB still lists {still_present}",
|
||||
)
|
||||
missing_files = []
|
||||
for path in self.added_package_files:
|
||||
result = self._run(guest, f"test -e {shlex_quote(path)}")
|
||||
if result.exit_code == 0:
|
||||
missing_files.append(path)
|
||||
ok &= self._check(
|
||||
"package-files-restored",
|
||||
not missing_files,
|
||||
f"package files survived the rollback: {missing_files[:5]}",
|
||||
)
|
||||
hostname = self._read_file(guest, HOSTNAME_FILE).strip()
|
||||
ok &= self._check(
|
||||
"breakage-reverted",
|
||||
hostname == (self.original_hostname or ""),
|
||||
f"{HOSTNAME_FILE} is {hostname!r}, expected {self.original_hostname!r}",
|
||||
)
|
||||
return ok
|
||||
|
||||
def _step_portage_db(self, guest: Guest) -> bool:
|
||||
self._active = self.results
|
||||
listing = parse_package_dirs(
|
||||
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||
)
|
||||
ok = self._check(
|
||||
"portage-db-generation-count",
|
||||
listing == self.package_dirs_before,
|
||||
f"/var/db/pkg has {len(listing)} entries, expected {len(self.package_dirs_before)}",
|
||||
)
|
||||
# Every entry must carry a CONTENTS file; an inconsistent database is
|
||||
# how a mismatched root/var pairing shows up.
|
||||
probe = self._run(
|
||||
guest,
|
||||
f"for d in {PKG_DB}/*/*; do [ -f \"$d/CONTENTS\" ] || echo \"$d\"; done",
|
||||
)
|
||||
missing_contents = [
|
||||
line.strip() for line in probe.stdout.splitlines() if line.strip()
|
||||
]
|
||||
ok &= self._check(
|
||||
"portage-db-contents",
|
||||
not missing_contents,
|
||||
f"package entries without CONTENTS: {missing_contents[:5]}",
|
||||
)
|
||||
return ok
|
||||
|
||||
def _step_kernel_modules(self, guest: Guest) -> bool:
|
||||
self._active = self.results
|
||||
kernel = parse_uname(self._run(guest, "uname -r").stdout)
|
||||
modules = self._read_file(guest, "/lib/modules")
|
||||
module_dirs = [line.strip() for line in modules.splitlines() if line.strip()]
|
||||
return self._check(
|
||||
"kernel-modules-compatible",
|
||||
bool(kernel) and kernel in module_dirs,
|
||||
f"no /lib/modules/{kernel} in the restored root ({module_dirs})",
|
||||
)
|
||||
|
||||
def _step_home_survives(self, guest: Guest) -> bool:
|
||||
self._active = self.results
|
||||
readback = self._read_file(guest, self.marker_path).strip()
|
||||
return self._check(
|
||||
"home-data-survives",
|
||||
readback == self.marker_token,
|
||||
"the /home marker did not survive the rollback",
|
||||
)
|
||||
|
||||
def _step_arm_recovery(self, guest: Guest) -> bool:
|
||||
# Boot the recovery entry exactly once, then fall back to the restored
|
||||
# root. This is a real systemd-boot interface, not a QEMU shortcut.
|
||||
self._active = self.results
|
||||
result = self._run(
|
||||
guest, "sudo bootctl set-oneshot phoebus-recovery", timeout=120.0
|
||||
)
|
||||
return self._check("recovery-entry-armed", result.ok, result.failure_reason)
|
||||
|
||||
def _step_recovery(self, guest: Guest) -> bool:
|
||||
self._active = self.recovery_results
|
||||
cmdline = self._read_file(guest, "/proc/cmdline")
|
||||
ok = self._check(
|
||||
"recovery-entry-booted",
|
||||
"systemd.unit=rescue.target" in cmdline,
|
||||
"the recovery entry did not boot into rescue.target",
|
||||
)
|
||||
addresses = parse_network_addresses(self._run(guest, "ip -brief addr").stdout)
|
||||
non_loopback = [a for a in addresses if a != "lo"]
|
||||
ok &= self._check(
|
||||
"recovery-no-network",
|
||||
not non_loopback,
|
||||
f"network interfaces were up: {non_loopback}",
|
||||
)
|
||||
# `sys` itself must work with no model: a status query is deterministic
|
||||
# and offline.
|
||||
status = self._run(guest, f"{self.sys_bin} undo status")
|
||||
ok &= self._check(
|
||||
"recovery-sys-offline",
|
||||
status.ok,
|
||||
status.failure_reason or "sys status failed offline",
|
||||
)
|
||||
ask = self._run(guest, f'{self.sys_bin} ask "show system status"', timeout=120.0)
|
||||
combined = (ask.stdout + ask.stderr).lower()
|
||||
ok &= self._check(
|
||||
"recovery-no-model-fallback",
|
||||
(not ask.ok) and "no model" in combined,
|
||||
"sys ask did not report the deterministic no-model fallback",
|
||||
)
|
||||
return ok
|
||||
|
||||
|
||||
def shlex_quote(value: str) -> str:
|
||||
"""Minimal single-quote shell escaping for a known-safe path."""
|
||||
return "'" + value.replace("'", "'\\''") + "'"
|
||||
|
||||
|
||||
def validate_acceptance_disks(disks, test_dir) -> list:
|
||||
"""Refuse anything but disposable qcow2 disks inside the test directory.
|
||||
|
||||
Kept here so the journey's safety entry point is unit-testable without QEMU.
|
||||
"""
|
||||
return vmsafety.validate_writable_drives(disks, test_dir)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SCENARIO_ROLLBACK",
|
||||
"SCENARIO_RECOVERY",
|
||||
"REQUIRED_INPUTS",
|
||||
"EXPECTED_STEP_NAMES",
|
||||
"CommandResult",
|
||||
"SnapshotRecord",
|
||||
"Guest",
|
||||
"BootTransport",
|
||||
"RollbackJourney",
|
||||
"validate_acceptance_disks",
|
||||
"extract_between",
|
||||
"normalize_subvol",
|
||||
"parse_snapshot_history",
|
||||
"newest_committed_coordinated",
|
||||
"parse_rollback_state",
|
||||
"parse_rootflags_subvol",
|
||||
"parse_fstab_subvol",
|
||||
"parse_package_dirs",
|
||||
"parse_contents_files",
|
||||
"parse_uname",
|
||||
"parse_network_addresses",
|
||||
]
|
||||
497
installer/tests/vm/test_rollback_acceptance.py
Normal file
497
installer/tests/vm/test_rollback_acceptance.py
Normal file
|
|
@ -0,0 +1,497 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Mocked-transport tests for the rollback acceptance orchestration.
|
||||
|
||||
These run on any host (no QEMU, no disk), so the journey's ordering, safety and
|
||||
failure semantics are checked in CI rather than only during a live run. Every
|
||||
transport here is a fake: the results are marked ``mocked`` and are explicitly
|
||||
**not** release evidence (the release tool refuses them).
|
||||
|
||||
python3 installer/tests/vm/test_rollback_acceptance.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import rollback_acceptance as ra
|
||||
from rollback_acceptance import (
|
||||
SCENARIO_RECOVERY,
|
||||
SCENARIO_ROLLBACK,
|
||||
CommandResult,
|
||||
RollbackJourney,
|
||||
validate_acceptance_disks,
|
||||
)
|
||||
from vmsafety import FAIL, MOCKED, NOT_RUN, Results, UnsafeDiskError
|
||||
|
||||
BASE_PKGS = [
|
||||
"/var/db/pkg/sys-apps/busybox-1.36.1",
|
||||
"/var/db/pkg/app-shells/bash-5.2_p26",
|
||||
]
|
||||
ADDED_PKG = "/var/db/pkg/app-misc/hello-2.12"
|
||||
HISTORY = """\
|
||||
[[records]]
|
||||
id = "snap-initial"
|
||||
created_unix = 1000
|
||||
operation = "initial known-good"
|
||||
source_subvol = "/"
|
||||
snapshot_subvol = "@-20260101T000000-initial-known-good"
|
||||
var_snapshot = "@var-20260101T000000-initial-known-good"
|
||||
status = "committed"
|
||||
"""
|
||||
ROLLBACK_STATE = """\
|
||||
status = "activated"
|
||||
generation = 1001
|
||||
target_snapshot = "@-20260101T000000-initial-known-good"
|
||||
previous_root = "@"
|
||||
root_subvol = "@restore-20260101T000000"
|
||||
var_subvol = "@var-restore-20260101T000000"
|
||||
started_unix = 1001
|
||||
finished_unix = 1001
|
||||
note = "restore from snap-initial"
|
||||
"""
|
||||
|
||||
|
||||
class FakeGuest:
|
||||
"""A scripted guest. First matching handler wins."""
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self.commands: list[str] = []
|
||||
self.handlers: list = []
|
||||
|
||||
def on(self, needle: str, handler):
|
||||
# A re-registered needle overrides the earlier one, so a test can tweak
|
||||
# one behaviour without disturbing the ordering of distinct needles.
|
||||
self.handlers = [(n, h) for (n, h) in self.handlers if n != needle]
|
||||
self.handlers.append((needle, handler))
|
||||
return self
|
||||
|
||||
def reply(self, needle: str, stdout: str = "", exit_code: int = 0):
|
||||
return self.on(needle, lambda cmd: CommandResult(cmd, stdout=stdout, exit_code=exit_code))
|
||||
|
||||
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
||||
self.commands.append(command)
|
||||
for needle, handler in self.handlers:
|
||||
if needle in command:
|
||||
return handler(command)
|
||||
return CommandResult(command=command, exit_code=0)
|
||||
|
||||
|
||||
class FakeTransport:
|
||||
def __init__(self, installed, recovery):
|
||||
# installed is a list of guests returned by successive boot_installed()
|
||||
# calls; the first is the change/undo boot, the second the restored boot.
|
||||
self._installed = list(installed)
|
||||
self._recovery = recovery
|
||||
self.boots: list[str] = []
|
||||
self.powered_off = 0
|
||||
|
||||
def boot_installed(self):
|
||||
self.boots.append("installed")
|
||||
return self._installed.pop(0)
|
||||
|
||||
def boot_recovery(self):
|
||||
self.boots.append("recovery")
|
||||
return self._recovery
|
||||
|
||||
def poweroff(self):
|
||||
self.powered_off += 1
|
||||
|
||||
|
||||
def make_phase1(state: dict) -> FakeGuest:
|
||||
g = FakeGuest("phase1")
|
||||
|
||||
def write_marker(cmd):
|
||||
match = re.search(r"printf '%s' '([^']*)' >", cmd)
|
||||
if match:
|
||||
state["token"] = match.group(1)
|
||||
return CommandResult(cmd, exit_code=0)
|
||||
|
||||
def read_marker(cmd):
|
||||
return CommandResult(cmd, stdout=state.get("token", ""))
|
||||
|
||||
def break_hostname(cmd):
|
||||
state["broken"] = True
|
||||
return CommandResult(cmd, exit_code=0)
|
||||
|
||||
def read_hostname(cmd):
|
||||
return CommandResult(
|
||||
cmd,
|
||||
stdout=ra.HOSTNAME_SENTINEL + "\n" if state.get("broken") else "phoebus\n",
|
||||
)
|
||||
|
||||
def install(cmd):
|
||||
state["installed"] = True
|
||||
return CommandResult(cmd, exit_code=0)
|
||||
|
||||
def pkg_dirs(cmd):
|
||||
dirs = list(BASE_PKGS)
|
||||
if state.get("installed"):
|
||||
dirs.append(ADDED_PKG)
|
||||
return CommandResult(cmd, stdout="\n".join(dirs) + "\n")
|
||||
|
||||
g.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=HISTORY))
|
||||
g.on("sha256sum", lambda c: CommandResult(c, exit_code=0, stdout="b" * 64 + " /usr/local/bin/sys\n"))
|
||||
g.on("> /home/phoebus/rollback-acceptance/marker", write_marker)
|
||||
g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker)
|
||||
g.on("tee /etc/hostname", break_hostname)
|
||||
g.on("cat /etc/hostname", read_hostname)
|
||||
g.on("sys install", install)
|
||||
g.on("find /var/db/pkg", pkg_dirs)
|
||||
g.on("cat " + ADDED_PKG + "/CONTENTS", lambda c: CommandResult(
|
||||
c, stdout="obj /usr/bin/hello abcdef\nobj /usr/share/hello/README def\n"
|
||||
))
|
||||
g.on("sys undo", lambda c: CommandResult(c, exit_code=0, stdout="rollback activated"))
|
||||
g.on("cat /var/lib/phoebus/rollback-state.toml", lambda c: CommandResult(c, stdout=ROLLBACK_STATE))
|
||||
return g
|
||||
|
||||
|
||||
def make_phase2(state: dict) -> FakeGuest:
|
||||
g = FakeGuest("phase2")
|
||||
|
||||
def read_marker(cmd):
|
||||
return CommandResult(cmd, stdout=state.get("token", ""))
|
||||
|
||||
fstab = (
|
||||
"/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n"
|
||||
"/dev/mapper/luks-root /var btrfs subvol=/@var-restore-20260101T000000 0 0\n"
|
||||
"/dev/mapper/luks-root /home btrfs subvol=/@home 0 0\n"
|
||||
)
|
||||
g.on("cat /etc/hostname", lambda c: CommandResult(c, stdout="phoebus\n"))
|
||||
g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker)
|
||||
g.on("find /var/db/pkg", lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS) + "\n"))
|
||||
g.on("test -e", lambda c: CommandResult(c, exit_code=1))
|
||||
g.on("cat /proc/cmdline", lambda c: CommandResult(
|
||||
c, stdout="rootflags=subvol=@restore-20260101T000000 quiet console=ttyS0\n"
|
||||
))
|
||||
g.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab))
|
||||
g.on("uname -r", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n"))
|
||||
g.on("cat /lib/modules", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n"))
|
||||
g.on("bootctl set-oneshot", lambda c: CommandResult(c, exit_code=0))
|
||||
return g
|
||||
|
||||
|
||||
def make_recovery() -> FakeGuest:
|
||||
g = FakeGuest("recovery")
|
||||
g.on("cat /proc/cmdline", lambda c: CommandResult(
|
||||
c, stdout="rootflags=subvol=@-20260101T000000-initial-known-good "
|
||||
"systemd.unit=rescue.target no-network\n"
|
||||
))
|
||||
g.on("ip -brief addr", lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8 ::1/128\n"))
|
||||
g.on("sys undo status", lambda c: CommandResult(c, exit_code=0, stdout="no active rollback\n"))
|
||||
g.on("sys ask", lambda c: CommandResult(
|
||||
c,
|
||||
exit_code=1,
|
||||
stderr="No model is connected. The deterministic interface still works:\n",
|
||||
))
|
||||
return g
|
||||
|
||||
|
||||
def build_journey(*, state=None, transport=None, **kwargs):
|
||||
state = state if state is not None else {}
|
||||
transport = transport or FakeTransport([make_phase1(state), make_phase2(state)], make_recovery())
|
||||
journey = RollbackJourney(
|
||||
transport,
|
||||
iso_sha256="a" * 64,
|
||||
sys_sha256="b" * 64,
|
||||
mocked=True,
|
||||
**kwargs,
|
||||
)
|
||||
state["token"] = journey.marker_token
|
||||
return journey
|
||||
|
||||
|
||||
def status_of(results: Results, name: str) -> str | None:
|
||||
for check in results.checks:
|
||||
if check.name == name:
|
||||
return check.status
|
||||
return None
|
||||
|
||||
|
||||
class HappyPathTests(unittest.TestCase):
|
||||
def test_full_journey_passes_in_expected_order(self):
|
||||
journey = build_journey()
|
||||
self.assertTrue(journey.run(), "happy path should pass")
|
||||
self.assertEqual(list(journey.executed), list(ra.EXPECTED_STEP_NAMES))
|
||||
self.assertEqual(journey.boots, ["installed", "installed", "recovery"])
|
||||
self.assertEqual(journey.results.exit_code(), 0)
|
||||
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
||||
|
||||
def test_happy_path_has_no_not_run_steps(self):
|
||||
journey = build_journey()
|
||||
journey.run()
|
||||
for stream in (journey.results, journey.recovery_results):
|
||||
self.assertFalse(
|
||||
any(c.status == NOT_RUN for c in stream.checks),
|
||||
"a complete run must not contain NOT RUN steps",
|
||||
)
|
||||
|
||||
def test_mocked_run_is_marked_and_never_live(self):
|
||||
journey = build_journey()
|
||||
journey.run()
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
|
||||
tmp.close()
|
||||
self.addCleanup(os.unlink, tmp.name)
|
||||
data = json.loads(journey.results.write(tmp.name).read_text())
|
||||
self.assertEqual(data["evidence"], MOCKED)
|
||||
self.assertEqual(data["inputs"]["iso"]["sha256"], "a" * 64)
|
||||
self.assertEqual(data["inputs"]["sys"]["sha256"], "b" * 64)
|
||||
self.assertEqual(data["scenario"], SCENARIO_ROLLBACK)
|
||||
|
||||
def test_recovery_scenario_is_written_separately(self):
|
||||
journey = build_journey()
|
||||
journey.run()
|
||||
self.assertEqual(journey.recovery_results.scenario, SCENARIO_RECOVERY)
|
||||
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
||||
|
||||
|
||||
class StopOnFailureTests(unittest.TestCase):
|
||||
def test_undo_failure_stops_and_marks_rest_unrun(self):
|
||||
state = {}
|
||||
phase1 = make_phase1(state)
|
||||
# Override undo to fail.
|
||||
phase1.on("sys undo", lambda c: CommandResult(c, exit_code=1, stderr="boom"))
|
||||
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||
state["token"] = journey.marker_token
|
||||
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "sys-undo-activates"), FAIL)
|
||||
self.assertEqual(status_of(journey.results, "journey-complete"), FAIL)
|
||||
self.assertEqual(status_of(journey.results, "restored-root-var-generations"), NOT_RUN)
|
||||
self.assertEqual(status_of(journey.recovery_results, "recovery-boot-no-network-no-model"), NOT_RUN)
|
||||
# The second installed boot and the recovery boot must not happen.
|
||||
self.assertEqual(transport.boots, ["installed"])
|
||||
self.assertEqual(journey.results.exit_code(), 1)
|
||||
|
||||
def test_timeout_is_a_failure_not_a_pass(self):
|
||||
state = {}
|
||||
phase1 = make_phase1(state)
|
||||
phase1.on("sys install", lambda c: CommandResult(c, timed_out=True))
|
||||
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||
state["token"] = journey.marker_token
|
||||
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "controlled-package-change"), FAIL)
|
||||
self.assertIn("timeout", "".join(c.detail for c in journey.results.checks))
|
||||
self.assertEqual(transport.boots, ["installed"])
|
||||
|
||||
def test_disconnect_is_a_failure(self):
|
||||
state = {}
|
||||
phase1 = make_phase1(state)
|
||||
phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, disconnected=True))
|
||||
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||
state["token"] = journey.marker_token
|
||||
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL)
|
||||
self.assertEqual(journey.results.exit_code(), 1)
|
||||
|
||||
def test_missing_evidence_is_refused_not_treated_as_empty(self):
|
||||
state = {}
|
||||
phase1 = make_phase1(state)
|
||||
phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=""))
|
||||
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||
state["token"] = journey.marker_token
|
||||
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL)
|
||||
|
||||
def test_missing_input_identities_block_the_run(self):
|
||||
# No iso/sys digests: the run must not boot anything.
|
||||
transport = FakeTransport([make_phase1({}), make_phase2({})], make_recovery())
|
||||
journey = RollbackJourney(transport, mocked=True)
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "input-identities"), FAIL)
|
||||
self.assertEqual(status_of(journey.results, "journey-complete"), FAIL)
|
||||
self.assertEqual(transport.boots, [])
|
||||
|
||||
|
||||
class RestorationMismatchTests(unittest.TestCase):
|
||||
def _journey_with_phase2(self, configure):
|
||||
state = {}
|
||||
phase1 = make_phase1(state)
|
||||
phase2 = make_phase2(state)
|
||||
configure(phase2)
|
||||
transport = FakeTransport([phase1, phase2], make_recovery())
|
||||
journey = RollbackJourney(
|
||||
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||
)
|
||||
state["token"] = journey.marker_token
|
||||
return journey
|
||||
|
||||
def test_wrong_restored_generation_fails(self):
|
||||
def configure(phase2):
|
||||
phase2.on("cat /proc/cmdline", lambda c: CommandResult(c, stdout="rootflags=subvol=@\n"))
|
||||
|
||||
journey = self._journey_with_phase2(configure)
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "restored-root-generation"), FAIL)
|
||||
self.assertEqual(journey.results.exit_code(), 1)
|
||||
|
||||
def test_wrong_var_generation_fails(self):
|
||||
def configure(phase2):
|
||||
fstab = "/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n"
|
||||
phase2.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab))
|
||||
|
||||
journey = self._journey_with_phase2(configure)
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "restored-var-generation"), FAIL)
|
||||
|
||||
def test_package_db_mismatch_fails(self):
|
||||
def configure(phase2):
|
||||
phase2.on(
|
||||
"find /var/db/pkg",
|
||||
lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS + [ADDED_PKG]) + "\n"),
|
||||
)
|
||||
|
||||
journey = self._journey_with_phase2(configure)
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "package-db-restored"), FAIL)
|
||||
|
||||
def test_surviving_package_file_fails(self):
|
||||
def configure(phase2):
|
||||
phase2.on("test -e", lambda c: CommandResult(c, exit_code=0))
|
||||
|
||||
journey = self._journey_with_phase2(configure)
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "package-files-restored"), FAIL)
|
||||
|
||||
def test_kernel_without_modules_fails(self):
|
||||
def configure(phase2):
|
||||
phase2.on("cat /lib/modules", lambda c: CommandResult(c, stdout="5.0.0-other\n"))
|
||||
|
||||
journey = self._journey_with_phase2(configure)
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "kernel-modules-compatible"), FAIL)
|
||||
|
||||
def test_lost_home_marker_fails(self):
|
||||
def configure(phase2):
|
||||
phase2.on("cat /home/phoebus/rollback-acceptance/marker", lambda c: CommandResult(c, stdout=""))
|
||||
|
||||
journey = self._journey_with_phase2(configure)
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.results, "home-data-survives"), FAIL)
|
||||
|
||||
def test_recovery_with_network_fails(self):
|
||||
state = {}
|
||||
recovery = make_recovery()
|
||||
recovery.on(
|
||||
"ip -brief addr",
|
||||
lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"),
|
||||
)
|
||||
transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery)
|
||||
journey = RollbackJourney(
|
||||
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||
)
|
||||
state["token"] = journey.marker_token
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.recovery_results, "recovery-no-network"), FAIL)
|
||||
|
||||
def test_recovery_without_no_model_fallback_fails(self):
|
||||
state = {}
|
||||
recovery = make_recovery()
|
||||
recovery.on("sys ask", lambda c: CommandResult(c, exit_code=0, stdout="here is a plan"))
|
||||
transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery)
|
||||
journey = RollbackJourney(
|
||||
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||
)
|
||||
state["token"] = journey.marker_token
|
||||
self.assertFalse(journey.run())
|
||||
self.assertEqual(status_of(journey.recovery_results, "recovery-no-model-fallback"), FAIL)
|
||||
|
||||
|
||||
class RecoveryOnlyTests(unittest.TestCase):
|
||||
def test_recovery_only_passes(self):
|
||||
transport = FakeTransport([], make_recovery())
|
||||
journey = RollbackJourney(
|
||||
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||
)
|
||||
self.assertTrue(journey.run_recovery())
|
||||
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
||||
self.assertEqual(transport.boots, ["recovery"])
|
||||
|
||||
def test_recovery_only_without_inputs_does_not_boot(self):
|
||||
transport = FakeTransport([], make_recovery())
|
||||
journey = RollbackJourney(transport, mocked=True)
|
||||
self.assertFalse(journey.run_recovery())
|
||||
self.assertEqual(transport.boots, [])
|
||||
self.assertEqual(
|
||||
status_of(journey.recovery_results, "recovery-boot-no-network-no-model"),
|
||||
NOT_RUN,
|
||||
)
|
||||
|
||||
|
||||
class SafetyTests(unittest.TestCase):
|
||||
def test_rejects_dev_disk(self):
|
||||
with self.assertRaises(UnsafeDiskError):
|
||||
validate_acceptance_disks(["/dev/sda"], "/var/tmp/phoebus-vm/disks")
|
||||
|
||||
def test_rejects_disk_outside_test_dir(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
outside = Path(tmp).parent / "rollback-outside.qcow2"
|
||||
outside.write_bytes(b"QFI\xfb" + b"\x00" * 64)
|
||||
self.addCleanup(outside.unlink)
|
||||
with self.assertRaises(UnsafeDiskError):
|
||||
validate_acceptance_disks([outside], tmp)
|
||||
|
||||
|
||||
class ParserTests(unittest.TestCase):
|
||||
def test_newest_committed_coordinated_skips_unpaired(self):
|
||||
records = ra.parse_snapshot_history(
|
||||
HISTORY
|
||||
+ """
|
||||
[[records]]
|
||||
id = "snap-varless"
|
||||
created_unix = 2000
|
||||
operation = "risk"
|
||||
source_subvol = "/"
|
||||
snapshot_subvol = "@-20260102T000000-risk"
|
||||
status = "committed"
|
||||
"""
|
||||
)
|
||||
chosen = ra.newest_committed_coordinated(records)
|
||||
self.assertIsNotNone(chosen)
|
||||
self.assertEqual(chosen.id, "snap-initial")
|
||||
|
||||
def test_parse_rootflags_subvol(self):
|
||||
self.assertEqual(
|
||||
ra.parse_rootflags_subvol("rootflags=subvol=@restore-x quiet"), "@restore-x"
|
||||
)
|
||||
self.assertEqual(ra.parse_rootflags_subvol("quiet"), None)
|
||||
|
||||
def test_parse_fstab_subvol(self):
|
||||
text = "/dev/x / btrfs subvol=/@restore-y 0 0\n"
|
||||
self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y")
|
||||
self.assertIsNone(ra.parse_fstab_subvol(text, "/var"))
|
||||
|
||||
def test_parse_fstab_subvol_without_leading_slash(self):
|
||||
text = "/dev/x / btrfs subvol=@restore-y 0 0\n"
|
||||
self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y")
|
||||
|
||||
def test_parse_contents_paths(self):
|
||||
paths = ra.parse_contents_files("obj /usr/bin/hello abc\ndir /usr/share/hello def\n")
|
||||
self.assertEqual(paths, ["/usr/bin/hello", "/usr/share/hello"])
|
||||
|
||||
def test_parse_network_addresses(self):
|
||||
self.assertEqual(ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\n"), ["lo"])
|
||||
self.assertEqual(
|
||||
ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"),
|
||||
["lo", "enp0s2"],
|
||||
)
|
||||
|
||||
def test_unreadable_history_raises(self):
|
||||
with self.assertRaises(ValueError):
|
||||
ra.parse_snapshot_history("not = = toml [[")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
|
|
@ -9,6 +9,7 @@ in CI, not only during a live install:
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
|
|
@ -16,9 +17,12 @@ import tempfile
|
|||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import vmsafety
|
||||
from vmsafety import (
|
||||
ARTIFACT_ENV_NAME,
|
||||
ARTIFACT_ENV_SHA256,
|
||||
BLOCKED,
|
||||
FAIL,
|
||||
PASS,
|
||||
|
|
@ -166,6 +170,53 @@ class ResultsTests(unittest.TestCase):
|
|||
return Path(tmp.name)
|
||||
|
||||
|
||||
class ArtifactBindingTests(unittest.TestCase):
|
||||
"""The release tool binds evidence to a candidate digest; a harness run
|
||||
must record the artifact it verified, or a stale pass could be replayed."""
|
||||
|
||||
def _tmp_results(self):
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
|
||||
tmp.close()
|
||||
self.addCleanup(os.unlink, tmp.name)
|
||||
return Path(tmp.name)
|
||||
|
||||
def test_artifact_from_env_is_written(self):
|
||||
with mock.patch.dict(
|
||||
os.environ,
|
||||
{ARTIFACT_ENV_NAME: "phoebus.iso", ARTIFACT_ENV_SHA256: "abc123"},
|
||||
):
|
||||
results = Results("unit")
|
||||
results.pass_("one")
|
||||
# write() is what records the binding, so exercise it directly.
|
||||
path = results.write(self._tmp_results())
|
||||
data = json.loads(path.read_text())
|
||||
self.assertEqual(data["artifact"], {"name": "phoebus.iso", "sha256": "abc123"})
|
||||
|
||||
def test_artifact_absent_is_null(self):
|
||||
clean = {
|
||||
k: v
|
||||
for k, v in os.environ.items()
|
||||
if k not in (ARTIFACT_ENV_NAME, ARTIFACT_ENV_SHA256)
|
||||
}
|
||||
with mock.patch.dict(os.environ, clean, clear=True):
|
||||
results = Results("unit")
|
||||
results.pass_("one")
|
||||
data = json.loads(results.write(self._tmp_results()).read_text())
|
||||
self.assertIsNone(data["artifact"])
|
||||
|
||||
def test_explicit_binding_is_not_overridden_by_env(self):
|
||||
with mock.patch.dict(
|
||||
os.environ,
|
||||
{ARTIFACT_ENV_NAME: "env.iso", ARTIFACT_ENV_SHA256: "env-digest"},
|
||||
):
|
||||
results = Results("unit")
|
||||
results.bind_artifact("explicit.iso", "explicit-digest")
|
||||
data = json.loads(results.write(self._tmp_results()).read_text())
|
||||
self.assertEqual(
|
||||
data["artifact"], {"name": "explicit.iso", "sha256": "explicit-digest"}
|
||||
)
|
||||
|
||||
|
||||
class FakeQmpServer(threading.Thread):
|
||||
"""A tiny QMP server that records HMP commands and answers them."""
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ host (see ``test_vmsafety.py``).
|
|||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
|
|
@ -43,7 +44,26 @@ BLOCKED = "BLOCKED"
|
|||
SKIPPED = "SKIPPED"
|
||||
NOT_RUN = "NOT RUN"
|
||||
|
||||
_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED)
|
||||
_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED, NOT_RUN)
|
||||
|
||||
# The candidate artifact a harness run is executed against. The release tool
|
||||
# binds a gate report to a specific digest; recording it here is what lets a
|
||||
# later verification step refuse stale evidence rather than trust any pass.
|
||||
ARTIFACT_ENV_NAME = "PHOEBUS_VM_ARTIFACT"
|
||||
ARTIFACT_ENV_SHA256 = "PHOEBUS_VM_ARTIFACT_SHA256"
|
||||
|
||||
# How a result set was produced. Only a `live` run is real release evidence; a
|
||||
# `mocked` run exercises the orchestration on fakes and must never be imported
|
||||
# as a gate pass. The release tool checks this field and refuses anything else.
|
||||
LIVE = "live"
|
||||
MOCKED = "mocked"
|
||||
|
||||
# The additional inputs a run may be bound to. The rollback acceptance journey
|
||||
# pins both the bootable ISO and the `sys` binary it exercised, because the
|
||||
# rollback semantics live in `sys`; a report that names only the ISO cannot say
|
||||
# which broker was tested.
|
||||
INPUT_ISO = "iso"
|
||||
INPUT_SYS = "sys"
|
||||
|
||||
|
||||
class UnsafeDiskError(Exception):
|
||||
|
|
@ -81,10 +101,67 @@ class Results:
|
|||
self.scenario = scenario
|
||||
self.run_id = run_id or new_run_id(scenario)
|
||||
self.checks: list[Check] = []
|
||||
# The candidate artifact this run verifies (name + sha256), if known.
|
||||
self.artifact: dict | None = None
|
||||
# Additional pinned inputs (e.g. the tested ISO and `sys` binary), keyed
|
||||
# by name. `artifact` remains the single release candidate for backwards
|
||||
# compatibility; `inputs` carries the wider identity set.
|
||||
self.inputs: dict[str, dict] = {}
|
||||
# `live` for a real run; `mocked` for a fake-transport unit run. A
|
||||
# mocked run is never valid release evidence.
|
||||
self.evidence_class = LIVE
|
||||
if strict is None:
|
||||
strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true")
|
||||
self.strict = bool(strict)
|
||||
|
||||
# -- candidate artifact binding ---------------------------------------
|
||||
def bind_artifact(self, name: str, sha256: str) -> dict:
|
||||
"""Record the candidate artifact (name + digest) this run verified."""
|
||||
self.artifact = {"name": name, "sha256": sha256}
|
||||
return self.artifact
|
||||
|
||||
def bind_artifact_from_env(self) -> dict | None:
|
||||
"""Bind from ``PHOEBUS_VM_ARTIFACT[_SHA256]`` when both are present.
|
||||
|
||||
The drivers call this through :meth:`write`, so a run always records the
|
||||
digest of the candidate it exercised (when the launcher supplied it).
|
||||
"""
|
||||
name = os.environ.get(ARTIFACT_ENV_NAME)
|
||||
sha256 = os.environ.get(ARTIFACT_ENV_SHA256)
|
||||
if name and sha256:
|
||||
return self.bind_artifact(name, sha256)
|
||||
return self.artifact
|
||||
|
||||
# -- additional pinned inputs -----------------------------------------
|
||||
def mark_mocked(self, reason: str = "") -> None:
|
||||
"""Declare this run a mocked one, so it can never be release evidence."""
|
||||
self.evidence_class = MOCKED
|
||||
if reason:
|
||||
self.inputs.setdefault("mocked_reason", {})["reason"] = reason
|
||||
|
||||
def bind_input(
|
||||
self, name: str, sha256: str, size_bytes: int | None = None
|
||||
) -> dict:
|
||||
"""Pin an input by digest (e.g. the ISO or the `sys` binary)."""
|
||||
entry: dict = {"sha256": sha256}
|
||||
if size_bytes is not None:
|
||||
entry["size_bytes"] = int(size_bytes)
|
||||
self.inputs[name] = entry
|
||||
return entry
|
||||
|
||||
def bind_input_file(self, name: str, path: str | os.PathLike) -> dict:
|
||||
"""Hash a local file and pin it as an input."""
|
||||
path = Path(path)
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return self.bind_input(name, digest.hexdigest(), path.stat().st_size)
|
||||
|
||||
def missing_inputs(self, names) -> list[str]:
|
||||
"""Input names that were required but not bound."""
|
||||
return [name for name in names if name not in self.inputs]
|
||||
|
||||
# -- recording ---------------------------------------------------------
|
||||
def _add(self, name: str, status: str, detail: str) -> Check:
|
||||
if status not in _TERMINAL:
|
||||
|
|
@ -114,6 +191,14 @@ class Results:
|
|||
def skip(self, name: str, detail: str = "") -> Check:
|
||||
return self._add(name, SKIPPED, detail)
|
||||
|
||||
def mark_not_run(self, name: str, detail: str = "") -> Check:
|
||||
"""Record a step that was expected but never reached.
|
||||
|
||||
A stopped journey must not look like a pass, so unexecuted steps are
|
||||
visible rather than simply absent.
|
||||
"""
|
||||
return self._add(name, NOT_RUN, detail)
|
||||
|
||||
# -- outcome -----------------------------------------------------------
|
||||
@property
|
||||
def failed(self) -> bool:
|
||||
|
|
@ -148,12 +233,17 @@ class Results:
|
|||
def write(self, path: str | Path) -> Path:
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if self.artifact is None:
|
||||
self.bind_artifact_from_env()
|
||||
payload = {
|
||||
"scenario": self.scenario,
|
||||
"run_id": self.run_id,
|
||||
"strict": self.strict,
|
||||
"evidence": self.evidence_class,
|
||||
"exit_code": self.exit_code(),
|
||||
"counts": self.counts(),
|
||||
"artifact": self.artifact,
|
||||
"inputs": self.inputs,
|
||||
"checks": [c.to_dict() for c in self.checks],
|
||||
}
|
||||
path.write_text(json.dumps(payload, indent=2) + "\n")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue