Compare commits
2 commits
main
...
codex/roll
| Author | SHA1 | Date | |
|---|---|---|---|
| fc36f637b4 | |||
| 1d37fbc1b8 |
21 changed files with 4157 additions and 184 deletions
|
|
@ -1,5 +1,13 @@
|
||||||
# Forgejo Actions CI. Runs only once an Actions runner is registered with this
|
# Forgejo Actions CI.
|
||||||
# label (see docs/DEVELOPMENT.md). It mirrors the local quality gates.
|
#
|
||||||
|
# This workflow mirrors the local quality gates exactly and is deliberately
|
||||||
|
# fail-closed: a required check that cannot run because a tool is missing fails
|
||||||
|
# the job rather than printing "skipping". A green job therefore means every
|
||||||
|
# listed check actually executed.
|
||||||
|
#
|
||||||
|
# The Rust toolchain is pinned in `installer-rs/rust-toolchain.toml`. The runner
|
||||||
|
# must provide either `rustup` (which installs the pin) or a cargo/rustfmt/clippy
|
||||||
|
# toolchain; the preflight below fails clearly otherwise.
|
||||||
#
|
#
|
||||||
# If your runner uses different labels, change `runs-on` accordingly.
|
# If your runner uses different labels, change `runs-on` accordingly.
|
||||||
name: ci
|
name: ci
|
||||||
|
|
@ -9,32 +17,35 @@ on:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
pull_request:
|
pull_request:
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ci-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
quality-gates:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 60
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Rust format
|
- name: Provision the pinned Rust toolchain
|
||||||
run: |
|
run: |
|
||||||
if command -v cargo >/dev/null 2>&1; then
|
set -euo pipefail
|
||||||
cd installer-rs && cargo fmt --all -- --check
|
if command -v rustup >/dev/null 2>&1; then
|
||||||
|
rustup show active-toolchain
|
||||||
|
rustup component add rustfmt clippy
|
||||||
|
rustup target add x86_64-unknown-linux-gnu
|
||||||
|
elif command -v cargo >/dev/null 2>&1; then
|
||||||
|
cargo --version
|
||||||
|
cargo fmt --version
|
||||||
|
cargo clippy --version
|
||||||
else
|
else
|
||||||
echo "cargo not installed on this runner; skipping"
|
echo "ERROR: no Rust toolchain on this runner." >&2
|
||||||
|
echo "Install rustup (recommended, honours the pinned toolchain) or" >&2
|
||||||
|
echo "provide cargo with the rustfmt and clippy components, then retry." >&2
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Rust clippy
|
# One command for the whole gate set, identical to local development.
|
||||||
run: |
|
- name: Quality gates (fmt, clippy, tests, harness tests, builds)
|
||||||
if command -v cargo >/dev/null 2>&1; then
|
run: make ci
|
||||||
cd installer-rs && cargo clippy --workspace --all-targets
|
|
||||||
else
|
|
||||||
echo "cargo not installed on this runner; skipping"
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Rust test suite
|
|
||||||
run: |
|
|
||||||
if command -v cargo >/dev/null 2>&1; then
|
|
||||||
cd installer-rs && cargo test --workspace
|
|
||||||
else
|
|
||||||
echo "cargo not installed on this runner; skipping"
|
|
||||||
fi
|
|
||||||
|
|
|
||||||
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -38,3 +38,8 @@ credentials
|
||||||
installer/live/build/
|
installer/live/build/
|
||||||
installer/live/*.iso
|
installer/live/*.iso
|
||||||
installer/live/*.iso.sha256
|
installer/live/*.iso.sha256
|
||||||
|
|
||||||
|
# --- Release verification reports and logs (regenerated; not source) --------
|
||||||
|
gate-report*.json
|
||||||
|
gate-logs/
|
||||||
|
*.minisig
|
||||||
|
|
|
||||||
28
Makefile
28
Makefile
|
|
@ -8,7 +8,10 @@
|
||||||
|
|
||||||
SHELL := /bin/bash
|
SHELL := /bin/bash
|
||||||
|
|
||||||
.PHONY: help context doctor test test-rust test-vm opencode-acceptance fmt fmt-check clippy check build static vm hooks commit publish backup forgejo-init public private protect release clean
|
# The release tool produced by `make build`, used by `make release-gates`.
|
||||||
|
RELEASE ?= installer-rs/target/release/phoebus-release
|
||||||
|
|
||||||
|
.PHONY: help context doctor test test-rust test-vm ci opencode-acceptance fmt fmt-check clippy check build static release-gates vm hooks commit publish backup forgejo-init public private protect release clean
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@printf 'Phoebus OS targets:\n'
|
@printf 'Phoebus OS targets:\n'
|
||||||
|
|
@ -17,6 +20,8 @@ help:
|
||||||
@printf ' make test run the Rust and VM-harness test suites\n'
|
@printf ' make test run the Rust and VM-harness test suites\n'
|
||||||
@printf ' make test-rust cargo test --workspace\n'
|
@printf ' make test-rust cargo test --workspace\n'
|
||||||
@printf ' make test-vm python3 installer/tests/vm/test_vmsafety.py\n'
|
@printf ' make test-vm python3 installer/tests/vm/test_vmsafety.py\n'
|
||||||
|
@printf ' make ci every CI quality gate (fmt, clippy, tests, builds)\n'
|
||||||
|
@printf ' make release-gates run the executable release gates (evidence report)\n'
|
||||||
@printf ' make opencode-acceptance live-model acceptance (needs PHOEBUS_OPENCODE_LIVE_*)\n'
|
@printf ' make opencode-acceptance live-model acceptance (needs PHOEBUS_OPENCODE_LIVE_*)\n'
|
||||||
@printf ' make fmt cargo fmt --all\n'
|
@printf ' make fmt cargo fmt --all\n'
|
||||||
@printf ' make check fmt-check + clippy + tests\n'
|
@printf ' make check fmt-check + clippy + tests\n'
|
||||||
|
|
@ -40,10 +45,15 @@ doctor:
|
||||||
test: test-rust test-vm
|
test: test-rust test-vm
|
||||||
|
|
||||||
test-rust:
|
test-rust:
|
||||||
cd installer-rs && cargo test --workspace
|
cd installer-rs && cargo test --workspace --locked
|
||||||
|
|
||||||
test-vm:
|
test-vm:
|
||||||
python3 installer/tests/vm/test_vmsafety.py
|
python3 installer/tests/vm/test_vmsafety.py
|
||||||
|
python3 installer/tests/vm/test_rollback_acceptance.py
|
||||||
|
|
||||||
|
# The complete set of required CI gates. Every target here must actually run;
|
||||||
|
# the workflow fails the job if a prerequisite is missing rather than skipping.
|
||||||
|
ci: fmt-check clippy test-rust test-vm build static
|
||||||
|
|
||||||
# A real-model acceptance run. Fails (BLOCKED) when the live server is not
|
# A real-model acceptance run. Fails (BLOCKED) when the live server is not
|
||||||
# configured, so it can never pass by silently not running.
|
# configured, so it can never pass by silently not running.
|
||||||
|
|
@ -53,7 +63,7 @@ opencode-acceptance:
|
||||||
exit 2; \
|
exit 2; \
|
||||||
fi
|
fi
|
||||||
cd installer-rs && PHOEBUS_OPENCODE_REQUIRE_LIVE=1 \
|
cd installer-rs && PHOEBUS_OPENCODE_REQUIRE_LIVE=1 \
|
||||||
cargo test -p phoebus-executor --test opencode_live -- --nocapture
|
cargo test -p phoebus-executor --test opencode_live --locked -- --nocapture
|
||||||
|
|
||||||
fmt:
|
fmt:
|
||||||
cd installer-rs && cargo fmt --all
|
cd installer-rs && cargo fmt --all
|
||||||
|
|
@ -62,16 +72,22 @@ fmt-check:
|
||||||
cd installer-rs && cargo fmt --all -- --check
|
cd installer-rs && cargo fmt --all -- --check
|
||||||
|
|
||||||
clippy:
|
clippy:
|
||||||
cd installer-rs && cargo clippy --workspace --all-targets
|
cd installer-rs && cargo clippy --workspace --all-targets --locked
|
||||||
|
|
||||||
check: fmt-check clippy test
|
check: fmt-check clippy test
|
||||||
|
|
||||||
build:
|
build:
|
||||||
cd installer-rs && cargo build --release -p phoebus-installer
|
cd installer-rs && cargo build --release -p phoebus-installer --locked
|
||||||
|
|
||||||
static:
|
static:
|
||||||
cd installer-rs && RUSTFLAGS="-C target-feature=+crt-static" \
|
cd installer-rs && RUSTFLAGS="-C target-feature=+crt-static" \
|
||||||
cargo build --release -p phoebus-installer --target x86_64-unknown-linux-gnu
|
cargo build --release -p phoebus-installer --target x86_64-unknown-linux-gnu --locked
|
||||||
|
|
||||||
|
# Run the executable release gates and write an evidence-bound report. Host
|
||||||
|
# gates run here; VM/seat/model gates require their environment and are
|
||||||
|
# recorded as blocked, never silently passed.
|
||||||
|
release-gates: build
|
||||||
|
$(RELEASE) gates --execute --output gate-report.json
|
||||||
|
|
||||||
hooks:
|
hooks:
|
||||||
scripts/setup-hooks.sh
|
scripts/setup-hooks.sh
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,27 @@ Format: Keep a Changelog. Dates are UTC.
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
- **Fail-closed CI with a full gate set.** `.forgejo/workflows/ci.yml` now runs
|
||||||
|
one command, `make ci` (fmt-check, clippy, workspace tests, VM-harness safety
|
||||||
|
tests, release build, static installer build). It provisions the pinned
|
||||||
|
toolchain or fails with a clear message; a missing Cargo/rustfmt/clippy is
|
||||||
|
never reported as a pass. The toolchain is pinned in
|
||||||
|
`installer-rs/rust-toolchain.toml` and dependencies are checked with
|
||||||
|
`--locked`. See ADR 0012.
|
||||||
|
- **Executable, evidence-bound release gates.** `phoebus-release gates` now
|
||||||
|
records `passed`/`failed`/`blocked`/`skipped`/`not_run` (only `passed` is a
|
||||||
|
pass), actually runs host gates, and imports evidence from the existing VM
|
||||||
|
harness `results.json` (`--harness-results`) instead of a competing harness.
|
||||||
|
`--candidate` binds the report to an artifact digest, and the harness records
|
||||||
|
the digest it verified from `PHOEBUS_VM_ARTIFACT[_SHA256]`, so stale evidence
|
||||||
|
is refused. Gates without automation (`no-model-recovery`,
|
||||||
|
`updates-and-rollback`, `desktop-session`) are always `blocked`.
|
||||||
|
- **Publication authorization.** `phoebus-release publish` refuses unless the
|
||||||
|
report is bound to the exact candidate digest, every gate passed, the
|
||||||
|
candidate matches its manifest, and a detached signature verifies. Check
|
||||||
|
failures happen before the destination is touched, and `atomic_publish`
|
||||||
|
restores the previous release if the final rename fails.
|
||||||
|
|
||||||
- **Real OpenCode agent-server adapter.** `phoebus-executor::opencode_client`
|
- **Real OpenCode agent-server adapter.** `phoebus-executor::opencode_client`
|
||||||
implements the OpenCode session/message protocol over the project's curl
|
implements the OpenCode session/message protocol over the project's curl
|
||||||
transport and the `ModelTransport` trait, so the strict proposal parser,
|
transport and the `ModelTransport` trait, so the strict proposal parser,
|
||||||
|
|
@ -34,6 +55,10 @@ Format: Keep a Changelog. Dates are UTC.
|
||||||
selection (`ai.backend`) in canonical configuration.
|
selection (`ai.backend`) in canonical configuration.
|
||||||
|
|
||||||
### Security
|
### Security
|
||||||
|
- **Release evidence cannot be replayed across builds.** A gate report and
|
||||||
|
imported harness evidence are bound to a candidate name and SHA-256; a report
|
||||||
|
for a different digest (or unbound evidence) fails rather than authorizing a
|
||||||
|
publication.
|
||||||
- **The mandatory-AI gate now requires a validated interaction.** Both the GUI
|
- **The mandatory-AI gate now requires a validated interaction.** Both the GUI
|
||||||
bridge and the installer CLI refuse to install when only a provider name is
|
bridge and the installer CLI refuse to install when only a provider name is
|
||||||
configured; a successful, validated model interaction bound to the current
|
configured; a successful, validated model interaction bound to the current
|
||||||
|
|
|
||||||
|
|
@ -230,10 +230,48 @@ prerequisites, and `make test`, `make check`, `make build`, `make static`,
|
||||||
|
|
||||||
## Continuous integration
|
## Continuous integration
|
||||||
|
|
||||||
`.forgejo/workflows/ci.yml` mirrors the local gates (`cargo fmt`, `cargo clippy`,
|
`.forgejo/workflows/ci.yml` runs the same checks as `make ci`:
|
||||||
`cargo test`). It runs only once a Forgejo Actions runner is registered for the
|
|
||||||
repository; see Forgejo's documentation for `act_runner`. Until then, run
|
```sh
|
||||||
`make test` before pushing.
|
make ci # fmt-check, clippy, test-rust, test-vm, build, static
|
||||||
|
```
|
||||||
|
|
||||||
|
The workflow is **fail-closed**: it provisions the pinned toolchain
|
||||||
|
(`installer-rs/rust-toolchain.toml`) or fails with a clear message, so a missing
|
||||||
|
Cargo/rustfmt/clippy is never reported as a pass. Dependencies are checked
|
||||||
|
against the committed `Cargo.lock` (`--locked`).
|
||||||
|
|
||||||
|
It runs only once a Forgejo Actions runner is registered for the repository; see
|
||||||
|
Forgejo's documentation for `act_runner`. Until then, run `make ci` before
|
||||||
|
pushing. A runner that uses different labels needs its `runs-on` value updated.
|
||||||
|
|
||||||
|
## Release verification
|
||||||
|
|
||||||
|
`phoebus-release` writes an evidence-bound gate report and refuses to publish
|
||||||
|
without one (ADR 0012):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Host gates run here; VM/seat/model gates are imported from existing harness
|
||||||
|
# results and are `blocked` without evidence.
|
||||||
|
make release-gates
|
||||||
|
installer-rs/target/release/phoebus-release gates --execute \
|
||||||
|
--candidate /path/phoebus-installer-amd64.iso \
|
||||||
|
--harness-results /var/tmp/phoebus-vm
|
||||||
|
|
||||||
|
# Publication requires: a candidate-bound all-passed report, a matching
|
||||||
|
# manifest, and a verified detached signature.
|
||||||
|
installer-rs/target/release/phoebus-release publish \
|
||||||
|
/path/phoebus-installer-amd64.iso /srv/releases/latest.iso \
|
||||||
|
--gate-report gate-report.json --manifest /path/phoebus-release.json \
|
||||||
|
--signature /path/phoebus-installer-amd64.iso.minisig
|
||||||
|
```
|
||||||
|
|
||||||
|
Gate outcomes are `passed`, `failed`, `blocked`, `skipped` and `not_run`; only
|
||||||
|
`passed` is a pass. A report for one artifact digest cannot authorize another
|
||||||
|
(`--candidate` binds the report; the harness records the digest it verified from
|
||||||
|
`PHOEBUS_VM_ARTIFACT[_SHA256]`). `no-model-recovery`, `updates-and-rollback` and
|
||||||
|
`desktop-session` have no automated harness yet, so they stay explicitly
|
||||||
|
`blocked` until one exists.
|
||||||
|
|
||||||
## Backups
|
## Backups
|
||||||
|
|
||||||
|
|
@ -247,7 +285,6 @@ Restore with `git clone /path/to/phoebus-<stamp>.bundle phoebus_os`.
|
||||||
|
|
||||||
## Quality gates before commit
|
## Quality gates before commit
|
||||||
|
|
||||||
1. `make test` (`cd installer-rs && cargo test --workspace`)
|
1. `make ci` (the complete gate set the CI workflow runs)
|
||||||
2. `cargo fmt --all -- --check` and `cargo clippy --workspace --all-targets`
|
2. Update `docs/CHANGELOG.md` and `docs/STATUS.md` when behavior changes.
|
||||||
3. Update `docs/CHANGELOG.md` and `docs/STATUS.md` when behavior changes.
|
3. Commit (see `docs/GIT-WORKFLOW.md`).
|
||||||
4. Commit (see `docs/GIT-WORKFLOW.md`).
|
|
||||||
|
|
|
||||||
|
|
@ -117,11 +117,10 @@ a staged file is too large, then runs the fast tests.
|
||||||
|
|
||||||
1. scans staged files for secrets (`scripts/check-secrets.sh`);
|
1. scans staged files for secrets (`scripts/check-secrets.sh`);
|
||||||
2. blocks staged `*.iso`, `*.qcow2` and files larger than 5 MiB;
|
2. blocks staged `*.iso`, `*.qcow2` and files larger than 5 MiB;
|
||||||
3. runs the fast Bash test suite.
|
3. optionally runs the Rust suite (set `PHOEBUS_PRECOMMIT_RUST=1` to enable).
|
||||||
|
|
||||||
Bypass for a genuine emergency with `git commit --no-verify`, or set
|
Bypass for a genuine emergency with `git commit --no-verify`, or set
|
||||||
`SKIP_PHOEBUS_HOOKS=1`. Set `PHOEBUS_PRECOMMIT_RUST=1` to also run
|
`SKIP_PHOEBUS_HOOKS=1`.
|
||||||
`cargo test --workspace` in the hook.
|
|
||||||
|
|
||||||
## Repository settings and hardening
|
## Repository settings and hardening
|
||||||
|
|
||||||
|
|
@ -136,13 +135,17 @@ Current state:
|
||||||
`enable DEST` at a synced/remote folder for a genuinely off-box copy, since
|
`enable DEST` at a synced/remote folder for a genuinely off-box copy, since
|
||||||
the only git remote is one VPS.
|
the only git remote is one VPS.
|
||||||
|
|
||||||
### CI runner (optional)
|
### CI runner (recommended)
|
||||||
|
|
||||||
`.forgejo/workflows/ci.yml` mirrors the local gates but only runs once a
|
`.forgejo/workflows/ci.yml` runs `make ci` (fmt, clippy, workspace tests,
|
||||||
Forgejo Actions runner is registered. Install `act_runner` on a machine,
|
VM-harness safety tests, release build, static installer build). It is
|
||||||
|
fail-closed: if the runner lacks a Rust toolchain it fails the job rather than
|
||||||
|
skipping checks, so a green result always means the checks ran. It runs once a
|
||||||
|
Forgejo Actions runner is registered; install `act_runner` on a machine,
|
||||||
register it (Site Administration -> Actions -> Runners, or
|
register it (Site Administration -> Actions -> Runners, or
|
||||||
`act_runner register --instance https://git.phoebus.club --token <TOKEN>`), and
|
`act_runner register --instance https://git.phoebus.club --token <TOKEN>`), and
|
||||||
run it as a service. Until then, run `make test` before pushing.
|
run it with a `rustup`-provisioned toolchain (or cargo + rustfmt + clippy).
|
||||||
|
Until then, run `make ci` before pushing.
|
||||||
|
|
||||||
### Commit signing (optional)
|
### Commit signing (optional)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,36 +3,43 @@
|
||||||
Rolling note between work sessions. **Update this file before you finish**, then
|
Rolling note between work sessions. **Update this file before you finish**, then
|
||||||
commit it. It should be short and current; use `git log` for detail.
|
commit it. It should be short and current; use `git log` for detail.
|
||||||
|
|
||||||
- **Last updated:** this session (2026-09-22), OpenCode adapter.
|
- **Last updated:** this session (2026-09-22), CI and release verification
|
||||||
- **Current focus:** Implemented a real OpenCode agent-server adapter
|
hardening (branch `codex/harden-ci-release-verification`).
|
||||||
(`phoebus-executor::opencode_client`) that speaks the session/message protocol
|
- **Current focus:** Made CI and release verification mean what they claim.
|
||||||
over curl and implements `ModelTransport`, wired into canonical config
|
(1) `.forgejo/workflows/ci.yml` is fail-closed: it provisions the pinned
|
||||||
(`ai.backend = "opencode"`), the GUI ("OpenCode agent" + Test connection),
|
toolchain or fails, and runs one `make ci` (fmt-check, clippy, workspace
|
||||||
the installer CLI (`--ai-backend opencode`, `--opencode-test`) and `sys ask`.
|
tests, VM-harness safety tests, release build, static installer build).
|
||||||
The mandatory-AI gate now requires a validated interaction with invalidated
|
`installer-rs/rust-toolchain.toml` pins stable 1.93.1; all cargo gates use
|
||||||
state. The probe/transport were hardened (loopback/TLS policy, DNS pinning,
|
`--locked`. (2) `phoebus-release` gate statuses are now
|
||||||
status checks, auth), and the GUI harness gained exact-session and
|
`passed`/`failed`/`blocked`/`skipped`/`not_run` (only `passed` is a pass);
|
||||||
frontend-state assertions.
|
host gates execute real commands; VM/seat/model gates import the existing VM
|
||||||
|
harness `results.json` (`--harness-results`). (3) Gate reports and harness
|
||||||
|
evidence are bound to a candidate name+SHA-256 (`--candidate`;
|
||||||
|
`PHOEBUS_VM_ARTIFACT[_SHA256]`), so stale evidence cannot authorize another
|
||||||
|
build. (4) `publish` refuses unless the report is candidate-bound and
|
||||||
|
all-passed, the manifest verifies the artifact and a detached signature
|
||||||
|
verifies; `atomic_publish` restores the previous release on failure.
|
||||||
- **Last verified (this session):**
|
- **Last verified (this session):**
|
||||||
- 336 Rust tests + 16 harness tests green; fmt/clippy clean.
|
- 364 Rust tests + 19 harness tests green; `cargo fmt --all -- --check` and
|
||||||
- Real inference through `opencode 1.18.32` with `deepseek-v4-flash`;
|
`cargo clippy --workspace --all-targets` clean (exit 0); `make ci` exit 0.
|
||||||
`--opencode-test` → `{"phoebus":"ok"}`; the GUI bridge `ai_check` applied 8
|
- Release build and static installer build succeed; the static binary is
|
||||||
reviewed changes; `sys ask` proposed and ran a read-only action.
|
`static-pie linked` (4.99 MB).
|
||||||
- Auth: none → `authentication_required`, wrong → `authentication_rejected`,
|
- New regression tests (20 Rust + 3 Python) cover: absent prerequisites are
|
||||||
correct → `model_catalog` + successful inference. Plain http to a LAN host
|
`blocked`; a failed host gate is `failed` with diagnostics; explicit skips
|
||||||
→ `insecure_transport`, exit 1, no request sent.
|
and `--only` are never passes; manual gates are `blocked`; stale or
|
||||||
- 7 real-HTTP integration tests (status, chunked multibyte, oversize,
|
unbound harness evidence is `failed`; `publish` refuses on not-run gates,
|
||||||
redirect, disconnect, auth) and 2 env-gated live tests.
|
unbound/stale reports, missing manifest, manifest mismatch, missing or
|
||||||
- GUI acceptance in a disposable VM: `gui-driver.py opencode` passed 17/17
|
failing signature, and a failed publication leaves the previous release
|
||||||
(selected OpenCode, entered the endpoint/model through the UI, real model
|
intact.
|
||||||
response + validated proposal); `gui-driver.py gui` passed 25/25.
|
- CLI smoke: `gates` (all `not_run`), `gates --execute --only rust-fmt`
|
||||||
ISO `sha256 4db98db3…`, static binary `sha256 72c2dcd0…`.
|
(executes and reports `passed`), harness-evidence import (`install`
|
||||||
- **Not yet verified this session:** the full graphical **installation** through
|
`passed`), and `publish` refusal on an unbound report.
|
||||||
OpenCode past the AI gate (disk/account/review/ERASE/install/detached boot),
|
- **Not yet verified this session:** no Forgejo Actions runner is registered,
|
||||||
and a separate physical computer. The pinned manifest is `v1.18.31` while the
|
so the workflow is unexecuted. The VM, graphical-seat and live-model gates
|
||||||
tested binary is `1.18.32`. On this server use `deepseek-flash`; the model
|
are `blocked` here (no disposable VM/model in this environment).
|
||||||
must match the server's catalog (the GUI default `deepseek-chat` does not
|
`no-model-recovery`, `updates-and-rollback` and `desktop-session` have no
|
||||||
exist on it).
|
automated harness yet and are always `blocked`, so a full publication is not
|
||||||
|
possible until they are automated (intentional: no false pass).
|
||||||
- **Public repo:** <https://git.phoebus.club/phoebus/phoebus_os>
|
- **Public repo:** <https://git.phoebus.club/phoebus/phoebus_os>
|
||||||
(remote `origin`, SSH port 2222).
|
(remote `origin`, SSH port 2222).
|
||||||
|
|
||||||
|
|
@ -55,11 +62,24 @@ commit it. It should be short and current; use `git log` for detail.
|
||||||
5. **Physical-seat desktop**: run `phoebus-session` on a real VT/seat and check
|
5. **Physical-seat desktop**: run `phoebus-session` on a real VT/seat and check
|
||||||
`phoebus-menu`, `phoebus-lock`, `phoebus-idle`, kitty, XWayland, the polkit
|
`phoebus-menu`, `phoebus-lock`, `phoebus-idle`, kitty, XWayland, the polkit
|
||||||
agent, logout/reap and reboot (`docs/DEVELOPMENT.md`).
|
agent, logout/reap and reboot (`docs/DEVELOPMENT.md`).
|
||||||
6. **Release gates on a CI runner**: `phoebus-release gates --execute` and
|
6. **Register a Forgejo Actions runner** and confirm `make ci` runs green
|
||||||
`publish` once a runner exists; no gate has executed.
|
there; no runner is registered, so `.forgejo/workflows/ci.yml` is
|
||||||
|
unexecuted. `make ci` passes locally, but that is not CI verification.
|
||||||
|
7. **Automate the three blocked release gates**: `no-model-recovery`,
|
||||||
|
`updates-and-rollback` and `desktop-session` have no harness scenario yet,
|
||||||
|
so they are always `blocked` and a full publication is impossible. Add
|
||||||
|
harness modes (or reviewed manual evidence) and re-run
|
||||||
|
`make release-gates`.
|
||||||
|
|
||||||
## Known blockers / risks
|
## Known blockers / risks
|
||||||
|
|
||||||
|
- **No Forgejo Actions runner is registered**, so `.forgejo/workflows/ci.yml`
|
||||||
|
is unexecuted; `make ci` is green locally only. Registering an `act_runner`
|
||||||
|
with a rustup toolchain is the external prerequisite (ADR 0012).
|
||||||
|
- **A full release cannot be authorized yet**: `no-model-recovery`,
|
||||||
|
`updates-and-rollback` and `desktop-session` have no automated harness, so
|
||||||
|
the gate report is never all-passed. This is deliberate until they are
|
||||||
|
automated.
|
||||||
- No live DeepSeek/OpenAI key and no local model are available; the **full GUI
|
- No live DeepSeek/OpenAI key and no local model are available; the **full GUI
|
||||||
install past the AI step** and two fresh-disk graphical installs are NOT
|
install past the AI step** and two fresh-disk graphical installs are NOT
|
||||||
verified (normal installation requires a connected model). The AI step is
|
verified (normal installation requires a connected model). The AI step is
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ more interesting. Status vocabulary: **Done**, **Partial**, **Next**, **Later**.
|
||||||
| 11 | AI privilege and privacy architecture | Partial | typed actions + enforced privacy boundary + guided proposal review + OS-level sandbox (systemd-run hardening) + approval-bound resolved effects; live call/sandbox not VM-verified |
|
| 11 | AI privilege and privacy architecture | Partial | typed actions + enforced privacy boundary + guided proposal review + OS-level sandbox (systemd-run hardening) + approval-bound resolved effects; live call/sandbox not VM-verified |
|
||||||
| 12 | `sys` operations | Partial | history, enable/disable, inspect, install, prepare-updates VM-verified; `undo`, `undo status`, `ask` added (unit-tested, VM gate pending) |
|
| 12 | `sys` operations | Partial | history, enable/disable, inspect, install, prepare-updates VM-verified; `undo`, `undo status`, `ask` added (unit-tested, VM gate pending) |
|
||||||
| 13 | Kernel optimization policy | Later | conservative fragments only; SAFE/OPTIMIZED/AGGRESSIVE selection exists |
|
| 13 | Kernel optimization policy | Later | conservative fragments only; SAFE/OPTIMIZED/AGGRESSIVE selection exists |
|
||||||
| 14 | ISO / release pipeline | Partial | manifest/verify/assemble-plan tooling; release gates, atomic publish and signing helpers added; no CI runner has executed a gate |
|
| 14 | ISO / release pipeline | Partial | manifest/verify/assemble tooling; executable evidence-bound gates + publication authorization (ADR 0012); no CI runner has executed a gate |
|
||||||
| 15 | Recovery environment | Later | recovery BLS entry generated; snapshot-based recovery entry written at install; full recovery image not built |
|
| 15 | Recovery environment | Later | recovery BLS entry generated; snapshot-based recovery entry written at install; full recovery image not built |
|
||||||
| — | Rust TUI | Partial | ratatui UI with plan/ERASE and AI provider selection; not yet the default on all paths |
|
| — | Rust TUI | Partial | ratatui UI with plan/ERASE and AI provider selection; not yet the default on all paths |
|
||||||
| — | Graphical installer (welcome + wizard) | Partial | Quickshell/QML over the typed `--gui-bridge`; live graphical VM verified, including a detached install+boot; live AI call pending (ADR 0010) |
|
| — | Graphical installer (welcome + wizard) | Partial | Quickshell/QML over the typed `--gui-bridge`; live graphical VM verified, including a detached install+boot; live AI call pending (ADR 0010) |
|
||||||
|
|
|
||||||
156
docs/ROLLBACK-ACCEPTANCE.md
Normal file
156
docs/ROLLBACK-ACCEPTANCE.md
Normal file
|
|
@ -0,0 +1,156 @@
|
||||||
|
# Rollback and no-model-recovery acceptance
|
||||||
|
|
||||||
|
This is the execution guide for the **rollback** (`updates-and-rollback`) and
|
||||||
|
**no-model recovery** (`no-model-recovery`) release gates. It describes how to
|
||||||
|
run the acceptance journey in the disposable VM harness, what it asserts, where
|
||||||
|
the evidence lands, and how a failure is diagnosed.
|
||||||
|
|
||||||
|
> **Status: NOT YET RUN.** The harness and its offline tests exist and pass on a
|
||||||
|
> plain host, but the live journey has **not** been executed. Until a real run
|
||||||
|
> succeeds and its `results.json` is imported, both release gates stay
|
||||||
|
> `blocked`. A run made with a fake transport is marked `mocked` and is
|
||||||
|
> **rejected** by the release tool.
|
||||||
|
|
||||||
|
## What the journey proves
|
||||||
|
|
||||||
|
The installer writes a coordinated `@` + `@var` known-good snapshot at install
|
||||||
|
time. The journey then deliberately changes the disk and asks the real
|
||||||
|
`sys undo` to put it back. The ordered steps (from
|
||||||
|
`installer/tests/vm/rollback_acceptance.py`) are:
|
||||||
|
|
||||||
|
1. `sys-binary-identity` — the installed `sys` matches the pinned digest.
|
||||||
|
2. `known-good-coordinated-snapshot` — a committed root snapshot with a paired
|
||||||
|
`@var` snapshot exists.
|
||||||
|
3. `home-data-marker` — a marker is written into `/home` (a separate subvolume).
|
||||||
|
4. `controlled-package-change` — a small package is installed, adding a
|
||||||
|
`/var/db/pkg` entry and files under `/`.
|
||||||
|
5. `deterministic-guest-breakage` — `/etc/hostname` is overwritten with a
|
||||||
|
sentinel.
|
||||||
|
6. `sys-undo-activates` — `sys undo <id>` writes an `activated` rollback state
|
||||||
|
for the next boot.
|
||||||
|
7. `reboot-detached` — the installed disk boots with the installer ISO absent.
|
||||||
|
8. `restored-root-var-generations` — the booted `rootflags`, `/etc/fstab` `/`
|
||||||
|
and `/var` subvolumes match the rollback state.
|
||||||
|
9. `package-files-restored` — the changed package's DB entry and files are gone,
|
||||||
|
and the sentinel hostname is reverted.
|
||||||
|
10. `portage-database-consistent` — `/var/db/pkg` equals its pre-change listing
|
||||||
|
and every entry has a `CONTENTS` file.
|
||||||
|
11. `kernel-modules-compatible` — the running kernel has a module tree in the
|
||||||
|
restored root.
|
||||||
|
12. `home-data-survives` — the `/home` marker is byte-identical.
|
||||||
|
13. `recovery-entry-armed` — `bootctl set-oneshot phoebus-recovery` succeeds.
|
||||||
|
14. `recovery-boot-no-network-no-model` — the recovery entry boots
|
||||||
|
`rescue.target`, no non-loopback interface is up, `sys` status works
|
||||||
|
offline, and `sys ask` reports the deterministic no-model fallback.
|
||||||
|
|
||||||
|
The package changed in step 4 is controlled by `ROLLBACK_PACKAGE` (default
|
||||||
|
`app-misc/hello`); pick a small, signed binary package for a fast run.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
- A **disposable qcow2** already installed by an existing harness run
|
||||||
|
(`serial-driver.py install` or `appliance-driver.py install`). This driver
|
||||||
|
never installs and never attaches the installer ISO.
|
||||||
|
- `/var/tmp/phoebus-vm/disks` (or `PHOEBUS_VM_DISKS`) has the disk. Never point
|
||||||
|
a writable disk at `/dev/*`; the harness refuses it.
|
||||||
|
- `qemu-system-x86_64`, `qemu-img`, OVMF firmware, Python `pexpect`.
|
||||||
|
- The candidate ISO and the `sys` binary under test, as files on the host, so
|
||||||
|
their digests can be pinned.
|
||||||
|
- Network access during the change phase (the package install); the recovery
|
||||||
|
phase uses `-net none`.
|
||||||
|
|
||||||
|
## Running it
|
||||||
|
|
||||||
|
Print the plan without starting a VM (safe on any host):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 installer/tests/vm/rollback-driver.py plan
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the full journey:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \
|
||||||
|
DISK=/var/tmp/phoebus-vm/disks/rollback.qcow2 \
|
||||||
|
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \
|
||||||
|
OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \
|
||||||
|
ISO=/var/tmp/phoebus-vm/phoebus-installer-amd64.iso \
|
||||||
|
PHOEBUS_VM_SYS=/var/tmp/phoebus-vm/sys \
|
||||||
|
PHOEBUS_VM_ARTIFACT=phoebus-installer-amd64.iso \
|
||||||
|
PHOEBUS_VM_ARTIFACT_SHA256="$(sha256sum /var/tmp/phoebus-vm/phoebus-installer-amd64.iso | cut -d' ' -f1)" \
|
||||||
|
PHOEBUS_VM_STRICT=1 \
|
||||||
|
LUKS_PASSPHRASE=phoebus-luks-test USER_PASSWORD=phoebus-user-test \
|
||||||
|
python3 installer/tests/vm/rollback-driver.py run
|
||||||
|
```
|
||||||
|
|
||||||
|
Re-verify only the recovery boot (boots normally once to arm the one-shot, then
|
||||||
|
reboots with no network):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# same environment, then:
|
||||||
|
python3 installer/tests/vm/rollback-driver.py recovery
|
||||||
|
```
|
||||||
|
|
||||||
|
`PHOEBUS_VM_ARTIFACT[_SHA256]` is what binds the evidence to a candidate; the
|
||||||
|
release tool refuses unbound or stale evidence. `PHOEBUS_VM_SYS` is hashed and
|
||||||
|
pinned as the `sys` identity (override with `PHOEBUS_VM_SYS_SHA256`). The guest
|
||||||
|
must report the same digest in step 1.
|
||||||
|
|
||||||
|
## Artifacts
|
||||||
|
|
||||||
|
Under `$PHOEBUS_VM_STATE/<run-id>/` (default `/var/tmp/phoebus-vm/`):
|
||||||
|
|
||||||
|
- `results.json` — the `rollback-acceptance` scenario.
|
||||||
|
- `recovery/results.json` — the `rollback-recovery` scenario (written by `run`).
|
||||||
|
- `serial-boot1.log`, `serial-boot2.log`, `serial-boot3.log` — per-boot serial
|
||||||
|
transcripts.
|
||||||
|
- `OVMF_VARS.fd` — the run's own writable firmware variables.
|
||||||
|
|
||||||
|
`results.json` records `scenario`, `run_id`, `evidence` (`live`), `exit_code`,
|
||||||
|
`counts`, `artifact` (name + ISO digest), `inputs` (`iso` and `sys` digests) and
|
||||||
|
the per-check `checks` list.
|
||||||
|
|
||||||
|
## Feeding the release gates
|
||||||
|
|
||||||
|
Import the run directory when producing a gate report:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
installer-rs/target/release/phoebus-release gates --execute \
|
||||||
|
--candidate /path/phoebus-installer-amd64.iso \
|
||||||
|
--harness-results /var/tmp/phoebus-vm/<run-id> \
|
||||||
|
--output gate-report.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The `updates-and-rollback` gate resolves `rollback-acceptance`; the
|
||||||
|
`no-model-recovery` gate resolves `rollback-recovery`. A gate **fails**
|
||||||
|
(never passes) when the evidence is:
|
||||||
|
|
||||||
|
- `mocked` (a fake-transport unit run),
|
||||||
|
- missing the `iso` or `sys` pinned input,
|
||||||
|
- bound to a different candidate digest (stale),
|
||||||
|
- unbound while a candidate is supplied, or
|
||||||
|
- present but with `exit_code != 0`.
|
||||||
|
|
||||||
|
## Failure diagnosis
|
||||||
|
|
||||||
|
| Symptom | Likely cause |
|
||||||
|
|---|---|
|
||||||
|
| `sys-binary-identity` fails | the installed `sys` is not the build under test; rebuild the ISO from the merged branch and re-install |
|
||||||
|
| `known-good-coordinated-snapshot` fails | the install did not record a paired `@var` snapshot, or `snapshot-history.toml` is unreadable (it is never treated as empty) |
|
||||||
|
| `controlled-package-change` fails/times out | no network, no signed binhost, or the atom needs a long compile; choose another `ROLLBACK_PACKAGE` |
|
||||||
|
| `sys-undo-activates` fails | `sys undo` refused (no committed snapshot, unreadable history) or the broker exited nonzero |
|
||||||
|
| `restored-root-var-generations` fails | `rootflags`/fstab were not rewritten to the restored generation |
|
||||||
|
| `portage-database-consistent` fails | root and `/var` came from different generations |
|
||||||
|
| `kernel-modules-compatible` fails | the selected boot entry does not match the restored `lib/modules` |
|
||||||
|
| `recovery-no-network` fails | QEMU was still given a network device; check `-net none` |
|
||||||
|
| `recovery-no-model-fallback` fails | a provider is configured, or `sys ask` did not fail closed offline |
|
||||||
|
| harness exits 2 with `disposable-disk` | the disk is not a disposable qcow2 inside `PHOEBUS_VM_DISKS` |
|
||||||
|
|
||||||
|
## Integration note
|
||||||
|
|
||||||
|
The branch this work is built on (`codex/rollback-acceptance`) is based on the
|
||||||
|
CI-hardening commit `1d37fbc` and does **not** include the sibling `sys`
|
||||||
|
hardening commit `4679f93`. The installed `sys` used in a live run must come
|
||||||
|
from a build that contains the sys hardening; otherwise step 1
|
||||||
|
(`sys-binary-identity`) will correctly fail. Build the acceptance ISO after the
|
||||||
|
two branches are integrated.
|
||||||
|
|
@ -94,7 +94,7 @@ still not VM-verified.
|
||||||
| Live desktop ISO (Mango + DMS) | yes | guards tested | graphical VM + GUI journey + detached boot | no |
|
| Live desktop ISO (Mango + DMS) | yes | guards tested | graphical VM + GUI journey + detached boot | no |
|
||||||
| Live welcome autostart + duplicate prevention | yes | guards tested | live graphical VM | no |
|
| Live welcome autostart + duplicate prevention | yes | guards tested | live graphical VM | no |
|
||||||
| Welcome primary action keyboard-focus | yes | guards tested | live graphical VM | no |
|
| Welcome primary action keyboard-focus | yes | guards tested | live graphical VM | no |
|
||||||
| VM harness disk safety + structured results | yes | yes (16 tests) | n/a | n/a |
|
| VM harness disk safety + structured results + artifact binding | yes | yes (19 tests) | n/a | n/a |
|
||||||
| Target disk revalidation before partition | yes | yes | no | n/a |
|
| Target disk revalidation before partition | yes | yes | no | n/a |
|
||||||
| Bridge replay/concurrency + input bounds | yes | yes | no | n/a |
|
| Bridge replay/concurrency + input bounds | yes | yes | no | n/a |
|
||||||
| Provider transport endpoint/size hardening | yes | yes | no | n/a |
|
| Provider transport endpoint/size hardening | yes | yes | no | n/a |
|
||||||
|
|
@ -105,7 +105,10 @@ still not VM-verified.
|
||||||
| OpenCode full graphical install | yes | yes | AI gate passed; install not run | n/a |
|
| OpenCode full graphical install | yes | yes | AI gate passed; install not run | n/a |
|
||||||
| Mandatory AI requires a validated interaction | yes | yes | bridge + CLI | n/a |
|
| Mandatory AI requires a validated interaction | yes | yes | bridge + CLI | n/a |
|
||||||
| GURU packaging (scoped keywords) | yes | yes | no (build/install unverified) | no |
|
| GURU packaging (scoped keywords) | yes | yes | no (build/install unverified) | no |
|
||||||
| Release gates + atomic publish | yes | yes | no | n/a |
|
| Fail-closed CI (`make ci`, pinned toolchain) | yes | yes (`make ci` green locally) | no runner registered | n/a |
|
||||||
|
| Executable release gates + evidence-bound publish | yes | yes | no (no VM/model here) | n/a |
|
||||||
|
| Harness evidence import + stale-evidence refusal | yes | yes | no | n/a |
|
||||||
|
| Atomic publish + previous-release preservation | yes | yes | no | n/a |
|
||||||
| `sys` verbs | yes | yes | yes | no |
|
| `sys` verbs | yes | yes | yes | no |
|
||||||
| OpenCode pinned bootstrap | yes | yes | no | n/a |
|
| OpenCode pinned bootstrap | yes | yes | no | n/a |
|
||||||
| Release manifest/verify | yes | yes | no | n/a |
|
| Release manifest/verify | yes | yes | no | n/a |
|
||||||
|
|
@ -160,9 +163,19 @@ still not VM-verified.
|
||||||
the argv level) and fails closed without `systemd-run`; it has not been
|
the argv level) and fails closed without `systemd-run`; it has not been
|
||||||
exercised in a VM. Only typed actions and the privacy boundary were previously
|
exercised in a VM. Only typed actions and the privacy boundary were previously
|
||||||
enforced.
|
enforced.
|
||||||
- **Release automation** now has gates, a report, atomic publish and signing
|
- **Release gates are executable and evidence-bound (ADR 0012).** `make ci`
|
||||||
command helpers, but no gate has been executed by a configured CI runner; the
|
runs fmt/clippy/tests/harness-tests/builds and is fail-closed. `phoebus-release
|
||||||
report records unexecuted gates as `NotRun`.
|
gates --execute` runs host gates and imports VM/seat/model evidence from the
|
||||||
|
existing harness `results.json`; unexecuted, blocked or skipped gates are
|
||||||
|
never passes. `publish` refuses unless the report is bound to the exact
|
||||||
|
candidate digest, every gate passed, the manifest verifies the artifact and a
|
||||||
|
detached signature verifies. **No CI runner is registered**, so the workflow
|
||||||
|
itself is unexecuted; the VM, graphical-seat and live-model gates are
|
||||||
|
**blocked** here (no disposable VM or model in this environment).
|
||||||
|
`no-model-recovery`, `updates-and-rollback` and `desktop-session` have no
|
||||||
|
automated harness yet and are always `blocked`, so a full release cannot be
|
||||||
|
authorized until they are automated (or given reviewed manual evidence). That
|
||||||
|
is intentional: no false pass.
|
||||||
- The desktop transaction compiles heavily because desktop USE does not match
|
- The desktop transaction compiles heavily because desktop USE does not match
|
||||||
the signed binhost (~26 min for the desktop set on 8 vCPUs in the test VM).
|
the signed binhost (~26 min for the desktop set on 8 vCPUs in the test VM).
|
||||||
The official Gentoo binhost is OpenRC/hardened, so a systemd system can only
|
The official Gentoo binhost is OpenRC/hardened, so a systemd system can only
|
||||||
|
|
|
||||||
78
docs/adr/0012-release-verification-and-publication.md
Normal file
78
docs/adr/0012-release-verification-and-publication.md
Normal file
|
|
@ -0,0 +1,78 @@
|
||||||
|
# ADR 0012: Release verification evidence and publication authorization
|
||||||
|
|
||||||
|
- **Status:** Accepted
|
||||||
|
- **Date:** 2026-09-22
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Two failure modes made a passing result meaningless:
|
||||||
|
|
||||||
|
1. **CI skipped required checks.** `.forgejo/workflows/ci.yml` wrapped every
|
||||||
|
Rust step in `if command -v cargo; then ...; else echo "skipping"; fi`. A
|
||||||
|
runner without Cargo reported green while running nothing.
|
||||||
|
2. **Release gates did not execute and were not bound to an artifact.**
|
||||||
|
`phoebus-release gates --execute` ran no command (there were no host gates,
|
||||||
|
and every non-host gate was recorded `not_run`). `publish` accepted any
|
||||||
|
report whose results were all `passed`, without checking that the report
|
||||||
|
described the artifact being published, that the artifact matched its
|
||||||
|
manifest, or that a signature had been verified. A stale report for an old
|
||||||
|
build could therefore authorize a different one.
|
||||||
|
|
||||||
|
The project already has a disposable VM harness (`installer/tests/vm/*.py`)
|
||||||
|
that writes structured `results.json`. A second, competing harness is not
|
||||||
|
wanted.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
1. **CI is fail-closed.** The workflow provisions the pinned toolchain or fails
|
||||||
|
with a clear message; it never treats a missing prerequisite as a pass. The
|
||||||
|
whole required set is one command, `make ci`, so CI and local development run
|
||||||
|
the same checks (fmt, clippy, workspace tests, VM-harness safety tests,
|
||||||
|
release build, static installer build). The toolchain is pinned in
|
||||||
|
`installer-rs/rust-toolchain.toml` and dependencies use the committed
|
||||||
|
`Cargo.lock` (`--locked`).
|
||||||
|
2. **Gate outcomes are distinct.** A gate is `passed`, `failed`, `blocked`,
|
||||||
|
`skipped` or `not_run`; **only `passed` is a pass**. `blocked` means a
|
||||||
|
prerequisite (VM, graphical seat, model, CI runner, or an automated command)
|
||||||
|
is missing; `skipped` means an operator excluded it; `not_run` means
|
||||||
|
execution was not requested.
|
||||||
|
3. **Gates perform their advertised check.** Host gates run their real command
|
||||||
|
through the command layer and record the exit status. VM/seat/model gates
|
||||||
|
name the real harness scenario and are resolved from that harness's
|
||||||
|
`results.json` (reusing the existing harness); with no evidence they are
|
||||||
|
`blocked`. Gates whose automation does not exist yet have an empty command
|
||||||
|
and can only ever be `blocked`.
|
||||||
|
4. **Evidence is bound to the artifact.** `phoebus-release gates --candidate
|
||||||
|
<file>` records the candidate's name and SHA-256 in the report; the harness
|
||||||
|
records the digest it verified from `PHOEBUS_VM_ARTIFACT[_SHA256]`. Imported
|
||||||
|
evidence whose digest differs from the candidate is reported as `failed`
|
||||||
|
("stale evidence"), never passed.
|
||||||
|
5. **Publication is authorized, not assumed.** `phoebus-release publish`
|
||||||
|
refuses unless the report is bound to the exact candidate digest, every gate
|
||||||
|
passed, the candidate matches its manifest entry, and a detached signature
|
||||||
|
verifies. Check failures happen before the destination is touched.
|
||||||
|
`atomic_publish` keeps `latest.iso.prev` and restores it if the final rename
|
||||||
|
fails, so a failed or interrupted publication never removes the working
|
||||||
|
release.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- A green CI job means the listed checks actually ran. There is no "skipping"
|
||||||
|
path for a required check.
|
||||||
|
- `make release-gates` writes a report that reflects reality. Because
|
||||||
|
`no-model-recovery`, `updates-and-rollback` and `desktop-session` have no
|
||||||
|
automated harness yet, they are always `blocked`, so a full release cannot be
|
||||||
|
authorized until those checks are automated (or given reviewed manual
|
||||||
|
evidence). This is deliberate: it is preferable to be unable to publish than
|
||||||
|
to publish on a false pass.
|
||||||
|
- The release report and per-gate logs are diagnostics, not source; they are
|
||||||
|
gitignored, redacted with the environment redactor, and bounded in size.
|
||||||
|
- The remaining external prerequisite is a registered Forgejo Actions runner;
|
||||||
|
until one exists the workflow is unexecuted and is recorded as such.
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted and implemented. The gate engine, harness-evidence import, artifact
|
||||||
|
binding and publication refusals are covered by unit tests; `make ci` passes
|
||||||
|
locally. CI execution and VM/model gates remain unverified here (no runner, no
|
||||||
|
disposable VM or model in this environment).
|
||||||
|
|
@ -17,5 +17,6 @@ and link it here.
|
||||||
| [0009](0009-mango-dankmaterialshell-desktop.md) | Mango + DankMaterialShell desktop option | Accepted |
|
| [0009](0009-mango-dankmaterialshell-desktop.md) | Mango + DankMaterialShell desktop option | Accepted |
|
||||||
| [0010](0010-graphical-installer-frontend.md) | Graphical installer frontend and typed bridge | Accepted |
|
| [0010](0010-graphical-installer-frontend.md) | Graphical installer frontend and typed bridge | Accepted |
|
||||||
| [0011](0011-opencode-connection-boundary.md) | OpenCode connection boundary | Accepted |
|
| [0011](0011-opencode-connection-boundary.md) | OpenCode connection boundary | Accepted |
|
||||||
|
| [0012](0012-release-verification-and-publication.md) | Release verification evidence and publication authorization | Accepted |
|
||||||
|
|
||||||
Format: Context / Decision / Consequences / Status.
|
Format: Context / Decision / Consequences / Status.
|
||||||
|
|
|
||||||
|
|
@ -8,9 +8,11 @@ use std::process::ExitCode;
|
||||||
|
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use phoebus_core::VERSION;
|
use phoebus_core::VERSION;
|
||||||
|
use phoebus_executor::{CommandSpec, ProcessRunner, RealRunner};
|
||||||
use phoebus_installer::release::{
|
use phoebus_installer::release::{
|
||||||
artifact_for, atomic_publish, default_manifest_path, run_gates, sha256_file, sign_command,
|
artifact_for, candidate_ref, default_manifest_path, publish_release, run_gates, sha256_file,
|
||||||
verify_signature_command, GateReport, ReleaseManifest,
|
sign_command, verify_signature_command, GateOptions, GateReport, HarnessEvidenceSet,
|
||||||
|
MinisignVerifier, PublishRequest, ReleaseManifest, SystemGateExecutor,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
|
|
@ -51,22 +53,49 @@ enum Command {
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
output: PathBuf,
|
output: PathBuf,
|
||||||
},
|
},
|
||||||
/// Write the candidate gate report. `--execute` runs host gates; VM,
|
/// Write the candidate gate report.
|
||||||
/// graphical-seat and model gates need their environment and are recorded
|
///
|
||||||
/// as not run otherwise.
|
/// Without `--execute` every gate is recorded as `not_run` and never as a
|
||||||
|
/// pass. With `--execute`, host gates run their real commands; VM, seat and
|
||||||
|
/// model gates are resolved from `--harness-results` when supplied, and are
|
||||||
|
/// otherwise recorded as `blocked` (never silently passed).
|
||||||
Gates {
|
Gates {
|
||||||
#[arg(long, default_value = "gate-report.json")]
|
#[arg(long, default_value = "gate-report.json")]
|
||||||
output: PathBuf,
|
output: PathBuf,
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
execute: bool,
|
execute: bool,
|
||||||
|
/// Bind the report to this candidate artifact's digest.
|
||||||
|
#[arg(long)]
|
||||||
|
candidate: Option<PathBuf>,
|
||||||
|
/// Import VM-harness `results.json` files from this file or directory.
|
||||||
|
#[arg(long)]
|
||||||
|
harness_results: Option<PathBuf>,
|
||||||
|
/// Record a gate as `skipped` (repeatable). A skipped gate is never a
|
||||||
|
/// pass, so this cannot be used to authorize a publication.
|
||||||
|
#[arg(long = "skip")]
|
||||||
|
skip: Vec<String>,
|
||||||
|
/// Run only these gates (repeatable); the rest are `skipped`.
|
||||||
|
#[arg(long = "only")]
|
||||||
|
only: Vec<String>,
|
||||||
|
/// Directory for redacted per-gate failure logs.
|
||||||
|
#[arg(long)]
|
||||||
|
log_dir: Option<PathBuf>,
|
||||||
},
|
},
|
||||||
/// Publish a candidate atomically, refusing unless a passing gate report is
|
/// Publish a candidate atomically. Refuses unless the gate report is bound
|
||||||
/// supplied. The previous release is preserved.
|
/// to the exact candidate digest, every gate passed, the manifest verifies
|
||||||
|
/// the artifact and a detached signature verifies. The previous release is
|
||||||
|
/// preserved.
|
||||||
Publish {
|
Publish {
|
||||||
candidate: PathBuf,
|
candidate: PathBuf,
|
||||||
dest: PathBuf,
|
dest: PathBuf,
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
gate_report: PathBuf,
|
gate_report: PathBuf,
|
||||||
|
/// The manifest the candidate is verified against.
|
||||||
|
#[arg(long)]
|
||||||
|
manifest: Option<PathBuf>,
|
||||||
|
/// The detached signature (`*.minisig`) for the candidate.
|
||||||
|
#[arg(long)]
|
||||||
|
signature: Option<PathBuf>,
|
||||||
},
|
},
|
||||||
/// Print the signing command for an artifact (key outside the repository).
|
/// Print the signing command for an artifact (key outside the repository).
|
||||||
Sign { artifact: PathBuf, key: PathBuf },
|
Sign { artifact: PathBuf, key: PathBuf },
|
||||||
|
|
@ -86,6 +115,20 @@ fn main() -> ExitCode {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The short source commit the tool is running against, for evidence binding.
|
||||||
|
fn git_commit() -> String {
|
||||||
|
let spec = CommandSpec::new("release", "git-rev-parse", "git")
|
||||||
|
.arg("rev-parse")
|
||||||
|
.arg("--short")
|
||||||
|
.arg("HEAD");
|
||||||
|
RealRunner
|
||||||
|
.execute(&spec)
|
||||||
|
.ok()
|
||||||
|
.filter(|output| output.success)
|
||||||
|
.map(|output| output.stdout_lossy().trim().to_string())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
fn run() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
fn run() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
match Cli::parse().command {
|
match Cli::parse().command {
|
||||||
Command::Manifest {
|
Command::Manifest {
|
||||||
|
|
@ -163,24 +206,65 @@ fn run() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
println!("{}", sha256_file(&file)?);
|
println!("{}", sha256_file(&file)?);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
Command::Gates { output, execute } => {
|
Command::Gates {
|
||||||
let report = run_gates(execute);
|
output,
|
||||||
|
execute,
|
||||||
|
candidate,
|
||||||
|
harness_results,
|
||||||
|
skip,
|
||||||
|
only,
|
||||||
|
log_dir,
|
||||||
|
} => {
|
||||||
|
let bound = match &candidate {
|
||||||
|
Some(path) => Some(candidate_ref(path)?),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
let evidence = match &harness_results {
|
||||||
|
Some(path) => Some(HarnessEvidenceSet::from_path(path)?),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
let log_dir = log_dir.or_else(|| {
|
||||||
|
Some(
|
||||||
|
output
|
||||||
|
.parent()
|
||||||
|
.unwrap_or_else(|| std::path::Path::new("."))
|
||||||
|
.join("gate-logs"),
|
||||||
|
)
|
||||||
|
});
|
||||||
|
let options = GateOptions {
|
||||||
|
execute,
|
||||||
|
skip,
|
||||||
|
only,
|
||||||
|
candidate: bound,
|
||||||
|
evidence: evidence.as_ref(),
|
||||||
|
log_dir,
|
||||||
|
git_commit: git_commit(),
|
||||||
|
build_id: std::env::var("PHOEBUS_BUILD_ID").unwrap_or_default(),
|
||||||
|
};
|
||||||
|
let report = run_gates(&SystemGateExecutor, &options);
|
||||||
report.save(&output)?;
|
report.save(&output)?;
|
||||||
|
println!("wrote {} ({})", output.display(), report.summary_line());
|
||||||
|
for result in &report.results {
|
||||||
println!(
|
println!(
|
||||||
"wrote {} ({} passed, {} not run)",
|
" {:<18} {:<8} {}",
|
||||||
output.display(),
|
result.name,
|
||||||
report
|
result.status.as_str(),
|
||||||
.results
|
result.detail
|
||||||
.iter()
|
|
||||||
.filter(|r| r.status == phoebus_installer::release::GateStatus::Passed)
|
|
||||||
.count(),
|
|
||||||
report.not_run()
|
|
||||||
);
|
);
|
||||||
|
}
|
||||||
if !execute {
|
if !execute {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"note: gates were not executed; pass --execute on a configured runner \
|
"note: gates were not executed; pass --execute on a configured runner \
|
||||||
to run them"
|
to run them. An unexecuted gate is never a pass."
|
||||||
);
|
);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
if !report.all_passed() {
|
||||||
|
return Err(format!(
|
||||||
|
"candidate checks did not all pass ({})",
|
||||||
|
report.summary_line()
|
||||||
|
)
|
||||||
|
.into());
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
@ -188,22 +272,22 @@ fn run() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
candidate,
|
candidate,
|
||||||
dest,
|
dest,
|
||||||
gate_report,
|
gate_report,
|
||||||
|
manifest,
|
||||||
|
signature,
|
||||||
} => {
|
} => {
|
||||||
let report = GateReport::load(&gate_report)?;
|
let report = GateReport::load(&gate_report)?;
|
||||||
if !report.all_passed() {
|
let manifest = match &manifest {
|
||||||
return Err(format!(
|
Some(path) => Some(ReleaseManifest::load(path)?),
|
||||||
"refusing to publish: {} gate(s) passed, {} not run \
|
None => None,
|
||||||
(run the gates on a configured runner and supply a passing report)",
|
};
|
||||||
report
|
let request = PublishRequest {
|
||||||
.results
|
candidate: &candidate,
|
||||||
.iter()
|
dest: &dest,
|
||||||
.filter(|r| r.status == phoebus_installer::release::GateStatus::Passed)
|
report: &report,
|
||||||
.count(),
|
manifest: manifest.as_ref(),
|
||||||
report.not_run()
|
signature: signature.as_deref(),
|
||||||
)
|
};
|
||||||
.into());
|
publish_release(&request, &MinisignVerifier)?;
|
||||||
}
|
|
||||||
atomic_publish(&candidate, &dest)?;
|
|
||||||
println!("published {} -> {}", candidate.display(), dest.display());
|
println!("published {} -> {}", candidate.display(), dest.display());
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
10
installer-rs/rust-toolchain.toml
Normal file
10
installer-rs/rust-toolchain.toml
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
# Pinned stable Rust toolchain for reproducible builds and CI.
|
||||||
|
#
|
||||||
|
# This is a stable release, not a nightly: the project's stability policy
|
||||||
|
# forbids release candidates and unstable defaults. Rustup-managed hosts will
|
||||||
|
# use this automatically; the distribution toolchain ignores the file.
|
||||||
|
[toolchain]
|
||||||
|
channel = "1.93.1"
|
||||||
|
profile = "minimal"
|
||||||
|
components = ["rustfmt", "clippy"]
|
||||||
|
targets = ["x86_64-unknown-linux-gnu"]
|
||||||
|
|
@ -8,6 +8,10 @@ touching a physical disk. The target is always a qcow2 file.
|
||||||
- `run-vm.sh` interactive QEMU launcher (manual testing).
|
- `run-vm.sh` interactive QEMU launcher (manual testing).
|
||||||
- `install.sh` guest-side, non-interactive installation (run in the VM).
|
- `install.sh` guest-side, non-interactive installation (run in the VM).
|
||||||
- `serial-driver.py` host-side pexpect driver (`install` and `boot` modes).
|
- `serial-driver.py` host-side pexpect driver (`install` and `boot` modes).
|
||||||
|
- `rollback-driver.py` host-side rollback + no-model-recovery driver (`plan`,
|
||||||
|
`run` and `recovery` modes); see `docs/ROLLBACK-ACCEPTANCE.md`.
|
||||||
|
- `rollback_acceptance.py` the ordered journey logic, testable with fakes.
|
||||||
|
- `vmsafety.py` shared disk-safety, structured-result and QMP helpers.
|
||||||
- `test.conf.example` installer configuration used by the driver.
|
- `test.conf.example` installer configuration used by the driver.
|
||||||
|
|
||||||
## Host prerequisites
|
## Host prerequisites
|
||||||
|
|
|
||||||
419
installer/tests/vm/rollback-driver.py
Normal file
419
installer/tests/vm/rollback-driver.py
Normal file
|
|
@ -0,0 +1,419 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Disposable-VM acceptance driver for rollback and no-model recovery.
|
||||||
|
|
||||||
|
This is the *live* counterpart to ``rollback_acceptance.py``: it boots an already
|
||||||
|
installed qcow2 disk (the installer ISO is **never** attached), drives the
|
||||||
|
serial console, and runs the ordered journey in
|
||||||
|
:mod:`rollback_acceptance`. Its ``results.json`` feeds the
|
||||||
|
``updates-and-rollback`` and ``no-model-recovery`` release gates.
|
||||||
|
|
||||||
|
Modes (the entry point is explicit; nothing runs by accident):
|
||||||
|
|
||||||
|
plan print the steps, the pinned input identities and the QEMU safety
|
||||||
|
checks, then exit 0 without starting a VM. Safe on any host.
|
||||||
|
run the full journey: snapshot, package change, breakage, `sys undo`,
|
||||||
|
detached reboot, restoration checks, then a recovery boot with
|
||||||
|
networking disabled, and write both scenarios' evidence.
|
||||||
|
recovery re-verify only the no-model recovery boot. Boots normally once to
|
||||||
|
arm the recovery entry, then reboots with no network. Writes the
|
||||||
|
``rollback-recovery`` evidence.
|
||||||
|
|
||||||
|
Safety: every writable disk is validated by :mod:`vmsafety` before QEMU starts.
|
||||||
|
It must be a qcow2 file inside ``PHOEBUS_VM_DISKS``; ``/dev/*`` paths, block and
|
||||||
|
character devices, raw images, escaping symlinks and outside backing chains are
|
||||||
|
refused. The installer ISO is deliberately not a boot input here, and the
|
||||||
|
recovery boot uses ``-net none``.
|
||||||
|
|
||||||
|
Environment (all paths must be local; no secrets on the command line):
|
||||||
|
|
||||||
|
OVMF_CODE, OVMF_VARS UEFI firmware and variable template (required)
|
||||||
|
DISK (or DISKS) disposable qcow2 installed disk(s)
|
||||||
|
PHOEBUS_VM_DISKS directory the disks must live in
|
||||||
|
ISO installer ISO whose digest is pinned (required)
|
||||||
|
PHOEBUS_VM_SYS host path to the `sys` binary under test
|
||||||
|
PHOEBUS_VM_SYS_SHA256 optional expected digest (otherwise computed)
|
||||||
|
PHOEBUS_VM_ARTIFACT[_SHA256] release tool candidate binding (optional here)
|
||||||
|
LUKS_PASSPHRASE, USER_PASSWORD, USERNAME
|
||||||
|
SYS_BIN guest `sys` path (default /usr/local/bin/sys)
|
||||||
|
ROLLBACK_PACKAGE controlled package change (default app-misc/hello)
|
||||||
|
QEMU_BIN, VM_MEM, VM_SMP, PHOEBUS_VM_STATE, PHOEBUS_VM_STRICT
|
||||||
|
|
||||||
|
Example (see docs/ROLLBACK-ACCEPTANCE.md):
|
||||||
|
|
||||||
|
PHOEBUS_VM_DISKS=/var/tmp/phoebus-vm/disks \\
|
||||||
|
DISK=/var/tmp/phoebus-vm/disks/rollback.qcow2 \\
|
||||||
|
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.fd \\
|
||||||
|
OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.fd \\
|
||||||
|
ISO=/path/phoebus-installer-amd64.iso \\
|
||||||
|
PHOEBUS_VM_SYS=/path/sys \\
|
||||||
|
python3 installer/tests/vm/rollback-driver.py run
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pexpect
|
||||||
|
|
||||||
|
import vmsafety
|
||||||
|
from rollback_acceptance import (
|
||||||
|
EXPECTED_STEP_NAMES,
|
||||||
|
REQUIRED_INPUTS,
|
||||||
|
RC_SENTINEL,
|
||||||
|
SCENARIO_RECOVERY,
|
||||||
|
SCENARIO_ROLLBACK,
|
||||||
|
CommandResult,
|
||||||
|
RollbackJourney,
|
||||||
|
validate_acceptance_disks,
|
||||||
|
)
|
||||||
|
from vmsafety import INPUT_ISO, INPUT_SYS, Results
|
||||||
|
|
||||||
|
QEMU = os.environ.get("QEMU_BIN", "qemu-system-x86_64")
|
||||||
|
LIBDIR = os.environ.get("QEMU_LIBDIR", "")
|
||||||
|
DATADIR = os.environ.get("QEMU_DATADIR", "")
|
||||||
|
OVMF_CODE = os.environ.get("OVMF_CODE", "")
|
||||||
|
OVMF_VARS_TEMPLATE = os.environ.get("OVMF_VARS", "")
|
||||||
|
DISKS = os.environ.get("DISKS") or os.environ.get("DISK", "")
|
||||||
|
ISO = os.environ.get("ISO", "")
|
||||||
|
SYS_HOST = os.environ.get("PHOEBUS_VM_SYS", "")
|
||||||
|
SYS_BIN = os.environ.get("SYS_BIN", "/usr/local/bin/sys")
|
||||||
|
PACKAGE = os.environ.get("ROLLBACK_PACKAGE", "app-misc/hello")
|
||||||
|
HOME = os.environ.get("ROLLBACK_HOME", "/home/phoebus")
|
||||||
|
LUKS_PW = os.environ.get("LUKS_PASSPHRASE", "phoebus-luks-test")
|
||||||
|
USER_PW = os.environ.get("USER_PASSWORD", "phoebus-user-test")
|
||||||
|
USER = os.environ.get("USERNAME", "phoebus")
|
||||||
|
MEM = os.environ.get("VM_MEM", "8192")
|
||||||
|
SMP = os.environ.get("VM_SMP", "8")
|
||||||
|
BOOT_TIMEOUT = int(os.environ.get("BOOT_TIMEOUT", "600"))
|
||||||
|
LOGIN_TIMEOUT = int(os.environ.get("LOGIN_TIMEOUT", "300"))
|
||||||
|
PROMPT = os.environ.get("GUEST_PROMPT", r"[$#] $")
|
||||||
|
|
||||||
|
|
||||||
|
def disposable_dir() -> Path:
|
||||||
|
base = os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm")
|
||||||
|
return Path(os.environ.get("PHOEBUS_VM_DISKS", base + "/disks"))
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: str | os.PathLike) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with open(path, "rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def strip_echo(text: str, command: str) -> str:
|
||||||
|
"""Drop the console echo of a typed command and the rc sentinel line."""
|
||||||
|
lines = text.splitlines()
|
||||||
|
if lines and command and command[:40] in lines[0]:
|
||||||
|
lines = lines[1:]
|
||||||
|
return "\n".join(line for line in lines if RC_SENTINEL not in line)
|
||||||
|
|
||||||
|
|
||||||
|
class PexpectGuest:
|
||||||
|
"""A :class:`rollback_acceptance.Guest` backed by a serial console."""
|
||||||
|
|
||||||
|
def __init__(self, child, log, user_pw: str):
|
||||||
|
self.child = child
|
||||||
|
self.log = log
|
||||||
|
self.user_pw = user_pw
|
||||||
|
|
||||||
|
def _ensure_sudo(self) -> None:
|
||||||
|
"""Refresh the sudo timestamp, feeding the password off the log.
|
||||||
|
|
||||||
|
``sudo -n true`` succeeds while the timestamp is valid; otherwise
|
||||||
|
``sudo -v`` prompts and the password is sent with the pexpect logfile
|
||||||
|
detached, so it never reaches the serial transcript.
|
||||||
|
"""
|
||||||
|
self.child.sendline("sudo -n true 2>/dev/null || sudo -v")
|
||||||
|
idx = self.child.expect(
|
||||||
|
[r"\[sudo\] password for [^:]+:", PROMPT, pexpect.TIMEOUT], timeout=30
|
||||||
|
)
|
||||||
|
if idx == 0:
|
||||||
|
saved = self.child.logfile
|
||||||
|
self.child.logfile = None
|
||||||
|
try:
|
||||||
|
self.child.sendline(self.user_pw)
|
||||||
|
self.child.expect(PROMPT, timeout=30)
|
||||||
|
finally:
|
||||||
|
self.child.logfile = saved
|
||||||
|
elif idx == 1:
|
||||||
|
return
|
||||||
|
else: # pragma: no cover - console fault
|
||||||
|
raise RuntimeError("sudo timestamp could not be refreshed")
|
||||||
|
|
||||||
|
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
||||||
|
try:
|
||||||
|
if "sudo" in command:
|
||||||
|
self._ensure_sudo()
|
||||||
|
end = f"echo {RC_SENTINEL}=$(printf '%s' $?)"
|
||||||
|
self.child.sendline(f"{command}; {end}")
|
||||||
|
idx = self.child.expect(
|
||||||
|
[rf"{RC_SENTINEL}=([0-9]+)", pexpect.TIMEOUT, pexpect.EOF],
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
except (pexpect.TIMEOUT, pexpect.EOF):
|
||||||
|
return CommandResult(command=command, disconnected=True, stdout="")
|
||||||
|
if idx == 0:
|
||||||
|
return CommandResult(
|
||||||
|
command=command,
|
||||||
|
stdout=strip_echo(self.child.before or "", command),
|
||||||
|
exit_code=int(self.child.match.group(1)),
|
||||||
|
)
|
||||||
|
if idx == 1:
|
||||||
|
return CommandResult(
|
||||||
|
command=command, timed_out=True, stdout=self.child.before or ""
|
||||||
|
)
|
||||||
|
return CommandResult(
|
||||||
|
command=command, disconnected=True, stdout=self.child.before or ""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PexpectTransport:
|
||||||
|
"""Boots the installed disk; the installer ISO is never attached."""
|
||||||
|
|
||||||
|
def __init__(self, disks, run_paths):
|
||||||
|
self.disks = disks
|
||||||
|
self.paths = run_paths
|
||||||
|
self.child = None
|
||||||
|
self.log = None
|
||||||
|
self._boot = 0
|
||||||
|
|
||||||
|
def _args(self, *, network: bool) -> list[str]:
|
||||||
|
env = dict(os.environ)
|
||||||
|
if LIBDIR:
|
||||||
|
env["LD_LIBRARY_PATH"] = LIBDIR
|
||||||
|
args = ["-machine", "q35,accel=kvm", "-cpu", "host", "-smp", SMP, "-m", MEM]
|
||||||
|
if DATADIR:
|
||||||
|
args += ["-L", DATADIR]
|
||||||
|
args += [
|
||||||
|
"-drive", f"if=pflash,format=raw,readonly=on,file={OVMF_CODE}",
|
||||||
|
"-drive", f"if=pflash,format=raw,file={self.paths.ovmf_vars}",
|
||||||
|
]
|
||||||
|
for disk in self.disks:
|
||||||
|
args += ["-drive", f"file={disk},if=virtio,format=qcow2"]
|
||||||
|
if network:
|
||||||
|
args += ["-netdev", "user,id=n0",
|
||||||
|
"-device", "virtio-net-pci,netdev=n0,romfile="]
|
||||||
|
else:
|
||||||
|
# Recovery must work with networking disabled.
|
||||||
|
args += ["-net", "none"]
|
||||||
|
args += [
|
||||||
|
"-device", "virtio-rng-pci", "-vga", "std", "-display", "none",
|
||||||
|
"-serial", "stdio", "-monitor", "none", "-no-reboot",
|
||||||
|
]
|
||||||
|
return args
|
||||||
|
|
||||||
|
def _spawn(self, *, network: bool) -> None:
|
||||||
|
self._boot += 1
|
||||||
|
log_path = self.paths.directory / f"serial-boot{self._boot}.log"
|
||||||
|
self.log = open(log_path, "w")
|
||||||
|
env = dict(os.environ)
|
||||||
|
if LIBDIR:
|
||||||
|
env["LD_LIBRARY_PATH"] = LIBDIR
|
||||||
|
self.child = pexpect.spawn(
|
||||||
|
QEMU,
|
||||||
|
self._args(network=network),
|
||||||
|
env=env,
|
||||||
|
timeout=BOOT_TIMEOUT,
|
||||||
|
encoding="utf-8",
|
||||||
|
codec_errors="replace",
|
||||||
|
maxread=65536,
|
||||||
|
)
|
||||||
|
self.child.delaybeforesend = 0.4
|
||||||
|
self.child.logfile = self.log
|
||||||
|
|
||||||
|
def _unlock_and_login(self) -> None:
|
||||||
|
assert self.child is not None
|
||||||
|
self.child.expect([r"[Pp]assphrase", r"login:"], timeout=BOOT_TIMEOUT)
|
||||||
|
if "assphrase" in (self.child.after or "").lower():
|
||||||
|
self.child.sendline(LUKS_PW)
|
||||||
|
logged_in = False
|
||||||
|
for _ in range(5):
|
||||||
|
self.child.expect(fr"{re.escape(USER)} login:", timeout=LOGIN_TIMEOUT)
|
||||||
|
self.child.sendline(USER)
|
||||||
|
idx = self.child.expect(
|
||||||
|
[r"[Pp]assword:", fr"{re.escape(USER)} login:", r"Login incorrect"],
|
||||||
|
timeout=60,
|
||||||
|
)
|
||||||
|
if idx == 0:
|
||||||
|
self.child.sendline(USER_PW)
|
||||||
|
if self.child.expect([PROMPT, r"Login incorrect", fr"{re.escape(USER)} login:"],
|
||||||
|
timeout=120) == 0:
|
||||||
|
logged_in = True
|
||||||
|
break
|
||||||
|
if not logged_in:
|
||||||
|
raise RuntimeError("could not log in on the installed system")
|
||||||
|
|
||||||
|
def boot_installed(self):
|
||||||
|
self._spawn(network=True)
|
||||||
|
self._unlock_and_login()
|
||||||
|
return PexpectGuest(self.child, self.log, USER_PW)
|
||||||
|
|
||||||
|
def boot_recovery(self):
|
||||||
|
self._spawn(network=False)
|
||||||
|
self._unlock_and_login()
|
||||||
|
return PexpectGuest(self.child, self.log, USER_PW)
|
||||||
|
|
||||||
|
def poweroff(self) -> None:
|
||||||
|
if self.child is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
PexpectGuest(self.child, self.log, USER_PW).run("sudo poweroff -f", timeout=60)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
self.child.close(force=True)
|
||||||
|
finally:
|
||||||
|
if self.log:
|
||||||
|
self.log.close()
|
||||||
|
self.child = None
|
||||||
|
self.log = None
|
||||||
|
|
||||||
|
|
||||||
|
def pin_inputs(results: Results, *, iso: str, sys_host: str) -> dict:
|
||||||
|
"""Bind the ISO and `sys` identities the run exercised."""
|
||||||
|
bound = {}
|
||||||
|
if iso:
|
||||||
|
results.bind_input_file(INPUT_ISO, iso)
|
||||||
|
bound[INPUT_ISO] = results.inputs[INPUT_ISO]
|
||||||
|
if sys_host:
|
||||||
|
expected = os.environ.get("PHOEBUS_VM_SYS_SHA256")
|
||||||
|
digest = expected or sha256_file(sys_host)
|
||||||
|
results.bind_input(INPUT_SYS, digest)
|
||||||
|
bound[INPUT_SYS] = results.inputs[INPUT_SYS]
|
||||||
|
results.bind_artifact_from_env()
|
||||||
|
return bound
|
||||||
|
|
||||||
|
|
||||||
|
def plan_text() -> str:
|
||||||
|
lines = [
|
||||||
|
"rollback acceptance journey (no VM started)",
|
||||||
|
"scenario: " + SCENARIO_ROLLBACK,
|
||||||
|
"recovery: " + SCENARIO_RECOVERY,
|
||||||
|
"steps:",
|
||||||
|
]
|
||||||
|
lines += [f" {i + 1:2d}. {name}" for i, name in enumerate(EXPECTED_STEP_NAMES)]
|
||||||
|
for scenario, required in REQUIRED_INPUTS.items():
|
||||||
|
lines.append(f"required inputs for {scenario}: {', '.join(required)}")
|
||||||
|
lines.append("safety: every DISK must be a qcow2 inside PHOEBUS_VM_DISKS;")
|
||||||
|
lines.append(" the installer ISO is never attached; recovery uses -net none.")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def build_results(scenario: str, mocked: bool = False) -> Results:
|
||||||
|
results = Results(scenario)
|
||||||
|
if mocked:
|
||||||
|
results.mark_mocked("fake transport")
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def run_full(disks, run_paths, sys_digest) -> int:
|
||||||
|
rollback = build_results(SCENARIO_ROLLBACK)
|
||||||
|
recovery = build_results(SCENARIO_RECOVERY)
|
||||||
|
pin_inputs(rollback, iso=ISO, sys_host=SYS_HOST)
|
||||||
|
pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST)
|
||||||
|
transport = PexpectTransport(disks, run_paths)
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport,
|
||||||
|
results=rollback,
|
||||||
|
recovery_results=recovery,
|
||||||
|
home=HOME,
|
||||||
|
sys_bin=SYS_BIN,
|
||||||
|
package=PACKAGE,
|
||||||
|
iso_sha256=os.environ.get("PHOEBUS_VM_ARTIFACT_SHA256") or (rollback.inputs.get(INPUT_ISO) or {}).get("sha256"),
|
||||||
|
sys_sha256=sys_digest,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
journey.run()
|
||||||
|
finally:
|
||||||
|
transport.poweroff()
|
||||||
|
rollback.write(run_paths.results_json)
|
||||||
|
# The release tool imports any results.json under a directory tree, so
|
||||||
|
# the recovery scenario lives beside the rollback run.
|
||||||
|
recovery_dir = run_paths.directory / "recovery"
|
||||||
|
recovery_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
recovery.write(recovery_dir / "results.json")
|
||||||
|
print(rollback.summary())
|
||||||
|
print(recovery.summary())
|
||||||
|
return 1 if (rollback.exit_code() or recovery.exit_code()) else 0
|
||||||
|
|
||||||
|
|
||||||
|
def run_recovery_only(disks, run_paths, sys_digest) -> int:
|
||||||
|
recovery = build_results(SCENARIO_RECOVERY)
|
||||||
|
pin_inputs(recovery, iso=ISO, sys_host=SYS_HOST)
|
||||||
|
# Arm the one-shot recovery entry on a normal boot first.
|
||||||
|
transport = PexpectTransport(disks, run_paths)
|
||||||
|
try:
|
||||||
|
guest = transport.boot_installed()
|
||||||
|
guest.run("sudo bootctl set-oneshot phoebus-recovery", timeout=120)
|
||||||
|
finally:
|
||||||
|
transport.poweroff()
|
||||||
|
transport = PexpectTransport(disks, run_paths)
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport,
|
||||||
|
results=build_results(SCENARIO_ROLLBACK),
|
||||||
|
recovery_results=recovery,
|
||||||
|
home=HOME,
|
||||||
|
sys_bin=SYS_BIN,
|
||||||
|
package=PACKAGE,
|
||||||
|
sys_sha256=sys_digest,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
journey.run_recovery()
|
||||||
|
finally:
|
||||||
|
transport.poweroff()
|
||||||
|
recovery.write(run_paths.results_json)
|
||||||
|
print(recovery.summary())
|
||||||
|
return recovery.exit_code()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
mode = sys.argv[1] if len(sys.argv) > 1 else ""
|
||||||
|
if mode == "plan":
|
||||||
|
print(plan_text())
|
||||||
|
return 0
|
||||||
|
if mode not in ("run", "recovery"):
|
||||||
|
print(__doc__)
|
||||||
|
return 2
|
||||||
|
if not OVMF_CODE or not OVMF_VARS_TEMPLATE:
|
||||||
|
print("harness error: OVMF_CODE and OVMF_VARS are required", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
if not ISO:
|
||||||
|
print("harness error: ISO is required to pin the candidate identity", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
if not SYS_HOST or not Path(SYS_HOST).is_file():
|
||||||
|
print(
|
||||||
|
"harness error: PHOEBUS_VM_SYS must point at the `sys` binary under test",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
scenario = SCENARIO_ROLLBACK if mode == "run" else SCENARIO_RECOVERY
|
||||||
|
run_paths = vmsafety.make_run_paths(
|
||||||
|
f"rollback-{mode}",
|
||||||
|
base=os.environ.get("PHOEBUS_VM_STATE", "/var/tmp/phoebus-vm"),
|
||||||
|
ovmf_vars_template=OVMF_VARS_TEMPLATE,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
disks = validate_acceptance_disks(DISKS.split(","), disposable_dir())
|
||||||
|
except vmsafety.UnsafeDiskError as exc:
|
||||||
|
results = build_results(scenario)
|
||||||
|
results.blocked("disposable-disk", str(exc))
|
||||||
|
results.write(run_paths.results_json)
|
||||||
|
print(f"harness error: {exc}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
sys_digest = os.environ.get("PHOEBUS_VM_SYS_SHA256") or sha256_file(SYS_HOST)
|
||||||
|
if mode == "run":
|
||||||
|
return run_full(disks, run_paths, sys_digest)
|
||||||
|
return run_recovery_only(disks, run_paths, sys_digest)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
852
installer/tests/vm/rollback_acceptance.py
Normal file
852
installer/tests/vm/rollback_acceptance.py
Normal file
|
|
@ -0,0 +1,852 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Offline orchestration for the rollback + no-model-recovery acceptance run.
|
||||||
|
|
||||||
|
This module holds the *logic* of the journey: the ordered steps, the guest
|
||||||
|
commands they issue, and the assertions that decide PASS/FAIL. It deliberately
|
||||||
|
imports neither pexpect nor QEMU, so the whole journey can be exercised on a
|
||||||
|
plain host against a fake transport (see ``test_rollback_acceptance.py``).
|
||||||
|
|
||||||
|
The real VM driver (``rollback-driver.py``) supplies a pexpect-backed transport
|
||||||
|
implementing :class:`BootTransport` and a :class:`Guest` that runs commands over
|
||||||
|
the serial console. The release gates import the ``results.json`` these runs
|
||||||
|
write; a run made with a fake transport is marked ``mocked`` and can never be a
|
||||||
|
gate pass.
|
||||||
|
|
||||||
|
Journey shape (the installer already created a coordinated ``@`` + ``@var``
|
||||||
|
known-good snapshot at install time):
|
||||||
|
|
||||||
|
boot installed (ISO detached)
|
||||||
|
1. identify the newest committed coordinated snapshot
|
||||||
|
2. write a marker into /home (a subvolume rollback must not touch)
|
||||||
|
3. apply a controlled package change (touches / and /var/db/pkg)
|
||||||
|
4. introduce deterministic guest-only breakage in /
|
||||||
|
5. run the real `sys undo` (restores root + var as the next boot root)
|
||||||
|
reboot, boot installed (ISO detached)
|
||||||
|
6. restored root/var generations match the rollback state
|
||||||
|
7. the changed package and its files are gone
|
||||||
|
8. /var/db/pkg matches / (consistent package database)
|
||||||
|
9. the booted kernel has matching modules in the restored root
|
||||||
|
10. /home data survived
|
||||||
|
arm the recovery entry, reboot, boot recovery (no network, no model)
|
||||||
|
11. recovery entry boots, no network, and `sys` still works with no model
|
||||||
|
|
||||||
|
Nothing here decides that a *live* run passed: only a real transport on a
|
||||||
|
disposable qcow2 disk can produce release evidence. The mocked run exists to
|
||||||
|
prove the orchestration itself.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import dataclasses
|
||||||
|
import re
|
||||||
|
import tomllib
|
||||||
|
from typing import Callable, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
import vmsafety
|
||||||
|
from vmsafety import INPUT_ISO, INPUT_SYS, Results
|
||||||
|
|
||||||
|
# Scenario names. Each is a release-gate evidence key.
|
||||||
|
SCENARIO_ROLLBACK = "rollback-acceptance"
|
||||||
|
SCENARIO_RECOVERY = "rollback-recovery"
|
||||||
|
|
||||||
|
# The input identities a gate must see before it may pass. The rollback
|
||||||
|
# semantics live in `sys`, so naming only the ISO is incomplete evidence.
|
||||||
|
REQUIRED_INPUTS: dict[str, tuple[str, ...]] = {
|
||||||
|
SCENARIO_ROLLBACK: (INPUT_ISO, INPUT_SYS),
|
||||||
|
SCENARIO_RECOVERY: (INPUT_ISO, INPUT_SYS),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Stable guest paths and sentinels.
|
||||||
|
SNAPSHOT_HISTORY = "/var/lib/phoebus/snapshot-history.toml"
|
||||||
|
ROLLBACK_STATE = "/var/lib/phoebus/rollback-state.toml"
|
||||||
|
PKG_DB = "/var/db/pkg"
|
||||||
|
HOSTNAME_FILE = "/etc/hostname"
|
||||||
|
HOME_MARKER_REL = "rollback-acceptance/marker"
|
||||||
|
HOME_MARKER_TOKEN = "phoebus-rollback-home-marker"
|
||||||
|
HOSTNAME_SENTINEL = "phoebus-rollback-broken-hostname"
|
||||||
|
DEFAULT_SYS = "/usr/local/bin/sys"
|
||||||
|
DEFAULT_PACKAGE = "app-misc/hello"
|
||||||
|
|
||||||
|
# File framing so a read is not confused with console noise.
|
||||||
|
FILE_FRAME = "__PHOEBUS_RB_FILE__"
|
||||||
|
# The driver appends this and parses the exit status from the console.
|
||||||
|
RC_SENTINEL = "__PHOEBUS_RB_RC"
|
||||||
|
|
||||||
|
MAX_OUTPUT_BYTES = 64 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Small, testable value types
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class CommandResult:
|
||||||
|
"""The outcome of one guest command.
|
||||||
|
|
||||||
|
``timed_out`` and ``disconnected`` are distinct from a nonzero exit so a
|
||||||
|
broken connection can never be mistaken for a clean failure that a later
|
||||||
|
step might tolerate.
|
||||||
|
"""
|
||||||
|
|
||||||
|
command: str
|
||||||
|
stdout: str = ""
|
||||||
|
stderr: str = ""
|
||||||
|
exit_code: int | None = None
|
||||||
|
timed_out: bool = False
|
||||||
|
disconnected: bool = False
|
||||||
|
|
||||||
|
def bounded(self) -> "CommandResult":
|
||||||
|
combined = self.stdout[-MAX_OUTPUT_BYTES:]
|
||||||
|
self.stdout = combined
|
||||||
|
self.stderr = self.stderr[-MAX_OUTPUT_BYTES:]
|
||||||
|
return self
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ok(self) -> bool:
|
||||||
|
return (
|
||||||
|
not self.timed_out
|
||||||
|
and not self.disconnected
|
||||||
|
and self.exit_code == 0
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def failure_reason(self) -> str:
|
||||||
|
if self.timed_out:
|
||||||
|
return f"timeout running `{self.command}`"
|
||||||
|
if self.disconnected:
|
||||||
|
return f"guest disconnected running `{self.command}`"
|
||||||
|
if self.exit_code is None:
|
||||||
|
return f"no exit status for `{self.command}`"
|
||||||
|
if self.exit_code != 0:
|
||||||
|
detail = (self.stderr or self.stdout).strip().splitlines()
|
||||||
|
tail = detail[-1] if detail else ""
|
||||||
|
return f"`{self.command}` exited {self.exit_code}: {tail}"
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class SnapshotRecord:
|
||||||
|
id: str
|
||||||
|
operation: str
|
||||||
|
snapshot_subvol: str
|
||||||
|
var_snapshot: str | None
|
||||||
|
status: str
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class Guest(Protocol):
|
||||||
|
"""A shell on a booted installed system."""
|
||||||
|
|
||||||
|
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class BootTransport(Protocol):
|
||||||
|
"""Boots the installed disk in the requested mode.
|
||||||
|
|
||||||
|
* ``boot_installed`` boots the normal entry with the installer ISO detached.
|
||||||
|
* ``boot_recovery`` boots the recovery entry with networking disabled.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def boot_installed(self) -> Guest:
|
||||||
|
...
|
||||||
|
|
||||||
|
def boot_recovery(self) -> Guest:
|
||||||
|
...
|
||||||
|
|
||||||
|
def poweroff(self) -> None:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Parsers (pure functions, unit-tested directly)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def extract_between(text: str, start: str, end: str) -> str:
|
||||||
|
"""Return the text between the first ``start`` and the next ``end``.
|
||||||
|
|
||||||
|
If the markers are absent the whole text is returned unchanged, so a fake
|
||||||
|
transport may return clean content directly.
|
||||||
|
"""
|
||||||
|
if start not in text or end not in text:
|
||||||
|
return text
|
||||||
|
body = text.split(start, 1)[1]
|
||||||
|
return body.split(end, 1)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_snapshot_history(text: str) -> list[SnapshotRecord]:
|
||||||
|
"""Parse the snapshot history TOML into records.
|
||||||
|
|
||||||
|
Raises :class:`ValueError` when the document is unreadable; a caller must
|
||||||
|
never treat "could not parse" as "no snapshots".
|
||||||
|
"""
|
||||||
|
text = text.strip()
|
||||||
|
if not text:
|
||||||
|
raise ValueError("empty snapshot history")
|
||||||
|
try:
|
||||||
|
data = tomllib.loads(text)
|
||||||
|
except tomllib.TOMLDecodeError as exc: # pragma: no cover - message detail
|
||||||
|
raise ValueError(f"invalid snapshot history: {exc}") from exc
|
||||||
|
records = data.get("records", [])
|
||||||
|
if not isinstance(records, list):
|
||||||
|
raise ValueError("snapshot history has no `records` list")
|
||||||
|
out: list[SnapshotRecord] = []
|
||||||
|
for entry in records:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
out.append(
|
||||||
|
SnapshotRecord(
|
||||||
|
id=str(entry.get("id", "")),
|
||||||
|
operation=str(entry.get("operation", "")),
|
||||||
|
snapshot_subvol=str(entry.get("snapshot_subvol", "")),
|
||||||
|
var_snapshot=entry.get("var_snapshot"),
|
||||||
|
status=str(entry.get("status", "")),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def newest_committed_coordinated(
|
||||||
|
records: list[SnapshotRecord],
|
||||||
|
) -> SnapshotRecord | None:
|
||||||
|
"""The newest committed snapshot that also captured ``/var``.
|
||||||
|
|
||||||
|
A root snapshot with no paired ``@var`` snapshot cannot restore a
|
||||||
|
consistent package database, so it is not a valid rollback target.
|
||||||
|
"""
|
||||||
|
candidates = [
|
||||||
|
r
|
||||||
|
for r in records
|
||||||
|
if r.status == "committed" and r.snapshot_subvol and r.var_snapshot
|
||||||
|
]
|
||||||
|
if not candidates:
|
||||||
|
return None
|
||||||
|
return candidates[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_rollback_state(text: str) -> dict:
|
||||||
|
"""Parse ``/var/lib/phoebus/rollback-state.toml`` into a dict."""
|
||||||
|
text = text.strip()
|
||||||
|
if not text:
|
||||||
|
raise ValueError("empty rollback state")
|
||||||
|
try:
|
||||||
|
return tomllib.loads(text)
|
||||||
|
except tomllib.TOMLDecodeError as exc: # pragma: no cover - message detail
|
||||||
|
raise ValueError(f"invalid rollback state: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_subvol(value: str | None) -> str | None:
|
||||||
|
"""Canonicalise a subvolume path for comparison.
|
||||||
|
|
||||||
|
Btrfs reports the same subvolume as ``@restore-x`` in ``rootflags`` and
|
||||||
|
``/@restore-x`` in a fstab mount option; comparison must ignore that.
|
||||||
|
"""
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
return value.lstrip("/")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_rootflags_subvol(cmdline: str) -> str | None:
|
||||||
|
"""Extract the ``subvol=`` value from the kernel ``rootflags=`` option."""
|
||||||
|
match = re.search(r"rootflags=[^ ]*?\bsubvol=([^,\s]+)", cmdline)
|
||||||
|
if match:
|
||||||
|
return normalize_subvol(match.group(1))
|
||||||
|
# Some kernels present rootflags only as the bare option.
|
||||||
|
match = re.search(r"\bsubvol=([^,\s]+)", cmdline)
|
||||||
|
return normalize_subvol(match.group(1)) if match else None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_fstab_subvol(text: str, mountpoint: str) -> str | None:
|
||||||
|
"""The ``subvol=`` value for a mountpoint in ``/etc/fstab``."""
|
||||||
|
for line in text.splitlines():
|
||||||
|
line = line.split("#", 1)[0].strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
fields = line.split()
|
||||||
|
if len(fields) < 4:
|
||||||
|
continue
|
||||||
|
if fields[1] != mountpoint:
|
||||||
|
continue
|
||||||
|
match = re.search(r"(?:^|,)subvol=([^,\s]+)", fields[3])
|
||||||
|
if match:
|
||||||
|
return normalize_subvol(match.group(1))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_package_dirs(text: str) -> list[str]:
|
||||||
|
"""Absolute ``/var/db/pkg/<cat>/<pkg>`` directories from a listing."""
|
||||||
|
return sorted({line.strip() for line in text.splitlines() if line.strip()})
|
||||||
|
|
||||||
|
|
||||||
|
def parse_contents_files(text: str) -> list[str]:
|
||||||
|
"""Installed file paths from a Portage ``CONTENTS`` file.
|
||||||
|
|
||||||
|
The format is ``<obj> <path> <hash...>`` per line.
|
||||||
|
"""
|
||||||
|
paths: list[str] = []
|
||||||
|
for line in text.splitlines():
|
||||||
|
fields = line.split()
|
||||||
|
if len(fields) >= 2 and fields[0] in ("obj", "dir", "sym"):
|
||||||
|
paths.append(fields[1])
|
||||||
|
return paths
|
||||||
|
|
||||||
|
|
||||||
|
def parse_uname(text: str) -> str:
|
||||||
|
return text.strip().splitlines()[-1].strip() if text.strip() else ""
|
||||||
|
|
||||||
|
|
||||||
|
def parse_network_addresses(text: str) -> list[str]:
|
||||||
|
"""Interfaces that have an address, from ``ip -brief addr`` output."""
|
||||||
|
out: list[str] = []
|
||||||
|
for line in text.splitlines():
|
||||||
|
fields = line.split()
|
||||||
|
if not fields:
|
||||||
|
continue
|
||||||
|
name = fields[0]
|
||||||
|
rest = fields[1:]
|
||||||
|
if any("." in f or ":" in f for f in rest):
|
||||||
|
out.append(name)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Journey
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class _Step:
|
||||||
|
name: str
|
||||||
|
method_name: str
|
||||||
|
boot: str | None = None
|
||||||
|
stream: str = "rollback"
|
||||||
|
|
||||||
|
|
||||||
|
# Order is the contract. A change here changes the release gate; keep every
|
||||||
|
# expected step in the list so a stopped run can mark the rest NOT RUN.
|
||||||
|
_STEPS: tuple[_Step, ...] = (
|
||||||
|
_Step("sys-binary-identity", "_step_sys_identity", boot="installed"),
|
||||||
|
_Step("known-good-coordinated-snapshot", "_step_known_good"),
|
||||||
|
_Step("home-data-marker", "_step_mark_home"),
|
||||||
|
_Step("controlled-package-change", "_step_package_change"),
|
||||||
|
_Step("deterministic-guest-breakage", "_step_break"),
|
||||||
|
_Step("sys-undo-activates", "_step_undo"),
|
||||||
|
_Step("reboot-detached", "_step_reboot", boot="installed"),
|
||||||
|
_Step("restored-root-var-generations", "_step_generations"),
|
||||||
|
_Step("package-files-restored", "_step_package_files"),
|
||||||
|
_Step("portage-database-consistent", "_step_portage_db"),
|
||||||
|
_Step("kernel-modules-compatible", "_step_kernel_modules"),
|
||||||
|
_Step("home-data-survives", "_step_home_survives"),
|
||||||
|
_Step("recovery-entry-armed", "_step_arm_recovery"),
|
||||||
|
_Step(
|
||||||
|
"recovery-boot-no-network-no-model",
|
||||||
|
"_step_recovery",
|
||||||
|
boot="recovery",
|
||||||
|
stream="recovery",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
EXPECTED_STEP_NAMES: tuple[str, ...] = tuple(step.name for step in _STEPS)
|
||||||
|
|
||||||
|
|
||||||
|
class RollbackJourney:
|
||||||
|
"""Run the ordered acceptance steps against a :class:`BootTransport`.
|
||||||
|
|
||||||
|
The class records every expected step, so a run that stops early reports the
|
||||||
|
remainder as NOT RUN and fails an explicit ``journey-complete`` check rather
|
||||||
|
than silently looking like a pass.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
transport: BootTransport,
|
||||||
|
*,
|
||||||
|
results: Results | None = None,
|
||||||
|
recovery_results: Results | None = None,
|
||||||
|
home: str = "/home/phoebus",
|
||||||
|
sys_bin: str = DEFAULT_SYS,
|
||||||
|
package: str = DEFAULT_PACKAGE,
|
||||||
|
iso_sha256: str | None = None,
|
||||||
|
sys_sha256: str | None = None,
|
||||||
|
mocked: bool = False,
|
||||||
|
):
|
||||||
|
self.transport = transport
|
||||||
|
self.results = results or Results(SCENARIO_ROLLBACK)
|
||||||
|
self.recovery_results = recovery_results or Results(SCENARIO_RECOVERY)
|
||||||
|
for stream in (self.results, self.recovery_results):
|
||||||
|
if iso_sha256:
|
||||||
|
stream.bind_input(INPUT_ISO, iso_sha256)
|
||||||
|
if sys_sha256:
|
||||||
|
stream.bind_input(INPUT_SYS, sys_sha256)
|
||||||
|
if mocked:
|
||||||
|
stream.mark_mocked("fake transport: not release evidence")
|
||||||
|
self.home = home
|
||||||
|
self.sys_bin = sys_bin
|
||||||
|
self.package = package
|
||||||
|
self.expected_sys_sha256 = sys_sha256
|
||||||
|
self.marker_path = f"{home.rstrip('/')}/{HOME_MARKER_REL}"
|
||||||
|
self.marker_token = f"{HOME_MARKER_TOKEN}:{self.results.run_id}"
|
||||||
|
self.boots: list[str] = []
|
||||||
|
self.executed: list[str] = []
|
||||||
|
|
||||||
|
# Findings carried between steps.
|
||||||
|
self.snapshot: SnapshotRecord | None = None
|
||||||
|
self.original_hostname: str | None = None
|
||||||
|
self.package_dirs_before: list[str] = []
|
||||||
|
self.added_package_dirs: list[str] = []
|
||||||
|
self.added_package_files: list[str] = []
|
||||||
|
self.rollback_state: dict = {}
|
||||||
|
|
||||||
|
self._active = self.results
|
||||||
|
|
||||||
|
# -- helpers -----------------------------------------------------------
|
||||||
|
def _check(self, name: str, condition: bool, detail: str = "") -> bool:
|
||||||
|
return self._active.assert_true(name, condition, detail)
|
||||||
|
|
||||||
|
def _run(
|
||||||
|
self, guest: Guest, script: str, timeout: float = 240.0
|
||||||
|
) -> CommandResult:
|
||||||
|
try:
|
||||||
|
return guest.run(script, timeout=timeout).bounded()
|
||||||
|
except Exception as exc: # a transport fault is a failure, not a crash
|
||||||
|
return CommandResult(
|
||||||
|
command=script, disconnected=True, stderr=str(exc)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _read_file(self, guest: Guest, path: str, timeout: float = 60.0) -> str:
|
||||||
|
result = self._run(
|
||||||
|
guest,
|
||||||
|
f"echo {FILE_FRAME}; cat {path} 2>/dev/null; echo {FILE_FRAME}",
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
return extract_between(result.stdout, FILE_FRAME, FILE_FRAME)
|
||||||
|
|
||||||
|
# -- public API --------------------------------------------------------
|
||||||
|
def required_inputs(self) -> tuple[str, ...]:
|
||||||
|
return REQUIRED_INPUTS[self.results.scenario]
|
||||||
|
|
||||||
|
def run(self) -> bool:
|
||||||
|
"""Execute the journey. Returns True only if every step passed."""
|
||||||
|
if not self._check_inputs():
|
||||||
|
for step in _STEPS:
|
||||||
|
stream = (
|
||||||
|
self.results
|
||||||
|
if step.stream == "rollback"
|
||||||
|
else self.recovery_results
|
||||||
|
)
|
||||||
|
stream.mark_not_run(step.name, "input identities were not pinned")
|
||||||
|
self.results.fail("journey-complete", "missing input identities")
|
||||||
|
self.recovery_results.fail("journey-complete", "missing input identities")
|
||||||
|
return False
|
||||||
|
|
||||||
|
guest: Guest | None = None
|
||||||
|
for index, step in enumerate(_STEPS):
|
||||||
|
self._active = (
|
||||||
|
self.results if step.stream == "rollback" else self.recovery_results
|
||||||
|
)
|
||||||
|
if step.boot is not None:
|
||||||
|
if guest is not None:
|
||||||
|
self.transport.poweroff()
|
||||||
|
guest = (
|
||||||
|
self.transport.boot_recovery()
|
||||||
|
if step.boot == "recovery"
|
||||||
|
else self.transport.boot_installed()
|
||||||
|
)
|
||||||
|
self.boots.append(step.boot)
|
||||||
|
method: Callable[[Guest], bool] = getattr(self, step.method_name)
|
||||||
|
self.executed.append(step.name)
|
||||||
|
if not method(guest):
|
||||||
|
self._finish_incomplete(index, step.name)
|
||||||
|
return False
|
||||||
|
|
||||||
|
self.results.assert_true("journey-complete", True)
|
||||||
|
self.recovery_results.assert_true("journey-complete", True)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def run_recovery(self) -> bool:
|
||||||
|
"""Verify only the no-model recovery boot (used by the `recovery` mode)."""
|
||||||
|
if not self._check_inputs():
|
||||||
|
self.recovery_results.mark_not_run(
|
||||||
|
"recovery-boot-no-network-no-model", "input identities were not pinned"
|
||||||
|
)
|
||||||
|
self.recovery_results.fail("journey-complete", "missing input identities")
|
||||||
|
return False
|
||||||
|
self._active = self.recovery_results
|
||||||
|
ok = self._step_recovery(self.transport.boot_recovery())
|
||||||
|
if ok:
|
||||||
|
self.recovery_results.assert_true("journey-complete", True)
|
||||||
|
else:
|
||||||
|
self.recovery_results.fail(
|
||||||
|
"journey-complete", "recovery verification failed"
|
||||||
|
)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
# -- input identity gate ----------------------------------------------
|
||||||
|
def _check_inputs(self) -> bool:
|
||||||
|
ok = True
|
||||||
|
for stream, scenario in (
|
||||||
|
(self.results, SCENARIO_ROLLBACK),
|
||||||
|
(self.recovery_results, SCENARIO_RECOVERY),
|
||||||
|
):
|
||||||
|
required = REQUIRED_INPUTS[scenario]
|
||||||
|
missing = stream.missing_inputs(required)
|
||||||
|
ok &= stream.assert_true(
|
||||||
|
"input-identities",
|
||||||
|
not missing,
|
||||||
|
f"missing pinned inputs for {scenario}: {', '.join(missing)}",
|
||||||
|
)
|
||||||
|
return bool(ok)
|
||||||
|
|
||||||
|
def _finish_incomplete(self, failed_index: int, failed_step: str) -> None:
|
||||||
|
for step in _STEPS[failed_index + 1:]:
|
||||||
|
stream = (
|
||||||
|
self.results if step.stream == "rollback" else self.recovery_results
|
||||||
|
)
|
||||||
|
stream.mark_not_run(step.name, "journey stopped before this step")
|
||||||
|
# The failed step's own stream already has a FAIL; make the incompleteness
|
||||||
|
# explicit on both so neither gate can read a partial run as a pass.
|
||||||
|
self.results.fail(
|
||||||
|
"journey-complete", f"stopped after `{failed_step}`"
|
||||||
|
)
|
||||||
|
self.recovery_results.fail(
|
||||||
|
"journey-complete", f"stopped after `{failed_step}`"
|
||||||
|
)
|
||||||
|
|
||||||
|
# -- steps -------------------------------------------------------------
|
||||||
|
def _step_sys_identity(self, guest: Guest) -> bool:
|
||||||
|
# The rollback semantics live in `sys`; pin the exact binary under test.
|
||||||
|
self._active = self.results
|
||||||
|
expected = self.expected_sys_sha256 or ""
|
||||||
|
result = self._run(guest, f"sha256sum {shlex_quote(self.sys_bin)}", timeout=60.0)
|
||||||
|
if not result.ok:
|
||||||
|
return self._check("sys-binary-identity", False, result.failure_reason)
|
||||||
|
digest = result.stdout.split()[0] if result.stdout.split() else ""
|
||||||
|
return self._check(
|
||||||
|
"sys-binary-identity",
|
||||||
|
bool(expected) and digest == expected,
|
||||||
|
f"guest sys digest {digest!r} != pinned {expected!r}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _step_known_good(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
text = self._read_file(guest, SNAPSHOT_HISTORY)
|
||||||
|
try:
|
||||||
|
records = parse_snapshot_history(text)
|
||||||
|
except ValueError as exc:
|
||||||
|
return self._check("known-good-coordinated-snapshot", False, str(exc))
|
||||||
|
snapshot = newest_committed_coordinated(records)
|
||||||
|
if snapshot is None:
|
||||||
|
return self._check(
|
||||||
|
"known-good-coordinated-snapshot",
|
||||||
|
False,
|
||||||
|
"no committed snapshot with a paired @var snapshot",
|
||||||
|
)
|
||||||
|
self.snapshot = snapshot
|
||||||
|
self._check("known-good-root-snapshot", bool(snapshot.snapshot_subvol), "empty root snapshot path")
|
||||||
|
self._check("known-good-var-snapshot", bool(snapshot.var_snapshot), "empty @var snapshot path")
|
||||||
|
self._check(
|
||||||
|
"known-good-coordinated-snapshot",
|
||||||
|
True,
|
||||||
|
f"{snapshot.id} ({snapshot.operation})",
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _step_mark_home(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
result = self._run(
|
||||||
|
guest,
|
||||||
|
f"mkdir -p {self.home.rstrip('/')}/{HOME_MARKER_REL.rsplit('/', 1)[0]} && "
|
||||||
|
f"printf '%s' '{self.marker_token}' > {self.marker_path}",
|
||||||
|
)
|
||||||
|
if not result.ok:
|
||||||
|
return self._check("home-data-marker", False, result.failure_reason)
|
||||||
|
readback = self._read_file(guest, self.marker_path)
|
||||||
|
if readback.strip() != self.marker_token:
|
||||||
|
return self._check(
|
||||||
|
"home-data-marker", False, "home marker did not read back"
|
||||||
|
)
|
||||||
|
self.original_hostname = self._read_file(guest, HOSTNAME_FILE).strip()
|
||||||
|
self.package_dirs_before = parse_package_dirs(
|
||||||
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||||
|
)
|
||||||
|
self._check("home-data-marker", True, self.marker_path)
|
||||||
|
self._check(
|
||||||
|
"hostname-recorded", bool(self.original_hostname), "could not read /etc/hostname"
|
||||||
|
)
|
||||||
|
return bool(self.original_hostname)
|
||||||
|
|
||||||
|
def _step_package_change(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
result = self._run(
|
||||||
|
guest, f"sudo {self.sys_bin} install {self.package} -y", timeout=1800.0
|
||||||
|
)
|
||||||
|
if not result.ok:
|
||||||
|
return self._check("controlled-package-change", False, result.failure_reason)
|
||||||
|
after = parse_package_dirs(
|
||||||
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||||
|
)
|
||||||
|
added = sorted(set(after) - set(self.package_dirs_before))
|
||||||
|
if not added:
|
||||||
|
return self._check(
|
||||||
|
"controlled-package-change",
|
||||||
|
False,
|
||||||
|
f"`{self.package}` produced no new /var/db/pkg entry",
|
||||||
|
)
|
||||||
|
self.added_package_dirs = added
|
||||||
|
contents = self._read_file(guest, f"{added[0]}/CONTENTS")
|
||||||
|
self.added_package_files = parse_contents_files(contents)
|
||||||
|
self._check(
|
||||||
|
"controlled-package-change", True, f"added {', '.join(added)}"
|
||||||
|
)
|
||||||
|
self._check(
|
||||||
|
"package-db-recorded",
|
||||||
|
bool(self.added_package_files),
|
||||||
|
"changed package has no recorded files",
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _step_break(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
result = self._run(
|
||||||
|
guest,
|
||||||
|
f"printf '%s\\n' '{HOSTNAME_SENTINEL}' | sudo tee {HOSTNAME_FILE} >/dev/null",
|
||||||
|
)
|
||||||
|
if not result.ok:
|
||||||
|
return self._check("deterministic-guest-breakage", False, result.failure_reason)
|
||||||
|
broken = self._read_file(guest, HOSTNAME_FILE).strip()
|
||||||
|
if broken != HOSTNAME_SENTINEL:
|
||||||
|
return self._check(
|
||||||
|
"deterministic-guest-breakage",
|
||||||
|
False,
|
||||||
|
"breakage did not take effect",
|
||||||
|
)
|
||||||
|
self._check(
|
||||||
|
"deterministic-guest-breakage",
|
||||||
|
True,
|
||||||
|
f"{HOSTNAME_FILE} -> {HOSTNAME_SENTINEL}",
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _step_undo(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
if self.snapshot is None:
|
||||||
|
return self._check("sys-undo-activates", False, "no snapshot selected")
|
||||||
|
result = self._run(
|
||||||
|
guest, f"sudo {self.sys_bin} undo {self.snapshot.id} -y", timeout=900.0
|
||||||
|
)
|
||||||
|
if not result.ok:
|
||||||
|
return self._check("sys-undo-activates", False, result.failure_reason)
|
||||||
|
state = self._read_file(guest, ROLLBACK_STATE)
|
||||||
|
try:
|
||||||
|
self.rollback_state = parse_rollback_state(state)
|
||||||
|
except ValueError as exc:
|
||||||
|
return self._check("sys-undo-activates", False, str(exc))
|
||||||
|
if self.rollback_state.get("status") != "activated":
|
||||||
|
return self._check(
|
||||||
|
"sys-undo-activates",
|
||||||
|
False,
|
||||||
|
f"rollback state is {self.rollback_state.get('status')!r}, not 'activated'",
|
||||||
|
)
|
||||||
|
self._check(
|
||||||
|
"sys-undo-activates",
|
||||||
|
True,
|
||||||
|
f"next boot uses {self.rollback_state.get('root_subvol')}",
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _step_reboot(self, guest: Guest) -> bool:
|
||||||
|
# The boot itself is performed by the transport; the driver asserts the
|
||||||
|
# ISO is detached before this returns. Here we only record the boundary.
|
||||||
|
self._active = self.results
|
||||||
|
self._check("reboot-detached", True, "booted installed disk with ISO detached")
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _step_generations(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
expected_root = str(self.rollback_state.get("root_subvol", ""))
|
||||||
|
expected_var = str(self.rollback_state.get("var_subvol", "") or "")
|
||||||
|
cmdline = self._read_file(guest, "/proc/cmdline")
|
||||||
|
booted_subvol = parse_rootflags_subvol(cmdline)
|
||||||
|
fstab = self._read_file(guest, "/etc/fstab")
|
||||||
|
fstab_root = parse_fstab_subvol(fstab, "/")
|
||||||
|
fstab_var = parse_fstab_subvol(fstab, "/var")
|
||||||
|
ok = True
|
||||||
|
ok &= self._check(
|
||||||
|
"restored-root-generation",
|
||||||
|
bool(expected_root) and booted_subvol == expected_root,
|
||||||
|
f"booted subvol {booted_subvol!r} != expected {expected_root!r}",
|
||||||
|
)
|
||||||
|
ok &= self._check(
|
||||||
|
"restored-fstab-root",
|
||||||
|
bool(expected_root) and fstab_root == expected_root,
|
||||||
|
f"fstab / subvol {fstab_root!r} != {expected_root!r}",
|
||||||
|
)
|
||||||
|
ok &= self._check(
|
||||||
|
"restored-var-generation",
|
||||||
|
bool(expected_var) and fstab_var == expected_var,
|
||||||
|
f"fstab /var subvol {fstab_var!r} != {expected_var!r}",
|
||||||
|
)
|
||||||
|
# The running system must not still be the pre-rollback generation.
|
||||||
|
ok &= self._check(
|
||||||
|
"generation-advanced",
|
||||||
|
booted_subvol.startswith("@restore-") if booted_subvol else False,
|
||||||
|
f"booted subvol {booted_subvol!r} is not a restored generation",
|
||||||
|
)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
def _step_package_files(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
after = parse_package_dirs(
|
||||||
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||||
|
)
|
||||||
|
still_present = [d for d in self.added_package_dirs if d in after]
|
||||||
|
ok = self._check(
|
||||||
|
"package-db-restored",
|
||||||
|
not still_present,
|
||||||
|
f"package DB still lists {still_present}",
|
||||||
|
)
|
||||||
|
missing_files = []
|
||||||
|
for path in self.added_package_files:
|
||||||
|
result = self._run(guest, f"test -e {shlex_quote(path)}")
|
||||||
|
if result.exit_code == 0:
|
||||||
|
missing_files.append(path)
|
||||||
|
ok &= self._check(
|
||||||
|
"package-files-restored",
|
||||||
|
not missing_files,
|
||||||
|
f"package files survived the rollback: {missing_files[:5]}",
|
||||||
|
)
|
||||||
|
hostname = self._read_file(guest, HOSTNAME_FILE).strip()
|
||||||
|
ok &= self._check(
|
||||||
|
"breakage-reverted",
|
||||||
|
hostname == (self.original_hostname or ""),
|
||||||
|
f"{HOSTNAME_FILE} is {hostname!r}, expected {self.original_hostname!r}",
|
||||||
|
)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
def _step_portage_db(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
listing = parse_package_dirs(
|
||||||
|
self._run(guest, f"find {PKG_DB} -mindepth 2 -maxdepth 2 -type d").stdout
|
||||||
|
)
|
||||||
|
ok = self._check(
|
||||||
|
"portage-db-generation-count",
|
||||||
|
listing == self.package_dirs_before,
|
||||||
|
f"/var/db/pkg has {len(listing)} entries, expected {len(self.package_dirs_before)}",
|
||||||
|
)
|
||||||
|
# Every entry must carry a CONTENTS file; an inconsistent database is
|
||||||
|
# how a mismatched root/var pairing shows up.
|
||||||
|
probe = self._run(
|
||||||
|
guest,
|
||||||
|
f"for d in {PKG_DB}/*/*; do [ -f \"$d/CONTENTS\" ] || echo \"$d\"; done",
|
||||||
|
)
|
||||||
|
missing_contents = [
|
||||||
|
line.strip() for line in probe.stdout.splitlines() if line.strip()
|
||||||
|
]
|
||||||
|
ok &= self._check(
|
||||||
|
"portage-db-contents",
|
||||||
|
not missing_contents,
|
||||||
|
f"package entries without CONTENTS: {missing_contents[:5]}",
|
||||||
|
)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
def _step_kernel_modules(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
kernel = parse_uname(self._run(guest, "uname -r").stdout)
|
||||||
|
modules = self._read_file(guest, "/lib/modules")
|
||||||
|
module_dirs = [line.strip() for line in modules.splitlines() if line.strip()]
|
||||||
|
return self._check(
|
||||||
|
"kernel-modules-compatible",
|
||||||
|
bool(kernel) and kernel in module_dirs,
|
||||||
|
f"no /lib/modules/{kernel} in the restored root ({module_dirs})",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _step_home_survives(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.results
|
||||||
|
readback = self._read_file(guest, self.marker_path).strip()
|
||||||
|
return self._check(
|
||||||
|
"home-data-survives",
|
||||||
|
readback == self.marker_token,
|
||||||
|
"the /home marker did not survive the rollback",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _step_arm_recovery(self, guest: Guest) -> bool:
|
||||||
|
# Boot the recovery entry exactly once, then fall back to the restored
|
||||||
|
# root. This is a real systemd-boot interface, not a QEMU shortcut.
|
||||||
|
self._active = self.results
|
||||||
|
result = self._run(
|
||||||
|
guest, "sudo bootctl set-oneshot phoebus-recovery", timeout=120.0
|
||||||
|
)
|
||||||
|
return self._check("recovery-entry-armed", result.ok, result.failure_reason)
|
||||||
|
|
||||||
|
def _step_recovery(self, guest: Guest) -> bool:
|
||||||
|
self._active = self.recovery_results
|
||||||
|
cmdline = self._read_file(guest, "/proc/cmdline")
|
||||||
|
ok = self._check(
|
||||||
|
"recovery-entry-booted",
|
||||||
|
"systemd.unit=rescue.target" in cmdline,
|
||||||
|
"the recovery entry did not boot into rescue.target",
|
||||||
|
)
|
||||||
|
addresses = parse_network_addresses(self._run(guest, "ip -brief addr").stdout)
|
||||||
|
non_loopback = [a for a in addresses if a != "lo"]
|
||||||
|
ok &= self._check(
|
||||||
|
"recovery-no-network",
|
||||||
|
not non_loopback,
|
||||||
|
f"network interfaces were up: {non_loopback}",
|
||||||
|
)
|
||||||
|
# `sys` itself must work with no model: a status query is deterministic
|
||||||
|
# and offline.
|
||||||
|
status = self._run(guest, f"{self.sys_bin} undo status")
|
||||||
|
ok &= self._check(
|
||||||
|
"recovery-sys-offline",
|
||||||
|
status.ok,
|
||||||
|
status.failure_reason or "sys status failed offline",
|
||||||
|
)
|
||||||
|
ask = self._run(guest, f'{self.sys_bin} ask "show system status"', timeout=120.0)
|
||||||
|
combined = (ask.stdout + ask.stderr).lower()
|
||||||
|
ok &= self._check(
|
||||||
|
"recovery-no-model-fallback",
|
||||||
|
(not ask.ok) and "no model" in combined,
|
||||||
|
"sys ask did not report the deterministic no-model fallback",
|
||||||
|
)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
|
||||||
|
def shlex_quote(value: str) -> str:
|
||||||
|
"""Minimal single-quote shell escaping for a known-safe path."""
|
||||||
|
return "'" + value.replace("'", "'\\''") + "'"
|
||||||
|
|
||||||
|
|
||||||
|
def validate_acceptance_disks(disks, test_dir) -> list:
|
||||||
|
"""Refuse anything but disposable qcow2 disks inside the test directory.
|
||||||
|
|
||||||
|
Kept here so the journey's safety entry point is unit-testable without QEMU.
|
||||||
|
"""
|
||||||
|
return vmsafety.validate_writable_drives(disks, test_dir)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"SCENARIO_ROLLBACK",
|
||||||
|
"SCENARIO_RECOVERY",
|
||||||
|
"REQUIRED_INPUTS",
|
||||||
|
"EXPECTED_STEP_NAMES",
|
||||||
|
"CommandResult",
|
||||||
|
"SnapshotRecord",
|
||||||
|
"Guest",
|
||||||
|
"BootTransport",
|
||||||
|
"RollbackJourney",
|
||||||
|
"validate_acceptance_disks",
|
||||||
|
"extract_between",
|
||||||
|
"normalize_subvol",
|
||||||
|
"parse_snapshot_history",
|
||||||
|
"newest_committed_coordinated",
|
||||||
|
"parse_rollback_state",
|
||||||
|
"parse_rootflags_subvol",
|
||||||
|
"parse_fstab_subvol",
|
||||||
|
"parse_package_dirs",
|
||||||
|
"parse_contents_files",
|
||||||
|
"parse_uname",
|
||||||
|
"parse_network_addresses",
|
||||||
|
]
|
||||||
497
installer/tests/vm/test_rollback_acceptance.py
Normal file
497
installer/tests/vm/test_rollback_acceptance.py
Normal file
|
|
@ -0,0 +1,497 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Mocked-transport tests for the rollback acceptance orchestration.
|
||||||
|
|
||||||
|
These run on any host (no QEMU, no disk), so the journey's ordering, safety and
|
||||||
|
failure semantics are checked in CI rather than only during a live run. Every
|
||||||
|
transport here is a fake: the results are marked ``mocked`` and are explicitly
|
||||||
|
**not** release evidence (the release tool refuses them).
|
||||||
|
|
||||||
|
python3 installer/tests/vm/test_rollback_acceptance.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import rollback_acceptance as ra
|
||||||
|
from rollback_acceptance import (
|
||||||
|
SCENARIO_RECOVERY,
|
||||||
|
SCENARIO_ROLLBACK,
|
||||||
|
CommandResult,
|
||||||
|
RollbackJourney,
|
||||||
|
validate_acceptance_disks,
|
||||||
|
)
|
||||||
|
from vmsafety import FAIL, MOCKED, NOT_RUN, Results, UnsafeDiskError
|
||||||
|
|
||||||
|
BASE_PKGS = [
|
||||||
|
"/var/db/pkg/sys-apps/busybox-1.36.1",
|
||||||
|
"/var/db/pkg/app-shells/bash-5.2_p26",
|
||||||
|
]
|
||||||
|
ADDED_PKG = "/var/db/pkg/app-misc/hello-2.12"
|
||||||
|
HISTORY = """\
|
||||||
|
[[records]]
|
||||||
|
id = "snap-initial"
|
||||||
|
created_unix = 1000
|
||||||
|
operation = "initial known-good"
|
||||||
|
source_subvol = "/"
|
||||||
|
snapshot_subvol = "@-20260101T000000-initial-known-good"
|
||||||
|
var_snapshot = "@var-20260101T000000-initial-known-good"
|
||||||
|
status = "committed"
|
||||||
|
"""
|
||||||
|
ROLLBACK_STATE = """\
|
||||||
|
status = "activated"
|
||||||
|
generation = 1001
|
||||||
|
target_snapshot = "@-20260101T000000-initial-known-good"
|
||||||
|
previous_root = "@"
|
||||||
|
root_subvol = "@restore-20260101T000000"
|
||||||
|
var_subvol = "@var-restore-20260101T000000"
|
||||||
|
started_unix = 1001
|
||||||
|
finished_unix = 1001
|
||||||
|
note = "restore from snap-initial"
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class FakeGuest:
|
||||||
|
"""A scripted guest. First matching handler wins."""
|
||||||
|
|
||||||
|
def __init__(self, name: str):
|
||||||
|
self.name = name
|
||||||
|
self.commands: list[str] = []
|
||||||
|
self.handlers: list = []
|
||||||
|
|
||||||
|
def on(self, needle: str, handler):
|
||||||
|
# A re-registered needle overrides the earlier one, so a test can tweak
|
||||||
|
# one behaviour without disturbing the ordering of distinct needles.
|
||||||
|
self.handlers = [(n, h) for (n, h) in self.handlers if n != needle]
|
||||||
|
self.handlers.append((needle, handler))
|
||||||
|
return self
|
||||||
|
|
||||||
|
def reply(self, needle: str, stdout: str = "", exit_code: int = 0):
|
||||||
|
return self.on(needle, lambda cmd: CommandResult(cmd, stdout=stdout, exit_code=exit_code))
|
||||||
|
|
||||||
|
def run(self, command: str, timeout: float = 180.0) -> CommandResult:
|
||||||
|
self.commands.append(command)
|
||||||
|
for needle, handler in self.handlers:
|
||||||
|
if needle in command:
|
||||||
|
return handler(command)
|
||||||
|
return CommandResult(command=command, exit_code=0)
|
||||||
|
|
||||||
|
|
||||||
|
class FakeTransport:
|
||||||
|
def __init__(self, installed, recovery):
|
||||||
|
# installed is a list of guests returned by successive boot_installed()
|
||||||
|
# calls; the first is the change/undo boot, the second the restored boot.
|
||||||
|
self._installed = list(installed)
|
||||||
|
self._recovery = recovery
|
||||||
|
self.boots: list[str] = []
|
||||||
|
self.powered_off = 0
|
||||||
|
|
||||||
|
def boot_installed(self):
|
||||||
|
self.boots.append("installed")
|
||||||
|
return self._installed.pop(0)
|
||||||
|
|
||||||
|
def boot_recovery(self):
|
||||||
|
self.boots.append("recovery")
|
||||||
|
return self._recovery
|
||||||
|
|
||||||
|
def poweroff(self):
|
||||||
|
self.powered_off += 1
|
||||||
|
|
||||||
|
|
||||||
|
def make_phase1(state: dict) -> FakeGuest:
|
||||||
|
g = FakeGuest("phase1")
|
||||||
|
|
||||||
|
def write_marker(cmd):
|
||||||
|
match = re.search(r"printf '%s' '([^']*)' >", cmd)
|
||||||
|
if match:
|
||||||
|
state["token"] = match.group(1)
|
||||||
|
return CommandResult(cmd, exit_code=0)
|
||||||
|
|
||||||
|
def read_marker(cmd):
|
||||||
|
return CommandResult(cmd, stdout=state.get("token", ""))
|
||||||
|
|
||||||
|
def break_hostname(cmd):
|
||||||
|
state["broken"] = True
|
||||||
|
return CommandResult(cmd, exit_code=0)
|
||||||
|
|
||||||
|
def read_hostname(cmd):
|
||||||
|
return CommandResult(
|
||||||
|
cmd,
|
||||||
|
stdout=ra.HOSTNAME_SENTINEL + "\n" if state.get("broken") else "phoebus\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
def install(cmd):
|
||||||
|
state["installed"] = True
|
||||||
|
return CommandResult(cmd, exit_code=0)
|
||||||
|
|
||||||
|
def pkg_dirs(cmd):
|
||||||
|
dirs = list(BASE_PKGS)
|
||||||
|
if state.get("installed"):
|
||||||
|
dirs.append(ADDED_PKG)
|
||||||
|
return CommandResult(cmd, stdout="\n".join(dirs) + "\n")
|
||||||
|
|
||||||
|
g.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=HISTORY))
|
||||||
|
g.on("sha256sum", lambda c: CommandResult(c, exit_code=0, stdout="b" * 64 + " /usr/local/bin/sys\n"))
|
||||||
|
g.on("> /home/phoebus/rollback-acceptance/marker", write_marker)
|
||||||
|
g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker)
|
||||||
|
g.on("tee /etc/hostname", break_hostname)
|
||||||
|
g.on("cat /etc/hostname", read_hostname)
|
||||||
|
g.on("sys install", install)
|
||||||
|
g.on("find /var/db/pkg", pkg_dirs)
|
||||||
|
g.on("cat " + ADDED_PKG + "/CONTENTS", lambda c: CommandResult(
|
||||||
|
c, stdout="obj /usr/bin/hello abcdef\nobj /usr/share/hello/README def\n"
|
||||||
|
))
|
||||||
|
g.on("sys undo", lambda c: CommandResult(c, exit_code=0, stdout="rollback activated"))
|
||||||
|
g.on("cat /var/lib/phoebus/rollback-state.toml", lambda c: CommandResult(c, stdout=ROLLBACK_STATE))
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
def make_phase2(state: dict) -> FakeGuest:
|
||||||
|
g = FakeGuest("phase2")
|
||||||
|
|
||||||
|
def read_marker(cmd):
|
||||||
|
return CommandResult(cmd, stdout=state.get("token", ""))
|
||||||
|
|
||||||
|
fstab = (
|
||||||
|
"/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n"
|
||||||
|
"/dev/mapper/luks-root /var btrfs subvol=/@var-restore-20260101T000000 0 0\n"
|
||||||
|
"/dev/mapper/luks-root /home btrfs subvol=/@home 0 0\n"
|
||||||
|
)
|
||||||
|
g.on("cat /etc/hostname", lambda c: CommandResult(c, stdout="phoebus\n"))
|
||||||
|
g.on("cat /home/phoebus/rollback-acceptance/marker", read_marker)
|
||||||
|
g.on("find /var/db/pkg", lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS) + "\n"))
|
||||||
|
g.on("test -e", lambda c: CommandResult(c, exit_code=1))
|
||||||
|
g.on("cat /proc/cmdline", lambda c: CommandResult(
|
||||||
|
c, stdout="rootflags=subvol=@restore-20260101T000000 quiet console=ttyS0\n"
|
||||||
|
))
|
||||||
|
g.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab))
|
||||||
|
g.on("uname -r", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n"))
|
||||||
|
g.on("cat /lib/modules", lambda c: CommandResult(c, stdout="6.6.0-phoebus\n"))
|
||||||
|
g.on("bootctl set-oneshot", lambda c: CommandResult(c, exit_code=0))
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
def make_recovery() -> FakeGuest:
|
||||||
|
g = FakeGuest("recovery")
|
||||||
|
g.on("cat /proc/cmdline", lambda c: CommandResult(
|
||||||
|
c, stdout="rootflags=subvol=@-20260101T000000-initial-known-good "
|
||||||
|
"systemd.unit=rescue.target no-network\n"
|
||||||
|
))
|
||||||
|
g.on("ip -brief addr", lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8 ::1/128\n"))
|
||||||
|
g.on("sys undo status", lambda c: CommandResult(c, exit_code=0, stdout="no active rollback\n"))
|
||||||
|
g.on("sys ask", lambda c: CommandResult(
|
||||||
|
c,
|
||||||
|
exit_code=1,
|
||||||
|
stderr="No model is connected. The deterministic interface still works:\n",
|
||||||
|
))
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
def build_journey(*, state=None, transport=None, **kwargs):
|
||||||
|
state = state if state is not None else {}
|
||||||
|
transport = transport or FakeTransport([make_phase1(state), make_phase2(state)], make_recovery())
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport,
|
||||||
|
iso_sha256="a" * 64,
|
||||||
|
sys_sha256="b" * 64,
|
||||||
|
mocked=True,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
return journey
|
||||||
|
|
||||||
|
|
||||||
|
def status_of(results: Results, name: str) -> str | None:
|
||||||
|
for check in results.checks:
|
||||||
|
if check.name == name:
|
||||||
|
return check.status
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class HappyPathTests(unittest.TestCase):
|
||||||
|
def test_full_journey_passes_in_expected_order(self):
|
||||||
|
journey = build_journey()
|
||||||
|
self.assertTrue(journey.run(), "happy path should pass")
|
||||||
|
self.assertEqual(list(journey.executed), list(ra.EXPECTED_STEP_NAMES))
|
||||||
|
self.assertEqual(journey.boots, ["installed", "installed", "recovery"])
|
||||||
|
self.assertEqual(journey.results.exit_code(), 0)
|
||||||
|
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
||||||
|
|
||||||
|
def test_happy_path_has_no_not_run_steps(self):
|
||||||
|
journey = build_journey()
|
||||||
|
journey.run()
|
||||||
|
for stream in (journey.results, journey.recovery_results):
|
||||||
|
self.assertFalse(
|
||||||
|
any(c.status == NOT_RUN for c in stream.checks),
|
||||||
|
"a complete run must not contain NOT RUN steps",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_mocked_run_is_marked_and_never_live(self):
|
||||||
|
journey = build_journey()
|
||||||
|
journey.run()
|
||||||
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
|
||||||
|
tmp.close()
|
||||||
|
self.addCleanup(os.unlink, tmp.name)
|
||||||
|
data = json.loads(journey.results.write(tmp.name).read_text())
|
||||||
|
self.assertEqual(data["evidence"], MOCKED)
|
||||||
|
self.assertEqual(data["inputs"]["iso"]["sha256"], "a" * 64)
|
||||||
|
self.assertEqual(data["inputs"]["sys"]["sha256"], "b" * 64)
|
||||||
|
self.assertEqual(data["scenario"], SCENARIO_ROLLBACK)
|
||||||
|
|
||||||
|
def test_recovery_scenario_is_written_separately(self):
|
||||||
|
journey = build_journey()
|
||||||
|
journey.run()
|
||||||
|
self.assertEqual(journey.recovery_results.scenario, SCENARIO_RECOVERY)
|
||||||
|
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
||||||
|
|
||||||
|
|
||||||
|
class StopOnFailureTests(unittest.TestCase):
|
||||||
|
def test_undo_failure_stops_and_marks_rest_unrun(self):
|
||||||
|
state = {}
|
||||||
|
phase1 = make_phase1(state)
|
||||||
|
# Override undo to fail.
|
||||||
|
phase1.on("sys undo", lambda c: CommandResult(c, exit_code=1, stderr="boom"))
|
||||||
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "sys-undo-activates"), FAIL)
|
||||||
|
self.assertEqual(status_of(journey.results, "journey-complete"), FAIL)
|
||||||
|
self.assertEqual(status_of(journey.results, "restored-root-var-generations"), NOT_RUN)
|
||||||
|
self.assertEqual(status_of(journey.recovery_results, "recovery-boot-no-network-no-model"), NOT_RUN)
|
||||||
|
# The second installed boot and the recovery boot must not happen.
|
||||||
|
self.assertEqual(transport.boots, ["installed"])
|
||||||
|
self.assertEqual(journey.results.exit_code(), 1)
|
||||||
|
|
||||||
|
def test_timeout_is_a_failure_not_a_pass(self):
|
||||||
|
state = {}
|
||||||
|
phase1 = make_phase1(state)
|
||||||
|
phase1.on("sys install", lambda c: CommandResult(c, timed_out=True))
|
||||||
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "controlled-package-change"), FAIL)
|
||||||
|
self.assertIn("timeout", "".join(c.detail for c in journey.results.checks))
|
||||||
|
self.assertEqual(transport.boots, ["installed"])
|
||||||
|
|
||||||
|
def test_disconnect_is_a_failure(self):
|
||||||
|
state = {}
|
||||||
|
phase1 = make_phase1(state)
|
||||||
|
phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, disconnected=True))
|
||||||
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL)
|
||||||
|
self.assertEqual(journey.results.exit_code(), 1)
|
||||||
|
|
||||||
|
def test_missing_evidence_is_refused_not_treated_as_empty(self):
|
||||||
|
state = {}
|
||||||
|
phase1 = make_phase1(state)
|
||||||
|
phase1.on("cat /var/lib/phoebus/snapshot-history.toml", lambda c: CommandResult(c, stdout=""))
|
||||||
|
transport = FakeTransport([phase1, make_phase2(state)], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "known-good-coordinated-snapshot"), FAIL)
|
||||||
|
|
||||||
|
def test_missing_input_identities_block_the_run(self):
|
||||||
|
# No iso/sys digests: the run must not boot anything.
|
||||||
|
transport = FakeTransport([make_phase1({}), make_phase2({})], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, mocked=True)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "input-identities"), FAIL)
|
||||||
|
self.assertEqual(status_of(journey.results, "journey-complete"), FAIL)
|
||||||
|
self.assertEqual(transport.boots, [])
|
||||||
|
|
||||||
|
|
||||||
|
class RestorationMismatchTests(unittest.TestCase):
|
||||||
|
def _journey_with_phase2(self, configure):
|
||||||
|
state = {}
|
||||||
|
phase1 = make_phase1(state)
|
||||||
|
phase2 = make_phase2(state)
|
||||||
|
configure(phase2)
|
||||||
|
transport = FakeTransport([phase1, phase2], make_recovery())
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||||
|
)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
return journey
|
||||||
|
|
||||||
|
def test_wrong_restored_generation_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
phase2.on("cat /proc/cmdline", lambda c: CommandResult(c, stdout="rootflags=subvol=@\n"))
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "restored-root-generation"), FAIL)
|
||||||
|
self.assertEqual(journey.results.exit_code(), 1)
|
||||||
|
|
||||||
|
def test_wrong_var_generation_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
fstab = "/dev/mapper/luks-root / btrfs subvol=/@restore-20260101T000000 0 0\n"
|
||||||
|
phase2.on("cat /etc/fstab", lambda c: CommandResult(c, stdout=fstab))
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "restored-var-generation"), FAIL)
|
||||||
|
|
||||||
|
def test_package_db_mismatch_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
phase2.on(
|
||||||
|
"find /var/db/pkg",
|
||||||
|
lambda c: CommandResult(c, stdout="\n".join(BASE_PKGS + [ADDED_PKG]) + "\n"),
|
||||||
|
)
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "package-db-restored"), FAIL)
|
||||||
|
|
||||||
|
def test_surviving_package_file_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
phase2.on("test -e", lambda c: CommandResult(c, exit_code=0))
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "package-files-restored"), FAIL)
|
||||||
|
|
||||||
|
def test_kernel_without_modules_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
phase2.on("cat /lib/modules", lambda c: CommandResult(c, stdout="5.0.0-other\n"))
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "kernel-modules-compatible"), FAIL)
|
||||||
|
|
||||||
|
def test_lost_home_marker_fails(self):
|
||||||
|
def configure(phase2):
|
||||||
|
phase2.on("cat /home/phoebus/rollback-acceptance/marker", lambda c: CommandResult(c, stdout=""))
|
||||||
|
|
||||||
|
journey = self._journey_with_phase2(configure)
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.results, "home-data-survives"), FAIL)
|
||||||
|
|
||||||
|
def test_recovery_with_network_fails(self):
|
||||||
|
state = {}
|
||||||
|
recovery = make_recovery()
|
||||||
|
recovery.on(
|
||||||
|
"ip -brief addr",
|
||||||
|
lambda c: CommandResult(c, stdout="lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"),
|
||||||
|
)
|
||||||
|
transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery)
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||||
|
)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.recovery_results, "recovery-no-network"), FAIL)
|
||||||
|
|
||||||
|
def test_recovery_without_no_model_fallback_fails(self):
|
||||||
|
state = {}
|
||||||
|
recovery = make_recovery()
|
||||||
|
recovery.on("sys ask", lambda c: CommandResult(c, exit_code=0, stdout="here is a plan"))
|
||||||
|
transport = FakeTransport([make_phase1(state), make_phase2(state)], recovery)
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||||
|
)
|
||||||
|
state["token"] = journey.marker_token
|
||||||
|
self.assertFalse(journey.run())
|
||||||
|
self.assertEqual(status_of(journey.recovery_results, "recovery-no-model-fallback"), FAIL)
|
||||||
|
|
||||||
|
|
||||||
|
class RecoveryOnlyTests(unittest.TestCase):
|
||||||
|
def test_recovery_only_passes(self):
|
||||||
|
transport = FakeTransport([], make_recovery())
|
||||||
|
journey = RollbackJourney(
|
||||||
|
transport, iso_sha256="a" * 64, sys_sha256="b" * 64, mocked=True
|
||||||
|
)
|
||||||
|
self.assertTrue(journey.run_recovery())
|
||||||
|
self.assertEqual(journey.recovery_results.exit_code(), 0)
|
||||||
|
self.assertEqual(transport.boots, ["recovery"])
|
||||||
|
|
||||||
|
def test_recovery_only_without_inputs_does_not_boot(self):
|
||||||
|
transport = FakeTransport([], make_recovery())
|
||||||
|
journey = RollbackJourney(transport, mocked=True)
|
||||||
|
self.assertFalse(journey.run_recovery())
|
||||||
|
self.assertEqual(transport.boots, [])
|
||||||
|
self.assertEqual(
|
||||||
|
status_of(journey.recovery_results, "recovery-boot-no-network-no-model"),
|
||||||
|
NOT_RUN,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SafetyTests(unittest.TestCase):
|
||||||
|
def test_rejects_dev_disk(self):
|
||||||
|
with self.assertRaises(UnsafeDiskError):
|
||||||
|
validate_acceptance_disks(["/dev/sda"], "/var/tmp/phoebus-vm/disks")
|
||||||
|
|
||||||
|
def test_rejects_disk_outside_test_dir(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
outside = Path(tmp).parent / "rollback-outside.qcow2"
|
||||||
|
outside.write_bytes(b"QFI\xfb" + b"\x00" * 64)
|
||||||
|
self.addCleanup(outside.unlink)
|
||||||
|
with self.assertRaises(UnsafeDiskError):
|
||||||
|
validate_acceptance_disks([outside], tmp)
|
||||||
|
|
||||||
|
|
||||||
|
class ParserTests(unittest.TestCase):
|
||||||
|
def test_newest_committed_coordinated_skips_unpaired(self):
|
||||||
|
records = ra.parse_snapshot_history(
|
||||||
|
HISTORY
|
||||||
|
+ """
|
||||||
|
[[records]]
|
||||||
|
id = "snap-varless"
|
||||||
|
created_unix = 2000
|
||||||
|
operation = "risk"
|
||||||
|
source_subvol = "/"
|
||||||
|
snapshot_subvol = "@-20260102T000000-risk"
|
||||||
|
status = "committed"
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
chosen = ra.newest_committed_coordinated(records)
|
||||||
|
self.assertIsNotNone(chosen)
|
||||||
|
self.assertEqual(chosen.id, "snap-initial")
|
||||||
|
|
||||||
|
def test_parse_rootflags_subvol(self):
|
||||||
|
self.assertEqual(
|
||||||
|
ra.parse_rootflags_subvol("rootflags=subvol=@restore-x quiet"), "@restore-x"
|
||||||
|
)
|
||||||
|
self.assertEqual(ra.parse_rootflags_subvol("quiet"), None)
|
||||||
|
|
||||||
|
def test_parse_fstab_subvol(self):
|
||||||
|
text = "/dev/x / btrfs subvol=/@restore-y 0 0\n"
|
||||||
|
self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y")
|
||||||
|
self.assertIsNone(ra.parse_fstab_subvol(text, "/var"))
|
||||||
|
|
||||||
|
def test_parse_fstab_subvol_without_leading_slash(self):
|
||||||
|
text = "/dev/x / btrfs subvol=@restore-y 0 0\n"
|
||||||
|
self.assertEqual(ra.parse_fstab_subvol(text, "/"), "@restore-y")
|
||||||
|
|
||||||
|
def test_parse_contents_paths(self):
|
||||||
|
paths = ra.parse_contents_files("obj /usr/bin/hello abc\ndir /usr/share/hello def\n")
|
||||||
|
self.assertEqual(paths, ["/usr/bin/hello", "/usr/share/hello"])
|
||||||
|
|
||||||
|
def test_parse_network_addresses(self):
|
||||||
|
self.assertEqual(ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\n"), ["lo"])
|
||||||
|
self.assertEqual(
|
||||||
|
ra.parse_network_addresses("lo UNKNOWN 127.0.0.1/8\nenp0s2 UP 10.0.2.15/24\n"),
|
||||||
|
["lo", "enp0s2"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_unreadable_history_raises(self):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
ra.parse_snapshot_history("not = = toml [[")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
|
|
@ -9,6 +9,7 @@ in CI, not only during a live install:
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import socket
|
import socket
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
@ -16,9 +17,12 @@ import tempfile
|
||||||
import threading
|
import threading
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
import vmsafety
|
import vmsafety
|
||||||
from vmsafety import (
|
from vmsafety import (
|
||||||
|
ARTIFACT_ENV_NAME,
|
||||||
|
ARTIFACT_ENV_SHA256,
|
||||||
BLOCKED,
|
BLOCKED,
|
||||||
FAIL,
|
FAIL,
|
||||||
PASS,
|
PASS,
|
||||||
|
|
@ -166,6 +170,53 @@ class ResultsTests(unittest.TestCase):
|
||||||
return Path(tmp.name)
|
return Path(tmp.name)
|
||||||
|
|
||||||
|
|
||||||
|
class ArtifactBindingTests(unittest.TestCase):
|
||||||
|
"""The release tool binds evidence to a candidate digest; a harness run
|
||||||
|
must record the artifact it verified, or a stale pass could be replayed."""
|
||||||
|
|
||||||
|
def _tmp_results(self):
|
||||||
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
|
||||||
|
tmp.close()
|
||||||
|
self.addCleanup(os.unlink, tmp.name)
|
||||||
|
return Path(tmp.name)
|
||||||
|
|
||||||
|
def test_artifact_from_env_is_written(self):
|
||||||
|
with mock.patch.dict(
|
||||||
|
os.environ,
|
||||||
|
{ARTIFACT_ENV_NAME: "phoebus.iso", ARTIFACT_ENV_SHA256: "abc123"},
|
||||||
|
):
|
||||||
|
results = Results("unit")
|
||||||
|
results.pass_("one")
|
||||||
|
# write() is what records the binding, so exercise it directly.
|
||||||
|
path = results.write(self._tmp_results())
|
||||||
|
data = json.loads(path.read_text())
|
||||||
|
self.assertEqual(data["artifact"], {"name": "phoebus.iso", "sha256": "abc123"})
|
||||||
|
|
||||||
|
def test_artifact_absent_is_null(self):
|
||||||
|
clean = {
|
||||||
|
k: v
|
||||||
|
for k, v in os.environ.items()
|
||||||
|
if k not in (ARTIFACT_ENV_NAME, ARTIFACT_ENV_SHA256)
|
||||||
|
}
|
||||||
|
with mock.patch.dict(os.environ, clean, clear=True):
|
||||||
|
results = Results("unit")
|
||||||
|
results.pass_("one")
|
||||||
|
data = json.loads(results.write(self._tmp_results()).read_text())
|
||||||
|
self.assertIsNone(data["artifact"])
|
||||||
|
|
||||||
|
def test_explicit_binding_is_not_overridden_by_env(self):
|
||||||
|
with mock.patch.dict(
|
||||||
|
os.environ,
|
||||||
|
{ARTIFACT_ENV_NAME: "env.iso", ARTIFACT_ENV_SHA256: "env-digest"},
|
||||||
|
):
|
||||||
|
results = Results("unit")
|
||||||
|
results.bind_artifact("explicit.iso", "explicit-digest")
|
||||||
|
data = json.loads(results.write(self._tmp_results()).read_text())
|
||||||
|
self.assertEqual(
|
||||||
|
data["artifact"], {"name": "explicit.iso", "sha256": "explicit-digest"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FakeQmpServer(threading.Thread):
|
class FakeQmpServer(threading.Thread):
|
||||||
"""A tiny QMP server that records HMP commands and answers them."""
|
"""A tiny QMP server that records HMP commands and answers them."""
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,7 @@ host (see ``test_vmsafety.py``).
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import dataclasses
|
import dataclasses
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
|
@ -43,7 +44,26 @@ BLOCKED = "BLOCKED"
|
||||||
SKIPPED = "SKIPPED"
|
SKIPPED = "SKIPPED"
|
||||||
NOT_RUN = "NOT RUN"
|
NOT_RUN = "NOT RUN"
|
||||||
|
|
||||||
_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED)
|
_TERMINAL = (PASS, FAIL, BLOCKED, SKIPPED, NOT_RUN)
|
||||||
|
|
||||||
|
# The candidate artifact a harness run is executed against. The release tool
|
||||||
|
# binds a gate report to a specific digest; recording it here is what lets a
|
||||||
|
# later verification step refuse stale evidence rather than trust any pass.
|
||||||
|
ARTIFACT_ENV_NAME = "PHOEBUS_VM_ARTIFACT"
|
||||||
|
ARTIFACT_ENV_SHA256 = "PHOEBUS_VM_ARTIFACT_SHA256"
|
||||||
|
|
||||||
|
# How a result set was produced. Only a `live` run is real release evidence; a
|
||||||
|
# `mocked` run exercises the orchestration on fakes and must never be imported
|
||||||
|
# as a gate pass. The release tool checks this field and refuses anything else.
|
||||||
|
LIVE = "live"
|
||||||
|
MOCKED = "mocked"
|
||||||
|
|
||||||
|
# The additional inputs a run may be bound to. The rollback acceptance journey
|
||||||
|
# pins both the bootable ISO and the `sys` binary it exercised, because the
|
||||||
|
# rollback semantics live in `sys`; a report that names only the ISO cannot say
|
||||||
|
# which broker was tested.
|
||||||
|
INPUT_ISO = "iso"
|
||||||
|
INPUT_SYS = "sys"
|
||||||
|
|
||||||
|
|
||||||
class UnsafeDiskError(Exception):
|
class UnsafeDiskError(Exception):
|
||||||
|
|
@ -81,10 +101,67 @@ class Results:
|
||||||
self.scenario = scenario
|
self.scenario = scenario
|
||||||
self.run_id = run_id or new_run_id(scenario)
|
self.run_id = run_id or new_run_id(scenario)
|
||||||
self.checks: list[Check] = []
|
self.checks: list[Check] = []
|
||||||
|
# The candidate artifact this run verifies (name + sha256), if known.
|
||||||
|
self.artifact: dict | None = None
|
||||||
|
# Additional pinned inputs (e.g. the tested ISO and `sys` binary), keyed
|
||||||
|
# by name. `artifact` remains the single release candidate for backwards
|
||||||
|
# compatibility; `inputs` carries the wider identity set.
|
||||||
|
self.inputs: dict[str, dict] = {}
|
||||||
|
# `live` for a real run; `mocked` for a fake-transport unit run. A
|
||||||
|
# mocked run is never valid release evidence.
|
||||||
|
self.evidence_class = LIVE
|
||||||
if strict is None:
|
if strict is None:
|
||||||
strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true")
|
strict = os.environ.get("PHOEBUS_VM_STRICT", "").lower() in ("1", "yes", "true")
|
||||||
self.strict = bool(strict)
|
self.strict = bool(strict)
|
||||||
|
|
||||||
|
# -- candidate artifact binding ---------------------------------------
|
||||||
|
def bind_artifact(self, name: str, sha256: str) -> dict:
|
||||||
|
"""Record the candidate artifact (name + digest) this run verified."""
|
||||||
|
self.artifact = {"name": name, "sha256": sha256}
|
||||||
|
return self.artifact
|
||||||
|
|
||||||
|
def bind_artifact_from_env(self) -> dict | None:
|
||||||
|
"""Bind from ``PHOEBUS_VM_ARTIFACT[_SHA256]`` when both are present.
|
||||||
|
|
||||||
|
The drivers call this through :meth:`write`, so a run always records the
|
||||||
|
digest of the candidate it exercised (when the launcher supplied it).
|
||||||
|
"""
|
||||||
|
name = os.environ.get(ARTIFACT_ENV_NAME)
|
||||||
|
sha256 = os.environ.get(ARTIFACT_ENV_SHA256)
|
||||||
|
if name and sha256:
|
||||||
|
return self.bind_artifact(name, sha256)
|
||||||
|
return self.artifact
|
||||||
|
|
||||||
|
# -- additional pinned inputs -----------------------------------------
|
||||||
|
def mark_mocked(self, reason: str = "") -> None:
|
||||||
|
"""Declare this run a mocked one, so it can never be release evidence."""
|
||||||
|
self.evidence_class = MOCKED
|
||||||
|
if reason:
|
||||||
|
self.inputs.setdefault("mocked_reason", {})["reason"] = reason
|
||||||
|
|
||||||
|
def bind_input(
|
||||||
|
self, name: str, sha256: str, size_bytes: int | None = None
|
||||||
|
) -> dict:
|
||||||
|
"""Pin an input by digest (e.g. the ISO or the `sys` binary)."""
|
||||||
|
entry: dict = {"sha256": sha256}
|
||||||
|
if size_bytes is not None:
|
||||||
|
entry["size_bytes"] = int(size_bytes)
|
||||||
|
self.inputs[name] = entry
|
||||||
|
return entry
|
||||||
|
|
||||||
|
def bind_input_file(self, name: str, path: str | os.PathLike) -> dict:
|
||||||
|
"""Hash a local file and pin it as an input."""
|
||||||
|
path = Path(path)
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with open(path, "rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return self.bind_input(name, digest.hexdigest(), path.stat().st_size)
|
||||||
|
|
||||||
|
def missing_inputs(self, names) -> list[str]:
|
||||||
|
"""Input names that were required but not bound."""
|
||||||
|
return [name for name in names if name not in self.inputs]
|
||||||
|
|
||||||
# -- recording ---------------------------------------------------------
|
# -- recording ---------------------------------------------------------
|
||||||
def _add(self, name: str, status: str, detail: str) -> Check:
|
def _add(self, name: str, status: str, detail: str) -> Check:
|
||||||
if status not in _TERMINAL:
|
if status not in _TERMINAL:
|
||||||
|
|
@ -114,6 +191,14 @@ class Results:
|
||||||
def skip(self, name: str, detail: str = "") -> Check:
|
def skip(self, name: str, detail: str = "") -> Check:
|
||||||
return self._add(name, SKIPPED, detail)
|
return self._add(name, SKIPPED, detail)
|
||||||
|
|
||||||
|
def mark_not_run(self, name: str, detail: str = "") -> Check:
|
||||||
|
"""Record a step that was expected but never reached.
|
||||||
|
|
||||||
|
A stopped journey must not look like a pass, so unexecuted steps are
|
||||||
|
visible rather than simply absent.
|
||||||
|
"""
|
||||||
|
return self._add(name, NOT_RUN, detail)
|
||||||
|
|
||||||
# -- outcome -----------------------------------------------------------
|
# -- outcome -----------------------------------------------------------
|
||||||
@property
|
@property
|
||||||
def failed(self) -> bool:
|
def failed(self) -> bool:
|
||||||
|
|
@ -148,12 +233,17 @@ class Results:
|
||||||
def write(self, path: str | Path) -> Path:
|
def write(self, path: str | Path) -> Path:
|
||||||
path = Path(path)
|
path = Path(path)
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if self.artifact is None:
|
||||||
|
self.bind_artifact_from_env()
|
||||||
payload = {
|
payload = {
|
||||||
"scenario": self.scenario,
|
"scenario": self.scenario,
|
||||||
"run_id": self.run_id,
|
"run_id": self.run_id,
|
||||||
"strict": self.strict,
|
"strict": self.strict,
|
||||||
|
"evidence": self.evidence_class,
|
||||||
"exit_code": self.exit_code(),
|
"exit_code": self.exit_code(),
|
||||||
"counts": self.counts(),
|
"counts": self.counts(),
|
||||||
|
"artifact": self.artifact,
|
||||||
|
"inputs": self.inputs,
|
||||||
"checks": [c.to_dict() for c in self.checks],
|
"checks": [c.to_dict() for c in self.checks],
|
||||||
}
|
}
|
||||||
path.write_text(json.dumps(payload, indent=2) + "\n")
|
path.write_text(json.dumps(payload, indent=2) + "\n")
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue