ci: enforce a shared strict repository quality gate
This commit is contained in:
@@ -15,15 +15,25 @@ jobs:
|
|||||||
|
|
||||||
- uses: dtolnay/rust-toolchain@stable
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
with:
|
with:
|
||||||
components: rustfmt, clippy
|
components: rustfmt, clippy, llvm-tools-preview
|
||||||
|
|
||||||
- uses: Swatinem/rust-cache@v2
|
- uses: Swatinem/rust-cache@v2
|
||||||
|
|
||||||
- name: Format check
|
- name: Discard reports from previous runs
|
||||||
run: cargo fmt -- --check
|
run: rm -rf target/quality-gate target/llvm-cov
|
||||||
|
|
||||||
- name: Clippy
|
- name: Provision quality-gate tools and native dependencies
|
||||||
run: cargo clippy --workspace --all-features
|
run: bash scripts/provision-quality-gate.sh
|
||||||
|
|
||||||
- name: Test
|
- name: Repository quality gate
|
||||||
run: cargo test --workspace --all-targets --all-features --no-fail-fast
|
run: make ci-domains
|
||||||
|
|
||||||
|
- name: Publish gate evidence (including failures)
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: quality-gate-${{ runner.os }}-${{ github.sha }}
|
||||||
|
path: |
|
||||||
|
target/quality-gate/**
|
||||||
|
target/llvm-cov/**
|
||||||
|
if-no-files-found: warn
|
||||||
|
|||||||
@@ -1,19 +1,19 @@
|
|||||||
.PHONY: fmt fmt-check clippy test-local test-debugger-socket clean coverage coverage-xml coverage-json coverage-report-json coverage-domain-list coverage-domain-evidence coverage-domain-audit coverage-domain-gate test ci ci-domains cobertura
|
.PHONY: fmt fmt-check clippy test-local test-debugger-socket clean coverage coverage-xml coverage-json coverage-report-json coverage-domain-list coverage-domain-evidence coverage-domain-audit coverage-domain-gate test ci ci-domains test-quality-gate cobertura
|
||||||
|
|
||||||
fmt:
|
fmt:
|
||||||
cargo fmt
|
cargo fmt --all
|
||||||
|
|
||||||
fmt-check:
|
fmt-check:
|
||||||
cargo fmt -- --check
|
cargo fmt --all -- --check
|
||||||
|
|
||||||
clippy:
|
clippy:
|
||||||
cargo clippy --workspace --all-features
|
cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||||
|
|
||||||
test-local:
|
test-local:
|
||||||
cargo test --workspace --all-targets --all-features --no-fail-fast
|
cargo test --workspace --all-targets --all-features --no-fail-fast
|
||||||
|
|
||||||
test-debugger-socket:
|
test-debugger-socket:
|
||||||
cargo test -p prometeu-host-desktop-winit --lib -- --ignored
|
cargo test -p prometeu-host-desktop-winit --lib runner::tests -- --ignored
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
cargo llvm-cov clean --workspace
|
cargo llvm-cov clean --workspace
|
||||||
@@ -47,6 +47,14 @@ coverage-domain-gate:
|
|||||||
./scripts/coverage-domain-evidence.sh --check-all
|
./scripts/coverage-domain-evidence.sh --check-all
|
||||||
|
|
||||||
test: fmt-check clippy test-local test-debugger-socket
|
test: fmt-check clippy test-local test-debugger-socket
|
||||||
ci: clean fmt-check clippy coverage coverage-report-json
|
# Keep the step list here: CI providers consume the repository contract.
|
||||||
ci-domains: clean fmt-check clippy coverage coverage-report-json coverage-xml coverage-json coverage-domain-gate
|
# The runner executes each step serially even if the caller supplies make -j.
|
||||||
|
CI_DOMAIN_STEPS = clean fmt-check clippy test-quality-gate test-local test-debugger-socket coverage coverage-report-json coverage-xml coverage-json coverage-domain-audit coverage-domain-gate
|
||||||
|
|
||||||
|
ci: ci-domains
|
||||||
|
ci-domains:
|
||||||
|
python3 scripts/run-quality-gate.py $(CI_DOMAIN_STEPS)
|
||||||
|
|
||||||
|
test-quality-gate:
|
||||||
|
python3 -B -m unittest discover -s scripts/tests -p 'test_quality_gate.py'
|
||||||
cobertura: coverage-xml coverage-json
|
cobertura: coverage-xml coverage-json
|
||||||
|
|||||||
@@ -107,19 +107,19 @@ Format, lint, and test the workspace:
|
|||||||
make test
|
make test
|
||||||
```
|
```
|
||||||
|
|
||||||
Run the CI-oriented non-domain gate:
|
Run the shared strict CI gate (`make ci` is an alias):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make ci
|
make ci
|
||||||
```
|
```
|
||||||
|
|
||||||
Run the CI-oriented domain coverage gate:
|
The authoritative target used by both GitHub and Jenkins is:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make ci-domains
|
make ci-domains
|
||||||
```
|
```
|
||||||
|
|
||||||
The debugger socket tests bind localhost sockets and are marked ignored in Rust test metadata; `make test` includes them through the dedicated `test-debugger-socket` target.
|
The five debugger socket tests bind localhost sockets and are marked ignored in Rust test metadata; both `make test` and `make ci-domains` select only `runner::tests` through `test-debugger-socket`. Physical audio and other manual probes remain opt-in. See [quality-gate prerequisites and commands](docs/vm-arch/COVERAGE_DOMAINS.md#commands-and-ci).
|
||||||
|
|
||||||
## Coverage Governance
|
## Coverage Governance
|
||||||
|
|
||||||
@@ -128,20 +128,14 @@ The repository uses a two-layer runtime-edge coverage model:
|
|||||||
- a mandatory CI gate based on canonical runtime-edge domains;
|
- a mandatory CI gate based on canonical runtime-edge domains;
|
||||||
- domain-oriented review evidence for the canonical runtime-edge domains.
|
- domain-oriented review evidence for the canonical runtime-edge domains.
|
||||||
|
|
||||||
Current canonical domains:
|
The fifteen canonical domains and fixed floors are defined in
|
||||||
|
[the repository configuration](files/config/runtime-edge-coverage-domains.json).
|
||||||
- `system/runtime`
|
Use `make coverage-domain-list` or the [ownership documentation](docs/vm-arch/COVERAGE_DOMAINS.md)
|
||||||
- `fs`
|
to inspect the current map.
|
||||||
- `asset/bank`
|
|
||||||
- `firmware`
|
|
||||||
- `host-dependent`
|
|
||||||
- `host/headless`
|
|
||||||
|
|
||||||
Global coverage artifacts:
|
Global coverage artifacts:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make ci
|
|
||||||
make cobertura
|
|
||||||
make ci-domains
|
make ci-domains
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -151,6 +145,7 @@ This produces:
|
|||||||
- `target/llvm-cov/report.json`
|
- `target/llvm-cov/report.json`
|
||||||
- `target/llvm-cov/summary.json`
|
- `target/llvm-cov/summary.json`
|
||||||
- `target/llvm-cov/cobertura.xml`
|
- `target/llvm-cov/cobertura.xml`
|
||||||
|
- `target/quality-gate/metadata.json` and per-step logs
|
||||||
|
|
||||||
Domain-oriented evidence helpers:
|
Domain-oriented evidence helpers:
|
||||||
|
|
||||||
@@ -161,12 +156,12 @@ make coverage-domain-evidence DOMAIN=host-dependent
|
|||||||
make coverage-domain-evidence DOMAIN=host/headless
|
make coverage-domain-evidence DOMAIN=host/headless
|
||||||
```
|
```
|
||||||
|
|
||||||
Per-domain baselines currently start around `60`.
|
`make ci-domains` runs formatting, strict workspace/all-target/all-feature Clippy,
|
||||||
That does not waive test obligations. It means the project is adopting explicit domain evidence immediately while tightening quantitative expectations incrementally over time.
|
all-target tests, the five debugger socket tests, fresh coverage and every domain
|
||||||
|
ownership/evidence gate. Both CI providers archive available evidence on failure.
|
||||||
`make ci-domains` is the CI entrypoint used by Jenkins.
|
A failing run never lowers a configured floor; local and CI checks use the same
|
||||||
It runs the standard coverage pipeline, produces the HTML/XML/JSON artifacts, and fails when any canonical domain falls below its configured `baseline_percent`.
|
repository-owned commands. Reports identify their revision, source/configuration
|
||||||
The CI gate currently compares domain `lines` coverage against each domain baseline.
|
and actual execution platform.
|
||||||
|
|
||||||
When a PR changes the observable contract of a canonical domain, it is expected to:
|
When a PR changes the observable contract of a canonical domain, it is expected to:
|
||||||
|
|
||||||
|
|||||||
@@ -1263,6 +1263,18 @@ pub(crate) fn metered_lookup_intrinsic(
|
|||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Runtime-owned catalogue, including input and scalar math.
|
||||||
|
pub fn intrinsic_catalog() -> impl Iterator<Item = &'static IntrinsicMeta> {
|
||||||
|
INTRINSICS
|
||||||
|
.iter()
|
||||||
|
.chain(crate::input_intrinsics::INPUT_INTRINSICS.iter())
|
||||||
|
.chain(crate::math_intrinsics::MATH_INTRINSICS.iter())
|
||||||
|
.chain(crate::random_intrinsics::RANDOM_INTRINSICS.iter())
|
||||||
|
}
|
||||||
|
pub fn builtin_constant_catalog() -> impl Iterator<Item = &'static BuiltinConstMeta> {
|
||||||
|
BUILTIN_CONSTS.iter()
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -1432,15 +1444,3 @@ mod tests {
|
|||||||
assert_eq!(pixel.field("color").map(|field| field.start_slot), Some(2));
|
assert_eq!(pixel.field("color").map(|field| field.start_slot), Some(2));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Runtime-owned catalogue, including input and scalar math.
|
|
||||||
pub fn intrinsic_catalog() -> impl Iterator<Item = &'static IntrinsicMeta> {
|
|
||||||
INTRINSICS
|
|
||||||
.iter()
|
|
||||||
.chain(crate::input_intrinsics::INPUT_INTRINSICS.iter())
|
|
||||||
.chain(crate::math_intrinsics::MATH_INTRINSICS.iter())
|
|
||||||
.chain(crate::random_intrinsics::RANDOM_INTRINSICS.iter())
|
|
||||||
}
|
|
||||||
pub fn builtin_constant_catalog() -> impl Iterator<Item = &'static BuiltinConstMeta> {
|
|
||||||
BUILTIN_CONSTS.iter()
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -61,7 +61,9 @@ fn integer_limits_are_exact_and_abs_does_not_wrap() {
|
|||||||
] {
|
] {
|
||||||
assert_eq!(call(base, &[b.clone(), a.clone()]).unwrap(), a);
|
assert_eq!(call(base, &[b.clone(), a.clone()]).unwrap(), a);
|
||||||
assert_eq!(call(base + 1, &[a.clone(), b.clone()]).unwrap(), b);
|
assert_eq!(call(base + 1, &[a.clone(), b.clone()]).unwrap(), b);
|
||||||
assert!(matches!(call(base+3,&[a.clone()]),Err(E::Numeric(e)) if e.code==RANGE));
|
assert!(
|
||||||
|
matches!(call(base+3,std::slice::from_ref(&a)),Err(E::Numeric(e)) if e.code==RANGE)
|
||||||
|
);
|
||||||
assert!(matches!(call(base+2,&[b.clone(),b,a]),Err(E::Numeric(e)) if e.code==DOMAIN));
|
assert!(matches!(call(base+2,&[b.clone(),b,a]),Err(E::Numeric(e)) if e.code==DOMAIN));
|
||||||
}
|
}
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ fn independent_words_signed_and_float_bits_for_all_seed_patterns() {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
code.push_str("RET");
|
code.push_str("RET");
|
||||||
let mut v = vm(&[(&code, sig(&[], &vec![if floating { 2 } else { 0 }; 8]), vec![], 1)]);
|
let mut v = vm(&[(&code, sig(&[], &[if floating { 2 } else { 0 }; 8]), vec![], 1)]);
|
||||||
let r = run(&mut v, 10000);
|
let r = run(&mut v, 10000);
|
||||||
assert_eq!(r.reason, End::EndOfRom);
|
assert_eq!(r.reason, End::EndOfRom);
|
||||||
let expected: Vec<_> = vectors(seed)
|
let expected: Vec<_> = vectors(seed)
|
||||||
|
|||||||
@@ -837,7 +837,7 @@ mod tests {
|
|||||||
// real socket bind/connect coordination. Deterministic debugger state changes
|
// real socket bind/connect coordination. Deterministic debugger state changes
|
||||||
// are covered below the host layer in `debugger.rs`.
|
// are covered below the host layer in `debugger.rs`.
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "requires localhost TCP bind/connect; run via `cargo test -p prometeu-host-desktop-winit --lib -- --ignored`"]
|
#[ignore = "requires localhost TCP bind/connect; run via `make test-debugger-socket`"]
|
||||||
fn test_debug_port_opens() {
|
fn test_debug_port_opens() {
|
||||||
let mut runner = HostRunner::new(None, None);
|
let mut runner = HostRunner::new(None, None);
|
||||||
let port = 9999;
|
let port = 9999;
|
||||||
@@ -897,7 +897,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "requires localhost TCP bind/connect; run via `cargo test -p prometeu-host-desktop-winit --lib -- --ignored`"]
|
#[ignore = "requires localhost TCP bind/connect; run via `make test-debugger-socket`"]
|
||||||
fn test_debug_reconnection() {
|
fn test_debug_reconnection() {
|
||||||
let mut runner = HostRunner::new(None, None);
|
let mut runner = HostRunner::new(None, None);
|
||||||
let port = 9998;
|
let port = 9998;
|
||||||
@@ -940,7 +940,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "requires localhost TCP bind/connect; run via `cargo test -p prometeu-host-desktop-winit --lib -- --ignored`"]
|
#[ignore = "requires localhost TCP bind/connect; run via `make test-debugger-socket`"]
|
||||||
fn test_debug_refuse_second_connection() {
|
fn test_debug_refuse_second_connection() {
|
||||||
let mut runner = HostRunner::new(None, None);
|
let mut runner = HostRunner::new(None, None);
|
||||||
let port = 9997;
|
let port = 9997;
|
||||||
@@ -976,7 +976,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "requires localhost TCP bind/connect; run via `cargo test -p prometeu-host-desktop-winit --lib -- --ignored`"]
|
#[ignore = "requires localhost TCP bind/connect; run via `make test-debugger-socket`"]
|
||||||
fn test_get_state_returns_response() {
|
fn test_get_state_returns_response() {
|
||||||
let mut runner = HostRunner::new(None, None);
|
let mut runner = HostRunner::new(None, None);
|
||||||
let port = 9996;
|
let port = 9996;
|
||||||
@@ -1021,7 +1021,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "requires localhost TCP bind/connect; run via `cargo test -p prometeu-host-desktop-winit --lib -- --ignored`"]
|
#[ignore = "requires localhost TCP bind/connect; run via `make test-debugger-socket`"]
|
||||||
fn test_debug_resume_on_disconnect() {
|
fn test_debug_resume_on_disconnect() {
|
||||||
let mut runner = HostRunner::new(None, None);
|
let mut runner = HostRunner::new(None, None);
|
||||||
let port = 9995;
|
let port = 9995;
|
||||||
|
|||||||
+37
@@ -142,3 +142,40 @@ must not be presented as completed strict workspace qualification.
|
|||||||
- Coverage gates that live in repository helpers are easier to evolve, audit, and keep consistent than CI-local logic.
|
- Coverage gates that live in repository helpers are easier to evolve, audit, and keep consistent than CI-local logic.
|
||||||
- Audit source ownership and evidence validity before comparing percentages.
|
- Audit source ownership and evidence validity before comparing percentages.
|
||||||
A passing percentage is useful only when its denominator has a clear owner.
|
A passing percentage is useful only when its denominator has a clear owner.
|
||||||
|
|
||||||
|
## A Shared Gate Must Share Failure Semantics Too
|
||||||
|
|
||||||
|
The strict-gate correction derives from DEC-0063. A common coverage helper did
|
||||||
|
not establish common CI policy while one pipeline only ran permissive lint/tests
|
||||||
|
and another produced domain reports. Both CI consumers now provision the declared
|
||||||
|
tools and invoke the same Makefile-owned `ci-domains` target. Strict Clippy covers
|
||||||
|
workspace targets and features with warnings denied; concrete lint fixes replace
|
||||||
|
warning suppression. The existing coverage map and floors remain unchanged.
|
||||||
|
|
||||||
|
The Makefile owns the ordered steps. A serial runner performs format, strict lint,
|
||||||
|
all-target tests, the five named debugger socket cases, fresh coverage/export and
|
||||||
|
all domain audits/gates. Serial child Make invocations avoid races between clean,
|
||||||
|
Cargo execution and report generation. A second independent gate in the same
|
||||||
|
checkout still requires operator coordination.
|
||||||
|
|
||||||
|
Failure publication is part of the contract. Before a run, old reports are
|
||||||
|
removed; every executed step receives a log, exit status, revision and source/config
|
||||||
|
identity. A later failure retains current partial reports. An early failure cannot
|
||||||
|
archive old HTML as current success. Both CI providers publish available evidence
|
||||||
|
on failure and preserve the gate's nonzero status. Regression fixtures exercise
|
||||||
|
failed execution, success and failure after partial report generation.
|
||||||
|
|
||||||
|
Ignored tests are not one qualification category. Five localhost socket tests
|
||||||
|
belong in automation, but physical audio, endurance, packaged installation and
|
||||||
|
manual timing/layout probes need their own named execution. Restricting the socket
|
||||||
|
filter prevents accidentally turning a device requirement into a CI prerequisite.
|
||||||
|
|
||||||
|
The local macOS/aarch64 strict gate passed 1,039 ordinary tests, five socket tests,
|
||||||
|
three runner regressions and all fifteen coverage gates; fresh coverage separately
|
||||||
|
replayed 1,039 tests. The ordinary phases reported thirteen ignored tests; five
|
||||||
|
of these ran in the separate socket step, while eight remained unselected. The initial sandbox
|
||||||
|
socket refusal is distinguished from the successful authorized run. Pipeline
|
||||||
|
configuration was inspected, but no remote or Linux gate was executed. Platform
|
||||||
|
equivalence concerns required commands, not identical measured coverage. See the
|
||||||
|
[published operational contract and evidence](../../../docs/vm-arch/COVERAGE_DOMAINS.md)
|
||||||
|
for prerequisites, manual commands and practical limits.
|
||||||
|
|||||||
@@ -119,30 +119,107 @@ fresh measurement.
|
|||||||
|
|
||||||
## Commands and CI
|
## Commands and CI
|
||||||
|
|
||||||
List domains, inspect source ownership and validate coverage evidence:
|
`make ci-domains` is the authoritative repository quality gate. Both
|
||||||
|
[GitHub CI](../../.github/workflows/ci.yml) and
|
||||||
|
[Jenkins](../../files/config/Jenkinsfile) provision dependencies and invoke that
|
||||||
|
same target. `make ci` is an alias. Invoke one gate per checkout at a time.
|
||||||
|
|
||||||
make coverage-domain-list
|
The Makefile supplies this ordered sequence to the serial evidence runner:
|
||||||
python3 scripts/audit-coverage-domains.py
|
|
||||||
make coverage-domain-audit
|
|
||||||
python3 scripts/audit-coverage-domains.py --report target/llvm-cov/report.json --json
|
|
||||||
|
|
||||||
Produce a fresh report and run all gates:
|
1. Clean LLVM coverage instrumentation from the previous run.
|
||||||
|
2. Check formatting with `cargo fmt --all -- --check`.
|
||||||
|
3. Run `cargo clippy --workspace --all-targets --all-features -- -D warnings`.
|
||||||
|
4. Test gate failure/report handling with the Python regression suite.
|
||||||
|
5. Run `cargo test --workspace --all-targets --all-features --no-fail-fast`.
|
||||||
|
6. Run the five ignored debugger socket tests through `make test-debugger-socket`.
|
||||||
|
7. Generate fresh workspace/all-feature LLVM coverage, HTML, full JSON,
|
||||||
|
Cobertura XML and summary JSON.
|
||||||
|
8. Audit source ownership/evidence and check every fixed coverage domain floor.
|
||||||
|
|
||||||
make ci-domains
|
The socket command is deliberately restricted:
|
||||||
|
|
||||||
The existing Jenkins configuration invokes this repository target. Its
|
```sh
|
||||||
coverage-domain-gate step also performs the ownership/evidence audit.
|
cargo test -p prometeu-host-desktop-winit --lib runner::tests -- --ignored
|
||||||
GitHub's workspace-test step discovers the quality regression tests; it
|
```
|
||||||
does not separately produce a domain coverage report.
|
|
||||||
|
|
||||||
Inspect a domain:
|
It selects `test_debug_port_opens`, `test_debug_reconnection`,
|
||||||
|
`test_debug_refuse_second_connection`, `test_get_state_returns_response` and
|
||||||
|
`test_debug_resume_on_disconnect`. It does not select the audio-device test.
|
||||||
|
Local sandbox TCP permissions must permit bind/connect; a permission denial is
|
||||||
|
not evidence of a debugger behavior regression.
|
||||||
|
|
||||||
make coverage-domain-evidence DOMAIN=vm/core
|
The runner executes child Make targets with `-j1` even if the outer invocation
|
||||||
scripts/coverage-domain-evidence.sh --check --verbose hal/core
|
uses `-j`. CI invokes `make ci-domains` serially. Do not run a separate Cargo,
|
||||||
scripts/coverage-domain-evidence.sh --check-all
|
coverage-clean or report process concurrently in the same checkout: serialized
|
||||||
|
steps inside one gate cannot protect against an independent process.
|
||||||
|
|
||||||
Dependencies are the existing Rust/LLVM coverage tooling, Bash, Git, jq and rg,
|
### Prerequisites and reports
|
||||||
plus Python 3 for the standard-library ownership auditor.
|
|
||||||
|
Both pipelines use `bash scripts/provision-quality-gate.sh`. A Debian/Ubuntu
|
||||||
|
agent needs root or sudo for apt provisioning, and an existing Rust/rustup
|
||||||
|
installation. GitHub installs stable Rust first; Jenkins must supply a supported
|
||||||
|
Rust toolchain and rustup in CARGO_HOME/bin. The script installs rustfmt, Clippy,
|
||||||
|
llvm-tools-preview and pinned cargo-llvm-cov 0.8.5 (`--locked`).
|
||||||
|
|
||||||
|
Linux packages include build-essential, pkg-config, libasound2-dev, libudev-dev,
|
||||||
|
libxkbcommon-dev, libwayland-dev, libx11-dev, libxrandr-dev, libxi-dev, Python 3,
|
||||||
|
jq, ripgrep, Git, curl and CA certificates. ALSA development files satisfy CPAL's
|
||||||
|
Linux build dependency; a running audio device is not needed. Non-Linux local
|
||||||
|
hosts must already provide native compiler/SDK tools, Bash, Make, pkg-config,
|
||||||
|
Python 3, jq, rg and Git. The provisioning script verifies these commands.
|
||||||
|
|
||||||
|
`target/quality-gate/metadata.json` records revision, dirty working tree, host,
|
||||||
|
Rust/coverage-tool versions, configuration/source fingerprints, ordered step
|
||||||
|
exit codes and report hashes. Each step has a separate combined-output log.
|
||||||
|
The runner stops on the first failed Make target and returns its nonzero exit
|
||||||
|
status; the log preserves underlying Cargo diagnostics. Three regression cases
|
||||||
|
check failure logs/identity, fresh success evidence and partial-report retention
|
||||||
|
when a later gate fails.
|
||||||
|
|
||||||
|
Both pipelines discard old generated report directories before provisioning and
|
||||||
|
publish current artifacts even after gate failure. An early failed gate has
|
||||||
|
metadata/logs but no old coverage report. Failure after generation preserves the
|
||||||
|
partial current reports. Provisioning failure can leave no report at all and
|
||||||
|
still fails CI. Reports are archived from `target/quality-gate/**` and
|
||||||
|
`target/llvm-cov/**`; Jenkins also publishes current HTML when present. Artifact
|
||||||
|
publication does not turn a failed gate into success.
|
||||||
|
|
||||||
|
List domains or inspect ownership and existing evidence:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make coverage-domain-list
|
||||||
|
python3 scripts/audit-coverage-domains.py
|
||||||
|
make coverage-domain-audit
|
||||||
|
make coverage-domain-evidence DOMAIN=vm/core
|
||||||
|
scripts/coverage-domain-evidence.sh --check --verbose hal/core
|
||||||
|
```
|
||||||
|
|
||||||
|
### Explicit manual qualification
|
||||||
|
|
||||||
|
Ordinary all-target tests and coverage leave ignored device/measurement tests
|
||||||
|
ignored. The shared target only opts in to the five socket tests above. It does
|
||||||
|
not certify the following manual workloads:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Physical desktop audio, exact named test.
|
||||||
|
cargo test -p prometeu-host-desktop-winit --lib audio::control_tests::desktop_audio_device_smoke -- --ignored --exact --nocapture
|
||||||
|
# Physical callback duration under host I/O pressure.
|
||||||
|
cargo test --release -p prometeu-host-desktop-winit --test audio_callback_allocations physical_callback_duration_16_voices_with_io_pressure -- --ignored --exact --nocapture
|
||||||
|
# Eight-hour active release soak.
|
||||||
|
cargo run --release -p prometeu-vm --features resource-diagnostics --example resource_qualification -- --seconds 28800
|
||||||
|
# Packaged installation; set PROMETEU_PROTOCOL_TEST_INSTALL to an extracted distribution first.
|
||||||
|
cargo test -p prometeu-cli --test automation_process packaged_installation_smoke -- --ignored --exact --nocapture
|
||||||
|
# File-host commit latency with the production asset decoder.
|
||||||
|
cargo test --release -p prometeu-system --all-features qualify_file_commits_with_production_asset_decoder -- --ignored --nocapture
|
||||||
|
# Explicit layout observation.
|
||||||
|
cargo test --release -p prometeu-bytecode --test registry_v1 reference_map_probe -- --ignored --exact --nocapture
|
||||||
|
```
|
||||||
|
|
||||||
|
Other ignored timing probes remain opt-in by their individual test names.
|
||||||
|
All-target execution can print ordinary benchmark observations; this does not
|
||||||
|
establish completion of ignored manual qualification. For scope and prior measured
|
||||||
|
results see [resource stability](resource-stability-qualification.md) and
|
||||||
|
[audio callback qualification](audio-callback-qualification.md).
|
||||||
|
|
||||||
## Historical local baseline
|
## Historical local baseline
|
||||||
|
|
||||||
@@ -194,9 +271,9 @@ The full local `make ci-domains` validation passed with 939 tests passed and
|
|||||||
in verification; its final floor uses the lower measurement. All gates were
|
in verification; its final floor uses the lower measurement. All gates were
|
||||||
rechecked against the verification report after that final adjustment.
|
rechecked against the verification report after that final adjustment.
|
||||||
|
|
||||||
## Current local measurement
|
## Ownership-repair measurement
|
||||||
|
|
||||||
The [current evidence](COVERAGE_DOMAINS_CURRENT.json) records the repaired
|
The [ownership-repair evidence](COVERAGE_DOMAINS_CURRENT.json) records the repaired
|
||||||
module-tree ownership, fixed floors, normalized source paths, actual line
|
module-tree ownership, fixed floors, normalized source paths, actual line
|
||||||
counts and source/config/report fingerprints. Both historical JSON records
|
counts and source/config/report fingerprints. Both historical JSON records
|
||||||
above remain unchanged. The measurement uses the implementation working tree
|
above remain unchanged. The measurement uses the implementation working tree
|
||||||
@@ -225,12 +302,42 @@ with 1,021 tests passed and 11 ignored. The 10 focused ownership/evidence
|
|||||||
tests and strict package Clippy also passed. No floor reduction or additional
|
tests and strict package Clippy also passed. No floor reduction or additional
|
||||||
runtime behavior test was needed to pass the repaired map.
|
runtime behavior test was needed to pass the repaired map.
|
||||||
|
|
||||||
The current Makefile target still uses non-strict workspace Clippy. A separate
|
At the time of that ownership measurement the target used non-strict workspace
|
||||||
`cargo clippy --workspace --all-targets --all-features -- -D warnings` run failed
|
Clippy, and a separate strict invocation failed on VM `useless_vec`,
|
||||||
on existing VM `useless_vec`, `cloned_ref_to_slice_refs` and
|
`cloned_ref_to_slice_refs` and `items_after_test_module` diagnostics. That historical
|
||||||
`items_after_test_module` diagnostics. This evidence certifies the corrected
|
record qualifies the repaired coverage map, not the later strict gate. The lint
|
||||||
coverage map and its gates; full strict workspace qualification remains pending.
|
sources have since been corrected without blanket suppression or floor changes.
|
||||||
|
|
||||||
Platform-specific compilation and host scenarios can change report totals.
|
Platform-specific compilation and host scenarios can change report totals.
|
||||||
Local evidence does not certify an unexecuted Linux or Intel Mac run; evaluate
|
Local evidence does not certify an unexecuted Linux or Intel Mac run; evaluate
|
||||||
those runs against the configured floors and investigate differences.
|
those runs against the configured floors and investigate differences.
|
||||||
|
|
||||||
|
## Strict shared-gate qualification
|
||||||
|
|
||||||
|
On 2026-10-14, macOS/aarch64 with rustc 1.98.1, the shared target passed strict
|
||||||
|
workspace/all-target/all-feature Clippy, all 1,039 ordinary tests (13 ignored),
|
||||||
|
the five explicitly selected socket tests, three gate-runner regression tests,
|
||||||
|
and all 15 fixed coverage domain gates. The separate fresh coverage execution
|
||||||
|
also passed 1,039 tests with 13 ignored. Those repeated runs are not distinct
|
||||||
|
behavioral tests and must not be added into a unique-test count.
|
||||||
|
|
||||||
|
The baseline strict invocation failed on the concrete VM lint sources above;
|
||||||
|
the repaired sources pass without new lint allowances. Socket tests initially
|
||||||
|
failed because the sandbox denied localhost TCP operations; the authorized
|
||||||
|
outside-sandbox run passed all five. Gate failure fixtures preserve nonzero
|
||||||
|
Make status, logs and commit/config identity, discard stale coverage, and retain
|
||||||
|
fresh partial reports when a later gate fails. Interrupted runners record failure
|
||||||
|
rather than success.
|
||||||
|
|
||||||
|
GitHub's YAML was parsed locally, and both pipeline definitions were inspected
|
||||||
|
for shared provisioning, `make ci-domains` and unconditional failure-artifact
|
||||||
|
publication. No remote GitHub or Jenkins build was launched. No Linux gate ran:
|
||||||
|
the local Docker API was denied by the sandbox and the unrestricted daemon query
|
||||||
|
did not return a usable result before interruption. Local macOS coverage does not
|
||||||
|
certify Linux compilation or domain percentages. Such failures must be investigated
|
||||||
|
on that platform without lowering the existing floors.
|
||||||
|
|
||||||
|
No physical audio, eight-hour soak, packaged-installation or ignored manual
|
||||||
|
latency/layout qualification was selected by this gate. Prior manual observations
|
||||||
|
remain separate evidence. Current executable metadata and reports are emitted
|
||||||
|
under `target/quality-gate/` and `target/llvm-cov/` by every invocation.
|
||||||
|
|||||||
@@ -29,11 +29,17 @@ cargo test --workspace --features prometeu-host-headless/resource-diagnostics
|
|||||||
cargo test --release --workspace --features prometeu-host-headless/resource-diagnostics
|
cargo test --release --workspace --features prometeu-host-headless/resource-diagnostics
|
||||||
cargo run --release -p prometeu-vm --features resource-diagnostics --example resource_qualification -- --frames 216000
|
cargo run --release -p prometeu-vm --features resource-diagnostics --example resource_qualification -- --frames 216000
|
||||||
cargo run --release -p prometeu-vm --features resource-diagnostics --example resource_qualification -- --seconds 28800
|
cargo run --release -p prometeu-vm --features resource-diagnostics --example resource_qualification -- --seconds 28800
|
||||||
cargo test --release -p prometeu-vm --features resource-diagnostics --lib -- --ignored --nocapture
|
cargo test --release -p prometeu-vm --features resource-diagnostics --lib sealed_image_load_probe -- --ignored --nocapture
|
||||||
|
cargo test --release -p prometeu-vm --features resource-diagnostics --lib storage_and_gc_probe -- --ignored --nocapture
|
||||||
cargo test --release -p prometeu-bytecode reference_map_probe -- --ignored --nocapture
|
cargo test --release -p prometeu-bytecode reference_map_probe -- --ignored --nocapture
|
||||||
cargo test -p prometeu-host-desktop-winit --lib -- --ignored
|
cargo test -p prometeu-host-desktop-winit --lib runner::tests -- --ignored
|
||||||
|
cargo test -p prometeu-host-desktop-winit --lib audio::control_tests::desktop_audio_device_smoke -- --ignored --exact --nocapture
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The reproduction commands name manual probes individually. The original recorded
|
||||||
|
desktop qualification grouped ignored tests; current automation selects only the
|
||||||
|
five debugger socket cases and leaves device audio explicit.
|
||||||
|
|
||||||
The `resource-diagnostics` feature adds engineering observations through Rust
|
The `resource-diagnostics` feature adds engineering observations through Rust
|
||||||
only. It adds no guest syscall. The eight-hour example is a manual command;
|
only. It adds no guest syscall. The eight-hour example is a manual command;
|
||||||
the repository CI, release workflow and Jenkins pipeline do not invoke it. Normal builds omit timing histograms, additional
|
the repository CI, release workflow and Jenkins pipeline do not invoke it. Normal builds omit timing histograms, additional
|
||||||
|
|||||||
Vendored
+34
-40
@@ -4,55 +4,49 @@ pipeline {
|
|||||||
environment {
|
environment {
|
||||||
CARGO_HOME = '/var/jenkins_home/.cargo'
|
CARGO_HOME = '/var/jenkins_home/.cargo'
|
||||||
CARGO_TARGET_DIR = 'target'
|
CARGO_TARGET_DIR = 'target'
|
||||||
|
|
||||||
CARGO_TERM_COLOR = 'always'
|
CARGO_TERM_COLOR = 'always'
|
||||||
}
|
}
|
||||||
|
|
||||||
stages {
|
stages {
|
||||||
stage('Build') {
|
stage('Prepare reports') {
|
||||||
|
steps {
|
||||||
|
dir('target/quality-gate') { deleteDir() }
|
||||||
|
dir('target/llvm-cov') { deleteDir() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stage('Provision') {
|
||||||
steps {
|
steps {
|
||||||
sh '''
|
sh '''
|
||||||
set -eux
|
set -eu
|
||||||
|
export PATH="$CARGO_HOME/bin:$PATH"
|
||||||
|
bash scripts/provision-quality-gate.sh
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stage('Quality gate') {
|
||||||
|
steps {
|
||||||
|
sh '''
|
||||||
|
set -eu
|
||||||
|
export PATH="$CARGO_HOME/bin:$PATH"
|
||||||
make ci-domains
|
make ci-domains
|
||||||
'''
|
'''
|
||||||
// withChecks(name: 'Test', includeStage: true) {
|
|
||||||
// sh '''
|
|
||||||
// set -eux
|
|
||||||
// make ci-domains
|
|
||||||
// '''
|
|
||||||
// recordCoverage(
|
|
||||||
// tools: [[parser: 'COBERTURA', pattern: 'target/llvm-cov/cobertura.xml']],
|
|
||||||
// sourceCodeRetention: 'LAST_BUILD',
|
|
||||||
// enabledForFailure: true,
|
|
||||||
// failOnError: false,
|
|
||||||
// checksAnnotationScope: 'MODIFIED_LINES',
|
|
||||||
// id: 'rust-coverage',
|
|
||||||
// name: 'Rust Coverage',
|
|
||||||
// checksName: 'Rust Coverage',
|
|
||||||
// qualityGates: [
|
|
||||||
// [metric: 'LINE', baseline: 'MODIFIED_LINES', threshold: 0.0],
|
|
||||||
// [metric: 'LINE', baseline: 'PROJECT', threshold: 0.0],
|
|
||||||
// [metric: 'BRANCH', baseline: 'PROJECT', threshold: 0.0]
|
|
||||||
// ]
|
|
||||||
// )
|
|
||||||
// }
|
|
||||||
}
|
}
|
||||||
} // Test
|
}
|
||||||
|
}
|
||||||
|
|
||||||
stage('Reports') {
|
post {
|
||||||
steps {
|
always {
|
||||||
publishHTML(target: [
|
archiveArtifacts artifacts: 'target/quality-gate/**,target/llvm-cov/**',
|
||||||
allowMissing: false,
|
fingerprint: true, allowEmptyArchive: true
|
||||||
alwaysLinkToLastBuild: true,
|
publishHTML(target: [
|
||||||
keepAll: true,
|
allowMissing: true,
|
||||||
reportDir: 'target/llvm-cov/html',
|
alwaysLinkToLastBuild: true,
|
||||||
reportFiles: 'index.html',
|
keepAll: true,
|
||||||
reportName: 'Rust Coverage HTML',
|
reportDir: 'target/llvm-cov/html',
|
||||||
reportTitles: 'Coverage Report'
|
reportFiles: 'index.html',
|
||||||
])
|
reportName: 'Rust Coverage HTML',
|
||||||
|
reportTitles: 'Coverage Report'
|
||||||
archiveArtifacts artifacts: 'target/llvm-cov/**', fingerprint: true
|
])
|
||||||
}
|
}
|
||||||
} // Reports
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Shared CI prerequisites. Run on a disposable Debian/Ubuntu agent with sudo
|
||||||
|
# (or root), or on an already provisioned macOS development machine.
|
||||||
|
set -euo pipefail
|
||||||
|
if [[ "$(uname -s)" == Linux ]]; then
|
||||||
|
admin=()
|
||||||
|
if [[ "$EUID" -ne 0 ]]; then admin=(sudo); fi
|
||||||
|
"${admin[@]}" apt-get update
|
||||||
|
"${admin[@]}" apt-get install -y --no-install-recommends \
|
||||||
|
build-essential pkg-config libasound2-dev libudev-dev libxkbcommon-dev \
|
||||||
|
libwayland-dev libx11-dev libxrandr-dev libxi-dev \
|
||||||
|
python3 jq ripgrep git curl ca-certificates
|
||||||
|
fi
|
||||||
|
for command in rustup cargo python3 jq rg git make pkg-config; do
|
||||||
|
command -v "$command" >/dev/null || { echo "Missing quality-gate prerequisite: $command" >&2; exit 1; }
|
||||||
|
done
|
||||||
|
rustup component add rustfmt clippy llvm-tools-preview
|
||||||
|
if [[ "$(cargo llvm-cov --version 2>/dev/null || true)" != 'cargo-llvm-cov 0.8.5' ]]; then
|
||||||
|
cargo install cargo-llvm-cov --version 0.8.5 --locked
|
||||||
|
fi
|
||||||
|
if [[ "$(uname -s)" == Linux ]]; then pkg-config --exists alsa; fi
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Run Makefile-owned gate steps serially and retain evidence on failure."""
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import platform
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
|
||||||
|
def capture(*command):
|
||||||
|
try:
|
||||||
|
result = subprocess.run(command, text=True, stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT, check=False)
|
||||||
|
return {"exit_code": result.returncode, "output": result.stdout.strip()}
|
||||||
|
except OSError as error:
|
||||||
|
return {"exit_code": 127, "output": str(error)}
|
||||||
|
|
||||||
|
|
||||||
|
def sha(path):
|
||||||
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
root = Path(__file__).resolve().parent.parent
|
||||||
|
os.chdir(root)
|
||||||
|
output = root / "target/quality-gate"
|
||||||
|
# These two directories contain generated reports only. Never archive a
|
||||||
|
# previous run's coverage as evidence for an early lint/test failure.
|
||||||
|
for directory in (output, root / "target/llvm-cov"):
|
||||||
|
if directory.exists():
|
||||||
|
shutil.rmtree(directory)
|
||||||
|
output.mkdir(parents=True)
|
||||||
|
inputs = [Path("Makefile"), Path("Cargo.lock"), Path(".github/workflows/ci.yml"),
|
||||||
|
Path("files/config/Jenkinsfile"), Path("files/config/runtime-edge-coverage-domains.json")]
|
||||||
|
inputs.extend(sorted(Path("scripts").rglob("*.py")))
|
||||||
|
inputs.extend(sorted(Path("scripts").glob("*.sh")))
|
||||||
|
fingerprints = {str(path): sha(path) for path in inputs if path.is_file()}
|
||||||
|
sources = "".join(f"{path}\0{sha(path)}\n" for path in sorted(Path("crates").rglob("*.rs")))
|
||||||
|
evidence = {
|
||||||
|
"started_utc": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"host": platform.platform(), "revision": capture("git", "rev-parse", "HEAD"),
|
||||||
|
"working_tree": capture("git", "status", "--porcelain"),
|
||||||
|
"rustc": capture("rustc", "-vV"), "cargo_llvm_cov": capture("cargo", "llvm-cov", "--version"),
|
||||||
|
"config_sha256": fingerprints,
|
||||||
|
"rust_source_sha256": hashlib.sha256(sources.encode()).hexdigest(),
|
||||||
|
"source_hash_algorithm": "SHA-256 of sorted path + NUL + file SHA-256 hex + newline records",
|
||||||
|
"steps": [], "status": "running",
|
||||||
|
}
|
||||||
|
metadata = output / "metadata.json"
|
||||||
|
|
||||||
|
def persist():
|
||||||
|
metadata.write_text(json.dumps(evidence, indent=2) + "\n")
|
||||||
|
|
||||||
|
persist()
|
||||||
|
exit_code = 0
|
||||||
|
try:
|
||||||
|
if not sys.argv[1:]:
|
||||||
|
raise ValueError("Makefile must supply the required gate steps")
|
||||||
|
for target in sys.argv[1:]:
|
||||||
|
if not target.replace("-", "").isalnum():
|
||||||
|
raise ValueError(f"invalid make target: {target!r}")
|
||||||
|
command = ["make", "--no-print-directory", "-j1", target]
|
||||||
|
log = output / f"{target}.log"
|
||||||
|
print(f"Quality gate: {target}", flush=True)
|
||||||
|
with log.open("w") as stream:
|
||||||
|
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||||
|
text=True, errors="replace")
|
||||||
|
for line in process.stdout:
|
||||||
|
stream.write(line)
|
||||||
|
print(line, end="", flush=True)
|
||||||
|
result = process.wait()
|
||||||
|
exit_code = result if result >= 0 else 128 - result
|
||||||
|
evidence["steps"].append({"target": target, "command": command,
|
||||||
|
"exit_code": exit_code, "log_sha256": sha(log)})
|
||||||
|
persist()
|
||||||
|
if exit_code:
|
||||||
|
break
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
evidence["error"] = "Interrupted"
|
||||||
|
exit_code = 130
|
||||||
|
except Exception as error:
|
||||||
|
evidence["error"] = str(error)
|
||||||
|
exit_code = 1
|
||||||
|
finally:
|
||||||
|
evidence["status"] = "passed" if exit_code == 0 else "failed"
|
||||||
|
evidence["exit_code"] = exit_code
|
||||||
|
evidence["finished_utc"] = datetime.now(timezone.utc).isoformat()
|
||||||
|
evidence["reports_sha256"] = {
|
||||||
|
str(path.relative_to(root)): sha(path)
|
||||||
|
for path in sorted((root / "target/llvm-cov").rglob("*")) if path.is_file()
|
||||||
|
}
|
||||||
|
persist()
|
||||||
|
return exit_code
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
"""Exercise real make failures without running Cargo or modifying the repository."""
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
class QualityGateTests(unittest.TestCase):
|
||||||
|
def run_gate(self, makefile, targets):
|
||||||
|
temporary = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(temporary.cleanup)
|
||||||
|
root = Path(temporary.name)
|
||||||
|
(root / "scripts").mkdir()
|
||||||
|
shutil.copy(Path(__file__).resolve().parents[1] / "run-quality-gate.py", root / "scripts")
|
||||||
|
(root / "Makefile").write_text(makefile)
|
||||||
|
(root / "target/llvm-cov").mkdir(parents=True)
|
||||||
|
(root / "target/llvm-cov/stale.json").write_text("old evidence")
|
||||||
|
result = subprocess.run([sys.executable, str(root / "scripts/run-quality-gate.py"), *targets],
|
||||||
|
capture_output=True, text=True)
|
||||||
|
evidence = json.loads((root / "target/quality-gate/metadata.json").read_text())
|
||||||
|
self.assertFalse((root / "target/llvm-cov/stale.json").exists())
|
||||||
|
return root, result, evidence
|
||||||
|
|
||||||
|
def test_failure_preserves_logs_identity_exit_and_stops_later_steps(self):
|
||||||
|
root, result, evidence = self.run_gate(
|
||||||
|
'fail:\n\t@echo diagnostic\n\t@exit 7\nlater:\n\t@touch should-not-exist\n',
|
||||||
|
['fail', 'later'])
|
||||||
|
self.assertNotEqual(result.returncode, 0)
|
||||||
|
self.assertEqual(evidence['exit_code'], result.returncode)
|
||||||
|
self.assertEqual(evidence['status'], 'failed')
|
||||||
|
self.assertEqual([step['target'] for step in evidence['steps']], ['fail'])
|
||||||
|
self.assertIn('diagnostic', (root / 'target/quality-gate/fail.log').read_text())
|
||||||
|
self.assertIn('Makefile', evidence['config_sha256'])
|
||||||
|
self.assertIn('revision', evidence)
|
||||||
|
self.assertEqual(evidence['reports_sha256'], {})
|
||||||
|
self.assertFalse((root / 'should-not-exist').exists())
|
||||||
|
|
||||||
|
def test_success_fingerprints_only_current_reports(self):
|
||||||
|
_, result, evidence = self.run_gate(
|
||||||
|
'report:\n\t@mkdir -p target/llvm-cov\n\t@echo fresh > target/llvm-cov/report.json\n',
|
||||||
|
['report'])
|
||||||
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||||
|
self.assertEqual(evidence['status'], 'passed')
|
||||||
|
self.assertEqual(list(evidence['reports_sha256']), ['target/llvm-cov/report.json'])
|
||||||
|
|
||||||
|
def test_partial_report_is_retained_when_a_later_gate_fails(self):
|
||||||
|
_, result, evidence = self.run_gate(
|
||||||
|
'report:\n\t@mkdir -p target/llvm-cov\n\t@echo fresh > target/llvm-cov/report.json\nfail:\n\t@exit 1\n',
|
||||||
|
['report', 'fail'])
|
||||||
|
self.assertNotEqual(result.returncode, 0)
|
||||||
|
self.assertEqual(evidence['status'], 'failed')
|
||||||
|
self.assertIn('target/llvm-cov/report.json', evidence['reports_sha256'])
|
||||||
Reference in New Issue
Block a user