diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31d5ff0..35d4ff8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,15 +15,25 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy + components: rustfmt, clippy, llvm-tools-preview - uses: Swatinem/rust-cache@v2 - - name: Format check - run: cargo fmt -- --check + - name: Discard reports from previous runs + run: rm -rf target/quality-gate target/llvm-cov - - name: Clippy - run: cargo clippy --workspace --all-features + - name: Provision quality-gate tools and native dependencies + run: bash scripts/provision-quality-gate.sh - - name: Test - run: cargo test --workspace --all-targets --all-features --no-fail-fast + - name: Repository quality gate + run: make ci-domains + + - name: Publish gate evidence (including failures) + if: always() + uses: actions/upload-artifact@v4 + with: + name: quality-gate-${{ runner.os }}-${{ github.sha }} + path: | + target/quality-gate/** + target/llvm-cov/** + if-no-files-found: warn diff --git a/Makefile b/Makefile index 8191dd4..2297c96 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,19 @@ -.PHONY: fmt fmt-check clippy test-local test-debugger-socket clean coverage coverage-xml coverage-json coverage-report-json coverage-domain-list coverage-domain-evidence coverage-domain-audit coverage-domain-gate test ci ci-domains cobertura +.PHONY: fmt fmt-check clippy test-local test-debugger-socket clean coverage coverage-xml coverage-json coverage-report-json coverage-domain-list coverage-domain-evidence coverage-domain-audit coverage-domain-gate test ci ci-domains test-quality-gate cobertura fmt: - cargo fmt + cargo fmt --all fmt-check: - cargo fmt -- --check + cargo fmt --all -- --check clippy: - cargo clippy --workspace --all-features + cargo clippy --workspace --all-targets --all-features -- -D warnings test-local: cargo test --workspace --all-targets --all-features --no-fail-fast test-debugger-socket: - cargo test -p prometeu-host-desktop-winit --lib -- --ignored + cargo test -p prometeu-host-desktop-winit --lib runner::tests -- --ignored clean: cargo llvm-cov clean --workspace @@ -47,6 +47,14 @@ coverage-domain-gate: ./scripts/coverage-domain-evidence.sh --check-all test: fmt-check clippy test-local test-debugger-socket -ci: clean fmt-check clippy coverage coverage-report-json -ci-domains: clean fmt-check clippy coverage coverage-report-json coverage-xml coverage-json coverage-domain-gate +# Keep the step list here: CI providers consume the repository contract. +# The runner executes each step serially even if the caller supplies make -j. +CI_DOMAIN_STEPS = clean fmt-check clippy test-quality-gate test-local test-debugger-socket coverage coverage-report-json coverage-xml coverage-json coverage-domain-audit coverage-domain-gate + +ci: ci-domains +ci-domains: + python3 scripts/run-quality-gate.py $(CI_DOMAIN_STEPS) + +test-quality-gate: + python3 -B -m unittest discover -s scripts/tests -p 'test_quality_gate.py' cobertura: coverage-xml coverage-json diff --git a/README.md b/README.md index be78993..1ab4920 100644 --- a/README.md +++ b/README.md @@ -107,19 +107,19 @@ Format, lint, and test the workspace: make test ``` -Run the CI-oriented non-domain gate: +Run the shared strict CI gate (`make ci` is an alias): ```bash make ci ``` -Run the CI-oriented domain coverage gate: +The authoritative target used by both GitHub and Jenkins is: ```bash make ci-domains ``` -The debugger socket tests bind localhost sockets and are marked ignored in Rust test metadata; `make test` includes them through the dedicated `test-debugger-socket` target. +The five debugger socket tests bind localhost sockets and are marked ignored in Rust test metadata; both `make test` and `make ci-domains` select only `runner::tests` through `test-debugger-socket`. Physical audio and other manual probes remain opt-in. See [quality-gate prerequisites and commands](docs/vm-arch/COVERAGE_DOMAINS.md#commands-and-ci). ## Coverage Governance @@ -128,20 +128,14 @@ The repository uses a two-layer runtime-edge coverage model: - a mandatory CI gate based on canonical runtime-edge domains; - domain-oriented review evidence for the canonical runtime-edge domains. -Current canonical domains: - -- `system/runtime` -- `fs` -- `asset/bank` -- `firmware` -- `host-dependent` -- `host/headless` +The fifteen canonical domains and fixed floors are defined in +[the repository configuration](files/config/runtime-edge-coverage-domains.json). +Use `make coverage-domain-list` or the [ownership documentation](docs/vm-arch/COVERAGE_DOMAINS.md) +to inspect the current map. Global coverage artifacts: ```bash -make ci -make cobertura make ci-domains ``` @@ -151,6 +145,7 @@ This produces: - `target/llvm-cov/report.json` - `target/llvm-cov/summary.json` - `target/llvm-cov/cobertura.xml` +- `target/quality-gate/metadata.json` and per-step logs Domain-oriented evidence helpers: @@ -161,12 +156,12 @@ make coverage-domain-evidence DOMAIN=host-dependent make coverage-domain-evidence DOMAIN=host/headless ``` -Per-domain baselines currently start around `60`. -That does not waive test obligations. It means the project is adopting explicit domain evidence immediately while tightening quantitative expectations incrementally over time. - -`make ci-domains` is the CI entrypoint used by Jenkins. -It runs the standard coverage pipeline, produces the HTML/XML/JSON artifacts, and fails when any canonical domain falls below its configured `baseline_percent`. -The CI gate currently compares domain `lines` coverage against each domain baseline. +`make ci-domains` runs formatting, strict workspace/all-target/all-feature Clippy, +all-target tests, the five debugger socket tests, fresh coverage and every domain +ownership/evidence gate. Both CI providers archive available evidence on failure. +A failing run never lowers a configured floor; local and CI checks use the same +repository-owned commands. Reports identify their revision, source/configuration +and actual execution platform. When a PR changes the observable contract of a canonical domain, it is expected to: diff --git a/crates/console/prometeu-vm/src/builtins.rs b/crates/console/prometeu-vm/src/builtins.rs index 6ec2894..56748ae 100644 --- a/crates/console/prometeu-vm/src/builtins.rs +++ b/crates/console/prometeu-vm/src/builtins.rs @@ -1263,6 +1263,18 @@ pub(crate) fn metered_lookup_intrinsic( Ok(None) } +/// Runtime-owned catalogue, including input and scalar math. +pub fn intrinsic_catalog() -> impl Iterator { + INTRINSICS + .iter() + .chain(crate::input_intrinsics::INPUT_INTRINSICS.iter()) + .chain(crate::math_intrinsics::MATH_INTRINSICS.iter()) + .chain(crate::random_intrinsics::RANDOM_INTRINSICS.iter()) +} +pub fn builtin_constant_catalog() -> impl Iterator { + BUILTIN_CONSTS.iter() +} + #[cfg(test)] mod tests { use super::*; @@ -1432,15 +1444,3 @@ mod tests { assert_eq!(pixel.field("color").map(|field| field.start_slot), Some(2)); } } - -/// Runtime-owned catalogue, including input and scalar math. -pub fn intrinsic_catalog() -> impl Iterator { - INTRINSICS - .iter() - .chain(crate::input_intrinsics::INPUT_INTRINSICS.iter()) - .chain(crate::math_intrinsics::MATH_INTRINSICS.iter()) - .chain(crate::random_intrinsics::RANDOM_INTRINSICS.iter()) -} -pub fn builtin_constant_catalog() -> impl Iterator { - BUILTIN_CONSTS.iter() -} diff --git a/crates/console/prometeu-vm/tests/numeric_contract.rs b/crates/console/prometeu-vm/tests/numeric_contract.rs index d0613fc..69c74f1 100644 --- a/crates/console/prometeu-vm/tests/numeric_contract.rs +++ b/crates/console/prometeu-vm/tests/numeric_contract.rs @@ -61,7 +61,9 @@ fn integer_limits_are_exact_and_abs_does_not_wrap() { ] { assert_eq!(call(base, &[b.clone(), a.clone()]).unwrap(), a); assert_eq!(call(base + 1, &[a.clone(), b.clone()]).unwrap(), b); - assert!(matches!(call(base+3,&[a.clone()]),Err(E::Numeric(e)) if e.code==RANGE)); + assert!( + matches!(call(base+3,std::slice::from_ref(&a)),Err(E::Numeric(e)) if e.code==RANGE) + ); assert!(matches!(call(base+2,&[b.clone(),b,a]),Err(E::Numeric(e)) if e.code==DOMAIN)); } assert_eq!( diff --git a/crates/console/prometeu-vm/tests/random_contract.rs b/crates/console/prometeu-vm/tests/random_contract.rs index f5b8ede..b939695 100644 --- a/crates/console/prometeu-vm/tests/random_contract.rs +++ b/crates/console/prometeu-vm/tests/random_contract.rs @@ -45,7 +45,7 @@ fn independent_words_signed_and_float_bits_for_all_seed_patterns() { }); } code.push_str("RET"); - let mut v = vm(&[(&code, sig(&[], &vec![if floating { 2 } else { 0 }; 8]), vec![], 1)]); + let mut v = vm(&[(&code, sig(&[], &[if floating { 2 } else { 0 }; 8]), vec![], 1)]); let r = run(&mut v, 10000); assert_eq!(r.reason, End::EndOfRom); let expected: Vec<_> = vectors(seed) diff --git a/crates/host/prometeu-host-desktop-winit/src/runner.rs b/crates/host/prometeu-host-desktop-winit/src/runner.rs index d0e7d69..bcf0719 100644 --- a/crates/host/prometeu-host-desktop-winit/src/runner.rs +++ b/crates/host/prometeu-host-desktop-winit/src/runner.rs @@ -837,7 +837,7 @@ mod tests { // real socket bind/connect coordination. Deterministic debugger state changes // are covered below the host layer in `debugger.rs`. #[test] - #[ignore = "requires localhost TCP bind/connect; run via `cargo test -p prometeu-host-desktop-winit --lib -- --ignored`"] + #[ignore = "requires localhost TCP bind/connect; run via `make test-debugger-socket`"] fn test_debug_port_opens() { let mut runner = HostRunner::new(None, None); let port = 9999; @@ -897,7 +897,7 @@ mod tests { } #[test] - #[ignore = "requires localhost TCP bind/connect; run via `cargo test -p prometeu-host-desktop-winit --lib -- --ignored`"] + #[ignore = "requires localhost TCP bind/connect; run via `make test-debugger-socket`"] fn test_debug_reconnection() { let mut runner = HostRunner::new(None, None); let port = 9998; @@ -940,7 +940,7 @@ mod tests { } #[test] - #[ignore = "requires localhost TCP bind/connect; run via `cargo test -p prometeu-host-desktop-winit --lib -- --ignored`"] + #[ignore = "requires localhost TCP bind/connect; run via `make test-debugger-socket`"] fn test_debug_refuse_second_connection() { let mut runner = HostRunner::new(None, None); let port = 9997; @@ -976,7 +976,7 @@ mod tests { } #[test] - #[ignore = "requires localhost TCP bind/connect; run via `cargo test -p prometeu-host-desktop-winit --lib -- --ignored`"] + #[ignore = "requires localhost TCP bind/connect; run via `make test-debugger-socket`"] fn test_get_state_returns_response() { let mut runner = HostRunner::new(None, None); let port = 9996; @@ -1021,7 +1021,7 @@ mod tests { } #[test] - #[ignore = "requires localhost TCP bind/connect; run via `cargo test -p prometeu-host-desktop-winit --lib -- --ignored`"] + #[ignore = "requires localhost TCP bind/connect; run via `make test-debugger-socket`"] fn test_debug_resume_on_disconnect() { let mut runner = HostRunner::new(None, None); let port = 9995; diff --git a/discussion/lessons/DSC-0002-runtime-edge-test-plan/LSN-0037-domain-gates-must-be-owned-by-the-repository.md b/discussion/lessons/DSC-0002-runtime-edge-test-plan/LSN-0037-domain-gates-must-be-owned-by-the-repository.md index 6635d9a..94a0ed6 100644 --- a/discussion/lessons/DSC-0002-runtime-edge-test-plan/LSN-0037-domain-gates-must-be-owned-by-the-repository.md +++ b/discussion/lessons/DSC-0002-runtime-edge-test-plan/LSN-0037-domain-gates-must-be-owned-by-the-repository.md @@ -142,3 +142,40 @@ must not be presented as completed strict workspace qualification. - Coverage gates that live in repository helpers are easier to evolve, audit, and keep consistent than CI-local logic. - Audit source ownership and evidence validity before comparing percentages. A passing percentage is useful only when its denominator has a clear owner. + +## A Shared Gate Must Share Failure Semantics Too + +The strict-gate correction derives from DEC-0063. A common coverage helper did +not establish common CI policy while one pipeline only ran permissive lint/tests +and another produced domain reports. Both CI consumers now provision the declared +tools and invoke the same Makefile-owned `ci-domains` target. Strict Clippy covers +workspace targets and features with warnings denied; concrete lint fixes replace +warning suppression. The existing coverage map and floors remain unchanged. + +The Makefile owns the ordered steps. A serial runner performs format, strict lint, +all-target tests, the five named debugger socket cases, fresh coverage/export and +all domain audits/gates. Serial child Make invocations avoid races between clean, +Cargo execution and report generation. A second independent gate in the same +checkout still requires operator coordination. + +Failure publication is part of the contract. Before a run, old reports are +removed; every executed step receives a log, exit status, revision and source/config +identity. A later failure retains current partial reports. An early failure cannot +archive old HTML as current success. Both CI providers publish available evidence +on failure and preserve the gate's nonzero status. Regression fixtures exercise +failed execution, success and failure after partial report generation. + +Ignored tests are not one qualification category. Five localhost socket tests +belong in automation, but physical audio, endurance, packaged installation and +manual timing/layout probes need their own named execution. Restricting the socket +filter prevents accidentally turning a device requirement into a CI prerequisite. + +The local macOS/aarch64 strict gate passed 1,039 ordinary tests, five socket tests, +three runner regressions and all fifteen coverage gates; fresh coverage separately +replayed 1,039 tests. The ordinary phases reported thirteen ignored tests; five +of these ran in the separate socket step, while eight remained unselected. The initial sandbox +socket refusal is distinguished from the successful authorized run. Pipeline +configuration was inspected, but no remote or Linux gate was executed. Platform +equivalence concerns required commands, not identical measured coverage. See the +[published operational contract and evidence](../../../docs/vm-arch/COVERAGE_DOMAINS.md) +for prerequisites, manual commands and practical limits. diff --git a/docs/vm-arch/COVERAGE_DOMAINS.md b/docs/vm-arch/COVERAGE_DOMAINS.md index de37d8d..68c0559 100644 --- a/docs/vm-arch/COVERAGE_DOMAINS.md +++ b/docs/vm-arch/COVERAGE_DOMAINS.md @@ -119,30 +119,107 @@ fresh measurement. ## Commands and CI -List domains, inspect source ownership and validate coverage evidence: +`make ci-domains` is the authoritative repository quality gate. Both +[GitHub CI](../../.github/workflows/ci.yml) and +[Jenkins](../../files/config/Jenkinsfile) provision dependencies and invoke that +same target. `make ci` is an alias. Invoke one gate per checkout at a time. - make coverage-domain-list - python3 scripts/audit-coverage-domains.py - make coverage-domain-audit - python3 scripts/audit-coverage-domains.py --report target/llvm-cov/report.json --json +The Makefile supplies this ordered sequence to the serial evidence runner: -Produce a fresh report and run all gates: +1. Clean LLVM coverage instrumentation from the previous run. +2. Check formatting with `cargo fmt --all -- --check`. +3. Run `cargo clippy --workspace --all-targets --all-features -- -D warnings`. +4. Test gate failure/report handling with the Python regression suite. +5. Run `cargo test --workspace --all-targets --all-features --no-fail-fast`. +6. Run the five ignored debugger socket tests through `make test-debugger-socket`. +7. Generate fresh workspace/all-feature LLVM coverage, HTML, full JSON, + Cobertura XML and summary JSON. +8. Audit source ownership/evidence and check every fixed coverage domain floor. - make ci-domains +The socket command is deliberately restricted: -The existing Jenkins configuration invokes this repository target. Its -coverage-domain-gate step also performs the ownership/evidence audit. -GitHub's workspace-test step discovers the quality regression tests; it -does not separately produce a domain coverage report. +```sh +cargo test -p prometeu-host-desktop-winit --lib runner::tests -- --ignored +``` -Inspect a domain: +It selects `test_debug_port_opens`, `test_debug_reconnection`, +`test_debug_refuse_second_connection`, `test_get_state_returns_response` and +`test_debug_resume_on_disconnect`. It does not select the audio-device test. +Local sandbox TCP permissions must permit bind/connect; a permission denial is +not evidence of a debugger behavior regression. - make coverage-domain-evidence DOMAIN=vm/core - scripts/coverage-domain-evidence.sh --check --verbose hal/core - scripts/coverage-domain-evidence.sh --check-all +The runner executes child Make targets with `-j1` even if the outer invocation +uses `-j`. CI invokes `make ci-domains` serially. Do not run a separate Cargo, +coverage-clean or report process concurrently in the same checkout: serialized +steps inside one gate cannot protect against an independent process. -Dependencies are the existing Rust/LLVM coverage tooling, Bash, Git, jq and rg, -plus Python 3 for the standard-library ownership auditor. +### Prerequisites and reports + +Both pipelines use `bash scripts/provision-quality-gate.sh`. A Debian/Ubuntu +agent needs root or sudo for apt provisioning, and an existing Rust/rustup +installation. GitHub installs stable Rust first; Jenkins must supply a supported +Rust toolchain and rustup in CARGO_HOME/bin. The script installs rustfmt, Clippy, +llvm-tools-preview and pinned cargo-llvm-cov 0.8.5 (`--locked`). + +Linux packages include build-essential, pkg-config, libasound2-dev, libudev-dev, +libxkbcommon-dev, libwayland-dev, libx11-dev, libxrandr-dev, libxi-dev, Python 3, +jq, ripgrep, Git, curl and CA certificates. ALSA development files satisfy CPAL's +Linux build dependency; a running audio device is not needed. Non-Linux local +hosts must already provide native compiler/SDK tools, Bash, Make, pkg-config, +Python 3, jq, rg and Git. The provisioning script verifies these commands. + +`target/quality-gate/metadata.json` records revision, dirty working tree, host, +Rust/coverage-tool versions, configuration/source fingerprints, ordered step +exit codes and report hashes. Each step has a separate combined-output log. +The runner stops on the first failed Make target and returns its nonzero exit +status; the log preserves underlying Cargo diagnostics. Three regression cases +check failure logs/identity, fresh success evidence and partial-report retention +when a later gate fails. + +Both pipelines discard old generated report directories before provisioning and +publish current artifacts even after gate failure. An early failed gate has +metadata/logs but no old coverage report. Failure after generation preserves the +partial current reports. Provisioning failure can leave no report at all and +still fails CI. Reports are archived from `target/quality-gate/**` and +`target/llvm-cov/**`; Jenkins also publishes current HTML when present. Artifact +publication does not turn a failed gate into success. + +List domains or inspect ownership and existing evidence: + +```sh +make coverage-domain-list +python3 scripts/audit-coverage-domains.py +make coverage-domain-audit +make coverage-domain-evidence DOMAIN=vm/core +scripts/coverage-domain-evidence.sh --check --verbose hal/core +``` + +### Explicit manual qualification + +Ordinary all-target tests and coverage leave ignored device/measurement tests +ignored. The shared target only opts in to the five socket tests above. It does +not certify the following manual workloads: + +```sh +# Physical desktop audio, exact named test. +cargo test -p prometeu-host-desktop-winit --lib audio::control_tests::desktop_audio_device_smoke -- --ignored --exact --nocapture +# Physical callback duration under host I/O pressure. +cargo test --release -p prometeu-host-desktop-winit --test audio_callback_allocations physical_callback_duration_16_voices_with_io_pressure -- --ignored --exact --nocapture +# Eight-hour active release soak. +cargo run --release -p prometeu-vm --features resource-diagnostics --example resource_qualification -- --seconds 28800 +# Packaged installation; set PROMETEU_PROTOCOL_TEST_INSTALL to an extracted distribution first. +cargo test -p prometeu-cli --test automation_process packaged_installation_smoke -- --ignored --exact --nocapture +# File-host commit latency with the production asset decoder. +cargo test --release -p prometeu-system --all-features qualify_file_commits_with_production_asset_decoder -- --ignored --nocapture +# Explicit layout observation. +cargo test --release -p prometeu-bytecode --test registry_v1 reference_map_probe -- --ignored --exact --nocapture +``` + +Other ignored timing probes remain opt-in by their individual test names. +All-target execution can print ordinary benchmark observations; this does not +establish completion of ignored manual qualification. For scope and prior measured +results see [resource stability](resource-stability-qualification.md) and +[audio callback qualification](audio-callback-qualification.md). ## Historical local baseline @@ -194,9 +271,9 @@ The full local `make ci-domains` validation passed with 939 tests passed and in verification; its final floor uses the lower measurement. All gates were rechecked against the verification report after that final adjustment. -## Current local measurement +## Ownership-repair measurement -The [current evidence](COVERAGE_DOMAINS_CURRENT.json) records the repaired +The [ownership-repair evidence](COVERAGE_DOMAINS_CURRENT.json) records the repaired module-tree ownership, fixed floors, normalized source paths, actual line counts and source/config/report fingerprints. Both historical JSON records above remain unchanged. The measurement uses the implementation working tree @@ -225,12 +302,42 @@ with 1,021 tests passed and 11 ignored. The 10 focused ownership/evidence tests and strict package Clippy also passed. No floor reduction or additional runtime behavior test was needed to pass the repaired map. -The current Makefile target still uses non-strict workspace Clippy. A separate -`cargo clippy --workspace --all-targets --all-features -- -D warnings` run failed -on existing VM `useless_vec`, `cloned_ref_to_slice_refs` and -`items_after_test_module` diagnostics. This evidence certifies the corrected -coverage map and its gates; full strict workspace qualification remains pending. +At the time of that ownership measurement the target used non-strict workspace +Clippy, and a separate strict invocation failed on VM `useless_vec`, +`cloned_ref_to_slice_refs` and `items_after_test_module` diagnostics. That historical +record qualifies the repaired coverage map, not the later strict gate. The lint +sources have since been corrected without blanket suppression or floor changes. Platform-specific compilation and host scenarios can change report totals. Local evidence does not certify an unexecuted Linux or Intel Mac run; evaluate those runs against the configured floors and investigate differences. + +## Strict shared-gate qualification + +On 2026-10-14, macOS/aarch64 with rustc 1.98.1, the shared target passed strict +workspace/all-target/all-feature Clippy, all 1,039 ordinary tests (13 ignored), +the five explicitly selected socket tests, three gate-runner regression tests, +and all 15 fixed coverage domain gates. The separate fresh coverage execution +also passed 1,039 tests with 13 ignored. Those repeated runs are not distinct +behavioral tests and must not be added into a unique-test count. + +The baseline strict invocation failed on the concrete VM lint sources above; +the repaired sources pass without new lint allowances. Socket tests initially +failed because the sandbox denied localhost TCP operations; the authorized +outside-sandbox run passed all five. Gate failure fixtures preserve nonzero +Make status, logs and commit/config identity, discard stale coverage, and retain +fresh partial reports when a later gate fails. Interrupted runners record failure +rather than success. + +GitHub's YAML was parsed locally, and both pipeline definitions were inspected +for shared provisioning, `make ci-domains` and unconditional failure-artifact +publication. No remote GitHub or Jenkins build was launched. No Linux gate ran: +the local Docker API was denied by the sandbox and the unrestricted daemon query +did not return a usable result before interruption. Local macOS coverage does not +certify Linux compilation or domain percentages. Such failures must be investigated +on that platform without lowering the existing floors. + +No physical audio, eight-hour soak, packaged-installation or ignored manual +latency/layout qualification was selected by this gate. Prior manual observations +remain separate evidence. Current executable metadata and reports are emitted +under `target/quality-gate/` and `target/llvm-cov/` by every invocation. diff --git a/docs/vm-arch/resource-stability-qualification.md b/docs/vm-arch/resource-stability-qualification.md index 1433347..9b8d9dc 100644 --- a/docs/vm-arch/resource-stability-qualification.md +++ b/docs/vm-arch/resource-stability-qualification.md @@ -29,11 +29,17 @@ cargo test --workspace --features prometeu-host-headless/resource-diagnostics cargo test --release --workspace --features prometeu-host-headless/resource-diagnostics cargo run --release -p prometeu-vm --features resource-diagnostics --example resource_qualification -- --frames 216000 cargo run --release -p prometeu-vm --features resource-diagnostics --example resource_qualification -- --seconds 28800 -cargo test --release -p prometeu-vm --features resource-diagnostics --lib -- --ignored --nocapture +cargo test --release -p prometeu-vm --features resource-diagnostics --lib sealed_image_load_probe -- --ignored --nocapture +cargo test --release -p prometeu-vm --features resource-diagnostics --lib storage_and_gc_probe -- --ignored --nocapture cargo test --release -p prometeu-bytecode reference_map_probe -- --ignored --nocapture -cargo test -p prometeu-host-desktop-winit --lib -- --ignored +cargo test -p prometeu-host-desktop-winit --lib runner::tests -- --ignored +cargo test -p prometeu-host-desktop-winit --lib audio::control_tests::desktop_audio_device_smoke -- --ignored --exact --nocapture ``` +The reproduction commands name manual probes individually. The original recorded +desktop qualification grouped ignored tests; current automation selects only the +five debugger socket cases and leaves device audio explicit. + The `resource-diagnostics` feature adds engineering observations through Rust only. It adds no guest syscall. The eight-hour example is a manual command; the repository CI, release workflow and Jenkins pipeline do not invoke it. Normal builds omit timing histograms, additional diff --git a/files/config/Jenkinsfile b/files/config/Jenkinsfile index c1f333f..17fc5ae 100644 --- a/files/config/Jenkinsfile +++ b/files/config/Jenkinsfile @@ -4,55 +4,49 @@ pipeline { environment { CARGO_HOME = '/var/jenkins_home/.cargo' CARGO_TARGET_DIR = 'target' - CARGO_TERM_COLOR = 'always' } stages { - stage('Build') { + stage('Prepare reports') { + steps { + dir('target/quality-gate') { deleteDir() } + dir('target/llvm-cov') { deleteDir() } + } + } + stage('Provision') { steps { sh ''' - set -eux + set -eu + export PATH="$CARGO_HOME/bin:$PATH" + bash scripts/provision-quality-gate.sh + ''' + } + } + stage('Quality gate') { + steps { + sh ''' + set -eu + export PATH="$CARGO_HOME/bin:$PATH" make ci-domains ''' -// withChecks(name: 'Test', includeStage: true) { -// sh ''' -// set -eux -// make ci-domains -// ''' -// recordCoverage( -// tools: [[parser: 'COBERTURA', pattern: 'target/llvm-cov/cobertura.xml']], -// sourceCodeRetention: 'LAST_BUILD', -// enabledForFailure: true, -// failOnError: false, -// checksAnnotationScope: 'MODIFIED_LINES', -// id: 'rust-coverage', -// name: 'Rust Coverage', -// checksName: 'Rust Coverage', -// qualityGates: [ -// [metric: 'LINE', baseline: 'MODIFIED_LINES', threshold: 0.0], -// [metric: 'LINE', baseline: 'PROJECT', threshold: 0.0], -// [metric: 'BRANCH', baseline: 'PROJECT', threshold: 0.0] -// ] -// ) -// } } - } // Test + } + } - stage('Reports') { - steps { - publishHTML(target: [ - allowMissing: false, - alwaysLinkToLastBuild: true, - keepAll: true, - reportDir: 'target/llvm-cov/html', - reportFiles: 'index.html', - reportName: 'Rust Coverage HTML', - reportTitles: 'Coverage Report' - ]) - - archiveArtifacts artifacts: 'target/llvm-cov/**', fingerprint: true - } - } // Reports + post { + always { + archiveArtifacts artifacts: 'target/quality-gate/**,target/llvm-cov/**', + fingerprint: true, allowEmptyArchive: true + publishHTML(target: [ + allowMissing: true, + alwaysLinkToLastBuild: true, + keepAll: true, + reportDir: 'target/llvm-cov/html', + reportFiles: 'index.html', + reportName: 'Rust Coverage HTML', + reportTitles: 'Coverage Report' + ]) + } } } diff --git a/scripts/provision-quality-gate.sh b/scripts/provision-quality-gate.sh new file mode 100644 index 0000000..de57988 --- /dev/null +++ b/scripts/provision-quality-gate.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Shared CI prerequisites. Run on a disposable Debian/Ubuntu agent with sudo +# (or root), or on an already provisioned macOS development machine. +set -euo pipefail +if [[ "$(uname -s)" == Linux ]]; then + admin=() + if [[ "$EUID" -ne 0 ]]; then admin=(sudo); fi + "${admin[@]}" apt-get update + "${admin[@]}" apt-get install -y --no-install-recommends \ + build-essential pkg-config libasound2-dev libudev-dev libxkbcommon-dev \ + libwayland-dev libx11-dev libxrandr-dev libxi-dev \ + python3 jq ripgrep git curl ca-certificates +fi +for command in rustup cargo python3 jq rg git make pkg-config; do + command -v "$command" >/dev/null || { echo "Missing quality-gate prerequisite: $command" >&2; exit 1; } +done +rustup component add rustfmt clippy llvm-tools-preview +if [[ "$(cargo llvm-cov --version 2>/dev/null || true)" != 'cargo-llvm-cov 0.8.5' ]]; then + cargo install cargo-llvm-cov --version 0.8.5 --locked +fi +if [[ "$(uname -s)" == Linux ]]; then pkg-config --exists alsa; fi diff --git a/scripts/run-quality-gate.py b/scripts/run-quality-gate.py new file mode 100644 index 0000000..f3e8d3c --- /dev/null +++ b/scripts/run-quality-gate.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Run Makefile-owned gate steps serially and retain evidence on failure.""" +import hashlib +import json +import os +from pathlib import Path +import platform +import shutil +import subprocess +import sys +from datetime import datetime, timezone + + +def capture(*command): + try: + result = subprocess.run(command, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False) + return {"exit_code": result.returncode, "output": result.stdout.strip()} + except OSError as error: + return {"exit_code": 127, "output": str(error)} + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(): + root = Path(__file__).resolve().parent.parent + os.chdir(root) + output = root / "target/quality-gate" + # These two directories contain generated reports only. Never archive a + # previous run's coverage as evidence for an early lint/test failure. + for directory in (output, root / "target/llvm-cov"): + if directory.exists(): + shutil.rmtree(directory) + output.mkdir(parents=True) + inputs = [Path("Makefile"), Path("Cargo.lock"), Path(".github/workflows/ci.yml"), + Path("files/config/Jenkinsfile"), Path("files/config/runtime-edge-coverage-domains.json")] + inputs.extend(sorted(Path("scripts").rglob("*.py"))) + inputs.extend(sorted(Path("scripts").glob("*.sh"))) + fingerprints = {str(path): sha(path) for path in inputs if path.is_file()} + sources = "".join(f"{path}\0{sha(path)}\n" for path in sorted(Path("crates").rglob("*.rs"))) + evidence = { + "started_utc": datetime.now(timezone.utc).isoformat(), + "host": platform.platform(), "revision": capture("git", "rev-parse", "HEAD"), + "working_tree": capture("git", "status", "--porcelain"), + "rustc": capture("rustc", "-vV"), "cargo_llvm_cov": capture("cargo", "llvm-cov", "--version"), + "config_sha256": fingerprints, + "rust_source_sha256": hashlib.sha256(sources.encode()).hexdigest(), + "source_hash_algorithm": "SHA-256 of sorted path + NUL + file SHA-256 hex + newline records", + "steps": [], "status": "running", + } + metadata = output / "metadata.json" + + def persist(): + metadata.write_text(json.dumps(evidence, indent=2) + "\n") + + persist() + exit_code = 0 + try: + if not sys.argv[1:]: + raise ValueError("Makefile must supply the required gate steps") + for target in sys.argv[1:]: + if not target.replace("-", "").isalnum(): + raise ValueError(f"invalid make target: {target!r}") + command = ["make", "--no-print-directory", "-j1", target] + log = output / f"{target}.log" + print(f"Quality gate: {target}", flush=True) + with log.open("w") as stream: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, errors="replace") + for line in process.stdout: + stream.write(line) + print(line, end="", flush=True) + result = process.wait() + exit_code = result if result >= 0 else 128 - result + evidence["steps"].append({"target": target, "command": command, + "exit_code": exit_code, "log_sha256": sha(log)}) + persist() + if exit_code: + break + except KeyboardInterrupt: + evidence["error"] = "Interrupted" + exit_code = 130 + except Exception as error: + evidence["error"] = str(error) + exit_code = 1 + finally: + evidence["status"] = "passed" if exit_code == 0 else "failed" + evidence["exit_code"] = exit_code + evidence["finished_utc"] = datetime.now(timezone.utc).isoformat() + evidence["reports_sha256"] = { + str(path.relative_to(root)): sha(path) + for path in sorted((root / "target/llvm-cov").rglob("*")) if path.is_file() + } + persist() + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_quality_gate.py b/scripts/tests/test_quality_gate.py new file mode 100644 index 0000000..33ffe5a --- /dev/null +++ b/scripts/tests/test_quality_gate.py @@ -0,0 +1,55 @@ +"""Exercise real make failures without running Cargo or modifying the repository.""" +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + + +class QualityGateTests(unittest.TestCase): + def run_gate(self, makefile, targets): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + (root / "scripts").mkdir() + shutil.copy(Path(__file__).resolve().parents[1] / "run-quality-gate.py", root / "scripts") + (root / "Makefile").write_text(makefile) + (root / "target/llvm-cov").mkdir(parents=True) + (root / "target/llvm-cov/stale.json").write_text("old evidence") + result = subprocess.run([sys.executable, str(root / "scripts/run-quality-gate.py"), *targets], + capture_output=True, text=True) + evidence = json.loads((root / "target/quality-gate/metadata.json").read_text()) + self.assertFalse((root / "target/llvm-cov/stale.json").exists()) + return root, result, evidence + + def test_failure_preserves_logs_identity_exit_and_stops_later_steps(self): + root, result, evidence = self.run_gate( + 'fail:\n\t@echo diagnostic\n\t@exit 7\nlater:\n\t@touch should-not-exist\n', + ['fail', 'later']) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(evidence['exit_code'], result.returncode) + self.assertEqual(evidence['status'], 'failed') + self.assertEqual([step['target'] for step in evidence['steps']], ['fail']) + self.assertIn('diagnostic', (root / 'target/quality-gate/fail.log').read_text()) + self.assertIn('Makefile', evidence['config_sha256']) + self.assertIn('revision', evidence) + self.assertEqual(evidence['reports_sha256'], {}) + self.assertFalse((root / 'should-not-exist').exists()) + + def test_success_fingerprints_only_current_reports(self): + _, result, evidence = self.run_gate( + 'report:\n\t@mkdir -p target/llvm-cov\n\t@echo fresh > target/llvm-cov/report.json\n', + ['report']) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(evidence['status'], 'passed') + self.assertEqual(list(evidence['reports_sha256']), ['target/llvm-cov/report.json']) + + def test_partial_report_is_retained_when_a_later_gate_fails(self): + _, result, evidence = self.run_gate( + 'report:\n\t@mkdir -p target/llvm-cov\n\t@echo fresh > target/llvm-cov/report.json\nfail:\n\t@exit 1\n', + ['report', 'fail']) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(evidence['status'], 'failed') + self.assertIn('target/llvm-cov/report.json', evidence['reports_sha256'])