Files
prometeu-runtime/scripts/tests/test_quality_gate.py
T

56 lines
2.8 KiB
Python

"""Exercise real make failures without running Cargo or modifying the repository."""
import json
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import unittest
class QualityGateTests(unittest.TestCase):
def run_gate(self, makefile, targets):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
root = Path(temporary.name)
(root / "scripts").mkdir()
shutil.copy(Path(__file__).resolve().parents[1] / "run-quality-gate.py", root / "scripts")
(root / "Makefile").write_text(makefile)
(root / "target/llvm-cov").mkdir(parents=True)
(root / "target/llvm-cov/stale.json").write_text("old evidence")
result = subprocess.run([sys.executable, str(root / "scripts/run-quality-gate.py"), *targets],
capture_output=True, text=True)
evidence = json.loads((root / "target/quality-gate/metadata.json").read_text())
self.assertFalse((root / "target/llvm-cov/stale.json").exists())
return root, result, evidence
def test_failure_preserves_logs_identity_exit_and_stops_later_steps(self):
root, result, evidence = self.run_gate(
'fail:\n\t@echo diagnostic\n\t@exit 7\nlater:\n\t@touch should-not-exist\n',
['fail', 'later'])
self.assertNotEqual(result.returncode, 0)
self.assertEqual(evidence['exit_code'], result.returncode)
self.assertEqual(evidence['status'], 'failed')
self.assertEqual([step['target'] for step in evidence['steps']], ['fail'])
self.assertIn('diagnostic', (root / 'target/quality-gate/fail.log').read_text())
self.assertIn('Makefile', evidence['config_sha256'])
self.assertIn('revision', evidence)
self.assertEqual(evidence['reports_sha256'], {})
self.assertFalse((root / 'should-not-exist').exists())
def test_success_fingerprints_only_current_reports(self):
_, result, evidence = self.run_gate(
'report:\n\t@mkdir -p target/llvm-cov\n\t@echo fresh > target/llvm-cov/report.json\n',
['report'])
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(evidence['status'], 'passed')
self.assertEqual(list(evidence['reports_sha256']), ['target/llvm-cov/report.json'])
def test_partial_report_is_retained_when_a_later_gate_fails(self):
_, result, evidence = self.run_gate(
'report:\n\t@mkdir -p target/llvm-cov\n\t@echo fresh > target/llvm-cov/report.json\nfail:\n\t@exit 1\n',
['report', 'fail'])
self.assertNotEqual(result.returncode, 0)
self.assertEqual(evidence['status'], 'failed')
self.assertIn('target/llvm-cov/report.json', evidence['reports_sha256'])