Files
bquarkz fa37d0d0c7
Test / Tests and coverage
Intrepid/Prometeu/Studio/pipeline/head This commit looks good
chore: retire Discussion Framework workflow and retain lessons
2026-11-16 16:45:34 +00:00

224 lines
14 KiB
Python

#!/usr/bin/env python3
"""Isolated production PBS regression matrix. Never writes the received baseline or runtime checkout."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import platform
import selectors
import signal
import subprocess
import tempfile
import time
import zipfile
ROOT = Path(__file__).resolve().parents[2]
MODULES = ["prometeu-infra", "prometeu-animation", "prometeu-compiler/prometeu-compiler-core",
"prometeu-compiler/prometeu-frontend-api", "prometeu-compiler/frontends/prometeu-frontend-pbs",
"prometeu-compiler/prometeu-frontend-registry", "prometeu-compiler/prometeu-deps",
"prometeu-compiler/prometeu-build-pipeline"]
def sha(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def write(path, data):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n")
class Session:
def __init__(self, binary, directory):
self.stderr_path = directory / "runtime.stderr"
self.stderr = self.stderr_path.open("wb")
self.process = subprocess.Popen([str(binary), "--stdio"], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=self.stderr, cwd=directory, start_new_session=True)
self.selector = selectors.DefaultSelector()
self.selector.register(self.process.stdout, selectors.EVENT_READ)
self.buffer = bytearray()
self.transcript = []
self.request = 0
def event(self):
deadline = time.monotonic() + 12
while b"\n" not in self.buffer:
remaining = deadline - time.monotonic()
if remaining <= 0 or not self.selector.select(remaining):
raise TimeoutError("Runtime reply deadline")
data = os.read(self.process.stdout.fileno(), 65536)
if not data:
raise EOFError("Runtime closed stdout")
self.buffer.extend(data)
line, _, self.buffer = self.buffer.partition(b"\n")
value = json.loads(line)
self.transcript.append({"direction": "received", "message": value})
return value
def call(self, command, params=None):
self.request += 1
request = {"requestId": f"pbs-corpus-{self.request}", "command": command, "params": params or {}}
self.transcript.append({"direction": "sent", "message": request})
self.process.stdin.write(json.dumps(request).encode() + b"\n")
self.process.stdin.flush()
while True:
response = self.event()
if response.get("requestId") != request["requestId"]:
raise AssertionError("Unexpected response correlation")
if response.get("type") == "progress":
continue
if "error" in response:
raise RuntimeError(json.dumps(response["error"]))
return response.get("result")
def close(self):
try:
if self.process.poll() is None:
try:
self.call("shutdown")
self.process.wait(timeout=5)
except Exception:
os.killpg(self.process.pid, signal.SIGKILL)
self.process.wait(timeout=5)
finally:
self.selector.close()
self.stderr.close()
self.process.stdin.close()
self.process.stdout.close()
def execute(binary, cartridge, work):
work.mkdir(parents=True)
result = {"observedRuntime": "infrastructure_failure"}
session = Session(binary, work)
try:
hello = session.event()
assert hello["type"] == "hello" and hello["protocolVersion"]["major"] == 1
required = ["boot", "advance", "get_state", "get_fault", "shutdown"]
assert set(required) <= set(hello["capabilities"])
result["hello"] = hello
session.call("initialize", {"protocolVersion": {"major": 1, "minor": min(1, hello["protocolVersion"]["minor"])},
"requiredCapabilities": required})
result["boot"] = session.call("boot", {"path": cartridge})
result["advance"] = advance = session.call("advance", {"frames": 3})
result["fault"] = fault = session.call("get_fault")
result["state"] = session.call("get_state")
fault = fault.get("fault") if isinstance(fault, dict) and "fault" in fault else fault
result["observedRuntime"] = "fault" if fault is not None else "passed" if advance.get("completed") == 3 else "incomplete"
except Exception as failure:
result["exception"] = {"class": type(failure).__name__, "message": str(failure)}
finally:
session.close()
result.update(workerTerminated=session.process.poll() is not None, processExitCode=session.process.returncode,
transcript=session.transcript, stderr=session.stderr_path.read_text(errors="replace"))
return result
def source_rejection(report):
issues = report.get("diagnostics", []) + [i for p in report.get("phases", []) for i in p.get("diagnostics", [])]
return report.get("status") in ("analysis_failed", "public_compile_rejected") and any(i.get("severity") == "ERROR" and (i.get("failureKind") == "program-rejection" or str(i.get("code", "")).startswith("E_")) for i in issues) \
and not any(i.get("failureKind") in ("internal-failure", "infrastructure-failure", "verification-limit") for i in issues)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--corpus", type=Path, required=True)
parser.add_argument("--distribution", type=Path, required=True)
parser.add_argument("--runtime-headless", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--select", help="Comma-separated IDs; omitted for the complete 168/108 gate")
parser.add_argument("--origins", choices=["checkout", "installed", "both"], default="both")
args = parser.parse_args()
corpus, distribution, binary, output = args.corpus.resolve(strict=True), args.distribution.resolve(strict=True), args.runtime_headless.resolve(strict=True), args.output.resolve()
if output.exists():
parser.error("Output must be a new directory; evidence is never overwritten")
lessons = ROOT / "discussion/lessons"
if output == lessons or lessons in output.parents:
parser.error("Published lessons and their evidence are read-only")
output.mkdir(parents=True)
cases = json.loads(corpus.read_text())["cases"]
if args.select:
selected = set(args.select.split(",")); assert selected <= {c["id"] for c in cases}
cases = [c for c in cases if c["id"] in selected]
scratch = Path(tempfile.mkdtemp(prefix="pbs-production-corpus-"))
assert not scratch.is_relative_to(ROOT)
java, javac = distribution / "runtime/bin/java", distribution / "runtime/bin/javac"
jars = sorted((distribution / "lib").glob("*.jar")); assert jars
# Verify candidate product classes before mixing checkout classes with third-party dependencies.
freshness = []
for module in MODULES:
jar = next(p for p in jars if p.name == Path(module).name + ".jar" or p.name.startswith(Path(module).name + "-"))
with zipfile.ZipFile(jar) as archive:
classes = ROOT / module / "build/classes/java/main"
compared = list(classes.rglob("*.class")); assert compared
different = [str(p.relative_to(classes)) for p in compared if archive.read(str(p.relative_to(classes))) != p.read_bytes()]
assert not different, (module, different)
freshness.append({"module": module, "jarSha256": sha(jar), "matchingClasses": len(compared)})
provenance = {"schemaVersion": 1, "corpusSha256": sha(corpus), "runtime": str(binary), "runtimeSha256": sha(binary),
"platform": platform.platform(), "gitHead": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip(),
"worktreeStatus": subprocess.check_output(["git", "status", "--short"], cwd=ROOT, text=True).splitlines(),
"java": subprocess.run([str(java), "-version"], capture_output=True, text=True, check=True).stderr,
"distribution": str(distribution), "jars": {p.name: sha(p) for p in jars}, "candidateFreshness": freshness,
"probeSha256": sha(Path(__file__).resolve().with_name("PbsProductionCorpusProbe.java")), "runnerSha256": sha(__file__),
"scratch": str(scratch), "unoptimizedVariant": "Investigation-only: omit OptimizeIRVMPipelineStage and emit original IRVM; not a public compiler flag"}
write(output / "provenance.json", provenance)
observations, executions = [], []
origins = ["checkout", "installed"] if args.origins == "both" else [args.origins]
for origin in origins:
installed = os.pathsep.join(map(str, jars))
entries = [ROOT / m / suffix for m in MODULES for suffix in ["build/classes/java/main", "build/resources/main"] if (ROOT / m / suffix).exists()]
cp = os.pathsep.join(map(str, entries)) + os.pathsep + installed if origin == "checkout" else installed
classes = scratch / origin / "classes"; classes.mkdir(parents=True)
subprocess.run([str(javac), "-proc:none", "-cp", cp, "-d", str(classes), str(Path(__file__).resolve().with_name("PbsProductionCorpusProbe.java"))], cwd=scratch, check=True)
for case in cases:
for variant in ["optimized", "unoptimized", "optimized-public"]:
project = scratch / origin / case["id"] / variant
for name, source in case["files"].items():
path = project / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_text(source)
report_path = output / "compilation" / origin / f"{case['id']}-{variant}.json"
cmd = [str(java), "-cp", str(classes) + os.pathsep + cp, "PbsProductionCorpusProbe", str(project), "unoptimized" if variant == "unoptimized" else "optimized", str(report_path)]
if variant == "optimized-public": cmd.append("public")
try:
completed = subprocess.run(cmd, cwd=scratch, capture_output=True, text=True, timeout=50, check=True)
report = json.loads(report_path.read_text()); report["stderr"] = completed.stderr
except Exception as failure:
report = {"status": "infrastructure_failure", "exception": {"class": type(failure).__name__, "message": str(failure)}}
expected = case["expectedCompile"]
matched = report["status"] == "compiled" if expected == "compiled" else source_rejection(report)
if origin == "installed" and report.get("frontendOrigin"):
assert all(str(distribution) in report.get(key, "") for key in ["frontendOrigin", "pipelineOrigin", "sdkOrigin"])
sources_unchanged = all((project / name).read_text() == source for name, source in case["files"].items())
assert sources_unchanged
identity = {"caseId": case["id"], "origin": origin, "variant": variant, "sourceSha256": case["sourceSha256"]}
report.update(identity, expectedCompile=expected, matchesExpectation=matched, sourcesUnchanged=sources_unchanged)
if report["status"] == "compiled":
cartridge = Path(report["cartridge"])
for name in ["manifest.json", "program.pbx"]:
target = output / "artifacts" / origin / case["id"] / variant / name
target.parent.mkdir(parents=True, exist_ok=True); target.write_bytes((cartridge / name).read_bytes())
report["manifestSha256"] = sha(cartridge / "manifest.json")
write(report_path, report); observations.append({**identity, "status": report["status"], "matchesExpectation": matched, "report": str(report_path.relative_to(output))})
print(origin, case["id"], variant, report["status"], matched, flush=True)
if report["status"] == "compiled" and variant != "optimized-public":
result = execute(binary, report["cartridge"], project / "execution")
result.update(identity, bytecodeSha256=report["bytecodeSha256"], expectedRuntime=case["expectedRuntime"])
result["matchesExpectation"] = result["observedRuntime"] == case["expectedRuntime"] and result["workerTerminated"]
path = output / "execution" / origin / f"{case['id']}-{variant}.json"; write(path, result)
executions.append({**identity, "observedRuntime": result["observedRuntime"], "matchesExpectation": result["matchesExpectation"], "report": str(path.relative_to(output))})
write(output / "summary.json", {"compilation": observations, "execution": executions})
failures = [x for x in observations + executions if not x["matchesExpectation"]]
expected_runs = sum(c["expectedCompile"] == "compiled" for c in cases) * len(origins) * 2
complete = len(observations) == len(cases)*len(origins)*3 and len(executions) == expected_runs
summary = {"compilation": observations, "execution": executions, "compileCount": len(observations), "executionCount": len(executions),
"normalSuccesses": sum(x["observedRuntime"] == "passed" for x in executions), "deliberateFaults": sum(x["observedRuntime"] == "fault" and x["caseId"] == "runtime-trap-control" for x in executions),
"complete": complete, "failures": failures, "passed": complete and not failures}
write(output / "summary.json", summary)
write(output / "checksums.json", {str(p.relative_to(output)): sha(p) for p in output.rglob("*") if p.is_file()})
print(json.dumps({k:v for k,v in summary.items() if k not in ("compilation", "execution", "failures")}))
return 0 if summary["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())