224 lines
14 KiB
Python
224 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Isolated production PBS regression matrix. Never writes the received baseline or runtime checkout."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import selectors
|
|
import signal
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import zipfile
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
MODULES = ["prometeu-infra", "prometeu-animation", "prometeu-compiler/prometeu-compiler-core",
|
|
"prometeu-compiler/prometeu-frontend-api", "prometeu-compiler/frontends/prometeu-frontend-pbs",
|
|
"prometeu-compiler/prometeu-frontend-registry", "prometeu-compiler/prometeu-deps",
|
|
"prometeu-compiler/prometeu-build-pipeline"]
|
|
|
|
|
|
def sha(path):
|
|
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
|
|
|
|
def write(path, data):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n")
|
|
|
|
|
|
class Session:
|
|
def __init__(self, binary, directory):
|
|
self.stderr_path = directory / "runtime.stderr"
|
|
self.stderr = self.stderr_path.open("wb")
|
|
self.process = subprocess.Popen([str(binary), "--stdio"], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=self.stderr, cwd=directory, start_new_session=True)
|
|
self.selector = selectors.DefaultSelector()
|
|
self.selector.register(self.process.stdout, selectors.EVENT_READ)
|
|
self.buffer = bytearray()
|
|
self.transcript = []
|
|
self.request = 0
|
|
|
|
def event(self):
|
|
deadline = time.monotonic() + 12
|
|
while b"\n" not in self.buffer:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0 or not self.selector.select(remaining):
|
|
raise TimeoutError("Runtime reply deadline")
|
|
data = os.read(self.process.stdout.fileno(), 65536)
|
|
if not data:
|
|
raise EOFError("Runtime closed stdout")
|
|
self.buffer.extend(data)
|
|
line, _, self.buffer = self.buffer.partition(b"\n")
|
|
value = json.loads(line)
|
|
self.transcript.append({"direction": "received", "message": value})
|
|
return value
|
|
|
|
def call(self, command, params=None):
|
|
self.request += 1
|
|
request = {"requestId": f"pbs-corpus-{self.request}", "command": command, "params": params or {}}
|
|
self.transcript.append({"direction": "sent", "message": request})
|
|
self.process.stdin.write(json.dumps(request).encode() + b"\n")
|
|
self.process.stdin.flush()
|
|
while True:
|
|
response = self.event()
|
|
if response.get("requestId") != request["requestId"]:
|
|
raise AssertionError("Unexpected response correlation")
|
|
if response.get("type") == "progress":
|
|
continue
|
|
if "error" in response:
|
|
raise RuntimeError(json.dumps(response["error"]))
|
|
return response.get("result")
|
|
|
|
def close(self):
|
|
try:
|
|
if self.process.poll() is None:
|
|
try:
|
|
self.call("shutdown")
|
|
self.process.wait(timeout=5)
|
|
except Exception:
|
|
os.killpg(self.process.pid, signal.SIGKILL)
|
|
self.process.wait(timeout=5)
|
|
finally:
|
|
self.selector.close()
|
|
self.stderr.close()
|
|
self.process.stdin.close()
|
|
self.process.stdout.close()
|
|
|
|
|
|
def execute(binary, cartridge, work):
|
|
work.mkdir(parents=True)
|
|
result = {"observedRuntime": "infrastructure_failure"}
|
|
session = Session(binary, work)
|
|
try:
|
|
hello = session.event()
|
|
assert hello["type"] == "hello" and hello["protocolVersion"]["major"] == 1
|
|
required = ["boot", "advance", "get_state", "get_fault", "shutdown"]
|
|
assert set(required) <= set(hello["capabilities"])
|
|
result["hello"] = hello
|
|
session.call("initialize", {"protocolVersion": {"major": 1, "minor": min(1, hello["protocolVersion"]["minor"])},
|
|
"requiredCapabilities": required})
|
|
result["boot"] = session.call("boot", {"path": cartridge})
|
|
result["advance"] = advance = session.call("advance", {"frames": 3})
|
|
result["fault"] = fault = session.call("get_fault")
|
|
result["state"] = session.call("get_state")
|
|
fault = fault.get("fault") if isinstance(fault, dict) and "fault" in fault else fault
|
|
result["observedRuntime"] = "fault" if fault is not None else "passed" if advance.get("completed") == 3 else "incomplete"
|
|
except Exception as failure:
|
|
result["exception"] = {"class": type(failure).__name__, "message": str(failure)}
|
|
finally:
|
|
session.close()
|
|
result.update(workerTerminated=session.process.poll() is not None, processExitCode=session.process.returncode,
|
|
transcript=session.transcript, stderr=session.stderr_path.read_text(errors="replace"))
|
|
return result
|
|
|
|
|
|
def source_rejection(report):
|
|
issues = report.get("diagnostics", []) + [i for p in report.get("phases", []) for i in p.get("diagnostics", [])]
|
|
return report.get("status") in ("analysis_failed", "public_compile_rejected") and any(i.get("severity") == "ERROR" and (i.get("failureKind") == "program-rejection" or str(i.get("code", "")).startswith("E_")) for i in issues) \
|
|
and not any(i.get("failureKind") in ("internal-failure", "infrastructure-failure", "verification-limit") for i in issues)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--corpus", type=Path, required=True)
|
|
parser.add_argument("--distribution", type=Path, required=True)
|
|
parser.add_argument("--runtime-headless", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--select", help="Comma-separated IDs; omitted for the complete 168/108 gate")
|
|
parser.add_argument("--origins", choices=["checkout", "installed", "both"], default="both")
|
|
args = parser.parse_args()
|
|
corpus, distribution, binary, output = args.corpus.resolve(strict=True), args.distribution.resolve(strict=True), args.runtime_headless.resolve(strict=True), args.output.resolve()
|
|
if output.exists():
|
|
parser.error("Output must be a new directory; evidence is never overwritten")
|
|
lessons = ROOT / "discussion/lessons"
|
|
if output == lessons or lessons in output.parents:
|
|
parser.error("Published lessons and their evidence are read-only")
|
|
output.mkdir(parents=True)
|
|
cases = json.loads(corpus.read_text())["cases"]
|
|
if args.select:
|
|
selected = set(args.select.split(",")); assert selected <= {c["id"] for c in cases}
|
|
cases = [c for c in cases if c["id"] in selected]
|
|
scratch = Path(tempfile.mkdtemp(prefix="pbs-production-corpus-"))
|
|
assert not scratch.is_relative_to(ROOT)
|
|
java, javac = distribution / "runtime/bin/java", distribution / "runtime/bin/javac"
|
|
jars = sorted((distribution / "lib").glob("*.jar")); assert jars
|
|
# Verify candidate product classes before mixing checkout classes with third-party dependencies.
|
|
freshness = []
|
|
for module in MODULES:
|
|
jar = next(p for p in jars if p.name == Path(module).name + ".jar" or p.name.startswith(Path(module).name + "-"))
|
|
with zipfile.ZipFile(jar) as archive:
|
|
classes = ROOT / module / "build/classes/java/main"
|
|
compared = list(classes.rglob("*.class")); assert compared
|
|
different = [str(p.relative_to(classes)) for p in compared if archive.read(str(p.relative_to(classes))) != p.read_bytes()]
|
|
assert not different, (module, different)
|
|
freshness.append({"module": module, "jarSha256": sha(jar), "matchingClasses": len(compared)})
|
|
provenance = {"schemaVersion": 1, "corpusSha256": sha(corpus), "runtime": str(binary), "runtimeSha256": sha(binary),
|
|
"platform": platform.platform(), "gitHead": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip(),
|
|
"worktreeStatus": subprocess.check_output(["git", "status", "--short"], cwd=ROOT, text=True).splitlines(),
|
|
"java": subprocess.run([str(java), "-version"], capture_output=True, text=True, check=True).stderr,
|
|
"distribution": str(distribution), "jars": {p.name: sha(p) for p in jars}, "candidateFreshness": freshness,
|
|
"probeSha256": sha(Path(__file__).resolve().with_name("PbsProductionCorpusProbe.java")), "runnerSha256": sha(__file__),
|
|
"scratch": str(scratch), "unoptimizedVariant": "Investigation-only: omit OptimizeIRVMPipelineStage and emit original IRVM; not a public compiler flag"}
|
|
write(output / "provenance.json", provenance)
|
|
observations, executions = [], []
|
|
origins = ["checkout", "installed"] if args.origins == "both" else [args.origins]
|
|
for origin in origins:
|
|
installed = os.pathsep.join(map(str, jars))
|
|
entries = [ROOT / m / suffix for m in MODULES for suffix in ["build/classes/java/main", "build/resources/main"] if (ROOT / m / suffix).exists()]
|
|
cp = os.pathsep.join(map(str, entries)) + os.pathsep + installed if origin == "checkout" else installed
|
|
classes = scratch / origin / "classes"; classes.mkdir(parents=True)
|
|
subprocess.run([str(javac), "-proc:none", "-cp", cp, "-d", str(classes), str(Path(__file__).resolve().with_name("PbsProductionCorpusProbe.java"))], cwd=scratch, check=True)
|
|
for case in cases:
|
|
for variant in ["optimized", "unoptimized", "optimized-public"]:
|
|
project = scratch / origin / case["id"] / variant
|
|
for name, source in case["files"].items():
|
|
path = project / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_text(source)
|
|
report_path = output / "compilation" / origin / f"{case['id']}-{variant}.json"
|
|
cmd = [str(java), "-cp", str(classes) + os.pathsep + cp, "PbsProductionCorpusProbe", str(project), "unoptimized" if variant == "unoptimized" else "optimized", str(report_path)]
|
|
if variant == "optimized-public": cmd.append("public")
|
|
try:
|
|
completed = subprocess.run(cmd, cwd=scratch, capture_output=True, text=True, timeout=50, check=True)
|
|
report = json.loads(report_path.read_text()); report["stderr"] = completed.stderr
|
|
except Exception as failure:
|
|
report = {"status": "infrastructure_failure", "exception": {"class": type(failure).__name__, "message": str(failure)}}
|
|
expected = case["expectedCompile"]
|
|
matched = report["status"] == "compiled" if expected == "compiled" else source_rejection(report)
|
|
if origin == "installed" and report.get("frontendOrigin"):
|
|
assert all(str(distribution) in report.get(key, "") for key in ["frontendOrigin", "pipelineOrigin", "sdkOrigin"])
|
|
sources_unchanged = all((project / name).read_text() == source for name, source in case["files"].items())
|
|
assert sources_unchanged
|
|
identity = {"caseId": case["id"], "origin": origin, "variant": variant, "sourceSha256": case["sourceSha256"]}
|
|
report.update(identity, expectedCompile=expected, matchesExpectation=matched, sourcesUnchanged=sources_unchanged)
|
|
if report["status"] == "compiled":
|
|
cartridge = Path(report["cartridge"])
|
|
for name in ["manifest.json", "program.pbx"]:
|
|
target = output / "artifacts" / origin / case["id"] / variant / name
|
|
target.parent.mkdir(parents=True, exist_ok=True); target.write_bytes((cartridge / name).read_bytes())
|
|
report["manifestSha256"] = sha(cartridge / "manifest.json")
|
|
write(report_path, report); observations.append({**identity, "status": report["status"], "matchesExpectation": matched, "report": str(report_path.relative_to(output))})
|
|
print(origin, case["id"], variant, report["status"], matched, flush=True)
|
|
if report["status"] == "compiled" and variant != "optimized-public":
|
|
result = execute(binary, report["cartridge"], project / "execution")
|
|
result.update(identity, bytecodeSha256=report["bytecodeSha256"], expectedRuntime=case["expectedRuntime"])
|
|
result["matchesExpectation"] = result["observedRuntime"] == case["expectedRuntime"] and result["workerTerminated"]
|
|
path = output / "execution" / origin / f"{case['id']}-{variant}.json"; write(path, result)
|
|
executions.append({**identity, "observedRuntime": result["observedRuntime"], "matchesExpectation": result["matchesExpectation"], "report": str(path.relative_to(output))})
|
|
write(output / "summary.json", {"compilation": observations, "execution": executions})
|
|
failures = [x for x in observations + executions if not x["matchesExpectation"]]
|
|
expected_runs = sum(c["expectedCompile"] == "compiled" for c in cases) * len(origins) * 2
|
|
complete = len(observations) == len(cases)*len(origins)*3 and len(executions) == expected_runs
|
|
summary = {"compilation": observations, "execution": executions, "compileCount": len(observations), "executionCount": len(executions),
|
|
"normalSuccesses": sum(x["observedRuntime"] == "passed" for x in executions), "deliberateFaults": sum(x["observedRuntime"] == "fault" and x["caseId"] == "runtime-trap-control" for x in executions),
|
|
"complete": complete, "failures": failures, "passed": complete and not failures}
|
|
write(output / "summary.json", summary)
|
|
write(output / "checksums.json", {str(p.relative_to(output)): sha(p) for p in output.rglob("*") if p.is_file()})
|
|
print(json.dumps({k:v for k,v in summary.items() if k not in ("compilation", "execution", "failures")}))
|
|
return 0 if summary["passed"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|