Files
prometeu-studio/tools/pbs-production-corpus/run.py
T
bquarkz ea1cc26fe3
Test / Tests and coverage occurred while executing withChecks step.
Intrepid/Prometeu/Studio/pipeline/pr-master There was a failure building this commit
feat(studio): improve compiler reliability and AI diagnostics
Preserve structured diagnostics, source attribution, phase outcomes and revision context through compiler, Studio, LSP, MCP and JavaFX. Add project service discovery and resilient VS Code reconnection.

Correct all eight reproduced PBS production failures in expression context, sibling executable bindings and nominal owner identity. Consolidate specs, lessons, immutable baseline evidence and the maintained source-to-guest qualification runner.

Validation: required Gradle gate recorded 724 tests, with 711 passed and 13 skipped. Full corpus passed 168 compilation observations and 108 guest runs (104 successes and four deliberate faults); supplemental controls passed 42 compilations and 16 guest runs. TypeScript compilation, two Node discovery tests and discussion validation passed.
2026-11-10 12:36:25 +00:00

224 lines
14 KiB
Python

#!/usr/bin/env python3
"""Isolated production PBS regression matrix. Never writes the received baseline or runtime checkout."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import platform
import selectors
import signal
import subprocess
import tempfile
import time
import zipfile
ROOT = Path(__file__).resolve().parents[2]
MODULES = ["prometeu-infra", "prometeu-animation", "prometeu-compiler/prometeu-compiler-core",
"prometeu-compiler/prometeu-frontend-api", "prometeu-compiler/frontends/prometeu-frontend-pbs",
"prometeu-compiler/prometeu-frontend-registry", "prometeu-compiler/prometeu-deps",
"prometeu-compiler/prometeu-build-pipeline"]
def sha(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def write(path, data):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n")
class Session:
def __init__(self, binary, directory):
self.stderr_path = directory / "runtime.stderr"
self.stderr = self.stderr_path.open("wb")
self.process = subprocess.Popen([str(binary), "--stdio"], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=self.stderr, cwd=directory, start_new_session=True)
self.selector = selectors.DefaultSelector()
self.selector.register(self.process.stdout, selectors.EVENT_READ)
self.buffer = bytearray()
self.transcript = []
self.request = 0
def event(self):
deadline = time.monotonic() + 12
while b"\n" not in self.buffer:
remaining = deadline - time.monotonic()
if remaining <= 0 or not self.selector.select(remaining):
raise TimeoutError("Runtime reply deadline")
data = os.read(self.process.stdout.fileno(), 65536)
if not data:
raise EOFError("Runtime closed stdout")
self.buffer.extend(data)
line, _, self.buffer = self.buffer.partition(b"\n")
value = json.loads(line)
self.transcript.append({"direction": "received", "message": value})
return value
def call(self, command, params=None):
self.request += 1
request = {"requestId": f"pbs-corpus-{self.request}", "command": command, "params": params or {}}
self.transcript.append({"direction": "sent", "message": request})
self.process.stdin.write(json.dumps(request).encode() + b"\n")
self.process.stdin.flush()
while True:
response = self.event()
if response.get("requestId") != request["requestId"]:
raise AssertionError("Unexpected response correlation")
if response.get("type") == "progress":
continue
if "error" in response:
raise RuntimeError(json.dumps(response["error"]))
return response.get("result")
def close(self):
try:
if self.process.poll() is None:
try:
self.call("shutdown")
self.process.wait(timeout=5)
except Exception:
os.killpg(self.process.pid, signal.SIGKILL)
self.process.wait(timeout=5)
finally:
self.selector.close()
self.stderr.close()
self.process.stdin.close()
self.process.stdout.close()
def execute(binary, cartridge, work):
work.mkdir(parents=True)
result = {"observedRuntime": "infrastructure_failure"}
session = Session(binary, work)
try:
hello = session.event()
assert hello["type"] == "hello" and hello["protocolVersion"]["major"] == 1
required = ["boot", "advance", "get_state", "get_fault", "shutdown"]
assert set(required) <= set(hello["capabilities"])
result["hello"] = hello
session.call("initialize", {"protocolVersion": {"major": 1, "minor": min(1, hello["protocolVersion"]["minor"])},
"requiredCapabilities": required})
result["boot"] = session.call("boot", {"path": cartridge})
result["advance"] = advance = session.call("advance", {"frames": 3})
result["fault"] = fault = session.call("get_fault")
result["state"] = session.call("get_state")
fault = fault.get("fault") if isinstance(fault, dict) and "fault" in fault else fault
result["observedRuntime"] = "fault" if fault is not None else "passed" if advance.get("completed") == 3 else "incomplete"
except Exception as failure:
result["exception"] = {"class": type(failure).__name__, "message": str(failure)}
finally:
session.close()
result.update(workerTerminated=session.process.poll() is not None, processExitCode=session.process.returncode,
transcript=session.transcript, stderr=session.stderr_path.read_text(errors="replace"))
return result
def source_rejection(report):
issues = report.get("diagnostics", []) + [i for p in report.get("phases", []) for i in p.get("diagnostics", [])]
return report.get("status") in ("analysis_failed", "public_compile_rejected") and any(i.get("severity") == "ERROR" and (i.get("failureKind") == "program-rejection" or str(i.get("code", "")).startswith("E_")) for i in issues) \
and not any(i.get("failureKind") in ("internal-failure", "infrastructure-failure", "verification-limit") for i in issues)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--corpus", type=Path, required=True)
parser.add_argument("--distribution", type=Path, required=True)
parser.add_argument("--runtime-headless", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--select", help="Comma-separated IDs; omitted for the complete 168/108 gate")
parser.add_argument("--origins", choices=["checkout", "installed", "both"], default="both")
args = parser.parse_args()
corpus, distribution, binary, output = args.corpus.resolve(strict=True), args.distribution.resolve(strict=True), args.runtime_headless.resolve(strict=True), args.output.resolve()
if output.exists():
parser.error("Output must be a new directory; evidence is never overwritten")
baseline = ROOT / "discussion/workflow/agendas/AGD-0113-evidence"
if output == baseline or baseline in output.parents:
parser.error("The frozen baseline is read-only")
output.mkdir(parents=True)
cases = json.loads(corpus.read_text())["cases"]
if args.select:
selected = set(args.select.split(",")); assert selected <= {c["id"] for c in cases}
cases = [c for c in cases if c["id"] in selected]
scratch = Path(tempfile.mkdtemp(prefix="pbs-production-corpus-"))
assert not scratch.is_relative_to(ROOT)
java, javac = distribution / "runtime/bin/java", distribution / "runtime/bin/javac"
jars = sorted((distribution / "lib").glob("*.jar")); assert jars
# Verify candidate product classes before mixing checkout classes with third-party dependencies.
freshness = []
for module in MODULES:
jar = next(p for p in jars if p.name == Path(module).name + ".jar" or p.name.startswith(Path(module).name + "-"))
with zipfile.ZipFile(jar) as archive:
classes = ROOT / module / "build/classes/java/main"
compared = list(classes.rglob("*.class")); assert compared
different = [str(p.relative_to(classes)) for p in compared if archive.read(str(p.relative_to(classes))) != p.read_bytes()]
assert not different, (module, different)
freshness.append({"module": module, "jarSha256": sha(jar), "matchingClasses": len(compared)})
provenance = {"schemaVersion": 1, "corpusSha256": sha(corpus), "runtime": str(binary), "runtimeSha256": sha(binary),
"platform": platform.platform(), "gitHead": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip(),
"worktreeStatus": subprocess.check_output(["git", "status", "--short"], cwd=ROOT, text=True).splitlines(),
"java": subprocess.run([str(java), "-version"], capture_output=True, text=True, check=True).stderr,
"distribution": str(distribution), "jars": {p.name: sha(p) for p in jars}, "candidateFreshness": freshness,
"probeSha256": sha(Path(__file__).resolve().with_name("PbsProductionCorpusProbe.java")), "runnerSha256": sha(__file__),
"scratch": str(scratch), "unoptimizedVariant": "Investigation-only: omit OptimizeIRVMPipelineStage and emit original IRVM; not a public compiler flag"}
write(output / "provenance.json", provenance)
observations, executions = [], []
origins = ["checkout", "installed"] if args.origins == "both" else [args.origins]
for origin in origins:
installed = os.pathsep.join(map(str, jars))
entries = [ROOT / m / suffix for m in MODULES for suffix in ["build/classes/java/main", "build/resources/main"] if (ROOT / m / suffix).exists()]
cp = os.pathsep.join(map(str, entries)) + os.pathsep + installed if origin == "checkout" else installed
classes = scratch / origin / "classes"; classes.mkdir(parents=True)
subprocess.run([str(javac), "-proc:none", "-cp", cp, "-d", str(classes), str(Path(__file__).resolve().with_name("PbsProductionCorpusProbe.java"))], cwd=scratch, check=True)
for case in cases:
for variant in ["optimized", "unoptimized", "optimized-public"]:
project = scratch / origin / case["id"] / variant
for name, source in case["files"].items():
path = project / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_text(source)
report_path = output / "compilation" / origin / f"{case['id']}-{variant}.json"
cmd = [str(java), "-cp", str(classes) + os.pathsep + cp, "PbsProductionCorpusProbe", str(project), "unoptimized" if variant == "unoptimized" else "optimized", str(report_path)]
if variant == "optimized-public": cmd.append("public")
try:
completed = subprocess.run(cmd, cwd=scratch, capture_output=True, text=True, timeout=50, check=True)
report = json.loads(report_path.read_text()); report["stderr"] = completed.stderr
except Exception as failure:
report = {"status": "infrastructure_failure", "exception": {"class": type(failure).__name__, "message": str(failure)}}
expected = case["expectedCompile"]
matched = report["status"] == "compiled" if expected == "compiled" else source_rejection(report)
if origin == "installed" and report.get("frontendOrigin"):
assert all(str(distribution) in report.get(key, "") for key in ["frontendOrigin", "pipelineOrigin", "sdkOrigin"])
sources_unchanged = all((project / name).read_text() == source for name, source in case["files"].items())
assert sources_unchanged
identity = {"caseId": case["id"], "origin": origin, "variant": variant, "sourceSha256": case["sourceSha256"]}
report.update(identity, expectedCompile=expected, matchesExpectation=matched, sourcesUnchanged=sources_unchanged)
if report["status"] == "compiled":
cartridge = Path(report["cartridge"])
for name in ["manifest.json", "program.pbx"]:
target = output / "artifacts" / origin / case["id"] / variant / name
target.parent.mkdir(parents=True, exist_ok=True); target.write_bytes((cartridge / name).read_bytes())
report["manifestSha256"] = sha(cartridge / "manifest.json")
write(report_path, report); observations.append({**identity, "status": report["status"], "matchesExpectation": matched, "report": str(report_path.relative_to(output))})
print(origin, case["id"], variant, report["status"], matched, flush=True)
if report["status"] == "compiled" and variant != "optimized-public":
result = execute(binary, report["cartridge"], project / "execution")
result.update(identity, bytecodeSha256=report["bytecodeSha256"], expectedRuntime=case["expectedRuntime"])
result["matchesExpectation"] = result["observedRuntime"] == case["expectedRuntime"] and result["workerTerminated"]
path = output / "execution" / origin / f"{case['id']}-{variant}.json"; write(path, result)
executions.append({**identity, "observedRuntime": result["observedRuntime"], "matchesExpectation": result["matchesExpectation"], "report": str(path.relative_to(output))})
write(output / "summary.json", {"compilation": observations, "execution": executions})
failures = [x for x in observations + executions if not x["matchesExpectation"]]
expected_runs = sum(c["expectedCompile"] == "compiled" for c in cases) * len(origins) * 2
complete = len(observations) == len(cases)*len(origins)*3 and len(executions) == expected_runs
summary = {"compilation": observations, "execution": executions, "compileCount": len(observations), "executionCount": len(executions),
"normalSuccesses": sum(x["observedRuntime"] == "passed" for x in executions), "deliberateFaults": sum(x["observedRuntime"] == "fault" and x["caseId"] == "runtime-trap-control" for x in executions),
"complete": complete, "failures": failures, "passed": complete and not failures}
write(output / "summary.json", summary)
write(output / "checksums.json", {str(p.relative_to(output)): sha(p) for p in output.rglob("*") if p.is_file()})
print(json.dumps({k:v for k,v in summary.items() if k not in ("compilation", "execution", "failures")}))
return 0 if summary["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())