feat(studio): improve compiler reliability and AI diagnostics
Preserve structured diagnostics, source attribution, phase outcomes and revision context through compiler, Studio, LSP, MCP and JavaFX. Add project service discovery and resilient VS Code reconnection. Correct all eight reproduced PBS production failures in expression context, sibling executable bindings and nominal owner identity. Consolidate specs, lessons, immutable baseline evidence and the maintained source-to-guest qualification runner. Validation: required Gradle gate recorded 724 tests, with 711 passed and 13 skipped. Full corpus passed 168 compilation observations and 108 guest runs (104 successes and four deliberate faults); supplemental controls passed 42 compilations and 16 guest runs. TypeScript compilation, two Node discovery tests and discussion validation passed.
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import p.studio.compiler.FrontendRegistryService;
|
||||
import p.studio.compiler.messages.BuildingIssue;
|
||||
import p.studio.compiler.messages.BuilderPipelineConfig;
|
||||
import p.studio.compiler.models.BuilderPipelineContext;
|
||||
import p.studio.compiler.source.identifiers.FileId;
|
||||
import p.studio.compiler.workspaces.BuilderPipelineService;
|
||||
import p.studio.compiler.workspaces.PipelineStage;
|
||||
import p.studio.compiler.workspaces.stages.*;
|
||||
import p.studio.utilities.logs.LogAggregator;
|
||||
|
||||
import java.nio.file.*;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.*;
|
||||
|
||||
/** Research-only probe: production stages, isolated source projects, no compiler changes. */
|
||||
public final class PbsProductionCorpusProbe {
|
||||
static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
static final String ANALYSIS_ID = UUID.randomUUID().toString();
|
||||
static int issueIndex;
|
||||
static final List<String> LOG = new ArrayList<>();
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
Path root = Path.of(args[0]).toRealPath();
|
||||
boolean optimize = args[1].equals("optimized");
|
||||
boolean publicEntry = args.length > 3 && args[3].equals("public");
|
||||
var result = new LinkedHashMap<String, Object>();
|
||||
result.put("projectRoot", root.toString());
|
||||
result.put("optimization", optimize ? "optimized" : "unoptimized");
|
||||
result.put("entrypoint", publicEntry ? "BuilderPipelineService.compile" : "public analysis + production compile stages");
|
||||
result.put("frontendOrigin", FrontendRegistryService.require("pbs").getClass().getProtectionDomain().getCodeSource().getLocation().toString());
|
||||
result.put("sdkOrigin", p.studio.compiler.pbs.stdlib.StdlibEnvironmentResolver.class.getProtectionDomain().getCodeSource().getLocation().toString());
|
||||
result.put("javaVersion", System.getProperty("java.version"));
|
||||
result.put("pipelineOrigin", BuilderPipelineService.class.getProtectionDomain().getCodeSource().getLocation().toString());
|
||||
var ctx = BuilderPipelineContext.fromConfig(new BuilderPipelineConfig(false, root.toString()));
|
||||
var logs = LogAggregator.with(LOG::add);
|
||||
var phases = new ArrayList<Map<String, Object>>();
|
||||
result.put("phases", phases);
|
||||
result.put("status", "infrastructure_failure");
|
||||
try {
|
||||
if (publicEntry) {
|
||||
var compiled = BuilderPipelineService.INSTANCE.compile(ctx, logs);
|
||||
result.put("diagnostics", compiled.diagnostics().stream().map(i -> issue(i, ctx)).toList());
|
||||
result.put("status", "compiled");
|
||||
} else {
|
||||
var analyzed = BuilderPipelineService.INSTANCE.analyze(ctx, logs);
|
||||
var phase = new LinkedHashMap<String, Object>();
|
||||
phase.put("phase", "analysis");
|
||||
phase.put("diagnostics", analyzed.diagnostics().stream().map(i -> issue(i, ctx)).toList());
|
||||
boolean failed = analyzed.diagnostics().stream().anyMatch(BuildingIssue::isError);
|
||||
phase.put("status", failed ? "failed" : "passed");
|
||||
phases.add(phase);
|
||||
if (failed) result.put("status", "analysis_failed");
|
||||
else {
|
||||
var stages = new ArrayList<PipelineStage>();
|
||||
stages.add(new LowerToIRVMPipelineStage());
|
||||
if (optimize) stages.add(new OptimizeIRVMPipelineStage());
|
||||
stages.add(new EmitBytecodePipelineStage());
|
||||
stages.add(new LinkBytecodePipelineStage());
|
||||
stages.add(new VerifyBytecodePipelineStage());
|
||||
result.put("status", "compiled");
|
||||
for (var stage : stages) {
|
||||
if (!optimize && stage instanceof EmitBytecodePipelineStage) ctx.optimizedIrvm = ctx.irvm;
|
||||
var issues = stage.run(ctx, logs).asCollection();
|
||||
var item = new LinkedHashMap<String, Object>();
|
||||
item.put("phase", stage.getClass().getSimpleName());
|
||||
item.put("diagnostics", issues.stream().map(i -> issue(i, ctx)).toList());
|
||||
boolean errors = issues.stream().anyMatch(BuildingIssue::isError);
|
||||
item.put("status", errors ? "failed" : "passed");
|
||||
phases.add(item);
|
||||
if (errors) { result.put("status", "compile_failed"); break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (result.get("status").equals("compiled")) {
|
||||
Path cartridge = root.resolve("cartridge");
|
||||
Files.createDirectories(cartridge);
|
||||
Files.write(cartridge.resolve("program.pbx"), ctx.bytecodeBytes);
|
||||
var manifest = new LinkedHashMap<String, Object>();
|
||||
manifest.put("magic", "PMTU"); manifest.put("cartridge_version", 1);
|
||||
manifest.put("app_id", 113); manifest.put("title", "PBS production assertion fixture");
|
||||
manifest.put("app_version", "1.0.0"); manifest.put("app_mode", "Game");
|
||||
manifest.put("capabilities", List.of()); manifest.put("asset_table", List.of()); manifest.put("preload", List.of());
|
||||
MAPPER.writerWithDefaultPrettyPrinter().writeValue(cartridge.resolve("manifest.json").toFile(), manifest);
|
||||
result.put("cartridge", cartridge.toString());
|
||||
result.put("bytecodeBytes", ctx.bytecodeBytes.length);
|
||||
result.put("bytecodeSha256", HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(ctx.bytecodeBytes)));
|
||||
}
|
||||
} catch (Throwable failure) {
|
||||
result.put("status", failure instanceof p.studio.compiler.exceptions.BuildException
|
||||
? "public_compile_rejected" : "pipeline_exception");
|
||||
result.put("exception", causes(failure));
|
||||
if (failure instanceof p.studio.compiler.exceptions.BuildException buildFailure)
|
||||
result.put("diagnostics", buildFailure.getDiagnostics().stream().map(i -> issue(i,ctx)).toList());
|
||||
}
|
||||
if (ctx.fileTable != null) {
|
||||
var files = new LinkedHashMap<String, Object>();
|
||||
for (int i = 0; i < ctx.fileTable.size(); i++) {
|
||||
var f = ctx.fileTable.get(new FileId(i));
|
||||
files.put(Integer.toString(i), f.getCanonPath().toString());
|
||||
}
|
||||
result.put("files", files);
|
||||
}
|
||||
if (ctx.irBackend != null) {
|
||||
var functions = new ArrayList<Map<String, Object>>();
|
||||
for (var fn : ctx.irBackend.getExecutableFunctions()) {
|
||||
var file = ctx.fileTable.get(fn.fileId()).getCanonPath();
|
||||
if (!file.startsWith(root)) continue;
|
||||
var f = new LinkedHashMap<String, Object>();
|
||||
f.put("callableId", fn.callableId().toString()); f.put("name", fn.callableName());
|
||||
f.put("file", root.relativize(file).toString());
|
||||
f.put("instructions", fn.instructions().stream().map(Object::toString).toList());
|
||||
functions.add(f);
|
||||
}
|
||||
result.put("projectFunctions", functions);
|
||||
}
|
||||
result.put("inputs", p.studio.compiler.source.diagnostics.CompilerDiagnosticProjection.inputs(
|
||||
p.studio.compiler.source.diagnostics.SourceReference.snapshot(ctx.fileTable), root));
|
||||
result.put("logs", LOG);
|
||||
Path report = Path.of(args[2]); Files.createDirectories(report.getParent());
|
||||
MAPPER.writerWithDefaultPrettyPrinter().writeValue(report.toFile(), result);
|
||||
System.out.println(result.get("status") + " " + root.getFileName());
|
||||
}
|
||||
|
||||
static Map<String, Object> issue(BuildingIssue i, BuilderPipelineContext ctx) {
|
||||
var sources = p.studio.compiler.source.diagnostics.SourceReference.snapshot(ctx.fileTable);
|
||||
return p.studio.compiler.source.diagnostics.CompilerDiagnosticProjection.issue(i, sources,
|
||||
p.studio.compiler.source.diagnostics.CompilerDiagnosticProjection.capturedText(ctx.fileTable),
|
||||
ctx.rootProjectPathCanon, ANALYSIS_ID + ":" + issueIndex++);
|
||||
}
|
||||
|
||||
static List<Map<String, String>> causes(Throwable t) {
|
||||
var result = new ArrayList<Map<String, String>>();
|
||||
for (int n = 0; t != null && n < 8; n++, t = t.getCause())
|
||||
result.add(Map.of("class", t.getClass().getName(), "message", String.valueOf(t.getMessage())));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
# PBS production corpus qualification
|
||||
|
||||
Owner: compiler/pbs. This maintained runner preserves the frozen 28-case input and writes each run to a new directory. It never rebuilds or writes the runtime provider and does not modify a game workspace or memcard.
|
||||
|
||||
Build the candidate with the required regression gate, then run the corpus:
|
||||
|
||||
```bash
|
||||
./gradlew :prometeu-compiler:frontends:prometeu-frontend-pbs:test :prometeu-compiler:prometeu-build-pipeline:test :prometeu-lsp:prometeu-lsp-v1:test :prometeu-app:test :prometeu-app:installedKnowledgeTest --console=plain
|
||||
```
|
||||
|
||||
```bash
|
||||
python3 tools/pbs-production-corpus/run.py --corpus prometeu-compiler/frontends/prometeu-frontend-pbs/src/test/resources/production-regressions/cases.json --distribution prometeu-app/build/install/prometeu-app --runtime-headless /absolute/existing/prometeu-headless --output build/pbs-production-corpus/new-run
|
||||
```
|
||||
|
||||
Each case is compiled in a fresh JVM/project outside the checkout. Checkout mode puts current classes/resources first; installed mode includes only candidate JARs and runs outside the checkout. The runner verifies candidate product class freshness and records Java, SDK/compiler origins, JAR/source/PBX/manifest/runtime hashes, Git/worktree state, diagnostic phases and protocol transcripts.
|
||||
|
||||
The required corpus produces 168 observations: 28 cases × checkout/installed × optimized production stages, investigation-only unoptimized stages, and the public optimized facade. Unoptimized means omitting OptimizeIRVMPipelineStage and emitting original IRVM; it is not a public compiler flag. The expected runtime matrix is 108 executions: 27 compiled cases × checkout/installed × two optimization configurations, with 104 normal three-frame successes and four deliberately faulting controls. The file-private case must remain a structured source rejection in all six compile configurations.
|
||||
|
||||
Use --select id1,id2 or --origins checkout/installed for focused investigation. Without those restrictions, exit zero requires the complete expected matrix and classifications. Output directories cannot be reused or target the original AGD-0113 baseline. Runtime processes are terminated and waited for on every path. Logs are local evidence, while summary.json and per-case JSON preserve structured results.
|
||||
|
||||
The focused-cases.json resource is an additional matrix for contextual overloads, noncommutative receiver/argument order, original global storage, service singleton identity and privacy. It does not replace or alter the frozen corpus. Successful compiler proof does not imply qualification of an installed runtime distribution or a live Studio session. The Studio tests independently preserve MCP/LSP diagnostic contracts.
|
||||
|
||||
The completed DEC-0088/PLN-0166 result, original baseline and separate post-fix reports are consolidated in [LSN-0106](../../discussion/lessons/DSC-0111-pbs-production-reproductions-and-fixes/LSN-0106-pbs-production-binding-and-lowering-corrections.md).
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Isolated production PBS regression matrix. Never writes the received baseline or runtime checkout."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import selectors
|
||||
import signal
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MODULES = ["prometeu-infra", "prometeu-animation", "prometeu-compiler/prometeu-compiler-core",
|
||||
"prometeu-compiler/prometeu-frontend-api", "prometeu-compiler/frontends/prometeu-frontend-pbs",
|
||||
"prometeu-compiler/prometeu-frontend-registry", "prometeu-compiler/prometeu-deps",
|
||||
"prometeu-compiler/prometeu-build-pipeline"]
|
||||
|
||||
|
||||
def sha(path):
|
||||
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def write(path, data):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
class Session:
|
||||
def __init__(self, binary, directory):
|
||||
self.stderr_path = directory / "runtime.stderr"
|
||||
self.stderr = self.stderr_path.open("wb")
|
||||
self.process = subprocess.Popen([str(binary), "--stdio"], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=self.stderr, cwd=directory, start_new_session=True)
|
||||
self.selector = selectors.DefaultSelector()
|
||||
self.selector.register(self.process.stdout, selectors.EVENT_READ)
|
||||
self.buffer = bytearray()
|
||||
self.transcript = []
|
||||
self.request = 0
|
||||
|
||||
def event(self):
|
||||
deadline = time.monotonic() + 12
|
||||
while b"\n" not in self.buffer:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0 or not self.selector.select(remaining):
|
||||
raise TimeoutError("Runtime reply deadline")
|
||||
data = os.read(self.process.stdout.fileno(), 65536)
|
||||
if not data:
|
||||
raise EOFError("Runtime closed stdout")
|
||||
self.buffer.extend(data)
|
||||
line, _, self.buffer = self.buffer.partition(b"\n")
|
||||
value = json.loads(line)
|
||||
self.transcript.append({"direction": "received", "message": value})
|
||||
return value
|
||||
|
||||
def call(self, command, params=None):
|
||||
self.request += 1
|
||||
request = {"requestId": f"pbs-corpus-{self.request}", "command": command, "params": params or {}}
|
||||
self.transcript.append({"direction": "sent", "message": request})
|
||||
self.process.stdin.write(json.dumps(request).encode() + b"\n")
|
||||
self.process.stdin.flush()
|
||||
while True:
|
||||
response = self.event()
|
||||
if response.get("requestId") != request["requestId"]:
|
||||
raise AssertionError("Unexpected response correlation")
|
||||
if response.get("type") == "progress":
|
||||
continue
|
||||
if "error" in response:
|
||||
raise RuntimeError(json.dumps(response["error"]))
|
||||
return response.get("result")
|
||||
|
||||
def close(self):
|
||||
try:
|
||||
if self.process.poll() is None:
|
||||
try:
|
||||
self.call("shutdown")
|
||||
self.process.wait(timeout=5)
|
||||
except Exception:
|
||||
os.killpg(self.process.pid, signal.SIGKILL)
|
||||
self.process.wait(timeout=5)
|
||||
finally:
|
||||
self.selector.close()
|
||||
self.stderr.close()
|
||||
self.process.stdin.close()
|
||||
self.process.stdout.close()
|
||||
|
||||
|
||||
def execute(binary, cartridge, work):
|
||||
work.mkdir(parents=True)
|
||||
result = {"observedRuntime": "infrastructure_failure"}
|
||||
session = Session(binary, work)
|
||||
try:
|
||||
hello = session.event()
|
||||
assert hello["type"] == "hello" and hello["protocolVersion"]["major"] == 1
|
||||
required = ["boot", "advance", "get_state", "get_fault", "shutdown"]
|
||||
assert set(required) <= set(hello["capabilities"])
|
||||
result["hello"] = hello
|
||||
session.call("initialize", {"protocolVersion": {"major": 1, "minor": min(1, hello["protocolVersion"]["minor"])},
|
||||
"requiredCapabilities": required})
|
||||
result["boot"] = session.call("boot", {"path": cartridge})
|
||||
result["advance"] = advance = session.call("advance", {"frames": 3})
|
||||
result["fault"] = fault = session.call("get_fault")
|
||||
result["state"] = session.call("get_state")
|
||||
fault = fault.get("fault") if isinstance(fault, dict) and "fault" in fault else fault
|
||||
result["observedRuntime"] = "fault" if fault is not None else "passed" if advance.get("completed") == 3 else "incomplete"
|
||||
except Exception as failure:
|
||||
result["exception"] = {"class": type(failure).__name__, "message": str(failure)}
|
||||
finally:
|
||||
session.close()
|
||||
result.update(workerTerminated=session.process.poll() is not None, processExitCode=session.process.returncode,
|
||||
transcript=session.transcript, stderr=session.stderr_path.read_text(errors="replace"))
|
||||
return result
|
||||
|
||||
|
||||
def source_rejection(report):
|
||||
issues = report.get("diagnostics", []) + [i for p in report.get("phases", []) for i in p.get("diagnostics", [])]
|
||||
return report.get("status") in ("analysis_failed", "public_compile_rejected") and any(i.get("severity") == "ERROR" and (i.get("failureKind") == "program-rejection" or str(i.get("code", "")).startswith("E_")) for i in issues) \
|
||||
and not any(i.get("failureKind") in ("internal-failure", "infrastructure-failure", "verification-limit") for i in issues)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--corpus", type=Path, required=True)
|
||||
parser.add_argument("--distribution", type=Path, required=True)
|
||||
parser.add_argument("--runtime-headless", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--select", help="Comma-separated IDs; omitted for the complete 168/108 gate")
|
||||
parser.add_argument("--origins", choices=["checkout", "installed", "both"], default="both")
|
||||
args = parser.parse_args()
|
||||
corpus, distribution, binary, output = args.corpus.resolve(strict=True), args.distribution.resolve(strict=True), args.runtime_headless.resolve(strict=True), args.output.resolve()
|
||||
if output.exists():
|
||||
parser.error("Output must be a new directory; evidence is never overwritten")
|
||||
baseline = ROOT / "discussion/workflow/agendas/AGD-0113-evidence"
|
||||
if output == baseline or baseline in output.parents:
|
||||
parser.error("The frozen baseline is read-only")
|
||||
output.mkdir(parents=True)
|
||||
cases = json.loads(corpus.read_text())["cases"]
|
||||
if args.select:
|
||||
selected = set(args.select.split(",")); assert selected <= {c["id"] for c in cases}
|
||||
cases = [c for c in cases if c["id"] in selected]
|
||||
scratch = Path(tempfile.mkdtemp(prefix="pbs-production-corpus-"))
|
||||
assert not scratch.is_relative_to(ROOT)
|
||||
java, javac = distribution / "runtime/bin/java", distribution / "runtime/bin/javac"
|
||||
jars = sorted((distribution / "lib").glob("*.jar")); assert jars
|
||||
# Verify candidate product classes before mixing checkout classes with third-party dependencies.
|
||||
freshness = []
|
||||
for module in MODULES:
|
||||
jar = next(p for p in jars if p.name == Path(module).name + ".jar" or p.name.startswith(Path(module).name + "-"))
|
||||
with zipfile.ZipFile(jar) as archive:
|
||||
classes = ROOT / module / "build/classes/java/main"
|
||||
compared = list(classes.rglob("*.class")); assert compared
|
||||
different = [str(p.relative_to(classes)) for p in compared if archive.read(str(p.relative_to(classes))) != p.read_bytes()]
|
||||
assert not different, (module, different)
|
||||
freshness.append({"module": module, "jarSha256": sha(jar), "matchingClasses": len(compared)})
|
||||
provenance = {"schemaVersion": 1, "corpusSha256": sha(corpus), "runtime": str(binary), "runtimeSha256": sha(binary),
|
||||
"platform": platform.platform(), "gitHead": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip(),
|
||||
"worktreeStatus": subprocess.check_output(["git", "status", "--short"], cwd=ROOT, text=True).splitlines(),
|
||||
"java": subprocess.run([str(java), "-version"], capture_output=True, text=True, check=True).stderr,
|
||||
"distribution": str(distribution), "jars": {p.name: sha(p) for p in jars}, "candidateFreshness": freshness,
|
||||
"probeSha256": sha(Path(__file__).resolve().with_name("PbsProductionCorpusProbe.java")), "runnerSha256": sha(__file__),
|
||||
"scratch": str(scratch), "unoptimizedVariant": "Investigation-only: omit OptimizeIRVMPipelineStage and emit original IRVM; not a public compiler flag"}
|
||||
write(output / "provenance.json", provenance)
|
||||
observations, executions = [], []
|
||||
origins = ["checkout", "installed"] if args.origins == "both" else [args.origins]
|
||||
for origin in origins:
|
||||
installed = os.pathsep.join(map(str, jars))
|
||||
entries = [ROOT / m / suffix for m in MODULES for suffix in ["build/classes/java/main", "build/resources/main"] if (ROOT / m / suffix).exists()]
|
||||
cp = os.pathsep.join(map(str, entries)) + os.pathsep + installed if origin == "checkout" else installed
|
||||
classes = scratch / origin / "classes"; classes.mkdir(parents=True)
|
||||
subprocess.run([str(javac), "-proc:none", "-cp", cp, "-d", str(classes), str(Path(__file__).resolve().with_name("PbsProductionCorpusProbe.java"))], cwd=scratch, check=True)
|
||||
for case in cases:
|
||||
for variant in ["optimized", "unoptimized", "optimized-public"]:
|
||||
project = scratch / origin / case["id"] / variant
|
||||
for name, source in case["files"].items():
|
||||
path = project / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_text(source)
|
||||
report_path = output / "compilation" / origin / f"{case['id']}-{variant}.json"
|
||||
cmd = [str(java), "-cp", str(classes) + os.pathsep + cp, "PbsProductionCorpusProbe", str(project), "unoptimized" if variant == "unoptimized" else "optimized", str(report_path)]
|
||||
if variant == "optimized-public": cmd.append("public")
|
||||
try:
|
||||
completed = subprocess.run(cmd, cwd=scratch, capture_output=True, text=True, timeout=50, check=True)
|
||||
report = json.loads(report_path.read_text()); report["stderr"] = completed.stderr
|
||||
except Exception as failure:
|
||||
report = {"status": "infrastructure_failure", "exception": {"class": type(failure).__name__, "message": str(failure)}}
|
||||
expected = case["expectedCompile"]
|
||||
matched = report["status"] == "compiled" if expected == "compiled" else source_rejection(report)
|
||||
if origin == "installed" and report.get("frontendOrigin"):
|
||||
assert all(str(distribution) in report.get(key, "") for key in ["frontendOrigin", "pipelineOrigin", "sdkOrigin"])
|
||||
sources_unchanged = all((project / name).read_text() == source for name, source in case["files"].items())
|
||||
assert sources_unchanged
|
||||
identity = {"caseId": case["id"], "origin": origin, "variant": variant, "sourceSha256": case["sourceSha256"]}
|
||||
report.update(identity, expectedCompile=expected, matchesExpectation=matched, sourcesUnchanged=sources_unchanged)
|
||||
if report["status"] == "compiled":
|
||||
cartridge = Path(report["cartridge"])
|
||||
for name in ["manifest.json", "program.pbx"]:
|
||||
target = output / "artifacts" / origin / case["id"] / variant / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True); target.write_bytes((cartridge / name).read_bytes())
|
||||
report["manifestSha256"] = sha(cartridge / "manifest.json")
|
||||
write(report_path, report); observations.append({**identity, "status": report["status"], "matchesExpectation": matched, "report": str(report_path.relative_to(output))})
|
||||
print(origin, case["id"], variant, report["status"], matched, flush=True)
|
||||
if report["status"] == "compiled" and variant != "optimized-public":
|
||||
result = execute(binary, report["cartridge"], project / "execution")
|
||||
result.update(identity, bytecodeSha256=report["bytecodeSha256"], expectedRuntime=case["expectedRuntime"])
|
||||
result["matchesExpectation"] = result["observedRuntime"] == case["expectedRuntime"] and result["workerTerminated"]
|
||||
path = output / "execution" / origin / f"{case['id']}-{variant}.json"; write(path, result)
|
||||
executions.append({**identity, "observedRuntime": result["observedRuntime"], "matchesExpectation": result["matchesExpectation"], "report": str(path.relative_to(output))})
|
||||
write(output / "summary.json", {"compilation": observations, "execution": executions})
|
||||
failures = [x for x in observations + executions if not x["matchesExpectation"]]
|
||||
expected_runs = sum(c["expectedCompile"] == "compiled" for c in cases) * len(origins) * 2
|
||||
complete = len(observations) == len(cases)*len(origins)*3 and len(executions) == expected_runs
|
||||
summary = {"compilation": observations, "execution": executions, "compileCount": len(observations), "executionCount": len(executions),
|
||||
"normalSuccesses": sum(x["observedRuntime"] == "passed" for x in executions), "deliberateFaults": sum(x["observedRuntime"] == "fault" and x["caseId"] == "runtime-trap-control" for x in executions),
|
||||
"complete": complete, "failures": failures, "passed": complete and not failures}
|
||||
write(output / "summary.json", summary)
|
||||
write(output / "checksums.json", {str(p.relative_to(output)): sha(p) for p in output.rglob("*") if p.is_file()})
|
||||
print(json.dumps({k:v for k,v in summary.items() if k not in ("compilation", "execution", "failures")}))
|
||||
return 0 if summary["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user