verify real MCP runtime and DEV framebuffer parity

This commit is contained in:
2026-09-29 11:40:02 +01:00
parent 379420e7a9
commit 4f148a50de
7 changed files with 419 additions and 9 deletions
@@ -0,0 +1,280 @@
package p.studio.app.runtime;
import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.databind.node.ObjectNode;
import javafx.application.Platform;
import javafx.scene.Scene;
import javafx.scene.control.*;
import javafx.scene.image.ImageView;
import javafx.stage.Stage;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.context.ConfigurableApplicationContext;
import p.studio.api.automation.*;
import p.studio.api.project.*;
import p.studio.api.session.*;
import p.studio.app.StudioApplicationConfiguration;
import p.studio.app.mcp.LocalMcpServer;
import p.studio.core.project.ProjectLocalStudioSetupStore;
import p.studio.ui.javafx.events.StudioEventBus;
import p.studio.ui.javafx.utilities.i18n.*;
import p.studio.ui.javafx.workspaces.dev.DevWorkspace;
import java.io.ByteArrayInputStream;
import java.net.http.*;
import java.nio.file.*;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.*;
import static org.junit.jupiter.api.Assertions.*;
/** Full real loop: external HTTP client -> app -> Rust JSONL -> MCP image and JavaFX DEV. */
class HeadlessMcpDesktopTest {
@TempDir Path root;
final ObjectMapper mapper = new ObjectMapper();
final Set<ProcessHandle> owned = new HashSet<>();
ConfigurableApplicationContext app;
ProjectSession project;
LocalMcpServer server;
HttpClient client;
String connection, task, build, session;
Path installation, source;
DevWorkspace dev;
Stage stage;
I18nService locale;
@BeforeAll static void toolkit() throws Exception {
var ready = new CompletableFuture<Void>();
Platform.startup(() -> { Platform.setImplicitExit(false); ready.complete(null); });
ready.get(10, TimeUnit.SECONDS);
}
@AfterAll static void stopToolkit() { Platform.exit(); }
static void fx(Runnable work) throws Exception {
var completed = new CompletableFuture<Void>();
Platform.runLater(() -> { try { work.run(); completed.complete(null); } catch (Throwable failure) { completed.completeExceptionally(failure); } });
completed.get(10, TimeUnit.SECONDS);
}
@BeforeEach void open() throws Exception {
String configured = System.getProperty("studio.test.runtimeInstall", "");
assertFalse(configured.isBlank(), "Supply -PstudioRuntimeInstall=<runtime distribution>");
installation = Path.of(configured).toRealPath(); root = root.toRealPath();
app = new SpringApplicationBuilder(StudioApplicationConfiguration.class).web(WebApplicationType.NONE).headless(false).run();
for (var ref : p.studio.compiler.FrontendRegistryService.require("pbs").documentation().orElseThrow().references(1, p.studio.AppMode.Game)) {
if (!ref.id().startsWith("pbs:example:game:1:")) continue;
Path file = root.resolve(ref.id().substring("pbs:example:game:1:".length()));
Files.createDirectories(file.getParent()); Files.writeString(file, ref.text());
}
source = root.resolve("src/main.pbs");
var reference = app.getBean(ProjectCatalog.class).openProject(root);
app.getBean(ProjectLocalStudioSetupStore.class).save(reference, new ProjectLocalStudioSetup(installation.toString(), null));
project = app.getBean(ProjectSessionFactory.class).open(reference);
server = app.getBean(LocalMcpServer.class); client = HttpClient.newHttpClient();
fx(() -> {
locale = new I18nService(); locale.setLocale(Locale.ENGLISH);
dev = new DevWorkspace(reference, project.automation(), locale, new StudioEventBus(), app.getBean(AgentConnectionGuide.class));
stage = new Stage(); stage.setTitle("Real MCP / headless / DEV conformance");
stage.setScene(new Scene((javafx.scene.Parent) dev.rootNode(), 1000, 800)); stage.show(); dev.load();
});
var initialize = request("initialize");
var params = initialize.putObject("params").put("protocolVersion", "2025-06-18");
params.putObject("capabilities"); params.putObject("clientInfo").put("name", "real-loop-test").put("version", "1");
var response = post(initialize); assertEquals(200, response.statusCode());
connection = response.headers().firstValue("Mcp-Session-Id").orElseThrow();
assertEquals(202, post(mapper.createObjectNode().put("jsonrpc", "2.0").put("method", "notifications/initialized")).statusCode());
}
@AfterEach void close() throws Exception {
try { fx(() -> { if (dev != null) dev.unLoad(); if (stage != null) stage.close(); }); }
finally {
try { if (project != null) project.close(); }
finally { if (client != null) client.close(); if (app != null) app.close(); }
}
assertTrue(owned.stream().noneMatch(ProcessHandle::isAlive), "No managed runtime may survive the test");
}
@Test void fullHttpLoopMatchesRawRuntimePixelsAndDisplayedDevAcrossResetAndRebuild() throws Exception {
assertEquals(17, mapper.readTree(post(request("tools/list")).body()).path("result").path("tools").size());
assertTrue(value(call("studio_project_context", args())).path("languages").toString().contains("pbs"));
value(call("studio_reference_read", args().put("referenceId", "pbs:quickstart").put("limit", 100)));
var sdk = value(call("studio_reference_read", args().put("referenceId", "pbs:sdk:game:1:sdk/input/main.pbs").put("limit", 1000)));
assertTrue(sdk.toString().contains("InputPad"), "Consult installed SDK source through the indexed read-only reference");
assertEquals("NOT_GRANTED", call("studio_control_acquire", args()).path("structuredContent").path("error").path("code").asText());
fx(() -> ((Button) dev.rootNode().lookup("#dev-grant")).fire());
assertEquals(ControlState.WAITING_FOR_AGENT, project.automation().snapshot().state());
task = value(call("studio_control_acquire", args())).path("taskId").asText();
assertFalse(task.isBlank());
// Agent-native edit of the installed example; SDK remains unchanged.
String edited = Files.readString(source).replace("import { Gfx } from @sdk:gfx;",
"import { Gfx } from @sdk:gfx;\nimport { Input } from @sdk:input;")
.replace("x = x + 1;", "if (Input.pad().right().down()) { x = x + 3; } else { x = x + 1; }");
Files.writeString(source, edited);
var query = args().put("operation", "diagnostics"); query.putObject("parameters").put("uri", source.toUri().toString());
assertTrue(value(call("studio_language_query", query)).path("result").isEmpty());
var symbols = args().put("operation", "documentSymbols"); symbols.putObject("parameters").put("uri", source.toUri().toString());
assertTrue(value(call("studio_language_query", symbols)).path("result").toString().contains("frame"));
int inputOffset = edited.indexOf("x = x + 3;") + "x = ".length();
String preceding = edited.substring(0, inputOffset);
var definition = args().put("operation", "definition");
definition.putObject("parameters").put("uri", source.toUri().toString()).putObject("position")
.put("line", preceding.chars().filter(c -> c == '\n').count())
.put("character", inputOffset - preceding.lastIndexOf('\n') - 1);
var locations = value(call("studio_language_query", definition)).path("result");
assertFalse(locations.isEmpty(), "Navigate the saved PBS source through the actual LSP");
assertEquals(source.toUri().toString(), locations.get(0).path("uri").asText());
build("first-build"); boot();
assertEquals("NO_FRAME_AVAILABLE", call("studio_headless_capture", args()).path("structuredContent").path("error").path("code").asText());
List<JsonNode> first;
try (var oracle = new DirectHeadlessOracle(installation, root.resolve("build"))) {
first = timeline("first", oracle);
}
value(call("studio_headless_reset", args())); assertStale(true);
String previousSession = session; boot(); assertEquals(previousSession, session);
try (var oracle = new DirectHeadlessOracle(installation, root.resolve("build"))) {
assertSameTimeline(first, timeline("reset", oracle));
}
value(call("studio_headless_shutdown", args())); assertStale(true);
boot(); assertNotEquals(previousSession, session);
try (var oracle = new DirectHeadlessOracle(installation, root.resolve("build"))) {
assertSameTimeline(first, timeline("fresh", oracle));
}
String oldBuild = build;
Files.writeString(source, edited.replace("Color.rgb(16, 20, 28)", "Color.rgb(80, 40, 20)"));
assertTrue(value(call("studio_language_query", query)).path("result").isEmpty());
build("rebuild"); assertNotEquals(oldBuild, build); assertStale(true);
var historical = value(call("studio_operation_get", args().put("clientRequestId", "fresh-0")));
assertTrue(historical.path("capture").path("stale").asBoolean());
boot();
try (var oracle = new DirectHeadlessOracle(installation, root.resolve("build"))) {
var rebuilt = timeline("rebuilt", oracle);
assertFalse(Arrays.equals(png(first.getFirst()), png(rebuilt.getFirst())));
}
value(call("studio_control_release", args()));
assertEquals(ControlState.MANUAL, project.automation().snapshot().state()); assertStale(true);
fx(() -> assertFalse(((Button) dev.rootNode().lookup("#dev-grant")).isDisabled()));
assertTrue(Files.readString(source).contains("Color.rgb(80, 40, 20)"));
}
List<JsonNode> timeline(String prefix, DirectHeadlessOracle oracle) throws Exception {
// RIGHT held for two frames, released for one, held for another three.
int[] frames = {2, 1, 3}; int[] logical = {2, 3, 6}; int[] x = {3, 6, 13};
var captures = new ArrayList<JsonNode>();
for (int i = 0; i < frames.length; i++) {
if (i == 1) { value(call("studio_headless_clear_input", args())); oracle.call("clear_input"); }
else {
var input = mapper.createObjectNode().put("right", true);
value(call("studio_headless_set_input", args().set("input", input))); oracle.call("set_input", input);
}
String requestId = prefix + "-" + i;
var receipt = value(call("studio_headless_advance", args().put("clientRequestId", requestId).put("frames", frames[i])));
var result = completed(requestId);
assertEquals("SUCCEEDED", value(result).path("result").path("status").asText());
assertEquals(frames[i], value(result).path("progress").path("completedFrames").asInt());
// Retrying the same receipt must not execute another batch of frames.
assertEquals(receipt.path("operationId"), value(call("studio_headless_advance",
args().put("clientRequestId", requestId).put("frames", frames[i]))).path("operationId"));
var advanced = oracle.call("advance", mapper.createObjectNode().put("frames", frames[i]));
assertEquals("completed", advanced.path("stop").asText());
var raw = oracle.call("capture");
var image = call("studio_headless_capture", args());
assertArrayEquals(png(result), png(image));
assertEquals(value(result).path("capture"), value(image));
compareCapture(raw, image, logical[i], x[i]);
captures.add(raw.deepCopy());
var actualState = value(call("studio_headless_state", args())).path("state");
assertEquals(oracle.call("get_state"), actualState);
assertEquals(oracle.call("get_fault"), value(call("studio_headless_fault", args())).path("fault"));
}
return captures;
}
void compareCapture(JsonNode raw, JsonNode image, int logical, int rectangleX) throws Exception {
var metadata = value(image);
for (String field : List.of("logicalFrameIndex", "frameId", "width", "height", "stridePixels", "pixelFormat", "mimeType"))
assertEquals(raw.path(field), metadata.path(field), field);
assertEquals(logical, metadata.path("logicalFrameIndex").asInt());
assertEquals(build, metadata.path("buildId").asText()); assertEquals(session, metadata.path("sessionId").asText());
byte[] expected = Base64.getDecoder().decode(raw.path("data").asText());
assertArrayEquals(expected, png(image));
final java.awt.image.BufferedImage decoded;
try (var encoded = new javax.imageio.stream.MemoryCacheImageInputStream(new ByteArrayInputStream(expected))) {
var decoder = javax.imageio.ImageIO.getImageReaders(encoded).next();
try { decoder.setInput(encoded); decoded = decoder.read(0); }
finally { decoder.dispose(); }
}
int width = decoded.getWidth(), height = decoded.getHeight();
assertEquals(metadata.path("width").asInt(), width); assertEquals(metadata.path("height").asInt(), height);
assertEquals(0xfff0a030, decoded.getRGB(rectangleX, 32)); // Actual input-sensitive rectangle, not only matching blank images.
assertNotEquals(0xfff0a030, decoded.getRGB(rectangleX - 1, 32));
int[] pixels = decoded.getRGB(0, 0, width, height, null, 0, width);
var snapshot = project.automation().snapshot();
assertArrayEquals(expected, snapshot.capture().png());
assertEquals(metadata.path("frameId").bigIntegerValue(), snapshot.capture().frameId());
fx(() -> {
var canvas = (ImageView) dev.rootNode().lookup("#dev-framebuffer");
assertNotNull(canvas.getImage()); assertFalse(canvas.getImage().isError());
assertEquals(width, (int) canvas.getImage().getWidth()); assertEquals(height, (int) canvas.getImage().getHeight());
var reader = canvas.getImage().getPixelReader();
int[] displayed = new int[width * height];
reader.getPixels(0, 0, width, height, javafx.scene.image.PixelFormat.getIntArgbInstance(), displayed, 0, width);
assertArrayEquals(pixels, displayed, "Every displayed framebuffer pixel must match the Rust PNG");
assertFalse(canvas.isSmooth());
String label = ((Label) dev.rootNode().lookup("#dev-capture-identity")).getText();
assertEquals("session=" + session + " | build=" + build + " | logical=" + logical
+ " | FrameId=" + metadata.path("frameId").asText() + " | " + width + "×" + height
+ " | stride=" + metadata.path("stridePixels").asText(), label);
});
assertStale(false);
}
void assertStale(boolean stale) throws Exception {
assertEquals(stale, project.automation().snapshot().stale());
fx(() -> assertEquals(locale.text(stale ? I18n.DEV_STALE_FRAME : I18n.DEV_CURRENT_FRAME),
((Label) dev.rootNode().lookup("#dev-image-status")).getText()));
}
void assertSameTimeline(List<JsonNode> expected, List<JsonNode> actual) {
assertEquals(expected, actual, "Raw runtime captures, pixels and both frame identities repeat exactly");
}
void build(String requestId) throws Exception {
value(call("studio_build_start", args().put("clientRequestId", requestId)));
var result = value(completed(requestId)).path("result");
assertEquals("SUCCESS", result.path("status").asText(), result.toString());
build = result.path("buildId").asText(); assertFalse(build.isBlank());
}
void boot() throws Exception {
session = value(call("studio_headless_boot", args().put("buildId", build))).path("sessionId").asText();
assertFalse(session.isBlank());
ProcessHandle.current().descendants().filter(p -> p.info().command().orElse("").startsWith(installation + "/")).forEach(owned::add);
assertFalse(owned.isEmpty(), "Observe the actual packaged runtime, not an in-process replacement");
}
JsonNode completed(String id) throws Exception {
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
do {
var reply = call("studio_operation_get", args().put("clientRequestId", id));
var result = value(reply);
if (result.path("workerTerminated").asBoolean()) { assertTrue(result.path("failure").isNull(), result.toString()); return reply; }
Thread.sleep(10);
} while (System.nanoTime() < deadline);
throw new AssertionError("Operation did not finish: " + id);
}
ObjectNode args() { var args = mapper.createObjectNode().put("projectRoot", root.toString()); if (task != null) args.put("taskId", task); return args; }
ObjectNode request(String method) { return mapper.createObjectNode().put("jsonrpc", "2.0").put("id", UUID.randomUUID().toString()).put("method", method); }
JsonNode call(String name, ObjectNode args) throws Exception {
var request = request("tools/call"); request.putObject("params").put("name", name).set("arguments", args);
var response = post(request); assertEquals(200, response.statusCode());
var body = mapper.readTree(response.body()); assertFalse(body.has("error"), body.toString()); return body.path("result");
}
HttpResponse<String> post(JsonNode body) throws Exception {
var request = HttpRequest.newBuilder(server.connection().endpoint()).timeout(Duration.ofSeconds(15))
.header("Authorization", server.connection().authorizationHeader()).header("Content-Type", "application/json")
.header("Accept", "application/json, text/event-stream").header("MCP-Protocol-Version", "2025-06-18");
if (connection != null) request.header("Mcp-Session-Id", connection);
return client.send(request.POST(HttpRequest.BodyPublishers.ofString(body.toString())).build(), HttpResponse.BodyHandlers.ofString());
}
JsonNode value(JsonNode result) { assertFalse(result.path("isError").asBoolean(), result.toString()); return result.path("structuredContent").path("value"); }
byte[] png(JsonNode result) {
if (result.has("data")) return Base64.getDecoder().decode(result.path("data").asText());
assertEquals("image", result.path("content").get(1).path("type").asText());
assertEquals("image/png", result.path("content").get(1).path("mimeType").asText());
return Base64.getDecoder().decode(result.path("content").get(1).path("data").asText());
}
}