implement real headless process gateway and supervision

This commit is contained in:
2026-09-29 11:02:21 +01:00
parent 9592c9b274
commit bc76f08bbd
13 changed files with 771 additions and 5 deletions
@@ -0,0 +1,306 @@
package p.studio.app.runtime;
import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.databind.*;
import p.studio.api.automation.*;
import p.studio.core.runtime.HeadlessGateway;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
import static p.studio.app.runtime.HeadlessWire.*;
/** One supervised process, bounded JSONL, no retry/replay and no simulated-time clock. */
final class HeadlessProcessSession implements HeadlessGateway.Session {
private final Process process;
private final ObjectMapper mapper;
private final HeadlessLimits limits;
private final String buildId;
private volatile String sessionId;
private volatile JsonNode hello;
private Path cartridge;
private final CompletableFuture<JsonNode> greeting = new CompletableFuture<>();
private final Map<String, Pending> pending = new ConcurrentHashMap<>();
private final Set<ProcessHandle> descendants = ConcurrentHashMap.newKeySet();
private final ExecutorService io = Executors.newVirtualThreadPerTaskExecutor();
private final ExecutorService writer = Executors.newSingleThreadExecutor(Thread.ofPlatform().daemon().name("headless-writer").factory());
private final ScheduledExecutorService timer = Executors.newSingleThreadScheduledExecutor(Thread.ofPlatform().daemon().name("headless-supervisor").factory());
private final ReentrantLock work = new ReentrantLock();
private final AtomicBoolean dead = new AtomicBoolean();
private final AtomicReference<Pending> advance = new AtomicReference<>();
private final Deque<String> stderr = new ArrayDeque<>();
private volatile boolean closing;
private CompletableFuture<Void> shutdown;
HeadlessProcessSession(Process process, ObjectMapper source, HeadlessLimits limits, String buildId) {
this.process = process; this.limits = limits; this.buildId = buildId;
mapper = source.copy().enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
mapper.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(limits.maxResponseBytes()).build());
timer.scheduleAtFixedRate(() -> { if (process.isAlive()) process.descendants().forEach(descendants::add); }, 0, 20, TimeUnit.MILLISECONDS);
io.execute(this::readOutput); io.execute(this::readErrors);
process.onExit().thenRun(() -> { if (!closing) abort("Headless process exited"); });
}
void initializeAndBoot(Path cartridge) throws Exception {
this.cartridge = cartridge;
long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(limits.handshakeTimeoutMs());
hello = await(greeting, limits.handshakeTimeoutMs());
version(hello.path("protocolVersion")); text(hello, "runtimeVersion"); capabilities(hello.path("capabilities"));
var expected = mapper.valueToTree(limits);
for (var fields = expected.fieldNames(); fields.hasNext();) {
String field = fields.next();
if (count(hello.path("limits"), field) != expected.path(field).longValue())
throw new AutomationException(AutomationFailure.Code.RUNTIME_INCOMPATIBLE, "Runtime limits do not match launch configuration");
}
var init = mapper.createObjectNode(); init.putObject("protocolVersion").put("major", 1).put("minor", 0);
init.set("requiredCapabilities", mapper.valueToTree(CAPABILITIES));
var negotiated = request("initialize", init, null, remaining(deadline));
version(negotiated.path("protocolVersion")); capabilities(negotiated.path("capabilities"));
state(request("boot", mapper.createObjectNode().put("path", cartridge.toString()), null, limits.callTimeoutMs()));
}
@Override public String sessionId() { return sessionId; }
@Override public long maxFramesPerAdvance() { return limits.maxFrames(); }
long pid() { return process.pid(); }
boolean alive() { return process.isAlive() || descendants.stream().anyMatch(ProcessHandle::isAlive); }
@Override public TaskTools.HeadlessReply call(TaskTools.HeadlessAction action, Map<String, Object> input) throws Exception {
if (!work.tryLock()) throw new AutomationException(AutomationFailure.Code.BUSY, "Headless operation active");
try {
String command = switch (action) {
case BOOT -> "boot";
case RESET -> "reset"; case SET_INPUT -> "set_input"; case CLEAR_INPUT -> "clear_input";
case CAPTURE -> "capture"; case STATE -> "get_state"; case FAULT -> "get_fault";
default -> throw new IllegalArgumentException("Use lifecycle methods for boot/shutdown");
};
var params = action == TaskTools.HeadlessAction.BOOT ? mapper.createObjectNode().put("path", cartridge.toString()) : mapper.valueToTree(input);
var result = request(command, params, null, limits.callTimeoutMs());
var values = new LinkedHashMap<String, Object>(); values.put("sessionId", sessionId); values.put("buildId", buildId);
CanonicalCapture capture = null;
switch (action) {
case CAPTURE -> capture = capture(result, sessionId, buildId);
case STATE, RESET, BOOT -> {
state(result); values.put("state", mapper.convertValue(result, Object.class));
values.put("runtimeVersion", text(hello, "runtimeVersion"));
values.put("limits", mapper.convertValue(hello.path("limits"), Object.class));
values.put("capabilities", mapper.convertValue(hello.path("capabilities"), Object.class));
}
case FAULT -> { fault(result); values.put("fault", mapper.convertValue(result, Object.class)); }
default -> { if (!result.isObject()) throw invalid("Invalid acknowledgement"); }
}
return new TaskTools.HeadlessReply(values, capture);
} catch (AutomationException failure) { invalidateMalformed(failure); throw failure; }
finally { work.unlock(); }
}
@Override public AutomationOutcome advance(long frames, Consumer<TrackedOperation.Progress> progress) throws Exception {
if (frames < 0 || frames > limits.maxFrames()) throw new AutomationException(AutomationFailure.Code.LIMIT_EXCEEDED, "Requested frames exceed effective limit");
if (!work.tryLock()) throw new AutomationException(AutomationFailure.Code.BUSY, "Headless operation active");
boolean interrupted = false;
try {
var item = new Pending("advance", frames, progress); advance.set(item);
send(item, mapper.createObjectNode().put("frames", frames));
JsonNode result;
try { result = await(item.result, limits.callTimeoutMs()); }
catch (InterruptedException cancelled) {
interrupted = true; cancel();
result = awaitUninterruptibly(item.result, limits.shutdownTimeoutMs());
} catch (TimeoutException timedOut) {
cancel();
try { awaitUninterruptibly(item.result, limits.shutdownTimeoutMs()); } catch (Exception ignored) { }
abort("Headless advance deadline exceeded"); throw infrastructure("Headless advance deadline exceeded");
}
long requested = count(result, "requested"), completed = count(result, "completed");
if (requested != frames || completed > frames || completed < item.completed) throw invalid("Inconsistent final progress");
progress.accept(new TrackedOperation.Progress(frames, completed));
var diagnostics = new ArrayList<String>();
var kind = switch (text(result, "stop")) {
case "completed" -> { if (completed != frames) throw invalid("Incomplete completed result"); yield AutomationOutcome.Kind.SUCCEEDED; }
case "canceled" -> AutomationOutcome.Kind.CANCELLED;
case "terminated" -> AutomationOutcome.Kind.TERMINATED;
case "faulted" -> { fault(result.path("fault")); if (result.path("fault").isNull()) throw invalid("Missing fault"); diagnostics.add(result.path("fault").toString()); yield AutomationOutcome.Kind.GUEST_FAULT; }
case "host_error" -> { error(result.path("error")); diagnostics.add(result.path("error").toString()); yield AutomationOutcome.Kind.INFRASTRUCTURE_FAILURE; }
default -> throw invalid("Unknown advance stop");
};
CanonicalCapture captured = null;
if (!closing) {
try { captured = capture(request("capture", mapper.createObjectNode(), null, limits.callTimeoutMs()), sessionId, buildId); }
catch (AutomationException noFrame) { if (noFrame.failure().code() != AutomationFailure.Code.NO_FRAME_AVAILABLE) throw noFrame; }
}
synchronized (stderr) { diagnostics.addAll(stderr); stderr.clear(); }
return new AutomationOutcome(kind, requested, completed, diagnostics, captured);
} catch (TimeoutException timeout) { abort("Cancellation deadline exceeded"); throw infrastructure("Cancellation deadline exceeded"); }
catch (AutomationException failure) { invalidateMalformed(failure); throw failure; }
finally { advance.set(null); work.unlock(); if (interrupted) Thread.currentThread().interrupt(); }
}
@Override public void cancel() {
var target = advance.get();
if (target == null || target.result.isDone() || !target.cancelled.compareAndSet(false, true) || dead.get()) return;
var cancel = new Pending("cancel", null, null);
send(cancel, mapper.createObjectNode().put("targetRequestId", target.id));
timer.schedule(() -> { if (!target.result.isDone() || !cancel.result.isDone()) abort("Runtime did not stop after cancellation"); }, limits.shutdownTimeoutMs(), TimeUnit.MILLISECONDS);
}
@Override public synchronized CompletionStage<Void> shutdown() {
if (shutdown != null) return shutdown;
closing = true;
shutdown = CompletableFuture.runAsync(() -> {
long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(limits.shutdownTimeoutMs());
try {
if (!dead.get()) {
var response = request("shutdown", mapper.createObjectNode(), null, remaining(deadline));
if (!response.path("closed").isBoolean() || !response.path("closed").booleanValue()) throw invalid("Missing shutdown acknowledgement");
if (!process.waitFor(remaining(deadline), TimeUnit.MILLISECONDS)) throw new TimeoutException();
}
} catch (Exception failure) {
abort("Headless shutdown failed"); throw infrastructure("Headless shutdown failed; owned process tree terminated");
} finally { terminate(); if (alive()) throw infrastructure("Owned runtime process tree did not terminate"); }
});
return shutdown;
}
private JsonNode request(String command, JsonNode params, Consumer<TrackedOperation.Progress> observer, long timeout) throws Exception {
var item = new Pending(command, null, observer); send(item, params);
try { return await(item.result, timeout); }
catch (InterruptedException interrupted) { abort("Headless request interrupted"); Thread.currentThread().interrupt(); throw infrastructure("Headless request interrupted"); }
catch (TimeoutException exceeded) {
if (!command.equals("shutdown") && !command.equals("initialize") && !dead.get()) {
closing = true;
try {
var stop = new Pending("shutdown", null, null); send(stop, mapper.createObjectNode());
awaitUninterruptibly(stop.result, limits.shutdownTimeoutMs());
} catch (Exception ignored) { /* Deadline escalation below always invalidates this process. */ }
}
abort("Headless request deadline exceeded"); throw infrastructure("Headless request deadline exceeded");
}
}
private void send(Pending item, JsonNode params) {
if (dead.get()) throw infrastructure("Headless session lost; create a new session explicitly");
var envelope = mapper.createObjectNode().put("requestId", item.id).put("command", item.command); envelope.set("params", params);
final byte[] bytes;
try { bytes = mapper.writeValueAsBytes(envelope); } catch (IOException invalid) { throw new IllegalArgumentException(invalid); }
if (bytes.length > limits.maxCommandBytes()) throw new AutomationException(AutomationFailure.Code.LIMIT_EXCEEDED, "Command exceeds runtime message limit");
item.params = params; pending.put(item.id, item);
try { writer.execute(() -> {
try { process.getOutputStream().write(bytes); process.getOutputStream().write('\n'); process.getOutputStream().flush(); }
catch (IOException failed) { abort("Headless stdin closed"); }
}); } catch (RejectedExecutionException closed) { abort("Headless writer closed"); throw infrastructure("Headless writer closed"); }
}
private void readOutput() {
try (var input = new BufferedInputStream(process.getInputStream())) {
var bytes = new ByteArrayOutputStream(); int next;
while ((next = input.read()) != -1) {
if (next == '\n') {
byte[] line = bytes.toByteArray(); bytes.reset();
String json = StandardCharsets.UTF_8.newDecoder().decode(java.nio.ByteBuffer.wrap(line)).toString();
receive(mapper.readTree(json));
} else {
if (bytes.size() >= limits.maxResponseBytes()) throw invalid("Response exceeds limit");
bytes.write(next);
}
}
if (bytes.size() != 0) throw invalid("Truncated JSONL response");
if (!closing || !pending.isEmpty()) abort("Headless stdout closed");
} catch (Exception failure) { abort("Malformed or lost headless output"); }
}
private void receive(JsonNode message) {
process.descendants().forEach(descendants::add);
if (message == null || !message.isObject()) throw invalid("Expected object");
if (sessionId == null) {
if (!text(message, "type").equals("hello")) throw invalid("Expected hello");
sessionId = text(message, "sessionId"); greeting.complete(message); return;
}
if (!text(message, "sessionId").equals(sessionId)) throw invalid("Wrong session identity");
var item = pending.get(text(message, "requestId"));
if (item == null) throw invalid("Uncorrelated response");
if (text(message, "type").equals("progress")) {
long requested = count(message, "requested"), completed = count(message, "completed");
if (item.frames == null || requested != item.frames || completed < item.completed || completed > requested) throw invalid("Invalid progress");
item.completed = completed; item.observer.accept(new TrackedOperation.Progress(requested, completed)); return;
}
if (!text(message, "type").equals("response") || message.has("result") == message.has("error")) throw invalid("Invalid response envelope");
if (message.has("error")) {
var error = message.path("error"); error(error);
var code = switch (text(error, "code")) {
case "no_frame_available" -> AutomationFailure.Code.NO_FRAME_AVAILABLE;
case "busy", "already_booted" -> AutomationFailure.Code.BUSY;
case "not_cancelable" -> AutomationFailure.Code.NOT_CANCELABLE;
case "incompatible_version", "unsupported_capability" -> AutomationFailure.Code.RUNTIME_INCOMPATIBLE;
case "limit_exceeded", "response_too_large" -> AutomationFailure.Code.LIMIT_EXCEEDED;
default -> AutomationFailure.Code.INFRASTRUCTURE_FAILURE;
};
pending.remove(item.id);
item.result.completeExceptionally(new AutomationException(code, "Runtime " + error.path("code").asText() + ": " + error.path("message").asText()));
} else {
var result = message.get("result");
if (item.command.equals("cancel") && (!result.path("requested").isBoolean() || !result.path("requested").booleanValue()
|| !result.path("targetRequestId").equals(item.params.path("targetRequestId")))) throw invalid("Invalid cancellation acknowledgement");
pending.remove(item.id); item.result.complete(result);
}
}
private void readErrors() {
try (var input = process.getErrorStream()) {
byte[] bytes = new byte[1024]; int read;
while ((read = input.read(bytes)) != -1) synchronized (stderr) {
stderr.addLast(new String(bytes, 0, read, StandardCharsets.UTF_8));
while (stderr.size() > 32) stderr.removeFirst();
}
} catch (IOException ignored) { }
}
void abort(String message) {
if (dead.compareAndSet(false, true)) {
terminate();
greeting.completeExceptionally(infrastructure(message));
pending.values().forEach(item -> item.result.completeExceptionally(infrastructure(message))); pending.clear();
} else terminate();
}
private void terminate() {
dead.set(true);
process.descendants().forEach(descendants::add);
descendants.forEach(ProcessHandle::destroyForcibly); process.destroyForcibly();
writer.shutdownNow(); timer.shutdownNow(); io.shutdown();
// Killing first unblocks pipe IO; closing a writer before kill can wait on its blocked monitor.
try { process.waitFor(1000, TimeUnit.MILLISECONDS); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); }
descendants.forEach(handle -> { if (handle.isAlive()) handle.destroyForcibly(); });
for (var handle : descendants) {
try { handle.onExit().get(1000, TimeUnit.MILLISECONDS); }
catch (Exception ignored) { /* A failed cleanup is surfaced by shutdown below. */ }
}
if (!process.isAlive()) {
try { process.getOutputStream().close(); } catch (IOException ignored) { }
}
}
private void invalidateMalformed(AutomationException failure) {
if (failure.getMessage().startsWith("Invalid headless response:")) abort("Invalid headless response");
}
private static AutomationException infrastructure(String message) { return new AutomationException(AutomationFailure.Code.INFRASTRUCTURE_FAILURE, message); }
private static <T> T await(CompletableFuture<T> future, long timeout) throws Exception {
try { return future.get(timeout, TimeUnit.MILLISECONDS); }
catch (ExecutionException error) { if (error.getCause() instanceof Exception cause) throw cause; throw error; }
}
private static long remaining(long deadline) throws TimeoutException {
long nanos = deadline - System.nanoTime();
if (nanos <= 0) throw new TimeoutException();
return Math.max(1, TimeUnit.NANOSECONDS.toMillis(nanos));
}
private static <T> T awaitUninterruptibly(CompletableFuture<T> future, long timeout) throws Exception {
long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeout);
while (true) {
try { return await(future, Math.max(1, TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime()))); }
catch (InterruptedException ignored) { if (System.nanoTime() >= deadline) throw new TimeoutException(); }
}
}
private static final class Pending {
final String id = UUID.randomUUID().toString(), command;
final Long frames;
final Consumer<TrackedOperation.Progress> observer;
final CompletableFuture<JsonNode> result = new CompletableFuture<>();
final AtomicBoolean cancelled = new AtomicBoolean();
volatile long completed;
JsonNode params;
Pending(String command, Long frames, Consumer<TrackedOperation.Progress> observer) { this.command = command; this.frames = frames; this.observer = observer; }
}
}