implement real headless process gateway and supervision
This commit is contained in:
@@ -0,0 +1,306 @@
|
||||
package p.studio.app.runtime;
|
||||
|
||||
import com.fasterxml.jackson.core.*;
|
||||
import com.fasterxml.jackson.databind.*;
|
||||
import p.studio.api.automation.*;
|
||||
import p.studio.core.runtime.HeadlessGateway;
|
||||
import java.io.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.*;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.function.Consumer;
|
||||
import static p.studio.app.runtime.HeadlessWire.*;
|
||||
|
||||
/** One supervised process, bounded JSONL, no retry/replay and no simulated-time clock. */
|
||||
final class HeadlessProcessSession implements HeadlessGateway.Session {
|
||||
private final Process process;
|
||||
private final ObjectMapper mapper;
|
||||
private final HeadlessLimits limits;
|
||||
private final String buildId;
|
||||
private volatile String sessionId;
|
||||
private volatile JsonNode hello;
|
||||
private Path cartridge;
|
||||
private final CompletableFuture<JsonNode> greeting = new CompletableFuture<>();
|
||||
private final Map<String, Pending> pending = new ConcurrentHashMap<>();
|
||||
private final Set<ProcessHandle> descendants = ConcurrentHashMap.newKeySet();
|
||||
private final ExecutorService io = Executors.newVirtualThreadPerTaskExecutor();
|
||||
private final ExecutorService writer = Executors.newSingleThreadExecutor(Thread.ofPlatform().daemon().name("headless-writer").factory());
|
||||
private final ScheduledExecutorService timer = Executors.newSingleThreadScheduledExecutor(Thread.ofPlatform().daemon().name("headless-supervisor").factory());
|
||||
private final ReentrantLock work = new ReentrantLock();
|
||||
private final AtomicBoolean dead = new AtomicBoolean();
|
||||
private final AtomicReference<Pending> advance = new AtomicReference<>();
|
||||
private final Deque<String> stderr = new ArrayDeque<>();
|
||||
private volatile boolean closing;
|
||||
private CompletableFuture<Void> shutdown;
|
||||
|
||||
HeadlessProcessSession(Process process, ObjectMapper source, HeadlessLimits limits, String buildId) {
|
||||
this.process = process; this.limits = limits; this.buildId = buildId;
|
||||
mapper = source.copy().enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
mapper.getFactory().setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(limits.maxResponseBytes()).build());
|
||||
timer.scheduleAtFixedRate(() -> { if (process.isAlive()) process.descendants().forEach(descendants::add); }, 0, 20, TimeUnit.MILLISECONDS);
|
||||
io.execute(this::readOutput); io.execute(this::readErrors);
|
||||
process.onExit().thenRun(() -> { if (!closing) abort("Headless process exited"); });
|
||||
}
|
||||
|
||||
void initializeAndBoot(Path cartridge) throws Exception {
|
||||
this.cartridge = cartridge;
|
||||
long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(limits.handshakeTimeoutMs());
|
||||
hello = await(greeting, limits.handshakeTimeoutMs());
|
||||
version(hello.path("protocolVersion")); text(hello, "runtimeVersion"); capabilities(hello.path("capabilities"));
|
||||
var expected = mapper.valueToTree(limits);
|
||||
for (var fields = expected.fieldNames(); fields.hasNext();) {
|
||||
String field = fields.next();
|
||||
if (count(hello.path("limits"), field) != expected.path(field).longValue())
|
||||
throw new AutomationException(AutomationFailure.Code.RUNTIME_INCOMPATIBLE, "Runtime limits do not match launch configuration");
|
||||
}
|
||||
var init = mapper.createObjectNode(); init.putObject("protocolVersion").put("major", 1).put("minor", 0);
|
||||
init.set("requiredCapabilities", mapper.valueToTree(CAPABILITIES));
|
||||
var negotiated = request("initialize", init, null, remaining(deadline));
|
||||
version(negotiated.path("protocolVersion")); capabilities(negotiated.path("capabilities"));
|
||||
state(request("boot", mapper.createObjectNode().put("path", cartridge.toString()), null, limits.callTimeoutMs()));
|
||||
}
|
||||
@Override public String sessionId() { return sessionId; }
|
||||
@Override public long maxFramesPerAdvance() { return limits.maxFrames(); }
|
||||
long pid() { return process.pid(); }
|
||||
boolean alive() { return process.isAlive() || descendants.stream().anyMatch(ProcessHandle::isAlive); }
|
||||
|
||||
@Override public TaskTools.HeadlessReply call(TaskTools.HeadlessAction action, Map<String, Object> input) throws Exception {
|
||||
if (!work.tryLock()) throw new AutomationException(AutomationFailure.Code.BUSY, "Headless operation active");
|
||||
try {
|
||||
String command = switch (action) {
|
||||
case BOOT -> "boot";
|
||||
case RESET -> "reset"; case SET_INPUT -> "set_input"; case CLEAR_INPUT -> "clear_input";
|
||||
case CAPTURE -> "capture"; case STATE -> "get_state"; case FAULT -> "get_fault";
|
||||
default -> throw new IllegalArgumentException("Use lifecycle methods for boot/shutdown");
|
||||
};
|
||||
var params = action == TaskTools.HeadlessAction.BOOT ? mapper.createObjectNode().put("path", cartridge.toString()) : mapper.valueToTree(input);
|
||||
var result = request(command, params, null, limits.callTimeoutMs());
|
||||
var values = new LinkedHashMap<String, Object>(); values.put("sessionId", sessionId); values.put("buildId", buildId);
|
||||
CanonicalCapture capture = null;
|
||||
switch (action) {
|
||||
case CAPTURE -> capture = capture(result, sessionId, buildId);
|
||||
case STATE, RESET, BOOT -> {
|
||||
state(result); values.put("state", mapper.convertValue(result, Object.class));
|
||||
values.put("runtimeVersion", text(hello, "runtimeVersion"));
|
||||
values.put("limits", mapper.convertValue(hello.path("limits"), Object.class));
|
||||
values.put("capabilities", mapper.convertValue(hello.path("capabilities"), Object.class));
|
||||
}
|
||||
case FAULT -> { fault(result); values.put("fault", mapper.convertValue(result, Object.class)); }
|
||||
default -> { if (!result.isObject()) throw invalid("Invalid acknowledgement"); }
|
||||
}
|
||||
return new TaskTools.HeadlessReply(values, capture);
|
||||
} catch (AutomationException failure) { invalidateMalformed(failure); throw failure; }
|
||||
finally { work.unlock(); }
|
||||
}
|
||||
|
||||
@Override public AutomationOutcome advance(long frames, Consumer<TrackedOperation.Progress> progress) throws Exception {
|
||||
if (frames < 0 || frames > limits.maxFrames()) throw new AutomationException(AutomationFailure.Code.LIMIT_EXCEEDED, "Requested frames exceed effective limit");
|
||||
if (!work.tryLock()) throw new AutomationException(AutomationFailure.Code.BUSY, "Headless operation active");
|
||||
boolean interrupted = false;
|
||||
try {
|
||||
var item = new Pending("advance", frames, progress); advance.set(item);
|
||||
send(item, mapper.createObjectNode().put("frames", frames));
|
||||
JsonNode result;
|
||||
try { result = await(item.result, limits.callTimeoutMs()); }
|
||||
catch (InterruptedException cancelled) {
|
||||
interrupted = true; cancel();
|
||||
result = awaitUninterruptibly(item.result, limits.shutdownTimeoutMs());
|
||||
} catch (TimeoutException timedOut) {
|
||||
cancel();
|
||||
try { awaitUninterruptibly(item.result, limits.shutdownTimeoutMs()); } catch (Exception ignored) { }
|
||||
abort("Headless advance deadline exceeded"); throw infrastructure("Headless advance deadline exceeded");
|
||||
}
|
||||
long requested = count(result, "requested"), completed = count(result, "completed");
|
||||
if (requested != frames || completed > frames || completed < item.completed) throw invalid("Inconsistent final progress");
|
||||
progress.accept(new TrackedOperation.Progress(frames, completed));
|
||||
var diagnostics = new ArrayList<String>();
|
||||
var kind = switch (text(result, "stop")) {
|
||||
case "completed" -> { if (completed != frames) throw invalid("Incomplete completed result"); yield AutomationOutcome.Kind.SUCCEEDED; }
|
||||
case "canceled" -> AutomationOutcome.Kind.CANCELLED;
|
||||
case "terminated" -> AutomationOutcome.Kind.TERMINATED;
|
||||
case "faulted" -> { fault(result.path("fault")); if (result.path("fault").isNull()) throw invalid("Missing fault"); diagnostics.add(result.path("fault").toString()); yield AutomationOutcome.Kind.GUEST_FAULT; }
|
||||
case "host_error" -> { error(result.path("error")); diagnostics.add(result.path("error").toString()); yield AutomationOutcome.Kind.INFRASTRUCTURE_FAILURE; }
|
||||
default -> throw invalid("Unknown advance stop");
|
||||
};
|
||||
CanonicalCapture captured = null;
|
||||
if (!closing) {
|
||||
try { captured = capture(request("capture", mapper.createObjectNode(), null, limits.callTimeoutMs()), sessionId, buildId); }
|
||||
catch (AutomationException noFrame) { if (noFrame.failure().code() != AutomationFailure.Code.NO_FRAME_AVAILABLE) throw noFrame; }
|
||||
}
|
||||
synchronized (stderr) { diagnostics.addAll(stderr); stderr.clear(); }
|
||||
return new AutomationOutcome(kind, requested, completed, diagnostics, captured);
|
||||
} catch (TimeoutException timeout) { abort("Cancellation deadline exceeded"); throw infrastructure("Cancellation deadline exceeded"); }
|
||||
catch (AutomationException failure) { invalidateMalformed(failure); throw failure; }
|
||||
finally { advance.set(null); work.unlock(); if (interrupted) Thread.currentThread().interrupt(); }
|
||||
}
|
||||
|
||||
@Override public void cancel() {
|
||||
var target = advance.get();
|
||||
if (target == null || target.result.isDone() || !target.cancelled.compareAndSet(false, true) || dead.get()) return;
|
||||
var cancel = new Pending("cancel", null, null);
|
||||
send(cancel, mapper.createObjectNode().put("targetRequestId", target.id));
|
||||
timer.schedule(() -> { if (!target.result.isDone() || !cancel.result.isDone()) abort("Runtime did not stop after cancellation"); }, limits.shutdownTimeoutMs(), TimeUnit.MILLISECONDS);
|
||||
}
|
||||
@Override public synchronized CompletionStage<Void> shutdown() {
|
||||
if (shutdown != null) return shutdown;
|
||||
closing = true;
|
||||
shutdown = CompletableFuture.runAsync(() -> {
|
||||
long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(limits.shutdownTimeoutMs());
|
||||
try {
|
||||
if (!dead.get()) {
|
||||
var response = request("shutdown", mapper.createObjectNode(), null, remaining(deadline));
|
||||
if (!response.path("closed").isBoolean() || !response.path("closed").booleanValue()) throw invalid("Missing shutdown acknowledgement");
|
||||
if (!process.waitFor(remaining(deadline), TimeUnit.MILLISECONDS)) throw new TimeoutException();
|
||||
}
|
||||
} catch (Exception failure) {
|
||||
abort("Headless shutdown failed"); throw infrastructure("Headless shutdown failed; owned process tree terminated");
|
||||
} finally { terminate(); if (alive()) throw infrastructure("Owned runtime process tree did not terminate"); }
|
||||
});
|
||||
return shutdown;
|
||||
}
|
||||
private JsonNode request(String command, JsonNode params, Consumer<TrackedOperation.Progress> observer, long timeout) throws Exception {
|
||||
var item = new Pending(command, null, observer); send(item, params);
|
||||
try { return await(item.result, timeout); }
|
||||
catch (InterruptedException interrupted) { abort("Headless request interrupted"); Thread.currentThread().interrupt(); throw infrastructure("Headless request interrupted"); }
|
||||
catch (TimeoutException exceeded) {
|
||||
if (!command.equals("shutdown") && !command.equals("initialize") && !dead.get()) {
|
||||
closing = true;
|
||||
try {
|
||||
var stop = new Pending("shutdown", null, null); send(stop, mapper.createObjectNode());
|
||||
awaitUninterruptibly(stop.result, limits.shutdownTimeoutMs());
|
||||
} catch (Exception ignored) { /* Deadline escalation below always invalidates this process. */ }
|
||||
}
|
||||
abort("Headless request deadline exceeded"); throw infrastructure("Headless request deadline exceeded");
|
||||
}
|
||||
}
|
||||
private void send(Pending item, JsonNode params) {
|
||||
if (dead.get()) throw infrastructure("Headless session lost; create a new session explicitly");
|
||||
var envelope = mapper.createObjectNode().put("requestId", item.id).put("command", item.command); envelope.set("params", params);
|
||||
final byte[] bytes;
|
||||
try { bytes = mapper.writeValueAsBytes(envelope); } catch (IOException invalid) { throw new IllegalArgumentException(invalid); }
|
||||
if (bytes.length > limits.maxCommandBytes()) throw new AutomationException(AutomationFailure.Code.LIMIT_EXCEEDED, "Command exceeds runtime message limit");
|
||||
item.params = params; pending.put(item.id, item);
|
||||
try { writer.execute(() -> {
|
||||
try { process.getOutputStream().write(bytes); process.getOutputStream().write('\n'); process.getOutputStream().flush(); }
|
||||
catch (IOException failed) { abort("Headless stdin closed"); }
|
||||
}); } catch (RejectedExecutionException closed) { abort("Headless writer closed"); throw infrastructure("Headless writer closed"); }
|
||||
}
|
||||
|
||||
private void readOutput() {
|
||||
try (var input = new BufferedInputStream(process.getInputStream())) {
|
||||
var bytes = new ByteArrayOutputStream(); int next;
|
||||
while ((next = input.read()) != -1) {
|
||||
if (next == '\n') {
|
||||
byte[] line = bytes.toByteArray(); bytes.reset();
|
||||
String json = StandardCharsets.UTF_8.newDecoder().decode(java.nio.ByteBuffer.wrap(line)).toString();
|
||||
receive(mapper.readTree(json));
|
||||
} else {
|
||||
if (bytes.size() >= limits.maxResponseBytes()) throw invalid("Response exceeds limit");
|
||||
bytes.write(next);
|
||||
}
|
||||
}
|
||||
if (bytes.size() != 0) throw invalid("Truncated JSONL response");
|
||||
if (!closing || !pending.isEmpty()) abort("Headless stdout closed");
|
||||
} catch (Exception failure) { abort("Malformed or lost headless output"); }
|
||||
}
|
||||
private void receive(JsonNode message) {
|
||||
process.descendants().forEach(descendants::add);
|
||||
if (message == null || !message.isObject()) throw invalid("Expected object");
|
||||
if (sessionId == null) {
|
||||
if (!text(message, "type").equals("hello")) throw invalid("Expected hello");
|
||||
sessionId = text(message, "sessionId"); greeting.complete(message); return;
|
||||
}
|
||||
if (!text(message, "sessionId").equals(sessionId)) throw invalid("Wrong session identity");
|
||||
var item = pending.get(text(message, "requestId"));
|
||||
if (item == null) throw invalid("Uncorrelated response");
|
||||
if (text(message, "type").equals("progress")) {
|
||||
long requested = count(message, "requested"), completed = count(message, "completed");
|
||||
if (item.frames == null || requested != item.frames || completed < item.completed || completed > requested) throw invalid("Invalid progress");
|
||||
item.completed = completed; item.observer.accept(new TrackedOperation.Progress(requested, completed)); return;
|
||||
}
|
||||
if (!text(message, "type").equals("response") || message.has("result") == message.has("error")) throw invalid("Invalid response envelope");
|
||||
if (message.has("error")) {
|
||||
var error = message.path("error"); error(error);
|
||||
var code = switch (text(error, "code")) {
|
||||
case "no_frame_available" -> AutomationFailure.Code.NO_FRAME_AVAILABLE;
|
||||
case "busy", "already_booted" -> AutomationFailure.Code.BUSY;
|
||||
case "not_cancelable" -> AutomationFailure.Code.NOT_CANCELABLE;
|
||||
case "incompatible_version", "unsupported_capability" -> AutomationFailure.Code.RUNTIME_INCOMPATIBLE;
|
||||
case "limit_exceeded", "response_too_large" -> AutomationFailure.Code.LIMIT_EXCEEDED;
|
||||
default -> AutomationFailure.Code.INFRASTRUCTURE_FAILURE;
|
||||
};
|
||||
pending.remove(item.id);
|
||||
item.result.completeExceptionally(new AutomationException(code, "Runtime " + error.path("code").asText() + ": " + error.path("message").asText()));
|
||||
} else {
|
||||
var result = message.get("result");
|
||||
if (item.command.equals("cancel") && (!result.path("requested").isBoolean() || !result.path("requested").booleanValue()
|
||||
|| !result.path("targetRequestId").equals(item.params.path("targetRequestId")))) throw invalid("Invalid cancellation acknowledgement");
|
||||
pending.remove(item.id); item.result.complete(result);
|
||||
}
|
||||
}
|
||||
private void readErrors() {
|
||||
try (var input = process.getErrorStream()) {
|
||||
byte[] bytes = new byte[1024]; int read;
|
||||
while ((read = input.read(bytes)) != -1) synchronized (stderr) {
|
||||
stderr.addLast(new String(bytes, 0, read, StandardCharsets.UTF_8));
|
||||
while (stderr.size() > 32) stderr.removeFirst();
|
||||
}
|
||||
} catch (IOException ignored) { }
|
||||
}
|
||||
void abort(String message) {
|
||||
if (dead.compareAndSet(false, true)) {
|
||||
terminate();
|
||||
greeting.completeExceptionally(infrastructure(message));
|
||||
pending.values().forEach(item -> item.result.completeExceptionally(infrastructure(message))); pending.clear();
|
||||
} else terminate();
|
||||
}
|
||||
private void terminate() {
|
||||
dead.set(true);
|
||||
process.descendants().forEach(descendants::add);
|
||||
descendants.forEach(ProcessHandle::destroyForcibly); process.destroyForcibly();
|
||||
writer.shutdownNow(); timer.shutdownNow(); io.shutdown();
|
||||
// Killing first unblocks pipe IO; closing a writer before kill can wait on its blocked monitor.
|
||||
try { process.waitFor(1000, TimeUnit.MILLISECONDS); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); }
|
||||
descendants.forEach(handle -> { if (handle.isAlive()) handle.destroyForcibly(); });
|
||||
for (var handle : descendants) {
|
||||
try { handle.onExit().get(1000, TimeUnit.MILLISECONDS); }
|
||||
catch (Exception ignored) { /* A failed cleanup is surfaced by shutdown below. */ }
|
||||
}
|
||||
if (!process.isAlive()) {
|
||||
try { process.getOutputStream().close(); } catch (IOException ignored) { }
|
||||
}
|
||||
}
|
||||
private void invalidateMalformed(AutomationException failure) {
|
||||
if (failure.getMessage().startsWith("Invalid headless response:")) abort("Invalid headless response");
|
||||
}
|
||||
private static AutomationException infrastructure(String message) { return new AutomationException(AutomationFailure.Code.INFRASTRUCTURE_FAILURE, message); }
|
||||
private static <T> T await(CompletableFuture<T> future, long timeout) throws Exception {
|
||||
try { return future.get(timeout, TimeUnit.MILLISECONDS); }
|
||||
catch (ExecutionException error) { if (error.getCause() instanceof Exception cause) throw cause; throw error; }
|
||||
}
|
||||
private static long remaining(long deadline) throws TimeoutException {
|
||||
long nanos = deadline - System.nanoTime();
|
||||
if (nanos <= 0) throw new TimeoutException();
|
||||
return Math.max(1, TimeUnit.NANOSECONDS.toMillis(nanos));
|
||||
}
|
||||
private static <T> T awaitUninterruptibly(CompletableFuture<T> future, long timeout) throws Exception {
|
||||
long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeout);
|
||||
while (true) {
|
||||
try { return await(future, Math.max(1, TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime()))); }
|
||||
catch (InterruptedException ignored) { if (System.nanoTime() >= deadline) throw new TimeoutException(); }
|
||||
}
|
||||
}
|
||||
private static final class Pending {
|
||||
final String id = UUID.randomUUID().toString(), command;
|
||||
final Long frames;
|
||||
final Consumer<TrackedOperation.Progress> observer;
|
||||
final CompletableFuture<JsonNode> result = new CompletableFuture<>();
|
||||
final AtomicBoolean cancelled = new AtomicBoolean();
|
||||
volatile long completed;
|
||||
JsonNode params;
|
||||
Pending(String command, Long frames, Consumer<TrackedOperation.Progress> observer) { this.command = command; this.frames = frames; this.observer = observer; }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user