implements PLN-0137 (5/7) semantic-token-overlay

Keep the lexical token provider as the baseline. When hover resolution
names a symbol and an existing semantic key matches it, replace the
weaker identifier token. A failed resolution, a broken file, or a
symbol without an existing key keeps the provider token. No new
semantic key is added.

docs/specs/compiler/23. Compiler Pipeline Entry Points Specification.md

@@ -294,6 +294,8 @@ When a frontend exposes folding ranges, the ranges MUST come from recovered synt
 
 When a frontend exposes selection ranges, the chain at a cursor MUST run from the inside outward in this order: identifier, argument or parameter list, block or `Doc` text block, declaration. A layer that does not contain the cursor MUST be omitted. A missing folding or selection capability MUST NOT be advertised and MUST produce an empty list rather than a protocol error.
 
+When a frontend exposes semantic tokens, the lexical token provider remains the baseline. An identifier token MAY be replaced with an existing semantic key when the same resolution used by hover names that symbol and an existing key matches it. Resolution failure, a broken file, or a symbol with no existing key MUST keep the baseline token. This overlay MUST NOT add a semantic-token key.
+
 When a frontend exposes formatting, it MUST be full-document formatting only. Range formatting and on-type formatting MUST NOT be advertised. The formatter MUST reprint the existing token stream. It MUST NOT reorder declarations, join lines, or split lines. Indentation MUST be four spaces and MUST change only where brace or parenthesis nesting changes. The interior of a `Doc` text block MUST be copied unchanged. A comment MUST stay on the line of the token it already follows. A missing formatting capability MUST NOT be advertised and MUST produce an empty edit list rather than a protocol error.
 
 `FrontendSpec` remains the source of static frontend-owned presentation metadata such as semantic vocabularies, host projections, and visual themes. Producing semantic tokens for a live document is an optional editor-facing capability; the existence of static presentation metadata MUST NOT imply that every frontend can provide live semantic-token results.

prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSFrontendLanguageService.java

@@ -17,6 +17,7 @@ import p.studio.compiler.models.SourceKind;
 import p.studio.compiler.pbs.semantics.PbsEditorialRename;
 import p.studio.compiler.pbs.semantics.PbsEditorialWorkspaceSymbol;
 import p.studio.compiler.pbs.PbsDocumentFormatter;
+import p.studio.compiler.pbs.PbsSemanticTokenOverlay;
 import p.studio.compiler.pbs.PbsStructureRanges;
 import p.studio.compiler.pbs.semantics.PbsQuickFix;
 import p.studio.compiler.pbs.semantics.PbsQuickFixCollector;
@@ -563,7 +564,31 @@ public final class PBSFrontendLanguageService implements FrontendLanguageService
 
     @Override
     public List<FrontendSemanticToken> semanticTokens(final FrontendDocumentRequest request) {
-        return semanticTokens(request == null ? "" : request.documentText());
+        return semanticTokens(request, null);
+    }
+
+    @Override
+    public List<FrontendSemanticToken> semanticTokens(
+            final FrontendDocumentRequest request,
+            final FrontendEditorialContext editorialContext) {
+        final String text = request == null || request.documentText() == null ? "" : request.documentText();
+        final List<FrontendSemanticToken> baseline = semanticTokenProvider.tokenize(text);
+        if (editorialContext == null
+                || !(editorialContext.syntaxTree() instanceof PbsAst.File ast)
+                || !(editorialContext.supplementalDeclarations() instanceof ReadOnlyList<?> supplementalTopDecls)
+                || !(editorialContext.semanticContext() instanceof FESurfaceContext feSurfaceContext)) {
+            return baseline;
+        }
+        @SuppressWarnings("unchecked")
+        final ReadOnlyList<PbsAst.TopDecl> typedSupplementalTopDecls =
+                (ReadOnlyList<PbsAst.TopDecl>) supplementalTopDecls;
+        return PbsSemanticTokenOverlay.apply(
+                baseline,
+                text,
+                ast,
+                typedSupplementalTopDecls,
+                feSurfaceContext,
+                editorialSupportService);
     }
 
     public List<FrontendSemanticToken> semanticTokens(final String documentText) {

prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/pbs/PbsSemanticTokenOverlay.java

@@ -0,0 +1,100 @@
+package p.studio.compiler.pbs;
+
+import p.studio.compiler.messages.FESurfaceContext;
+import p.studio.compiler.models.FrontendSemanticToken;
+import p.studio.compiler.pbs.ast.PbsAst;
+import p.studio.compiler.pbs.semantics.PbsEditorialResolvedSymbol;
+import p.studio.compiler.pbs.semantics.PbsEditorialSupportService;
+import p.studio.compiler.pbs.semantics.PbsEditorialSymbolKind;
+import p.studio.utilities.structures.ReadOnlyList;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+public final class PbsSemanticTokenOverlay {
+    private static final Set<String> LEXICAL_KEYS = Set.of(
+            PbsSemanticKind.COMMENT.semanticKey(),
+            PbsSemanticKind.STRING.semanticKey(),
+            PbsSemanticKind.NUMBER.semanticKey(),
+            PbsSemanticKind.LITERAL.semanticKey(),
+            PbsSemanticKind.KEYWORD.semanticKey(),
+            PbsSemanticKind.OPERATOR.semanticKey(),
+            PbsSemanticKind.PUNCTUATION.semanticKey());
+
+    private PbsSemanticTokenOverlay() {
+    }
+
+    public static List<FrontendSemanticToken> apply(
+            final List<FrontendSemanticToken> baseline,
+            final String documentText,
+            final PbsAst.File ast,
+            final ReadOnlyList<PbsAst.TopDecl> supplementalTopDecls,
+            final FESurfaceContext feSurfaceContext,
+            final PbsEditorialSupportService editorialSupportService) {
+        if (baseline == null || baseline.isEmpty() || ast == null || editorialSupportService == null) {
+            return baseline == null ? List.of() : List.copyOf(baseline);
+        }
+        final String text = documentText == null ? "" : documentText;
+        final ReadOnlyList<PbsAst.TopDecl> supplementals = supplementalTopDecls == null
+                ? ReadOnlyList.empty()
+                : supplementalTopDecls;
+        final FESurfaceContext surface = feSurfaceContext == null ? FESurfaceContext.empty() : feSurfaceContext;
+        final ArrayList<FrontendSemanticToken> overlaid = new ArrayList<>(baseline.size());
+        for (final FrontendSemanticToken token : baseline) {
+            overlaid.add(overlay(token, text, ast, supplementals, surface, editorialSupportService));
+        }
+        return List.copyOf(overlaid);
+    }
+
+    private static FrontendSemanticToken overlay(
+            final FrontendSemanticToken token,
+            final String text,
+            final PbsAst.File ast,
+            final ReadOnlyList<PbsAst.TopDecl> supplementals,
+            final FESurfaceContext surface,
+            final PbsEditorialSupportService editorialSupportService) {
+        if (token == null || LEXICAL_KEYS.contains(token.semanticKey())) {
+            return token;
+        }
+        try {
+            final String resolvedKey = editorialSupportService.hover(
+                            text,
+                            ast,
+                            supplementals,
+                            surface,
+                            token.startOffset())
+                    .map(PbsSemanticTokenOverlay::semanticKey)
+                    .orElse(null);
+            if (resolvedKey == null || resolvedKey.equals(token.semanticKey())) {
+                return token;
+            }
+            return new FrontendSemanticToken(token.startOffset(), token.endOffset(), resolvedKey);
+        } catch (final RuntimeException ignored) {
+            return token;
+        }
+    }
+
+    private static String semanticKey(final PbsEditorialResolvedSymbol symbol) {
+        if (symbol == null || symbol.kind() == null) {
+            return null;
+        }
+        final PbsSemanticKind kind = switch (symbol.kind()) {
+            case FUNCTION -> PbsSemanticKind.FUNCTION;
+            case METHOD -> PbsSemanticKind.METHOD;
+            case CONSTRUCTOR -> PbsSemanticKind.CONSTRUCTOR;
+            case STRUCT -> PbsSemanticKind.STRUCT;
+            case BUILTIN_TYPE -> PbsSemanticKind.BUILTIN_TYPE;
+            case SERVICE -> PbsSemanticKind.SERVICE;
+            case HOST -> PbsSemanticKind.HOST;
+            case CONTRACT -> PbsSemanticKind.CONTRACT;
+            case CALLBACK -> PbsSemanticKind.CALLBACK;
+            case ENUM -> PbsSemanticKind.ENUM;
+            case ERROR -> PbsSemanticKind.ERROR;
+            case GLOBAL -> PbsSemanticKind.GLOBAL;
+            case CONST -> PbsSemanticKind.CONST;
+            case KEYWORD, LOCAL, PARAMETER, FIELD -> null;
+        };
+        return kind == null ? null : kind.semanticKey();
+    }
+}

prometeu-compiler/frontends/prometeu-frontend-pbs/src/test/java/p/studio/compiler/pbs/PbsSemanticTokenOverlayTest.java

@@ -0,0 +1,91 @@
+package p.studio.compiler.pbs;
+
+import org.junit.jupiter.api.Test;
+import p.studio.compiler.PBSFrontendLanguageService;
+import p.studio.compiler.PBSSemanticTokenProvider;
+import p.studio.compiler.messages.FESurfaceContext;
+import p.studio.compiler.models.FrontendSemanticToken;
+import p.studio.compiler.pbs.ast.PbsAst;
+import p.studio.compiler.pbs.lexer.PbsLexer;
+import p.studio.compiler.pbs.parser.PbsParser;
+import p.studio.compiler.services.FrontendDocumentRequest;
+import p.studio.compiler.services.FrontendEditorialContext;
+import p.studio.compiler.source.diagnostics.DiagnosticSink;
+import p.studio.compiler.source.identifiers.FileId;
+import p.studio.utilities.structures.ReadOnlyList;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class PbsSemanticTokenOverlayTest {
+
+    @Test
+    void resolvedFunctionUseReplacesTheIdentifierBaseline() {
+        final String source = """
+                fn helper() -> int {
+                  return 1;
+                }
+
+                fn main() -> int {
+                  return helper;
+                }
+                """;
+        final PbsAst.File ast = PbsParser.parse(
+                PbsLexer.lex(source, FileId.none(), DiagnosticSink.empty()),
+                FileId.none(),
+                DiagnosticSink.empty());
+        final var service = new PBSFrontendLanguageService();
+        final var tokens = service.semanticTokens(
+                new FrontendDocumentRequest(Path.of("."), Path.of("demo.pbs"), source),
+                new FrontendEditorialContext(ast, ReadOnlyList.empty(), FESurfaceContext.empty()));
+
+        assertEquals(PbsSemanticKind.FUNCTION.semanticKey(), keyAt(source, tokens, "return helper"));
+        assertTrue(tokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.KEYWORD.semanticKey())));
+    }
+
+    @Test
+    void localBindingAndBrokenTextKeepTheBaselineToken() {
+        final String source = """
+                fn main() -> int {
+                  let value: int = 1;
+                  return value;
+                }
+                """;
+        final PbsAst.File ast = PbsParser.parse(
+                PbsLexer.lex(source, FileId.none(), DiagnosticSink.empty()),
+                FileId.none(),
+                DiagnosticSink.empty());
+        final var service = new PBSFrontendLanguageService();
+        final var request = new FrontendDocumentRequest(Path.of("."), Path.of("demo.pbs"), source);
+        final var overlaid = service.semanticTokens(
+                request,
+                new FrontendEditorialContext(ast, ReadOnlyList.empty(), FESurfaceContext.empty()));
+
+        assertEquals(PbsSemanticKind.IDENTIFIER.semanticKey(), keyAt(source, overlaid, "return value"));
+
+        final String broken = "fn main( {";
+        final List<FrontendSemanticToken> baseline = new PBSSemanticTokenProvider().tokenize(broken);
+        final List<FrontendSemanticToken> withoutContext = service.semanticTokens(
+                new FrontendDocumentRequest(Path.of("."), Path.of("broken.pbs"), broken));
+        assertEquals(baseline, withoutContext);
+        assertTrue(withoutContext.stream().anyMatch(token ->
+                token.semanticKey().equals(PbsSemanticKind.KEYWORD.semanticKey())));
+    }
+
+    private static String keyAt(
+            final String source,
+            final List<FrontendSemanticToken> tokens,
+            final String marker) {
+        final int nameOffset = source.indexOf(marker) + marker.lastIndexOf(' ') + 1;
+        final int byteOffset = source.substring(0, nameOffset).getBytes(StandardCharsets.UTF_8).length;
+        return tokens.stream()
+                .filter(token -> token.startOffset() == byteOffset)
+                .map(FrontendSemanticToken::semanticKey)
+                .findFirst()
+                .orElseThrow();
+    }
+}

prometeu-compiler/prometeu-frontend-api/src/main/java/p/studio/compiler/services/FrontendLanguageService.java

@@ -26,6 +26,12 @@ public interface FrontendLanguageService {
         return List.of();
     }
 
+    default List<FrontendSemanticToken> semanticTokens(
+            final FrontendDocumentRequest request,
+            final FrontendEditorialContext editorialContext) {
+        return semanticTokens(request);
+    }
+
     default List<FrontendCompletionCandidate> completion(
             final FrontendDocumentRequest request,
             final int offset) {

prometeu-lsp/prometeu-lsp-v1/src/main/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridge.java

@@ -2,6 +2,7 @@ package p.studio.lsp.services.compiler;
 
 import org.apache.commons.lang3.StringUtils;
 import p.studio.compiler.FrontendRegistryService;
+import p.studio.compiler.exceptions.BuildException;
 import p.studio.compiler.messages.*;
 import p.studio.compiler.models.*;
 import p.studio.compiler.services.FrontendCodeAction;
@@ -628,7 +629,7 @@ public final class CompilerLanguageServiceBridge implements LanguageServiceBridg
                 documentPath,
                 effectiveText);
         final List<FrontendSemanticToken> tokens = provider.languageService()
-                .map(languageService -> languageService.semanticTokens(request))
+                .map(languageService -> semanticTokens(languageService, context, documentUri, documentPath, request))
                 .orElseGet(List::of);
         final var positionMapper = new DocumentPositionMapper(effectiveText);
         final var semanticTokens = new ArrayList<BaselineSemanticToken>();
@@ -647,6 +648,26 @@ public final class CompilerLanguageServiceBridge implements LanguageServiceBridg
         return new BaselineSemanticTokens(presentation.semanticKeys(), semanticTokens);
     }
 
+    private List<FrontendSemanticToken> semanticTokens(
+            final FrontendLanguageService languageService,
+            final LspProjectContext context,
+            final String documentUri,
+            final Path documentPath,
+            final FrontendDocumentRequest request) {
+        try {
+            return editorialDocument(context, documentUri, request.documentText())
+                    .map(document -> languageService.semanticTokens(
+                            new FrontendDocumentRequest(
+                                    context.projectRoot(),
+                                    documentPath,
+                                    document.text()),
+                            document.editorialContext()))
+                    .orElseGet(() -> languageService.semanticTokens(request));
+        } catch (final BuildException ignored) {
+            return languageService.semanticTokens(request);
+        }
+    }
+
     @Override
     public String onSave(
             final LspProjectContext context,

prometeu-lsp/prometeu-lsp-v1/src/test/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridgeTest.java

@@ -172,6 +172,56 @@ class CompilerLanguageServiceBridgeTest {
         assertTrue(semanticTokens.tokens().stream().anyMatch(token -> token.semanticKey().equals("pbs-builtin-type")));
     }
 
+    @Test
+    void semanticTokenOverlayReplacesAResolvedUseAndKeepsBrokenText(@TempDir final Path tempDir) throws IOException {
+        final Path projectRoot = tempDir.resolve("app");
+        final Path source = projectRoot.resolve("src").resolve("main.pbs");
+        Files.createDirectories(source.getParent());
+        Files.writeString(projectRoot.resolve("prometeu.json"), """
+                {
+                  "name": "app",
+                  "version": "1.0.0",
+                  "language": "pbs",
+                  "stdlib": "1",
+                  "target": "Game",
+                  "dependencies": []
+                }
+                """);
+        final String text = """
+                fn helper() -> int {
+                  return 1;
+                }
+
+                fn main() -> int {
+                  return helper;
+                }
+                """;
+        Files.writeString(source, text);
+        Files.writeString(source.getParent().resolve("mod.barrel"), """
+                pub fn helper() -> int;
+                pub fn main() -> int;
+                """);
+        final CompilerLanguageServiceBridge bridge = new CompilerLanguageServiceBridge();
+        final Path realSource = source.toRealPath();
+        final var tokens = bridge.semanticTokens(
+                new LspProjectContext("app", "pbs", projectRoot.toRealPath()),
+                realSource.toUri().toString(),
+                text);
+        final int helperUse = text.indexOf("return helper") + "return ".length();
+        assertTrue(tokens.tokens().stream().anyMatch(token ->
+                token.semanticKey().equals("pbs-function")
+                        && token.line() == new DocumentPositionMapper(text).positionOf(helperUse).line()
+                        && token.startCharacter() == new DocumentPositionMapper(text).positionOf(helperUse).character()));
+
+        final String broken = "fn main( {";
+        final var brokenTokens = bridge.semanticTokens(
+                new LspProjectContext("app", "pbs", projectRoot.toRealPath()),
+                realSource.toUri().toString(),
+                broken);
+        assertTrue(brokenTokens.tokens().stream().anyMatch(token -> token.semanticKey().equals("pbs-keyword")));
+        assertTrue(brokenTokens.tokens().stream().anyMatch(token -> token.semanticKey().equals("pbs-punctuation")));
+    }
+
     @Test
     void completionHoverAndSignatureHelpUseCompilerBackedEditorialResolution() {
         final Path projectRoot = findRepoRoot(Path.of("").toAbsolutePath().normalize())