diff --git a/docs/specs/compiler/23. Compiler Pipeline Entry Points Specification.md b/docs/specs/compiler/23. Compiler Pipeline Entry Points Specification.md index 13ebaece..84eba27a 100644 --- a/docs/specs/compiler/23. Compiler Pipeline Entry Points Specification.md +++ b/docs/specs/compiler/23. Compiler Pipeline Entry Points Specification.md @@ -294,6 +294,8 @@ When a frontend exposes folding ranges, the ranges MUST come from recovered synt When a frontend exposes selection ranges, the chain at a cursor MUST run from the inside outward in this order: identifier, argument or parameter list, block or `Doc` text block, declaration. A layer that does not contain the cursor MUST be omitted. A missing folding or selection capability MUST NOT be advertised and MUST produce an empty list rather than a protocol error. +When a frontend exposes semantic tokens, the lexical token provider remains the baseline. An identifier token MAY be replaced with an existing semantic key when the same resolution used by hover names that symbol and an existing key matches it. Resolution failure, a broken file, or a symbol with no existing key MUST keep the baseline token. This overlay MUST NOT add a semantic-token key. + When a frontend exposes formatting, it MUST be full-document formatting only. Range formatting and on-type formatting MUST NOT be advertised. The formatter MUST reprint the existing token stream. It MUST NOT reorder declarations, join lines, or split lines. Indentation MUST be four spaces and MUST change only where brace or parenthesis nesting changes. The interior of a `Doc` text block MUST be copied unchanged. A comment MUST stay on the line of the token it already follows. A missing formatting capability MUST NOT be advertised and MUST produce an empty edit list rather than a protocol error. `FrontendSpec` remains the source of static frontend-owned presentation metadata such as semantic vocabularies, host projections, and visual themes. Producing semantic tokens for a live document is an optional editor-facing capability; the existence of static presentation metadata MUST NOT imply that every frontend can provide live semantic-token results. diff --git a/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSFrontendLanguageService.java b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSFrontendLanguageService.java index b8d1ca1e..9b674375 100644 --- a/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSFrontendLanguageService.java +++ b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSFrontendLanguageService.java @@ -17,6 +17,7 @@ import p.studio.compiler.models.SourceKind; import p.studio.compiler.pbs.semantics.PbsEditorialRename; import p.studio.compiler.pbs.semantics.PbsEditorialWorkspaceSymbol; import p.studio.compiler.pbs.PbsDocumentFormatter; +import p.studio.compiler.pbs.PbsSemanticTokenOverlay; import p.studio.compiler.pbs.PbsStructureRanges; import p.studio.compiler.pbs.semantics.PbsQuickFix; import p.studio.compiler.pbs.semantics.PbsQuickFixCollector; @@ -563,7 +564,31 @@ public final class PBSFrontendLanguageService implements FrontendLanguageService @Override public List semanticTokens(final FrontendDocumentRequest request) { - return semanticTokens(request == null ? "" : request.documentText()); + return semanticTokens(request, null); + } + + @Override + public List semanticTokens( + final FrontendDocumentRequest request, + final FrontendEditorialContext editorialContext) { + final String text = request == null || request.documentText() == null ? "" : request.documentText(); + final List baseline = semanticTokenProvider.tokenize(text); + if (editorialContext == null + || !(editorialContext.syntaxTree() instanceof PbsAst.File ast) + || !(editorialContext.supplementalDeclarations() instanceof ReadOnlyList supplementalTopDecls) + || !(editorialContext.semanticContext() instanceof FESurfaceContext feSurfaceContext)) { + return baseline; + } + @SuppressWarnings("unchecked") + final ReadOnlyList typedSupplementalTopDecls = + (ReadOnlyList) supplementalTopDecls; + return PbsSemanticTokenOverlay.apply( + baseline, + text, + ast, + typedSupplementalTopDecls, + feSurfaceContext, + editorialSupportService); } public List semanticTokens(final String documentText) { diff --git a/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/pbs/PbsSemanticTokenOverlay.java b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/pbs/PbsSemanticTokenOverlay.java new file mode 100644 index 00000000..bcc70047 --- /dev/null +++ b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/pbs/PbsSemanticTokenOverlay.java @@ -0,0 +1,100 @@ +package p.studio.compiler.pbs; + +import p.studio.compiler.messages.FESurfaceContext; +import p.studio.compiler.models.FrontendSemanticToken; +import p.studio.compiler.pbs.ast.PbsAst; +import p.studio.compiler.pbs.semantics.PbsEditorialResolvedSymbol; +import p.studio.compiler.pbs.semantics.PbsEditorialSupportService; +import p.studio.compiler.pbs.semantics.PbsEditorialSymbolKind; +import p.studio.utilities.structures.ReadOnlyList; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public final class PbsSemanticTokenOverlay { + private static final Set LEXICAL_KEYS = Set.of( + PbsSemanticKind.COMMENT.semanticKey(), + PbsSemanticKind.STRING.semanticKey(), + PbsSemanticKind.NUMBER.semanticKey(), + PbsSemanticKind.LITERAL.semanticKey(), + PbsSemanticKind.KEYWORD.semanticKey(), + PbsSemanticKind.OPERATOR.semanticKey(), + PbsSemanticKind.PUNCTUATION.semanticKey()); + + private PbsSemanticTokenOverlay() { + } + + public static List apply( + final List baseline, + final String documentText, + final PbsAst.File ast, + final ReadOnlyList supplementalTopDecls, + final FESurfaceContext feSurfaceContext, + final PbsEditorialSupportService editorialSupportService) { + if (baseline == null || baseline.isEmpty() || ast == null || editorialSupportService == null) { + return baseline == null ? List.of() : List.copyOf(baseline); + } + final String text = documentText == null ? "" : documentText; + final ReadOnlyList supplementals = supplementalTopDecls == null + ? ReadOnlyList.empty() + : supplementalTopDecls; + final FESurfaceContext surface = feSurfaceContext == null ? FESurfaceContext.empty() : feSurfaceContext; + final ArrayList overlaid = new ArrayList<>(baseline.size()); + for (final FrontendSemanticToken token : baseline) { + overlaid.add(overlay(token, text, ast, supplementals, surface, editorialSupportService)); + } + return List.copyOf(overlaid); + } + + private static FrontendSemanticToken overlay( + final FrontendSemanticToken token, + final String text, + final PbsAst.File ast, + final ReadOnlyList supplementals, + final FESurfaceContext surface, + final PbsEditorialSupportService editorialSupportService) { + if (token == null || LEXICAL_KEYS.contains(token.semanticKey())) { + return token; + } + try { + final String resolvedKey = editorialSupportService.hover( + text, + ast, + supplementals, + surface, + token.startOffset()) + .map(PbsSemanticTokenOverlay::semanticKey) + .orElse(null); + if (resolvedKey == null || resolvedKey.equals(token.semanticKey())) { + return token; + } + return new FrontendSemanticToken(token.startOffset(), token.endOffset(), resolvedKey); + } catch (final RuntimeException ignored) { + return token; + } + } + + private static String semanticKey(final PbsEditorialResolvedSymbol symbol) { + if (symbol == null || symbol.kind() == null) { + return null; + } + final PbsSemanticKind kind = switch (symbol.kind()) { + case FUNCTION -> PbsSemanticKind.FUNCTION; + case METHOD -> PbsSemanticKind.METHOD; + case CONSTRUCTOR -> PbsSemanticKind.CONSTRUCTOR; + case STRUCT -> PbsSemanticKind.STRUCT; + case BUILTIN_TYPE -> PbsSemanticKind.BUILTIN_TYPE; + case SERVICE -> PbsSemanticKind.SERVICE; + case HOST -> PbsSemanticKind.HOST; + case CONTRACT -> PbsSemanticKind.CONTRACT; + case CALLBACK -> PbsSemanticKind.CALLBACK; + case ENUM -> PbsSemanticKind.ENUM; + case ERROR -> PbsSemanticKind.ERROR; + case GLOBAL -> PbsSemanticKind.GLOBAL; + case CONST -> PbsSemanticKind.CONST; + case KEYWORD, LOCAL, PARAMETER, FIELD -> null; + }; + return kind == null ? null : kind.semanticKey(); + } +} diff --git a/prometeu-compiler/frontends/prometeu-frontend-pbs/src/test/java/p/studio/compiler/pbs/PbsSemanticTokenOverlayTest.java b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/test/java/p/studio/compiler/pbs/PbsSemanticTokenOverlayTest.java new file mode 100644 index 00000000..0960695c --- /dev/null +++ b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/test/java/p/studio/compiler/pbs/PbsSemanticTokenOverlayTest.java @@ -0,0 +1,91 @@ +package p.studio.compiler.pbs; + +import org.junit.jupiter.api.Test; +import p.studio.compiler.PBSFrontendLanguageService; +import p.studio.compiler.PBSSemanticTokenProvider; +import p.studio.compiler.messages.FESurfaceContext; +import p.studio.compiler.models.FrontendSemanticToken; +import p.studio.compiler.pbs.ast.PbsAst; +import p.studio.compiler.pbs.lexer.PbsLexer; +import p.studio.compiler.pbs.parser.PbsParser; +import p.studio.compiler.services.FrontendDocumentRequest; +import p.studio.compiler.services.FrontendEditorialContext; +import p.studio.compiler.source.diagnostics.DiagnosticSink; +import p.studio.compiler.source.identifiers.FileId; +import p.studio.utilities.structures.ReadOnlyList; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PbsSemanticTokenOverlayTest { + + @Test + void resolvedFunctionUseReplacesTheIdentifierBaseline() { + final String source = """ + fn helper() -> int { + return 1; + } + + fn main() -> int { + return helper; + } + """; + final PbsAst.File ast = PbsParser.parse( + PbsLexer.lex(source, FileId.none(), DiagnosticSink.empty()), + FileId.none(), + DiagnosticSink.empty()); + final var service = new PBSFrontendLanguageService(); + final var tokens = service.semanticTokens( + new FrontendDocumentRequest(Path.of("."), Path.of("demo.pbs"), source), + new FrontendEditorialContext(ast, ReadOnlyList.empty(), FESurfaceContext.empty())); + + assertEquals(PbsSemanticKind.FUNCTION.semanticKey(), keyAt(source, tokens, "return helper")); + assertTrue(tokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.KEYWORD.semanticKey()))); + } + + @Test + void localBindingAndBrokenTextKeepTheBaselineToken() { + final String source = """ + fn main() -> int { + let value: int = 1; + return value; + } + """; + final PbsAst.File ast = PbsParser.parse( + PbsLexer.lex(source, FileId.none(), DiagnosticSink.empty()), + FileId.none(), + DiagnosticSink.empty()); + final var service = new PBSFrontendLanguageService(); + final var request = new FrontendDocumentRequest(Path.of("."), Path.of("demo.pbs"), source); + final var overlaid = service.semanticTokens( + request, + new FrontendEditorialContext(ast, ReadOnlyList.empty(), FESurfaceContext.empty())); + + assertEquals(PbsSemanticKind.IDENTIFIER.semanticKey(), keyAt(source, overlaid, "return value")); + + final String broken = "fn main( {"; + final List baseline = new PBSSemanticTokenProvider().tokenize(broken); + final List withoutContext = service.semanticTokens( + new FrontendDocumentRequest(Path.of("."), Path.of("broken.pbs"), broken)); + assertEquals(baseline, withoutContext); + assertTrue(withoutContext.stream().anyMatch(token -> + token.semanticKey().equals(PbsSemanticKind.KEYWORD.semanticKey()))); + } + + private static String keyAt( + final String source, + final List tokens, + final String marker) { + final int nameOffset = source.indexOf(marker) + marker.lastIndexOf(' ') + 1; + final int byteOffset = source.substring(0, nameOffset).getBytes(StandardCharsets.UTF_8).length; + return tokens.stream() + .filter(token -> token.startOffset() == byteOffset) + .map(FrontendSemanticToken::semanticKey) + .findFirst() + .orElseThrow(); + } +} diff --git a/prometeu-compiler/prometeu-frontend-api/src/main/java/p/studio/compiler/services/FrontendLanguageService.java b/prometeu-compiler/prometeu-frontend-api/src/main/java/p/studio/compiler/services/FrontendLanguageService.java index cc10e7f8..50e4db4f 100644 --- a/prometeu-compiler/prometeu-frontend-api/src/main/java/p/studio/compiler/services/FrontendLanguageService.java +++ b/prometeu-compiler/prometeu-frontend-api/src/main/java/p/studio/compiler/services/FrontendLanguageService.java @@ -26,6 +26,12 @@ public interface FrontendLanguageService { return List.of(); } + default List semanticTokens( + final FrontendDocumentRequest request, + final FrontendEditorialContext editorialContext) { + return semanticTokens(request); + } + default List completion( final FrontendDocumentRequest request, final int offset) { diff --git a/prometeu-lsp/prometeu-lsp-v1/src/main/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridge.java b/prometeu-lsp/prometeu-lsp-v1/src/main/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridge.java index 9815b19d..dd54c7fb 100644 --- a/prometeu-lsp/prometeu-lsp-v1/src/main/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridge.java +++ b/prometeu-lsp/prometeu-lsp-v1/src/main/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridge.java @@ -2,6 +2,7 @@ package p.studio.lsp.services.compiler; import org.apache.commons.lang3.StringUtils; import p.studio.compiler.FrontendRegistryService; +import p.studio.compiler.exceptions.BuildException; import p.studio.compiler.messages.*; import p.studio.compiler.models.*; import p.studio.compiler.services.FrontendCodeAction; @@ -628,7 +629,7 @@ public final class CompilerLanguageServiceBridge implements LanguageServiceBridg documentPath, effectiveText); final List tokens = provider.languageService() - .map(languageService -> languageService.semanticTokens(request)) + .map(languageService -> semanticTokens(languageService, context, documentUri, documentPath, request)) .orElseGet(List::of); final var positionMapper = new DocumentPositionMapper(effectiveText); final var semanticTokens = new ArrayList(); @@ -647,6 +648,26 @@ public final class CompilerLanguageServiceBridge implements LanguageServiceBridg return new BaselineSemanticTokens(presentation.semanticKeys(), semanticTokens); } + private List semanticTokens( + final FrontendLanguageService languageService, + final LspProjectContext context, + final String documentUri, + final Path documentPath, + final FrontendDocumentRequest request) { + try { + return editorialDocument(context, documentUri, request.documentText()) + .map(document -> languageService.semanticTokens( + new FrontendDocumentRequest( + context.projectRoot(), + documentPath, + document.text()), + document.editorialContext())) + .orElseGet(() -> languageService.semanticTokens(request)); + } catch (final BuildException ignored) { + return languageService.semanticTokens(request); + } + } + @Override public String onSave( final LspProjectContext context, diff --git a/prometeu-lsp/prometeu-lsp-v1/src/test/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridgeTest.java b/prometeu-lsp/prometeu-lsp-v1/src/test/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridgeTest.java index 285103fe..68cf5649 100644 --- a/prometeu-lsp/prometeu-lsp-v1/src/test/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridgeTest.java +++ b/prometeu-lsp/prometeu-lsp-v1/src/test/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridgeTest.java @@ -172,6 +172,56 @@ class CompilerLanguageServiceBridgeTest { assertTrue(semanticTokens.tokens().stream().anyMatch(token -> token.semanticKey().equals("pbs-builtin-type"))); } + @Test + void semanticTokenOverlayReplacesAResolvedUseAndKeepsBrokenText(@TempDir final Path tempDir) throws IOException { + final Path projectRoot = tempDir.resolve("app"); + final Path source = projectRoot.resolve("src").resolve("main.pbs"); + Files.createDirectories(source.getParent()); + Files.writeString(projectRoot.resolve("prometeu.json"), """ + { + "name": "app", + "version": "1.0.0", + "language": "pbs", + "stdlib": "1", + "target": "Game", + "dependencies": [] + } + """); + final String text = """ + fn helper() -> int { + return 1; + } + + fn main() -> int { + return helper; + } + """; + Files.writeString(source, text); + Files.writeString(source.getParent().resolve("mod.barrel"), """ + pub fn helper() -> int; + pub fn main() -> int; + """); + final CompilerLanguageServiceBridge bridge = new CompilerLanguageServiceBridge(); + final Path realSource = source.toRealPath(); + final var tokens = bridge.semanticTokens( + new LspProjectContext("app", "pbs", projectRoot.toRealPath()), + realSource.toUri().toString(), + text); + final int helperUse = text.indexOf("return helper") + "return ".length(); + assertTrue(tokens.tokens().stream().anyMatch(token -> + token.semanticKey().equals("pbs-function") + && token.line() == new DocumentPositionMapper(text).positionOf(helperUse).line() + && token.startCharacter() == new DocumentPositionMapper(text).positionOf(helperUse).character())); + + final String broken = "fn main( {"; + final var brokenTokens = bridge.semanticTokens( + new LspProjectContext("app", "pbs", projectRoot.toRealPath()), + realSource.toUri().toString(), + broken); + assertTrue(brokenTokens.tokens().stream().anyMatch(token -> token.semanticKey().equals("pbs-keyword"))); + assertTrue(brokenTokens.tokens().stream().anyMatch(token -> token.semanticKey().equals("pbs-punctuation"))); + } + @Test void completionHoverAndSignatureHelpUseCompilerBackedEditorialResolution() { final Path projectRoot = findRepoRoot(Path.of("").toAbsolutePath().normalize())