diff --git a/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSDefinitions.java b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSDefinitions.java index e55a89ef..ea3135f3 100644 --- a/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSDefinitions.java +++ b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSDefinitions.java @@ -17,5 +17,6 @@ public class PBSDefinitions { List.of(PBSFrontendThemes.DEFAULT_PBS), "pbs-default", List.of(PBSFrontendHostProjections.VSCODE))) + .semanticTokenProvider(new PBSSemanticTokenProvider()) .build(); } diff --git a/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSSemanticTokenProvider.java b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSSemanticTokenProvider.java new file mode 100644 index 00000000..b9c082e5 --- /dev/null +++ b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSSemanticTokenProvider.java @@ -0,0 +1,158 @@ +package p.studio.compiler; + +import p.studio.compiler.models.FrontendSemanticToken; +import p.studio.compiler.models.FrontendSemanticTokenProvider; +import p.studio.compiler.pbs.PbsSemanticKind; +import p.studio.compiler.pbs.lexer.PbsLexer; +import p.studio.compiler.pbs.lexer.PbsToken; +import p.studio.compiler.pbs.lexer.PbsTokenKind; +import p.studio.compiler.source.diagnostics.DiagnosticSink; +import p.studio.compiler.source.identifiers.FileId; +import p.studio.utilities.structures.ReadOnlyList; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public final class PBSSemanticTokenProvider implements FrontendSemanticTokenProvider { + private static final Set BUILTIN_TYPE_NAMES = Set.of("int", "float", "bool", "string", "i32", "i64", "f32", "f64"); + + @Override + public List tokenize(final String documentText) { + final String effectiveText = documentText == null ? "" : documentText; + final ReadOnlyList tokens = PbsLexer.lex(effectiveText, FileId.none(), DiagnosticSink.empty()); + final ArrayList semanticTokens = new ArrayList<>(); + + for (int index = 0; index < tokens.size(); index += 1) { + final PbsToken token = tokens.get(index); + final PbsSemanticKind semanticKind = classifyToken(tokens, index); + if (semanticKind == null) { + continue; + } + semanticTokens.add(new FrontendSemanticToken(token.start(), token.end(), semanticKind.semanticKey())); + } + + return List.copyOf(semanticTokens); + } + + private PbsSemanticKind classifyToken( + final ReadOnlyList tokens, + final int index) { + final PbsToken token = tokens.get(index); + final PbsSemanticKind lexicalKind = PbsSemanticKind.forToken(token); + if (lexicalKind != null) { + return lexicalKind; + } + if (token.kind() == PbsTokenKind.IDENTIFIER) { + if (isLifecycleToken(tokens, index, token)) { + return PbsSemanticKind.LIFECYCLE; + } + return classifyIdentifierToken(tokens, index, token); + } + return null; + } + + private PbsSemanticKind classifyIdentifierToken( + final ReadOnlyList tokens, + final int index, + final PbsToken token) { + final PbsTokenKind previousKind = previousSignificantKind(tokens, index); + final PbsTokenKind nextKind = nextSignificantKind(tokens, index); + + if (previousKind == PbsTokenKind.STRUCT) { + return PbsSemanticKind.STRUCT; + } + if (previousKind == PbsTokenKind.CONTRACT) { + return PbsSemanticKind.CONTRACT; + } + if (previousKind == PbsTokenKind.HOST) { + return PbsSemanticKind.HOST; + } + if (previousKind == PbsTokenKind.SERVICE) { + return PbsSemanticKind.SERVICE; + } + if (previousKind == PbsTokenKind.ENUM || previousKind == PbsTokenKind.ERROR) { + return PbsSemanticKind.ENUM; + } + if (previousKind == PbsTokenKind.CALLBACK) { + return PbsSemanticKind.CALLBACK; + } + if (previousKind == PbsTokenKind.GLOBAL) { + return PbsSemanticKind.GLOBAL; + } + if (previousKind == PbsTokenKind.CONST) { + return PbsSemanticKind.CONST; + } + if (previousKind == PbsTokenKind.CTOR || previousKind == PbsTokenKind.NEW) { + return PbsSemanticKind.CONSTRUCTOR; + } + if (previousKind == PbsTokenKind.BUILTIN) { + return PbsSemanticKind.BUILTIN_TYPE; + } + if (previousKind == PbsTokenKind.DOT && nextKind == PbsTokenKind.LEFT_PAREN) { + return PbsSemanticKind.METHOD; + } + if (nextKind == PbsTokenKind.LEFT_PAREN) { + return PbsSemanticKind.FUNCTION; + } + if (isTypeReferenceContext(previousKind, token.lexeme())) { + return PbsSemanticKind.BUILTIN_TYPE; + } + return PbsSemanticKind.IDENTIFIER; + } + + private boolean isTypeReferenceContext( + final PbsTokenKind previousKind, + final String lexeme) { + if (previousKind == null) { + return false; + } + if (previousKind != PbsTokenKind.COLON + && previousKind != PbsTokenKind.ARROW + && previousKind != PbsTokenKind.AS + && previousKind != PbsTokenKind.IMPLEMENTS) { + return false; + } + return BUILTIN_TYPE_NAMES.contains(lexeme); + } + + private PbsTokenKind previousSignificantKind( + final ReadOnlyList tokens, + final int index) { + for (int cursor = index - 1; cursor >= 0; cursor -= 1) { + final PbsTokenKind kind = tokens.get(cursor).kind(); + if (kind == PbsTokenKind.COMMENT || kind == PbsTokenKind.EOF) { + continue; + } + return kind; + } + return null; + } + + private PbsTokenKind nextSignificantKind( + final ReadOnlyList tokens, + final int index) { + for (int cursor = index + 1; cursor < tokens.size(); cursor += 1) { + final PbsTokenKind kind = tokens.get(cursor).kind(); + if (kind == PbsTokenKind.COMMENT || kind == PbsTokenKind.EOF) { + continue; + } + return kind; + } + return null; + } + + private boolean isLifecycleToken( + final ReadOnlyList tokens, + final int index, + final PbsToken token) { + if (!"Init".equals(token.lexeme()) && !"Frame".equals(token.lexeme())) { + return false; + } + if (index == 0 || index + 1 >= tokens.size()) { + return false; + } + return tokens.get(index - 1).kind() == PbsTokenKind.LEFT_BRACKET + && tokens.get(index + 1).kind() == PbsTokenKind.RIGHT_BRACKET; + } +} diff --git a/prometeu-compiler/frontends/prometeu-frontend-pbs/src/test/java/p/studio/compiler/pbs/PbsSemanticPresentationContractTest.java b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/test/java/p/studio/compiler/pbs/PbsSemanticPresentationContractTest.java index 743b144f..6ef7400b 100644 --- a/prometeu-compiler/frontends/prometeu-frontend-pbs/src/test/java/p/studio/compiler/pbs/PbsSemanticPresentationContractTest.java +++ b/prometeu-compiler/frontends/prometeu-frontend-pbs/src/test/java/p/studio/compiler/pbs/PbsSemanticPresentationContractTest.java @@ -3,6 +3,9 @@ package p.studio.compiler.pbs; import org.junit.jupiter.api.Test; import p.studio.compiler.PBSFrontendHostProjections; import p.studio.compiler.PBSDefinitions; +import p.studio.compiler.models.FrontendSemanticToken; + +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -50,4 +53,24 @@ final class PbsSemanticPresentationContractTest { java.util.List.of("readonly"), vscodeProjection.requireProjection(PbsSemanticKind.GLOBAL.semanticKey()).hostTokenModifiers()); } + + @Test + void shouldPublishPbsOwnedSemanticTokenProvider() { + final List semanticTokens = PBSDefinitions.PBS.getSemanticTokenProvider().tokenize(""" + struct Vec2 {} + + fn main() -> void { + const answer: int = 42; + let value = new Vec2(); + value.length(); + } + """); + + assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.STRUCT.semanticKey()))); + assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.FUNCTION.semanticKey()))); + assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.CONST.semanticKey()))); + assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.BUILTIN_TYPE.semanticKey()))); + assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.CONSTRUCTOR.semanticKey()))); + assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.METHOD.semanticKey()))); + } } diff --git a/prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSemanticToken.java b/prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSemanticToken.java new file mode 100644 index 00000000..fef83228 --- /dev/null +++ b/prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSemanticToken.java @@ -0,0 +1,29 @@ +package p.studio.compiler.models; + +import java.util.Objects; + +public record FrontendSemanticToken( + int startOffset, + int endOffset, + String semanticKey) { + + public FrontendSemanticToken { + if (startOffset < 0) { + throw new IllegalArgumentException("startOffset must be >= 0"); + } + if (endOffset < startOffset) { + throw new IllegalArgumentException("endOffset must be >= startOffset"); + } + semanticKey = requireText(semanticKey, "semanticKey"); + } + + private static String requireText( + final String value, + final String field) { + final String candidate = Objects.requireNonNull(value, field).trim(); + if (candidate.isEmpty()) { + throw new IllegalArgumentException(field + " must not be blank"); + } + return candidate; + } +} diff --git a/prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSemanticTokenProvider.java b/prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSemanticTokenProvider.java new file mode 100644 index 00000000..6e52f6f3 --- /dev/null +++ b/prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSemanticTokenProvider.java @@ -0,0 +1,14 @@ +package p.studio.compiler.models; + +import java.util.List; + +@FunctionalInterface +public interface FrontendSemanticTokenProvider { + FrontendSemanticTokenProvider NOOP = documentText -> List.of(); + + List tokenize(String documentText); + + static FrontendSemanticTokenProvider noop() { + return NOOP; + } +} diff --git a/prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSpec.java b/prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSpec.java index f09e24fc..72a42cab 100644 --- a/prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSpec.java +++ b/prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSpec.java @@ -17,6 +17,8 @@ public class FrontendSpec { private final List stdlibVersions = List.of(); @Builder.Default private final FrontendSemanticPresentationSpec semanticPresentation = FrontendSemanticPresentationSpec.empty(); + @Builder.Default + private final FrontendSemanticTokenProvider semanticTokenProvider = FrontendSemanticTokenProvider.noop(); public String toString() { return String.format("FrontendSpec(language=%s)", languageId); diff --git a/prometeu-lsp/prometeu-lsp-v1/build.gradle.kts b/prometeu-lsp/prometeu-lsp-v1/build.gradle.kts index a7ccc7b3..addf13e3 100644 --- a/prometeu-lsp/prometeu-lsp-v1/build.gradle.kts +++ b/prometeu-lsp/prometeu-lsp-v1/build.gradle.kts @@ -6,6 +6,6 @@ dependencies { implementation(project(":prometeu-lsp:prometeu-lsp-api")) implementation(project(":prometeu-compiler:prometeu-compiler-core")) implementation(project(":prometeu-compiler:prometeu-build-pipeline")) - implementation(project(":prometeu-compiler:frontends:prometeu-frontend-pbs")) + implementation(project(":prometeu-compiler:prometeu-frontend-registry")) implementation(libs.lsp4j) } diff --git a/prometeu-lsp/prometeu-lsp-v1/src/main/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridge.java b/prometeu-lsp/prometeu-lsp-v1/src/main/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridge.java index 9a2d14a7..8152a91d 100644 --- a/prometeu-lsp/prometeu-lsp-v1/src/main/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridge.java +++ b/prometeu-lsp/prometeu-lsp-v1/src/main/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridge.java @@ -1,6 +1,7 @@ package p.studio.lsp.services.compiler; import org.apache.commons.lang3.StringUtils; +import p.studio.compiler.FrontendRegistryService; import p.studio.compiler.messages.BuilderPipelineConfig; import p.studio.compiler.messages.BuildingIssue; import p.studio.compiler.models.AnalysisSnapshot; @@ -8,21 +9,16 @@ import p.studio.compiler.models.BuilderPipelineContext; import p.studio.compiler.models.FrontendEditorPaletteSpec; import p.studio.compiler.models.FrontendHostProjectionEntrySpec; import p.studio.compiler.models.FrontendHostProjectionSpec; -import p.studio.compiler.PBSDefinitions; +import p.studio.compiler.models.FrontendSemanticPresentationSpec; +import p.studio.compiler.models.FrontendSemanticToken; import p.studio.compiler.models.FrontendTokenStyleSpec; import p.studio.compiler.models.FrontendVisualThemeSpec; -import p.studio.compiler.pbs.PbsSemanticKind; -import p.studio.compiler.pbs.lexer.PbsLexer; -import p.studio.compiler.pbs.lexer.PbsToken; -import p.studio.compiler.pbs.lexer.PbsTokenKind; import p.studio.compiler.source.identifiers.FileId; -import p.studio.compiler.source.diagnostics.DiagnosticSink; import p.studio.compiler.utilities.SourceProviderFactory; import p.studio.compiler.workspaces.BuilderPipelineService; import p.studio.lsp.messages.*; import p.studio.lsp.services.LanguageServiceBridge; import p.studio.utilities.logs.LogAggregator; -import p.studio.utilities.structures.ReadOnlyList; import java.net.URI; import java.nio.file.Path; @@ -30,16 +26,13 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; - -public record CompilerLanguageServiceBridge() implements LanguageServiceBridge { - private static final Set BUILTIN_TYPE_NAMES = Set.of("int", "float", "bool", "string", "i32", "i64", "f32", "f64"); - +public final class CompilerLanguageServiceBridge implements LanguageServiceBridge { @Override public BaselineServerDescription describeServer(final LspProjectContext context) { Objects.requireNonNull(context, "context"); - final var presentation = PBSDefinitions.PBS.getSemanticPresentation(); + final var frontend = frontendSpec(context); + final var presentation = frontend.getSemanticPresentation(); return new BaselineServerDescription( "Prometeu Studio LSP", "0.1.0", @@ -86,28 +79,24 @@ public record CompilerLanguageServiceBridge() implements LanguageServiceBridge { final String text) { Objects.requireNonNull(context, "context"); final String effectiveText = text == null ? "" : text; - final var legend = PBSDefinitions.PBS.getSemanticPresentation().semanticKeys(); - final var tokens = PbsLexer.lex(effectiveText, FileId.none(), DiagnosticSink.empty()); + final var frontend = frontendSpec(context); + final FrontendSemanticPresentationSpec presentation = frontend.getSemanticPresentation(); + final List tokens = frontend.getSemanticTokenProvider().tokenize(effectiveText); final var positionMapper = new DocumentPositionMapper(effectiveText); final var semanticTokens = new ArrayList(); - for (int index = 0; index < tokens.size(); index += 1) { - final PbsToken token = tokens.get(index); - final PbsSemanticKind semanticKind = classifyToken(tokens, index); - if (semanticKind == null) { - continue; - } - final DocumentPosition start = positionMapper.positionOf(token.start()); - final DocumentPosition end = positionMapper.positionOf(token.end()); + for (final FrontendSemanticToken token : tokens) { + final DocumentPosition start = positionMapper.positionOf(token.startOffset()); + final DocumentPosition end = positionMapper.positionOf(token.endOffset()); final int length = Math.max(1, end.character() - start.character()); semanticTokens.add(new BaselineSemanticToken( start.line(), start.character(), length, - semanticKind.semanticKey())); + token.semanticKey())); } - return new BaselineSemanticTokens(legend, semanticTokens); + return new BaselineSemanticTokens(presentation.semanticKeys(), semanticTokens); } @Override @@ -174,127 +163,6 @@ public record CompilerLanguageServiceBridge() implements LanguageServiceBridge { return Path.of(URI.create(Objects.requireNonNull(documentUri, "documentUri"))).toAbsolutePath().normalize(); } - private PbsSemanticKind classifyToken( - final ReadOnlyList tokens, - final int index) { - final PbsToken token = tokens.get(index); - final PbsSemanticKind lexicalKind = PbsSemanticKind.forToken(token); - if (lexicalKind != null) { - return lexicalKind; - } - if (token.kind() == p.studio.compiler.pbs.lexer.PbsTokenKind.IDENTIFIER) { - if (isLifecycleToken(tokens, index, token)) { - return PbsSemanticKind.LIFECYCLE; - } - return classifyIdentifierToken(tokens, index, token); - } - return null; - } - - private PbsSemanticKind classifyIdentifierToken( - final ReadOnlyList tokens, - final int index, - final PbsToken token) { - final PbsTokenKind previousKind = previousSignificantKind(tokens, index); - final PbsTokenKind nextKind = nextSignificantKind(tokens, index); - - if (previousKind == PbsTokenKind.STRUCT) { - return PbsSemanticKind.STRUCT; - } - if (previousKind == PbsTokenKind.CONTRACT) { - return PbsSemanticKind.CONTRACT; - } - if (previousKind == PbsTokenKind.HOST) { - return PbsSemanticKind.HOST; - } - if (previousKind == PbsTokenKind.SERVICE) { - return PbsSemanticKind.SERVICE; - } - if (previousKind == PbsTokenKind.ENUM || previousKind == PbsTokenKind.ERROR) { - return PbsSemanticKind.ENUM; - } - if (previousKind == PbsTokenKind.CALLBACK) { - return PbsSemanticKind.CALLBACK; - } - if (previousKind == PbsTokenKind.GLOBAL) { - return PbsSemanticKind.GLOBAL; - } - if (previousKind == PbsTokenKind.CONST) { - return PbsSemanticKind.CONST; - } - if (previousKind == PbsTokenKind.CTOR || previousKind == PbsTokenKind.NEW) { - return PbsSemanticKind.CONSTRUCTOR; - } - if (previousKind == PbsTokenKind.BUILTIN) { - return PbsSemanticKind.BUILTIN_TYPE; - } - if (previousKind == PbsTokenKind.DOT && nextKind == PbsTokenKind.LEFT_PAREN) { - return PbsSemanticKind.METHOD; - } - if (nextKind == PbsTokenKind.LEFT_PAREN) { - return PbsSemanticKind.FUNCTION; - } - if (isTypeReferenceContext(previousKind, token.lexeme())) { - return PbsSemanticKind.BUILTIN_TYPE; - } - return PbsSemanticKind.IDENTIFIER; - } - - private boolean isTypeReferenceContext( - final PbsTokenKind previousKind, - final String lexeme) { - if (previousKind == null) { - return false; - } - if (previousKind != PbsTokenKind.COLON - && previousKind != PbsTokenKind.ARROW - && previousKind != PbsTokenKind.AS - && previousKind != PbsTokenKind.IMPLEMENTS) { - return false; - } - return BUILTIN_TYPE_NAMES.contains(lexeme); - } - - private PbsTokenKind previousSignificantKind( - final ReadOnlyList tokens, - final int index) { - for (int cursor = index - 1; cursor >= 0; cursor -= 1) { - final PbsTokenKind kind = tokens.get(cursor).kind(); - if (kind == PbsTokenKind.COMMENT || kind == PbsTokenKind.EOF) { - continue; - } - return kind; - } - return null; - } - - private PbsTokenKind nextSignificantKind( - final ReadOnlyList tokens, - final int index) { - for (int cursor = index + 1; cursor < tokens.size(); cursor += 1) { - final PbsTokenKind kind = tokens.get(cursor).kind(); - if (kind == PbsTokenKind.COMMENT || kind == PbsTokenKind.EOF) { - continue; - } - return kind; - } - return null; - } - - private boolean isLifecycleToken( - final ReadOnlyList tokens, - final int index, - final PbsToken token) { - if (!"Init".equals(token.lexeme()) && !"Frame".equals(token.lexeme())) { - return false; - } - if (index == 0 || index + 1 >= tokens.size()) { - return false; - } - return tokens.get(index - 1).kind() == p.studio.compiler.pbs.lexer.PbsTokenKind.LEFT_BRACKET - && tokens.get(index + 1).kind() == p.studio.compiler.pbs.lexer.PbsTokenKind.RIGHT_BRACKET; - } - private BaselineVisualTheme mapVisualTheme(final FrontendVisualThemeSpec theme) { return new BaselineVisualTheme( theme.themeId(), @@ -337,4 +205,9 @@ public record CompilerLanguageServiceBridge() implements LanguageServiceBridge { tokenStyle.bold(), tokenStyle.underline()); } + + private p.studio.compiler.models.FrontendSpec frontendSpec(final LspProjectContext context) { + return FrontendRegistryService.getFrontendSpec(context.languageId()) + .orElseThrow(() -> new IllegalArgumentException("no frontend registered for languageId: " + context.languageId())); + } }