Frontend Visual Theme Spec and Retirement of Host-Consumed Semantic CSS

prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSDefinitions.java

@@ -17,5 +17,6 @@ public class PBSDefinitions {
                     List.of(PBSFrontendThemes.DEFAULT_PBS),
                     "pbs-default",
                     List.of(PBSFrontendHostProjections.VSCODE)))
+            .semanticTokenProvider(new PBSSemanticTokenProvider())
             .build();
 }

prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSSemanticTokenProvider.java

@@ -0,0 +1,158 @@
+package p.studio.compiler;
+
+import p.studio.compiler.models.FrontendSemanticToken;
+import p.studio.compiler.models.FrontendSemanticTokenProvider;
+import p.studio.compiler.pbs.PbsSemanticKind;
+import p.studio.compiler.pbs.lexer.PbsLexer;
+import p.studio.compiler.pbs.lexer.PbsToken;
+import p.studio.compiler.pbs.lexer.PbsTokenKind;
+import p.studio.compiler.source.diagnostics.DiagnosticSink;
+import p.studio.compiler.source.identifiers.FileId;
+import p.studio.utilities.structures.ReadOnlyList;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+public final class PBSSemanticTokenProvider implements FrontendSemanticTokenProvider {
+    private static final Set<String> BUILTIN_TYPE_NAMES = Set.of("int", "float", "bool", "string", "i32", "i64", "f32", "f64");
+
+    @Override
+    public List<FrontendSemanticToken> tokenize(final String documentText) {
+        final String effectiveText = documentText == null ? "" : documentText;
+        final ReadOnlyList<PbsToken> tokens = PbsLexer.lex(effectiveText, FileId.none(), DiagnosticSink.empty());
+        final ArrayList<FrontendSemanticToken> semanticTokens = new ArrayList<>();
+
+        for (int index = 0; index < tokens.size(); index += 1) {
+            final PbsToken token = tokens.get(index);
+            final PbsSemanticKind semanticKind = classifyToken(tokens, index);
+            if (semanticKind == null) {
+                continue;
+            }
+            semanticTokens.add(new FrontendSemanticToken(token.start(), token.end(), semanticKind.semanticKey()));
+        }
+
+        return List.copyOf(semanticTokens);
+    }
+
+    private PbsSemanticKind classifyToken(
+            final ReadOnlyList<PbsToken> tokens,
+            final int index) {
+        final PbsToken token = tokens.get(index);
+        final PbsSemanticKind lexicalKind = PbsSemanticKind.forToken(token);
+        if (lexicalKind != null) {
+            return lexicalKind;
+        }
+        if (token.kind() == PbsTokenKind.IDENTIFIER) {
+            if (isLifecycleToken(tokens, index, token)) {
+                return PbsSemanticKind.LIFECYCLE;
+            }
+            return classifyIdentifierToken(tokens, index, token);
+        }
+        return null;
+    }
+
+    private PbsSemanticKind classifyIdentifierToken(
+            final ReadOnlyList<PbsToken> tokens,
+            final int index,
+            final PbsToken token) {
+        final PbsTokenKind previousKind = previousSignificantKind(tokens, index);
+        final PbsTokenKind nextKind = nextSignificantKind(tokens, index);
+
+        if (previousKind == PbsTokenKind.STRUCT) {
+            return PbsSemanticKind.STRUCT;
+        }
+        if (previousKind == PbsTokenKind.CONTRACT) {
+            return PbsSemanticKind.CONTRACT;
+        }
+        if (previousKind == PbsTokenKind.HOST) {
+            return PbsSemanticKind.HOST;
+        }
+        if (previousKind == PbsTokenKind.SERVICE) {
+            return PbsSemanticKind.SERVICE;
+        }
+        if (previousKind == PbsTokenKind.ENUM || previousKind == PbsTokenKind.ERROR) {
+            return PbsSemanticKind.ENUM;
+        }
+        if (previousKind == PbsTokenKind.CALLBACK) {
+            return PbsSemanticKind.CALLBACK;
+        }
+        if (previousKind == PbsTokenKind.GLOBAL) {
+            return PbsSemanticKind.GLOBAL;
+        }
+        if (previousKind == PbsTokenKind.CONST) {
+            return PbsSemanticKind.CONST;
+        }
+        if (previousKind == PbsTokenKind.CTOR || previousKind == PbsTokenKind.NEW) {
+            return PbsSemanticKind.CONSTRUCTOR;
+        }
+        if (previousKind == PbsTokenKind.BUILTIN) {
+            return PbsSemanticKind.BUILTIN_TYPE;
+        }
+        if (previousKind == PbsTokenKind.DOT && nextKind == PbsTokenKind.LEFT_PAREN) {
+            return PbsSemanticKind.METHOD;
+        }
+        if (nextKind == PbsTokenKind.LEFT_PAREN) {
+            return PbsSemanticKind.FUNCTION;
+        }
+        if (isTypeReferenceContext(previousKind, token.lexeme())) {
+            return PbsSemanticKind.BUILTIN_TYPE;
+        }
+        return PbsSemanticKind.IDENTIFIER;
+    }
+
+    private boolean isTypeReferenceContext(
+            final PbsTokenKind previousKind,
+            final String lexeme) {
+        if (previousKind == null) {
+            return false;
+        }
+        if (previousKind != PbsTokenKind.COLON
+                && previousKind != PbsTokenKind.ARROW
+                && previousKind != PbsTokenKind.AS
+                && previousKind != PbsTokenKind.IMPLEMENTS) {
+            return false;
+        }
+        return BUILTIN_TYPE_NAMES.contains(lexeme);
+    }
+
+    private PbsTokenKind previousSignificantKind(
+            final ReadOnlyList<PbsToken> tokens,
+            final int index) {
+        for (int cursor = index - 1; cursor >= 0; cursor -= 1) {
+            final PbsTokenKind kind = tokens.get(cursor).kind();
+            if (kind == PbsTokenKind.COMMENT || kind == PbsTokenKind.EOF) {
+                continue;
+            }
+            return kind;
+        }
+        return null;
+    }
+
+    private PbsTokenKind nextSignificantKind(
+            final ReadOnlyList<PbsToken> tokens,
+            final int index) {
+        for (int cursor = index + 1; cursor < tokens.size(); cursor += 1) {
+            final PbsTokenKind kind = tokens.get(cursor).kind();
+            if (kind == PbsTokenKind.COMMENT || kind == PbsTokenKind.EOF) {
+                continue;
+            }
+            return kind;
+        }
+        return null;
+    }
+
+    private boolean isLifecycleToken(
+            final ReadOnlyList<PbsToken> tokens,
+            final int index,
+            final PbsToken token) {
+        if (!"Init".equals(token.lexeme()) && !"Frame".equals(token.lexeme())) {
+            return false;
+        }
+        if (index == 0 || index + 1 >= tokens.size()) {
+            return false;
+        }
+        return tokens.get(index - 1).kind() == PbsTokenKind.LEFT_BRACKET
+                && tokens.get(index + 1).kind() == PbsTokenKind.RIGHT_BRACKET;
+    }
+}

prometeu-compiler/frontends/prometeu-frontend-pbs/src/test/java/p/studio/compiler/pbs/PbsSemanticPresentationContractTest.java

@@ -3,6 +3,9 @@ package p.studio.compiler.pbs;
 import org.junit.jupiter.api.Test;
 import p.studio.compiler.PBSFrontendHostProjections;
 import p.studio.compiler.PBSDefinitions;
+import p.studio.compiler.models.FrontendSemanticToken;
+
+import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -50,4 +53,24 @@ final class PbsSemanticPresentationContractTest {
                 java.util.List.of("readonly"),
                 vscodeProjection.requireProjection(PbsSemanticKind.GLOBAL.semanticKey()).hostTokenModifiers());
     }
+
+    @Test
+    void shouldPublishPbsOwnedSemanticTokenProvider() {
+        final List<FrontendSemanticToken> semanticTokens = PBSDefinitions.PBS.getSemanticTokenProvider().tokenize("""
+                struct Vec2 {}
+
+                fn main() -> void {
+                  const answer: int = 42;
+                  let value = new Vec2();
+                  value.length();
+                }
+                """);
+
+        assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.STRUCT.semanticKey())));
+        assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.FUNCTION.semanticKey())));
+        assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.CONST.semanticKey())));
+        assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.BUILTIN_TYPE.semanticKey())));
+        assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.CONSTRUCTOR.semanticKey())));
+        assertTrue(semanticTokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.METHOD.semanticKey())));
+    }
 }

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSemanticToken.java

@@ -0,0 +1,29 @@
+package p.studio.compiler.models;
+
+import java.util.Objects;
+
+public record FrontendSemanticToken(
+        int startOffset,
+        int endOffset,
+        String semanticKey) {
+
+    public FrontendSemanticToken {
+        if (startOffset < 0) {
+            throw new IllegalArgumentException("startOffset must be >= 0");
+        }
+        if (endOffset < startOffset) {
+            throw new IllegalArgumentException("endOffset must be >= startOffset");
+        }
+        semanticKey = requireText(semanticKey, "semanticKey");
+    }
+
+    private static String requireText(
+            final String value,
+            final String field) {
+        final String candidate = Objects.requireNonNull(value, field).trim();
+        if (candidate.isEmpty()) {
+            throw new IllegalArgumentException(field + " must not be blank");
+        }
+        return candidate;
+    }
+}

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSemanticTokenProvider.java

@@ -0,0 +1,14 @@
+package p.studio.compiler.models;
+
+import java.util.List;
+
+@FunctionalInterface
+public interface FrontendSemanticTokenProvider {
+    FrontendSemanticTokenProvider NOOP = documentText -> List.of();
+
+    List<FrontendSemanticToken> tokenize(String documentText);
+
+    static FrontendSemanticTokenProvider noop() {
+        return NOOP;
+    }
+}

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/FrontendSpec.java

@@ -17,6 +17,8 @@ public class FrontendSpec {
     private final List<Stdlib> stdlibVersions = List.of();
     @Builder.Default
     private final FrontendSemanticPresentationSpec semanticPresentation = FrontendSemanticPresentationSpec.empty();
+    @Builder.Default
+    private final FrontendSemanticTokenProvider semanticTokenProvider = FrontendSemanticTokenProvider.noop();
 
     public String toString() {
         return String.format("FrontendSpec(language=%s)", languageId);

prometeu-lsp/prometeu-lsp-v1/build.gradle.kts

@@ -6,6 +6,6 @@ dependencies {
     implementation(project(":prometeu-lsp:prometeu-lsp-api"))
     implementation(project(":prometeu-compiler:prometeu-compiler-core"))
     implementation(project(":prometeu-compiler:prometeu-build-pipeline"))
-    implementation(project(":prometeu-compiler:frontends:prometeu-frontend-pbs"))
+    implementation(project(":prometeu-compiler:prometeu-frontend-registry"))
     implementation(libs.lsp4j)
 }

prometeu-lsp/prometeu-lsp-v1/src/main/java/p/studio/lsp/services/compiler/CompilerLanguageServiceBridge.java

@@ -1,6 +1,7 @@
 package p.studio.lsp.services.compiler;
 
 import org.apache.commons.lang3.StringUtils;
+import p.studio.compiler.FrontendRegistryService;
 import p.studio.compiler.messages.BuilderPipelineConfig;
 import p.studio.compiler.messages.BuildingIssue;
 import p.studio.compiler.models.AnalysisSnapshot;
@@ -8,21 +9,16 @@ import p.studio.compiler.models.BuilderPipelineContext;
 import p.studio.compiler.models.FrontendEditorPaletteSpec;
 import p.studio.compiler.models.FrontendHostProjectionEntrySpec;
 import p.studio.compiler.models.FrontendHostProjectionSpec;
-import p.studio.compiler.PBSDefinitions;
+import p.studio.compiler.models.FrontendSemanticPresentationSpec;
+import p.studio.compiler.models.FrontendSemanticToken;
 import p.studio.compiler.models.FrontendTokenStyleSpec;
 import p.studio.compiler.models.FrontendVisualThemeSpec;
-import p.studio.compiler.pbs.PbsSemanticKind;
-import p.studio.compiler.pbs.lexer.PbsLexer;
-import p.studio.compiler.pbs.lexer.PbsToken;
-import p.studio.compiler.pbs.lexer.PbsTokenKind;
 import p.studio.compiler.source.identifiers.FileId;
-import p.studio.compiler.source.diagnostics.DiagnosticSink;
 import p.studio.compiler.utilities.SourceProviderFactory;
 import p.studio.compiler.workspaces.BuilderPipelineService;
 import p.studio.lsp.messages.*;
 import p.studio.lsp.services.LanguageServiceBridge;
 import p.studio.utilities.logs.LogAggregator;
-import p.studio.utilities.structures.ReadOnlyList;
 
 import java.net.URI;
 import java.nio.file.Path;
@@ -30,16 +26,13 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
-
-public record CompilerLanguageServiceBridge() implements LanguageServiceBridge {
-    private static final Set<String> BUILTIN_TYPE_NAMES = Set.of("int", "float", "bool", "string", "i32", "i64", "f32", "f64");
-
 
+public final class CompilerLanguageServiceBridge implements LanguageServiceBridge {
     @Override
     public BaselineServerDescription describeServer(final LspProjectContext context) {
         Objects.requireNonNull(context, "context");
-        final var presentation = PBSDefinitions.PBS.getSemanticPresentation();
+        final var frontend = frontendSpec(context);
+        final var presentation = frontend.getSemanticPresentation();
         return new BaselineServerDescription(
                 "Prometeu Studio LSP",
                 "0.1.0",
@@ -86,28 +79,24 @@ public record CompilerLanguageServiceBridge() implements LanguageServiceBridge {
             final String text) {
         Objects.requireNonNull(context, "context");
         final String effectiveText = text == null ? "" : text;
-        final var legend = PBSDefinitions.PBS.getSemanticPresentation().semanticKeys();
+        final var frontend = frontendSpec(context);
-        final var tokens = PbsLexer.lex(effectiveText, FileId.none(), DiagnosticSink.empty());
+        final FrontendSemanticPresentationSpec presentation = frontend.getSemanticPresentation();
+        final List<FrontendSemanticToken> tokens = frontend.getSemanticTokenProvider().tokenize(effectiveText);
         final var positionMapper = new DocumentPositionMapper(effectiveText);
         final var semanticTokens = new ArrayList<BaselineSemanticToken>();
 
-        for (int index = 0; index < tokens.size(); index += 1) {
+        for (final FrontendSemanticToken token : tokens) {
-            final PbsToken token = tokens.get(index);
+            final DocumentPosition start = positionMapper.positionOf(token.startOffset());
-            final PbsSemanticKind semanticKind = classifyToken(tokens, index);
+            final DocumentPosition end = positionMapper.positionOf(token.endOffset());
-            if (semanticKind == null) {
-                continue;
-            }
-            final DocumentPosition start = positionMapper.positionOf(token.start());
-            final DocumentPosition end = positionMapper.positionOf(token.end());
             final int length = Math.max(1, end.character() - start.character());
             semanticTokens.add(new BaselineSemanticToken(
                     start.line(),
                     start.character(),
                     length,
-                    semanticKind.semanticKey()));
+                    token.semanticKey()));
         }
 
-        return new BaselineSemanticTokens(legend, semanticTokens);
+        return new BaselineSemanticTokens(presentation.semanticKeys(), semanticTokens);
     }
 
     @Override
@@ -174,127 +163,6 @@ public record CompilerLanguageServiceBridge() implements LanguageServiceBridge {
         return Path.of(URI.create(Objects.requireNonNull(documentUri, "documentUri"))).toAbsolutePath().normalize();
     }
 
-    private PbsSemanticKind classifyToken(
-            final ReadOnlyList<PbsToken> tokens,
-            final int index) {
-        final PbsToken token = tokens.get(index);
-        final PbsSemanticKind lexicalKind = PbsSemanticKind.forToken(token);
-        if (lexicalKind != null) {
-            return lexicalKind;
-        }
-        if (token.kind() == p.studio.compiler.pbs.lexer.PbsTokenKind.IDENTIFIER) {
-            if (isLifecycleToken(tokens, index, token)) {
-                return PbsSemanticKind.LIFECYCLE;
-            }
-            return classifyIdentifierToken(tokens, index, token);
-        }
-        return null;
-    }
-
-    private PbsSemanticKind classifyIdentifierToken(
-            final ReadOnlyList<PbsToken> tokens,
-            final int index,
-            final PbsToken token) {
-        final PbsTokenKind previousKind = previousSignificantKind(tokens, index);
-        final PbsTokenKind nextKind = nextSignificantKind(tokens, index);
-
-        if (previousKind == PbsTokenKind.STRUCT) {
-            return PbsSemanticKind.STRUCT;
-        }
-        if (previousKind == PbsTokenKind.CONTRACT) {
-            return PbsSemanticKind.CONTRACT;
-        }
-        if (previousKind == PbsTokenKind.HOST) {
-            return PbsSemanticKind.HOST;
-        }
-        if (previousKind == PbsTokenKind.SERVICE) {
-            return PbsSemanticKind.SERVICE;
-        }
-        if (previousKind == PbsTokenKind.ENUM || previousKind == PbsTokenKind.ERROR) {
-            return PbsSemanticKind.ENUM;
-        }
-        if (previousKind == PbsTokenKind.CALLBACK) {
-            return PbsSemanticKind.CALLBACK;
-        }
-        if (previousKind == PbsTokenKind.GLOBAL) {
-            return PbsSemanticKind.GLOBAL;
-        }
-        if (previousKind == PbsTokenKind.CONST) {
-            return PbsSemanticKind.CONST;
-        }
-        if (previousKind == PbsTokenKind.CTOR || previousKind == PbsTokenKind.NEW) {
-            return PbsSemanticKind.CONSTRUCTOR;
-        }
-        if (previousKind == PbsTokenKind.BUILTIN) {
-            return PbsSemanticKind.BUILTIN_TYPE;
-        }
-        if (previousKind == PbsTokenKind.DOT && nextKind == PbsTokenKind.LEFT_PAREN) {
-            return PbsSemanticKind.METHOD;
-        }
-        if (nextKind == PbsTokenKind.LEFT_PAREN) {
-            return PbsSemanticKind.FUNCTION;
-        }
-        if (isTypeReferenceContext(previousKind, token.lexeme())) {
-            return PbsSemanticKind.BUILTIN_TYPE;
-        }
-        return PbsSemanticKind.IDENTIFIER;
-    }
-
-    private boolean isTypeReferenceContext(
-            final PbsTokenKind previousKind,
-            final String lexeme) {
-        if (previousKind == null) {
-            return false;
-        }
-        if (previousKind != PbsTokenKind.COLON
-                && previousKind != PbsTokenKind.ARROW
-                && previousKind != PbsTokenKind.AS
-                && previousKind != PbsTokenKind.IMPLEMENTS) {
-            return false;
-        }
-        return BUILTIN_TYPE_NAMES.contains(lexeme);
-    }
-
-    private PbsTokenKind previousSignificantKind(
-            final ReadOnlyList<PbsToken> tokens,
-            final int index) {
-        for (int cursor = index - 1; cursor >= 0; cursor -= 1) {
-            final PbsTokenKind kind = tokens.get(cursor).kind();
-            if (kind == PbsTokenKind.COMMENT || kind == PbsTokenKind.EOF) {
-                continue;
-            }
-            return kind;
-        }
-        return null;
-    }
-
-    private PbsTokenKind nextSignificantKind(
-            final ReadOnlyList<PbsToken> tokens,
-            final int index) {
-        for (int cursor = index + 1; cursor < tokens.size(); cursor += 1) {
-            final PbsTokenKind kind = tokens.get(cursor).kind();
-            if (kind == PbsTokenKind.COMMENT || kind == PbsTokenKind.EOF) {
-                continue;
-            }
-            return kind;
-        }
-        return null;
-    }
-
-    private boolean isLifecycleToken(
-            final ReadOnlyList<PbsToken> tokens,
-            final int index,
-            final PbsToken token) {
-        if (!"Init".equals(token.lexeme()) && !"Frame".equals(token.lexeme())) {
-            return false;
-        }
-        if (index == 0 || index + 1 >= tokens.size()) {
-            return false;
-        }
-        return tokens.get(index - 1).kind() == p.studio.compiler.pbs.lexer.PbsTokenKind.LEFT_BRACKET
-                && tokens.get(index + 1).kind() == p.studio.compiler.pbs.lexer.PbsTokenKind.RIGHT_BRACKET;
-    }
-
     private BaselineVisualTheme mapVisualTheme(final FrontendVisualThemeSpec theme) {
         return new BaselineVisualTheme(
                 theme.themeId(),
@@ -337,4 +205,9 @@ public record CompilerLanguageServiceBridge() implements LanguageServiceBridge {
                 tokenStyle.bold(),
                 tokenStyle.underline());
     }
+
+    private p.studio.compiler.models.FrontendSpec frontendSpec(final LspProjectContext context) {
+        return FrontendRegistryService.getFrontendSpec(context.languageId())
+                .orElseThrow(() -> new IllegalArgumentException("no frontend registered for languageId: " + context.languageId()));
+    }
 }