implements PLN-0137 (5/7) semantic-token-overlay

Keep the lexical token provider as the baseline. When hover resolution
names a symbol and an existing semantic key matches it, replace the
weaker identifier token. A failed resolution, a broken file, or a
symbol without an existing key keeps the provider token. No new
semantic key is added.
This commit is contained in:
2026-09-22 09:09:43 +01:00
parent 7629a73e0a
commit 202c785f94
7 changed files with 297 additions and 2 deletions
@@ -17,6 +17,7 @@ import p.studio.compiler.models.SourceKind;
import p.studio.compiler.pbs.semantics.PbsEditorialRename;
import p.studio.compiler.pbs.semantics.PbsEditorialWorkspaceSymbol;
import p.studio.compiler.pbs.PbsDocumentFormatter;
import p.studio.compiler.pbs.PbsSemanticTokenOverlay;
import p.studio.compiler.pbs.PbsStructureRanges;
import p.studio.compiler.pbs.semantics.PbsQuickFix;
import p.studio.compiler.pbs.semantics.PbsQuickFixCollector;
@@ -563,7 +564,31 @@ public final class PBSFrontendLanguageService implements FrontendLanguageService
@Override
public List<FrontendSemanticToken> semanticTokens(final FrontendDocumentRequest request) {
return semanticTokens(request == null ? "" : request.documentText());
return semanticTokens(request, null);
}
@Override
public List<FrontendSemanticToken> semanticTokens(
final FrontendDocumentRequest request,
final FrontendEditorialContext editorialContext) {
final String text = request == null || request.documentText() == null ? "" : request.documentText();
final List<FrontendSemanticToken> baseline = semanticTokenProvider.tokenize(text);
if (editorialContext == null
|| !(editorialContext.syntaxTree() instanceof PbsAst.File ast)
|| !(editorialContext.supplementalDeclarations() instanceof ReadOnlyList<?> supplementalTopDecls)
|| !(editorialContext.semanticContext() instanceof FESurfaceContext feSurfaceContext)) {
return baseline;
}
@SuppressWarnings("unchecked")
final ReadOnlyList<PbsAst.TopDecl> typedSupplementalTopDecls =
(ReadOnlyList<PbsAst.TopDecl>) supplementalTopDecls;
return PbsSemanticTokenOverlay.apply(
baseline,
text,
ast,
typedSupplementalTopDecls,
feSurfaceContext,
editorialSupportService);
}
public List<FrontendSemanticToken> semanticTokens(final String documentText) {
@@ -0,0 +1,100 @@
package p.studio.compiler.pbs;
import p.studio.compiler.messages.FESurfaceContext;
import p.studio.compiler.models.FrontendSemanticToken;
import p.studio.compiler.pbs.ast.PbsAst;
import p.studio.compiler.pbs.semantics.PbsEditorialResolvedSymbol;
import p.studio.compiler.pbs.semantics.PbsEditorialSupportService;
import p.studio.compiler.pbs.semantics.PbsEditorialSymbolKind;
import p.studio.utilities.structures.ReadOnlyList;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
public final class PbsSemanticTokenOverlay {
private static final Set<String> LEXICAL_KEYS = Set.of(
PbsSemanticKind.COMMENT.semanticKey(),
PbsSemanticKind.STRING.semanticKey(),
PbsSemanticKind.NUMBER.semanticKey(),
PbsSemanticKind.LITERAL.semanticKey(),
PbsSemanticKind.KEYWORD.semanticKey(),
PbsSemanticKind.OPERATOR.semanticKey(),
PbsSemanticKind.PUNCTUATION.semanticKey());
private PbsSemanticTokenOverlay() {
}
public static List<FrontendSemanticToken> apply(
final List<FrontendSemanticToken> baseline,
final String documentText,
final PbsAst.File ast,
final ReadOnlyList<PbsAst.TopDecl> supplementalTopDecls,
final FESurfaceContext feSurfaceContext,
final PbsEditorialSupportService editorialSupportService) {
if (baseline == null || baseline.isEmpty() || ast == null || editorialSupportService == null) {
return baseline == null ? List.of() : List.copyOf(baseline);
}
final String text = documentText == null ? "" : documentText;
final ReadOnlyList<PbsAst.TopDecl> supplementals = supplementalTopDecls == null
? ReadOnlyList.empty()
: supplementalTopDecls;
final FESurfaceContext surface = feSurfaceContext == null ? FESurfaceContext.empty() : feSurfaceContext;
final ArrayList<FrontendSemanticToken> overlaid = new ArrayList<>(baseline.size());
for (final FrontendSemanticToken token : baseline) {
overlaid.add(overlay(token, text, ast, supplementals, surface, editorialSupportService));
}
return List.copyOf(overlaid);
}
private static FrontendSemanticToken overlay(
final FrontendSemanticToken token,
final String text,
final PbsAst.File ast,
final ReadOnlyList<PbsAst.TopDecl> supplementals,
final FESurfaceContext surface,
final PbsEditorialSupportService editorialSupportService) {
if (token == null || LEXICAL_KEYS.contains(token.semanticKey())) {
return token;
}
try {
final String resolvedKey = editorialSupportService.hover(
text,
ast,
supplementals,
surface,
token.startOffset())
.map(PbsSemanticTokenOverlay::semanticKey)
.orElse(null);
if (resolvedKey == null || resolvedKey.equals(token.semanticKey())) {
return token;
}
return new FrontendSemanticToken(token.startOffset(), token.endOffset(), resolvedKey);
} catch (final RuntimeException ignored) {
return token;
}
}
private static String semanticKey(final PbsEditorialResolvedSymbol symbol) {
if (symbol == null || symbol.kind() == null) {
return null;
}
final PbsSemanticKind kind = switch (symbol.kind()) {
case FUNCTION -> PbsSemanticKind.FUNCTION;
case METHOD -> PbsSemanticKind.METHOD;
case CONSTRUCTOR -> PbsSemanticKind.CONSTRUCTOR;
case STRUCT -> PbsSemanticKind.STRUCT;
case BUILTIN_TYPE -> PbsSemanticKind.BUILTIN_TYPE;
case SERVICE -> PbsSemanticKind.SERVICE;
case HOST -> PbsSemanticKind.HOST;
case CONTRACT -> PbsSemanticKind.CONTRACT;
case CALLBACK -> PbsSemanticKind.CALLBACK;
case ENUM -> PbsSemanticKind.ENUM;
case ERROR -> PbsSemanticKind.ERROR;
case GLOBAL -> PbsSemanticKind.GLOBAL;
case CONST -> PbsSemanticKind.CONST;
case KEYWORD, LOCAL, PARAMETER, FIELD -> null;
};
return kind == null ? null : kind.semanticKey();
}
}
@@ -0,0 +1,91 @@
package p.studio.compiler.pbs;
import org.junit.jupiter.api.Test;
import p.studio.compiler.PBSFrontendLanguageService;
import p.studio.compiler.PBSSemanticTokenProvider;
import p.studio.compiler.messages.FESurfaceContext;
import p.studio.compiler.models.FrontendSemanticToken;
import p.studio.compiler.pbs.ast.PbsAst;
import p.studio.compiler.pbs.lexer.PbsLexer;
import p.studio.compiler.pbs.parser.PbsParser;
import p.studio.compiler.services.FrontendDocumentRequest;
import p.studio.compiler.services.FrontendEditorialContext;
import p.studio.compiler.source.diagnostics.DiagnosticSink;
import p.studio.compiler.source.identifiers.FileId;
import p.studio.utilities.structures.ReadOnlyList;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class PbsSemanticTokenOverlayTest {
@Test
void resolvedFunctionUseReplacesTheIdentifierBaseline() {
final String source = """
fn helper() -> int {
return 1;
}
fn main() -> int {
return helper;
}
""";
final PbsAst.File ast = PbsParser.parse(
PbsLexer.lex(source, FileId.none(), DiagnosticSink.empty()),
FileId.none(),
DiagnosticSink.empty());
final var service = new PBSFrontendLanguageService();
final var tokens = service.semanticTokens(
new FrontendDocumentRequest(Path.of("."), Path.of("demo.pbs"), source),
new FrontendEditorialContext(ast, ReadOnlyList.empty(), FESurfaceContext.empty()));
assertEquals(PbsSemanticKind.FUNCTION.semanticKey(), keyAt(source, tokens, "return helper"));
assertTrue(tokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.KEYWORD.semanticKey())));
}
@Test
void localBindingAndBrokenTextKeepTheBaselineToken() {
final String source = """
fn main() -> int {
let value: int = 1;
return value;
}
""";
final PbsAst.File ast = PbsParser.parse(
PbsLexer.lex(source, FileId.none(), DiagnosticSink.empty()),
FileId.none(),
DiagnosticSink.empty());
final var service = new PBSFrontendLanguageService();
final var request = new FrontendDocumentRequest(Path.of("."), Path.of("demo.pbs"), source);
final var overlaid = service.semanticTokens(
request,
new FrontendEditorialContext(ast, ReadOnlyList.empty(), FESurfaceContext.empty()));
assertEquals(PbsSemanticKind.IDENTIFIER.semanticKey(), keyAt(source, overlaid, "return value"));
final String broken = "fn main( {";
final List<FrontendSemanticToken> baseline = new PBSSemanticTokenProvider().tokenize(broken);
final List<FrontendSemanticToken> withoutContext = service.semanticTokens(
new FrontendDocumentRequest(Path.of("."), Path.of("broken.pbs"), broken));
assertEquals(baseline, withoutContext);
assertTrue(withoutContext.stream().anyMatch(token ->
token.semanticKey().equals(PbsSemanticKind.KEYWORD.semanticKey())));
}
private static String keyAt(
final String source,
final List<FrontendSemanticToken> tokens,
final String marker) {
final int nameOffset = source.indexOf(marker) + marker.lastIndexOf(' ') + 1;
final int byteOffset = source.substring(0, nameOffset).getBytes(StandardCharsets.UTF_8).length;
return tokens.stream()
.filter(token -> token.startOffset() == byteOffset)
.map(FrontendSemanticToken::semanticKey)
.findFirst()
.orElseThrow();
}
}