implements PLN-0137 (5/7) semantic-token-overlay
Keep the lexical token provider as the baseline. When hover resolution names a symbol and an existing semantic key matches it, replace the weaker identifier token. A failed resolution, a broken file, or a symbol without an existing key keeps the provider token. No new semantic key is added.
This commit is contained in:
+26
-1
@@ -17,6 +17,7 @@ import p.studio.compiler.models.SourceKind;
|
||||
import p.studio.compiler.pbs.semantics.PbsEditorialRename;
|
||||
import p.studio.compiler.pbs.semantics.PbsEditorialWorkspaceSymbol;
|
||||
import p.studio.compiler.pbs.PbsDocumentFormatter;
|
||||
import p.studio.compiler.pbs.PbsSemanticTokenOverlay;
|
||||
import p.studio.compiler.pbs.PbsStructureRanges;
|
||||
import p.studio.compiler.pbs.semantics.PbsQuickFix;
|
||||
import p.studio.compiler.pbs.semantics.PbsQuickFixCollector;
|
||||
@@ -563,7 +564,31 @@ public final class PBSFrontendLanguageService implements FrontendLanguageService
|
||||
|
||||
@Override
|
||||
public List<FrontendSemanticToken> semanticTokens(final FrontendDocumentRequest request) {
|
||||
return semanticTokens(request == null ? "" : request.documentText());
|
||||
return semanticTokens(request, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<FrontendSemanticToken> semanticTokens(
|
||||
final FrontendDocumentRequest request,
|
||||
final FrontendEditorialContext editorialContext) {
|
||||
final String text = request == null || request.documentText() == null ? "" : request.documentText();
|
||||
final List<FrontendSemanticToken> baseline = semanticTokenProvider.tokenize(text);
|
||||
if (editorialContext == null
|
||||
|| !(editorialContext.syntaxTree() instanceof PbsAst.File ast)
|
||||
|| !(editorialContext.supplementalDeclarations() instanceof ReadOnlyList<?> supplementalTopDecls)
|
||||
|| !(editorialContext.semanticContext() instanceof FESurfaceContext feSurfaceContext)) {
|
||||
return baseline;
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
final ReadOnlyList<PbsAst.TopDecl> typedSupplementalTopDecls =
|
||||
(ReadOnlyList<PbsAst.TopDecl>) supplementalTopDecls;
|
||||
return PbsSemanticTokenOverlay.apply(
|
||||
baseline,
|
||||
text,
|
||||
ast,
|
||||
typedSupplementalTopDecls,
|
||||
feSurfaceContext,
|
||||
editorialSupportService);
|
||||
}
|
||||
|
||||
public List<FrontendSemanticToken> semanticTokens(final String documentText) {
|
||||
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package p.studio.compiler.pbs;
|
||||
|
||||
import p.studio.compiler.messages.FESurfaceContext;
|
||||
import p.studio.compiler.models.FrontendSemanticToken;
|
||||
import p.studio.compiler.pbs.ast.PbsAst;
|
||||
import p.studio.compiler.pbs.semantics.PbsEditorialResolvedSymbol;
|
||||
import p.studio.compiler.pbs.semantics.PbsEditorialSupportService;
|
||||
import p.studio.compiler.pbs.semantics.PbsEditorialSymbolKind;
|
||||
import p.studio.utilities.structures.ReadOnlyList;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public final class PbsSemanticTokenOverlay {
|
||||
private static final Set<String> LEXICAL_KEYS = Set.of(
|
||||
PbsSemanticKind.COMMENT.semanticKey(),
|
||||
PbsSemanticKind.STRING.semanticKey(),
|
||||
PbsSemanticKind.NUMBER.semanticKey(),
|
||||
PbsSemanticKind.LITERAL.semanticKey(),
|
||||
PbsSemanticKind.KEYWORD.semanticKey(),
|
||||
PbsSemanticKind.OPERATOR.semanticKey(),
|
||||
PbsSemanticKind.PUNCTUATION.semanticKey());
|
||||
|
||||
private PbsSemanticTokenOverlay() {
|
||||
}
|
||||
|
||||
public static List<FrontendSemanticToken> apply(
|
||||
final List<FrontendSemanticToken> baseline,
|
||||
final String documentText,
|
||||
final PbsAst.File ast,
|
||||
final ReadOnlyList<PbsAst.TopDecl> supplementalTopDecls,
|
||||
final FESurfaceContext feSurfaceContext,
|
||||
final PbsEditorialSupportService editorialSupportService) {
|
||||
if (baseline == null || baseline.isEmpty() || ast == null || editorialSupportService == null) {
|
||||
return baseline == null ? List.of() : List.copyOf(baseline);
|
||||
}
|
||||
final String text = documentText == null ? "" : documentText;
|
||||
final ReadOnlyList<PbsAst.TopDecl> supplementals = supplementalTopDecls == null
|
||||
? ReadOnlyList.empty()
|
||||
: supplementalTopDecls;
|
||||
final FESurfaceContext surface = feSurfaceContext == null ? FESurfaceContext.empty() : feSurfaceContext;
|
||||
final ArrayList<FrontendSemanticToken> overlaid = new ArrayList<>(baseline.size());
|
||||
for (final FrontendSemanticToken token : baseline) {
|
||||
overlaid.add(overlay(token, text, ast, supplementals, surface, editorialSupportService));
|
||||
}
|
||||
return List.copyOf(overlaid);
|
||||
}
|
||||
|
||||
private static FrontendSemanticToken overlay(
|
||||
final FrontendSemanticToken token,
|
||||
final String text,
|
||||
final PbsAst.File ast,
|
||||
final ReadOnlyList<PbsAst.TopDecl> supplementals,
|
||||
final FESurfaceContext surface,
|
||||
final PbsEditorialSupportService editorialSupportService) {
|
||||
if (token == null || LEXICAL_KEYS.contains(token.semanticKey())) {
|
||||
return token;
|
||||
}
|
||||
try {
|
||||
final String resolvedKey = editorialSupportService.hover(
|
||||
text,
|
||||
ast,
|
||||
supplementals,
|
||||
surface,
|
||||
token.startOffset())
|
||||
.map(PbsSemanticTokenOverlay::semanticKey)
|
||||
.orElse(null);
|
||||
if (resolvedKey == null || resolvedKey.equals(token.semanticKey())) {
|
||||
return token;
|
||||
}
|
||||
return new FrontendSemanticToken(token.startOffset(), token.endOffset(), resolvedKey);
|
||||
} catch (final RuntimeException ignored) {
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
private static String semanticKey(final PbsEditorialResolvedSymbol symbol) {
|
||||
if (symbol == null || symbol.kind() == null) {
|
||||
return null;
|
||||
}
|
||||
final PbsSemanticKind kind = switch (symbol.kind()) {
|
||||
case FUNCTION -> PbsSemanticKind.FUNCTION;
|
||||
case METHOD -> PbsSemanticKind.METHOD;
|
||||
case CONSTRUCTOR -> PbsSemanticKind.CONSTRUCTOR;
|
||||
case STRUCT -> PbsSemanticKind.STRUCT;
|
||||
case BUILTIN_TYPE -> PbsSemanticKind.BUILTIN_TYPE;
|
||||
case SERVICE -> PbsSemanticKind.SERVICE;
|
||||
case HOST -> PbsSemanticKind.HOST;
|
||||
case CONTRACT -> PbsSemanticKind.CONTRACT;
|
||||
case CALLBACK -> PbsSemanticKind.CALLBACK;
|
||||
case ENUM -> PbsSemanticKind.ENUM;
|
||||
case ERROR -> PbsSemanticKind.ERROR;
|
||||
case GLOBAL -> PbsSemanticKind.GLOBAL;
|
||||
case CONST -> PbsSemanticKind.CONST;
|
||||
case KEYWORD, LOCAL, PARAMETER, FIELD -> null;
|
||||
};
|
||||
return kind == null ? null : kind.semanticKey();
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
package p.studio.compiler.pbs;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import p.studio.compiler.PBSFrontendLanguageService;
|
||||
import p.studio.compiler.PBSSemanticTokenProvider;
|
||||
import p.studio.compiler.messages.FESurfaceContext;
|
||||
import p.studio.compiler.models.FrontendSemanticToken;
|
||||
import p.studio.compiler.pbs.ast.PbsAst;
|
||||
import p.studio.compiler.pbs.lexer.PbsLexer;
|
||||
import p.studio.compiler.pbs.parser.PbsParser;
|
||||
import p.studio.compiler.services.FrontendDocumentRequest;
|
||||
import p.studio.compiler.services.FrontendEditorialContext;
|
||||
import p.studio.compiler.source.diagnostics.DiagnosticSink;
|
||||
import p.studio.compiler.source.identifiers.FileId;
|
||||
import p.studio.utilities.structures.ReadOnlyList;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class PbsSemanticTokenOverlayTest {
|
||||
|
||||
@Test
|
||||
void resolvedFunctionUseReplacesTheIdentifierBaseline() {
|
||||
final String source = """
|
||||
fn helper() -> int {
|
||||
return 1;
|
||||
}
|
||||
|
||||
fn main() -> int {
|
||||
return helper;
|
||||
}
|
||||
""";
|
||||
final PbsAst.File ast = PbsParser.parse(
|
||||
PbsLexer.lex(source, FileId.none(), DiagnosticSink.empty()),
|
||||
FileId.none(),
|
||||
DiagnosticSink.empty());
|
||||
final var service = new PBSFrontendLanguageService();
|
||||
final var tokens = service.semanticTokens(
|
||||
new FrontendDocumentRequest(Path.of("."), Path.of("demo.pbs"), source),
|
||||
new FrontendEditorialContext(ast, ReadOnlyList.empty(), FESurfaceContext.empty()));
|
||||
|
||||
assertEquals(PbsSemanticKind.FUNCTION.semanticKey(), keyAt(source, tokens, "return helper"));
|
||||
assertTrue(tokens.stream().anyMatch(token -> token.semanticKey().equals(PbsSemanticKind.KEYWORD.semanticKey())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void localBindingAndBrokenTextKeepTheBaselineToken() {
|
||||
final String source = """
|
||||
fn main() -> int {
|
||||
let value: int = 1;
|
||||
return value;
|
||||
}
|
||||
""";
|
||||
final PbsAst.File ast = PbsParser.parse(
|
||||
PbsLexer.lex(source, FileId.none(), DiagnosticSink.empty()),
|
||||
FileId.none(),
|
||||
DiagnosticSink.empty());
|
||||
final var service = new PBSFrontendLanguageService();
|
||||
final var request = new FrontendDocumentRequest(Path.of("."), Path.of("demo.pbs"), source);
|
||||
final var overlaid = service.semanticTokens(
|
||||
request,
|
||||
new FrontendEditorialContext(ast, ReadOnlyList.empty(), FESurfaceContext.empty()));
|
||||
|
||||
assertEquals(PbsSemanticKind.IDENTIFIER.semanticKey(), keyAt(source, overlaid, "return value"));
|
||||
|
||||
final String broken = "fn main( {";
|
||||
final List<FrontendSemanticToken> baseline = new PBSSemanticTokenProvider().tokenize(broken);
|
||||
final List<FrontendSemanticToken> withoutContext = service.semanticTokens(
|
||||
new FrontendDocumentRequest(Path.of("."), Path.of("broken.pbs"), broken));
|
||||
assertEquals(baseline, withoutContext);
|
||||
assertTrue(withoutContext.stream().anyMatch(token ->
|
||||
token.semanticKey().equals(PbsSemanticKind.KEYWORD.semanticKey())));
|
||||
}
|
||||
|
||||
private static String keyAt(
|
||||
final String source,
|
||||
final List<FrontendSemanticToken> tokens,
|
||||
final String marker) {
|
||||
final int nameOffset = source.indexOf(marker) + marker.lastIndexOf(' ') + 1;
|
||||
final int byteOffset = source.substring(0, nameOffset).getBytes(StandardCharsets.UTF_8).length;
|
||||
return tokens.stream()
|
||||
.filter(token -> token.startOffset() == byteOffset)
|
||||
.map(FrontendSemanticToken::semanticKey)
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user