implements PLN-0093

This commit is contained in:
2026-07-15 06:57:19 +01:00
parent 917b0e1b9a
commit 2b25364252
7 changed files with 93 additions and 4 deletions
@@ -50,7 +50,7 @@ public enum PbsSemanticKind {
public static PbsSemanticKind forToken(final PbsToken token) {
return switch (token.kind()) {
case COMMENT -> COMMENT;
case STRING_LITERAL -> STRING;
case STRING_LITERAL, DOC_TEXT_BLOCK -> STRING;
case INT_LITERAL, FLOAT_LITERAL, BOUNDED_LITERAL -> NUMBER;
case TRUE, FALSE, NONE, SOME, OK, ERR -> LITERAL;
case VOID, OPTIONAL, RESULT -> BUILTIN_TYPE;
@@ -3,5 +3,6 @@ package p.studio.compiler.pbs.lexer;
public enum LexErrors {
E_LEX_INVALID_CHAR,
E_LEX_UNTERMINATED_STRING,
E_LEX_UNTERMINATED_DOC_TEXT_BLOCK,
E_LEX_INVALID_STRING_ESCAPE,
}
@@ -17,6 +17,7 @@ public final class PbsLexer {
IDENTIFIER,
NUMBER,
STRING,
DOC_TEXT_BLOCK,
LINE_COMMENT
}
@@ -52,6 +53,7 @@ public final class PbsLexer {
case IDENTIFIER -> scanIdentifierState();
case NUMBER -> scanNumberState();
case STRING -> scanStringState();
case DOC_TEXT_BLOCK -> scanDocTextBlockState();
case LINE_COMMENT -> scanLineCommentState();
}
}
@@ -123,7 +125,15 @@ public final class PbsLexer {
}
addToken(match('=') ? PbsTokenKind.SLASH_EQUAL : PbsTokenKind.SLASH);
}
case '"' -> state = LexerState.STRING;
case '"' -> {
if (peek() == '"' && peekNext() == '"') {
advance();
advance();
state = LexerState.DOC_TEXT_BLOCK;
return;
}
state = LexerState.STRING;
}
default -> {
if (isDigit(c)) {
state = LexerState.NUMBER;
@@ -206,6 +216,23 @@ public final class PbsLexer {
state = LexerState.DEFAULT;
}
private void scanDocTextBlockState() {
while (!isAtEnd()) {
if (peek() == '"' && peekNext() == '"' && peekAfterNext() == '"') {
advance();
advance();
advance();
addToken(PbsTokenKind.DOC_TEXT_BLOCK);
state = LexerState.DEFAULT;
return;
}
advance();
}
report(LexErrors.E_LEX_UNTERMINATED_DOC_TEXT_BLOCK, "Unterminated documentation text block");
state = LexerState.DEFAULT;
}
private void scanLineCommentState() {
while (!isAtEnd() && peek() != '\n') {
advance();
@@ -250,6 +277,17 @@ public final class PbsLexer {
return source.codePointAt(nextIndex);
}
private int peekAfterNext() {
if (isAtEnd()) return '\0';
final int first = source.codePointAt(current);
final int secondIndex = current + Character.charCount(first);
if (secondIndex >= source.length()) return '\0';
final int second = source.codePointAt(secondIndex);
final int thirdIndex = secondIndex + Character.charCount(second);
if (thirdIndex >= source.length()) return '\0';
return source.codePointAt(thirdIndex);
}
private boolean isAtEnd() {
return current >= source.length();
}
@@ -17,6 +17,7 @@ public enum PbsTokenKind {
FLOAT_LITERAL,
BOUNDED_LITERAL,
STRING_LITERAL,
DOC_TEXT_BLOCK,
COMMENT,
// Import keywords.
@@ -113,6 +113,55 @@ class PbsLexerTest {
diagnostics.stream().findFirst().orElseThrow().getCode());
}
@Test
void shouldLexDocumentationTextBlock() {
final var source = """
[Doc(markdown = \"""
Draws a sprite.
- `x`: screen x
- `y`: screen y
\""")]
""";
final var diagnostics = DiagnosticSink.empty();
final var tokens = PbsLexer.lex(source, new FileId(0), diagnostics);
final var docTextBlock = tokens.stream()
.filter(t -> t.kind() == PbsTokenKind.DOC_TEXT_BLOCK)
.findFirst()
.orElseThrow();
assertTrue(docTextBlock.lexeme().startsWith("\"\"\""));
assertTrue(docTextBlock.lexeme().endsWith("\"\"\""));
assertTrue(docTextBlock.lexeme().contains("Draws a sprite."));
assertTrue(docTextBlock.lexeme().contains("- `x`: screen x"));
assertTrue(diagnostics.isEmpty(), "Documentation text block should lex without diagnostics");
}
@Test
void shouldAllowSingleAndDoubleQuotesInsideDocumentationTextBlock() {
final var source = "[Doc(markdown = \"\"\"Uses \"quoted\" text and 'single' quotes.\"\"\")]";
final var diagnostics = DiagnosticSink.empty();
final var tokens = PbsLexer.lex(source, new FileId(0), diagnostics);
final var kinds = tokens.stream().map(PbsToken::kind).toList();
assertTrue(kinds.contains(PbsTokenKind.DOC_TEXT_BLOCK));
assertTrue(diagnostics.isEmpty(), "Quotes inside documentation text block should not need escaping");
}
@Test
void shouldReportUnterminatedDocumentationTextBlock() {
final var source = "[Doc(markdown = \"\"\"unterminated)]";
final var diagnostics = DiagnosticSink.empty();
PbsLexer.lex(source, new FileId(0), diagnostics);
assertTrue(diagnostics.hasErrors(), "Lexer should report unterminated documentation text blocks");
assertEquals(LexErrors.E_LEX_UNTERMINATED_DOC_TEXT_BLOCK.name(),
diagnostics.stream().findFirst().orElseThrow().getCode());
}
@Test
void shouldReportInvalidStringEscape() {
final var source = "\"bad\\q\"";