implements PLN-0093
This commit is contained in:
+1
-1
@@ -50,7 +50,7 @@ public enum PbsSemanticKind {
|
||||
public static PbsSemanticKind forToken(final PbsToken token) {
|
||||
return switch (token.kind()) {
|
||||
case COMMENT -> COMMENT;
|
||||
case STRING_LITERAL -> STRING;
|
||||
case STRING_LITERAL, DOC_TEXT_BLOCK -> STRING;
|
||||
case INT_LITERAL, FLOAT_LITERAL, BOUNDED_LITERAL -> NUMBER;
|
||||
case TRUE, FALSE, NONE, SOME, OK, ERR -> LITERAL;
|
||||
case VOID, OPTIONAL, RESULT -> BUILTIN_TYPE;
|
||||
|
||||
+1
@@ -3,5 +3,6 @@ package p.studio.compiler.pbs.lexer;
|
||||
public enum LexErrors {
|
||||
E_LEX_INVALID_CHAR,
|
||||
E_LEX_UNTERMINATED_STRING,
|
||||
E_LEX_UNTERMINATED_DOC_TEXT_BLOCK,
|
||||
E_LEX_INVALID_STRING_ESCAPE,
|
||||
}
|
||||
|
||||
+39
-1
@@ -17,6 +17,7 @@ public final class PbsLexer {
|
||||
IDENTIFIER,
|
||||
NUMBER,
|
||||
STRING,
|
||||
DOC_TEXT_BLOCK,
|
||||
LINE_COMMENT
|
||||
}
|
||||
|
||||
@@ -52,6 +53,7 @@ public final class PbsLexer {
|
||||
case IDENTIFIER -> scanIdentifierState();
|
||||
case NUMBER -> scanNumberState();
|
||||
case STRING -> scanStringState();
|
||||
case DOC_TEXT_BLOCK -> scanDocTextBlockState();
|
||||
case LINE_COMMENT -> scanLineCommentState();
|
||||
}
|
||||
}
|
||||
@@ -123,7 +125,15 @@ public final class PbsLexer {
|
||||
}
|
||||
addToken(match('=') ? PbsTokenKind.SLASH_EQUAL : PbsTokenKind.SLASH);
|
||||
}
|
||||
case '"' -> state = LexerState.STRING;
|
||||
case '"' -> {
|
||||
if (peek() == '"' && peekNext() == '"') {
|
||||
advance();
|
||||
advance();
|
||||
state = LexerState.DOC_TEXT_BLOCK;
|
||||
return;
|
||||
}
|
||||
state = LexerState.STRING;
|
||||
}
|
||||
default -> {
|
||||
if (isDigit(c)) {
|
||||
state = LexerState.NUMBER;
|
||||
@@ -206,6 +216,23 @@ public final class PbsLexer {
|
||||
state = LexerState.DEFAULT;
|
||||
}
|
||||
|
||||
private void scanDocTextBlockState() {
|
||||
while (!isAtEnd()) {
|
||||
if (peek() == '"' && peekNext() == '"' && peekAfterNext() == '"') {
|
||||
advance();
|
||||
advance();
|
||||
advance();
|
||||
addToken(PbsTokenKind.DOC_TEXT_BLOCK);
|
||||
state = LexerState.DEFAULT;
|
||||
return;
|
||||
}
|
||||
advance();
|
||||
}
|
||||
|
||||
report(LexErrors.E_LEX_UNTERMINATED_DOC_TEXT_BLOCK, "Unterminated documentation text block");
|
||||
state = LexerState.DEFAULT;
|
||||
}
|
||||
|
||||
private void scanLineCommentState() {
|
||||
while (!isAtEnd() && peek() != '\n') {
|
||||
advance();
|
||||
@@ -250,6 +277,17 @@ public final class PbsLexer {
|
||||
return source.codePointAt(nextIndex);
|
||||
}
|
||||
|
||||
private int peekAfterNext() {
|
||||
if (isAtEnd()) return '\0';
|
||||
final int first = source.codePointAt(current);
|
||||
final int secondIndex = current + Character.charCount(first);
|
||||
if (secondIndex >= source.length()) return '\0';
|
||||
final int second = source.codePointAt(secondIndex);
|
||||
final int thirdIndex = secondIndex + Character.charCount(second);
|
||||
if (thirdIndex >= source.length()) return '\0';
|
||||
return source.codePointAt(thirdIndex);
|
||||
}
|
||||
|
||||
private boolean isAtEnd() {
|
||||
return current >= source.length();
|
||||
}
|
||||
|
||||
+1
@@ -17,6 +17,7 @@ public enum PbsTokenKind {
|
||||
FLOAT_LITERAL,
|
||||
BOUNDED_LITERAL,
|
||||
STRING_LITERAL,
|
||||
DOC_TEXT_BLOCK,
|
||||
COMMENT,
|
||||
|
||||
// Import keywords.
|
||||
|
||||
+49
@@ -113,6 +113,55 @@ class PbsLexerTest {
|
||||
diagnostics.stream().findFirst().orElseThrow().getCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldLexDocumentationTextBlock() {
|
||||
final var source = """
|
||||
[Doc(markdown = \"""
|
||||
Draws a sprite.
|
||||
|
||||
- `x`: screen x
|
||||
- `y`: screen y
|
||||
\""")]
|
||||
""";
|
||||
final var diagnostics = DiagnosticSink.empty();
|
||||
|
||||
final var tokens = PbsLexer.lex(source, new FileId(0), diagnostics);
|
||||
final var docTextBlock = tokens.stream()
|
||||
.filter(t -> t.kind() == PbsTokenKind.DOC_TEXT_BLOCK)
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
|
||||
assertTrue(docTextBlock.lexeme().startsWith("\"\"\""));
|
||||
assertTrue(docTextBlock.lexeme().endsWith("\"\"\""));
|
||||
assertTrue(docTextBlock.lexeme().contains("Draws a sprite."));
|
||||
assertTrue(docTextBlock.lexeme().contains("- `x`: screen x"));
|
||||
assertTrue(diagnostics.isEmpty(), "Documentation text block should lex without diagnostics");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldAllowSingleAndDoubleQuotesInsideDocumentationTextBlock() {
|
||||
final var source = "[Doc(markdown = \"\"\"Uses \"quoted\" text and 'single' quotes.\"\"\")]";
|
||||
final var diagnostics = DiagnosticSink.empty();
|
||||
|
||||
final var tokens = PbsLexer.lex(source, new FileId(0), diagnostics);
|
||||
final var kinds = tokens.stream().map(PbsToken::kind).toList();
|
||||
|
||||
assertTrue(kinds.contains(PbsTokenKind.DOC_TEXT_BLOCK));
|
||||
assertTrue(diagnostics.isEmpty(), "Quotes inside documentation text block should not need escaping");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReportUnterminatedDocumentationTextBlock() {
|
||||
final var source = "[Doc(markdown = \"\"\"unterminated)]";
|
||||
final var diagnostics = DiagnosticSink.empty();
|
||||
|
||||
PbsLexer.lex(source, new FileId(0), diagnostics);
|
||||
|
||||
assertTrue(diagnostics.hasErrors(), "Lexer should report unterminated documentation text blocks");
|
||||
assertEquals(LexErrors.E_LEX_UNTERMINATED_DOC_TEXT_BLOCK.name(),
|
||||
diagnostics.stream().findFirst().orElseThrow().getCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReportInvalidStringEscape() {
|
||||
final var source = "\"bad\\q\"";
|
||||
|
||||
Reference in New Issue
Block a user