implements PR012

This commit is contained in:
2026-03-24 13:42:21 +00:00
parent c83a31c3b6
commit 7dfaf6b49b
4 changed files with 85 additions and 67 deletions
@@ -27,6 +27,8 @@ public final class PbsLexer {
private int start;
private int current;
private int startByte;
private int currentByte;
private LexerState state = LexerState.DEFAULT;
private PbsLexer(final String source, final FileId fileId, final DiagnosticSink diagnostics) {
@@ -53,7 +55,7 @@ public final class PbsLexer {
case LINE_COMMENT -> scanLineCommentState();
}
}
tokens.add(new PbsToken(PbsTokenKind.EOF, "", current, current));
tokens.add(new PbsToken(PbsTokenKind.EOF, "", currentByte, currentByte));
return ReadOnlyList.wrap(tokens);
}
@@ -62,7 +64,8 @@ public final class PbsLexer {
return;
}
start = current;
final char c = advance();
startByte = currentByte;
final int c = advance();
switch (c) {
case ' ', '\r', '\t', '\n' -> {
// Deliberately ignored.
@@ -130,7 +133,7 @@ public final class PbsLexer {
state = LexerState.IDENTIFIER;
return;
}
report(LexErrors.E_LEX_INVALID_CHAR, "Invalid character: '%s'".formatted(c));
report(LexErrors.E_LEX_INVALID_CHAR, "Invalid character: '%s'".formatted(Character.toString(c)));
}
}
}
@@ -174,16 +177,16 @@ public final class PbsLexer {
private void scanStringState() {
while (!isAtEnd() && peek() != '"') {
if (peek() == '\\') {
final var escapeStart = current;
final var escapeStart = currentByte;
advance();
if (!isAtEnd()) {
final var escaped = advance();
if (!isValidStringEscape(escaped)) {
report(
LexErrors.E_LEX_INVALID_STRING_ESCAPE,
"Invalid string escape '\\%s'".formatted(escaped),
"Invalid string escape '\\%s'".formatted(Character.toString(escaped)),
escapeStart,
current);
currentByte);
}
}
continue;
@@ -213,47 +216,57 @@ public final class PbsLexer {
private void addToken(final PbsTokenKind kind) {
final String lexeme = source.substring(start, current);
tokens.add(new PbsToken(kind, lexeme, start, current));
tokens.add(new PbsToken(kind, lexeme, startByte, currentByte));
}
private boolean match(final char expected) {
if (isAtEnd()) return false;
if (source.charAt(current) != expected) return false;
final var codePoint = source.codePointAt(current);
if (Character.charCount(codePoint) != 1 || codePoint != expected) {
return false;
}
current++;
currentByte++;
return true;
}
private char advance() {
return source.charAt(current++);
private int advance() {
final int codePoint = source.codePointAt(current);
current += Character.charCount(codePoint);
currentByte += utf8Bytes(codePoint);
return codePoint;
}
private char peek() {
private int peek() {
if (isAtEnd()) return '\0';
return source.charAt(current);
return source.codePointAt(current);
}
private char peekNext() {
if (current + 1 >= source.length()) return '\0';
return source.charAt(current + 1);
private int peekNext() {
if (isAtEnd()) return '\0';
final int first = source.codePointAt(current);
final int nextIndex = current + Character.charCount(first);
if (nextIndex >= source.length()) return '\0';
return source.codePointAt(nextIndex);
}
private boolean isAtEnd() {
return current >= source.length();
}
private boolean isDigit(final char c) {
private boolean isDigit(final int c) {
return c >= '0' && c <= '9';
}
private boolean isIdentifierStart(final char c) {
private boolean isIdentifierStart(final int c) {
return c == '_' || Character.isAlphabetic(c);
}
private boolean isIdentifierPart(final char c) {
private boolean isIdentifierPart(final int c) {
return c == '_' || Character.isAlphabetic(c) || Character.isDigit(c);
}
private boolean isValidStringEscape(final char escaped) {
private boolean isValidStringEscape(final int escaped) {
return escaped == '\\'
|| escaped == '"'
|| escaped == 'n'
@@ -262,17 +275,30 @@ public final class PbsLexer {
}
private void report(final LexErrors lexErrors, final String message) {
report(lexErrors, message, start, current);
report(lexErrors, message, startByte, currentByte);
}
private void report(
final LexErrors lexErrors,
final String message,
final long spanStart,
final long spanEnd) {
final int spanStart,
final int spanEnd) {
p.studio.compiler.source.diagnostics.Diagnostics.error(diagnostics, lexErrors.name(), message, new Span(fileId, spanStart, spanEnd));
}
private int utf8Bytes(final int codePoint) {
if (codePoint <= 0x7F) {
return 1;
}
if (codePoint <= 0x7FF) {
return 2;
}
if (codePoint <= 0xFFFF) {
return 3;
}
return 4;
}
private static Map<String, PbsTokenKind> buildKeywords() {
final var map = new HashMap<String, PbsTokenKind>();
map.put("import", PbsTokenKind.IMPORT);