implements PR012
This commit is contained in:
+19
@@ -4,6 +4,7 @@ import org.junit.jupiter.api.Test;
|
||||
import p.studio.compiler.source.diagnostics.DiagnosticSink;
|
||||
import p.studio.compiler.source.identifiers.FileId;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
@@ -134,4 +135,22 @@ class PbsLexerTest {
|
||||
assertEquals(LexErrors.E_LEX_INVALID_CHAR.name(),
|
||||
diagnostics.stream().findFirst().orElseThrow().getCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldEmitUtf8ByteOffsetsForNonAsciiIdentifier() {
|
||||
final var source = "fn cafe\u00E9() -> int { return 1; }";
|
||||
final var diagnostics = DiagnosticSink.empty();
|
||||
|
||||
final var tokens = PbsLexer.lex(source, new FileId(0), diagnostics);
|
||||
|
||||
assertTrue(diagnostics.isEmpty(), "Valid non-ASCII identifier should lex without diagnostics");
|
||||
|
||||
final var identifier = tokens.get(1);
|
||||
final int expectedStart = "fn ".getBytes(StandardCharsets.UTF_8).length;
|
||||
final int expectedEnd = "fn cafe\u00E9".getBytes(StandardCharsets.UTF_8).length;
|
||||
|
||||
assertEquals(PbsTokenKind.IDENTIFIER, identifier.kind());
|
||||
assertEquals(expectedStart, identifier.start());
|
||||
assertEquals(expectedEnd, identifier.end());
|
||||
}
|
||||
}
|
||||
|
||||
+18
@@ -6,6 +6,8 @@ import p.studio.compiler.pbs.lexer.PbsLexer;
|
||||
import p.studio.compiler.source.diagnostics.DiagnosticSink;
|
||||
import p.studio.compiler.source.identifiers.FileId;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
@@ -262,4 +264,20 @@ class PbsParserTest {
|
||||
assertInstanceOf(PbsAst.InvalidDecl.class, ast.topDecls().get(0));
|
||||
assertInstanceOf(PbsAst.FunctionDecl.class, ast.topDecls().get(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldPropagateUtf8ByteOffsetsToAstSpans() {
|
||||
final var source = "fn cafe\u00E9() -> int { return 1; }";
|
||||
final var diagnostics = DiagnosticSink.empty();
|
||||
final var fileId = new FileId(0);
|
||||
|
||||
final PbsAst.File ast = PbsParser.parse(PbsLexer.lex(source, fileId, diagnostics), fileId, diagnostics);
|
||||
|
||||
assertTrue(diagnostics.isEmpty(), "Valid non-ASCII source should parse without diagnostics");
|
||||
|
||||
final int expectedEnd = source.getBytes(StandardCharsets.UTF_8).length;
|
||||
final var fn = ast.functions().getFirst();
|
||||
assertEquals(expectedEnd, ast.span().getEnd());
|
||||
assertEquals(expectedEnd, fn.span().getEnd());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user