implements PR012

This commit is contained in:
2026-03-24 13:42:21 +00:00
parent c83a31c3b6
commit 7dfaf6b49b
4 changed files with 85 additions and 67 deletions
@@ -4,6 +4,7 @@ import org.junit.jupiter.api.Test;
import p.studio.compiler.source.diagnostics.DiagnosticSink;
import p.studio.compiler.source.identifiers.FileId;
import java.nio.charset.StandardCharsets;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -134,4 +135,22 @@ class PbsLexerTest {
assertEquals(LexErrors.E_LEX_INVALID_CHAR.name(),
diagnostics.stream().findFirst().orElseThrow().getCode());
}
@Test
void shouldEmitUtf8ByteOffsetsForNonAsciiIdentifier() {
final var source = "fn cafe\u00E9() -> int { return 1; }";
final var diagnostics = DiagnosticSink.empty();
final var tokens = PbsLexer.lex(source, new FileId(0), diagnostics);
assertTrue(diagnostics.isEmpty(), "Valid non-ASCII identifier should lex without diagnostics");
final var identifier = tokens.get(1);
final int expectedStart = "fn ".getBytes(StandardCharsets.UTF_8).length;
final int expectedEnd = "fn cafe\u00E9".getBytes(StandardCharsets.UTF_8).length;
assertEquals(PbsTokenKind.IDENTIFIER, identifier.kind());
assertEquals(expectedStart, identifier.start());
assertEquals(expectedEnd, identifier.end());
}
}
@@ -6,6 +6,8 @@ import p.studio.compiler.pbs.lexer.PbsLexer;
import p.studio.compiler.source.diagnostics.DiagnosticSink;
import p.studio.compiler.source.identifiers.FileId;
import java.nio.charset.StandardCharsets;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -262,4 +264,20 @@ class PbsParserTest {
assertInstanceOf(PbsAst.InvalidDecl.class, ast.topDecls().get(0));
assertInstanceOf(PbsAst.FunctionDecl.class, ast.topDecls().get(1));
}
@Test
void shouldPropagateUtf8ByteOffsetsToAstSpans() {
final var source = "fn cafe\u00E9() -> int { return 1; }";
final var diagnostics = DiagnosticSink.empty();
final var fileId = new FileId(0);
final PbsAst.File ast = PbsParser.parse(PbsLexer.lex(source, fileId, diagnostics), fileId, diagnostics);
assertTrue(diagnostics.isEmpty(), "Valid non-ASCII source should parse without diagnostics");
final int expectedEnd = source.getBytes(StandardCharsets.UTF_8).length;
final var fn = ast.functions().getFirst();
assertEquals(expectedEnd, ast.span().getEnd());
assertEquals(expectedEnd, fn.span().getEnd());
}
}