code improvements, add pipeline stages

This commit is contained in:
2026-03-24 13:42:15 +00:00
parent 47a077bffc
commit 31c59f670e
16 changed files with 289 additions and 179 deletions
@@ -13,7 +13,7 @@ public class BuilderPipelineContext {
public Path rootProjectPathCanon;
public ResolvedWorkspace resolvedWorkspace;
public final FileTable fileTable = new FileTable();
public FileTable fileTable;
private BuilderPipelineContext(
final BuilderPipelineConfig config,
@@ -4,8 +4,7 @@ import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.exceptions.BuildException;
import p.studio.compiler.messages.BuilderPipelineConfig;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.workspaces.stages.LoadPipelineStage;
import p.studio.compiler.workspaces.stages.ResolvePipelineStage;
import p.studio.compiler.workspaces.stages.*;
import p.studio.utilities.logs.LogAggregator;
import p.studio.utilities.structures.ReadOnlyCollection;
@@ -17,8 +16,11 @@ public class BuilderPipelineService {
static {
final var stages = List.<PipelineStage>of(
new ResolvePipelineStage(),
new LoadPipelineStage()
new ResolveDepsPipelineStage(),
new LoadSourcesPipelineStage(),
new FrontendPhasePipelineStage(),
new LowerToVMPipelineStage(),
new EmitBytecodePipelineStage()
);
INSTANCE = new BuilderPipelineService(stages);
}
@@ -0,0 +1,15 @@
package p.studio.compiler.workspaces.stages;
import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
@Slf4j
public class EmitBytecodePipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(BuilderPipelineContext ctx, LogAggregator logs) {
return BuildingIssueSink.empty();
}
}
@@ -0,0 +1,15 @@
package p.studio.compiler.workspaces.stages;
import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
@Slf4j
public class FrontendPhasePipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(BuilderPipelineContext ctx, LogAggregator logs) {
return BuildingIssueSink.empty();
}
}
@@ -1,145 +0,0 @@
package p.studio.compiler.workspaces.stages;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.models.SourceHandle;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
import p.studio.utilities.structures.ReadOnlySet;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
public class LoadPipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(final BuilderPipelineContext ctx, final LogAggregator logs) {
final var issues = BuildingIssueSink.empty();
// Iterates projects; "loads sources"; registers files
ctx.resolvedWorkspace.topologicalOrder().forEach(pId -> {
final var pd = ctx.resolvedWorkspace.graph().projectTable().get(pId);
logs.using(log).info("Project [ " + pd.getName() + " ] source loading...");
final var allowedExtensions = normalize(ctx.resolvedWorkspace.frontendSpec().getAllowedExtensions());
for (final var sourceRootPath : pd.getSourceRoots()) {
logs.using(log).debug("Walking source root [ " + sourceRootPath + " ]");
try {
final List<Path> paths = new ArrayList<>();
Files.walkFileTree(sourceRootPath, new SourceCrawler(allowedExtensions, paths));
paths.sort(Path::compareTo);
for (var path : paths) {
logs.using(log).debug("file tabling [ " + path + " ]");
final var attributes = Files.readAttributes(path, BasicFileAttributes.class);
final var size = attributes.size();
final var lastModified = attributes.lastModifiedTime().toMillis();
final var rawFile = new SourceHandle(pId, path, size, lastModified, ctx.sourceProviderFactory);
// register in dense tables
var fileId = ctx.fileTable.register(rawFile);
}
} catch (IOException e) {
issues.report(builder -> builder
.error(true)
.message("Failed to load project [ " + pd.getName() + " ]")
.exception(e));
}
}
});
return BuildingIssueSink.empty();
}
private static ReadOnlySet<String> normalize(final ReadOnlySet<String> extensions) {
return ReadOnlySet
.wrap(extensions
.map(String::toLowerCase)
.map(s -> s.startsWith(".") ? s.substring(1) : s)
.collect(Collectors.toSet()));
}
private static final class SourceCrawler extends SimpleFileVisitor<Path> {
private static final Set<String> IGNORED_DIRS = Set.of(
".git",
".prometeu",
".workspace",
"target",
"build",
"out",
"node_modules"
);
private final ReadOnlySet<String> allowedExtensions;
private final List<Path> paths;
public SourceCrawler(
final ReadOnlySet<String> allowedExtensions,
final List<Path> paths) {
this.allowedExtensions = allowedExtensions;
this.paths = paths;
}
@Override
public FileVisitResult preVisitDirectory(final Path path, final BasicFileAttributes attrs) {
for (final var directory : path) {
if (!isAllowedPath(directory)) {
return FileVisitResult.SKIP_SUBTREE;
}
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs) {
if (!attrs.isRegularFile()) {
return FileVisitResult.CONTINUE;
}
if (hasAllowedExt(path, allowedExtensions)) {
paths.add(path);
}
return FileVisitResult.CONTINUE;
}
private static boolean isAllowedPath(
final Path path) {
for (Path part : path) {
if (IGNORED_DIRS.contains(part.toString())) {
return false;
}
}
return true;
}
private static boolean hasAllowedExt(
final Path path,
final ReadOnlySet<String> allowedExtensions) {
final var fileName = path.getFileName();
if (Objects.isNull(fileName)) {
return false;
}
final var extension = FilenameUtils.getExtension(fileName.toString()).toLowerCase();
if (StringUtils.isBlank(extension)) {
return false;
}
return allowedExtensions.contains(extension);
}
}
}
@@ -0,0 +1,97 @@
package p.studio.compiler.workspaces.stages;
import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.models.ProjectDescriptor;
import p.studio.compiler.models.SourceHandle;
import p.studio.compiler.source.identifiers.ProjectId;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
import p.studio.utilities.structures.ReadOnlySet;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
public class LoadSourcesPipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(final BuilderPipelineContext ctx, final LogAggregator logs) {
final var issues = BuildingIssueSink.empty();
ctx.resolvedWorkspace.stack().topologicalOrder.forEach(pId -> visitProjectSources(pId, ctx, logs, issues));
return issues;
}
private static void visitProjectSources(
final ProjectId pId,
final BuilderPipelineContext ctx,
final LogAggregator logs,
final BuildingIssueSink issues) {
final var pd = ctx.resolvedWorkspace.graph().projectTable().get(pId);
logs.using(log).debug("Project [ " + pd.getName() + " ] source loading...");
final var allowedExtensions = normalize(ctx.resolvedWorkspace.frontendSpec().getAllowedExtensions());
// Iterates source roots; crawls files; registers them
for (final var sourceRootPath : pd.getSourceRoots()) {
logs.using(log).debug("Walking source root [ " + sourceRootPath + " ]");
final List<Path> paths = new ArrayList<>();
final var sourceCrawler = new SourceCrawler(allowedExtensions, paths);
try {
Files.walkFileTree(sourceRootPath, sourceCrawler);
paths.sort(Path::compareTo); // do we really need this for deterministic builds?
} catch (IOException e) {
issues.report(builder -> builder
.error(true)
.message("Failed to load project [ " + pd.getName() + " ]")
.exception(e));
}
registerFiles(pId, pd, paths, ctx, logs, issues);
}
}
private static void registerFiles(
final ProjectId pId,
final ProjectDescriptor pd,
final List<Path> canonPaths,
final BuilderPipelineContext ctx,
final LogAggregator logs,
final BuildingIssueSink issues) {
for (var canonPath : canonPaths) {
final long size;
final long lastModified;
try {
final BasicFileAttributes attributes = Files.readAttributes(canonPath, BasicFileAttributes.class);
size = attributes.size();
lastModified = attributes.lastModifiedTime().toMillis();
} catch (IOException e) {
issues.report(builder -> builder
.error(true)
.message("Failed to read attributes for file: " + canonPath)
.exception(e));
continue;
}
// register in dense tables
final var relativePath = pd.getRootPath().relativize(canonPath);
final var sourceHandle = new SourceHandle(pId, relativePath, canonPath, size, lastModified, ctx.sourceProviderFactory);
logs.using(log).debug("Registering: " + sourceHandle);
ctx.fileTable.register(sourceHandle);
}
}
private static ReadOnlySet<String> normalize(final ReadOnlySet<String> extensions) {
return ReadOnlySet
.wrap(extensions
.map(String::toLowerCase)
.map(s -> s.startsWith(".") ? s.substring(1) : s)
.collect(Collectors.toSet()));
}
}
@@ -0,0 +1,15 @@
package p.studio.compiler.workspaces.stages;
import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
@Slf4j
public class LowerToVMPipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(BuilderPipelineContext ctx, LogAggregator logs) {
return BuildingIssueSink.empty();
}
}
@@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.messages.DependencyConfig;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.source.tables.FileTable;
import p.studio.compiler.workspaces.DependencyService;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
@@ -12,7 +13,7 @@ import java.io.IOException;
import java.nio.file.Paths;
@Slf4j
public class ResolvePipelineStage implements PipelineStage {
public class ResolveDepsPipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(final BuilderPipelineContext ctx, LogAggregator logs) {
try {
@@ -26,6 +27,7 @@ public class ResolvePipelineStage implements PipelineStage {
}
final var dependencyConfig = new DependencyConfig(ctx.config.explain(), ctx.rootProjectPathCanon);
ctx.resolvedWorkspace = DependencyService.INSTANCE.run(dependencyConfig, logs);
ctx.fileTable = new FileTable(ctx.resolvedWorkspace.graph().projectTable().size());
return BuildingIssueSink.empty();
}
}
@@ -0,0 +1,86 @@
package p.studio.compiler.workspaces.stages;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import p.studio.utilities.structures.ReadOnlySet;
import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
import java.util.Objects;
import java.util.Set;
final class SourceCrawler extends SimpleFileVisitor<Path> {
private static final Set<String> IGNORED_DIRS = Set.of(
".git",
".prometeu",
".workspace",
"target",
"build",
"out",
"node_modules"
);
private final ReadOnlySet<String> allowedExtensions;
private final List<Path> paths;
public SourceCrawler(
final ReadOnlySet<String> allowedExtensions,
final List<Path> paths) {
this.allowedExtensions = allowedExtensions;
this.paths = paths;
}
@Override
public FileVisitResult preVisitDirectory(final Path path, final BasicFileAttributes attrs) {
for (final var directory : path) {
if (!isAllowedPath(directory)) {
return FileVisitResult.SKIP_SUBTREE;
}
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs) {
if (!attrs.isRegularFile()) {
return FileVisitResult.CONTINUE;
}
if (hasAllowedExt(path, allowedExtensions)) {
paths.add(path);
}
return FileVisitResult.CONTINUE;
}
private static boolean isAllowedPath(
final Path path) {
for (Path part : path) {
if (IGNORED_DIRS.contains(part.toString())) {
return false;
}
}
return true;
}
private static boolean hasAllowedExt(
final Path path,
final ReadOnlySet<String> allowedExtensions) {
final var fileName = path.getFileName();
if (Objects.isNull(fileName)) {
return false;
}
final var extension = FilenameUtils.getExtension(fileName.toString()).toLowerCase();
if (StringUtils.isBlank(extension)) {
return false;
}
return allowedExtensions.contains(extension);
}
}