Intrepid/prometeu-studio
Intrepid/prometeu-studio
3
0
Fork 0
Code Pull Requests 1 Activity

code improvements, add pipeline stages

Browse Source
This commit is contained in:
bQUARKz 2026-02-26 07:13:02 +00:00
parent 47a077bffc
commit 31c59f670e
Signed by: bquarkz
SSH Key Fingerprint: SHA256:Z7dgqoglWwoK6j6u4QC87OveEq74WOhFN+gitsxtkf8
16 changed files with 289 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/models/BuilderPipelineContext.java
View File

@ -13,7 +13,7 @@ public class BuilderPipelineContext {
public Path rootProjectPathCanon;
public ResolvedWorkspace resolvedWorkspace;
public final FileTable fileTable = new FileTable();
public FileTable fileTable;
private BuilderPipelineContext(
final BuilderPipelineConfig config,

10
prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/BuilderPipelineService.java
View File

@ -4,8 +4,7 @@ import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.exceptions.BuildException;
import p.studio.compiler.messages.BuilderPipelineConfig;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.workspaces.stages.LoadPipelineStage;
import p.studio.compiler.workspaces.stages.ResolvePipelineStage;
import p.studio.compiler.workspaces.stages.*;
import p.studio.utilities.logs.LogAggregator;
import p.studio.utilities.structures.ReadOnlyCollection;
@ -17,8 +16,11 @@ public class BuilderPipelineService {
static {
final var stages = List.<PipelineStage>of(
new ResolvePipelineStage(),
new LoadPipelineStage()
new ResolveDepsPipelineStage(),
new LoadSourcesPipelineStage(),
new FrontendPhasePipelineStage(),
new LowerToVMPipelineStage(),
new EmitBytecodePipelineStage()
);
INSTANCE = new BuilderPipelineService(stages);
}

15
prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/EmitBytecodePipelineStage.java Normal file
View File

@ -0,0 +1,15 @@
package p.studio.compiler.workspaces.stages;
import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
@Slf4j
public class EmitBytecodePipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(BuilderPipelineContext ctx, LogAggregator logs) {
return BuildingIssueSink.empty();
}
}

15
prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/FrontendPhasePipelineStage.java Normal file
View File

@ -0,0 +1,15 @@
package p.studio.compiler.workspaces.stages;
import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
@Slf4j
public class FrontendPhasePipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(BuilderPipelineContext ctx, LogAggregator logs) {
return BuildingIssueSink.empty();
}
}

145
prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/LoadPipelineStage.java
View File

@ -1,145 +0,0 @@
package p.studio.compiler.workspaces.stages;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.models.SourceHandle;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
import p.studio.utilities.structures.ReadOnlySet;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
public class LoadPipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(final BuilderPipelineContext ctx, final LogAggregator logs) {
final var issues = BuildingIssueSink.empty();
// Iterates projects; "loads sources"; registers files
ctx.resolvedWorkspace.topologicalOrder().forEach(pId -> {
final var pd = ctx.resolvedWorkspace.graph().projectTable().get(pId);
logs.using(log).info("Project [ " + pd.getName() + " ] source loading...");
final var allowedExtensions = normalize(ctx.resolvedWorkspace.frontendSpec().getAllowedExtensions());
for (final var sourceRootPath : pd.getSourceRoots()) {
logs.using(log).debug("Walking source root [ " + sourceRootPath + " ]");
try {
final List<Path> paths = new ArrayList<>();
Files.walkFileTree(sourceRootPath, new SourceCrawler(allowedExtensions, paths));
paths.sort(Path::compareTo);
for (var path : paths) {
logs.using(log).debug("file tabling [ " + path + " ]");
final var attributes = Files.readAttributes(path, BasicFileAttributes.class);
final var size = attributes.size();
final var lastModified = attributes.lastModifiedTime().toMillis();
final var rawFile = new SourceHandle(pId, path, size, lastModified, ctx.sourceProviderFactory);
// register in dense tables
var fileId = ctx.fileTable.register(rawFile);
}
} catch (IOException e) {
issues.report(builder -> builder
.error(true)
.message("Failed to load project [ " + pd.getName() + " ]")
.exception(e));
}
}
});
return BuildingIssueSink.empty();
}
private static ReadOnlySet<String> normalize(final ReadOnlySet<String> extensions) {
return ReadOnlySet
.wrap(extensions
.map(String::toLowerCase)
.map(s -> s.startsWith(".") ? s.substring(1) : s)
.collect(Collectors.toSet()));
}
private static final class SourceCrawler extends SimpleFileVisitor<Path> {
private static final Set<String> IGNORED_DIRS = Set.of(
".git",
".prometeu",
".workspace",
"target",
"build",
"out",
"node_modules"
);
private final ReadOnlySet<String> allowedExtensions;
private final List<Path> paths;
public SourceCrawler(
final ReadOnlySet<String> allowedExtensions,
final List<Path> paths) {
this.allowedExtensions = allowedExtensions;
this.paths = paths;
}
@Override
public FileVisitResult preVisitDirectory(final Path path, final BasicFileAttributes attrs) {
for (final var directory : path) {
if (!isAllowedPath(directory)) {
return FileVisitResult.SKIP_SUBTREE;
}
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs) {
if (!attrs.isRegularFile()) {
return FileVisitResult.CONTINUE;
}
if (hasAllowedExt(path, allowedExtensions)) {
paths.add(path);
}
return FileVisitResult.CONTINUE;
}
private static boolean isAllowedPath(
final Path path) {
for (Path part : path) {
if (IGNORED_DIRS.contains(part.toString())) {
return false;
}
}
return true;
}
private static boolean hasAllowedExt(
final Path path,
final ReadOnlySet<String> allowedExtensions) {
final var fileName = path.getFileName();
if (Objects.isNull(fileName)) {
return false;
}
final var extension = FilenameUtils.getExtension(fileName.toString()).toLowerCase();
if (StringUtils.isBlank(extension)) {
return false;
}
return allowedExtensions.contains(extension);
}
}
}

97
prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/LoadSourcesPipelineStage.java Normal file
View File

@ -0,0 +1,97 @@
package p.studio.compiler.workspaces.stages;
import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.models.ProjectDescriptor;
import p.studio.compiler.models.SourceHandle;
import p.studio.compiler.source.identifiers.ProjectId;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
import p.studio.utilities.structures.ReadOnlySet;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
public class LoadSourcesPipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(final BuilderPipelineContext ctx, final LogAggregator logs) {
final var issues = BuildingIssueSink.empty();
ctx.resolvedWorkspace.stack().topologicalOrder.forEach(pId -> visitProjectSources(pId, ctx, logs, issues));
return issues;
}
private static void visitProjectSources(
final ProjectId pId,
final BuilderPipelineContext ctx,
final LogAggregator logs,
final BuildingIssueSink issues) {
final var pd = ctx.resolvedWorkspace.graph().projectTable().get(pId);
logs.using(log).debug("Project [ " + pd.getName() + " ] source loading...");
final var allowedExtensions = normalize(ctx.resolvedWorkspace.frontendSpec().getAllowedExtensions());
// Iterates source roots; crawls files; registers them
for (final var sourceRootPath : pd.getSourceRoots()) {
logs.using(log).debug("Walking source root [ " + sourceRootPath + " ]");
final List<Path> paths = new ArrayList<>();
final var sourceCrawler = new SourceCrawler(allowedExtensions, paths);
try {
Files.walkFileTree(sourceRootPath, sourceCrawler);
paths.sort(Path::compareTo); // do we really need this for deterministic builds?
} catch (IOException e) {
issues.report(builder -> builder
.error(true)
.message("Failed to load project [ " + pd.getName() + " ]")
.exception(e));
}
registerFiles(pId, pd, paths, ctx, logs, issues);
}
}
private static void registerFiles(
final ProjectId pId,
final ProjectDescriptor pd,
final List<Path> canonPaths,
final BuilderPipelineContext ctx,
final LogAggregator logs,
final BuildingIssueSink issues) {
for (var canonPath : canonPaths) {
final long size;
final long lastModified;
try {
final BasicFileAttributes attributes = Files.readAttributes(canonPath, BasicFileAttributes.class);
size = attributes.size();
lastModified = attributes.lastModifiedTime().toMillis();
} catch (IOException e) {
issues.report(builder -> builder
.error(true)
.message("Failed to read attributes for file: " + canonPath)
.exception(e));
continue;
}
// register in dense tables
final var relativePath = pd.getRootPath().relativize(canonPath);
final var sourceHandle = new SourceHandle(pId, relativePath, canonPath, size, lastModified, ctx.sourceProviderFactory);
logs.using(log).debug("Registering: " + sourceHandle);
ctx.fileTable.register(sourceHandle);
}
}
private static ReadOnlySet<String> normalize(final ReadOnlySet<String> extensions) {
return ReadOnlySet
.wrap(extensions
.map(String::toLowerCase)
.map(s -> s.startsWith(".") ? s.substring(1) : s)
.collect(Collectors.toSet()));
}
}

15
prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/LowerToVMPipelineStage.java Normal file
View File

@ -0,0 +1,15 @@
package p.studio.compiler.workspaces.stages;
import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
@Slf4j
public class LowerToVMPipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(BuilderPipelineContext ctx, LogAggregator logs) {
return BuildingIssueSink.empty();
}
}

4
prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/ResolvePipelineStage.java → prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/ResolveDepsPipelineStage.java
View File

@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j;
import p.studio.compiler.messages.BuildingIssueSink;
import p.studio.compiler.messages.DependencyConfig;
import p.studio.compiler.models.BuilderPipelineContext;
import p.studio.compiler.source.tables.FileTable;
import p.studio.compiler.workspaces.DependencyService;
import p.studio.compiler.workspaces.PipelineStage;
import p.studio.utilities.logs.LogAggregator;
@ -12,7 +13,7 @@ import java.io.IOException;
import java.nio.file.Paths;
@Slf4j
public class ResolvePipelineStage implements PipelineStage {
public class ResolveDepsPipelineStage implements PipelineStage {
@Override
public BuildingIssueSink run(final BuilderPipelineContext ctx, LogAggregator logs) {
try {
@ -26,6 +27,7 @@ public class ResolvePipelineStage implements PipelineStage {
}
final var dependencyConfig = new DependencyConfig(ctx.config.explain(), ctx.rootProjectPathCanon);
ctx.resolvedWorkspace = DependencyService.INSTANCE.run(dependencyConfig, logs);
ctx.fileTable = new FileTable(ctx.resolvedWorkspace.graph().projectTable().size());
return BuildingIssueSink.empty();
}
}

86
prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/SourceCrawler.java Normal file
View File

@ -0,0 +1,86 @@
package p.studio.compiler.workspaces.stages;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import p.studio.utilities.structures.ReadOnlySet;
import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
import java.util.Objects;
import java.util.Set;
final class SourceCrawler extends SimpleFileVisitor<Path> {
private static final Set<String> IGNORED_DIRS = Set.of(
".git",
".prometeu",
".workspace",
"target",
"build",
"out",
"node_modules"
);
private final ReadOnlySet<String> allowedExtensions;
private final List<Path> paths;
public SourceCrawler(
final ReadOnlySet<String> allowedExtensions,
final List<Path> paths) {
this.allowedExtensions = allowedExtensions;
this.paths = paths;
}
@Override
public FileVisitResult preVisitDirectory(final Path path, final BasicFileAttributes attrs) {
for (final var directory : path) {
if (!isAllowedPath(directory)) {
return FileVisitResult.SKIP_SUBTREE;
}
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs) {
if (!attrs.isRegularFile()) {
return FileVisitResult.CONTINUE;
}
if (hasAllowedExt(path, allowedExtensions)) {
paths.add(path);
}
return FileVisitResult.CONTINUE;
}
private static boolean isAllowedPath(
final Path path) {
for (Path part : path) {
if (IGNORED_DIRS.contains(part.toString())) {
return false;
}
}
return true;
}
private static boolean hasAllowedExt(
final Path path,
final ReadOnlySet<String> allowedExtensions) {
final var fileName = path.getFileName();
if (Objects.isNull(fileName)) {
return false;
}
final var extension = FilenameUtils.getExtension(fileName.toString()).toLowerCase();
if (StringUtils.isBlank(extension)) {
return false;
}
return allowedExtensions.contains(extension);
}
}

25
prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/SourceHandle.java
View File

@ -9,14 +9,19 @@ import p.studio.compiler.utilities.SourceProviderFactory;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class SourceHandle {
@Getter
private final ProjectId projectId;
@Getter
private final Path relativePath;
@EqualsAndHashCode.Include
@Getter
private final Path path; // canon path to file
private final Path canonPath;
@Getter
private final String filename;
@Getter
@ -27,16 +32,18 @@ public class SourceHandle {
public SourceHandle(
final ProjectId projectId,
final Path path,
final Path relativePath,
final Path canonPath,
final long size,
final long lastModified,
final SourceProviderFactory factory) {
this.projectId = projectId;
this.path = path;
this.filename = path.getFileName().toString();
this.relativePath = relativePath;
this.canonPath = canonPath;
this.filename = canonPath.getFileName().toString();
this.size = size;
this.lastModified = lastModified;
this.provider = factory.create(path);
this.provider = factory.create(canonPath);
}
public byte[] readBytes() throws IOException {
@ -46,4 +53,12 @@ public class SourceHandle {
public String readUtf8() throws IOException {
return new String(readBytes(), StandardCharsets.UTF_8);
}
@Override
public String toString() {
return "SourceHandle{ %s, %s, %sKb, mod: %s }".formatted(projectId,
relativePath,
size / 1024,
LocalDateTime.ofInstant(Instant.ofEpochMilli(lastModified), ZoneId.systemDefault()));
}
}

22
prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/source/tables/FileTable.java
View File

@ -6,29 +6,37 @@ import p.studio.compiler.source.identifiers.FileId;
import p.studio.compiler.source.identifiers.ProjectId;
import p.studio.utilities.structures.ReadOnlyList;
import java.util.*;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
public class FileTable extends InternTable<FileId, SourceHandle> implements FileTableReader {
private final Map<ProjectId, Set<FileId>> projectFiles = new HashMap<>();
private final FileIds[] projectFiles;
public FileTable() {
public FileTable(int projectCapacity) {
super(FileId::new);
projectFiles = new FileIds[projectCapacity];
Arrays.setAll(projectFiles, ignored -> new FileIds());
}
@Override
public FileId register(final SourceHandle value) {
final var fileId = super.register(value);
projectFiles.computeIfAbsent(value.getProjectId(), ignored -> new HashSet<>()).add(fileId);
projectFiles[value.getProjectId().getIndex()].set.add(fileId);
return fileId;
}
@Override
public ReadOnlyList<FileId> getFiles(final ProjectId projectId) {
final var fileIds = projectFiles.get(projectId);
if (CollectionUtils.isEmpty(fileIds)) {
final var fileIds = projectFiles[projectId.getIndex()];
if (CollectionUtils.isEmpty(fileIds.set)) {
return ReadOnlyList.empty();
}
return ReadOnlyList.wrap(fileIds);
return ReadOnlyList.wrap(fileIds.set);
}
private static class FileIds {
public final Set<FileId> set = new HashSet<>();
}
}

2
prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/models/BuildStack.java
View File

@ -7,7 +7,7 @@ public class BuildStack {
public final ReadOnlyList<ProjectId> topologicalOrder;
public final ReadOnlyList<ProjectId> reverseTopologicalOrder;
public BuildStack(ReadOnlyList<ProjectId> topologicalOrder) {
public BuildStack(final ReadOnlyList<ProjectId> topologicalOrder) {
this.topologicalOrder = topologicalOrder;
this.reverseTopologicalOrder = topologicalOrder.invert();
}

15
prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/models/ResolvedWorkspace.java
View File

@ -2,23 +2,12 @@ package p.studio.compiler.models;
import p.studio.compiler.source.identifiers.ProjectId;
import java.util.stream.Stream;
public record ResolvedWorkspace(
ProjectId projectId,
ProjectId mainProjectId,
FrontendSpec frontendSpec,
WorkspaceGraph graph,
BuildStack stack) {
public ProjectDescriptor mainProject() {
return graph.projectDescriptor(projectId);
}
public Stream<ProjectId> topologicalOrder() {
return stack.topologicalOrder.stream();
}
public Stream<ProjectId> reverseTopologicalOrder() {
return stack.reverseTopologicalOrder.stream();
return graph.projectDescriptor(mainProjectId);
}
}

4
prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/models/WorkspaceGraph.java
View File

@ -1,11 +1,11 @@
package p.studio.compiler.models;
import p.studio.compiler.source.identifiers.ProjectId;
import p.studio.compiler.source.tables.ProjectTable;
import p.studio.compiler.source.tables.ProjectTableReader;
import p.studio.utilities.structures.ReadOnlyList;
public record WorkspaceGraph(
ProjectTable projectTable,
ProjectTableReader projectTable,
ReadOnlyList<ReadOnlyList<ProjectId>> dependenciesByProjectId) {
public ProjectDescriptor projectDescriptor(final ProjectId projectId) {

2
prometeu-compiler/prometeu-frontend-api/build.gradle.kts
View File

@ -4,4 +4,6 @@ plugins {
dependencies {
api(project(":prometeu-infra"))
implementation(project(":prometeu-compiler:prometeu-compiler-core"))
}

9
prometeu-compiler/prometeu-frontend-api/src/main/java/p/studio/compiler/messages/FrontendPhaseRequest.java Normal file
View File

@ -0,0 +1,9 @@
package p.studio.compiler.messages;
import p.studio.compiler.source.tables.FileTableReader;
import p.studio.compiler.source.tables.ProjectTableReader;
public record FrontendPhaseRequest(
ProjectTableReader projectTable,
FileTableReader fileTable) {
}