code improvements, add pipeline stages

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/models/BuilderPipelineContext.java

@@ -13,7 +13,7 @@ public class BuilderPipelineContext {
     public Path rootProjectPathCanon;
     public ResolvedWorkspace resolvedWorkspace;
 
-    public final FileTable fileTable = new FileTable();
+    public FileTable fileTable;
 
     private BuilderPipelineContext(
             final BuilderPipelineConfig config,

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/BuilderPipelineService.java

@@ -4,8 +4,7 @@ import lombok.extern.slf4j.Slf4j;
 import p.studio.compiler.exceptions.BuildException;
 import p.studio.compiler.messages.BuilderPipelineConfig;
 import p.studio.compiler.models.BuilderPipelineContext;
-import p.studio.compiler.workspaces.stages.LoadPipelineStage;
+import p.studio.compiler.workspaces.stages.*;
-import p.studio.compiler.workspaces.stages.ResolvePipelineStage;
 import p.studio.utilities.logs.LogAggregator;
 import p.studio.utilities.structures.ReadOnlyCollection;
 
@@ -17,8 +16,11 @@ public class BuilderPipelineService {
 
     static {
         final var stages = List.<PipelineStage>of(
-                new ResolvePipelineStage(),
+                new ResolveDepsPipelineStage(),
-                new LoadPipelineStage()
+                new LoadSourcesPipelineStage(),
+                new FrontendPhasePipelineStage(),
+                new LowerToVMPipelineStage(),
+                new EmitBytecodePipelineStage()
         );
         INSTANCE = new BuilderPipelineService(stages);
     }

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/EmitBytecodePipelineStage.java

@@ -0,0 +1,15 @@
+package p.studio.compiler.workspaces.stages;
+
+import lombok.extern.slf4j.Slf4j;
+import p.studio.compiler.messages.BuildingIssueSink;
+import p.studio.compiler.models.BuilderPipelineContext;
+import p.studio.compiler.workspaces.PipelineStage;
+import p.studio.utilities.logs.LogAggregator;
+
+@Slf4j
+public class EmitBytecodePipelineStage implements PipelineStage {
+    @Override
+    public BuildingIssueSink run(BuilderPipelineContext ctx, LogAggregator logs) {
+        return BuildingIssueSink.empty();
+    }
+}

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/FrontendPhasePipelineStage.java

@@ -0,0 +1,15 @@
+package p.studio.compiler.workspaces.stages;
+
+import lombok.extern.slf4j.Slf4j;
+import p.studio.compiler.messages.BuildingIssueSink;
+import p.studio.compiler.models.BuilderPipelineContext;
+import p.studio.compiler.workspaces.PipelineStage;
+import p.studio.utilities.logs.LogAggregator;
+
+@Slf4j
+public class FrontendPhasePipelineStage implements PipelineStage {
+    @Override
+    public BuildingIssueSink run(BuilderPipelineContext ctx, LogAggregator logs) {
+        return BuildingIssueSink.empty();
+    }
+}

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/LoadPipelineStage.java

@@ -1,145 +0,0 @@
-package p.studio.compiler.workspaces.stages;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang3.StringUtils;
-import p.studio.compiler.messages.BuildingIssueSink;
-import p.studio.compiler.models.BuilderPipelineContext;
-import p.studio.compiler.models.SourceHandle;
-import p.studio.compiler.workspaces.PipelineStage;
-import p.studio.utilities.logs.LogAggregator;
-import p.studio.utilities.structures.ReadOnlySet;
-
-import java.io.IOException;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-@Slf4j
-public class LoadPipelineStage implements PipelineStage {
-
-    @Override
-    public BuildingIssueSink run(final BuilderPipelineContext ctx, final LogAggregator logs) {
-        final var issues = BuildingIssueSink.empty();
-
-        // Iterates projects; "loads sources"; registers files
-        ctx.resolvedWorkspace.topologicalOrder().forEach(pId -> {
-            final var pd = ctx.resolvedWorkspace.graph().projectTable().get(pId);
-            logs.using(log).info("Project [ " + pd.getName() + " ] source loading...");
-
-            final var allowedExtensions = normalize(ctx.resolvedWorkspace.frontendSpec().getAllowedExtensions());
-            for (final var sourceRootPath : pd.getSourceRoots()) {
-                logs.using(log).debug("Walking source root [ " + sourceRootPath + " ]");
-
-                try {
-                    final List<Path> paths = new ArrayList<>();
-                    Files.walkFileTree(sourceRootPath, new SourceCrawler(allowedExtensions, paths));
-                    paths.sort(Path::compareTo);
-                    for (var path : paths) {
-                        logs.using(log).debug("file tabling [ " + path + " ]");
-                        final var attributes = Files.readAttributes(path, BasicFileAttributes.class);
-                        final var size = attributes.size();
-                        final var lastModified = attributes.lastModifiedTime().toMillis();
-                        final var rawFile = new SourceHandle(pId, path, size, lastModified, ctx.sourceProviderFactory);
-                        // register in dense tables
-                        var fileId = ctx.fileTable.register(rawFile);
-                    }
-                } catch (IOException e) {
-                    issues.report(builder -> builder
-                            .error(true)
-                            .message("Failed to load project [ " + pd.getName() + " ]")
-                            .exception(e));
-                }
-            }
-        });
-        return BuildingIssueSink.empty();
-    }
-
-    private static ReadOnlySet<String> normalize(final ReadOnlySet<String> extensions) {
-        return ReadOnlySet
-                .wrap(extensions
-                .map(String::toLowerCase)
-                .map(s -> s.startsWith(".") ? s.substring(1) : s)
-                .collect(Collectors.toSet()));
-    }
-
-    private static final class SourceCrawler extends SimpleFileVisitor<Path>  {
-
-        private static final Set<String> IGNORED_DIRS = Set.of(
-                ".git",
-                ".prometeu",
-                ".workspace",
-                "target",
-                "build",
-                "out",
-                "node_modules"
-        );
-
-        private final ReadOnlySet<String> allowedExtensions;
-        private final List<Path> paths;
-
-        public SourceCrawler(
-                final ReadOnlySet<String> allowedExtensions,
-                final List<Path> paths) {
-            this.allowedExtensions = allowedExtensions;
-            this.paths = paths;
-        }
-
-        @Override
-        public FileVisitResult preVisitDirectory(final Path path, final BasicFileAttributes attrs) {
-            for (final var directory : path) {
-                if (!isAllowedPath(directory)) {
-                    return FileVisitResult.SKIP_SUBTREE;
-                }
-            }
-            return FileVisitResult.CONTINUE;
-        }
-
-        @Override
-        public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs) {
-            if (!attrs.isRegularFile()) {
-                return FileVisitResult.CONTINUE;
-            }
-
-            if (hasAllowedExt(path, allowedExtensions)) {
-                paths.add(path);
-            }
-
-            return FileVisitResult.CONTINUE;
-        }
-
-        private static boolean isAllowedPath(
-                final Path path) {
-            for (Path part : path) {
-                if (IGNORED_DIRS.contains(part.toString())) {
-                    return false;
-                }
-            }
-            return true;
-        }
-
-        private static boolean hasAllowedExt(
-                final Path path,
-                final ReadOnlySet<String> allowedExtensions) {
-
-            final var fileName = path.getFileName();
-            if (Objects.isNull(fileName)) {
-                return false;
-            }
-
-            final var extension = FilenameUtils.getExtension(fileName.toString()).toLowerCase();
-            if (StringUtils.isBlank(extension)) {
-                return false;
-            }
-
-            return allowedExtensions.contains(extension);
-        }
-    }
-}

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/LoadSourcesPipelineStage.java

@@ -0,0 +1,97 @@
+package p.studio.compiler.workspaces.stages;
+
+import lombok.extern.slf4j.Slf4j;
+import p.studio.compiler.messages.BuildingIssueSink;
+import p.studio.compiler.models.BuilderPipelineContext;
+import p.studio.compiler.models.ProjectDescriptor;
+import p.studio.compiler.models.SourceHandle;
+import p.studio.compiler.source.identifiers.ProjectId;
+import p.studio.compiler.workspaces.PipelineStage;
+import p.studio.utilities.logs.LogAggregator;
+import p.studio.utilities.structures.ReadOnlySet;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class LoadSourcesPipelineStage implements PipelineStage {
+
+    @Override
+    public BuildingIssueSink run(final BuilderPipelineContext ctx, final LogAggregator logs) {
+        final var issues = BuildingIssueSink.empty();
+        ctx.resolvedWorkspace.stack().topologicalOrder.forEach(pId -> visitProjectSources(pId, ctx, logs, issues));
+        return issues;
+    }
+
+    private static void visitProjectSources(
+            final ProjectId pId,
+            final BuilderPipelineContext ctx,
+            final LogAggregator logs,
+            final BuildingIssueSink issues) {
+        final var pd = ctx.resolvedWorkspace.graph().projectTable().get(pId);
+        logs.using(log).debug("Project [ " + pd.getName() + " ] source loading...");
+
+        final var allowedExtensions = normalize(ctx.resolvedWorkspace.frontendSpec().getAllowedExtensions());
+        // Iterates source roots; crawls files; registers them
+        for (final var sourceRootPath : pd.getSourceRoots()) {
+            logs.using(log).debug("Walking source root [ " + sourceRootPath + " ]");
+
+            final List<Path> paths = new ArrayList<>();
+            final var sourceCrawler = new SourceCrawler(allowedExtensions, paths);
+
+            try {
+                Files.walkFileTree(sourceRootPath, sourceCrawler);
+                paths.sort(Path::compareTo); // do we really need this for deterministic builds?
+            } catch (IOException e) {
+                issues.report(builder -> builder
+                        .error(true)
+                        .message("Failed to load project [ " + pd.getName() + " ]")
+                        .exception(e));
+            }
+
+            registerFiles(pId, pd, paths, ctx, logs, issues);
+        }
+    }
+
+    private static void registerFiles(
+            final ProjectId pId,
+            final ProjectDescriptor pd,
+            final List<Path> canonPaths,
+            final BuilderPipelineContext ctx,
+            final LogAggregator logs,
+            final BuildingIssueSink issues) {
+        for (var canonPath : canonPaths) {
+            final long size;
+            final long lastModified;
+            try {
+                final BasicFileAttributes attributes = Files.readAttributes(canonPath, BasicFileAttributes.class);
+                size = attributes.size();
+                lastModified = attributes.lastModifiedTime().toMillis();
+            } catch (IOException e) {
+                issues.report(builder -> builder
+                        .error(true)
+                        .message("Failed to read attributes for file: " + canonPath)
+                        .exception(e));
+                continue;
+            }
+            // register in dense tables
+            final var relativePath = pd.getRootPath().relativize(canonPath);
+            final var sourceHandle = new SourceHandle(pId, relativePath, canonPath, size, lastModified, ctx.sourceProviderFactory);
+            logs.using(log).debug("Registering: " + sourceHandle);
+            ctx.fileTable.register(sourceHandle);
+        }
+    }
+
+    private static ReadOnlySet<String> normalize(final ReadOnlySet<String> extensions) {
+        return ReadOnlySet
+                .wrap(extensions
+                .map(String::toLowerCase)
+                .map(s -> s.startsWith(".") ? s.substring(1) : s)
+                .collect(Collectors.toSet()));
+    }
+}

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/LowerToVMPipelineStage.java

@@ -0,0 +1,15 @@
+package p.studio.compiler.workspaces.stages;
+
+import lombok.extern.slf4j.Slf4j;
+import p.studio.compiler.messages.BuildingIssueSink;
+import p.studio.compiler.models.BuilderPipelineContext;
+import p.studio.compiler.workspaces.PipelineStage;
+import p.studio.utilities.logs.LogAggregator;
+
+@Slf4j
+public class LowerToVMPipelineStage implements PipelineStage {
+    @Override
+    public BuildingIssueSink run(BuilderPipelineContext ctx, LogAggregator logs) {
+        return BuildingIssueSink.empty();
+    }
+}

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/ResolveDepsPipelineStage.java

@@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j;
 import p.studio.compiler.messages.BuildingIssueSink;
 import p.studio.compiler.messages.DependencyConfig;
 import p.studio.compiler.models.BuilderPipelineContext;
+import p.studio.compiler.source.tables.FileTable;
 import p.studio.compiler.workspaces.DependencyService;
 import p.studio.compiler.workspaces.PipelineStage;
 import p.studio.utilities.logs.LogAggregator;
@@ -12,7 +13,7 @@ import java.io.IOException;
 import java.nio.file.Paths;
 
 @Slf4j
-public class ResolvePipelineStage implements PipelineStage {
+public class ResolveDepsPipelineStage implements PipelineStage {
     @Override
     public BuildingIssueSink run(final BuilderPipelineContext ctx, LogAggregator logs) {
         try {
@@ -26,6 +27,7 @@ public class ResolvePipelineStage implements PipelineStage {
         }
         final var dependencyConfig = new DependencyConfig(ctx.config.explain(), ctx.rootProjectPathCanon);
         ctx.resolvedWorkspace = DependencyService.INSTANCE.run(dependencyConfig, logs);
+        ctx.fileTable = new FileTable(ctx.resolvedWorkspace.graph().projectTable().size());
         return BuildingIssueSink.empty();
     }
 }

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/SourceCrawler.java

@@ -0,0 +1,86 @@
+package p.studio.compiler.workspaces.stages;
+
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.StringUtils;
+import p.studio.utilities.structures.ReadOnlySet;
+
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+final class SourceCrawler extends SimpleFileVisitor<Path> {
+
+    private static final Set<String> IGNORED_DIRS = Set.of(
+            ".git",
+            ".prometeu",
+            ".workspace",
+            "target",
+            "build",
+            "out",
+            "node_modules"
+    );
+
+    private final ReadOnlySet<String> allowedExtensions;
+    private final List<Path> paths;
+
+    public SourceCrawler(
+            final ReadOnlySet<String> allowedExtensions,
+            final List<Path> paths) {
+        this.allowedExtensions = allowedExtensions;
+        this.paths = paths;
+    }
+
+    @Override
+    public FileVisitResult preVisitDirectory(final Path path, final BasicFileAttributes attrs) {
+        for (final var directory : path) {
+            if (!isAllowedPath(directory)) {
+                return FileVisitResult.SKIP_SUBTREE;
+            }
+        }
+        return FileVisitResult.CONTINUE;
+    }
+
+    @Override
+    public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs) {
+        if (!attrs.isRegularFile()) {
+            return FileVisitResult.CONTINUE;
+        }
+
+        if (hasAllowedExt(path, allowedExtensions)) {
+            paths.add(path);
+        }
+
+        return FileVisitResult.CONTINUE;
+    }
+
+    private static boolean isAllowedPath(
+            final Path path) {
+        for (Path part : path) {
+            if (IGNORED_DIRS.contains(part.toString())) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private static boolean hasAllowedExt(
+            final Path path,
+            final ReadOnlySet<String> allowedExtensions) {
+
+        final var fileName = path.getFileName();
+        if (Objects.isNull(fileName)) {
+            return false;
+        }
+
+        final var extension = FilenameUtils.getExtension(fileName.toString()).toLowerCase();
+        if (StringUtils.isBlank(extension)) {
+            return false;
+        }
+
+        return allowedExtensions.contains(extension);
+    }
+}

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/SourceHandle.java

@@ -9,14 +9,19 @@ import p.studio.compiler.utilities.SourceProviderFactory;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
 
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class SourceHandle {
     @Getter
     private final ProjectId projectId;
+    @Getter
+    private final Path relativePath;
     @EqualsAndHashCode.Include
     @Getter
-    private final Path path; // canon path to file
+    private final Path canonPath;
     @Getter
     private final String filename;
     @Getter
@@ -27,16 +32,18 @@ public class SourceHandle {
 
     public SourceHandle(
             final ProjectId projectId,
-            final Path path,
+            final Path relativePath,
+            final Path canonPath,
             final long size,
             final long lastModified,
             final SourceProviderFactory factory) {
         this.projectId = projectId;
-        this.path = path;
+        this.relativePath = relativePath;
-        this.filename = path.getFileName().toString();
+        this.canonPath = canonPath;
+        this.filename = canonPath.getFileName().toString();
         this.size = size;
         this.lastModified = lastModified;
-        this.provider = factory.create(path);
+        this.provider = factory.create(canonPath);
     }
 
     public byte[] readBytes() throws IOException {
@@ -46,4 +53,12 @@ public class SourceHandle {
     public String readUtf8() throws IOException {
         return new String(readBytes(), StandardCharsets.UTF_8);
     }
+
+    @Override
+    public String toString() {
+        return "SourceHandle{ %s, %s, %sKb, mod: %s }".formatted(projectId,
+                relativePath,
+                size / 1024,
+                LocalDateTime.ofInstant(Instant.ofEpochMilli(lastModified), ZoneId.systemDefault()));
+    }
 }

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/source/tables/FileTable.java

@@ -6,29 +6,37 @@ import p.studio.compiler.source.identifiers.FileId;
 import p.studio.compiler.source.identifiers.ProjectId;
 import p.studio.utilities.structures.ReadOnlyList;
 
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
 
 public class FileTable extends InternTable<FileId, SourceHandle> implements FileTableReader {
 
-    private final Map<ProjectId, Set<FileId>> projectFiles = new HashMap<>();
+    private final FileIds[] projectFiles;
 
-    public FileTable() {
+    public FileTable(int projectCapacity) {
         super(FileId::new);
+        projectFiles = new FileIds[projectCapacity];
+        Arrays.setAll(projectFiles, ignored -> new FileIds());
     }
 
     @Override
     public FileId register(final SourceHandle value) {
         final var fileId = super.register(value);
-        projectFiles.computeIfAbsent(value.getProjectId(), ignored -> new HashSet<>()).add(fileId);
+        projectFiles[value.getProjectId().getIndex()].set.add(fileId);
         return fileId;
     }
 
     @Override
     public ReadOnlyList<FileId> getFiles(final ProjectId projectId) {
-        final var fileIds = projectFiles.get(projectId);
+        final var fileIds = projectFiles[projectId.getIndex()];
-        if (CollectionUtils.isEmpty(fileIds)) {
+        if (CollectionUtils.isEmpty(fileIds.set)) {
             return ReadOnlyList.empty();
         }
-        return ReadOnlyList.wrap(fileIds);
+        return ReadOnlyList.wrap(fileIds.set);
+    }
+
+    private static class FileIds {
+        public final Set<FileId> set = new HashSet<>();
     }
 }

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/models/BuildStack.java

@@ -7,7 +7,7 @@ public class BuildStack {
     public final ReadOnlyList<ProjectId> topologicalOrder;
     public final ReadOnlyList<ProjectId> reverseTopologicalOrder;
 
-    public BuildStack(ReadOnlyList<ProjectId> topologicalOrder) {
+    public BuildStack(final ReadOnlyList<ProjectId> topologicalOrder) {
         this.topologicalOrder = topologicalOrder;
         this.reverseTopologicalOrder = topologicalOrder.invert();
     }

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/models/ResolvedWorkspace.java

@@ -2,23 +2,12 @@ package p.studio.compiler.models;
 
 import p.studio.compiler.source.identifiers.ProjectId;
 
-import java.util.stream.Stream;
-
 public record ResolvedWorkspace(
-        ProjectId projectId,
+        ProjectId mainProjectId,
         FrontendSpec frontendSpec,
         WorkspaceGraph graph,
         BuildStack stack) {
-
     public ProjectDescriptor mainProject() {
-        return graph.projectDescriptor(projectId);
+        return graph.projectDescriptor(mainProjectId);
-    }
-
-    public Stream<ProjectId> topologicalOrder() {
-        return stack.topologicalOrder.stream();
-    }
-
-    public Stream<ProjectId> reverseTopologicalOrder() {
-        return stack.reverseTopologicalOrder.stream();
     }
 }

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/models/WorkspaceGraph.java

@@ -1,11 +1,11 @@
 package p.studio.compiler.models;
 
 import p.studio.compiler.source.identifiers.ProjectId;
-import p.studio.compiler.source.tables.ProjectTable;
+import p.studio.compiler.source.tables.ProjectTableReader;
 import p.studio.utilities.structures.ReadOnlyList;
 
 public record WorkspaceGraph(
-        ProjectTable projectTable,
+        ProjectTableReader projectTable,
         ReadOnlyList<ReadOnlyList<ProjectId>> dependenciesByProjectId) {
 
     public ProjectDescriptor projectDescriptor(final ProjectId projectId) {

prometeu-compiler/prometeu-frontend-api/build.gradle.kts

@@ -4,4 +4,6 @@ plugins {
 
 dependencies {
     api(project(":prometeu-infra"))
+
+    implementation(project(":prometeu-compiler:prometeu-compiler-core"))
 }

prometeu-compiler/prometeu-frontend-api/src/main/java/p/studio/compiler/messages/FrontendPhaseRequest.java

@@ -0,0 +1,9 @@
+package p.studio.compiler.messages;
+
+import p.studio.compiler.source.tables.FileTableReader;
+import p.studio.compiler.source.tables.ProjectTableReader;
+
+public record FrontendPhaseRequest(
+        ProjectTableReader projectTable,
+        FileTableReader fileTable) {
+}