added Load into compiler pipeline

gradle/libs.versions.toml

@@ -8,7 +8,7 @@ javafx-controls = { group = "org.openjfx", name = "javafx-controls", version.ref
 javafx-fxml = { group = "org.openjfx", name = "javafx-fxml", version.ref = "javafx" }
 richtextfx = { group = "org.fxmisc.richtext", name = "richtextfx", version.ref = "richtextfx" }
 jackson-databind = { group = "com.fasterxml.jackson.core", name = "jackson-databind", version.ref = "jackson" }
-apache-commons-lang3 = { group = "org.apache.commons", name = "commons-lang3", version = "3.13.0" }
+apache-commons-lang3 = { group = "org.apache.commons", name = "commons-lang3", version = "3.18.0" }
 apache-commons-io = { group = "commons-io", name = "commons-io", version = "2.13.0" }
 apache-commons-collections = { group = "org.apache.commons", name = "commons-collections4", version = "4.4" }
 

prometeu-compiler/frontends/prometeu-frontend-pbs/src/main/java/p/studio/compiler/PBSDefinitions.java

@@ -7,7 +7,7 @@ public class PBSDefinitions {
     public static final FrontendSpec PBS = FrontendSpec
             .builder()
             .languageId("pbs")
-            .allowedExtensions(ReadOnlySet.from("pbs"))
+            .allowedExtensions(ReadOnlySet.from("pbs", "barrel"))
             .sourceRoots(ReadOnlySet.from("src"))
             .build();
 }

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/models/BuilderPipelineContext.java

@@ -1,18 +1,28 @@
 package p.studio.compiler.models;
 
 import p.studio.compiler.messages.BuilderPipelineConfig;
+import p.studio.compiler.source.tables.FileTable;
+import p.studio.compiler.utilities.SourceProviderFactory;
+
+import java.nio.file.Path;
 
 public class BuilderPipelineContext {
     public final BuilderPipelineConfig config;
+    public final SourceProviderFactory sourceProviderFactory;
 
+    public Path rootProjectPathCanon;
     public ResolvedWorkspace resolvedWorkspace;
 
+    public final FileTable fileTable = new FileTable();
+
     private BuilderPipelineContext(
-            final BuilderPipelineConfig config) {
+            final BuilderPipelineConfig config,
+            final SourceProviderFactory factory) {
         this.config = config;
+        this.sourceProviderFactory = factory;
     }
 
-    public static BuilderPipelineContext basedOn(BuilderPipelineConfig config) {
-        return new BuilderPipelineContext(config);
+    public static BuilderPipelineContext compilerContext(final BuilderPipelineConfig config) {
+        return new BuilderPipelineContext(config, SourceProviderFactory.filesystem());
     }
 }

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/BuilderPipelineService.java

@@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j;
 import p.studio.compiler.exceptions.BuildException;
 import p.studio.compiler.messages.BuilderPipelineConfig;
 import p.studio.compiler.models.BuilderPipelineContext;
+import p.studio.compiler.workspaces.stages.LoadPipelineStage;
 import p.studio.compiler.workspaces.stages.ResolvePipelineStage;
 import p.studio.utilities.logs.LogAggregator;
 import p.studio.utilities.structures.ReadOnlyCollection;
@@ -16,7 +17,8 @@ public class BuilderPipelineService {
 
     static {
         final var stages = List.<PipelineStage>of(
-                new ResolvePipelineStage()
+                new ResolvePipelineStage(),
+                new LoadPipelineStage()
         );
         INSTANCE = new BuilderPipelineService(stages);
     }
@@ -30,7 +32,7 @@ public class BuilderPipelineService {
     public void run(
             final BuilderPipelineConfig config,
             final LogAggregator logs) {
-        final var ctx = BuilderPipelineContext.basedOn(config);
+        final var ctx = BuilderPipelineContext.compilerContext(config);
 
         for (final var builderPipelineStage : stages) {
             final var issues = builderPipelineStage.run(ctx, logs);

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/LoadPipelineStage.java

@@ -0,0 +1,145 @@
+package p.studio.compiler.workspaces.stages;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.StringUtils;
+import p.studio.compiler.messages.BuildingIssues;
+import p.studio.compiler.models.BuilderPipelineContext;
+import p.studio.compiler.models.SourceHandle;
+import p.studio.compiler.workspaces.PipelineStage;
+import p.studio.utilities.logs.LogAggregator;
+import p.studio.utilities.structures.ReadOnlySet;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class LoadPipelineStage implements PipelineStage {
+
+    @Override
+    public BuildingIssues run(final BuilderPipelineContext ctx, final LogAggregator logs) {
+        final var issues = BuildingIssues.empty();
+
+        // Iterates projects; "loads sources"; registers files
+        ctx.resolvedWorkspace.topologicalOrder().forEach(pId -> {
+            final var pd = ctx.resolvedWorkspace.graph().projectTable().get(pId);
+            logs.using(log).info("Project [ " + pd.getName() + " ] source loading...");
+
+            final var allowedExtensions = normalize(pd.getFrontendSpec().getAllowedExtensions());
+            for (final var sourceRootPath : pd.getSourceRoots()) {
+                logs.using(log).debug("Walking source root [ " + sourceRootPath + " ]");
+
+                try {
+                    final List<Path> paths = new ArrayList<>();
+                    Files.walkFileTree(sourceRootPath, new SourceCrawler(allowedExtensions, paths));
+                    paths.sort(Path::compareTo);
+                    for (var path : paths) {
+                        logs.using(log).debug("file tabling [ " + path + " ]");
+                        final var attributes = Files.readAttributes(path, BasicFileAttributes.class);
+                        final var size = attributes.size();
+                        final var lastModified = attributes.lastModifiedTime().toMillis();
+                        final var rawFile = new SourceHandle(pId, path, size, lastModified, ctx.sourceProviderFactory);
+                        // register in dense tables
+                        var fileId = ctx.fileTable.register(rawFile);
+                    }
+                } catch (IOException e) {
+                    issues.add(builder -> builder
+                            .error(true)
+                            .message("Failed to load project [ " + pd.getName() + " ]")
+                            .exception(e));
+                }
+            }
+        });
+        return BuildingIssues.empty();
+    }
+
+    private static ReadOnlySet<String> normalize(final ReadOnlySet<String> extensions) {
+        return ReadOnlySet
+                .wrap(extensions
+                .map(String::toLowerCase)
+                .map(s -> s.startsWith(".") ? s.substring(1) : s)
+                .collect(Collectors.toSet()));
+    }
+
+    private static final class SourceCrawler extends SimpleFileVisitor<Path>  {
+
+        private static final Set<String> IGNORED_DIRS = Set.of(
+                ".git",
+                ".prometeu",
+                ".workspace",
+                "target",
+                "build",
+                "out",
+                "node_modules"
+        );
+
+        private final ReadOnlySet<String> allowedExtensions;
+        private final List<Path> paths;
+
+        public SourceCrawler(
+                final ReadOnlySet<String> allowedExtensions,
+                final List<Path> paths) {
+            this.allowedExtensions = allowedExtensions;
+            this.paths = paths;
+        }
+
+        @Override
+        public FileVisitResult preVisitDirectory(final Path path, final BasicFileAttributes attrs) {
+            for (final var directory : path) {
+                if (!isAllowedPath(directory)) {
+                    return FileVisitResult.SKIP_SUBTREE;
+                }
+            }
+            return FileVisitResult.CONTINUE;
+        }
+
+        @Override
+        public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs) {
+            if (!attrs.isRegularFile()) {
+                return FileVisitResult.CONTINUE;
+            }
+
+            if (hasAllowedExt(path, allowedExtensions)) {
+                paths.add(path);
+            }
+
+            return FileVisitResult.CONTINUE;
+        }
+
+        private static boolean isAllowedPath(
+                final Path path) {
+            for (Path part : path) {
+                if (IGNORED_DIRS.contains(part.toString())) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static boolean hasAllowedExt(
+                final Path path,
+                final ReadOnlySet<String> allowedExtensions) {
+
+            final var fileName = path.getFileName();
+            if (Objects.isNull(fileName)) {
+                return false;
+            }
+
+            final var extension = FilenameUtils.getExtension(fileName.toString()).toLowerCase();
+            if (StringUtils.isBlank(extension)) {
+                return false;
+            }
+
+            return allowedExtensions.contains(extension);
+        }
+    }
+}

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/ResolvePipelineStage.java

@@ -9,16 +9,14 @@ import p.studio.compiler.workspaces.PipelineStage;
 import p.studio.utilities.logs.LogAggregator;
 
 import java.io.IOException;
-import java.nio.file.Path;
 import java.nio.file.Paths;
 
 @Slf4j
 public class ResolvePipelineStage implements PipelineStage {
     @Override
     public BuildingIssues run(final BuilderPipelineContext ctx, LogAggregator logs) {
-        final Path rootCanonPath;
         try {
-            rootCanonPath = Paths.get(ctx.config.rootProjectPath()).toRealPath();
+            ctx.rootProjectPathCanon = Paths.get(ctx.config.rootProjectPath()).toRealPath();
         } catch (IOException e) {
             return BuildingIssues.empty()
                     .add(builder -> builder
@@ -26,9 +24,8 @@ public class ResolvePipelineStage implements PipelineStage {
                             .message("[BUILD]: root project directory no found: " + ctx.config.rootProjectPath())
                             .exception(e));
         }
-        final var dependencyConfig = new DependencyConfig(ctx.config.explain(), rootCanonPath);
+        final var dependencyConfig = new DependencyConfig(ctx.config.explain(), ctx.rootProjectPathCanon);
         ctx.resolvedWorkspace = DependencyService.INSTANCE.run(dependencyConfig, logs);
-        ctx.resolvedWorkspace.topologicalOrder().forEach(pd -> logs.using(log).info("Project [ " + pd.getName() + " ] read"));
         return BuildingIssues.empty();
     }
 }

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/ProjectDescriptor.java

@@ -9,9 +9,9 @@ import java.nio.file.Path;
 @Builder
 @Getter
 public final class ProjectDescriptor {
-    private final Path rootPath;
-    private final String name;
-    private final String version;
+    private final Path rootPath; // canon root path
+    private final String name; // project name
+    private final String version; // project version
     private final ReadOnlyList<Path> sourceRoots;
     private final FrontendSpec frontendSpec;
 }

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/SourceHandle.java

@@ -0,0 +1,49 @@
+package p.studio.compiler.models;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import p.studio.compiler.source.identifiers.ProjectId;
+import p.studio.compiler.utilities.SourceProvider;
+import p.studio.compiler.utilities.SourceProviderFactory;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
+public class SourceHandle {
+    @Getter
+    private final ProjectId projectId;
+    @EqualsAndHashCode.Include
+    @Getter
+    private final Path path; // canon path to file
+    @Getter
+    private final String filename;
+    @Getter
+    private final long size;
+    @Getter
+    private final long lastModified;
+    private final SourceProvider provider;
+
+    public SourceHandle(
+            final ProjectId projectId,
+            final Path path,
+            final long size,
+            final long lastModified,
+            final SourceProviderFactory factory) {
+        this.projectId = projectId;
+        this.path = path;
+        this.filename = path.getFileName().toString();
+        this.size = size;
+        this.lastModified = lastModified;
+        this.provider = factory.create(path);
+    }
+
+    public byte[] readBytes() throws IOException {
+        return provider.read();
+    }
+
+    public String readUtf8() throws IOException {
+        return new String(readBytes(), StandardCharsets.UTF_8);
+    }
+}

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/source/SourceFile.java

@@ -1,15 +0,0 @@
-package p.studio.compiler.source;
-
-import lombok.Builder;
-import lombok.Getter;
-import p.studio.compiler.source.identifiers.ProjectId;
-
-@Builder
-@Getter
-public class SourceFile {
-    private final ProjectId projectId;
-    private final String module;
-    private final String name;
-    private final String extension;
-    private final byte[] content;
-}

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/source/tables/FileTable.java

@@ -1,13 +1,14 @@
 package p.studio.compiler.source.tables;
 
 import org.apache.commons.collections4.CollectionUtils;
-import p.studio.compiler.source.SourceFile;
+import p.studio.compiler.models.SourceHandle;
 import p.studio.compiler.source.identifiers.FileId;
 import p.studio.compiler.source.identifiers.ProjectId;
+import p.studio.utilities.structures.ReadOnlyList;
 
 import java.util.*;
 
-public class FileTable extends InternTable<FileId, SourceFile> {
+public class FileTable extends InternTable<FileId, SourceHandle> {
 
     private final Map<ProjectId, Set<FileId>> projectFiles = new HashMap<>();
 
@@ -16,17 +17,17 @@ public class FileTable extends InternTable<FileId, SourceFile> {
     }
 
     @Override
-    public FileId register(final SourceFile value) {
+    public FileId register(final SourceHandle value) {
         final var fileId = super.register(value);
         projectFiles.computeIfAbsent(value.getProjectId(), ignored -> new HashSet<>()).add(fileId);
         return fileId;
     }
 
-    public List<SourceFile> getSourceFiles(final ProjectId projectId) {
+    public ReadOnlyList<FileId> getFiles(final ProjectId projectId) {
         final var fileIds = projectFiles.get(projectId);
         if (CollectionUtils.isEmpty(fileIds)) {
-            return List.of();
+            return ReadOnlyList.empty();
         }
-        return fileIds.stream().map(this::get).toList();
+        return ReadOnlyList.wrap(fileIds);
     }
 }

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/utilities/SourceProvider.java

@@ -0,0 +1,8 @@
+package p.studio.compiler.utilities;
+
+import java.io.IOException;
+
+@FunctionalInterface
+public interface SourceProvider {
+    byte[] read() throws IOException;
+}

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/utilities/SourceProviderFactory.java

@@ -0,0 +1,28 @@
+package p.studio.compiler.utilities;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Objects;
+
+@FunctionalInterface
+public interface SourceProviderFactory {
+    SourceProvider create(Path path);
+
+    static SourceProviderFactory filesystem() {
+        return path -> new SourceProvider() {
+
+            private volatile byte[] cachedBytes = null;
+
+            @Override
+            public byte[] read() throws IOException {
+                if (Objects.isNull(cachedBytes)) {
+                    cachedBytes = Files.readAllBytes(path);
+                }
+                return cachedBytes;
+            }
+        };
+    }
+
+    // more providers can be added here, e.g., in-memory, etc.
+}

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/models/ResolvedWorkspace.java

@@ -8,15 +8,16 @@ public record ResolvedWorkspace(
         ProjectId projectId,
         WorkspaceGraph graph,
         BuildStack stack) {
+
     public ProjectDescriptor mainProject() {
         return graph.projectDescriptor(projectId);
     }
 
-    public Stream<ProjectDescriptor> topologicalOrder() {
-        return stack.topologicalOrder.stream().map(graph::projectDescriptor);
+    public Stream<ProjectId> topologicalOrder() {
+        return stack.topologicalOrder.stream();
     }
 
-    public Stream<ProjectDescriptor> reverseTopologicalOrder() {
-        return stack.reverseTopologicalOrder.stream().map(graph::projectDescriptor);
+    public Stream<ProjectId> reverseTopologicalOrder() {
+        return stack.reverseTopologicalOrder.stream();
     }
 }

prometeu-infra/build.gradle.kts

@@ -5,5 +5,6 @@ plugins {
 dependencies {
     api(libs.jackson.databind)
     api(libs.apache.commons.lang3)
+    api(libs.apache.commons.io)
     api(libs.apache.commons.collections)
 }

prometeu-infra/src/main/java/p/studio/utilities/structures/ReadOnlyList.java

@@ -1,7 +1,5 @@
 package p.studio.utilities.structures;
 
-import org.apache.commons.collections4.ListUtils;
-
 import java.util.*;
 import java.util.stream.Stream;
 
@@ -23,8 +21,8 @@ public class ReadOnlyList<T> extends ReadOnlyCollection<T> {
         return ReadOnlyList.wrap(new ArrayList<>(c.collection));
     }
 
-    public static <T> ReadOnlyCollection<T> wrap(final Set<T> set) {
-        return new ReadOnlyCollection<>(List.copyOf(set));
+    public static <T> ReadOnlyList<T> wrap(final Set<T> set) {
+        return new ReadOnlyList<>(List.copyOf(set));
     }
 
     @SafeVarargs