more clean up, and improve FileTable

prometeu-compiler/prometeu-build-pipeline/src/main/java/p/studio/compiler/workspaces/stages/DepsPipelineStage.java

@@ -28,7 +28,7 @@ public class DepsPipelineStage implements PipelineStage {
         final var cfg = new DependencyConfig(false, rootCanonPath);
         final var resolvedWorkspace = DependencyService.INSTANCE.run(cfg, ctx.getLogs());
         for (final var pId : resolvedWorkspace.stack().projects()) {
-            final var pd = resolvedWorkspace.graph().projects().get(pId.getIndex());
+            final var pd = resolvedWorkspace.graph().projectTable().get(pId);
             ctx.getLogs().info("Project [ " + pd.getName() + " ] read");
         }
         ctx.setResolvedWorkspace(resolvedWorkspace);

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/models/ProjectDescriptor.java

@@ -2,8 +2,6 @@ package p.studio.compiler.models;
 
 import lombok.Builder;
 import lombok.Getter;
-import p.studio.compiler.source.identifiers.ProjectId;
-import p.studio.compiler.workspaces.DependencyReference;
 import p.studio.utilities.structures.ReadOnlyList;
 
 import java.nio.file.Path;
@@ -11,11 +9,9 @@ import java.nio.file.Path;
 @Builder
 @Getter
 public final class ProjectDescriptor {
-    private final ProjectId projectId;
+    private final Path rootPath;
-    private final Path projectRootPath;
     private final String name;
     private final String version;
     private final ReadOnlyList<Path> sourceRoots;
-    private final ReadOnlyList<DependencyReference> dependencies;
     private final FrontendSpec frontendSpec;
 }

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/source/SourceFile.java

@@ -0,0 +1,15 @@
+package p.studio.compiler.source;
+
+import lombok.Builder;
+import lombok.Getter;
+import p.studio.compiler.source.identifiers.ProjectId;
+
+@Builder
+@Getter
+public class SourceFile {
+    private final ProjectId projectId;
+    private final String module;
+    private final String name;
+    private final String extension;
+    private final byte[] content;
+}

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/source/tables/FileTable.java

@@ -1,11 +1,32 @@
 package p.studio.compiler.source.tables;
 
+import org.apache.commons.collections4.CollectionUtils;
+import p.studio.compiler.source.SourceFile;
 import p.studio.compiler.source.identifiers.FileId;
+import p.studio.compiler.source.identifiers.ProjectId;
 
-import java.nio.file.Path;
+import java.util.*;
+
+public class FileTable extends InternTable<FileId, SourceFile> {
+
+    private final Map<ProjectId, Set<FileId>> projectFiles = new HashMap<>();
 
-public class FileTable extends InternTable<FileId, Path> {
     public FileTable() {
         super(FileId::new);
     }
+
+    @Override
+    public FileId register(final SourceFile value) {
+        final var fileId = super.register(value);
+        projectFiles.computeIfAbsent(value.getProjectId(), ignored -> new HashSet<>()).add(fileId);
+        return fileId;
+    }
+
+    public List<SourceFile> getSourceFiles(final ProjectId projectId) {
+        final var fileIds = projectFiles.get(projectId);
+        if (CollectionUtils.isEmpty(fileIds)) {
+            return List.of();
+        }
+        return fileIds.stream().map(this::get).toList();
+    }
 }

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/source/tables/InternTable.java

@@ -2,9 +2,9 @@ package p.studio.compiler.source.tables;
 
 import p.studio.compiler.source.identifiers.SourceIdentifier;
 
-import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Function;
 
 public abstract class InternTable<IDENTIFIER extends SourceIdentifier, VALUE>
@@ -21,8 +21,8 @@ public abstract class InternTable<IDENTIFIER extends SourceIdentifier, VALUE>
         return identifierByValue.computeIfAbsent(value, super::register);
     }
 
-    public IDENTIFIER get(final VALUE value) {
+    public Optional<IDENTIFIER> optional(final VALUE value) {
-        return identifierByValue.get(value);
+        return Optional.ofNullable(identifierByValue.get(value));
     }
 
     @Override

prometeu-compiler/prometeu-compiler-core/src/main/java/p/studio/compiler/source/tables/ProjectTable.java

@@ -1,11 +1,37 @@
 package p.studio.compiler.source.tables;
 
+import p.studio.compiler.models.ProjectDescriptor;
 import p.studio.compiler.source.identifiers.ProjectId;
 
 import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public class ProjectTable extends DenseTable<ProjectId, ProjectDescriptor> {
+
+    private final Map<Path, ProjectId> projectIdByPath = new HashMap<>();
 
-public class ProjectTable extends InternTable<ProjectId, Path> {
     public ProjectTable() {
         super(ProjectId::new);
     }
+
+    @Override
+    public ProjectId register(final ProjectDescriptor value) {
+        return projectIdByPath.computeIfAbsent(value.getRootPath(), ignored -> super.register(value));
+    }
+
+    @Override
+    public void clear() {
+        super.clear();
+        projectIdByPath.clear();
+    }
+
+    public Optional<ProjectDescriptor> optional(Path pathCanon) {
+        return optionalId(pathCanon).map(this::get);
+    }
+
+    public Optional<ProjectId> optionalId(Path pathCanon) {
+        return Optional.ofNullable(projectIdByPath.get(pathCanon));
+    }
 }

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/dtos/PrometeuManifestDTO.java

@@ -1,7 +1,6 @@
 package p.studio.compiler.dtos;
 
 import com.fasterxml.jackson.annotation.*;
-import lombok.Getter;
 
 import java.util.List;
 
@@ -18,10 +17,7 @@ public record PrometeuManifestDTO(
             @JsonSubTypes.Type(value = DependencyDeclaration.Git.class)
     })
     public interface DependencyDeclaration {
-        @Getter
+        record Local(String path) implements DependencyDeclaration {
-        class Local implements DependencyDeclaration {
-            private final String path;
-
             @JsonCreator
             public Local(@JsonProperty("path") final String path) {
                 this.path = path;
@@ -29,11 +25,7 @@ public record PrometeuManifestDTO(
         }
 
         @JsonIgnoreProperties(ignoreUnknown = true)
-        @Getter
+        record Git(String url, String rev) implements DependencyDeclaration {
-        class Git implements DependencyDeclaration {
-            private final String url;
-            private final String rev;
-
                     @JsonCreator
                     public Git(@JsonProperty("url") final String url, @JsonProperty("rev") final String rev) {
                         this.url = url;

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/models/DependencyContext.java

@@ -3,6 +3,7 @@ package p.studio.compiler.models;
 import p.studio.compiler.exceptions.BuildException;
 import p.studio.compiler.messages.DependencyConfig;
 import p.studio.compiler.source.identifiers.ProjectId;
+import p.studio.compiler.source.tables.FileTable;
 import p.studio.compiler.source.tables.ProjectTable;
 import p.studio.utilities.structures.ReadOnlyList;
 
@@ -22,11 +23,12 @@ public final class DependencyContext {
     public final ProjectTable projectTable = new ProjectTable();
 
     public final Map<String, Set<String>> projectNameAndVersions = new HashMap<>();
-    public final List<ProjectDescriptor> projectDescriptors = new ArrayList<>();
+    public final List<ProjectId> projectIds = new ArrayList<>();
     public final List<List<ProjectId>> dependenciesByProject = new ArrayList<>();
 
     public ProjectId rootProjectId;
     public BuildStack stack;
+    public FileTable fileTable;
 
     private DependencyContext(DependencyConfig config) {
         this.config = config;
@@ -40,17 +42,22 @@ public final class DependencyContext {
         return config;
     }
 
-    public ResolvedWorkspace toResolvedWorkspace() {
+    private ReadOnlyList<ReadOnlyList<ProjectId>> buildDependenciesByProject() {
-        if (rootProjectId == null) {
+        return ReadOnlyList.wrap(this
-            throw new BuildException("dependenciesByProjectId: internal error: rootProjectId ProjectId not set");
-        }
-        final var projectDescriptors = ReadOnlyList.wrap(this.projectDescriptors);
-        final var dependenciesByProject = ReadOnlyList.wrap(this
                 .dependenciesByProject
                 .stream()
                 .map(ReadOnlyList::wrap)
                 .toList());
-        final var workspaceGraph = new WorkspaceGraph(projectDescriptors, dependenciesByProject);
-        return new ResolvedWorkspace(rootProjectId, workspaceGraph, stack);
     }
+
+    public ResolvedWorkspace toResolvedWorkspace() {
+        if (rootProjectId == null) {
+            throw new BuildException("dependenciesByProjectId: internal error: rootProjectId ProjectId not set");
+        }
+        final var dependenciesByProject = buildDependenciesByProject();
+        final var workspaceGraph = new WorkspaceGraph(projectTable, dependenciesByProject);
+        return new ResolvedWorkspace(rootProjectId, workspaceGraph, stack, fileTable);
+    }
+
+
 }

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/models/ResolvedWorkspace.java

@@ -1,9 +1,11 @@
 package p.studio.compiler.models;
 
 import p.studio.compiler.source.identifiers.ProjectId;
+import p.studio.compiler.source.tables.FileTable;
 
 public record ResolvedWorkspace(
         ProjectId projectId,
         WorkspaceGraph graph,
-        BuildStack stack) {
+        BuildStack stack,
+        FileTable fileTable) {
 }

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/models/WorkspaceGraph.java

@@ -1,14 +1,15 @@
 package p.studio.compiler.models;
 
 import p.studio.compiler.source.identifiers.ProjectId;
+import p.studio.compiler.source.tables.ProjectTable;
 import p.studio.utilities.structures.ReadOnlyList;
 
 public record WorkspaceGraph(
-        ReadOnlyList<ProjectDescriptor> projects,
+        ProjectTable projectTable,
         ReadOnlyList<ReadOnlyList<ProjectId>> dependenciesByProjectId) {
 
     public ProjectDescriptor getProjectDescriptor(final ProjectId projectId) {
-        return projects.get(projectId.getIndex());
+        return projectTable.get(projectId);
     }
 
     public ReadOnlyList<ProjectId> getDependencies(final ProjectId projectId) {

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/workspaces/DependencyService.java

@@ -19,11 +19,9 @@ public final class DependencyService {
         final var phases = List.of(
                 new SeedPhase(),
                 new DiscoverPhase(),
-                new MaterializePhase(),
+                new WireProjectsPhase(),
-                new LocalizePhase(),
                 new ValidatePhase(),
-                new StackPhase(),
+                new StackPhase()
-                new PolicyPhase()
         );
         INSTANCE = new DependencyService(phases);
     }

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/workspaces/phases/DiscoverPhase.java

@@ -143,13 +143,13 @@ public class DiscoverPhase implements DependencyPhase {
             switch (dependency) {
                 case PrometeuManifestDTO.DependencyDeclaration.Local local -> {
                     try {
-                        final Path dependencyPathCanon = rootProjectCanonPath.resolve(local.getPath()).toRealPath();
+                        final Path dependencyPathCanon = rootProjectCanonPath.resolve(local.path()).toRealPath();
                         deps.add(new DependencyReference(dependencyPathCanon));
                     } catch (IOException e) {
                         final var issue = BuildingIssue
                                 .builder()
                                 .error(true)
-                                .message("[DEPS]: failed to canonicalize dependency path: " + local.getPath() + " from (" + rootProjectCanonPath + ")")
+                                .message("[DEPS]: failed to canonicalize dependency path: " + local.path() + " from (" + rootProjectCanonPath + ")")
                                 .exception(e)
                                 .build();
                         buildingIssues.add(issue);
@@ -160,7 +160,7 @@ public class DiscoverPhase implements DependencyPhase {
                     final var issue = BuildingIssue
                             .builder()
                             .error(true)
-                            .message("[DEPS]: git dependencies are not supported yet: " + git.getUrl() + " from (" + rootProjectCanonPath + ")")
+                            .message("[DEPS]: git dependencies are not supported yet: " + git.url() + " from (" + rootProjectCanonPath + ")")
                             .build();
                     buildingIssues.add(issue);
                 }

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/workspaces/phases/LocalizePhase.java

@@ -1,35 +0,0 @@
-package p.studio.compiler.workspaces.phases;
-
-import p.studio.compiler.messages.BuildingIssue;
-import p.studio.compiler.models.DependencyContext;
-import p.studio.compiler.workspaces.DependencyPhase;
-import p.studio.utilities.structures.ReadOnlyCollection;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-
-public final class LocalizePhase implements DependencyPhase {
-    @Override
-    public ReadOnlyCollection<BuildingIssue> run(final DependencyContext ctx) {
-        final List<BuildingIssue> issues = new ArrayList<>();
-        for (int i = 0; i < ctx.projectDescriptors.size(); i++) {
-            final var fromProjectNode = ctx.projectDescriptors.get(i);
-            for (final var dependencyReference : fromProjectNode.getDependencies()) {
-                final var dependencyReferenceCanonPath = dependencyReference.canonPath();
-                final var projectId = ctx.projectTable.get(dependencyReferenceCanonPath);
-                if (Objects.isNull(projectId)) {
-                    final var issue = BuildingIssue
-                            .builder()
-                            .error(true)
-                            .message("[DEPS]: dependency not found: " + dependencyReferenceCanonPath)
-                            .build();
-                    issues.add(issue);
-                    continue;
-                }
-                ctx.dependenciesByProject.get(fromProjectNode.getProjectId().getIndex()).add(projectId);
-            }
-        }
-        return ReadOnlyCollection.wrap(issues);
-    }
-}

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/workspaces/phases/PolicyPhase.java

@@ -1,18 +0,0 @@
-package p.studio.compiler.workspaces.phases;
-
-import p.studio.compiler.messages.BuildingIssue;
-import p.studio.compiler.workspaces.DependencyPhase;
-import p.studio.compiler.models.DependencyContext;
-import p.studio.utilities.structures.ReadOnlyCollection;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public final class PolicyPhase implements DependencyPhase {
-    @Override
-    public ReadOnlyCollection<BuildingIssue> run(DependencyContext state) {
-        final List<BuildingIssue> issues = new ArrayList<>();
-        // No-op for now; in Rust this applies source policies
-        return ReadOnlyCollection.wrap(issues);
-    }
-}

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/workspaces/phases/StackPhase.java

@@ -22,7 +22,7 @@ public final class StackPhase implements DependencyPhase {
      */
     @Override
     public ReadOnlyCollection<BuildingIssue> run(final DependencyContext ctx) {
-        final int n = ctx.projectDescriptors.size();
+        final int n = ctx.projectTable.size();
         final int[] indeg = new int[n];
         for (int from = 0; from < n; from++) {
             for (final ProjectId to : ctx.dependenciesByProject.get(from)) {
@@ -39,10 +39,11 @@ public final class StackPhase implements DependencyPhase {
         // Performs topological sort using Kahn's algorithm
         while (!q.isEmpty()) {
             final int u = q.removeFirst();
-            travesalOrder.add(ctx.projectDescriptors.get(u).getProjectId());
+            //travesalOrder.add(ctx.projectDescriptors.get(u).getProjectId());
+            travesalOrder.add(new ProjectId(u));
             for (final ProjectId v : ctx.dependenciesByProject.get(u)) {
-                if (--indeg[(int) v.getId()] == 0) {
+                if (--indeg[v.getId()] == 0) {
-                    q.addLast((int) v.getId());
+                    q.addLast(v.getId());
                 }
             }
         }
@@ -57,15 +58,17 @@ public final class StackPhase implements DependencyPhase {
             final List<List<ProjectId>> cycles = new ArrayList<>();
             for (var scc : sccs) {
                 if (scc.size() > 1) {
-                    final var cycle = scc
+//                    final var cycle = scc
-                            .stream()
+//                            .stream()
-                            .map(i -> ctx.projectDescriptors.get(i).getProjectId())
+//                            .map(i -> ctx.projectDescriptors.get(i).getProjectId())
-                            .toList();
+//                            .toList();
+                    final var cycle = scc.stream().map(ProjectId::new).toList();
                     cycles.add(cycle);
                 } else {
                     // size==1: cycle only if self-loop exists
                     final var u = scc.getFirst();
-                    final var projectId = ctx.projectDescriptors.get(u).getProjectId();
+//                    final var projectId = ctx.projectDescriptors.get(u).getProjectId();
+                    final var projectId = new ProjectId(u);
                     boolean selfLoop = false;
                     for (final var pu : ctx.dependenciesByProject.get(u)) {
                         if (pu.getIndex() == u) {

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/workspaces/phases/ValidatePhase.java

@@ -10,6 +10,7 @@ import java.util.List;
 public final class ValidatePhase implements DependencyPhase {
     @Override
     public ReadOnlyCollection<BuildingIssue> run(DependencyContext ctx) {
+        // ensure rootProjectId is set
         if (ctx.rootProjectId == null) {
             final var issue = BuildingIssue
                     .builder()
@@ -19,17 +20,17 @@ public final class ValidatePhase implements DependencyPhase {
             return ReadOnlyCollection.wrap(List.of(issue));
         }
 
-        // Ensure the dependenciesByProjectId list matches the number of project descriptors
+        // ensure the dependenciesByProject matches the number of projectDescriptors
-        if (ctx.dependenciesByProject.size() != ctx.projectDescriptors.size()) {
+        if (ctx.dependenciesByProject.size() != ctx.projectTable.size()) {
             final var issue = BuildingIssue
                     .builder()
                     .error(true)
-                    .message("[DEPS]: internal error: dependenciesByProjectId list size mismatch")
+                    .message("[DEPS]: internal error: dependenciesByProject and projectDescriptors size mismatch")
                     .build();
             return ReadOnlyCollection.wrap(List.of(issue));
         }
 
-        // we should check and ensure uniformity to version across the same projects
+        // ensure uniformity to version across the same projects (associated by name)
         for (final var entry : ctx.projectNameAndVersions.entrySet()) {
             final var name = entry.getKey();
             final var versions = entry.getValue();
@@ -43,6 +44,8 @@ public final class ValidatePhase implements DependencyPhase {
             }
         }
 
+        // run check over source policy (if any) from here (FrontedSpec)
+
         return ReadOnlyCollection.empty();
     }
 }

prometeu-compiler/prometeu-deps/src/main/java/p/studio/compiler/workspaces/phases/WireProjectsPhase.java

@@ -15,25 +15,26 @@ import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 
-public final class MaterializePhase implements DependencyPhase {
+public final class WireProjectsPhase implements DependencyPhase {
     @Override
     public ReadOnlyCollection<BuildingIssue> run(final DependencyContext ctx) {
         // to start all over again, we will re-populate the project nodes and dependenciesByProjectId based on the project infos
         ctx.rootProjectId = null;
-        ctx.projectDescriptors.clear();
+        ctx.projectIds.clear();
         ctx.projectTable.clear();
         ctx.dependenciesByProject.clear();
 
         final List<BuildingIssue> issues = new ArrayList<>();
+
         for (int index = 0; index < ctx.projectInfoTable.size(); index++) {
             final var projectInfo = ctx.projectInfoTable.get(new ProjectInfoId(index));
-            final var projectId = ctx.projectTable.register(projectInfo.rootDirectory);
+            final var projectDescriptor = buildProjectDescriptor(projectInfo, issues);
-            ctx.projectDescriptors.add(buildProjectDescriptor(projectId, projectInfo, issues));
+            final var projectId = ctx.projectTable.register(projectDescriptor);
-            ctx.dependenciesByProject.add(new ArrayList<>());
+            ctx.projectIds.add(projectId);
         }
 
-        final var rootProjectId = ctx.projectTable.get(ctx.mainProjectRootPathCanon);
+        final var rootProjectId = ctx.projectTable.optionalId(ctx.mainProjectRootPathCanon);
-        if (rootProjectId == null) {
+        if (rootProjectId.isEmpty()) {
             final var issue = BuildingIssue
                     .builder()
                     .message("[DEPS]: rootProjectId project dir " + ctx.mainProjectRootPathCanon + " was not discovered/materialized")
@@ -42,13 +43,33 @@ public final class MaterializePhase implements DependencyPhase {
             return ReadOnlyCollection.wrap(issues);
         }
 
-        ctx.rootProjectId = rootProjectId;
+        // since it is not ordered, we have to iterate it over again, but now associating dependencies with projects
+        for (int i = 0; i < ctx.projectInfoTable.size(); i++) {
+            final var projectInfo = ctx.projectInfoTable.get(new ProjectInfoId(i));
+            final var dependencies = new ArrayList<ProjectId>();
+            for (final var dependency : projectInfo.manifest.dependencies()) {
+                final var dependencyCanonPath = dependency.canonPath();
+                final var projectId = ctx.projectTable.optionalId(dependencyCanonPath);
+                if (projectId.isEmpty()) {
+                    final var issue = BuildingIssue
+                            .builder()
+                            .error(true)
+                            .message("[DEPS]: dependency not found: " + dependencyCanonPath)
+                            .build();
+                    issues.add(issue);
+                    continue;
+                }
+                dependencies.add(projectId.get());
+            }
+            ctx.dependenciesByProject.add(dependencies);
+        }
+
+        ctx.rootProjectId = rootProjectId.get();
 
         return ReadOnlyCollection.wrap(issues);
     }
 
     private static ProjectDescriptor buildProjectDescriptor(
-            final ProjectId projectId,
             final ProjectInfo projectInfo,
             final List<BuildingIssue> issues) {
         final List<BuildingIssue> sourceRootIssues = new ArrayList<>();
@@ -62,7 +83,7 @@ public final class MaterializePhase implements DependencyPhase {
                 final var issue = BuildingIssue
                         .builder()
                         .error(true)
-                        .message("[DEPS]: source rootProjectId canonPath does not exist: " + sourceRootPath + " (from " + projectInfo.rootDirectory + ")")
+                        .message("[DEPS]: source project canonPath does not exist: " + sourceRootPath + " (from " + projectInfo.rootDirectory + ")")
                         .exception(e)
                         .build();
                 sourceRootIssues.add(issue);
@@ -75,13 +96,11 @@ public final class MaterializePhase implements DependencyPhase {
 
         return ProjectDescriptor
                 .builder()
-                .projectId(projectId)
+                .rootPath(projectInfo.rootDirectory)
-                .projectRootPath(projectInfo.rootDirectory)
                 .name(projectInfo.manifest.name())
                 .version(projectInfo.manifest.version())
                 .frontendSpec(projectInfo.getFrontendSpec())
                 .sourceRoots(ReadOnlyList.wrap(sourceRoots))
-                .dependencies(projectInfo.manifest.dependencies())
                 .build();
     }
 }