diff --git a/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/models/PackerFileProbe.java b/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/models/PackerFileProbe.java index 6d234389..4bba69e8 100644 --- a/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/models/PackerFileProbe.java +++ b/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/models/PackerFileProbe.java @@ -5,6 +5,7 @@ import java.nio.file.Path; public record PackerFileProbe( Path path, String mimeType, + long size, long lastModified, byte[] content) { } diff --git a/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/repositories/PackerAbstractBankWalker.java b/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/repositories/PackerAbstractBankWalker.java index 4bc8dcdf..5741174b 100644 --- a/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/repositories/PackerAbstractBankWalker.java +++ b/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/repositories/PackerAbstractBankWalker.java @@ -4,7 +4,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.tika.Tika; import p.packer.messages.diagnostics.PackerDiagnosticCategory; import p.packer.messages.diagnostics.PackerDiagnosticSeverity; +import p.packer.models.PackerAssetCacheEntry; import p.packer.models.PackerDiagnostic; +import p.packer.models.PackerFileCacheEntry; import p.packer.models.PackerFileProbe; import p.packer.models.PackerProbeResult; import p.packer.models.PackerWalkResult; @@ -12,8 +14,12 @@ import p.packer.models.PackerWalkResult; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -37,7 +43,7 @@ public abstract class PackerAbstractBankWalker { return Optional.empty(); } final var lastModified = filePath.toFile().lastModified(); - final var fileProbe = new PackerFileProbe(filePath, mimeType, lastModified, bytes); + final var fileProbe = new PackerFileProbe(filePath, mimeType, bytes.length, lastModified, bytes); return Optional.of(fileProbe); } @@ -53,11 +59,13 @@ public abstract class PackerAbstractBankWalker { return path.getFileName().toString().equalsIgnoreCase("asset.json"); } - private List processFileProbeWhenSupported( + private List processFileProbeWhenSupported( final Path assetRoot, + final Optional priorAssetCache, + final R requirements, final Set supportedMimeTypes, final List diagnostics) { - final List supportedFileProbes = new ArrayList<>(); + final List probeResults = new ArrayList<>(); try { final var validPaths = retrieveValidPaths(assetRoot); for (final var filePath : validPaths) { @@ -66,7 +74,12 @@ public abstract class PackerAbstractBankWalker { continue; } final var fileProbe = fileProbeMaybe.get(); - supportedFileProbes.add(fileProbe); + final var cachedEntry = resolvePriorFileCache(assetRoot, fileProbe.path(), priorAssetCache); + if (cachedEntry.isPresent() && canReuseMetadata(fileProbe, cachedEntry.get())) { + probeResults.add(new PackerProbeResult(fileProbe, cachedEntry.get().metadata(), List.of())); + continue; + } + probeResults.add(processFileProbe(fileProbe, requirements)); } } catch (IOException exception) { diagnostics.add(new PackerDiagnostic( @@ -77,16 +90,68 @@ public abstract class PackerAbstractBankWalker { true )); } - return supportedFileProbes; + return probeResults; } public PackerWalkResult walk( final Path assetRoot, final R requirements) { + return walk(assetRoot, requirements, Optional.empty()); + } + + public PackerWalkResult walk( + final Path assetRoot, + final R requirements, + final Optional priorAssetCache) { final List rootDiagnostics = new ArrayList<>(); - final List probeResults = new ArrayList<>(); - processFileProbeWhenSupported(assetRoot, getSupportedMimeTypes(), rootDiagnostics) - .forEach(fileProbe -> probeResults.add(processFileProbe(fileProbe, requirements))); + final List probeResults = processFileProbeWhenSupported( + assetRoot, + Objects.requireNonNull(priorAssetCache, "priorAssetCache"), + requirements, + getSupportedMimeTypes(), + rootDiagnostics); return new PackerWalkResult(probeResults, rootDiagnostics); } + + protected boolean canReuseMetadata( + final PackerFileProbe fileProbe, + final PackerFileCacheEntry cachedEntry) { + if (fileProbe.size() != cachedEntry.size()) { + return false; + } + if (fileProbe.lastModified() > cachedEntry.lastModified()) { + return false; + } + if (cachedEntry.mimeType() != null && !cachedEntry.mimeType().equals(fileProbe.mimeType())) { + return false; + } + if (cachedEntry.fingerprint() == null || cachedEntry.fingerprint().isBlank()) { + return true; + } + return cachedEntry.fingerprint().equals(computeFingerprint(fileProbe)); + } + + protected String computeFingerprint(final PackerFileProbe fileProbe) { + try { + final MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return HexFormat.of().formatHex(digest.digest(fileProbe.content())); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 fingerprint is unavailable", exception); + } + } + + private Optional resolvePriorFileCache( + final Path assetRoot, + final Path filePath, + final Optional priorAssetCache) { + if (priorAssetCache.isEmpty()) { + return Optional.empty(); + } + final String relativePath = assetRoot.toAbsolutePath() + .normalize() + .relativize(filePath.toAbsolutePath().normalize()) + .toString() + .replace('\\', '/'); + return priorAssetCache.get().findFile(relativePath); + } } diff --git a/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/repositories/PackerAssetWalker.java b/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/repositories/PackerAssetWalker.java index dff61f35..2233b697 100644 --- a/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/repositories/PackerAssetWalker.java +++ b/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/repositories/PackerAssetWalker.java @@ -12,6 +12,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; public class PackerAssetWalker { @@ -34,6 +35,13 @@ public class PackerAssetWalker { public PackerWalkResult walk( final Path assetRoot, final PackerAssetDeclaration declaration) { + return walk(assetRoot, declaration, Optional.empty()); + } + + public PackerWalkResult walk( + final Path assetRoot, + final PackerAssetDeclaration declaration, + final Optional priorAssetCache) { final List diagnostics = new ArrayList<>(); switch (declaration.assetFamily()) { case TILE_BANK -> { @@ -57,7 +65,7 @@ public class PackerAssetWalker { true)); return new PackerWalkResult(List.of(), diagnostics); } - final var walkResult = tileBankWalker.walk(assetRoot, requirementBuildResult.requirements); + final var walkResult = tileBankWalker.walk(assetRoot, requirementBuildResult.requirements, priorAssetCache); diagnostics.addAll(walkResult.diagnostics()); return new PackerWalkResult(walkResult.probeResults(), diagnostics); } @@ -82,7 +90,7 @@ public class PackerAssetWalker { true)); return new PackerWalkResult(List.of(), diagnostics); } - final var walkResult = soundBankWalker.walk(assetRoot, result.requirements); + final var walkResult = soundBankWalker.walk(assetRoot, result.requirements, priorAssetCache); diagnostics.addAll(walkResult.diagnostics()); return new PackerWalkResult(walkResult.probeResults(), diagnostics); } diff --git a/prometeu-packer/prometeu-packer-v1/src/test/java/p/packer/repositories/PackerAbstractBankWalkerTest.java b/prometeu-packer/prometeu-packer-v1/src/test/java/p/packer/repositories/PackerAbstractBankWalkerTest.java new file mode 100644 index 00000000..b7d78a11 --- /dev/null +++ b/prometeu-packer/prometeu-packer-v1/src/test/java/p/packer/repositories/PackerAbstractBankWalkerTest.java @@ -0,0 +1,177 @@ +package p.packer.repositories; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import p.packer.models.*; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.*; + +final class PackerAbstractBankWalkerTest { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @TempDir + Path tempDir; + + @Test + void invalidatesReuseImmediatelyWhenSizeChanges() throws Exception { + final var walker = new CountingWalker(); + final var assetRoot = Files.createDirectories(tempDir.resolve("asset")); + final var filePath = writeText(assetRoot.resolve("entry.txt"), "hello world"); + final long lastModified = Files.getLastModifiedTime(filePath).toMillis(); + final var priorCache = new PackerAssetCacheEntry(1, List.of( + new PackerFileCacheEntry("entry.txt", "text/plain", 1L, lastModified, "cached", Map.of("source", "cache")))); + + final var result = walker.walk(assetRoot, "requirements", java.util.Optional.of(priorCache)); + + assertEquals(1, walker.processCount()); + assertEquals(0, walker.fingerprintCount()); + assertEquals(Boolean.TRUE, result.probeResults().getFirst().metadata().get("processed")); + } + + @Test + void invalidatesReuseImmediatelyWhenCurrentLastModifiedIsNewer() throws Exception { + final var walker = new CountingWalker(); + final var assetRoot = Files.createDirectories(tempDir.resolve("asset")); + final var filePath = writeText(assetRoot.resolve("entry.txt"), "hello world"); + final long currentLastModified = Files.getLastModifiedTime(filePath).toMillis(); + final var priorCache = new PackerAssetCacheEntry(1, List.of( + new PackerFileCacheEntry( + "entry.txt", + "text/plain", + Files.size(filePath), + currentLastModified - 1L, + "cached", + Map.of("source", "cache")))); + + final var result = walker.walk(assetRoot, "requirements", java.util.Optional.of(priorCache)); + + assertEquals(1, walker.processCount()); + assertEquals(0, walker.fingerprintCount()); + assertEquals(Boolean.TRUE, result.probeResults().getFirst().metadata().get("processed")); + } + + @Test + void evaluatesFingerprintOnlyAfterCheaperChecksRemainValid() throws Exception { + final var walker = new CountingWalker(); + final var assetRoot = Files.createDirectories(tempDir.resolve("asset")); + final var filePath = writeText(assetRoot.resolve("entry.txt"), "hello world"); + final long lastModified = Files.getLastModifiedTime(filePath).toMillis(); + final String fingerprint = sha256(Files.readAllBytes(filePath)); + final var priorCache = new PackerAssetCacheEntry(1, List.of( + new PackerFileCacheEntry( + "entry.txt", + "text/plain", + Files.size(filePath), + lastModified, + fingerprint, + Map.of("source", "cache")))); + + final var result = walker.walk(assetRoot, "requirements", java.util.Optional.of(priorCache)); + + assertEquals(0, walker.processCount()); + assertEquals(1, walker.fingerprintCount()); + assertEquals("cache", result.probeResults().getFirst().metadata().get("source")); + } + + @Test + void reusesStableMetadataWithoutFingerprintWhenCheaperChecksStayValid() throws Exception { + final var walker = new CountingWalker(); + final var assetRoot = Files.createDirectories(tempDir.resolve("asset")); + final var filePath = writeText(assetRoot.resolve("entry.txt"), "hello world"); + final long lastModified = Files.getLastModifiedTime(filePath).toMillis(); + final var priorCache = new PackerAssetCacheEntry(1, List.of( + new PackerFileCacheEntry( + "entry.txt", + "text/plain", + Files.size(filePath), + lastModified, + null, + Map.of("source", "cache")))); + + final var result = walker.walk(assetRoot, "requirements", java.util.Optional.of(priorCache)); + + assertEquals(0, walker.processCount()); + assertEquals(0, walker.fingerprintCount()); + assertEquals("cache", result.probeResults().getFirst().metadata().get("source")); + } + + @Test + void preservesFileScopedDiagnosticsInFreshWalkOutput() throws Exception { + final var walker = new CountingWalker(); + final var assetRoot = Files.createDirectories(tempDir.resolve("asset")); + writeText(assetRoot.resolve("entry.txt"), "hello world"); + + final var result = walker.walk(assetRoot, "requirements"); + + assertEquals(1, result.probeResults().size()); + assertEquals(1, result.probeResults().getFirst().diagnostics().size()); + assertTrue(result.probeResults().getFirst().diagnostics().getFirst().message().contains("file-scoped")); + } + + private Path writeText(Path path, String content) throws Exception { + Files.createDirectories(path.getParent()); + Files.writeString(path, content); + Files.setLastModifiedTime(path, FileTime.fromMillis(Math.max(1L, Files.getLastModifiedTime(path).toMillis()))); + return path; + } + + private static String sha256(byte[] bytes) throws Exception { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); + } + + private static final class CountingWalker extends PackerAbstractBankWalker { + private int processCount; + private int fingerprintCount; + + private CountingWalker() { + super(MAPPER); + } + + @Override + protected Set getSupportedMimeTypes() { + return Set.of("text/plain"); + } + + @Override + protected PackerProbeResult processFileProbe(PackerFileProbe fileProbe, String requirements) { + processCount += 1; + return new PackerProbeResult( + fileProbe, + Map.of("processed", true), + List.of(new PackerDiagnostic( + p.packer.messages.diagnostics.PackerDiagnosticSeverity.WARNING, + p.packer.messages.diagnostics.PackerDiagnosticCategory.HYGIENE, + "file-scoped diagnostic", + fileProbe.path(), + false))); + } + + @Override + protected String computeFingerprint(PackerFileProbe fileProbe) { + fingerprintCount += 1; + try { + return sha256(fileProbe.content()); + } catch (Exception exception) { + throw new RuntimeException(exception); + } + } + + private int processCount() { + return processCount; + } + + private int fingerprintCount() { + return fingerprintCount; + } + } +}