From 76b5717cd8027137b7243cedabc3473084e20d76 Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Fri, 10 Apr 2026 05:58:27 +0100 Subject: [PATCH] packer-pipeline-metadata-ownership --- discussion/index.ndjson | 3 +- ...ary-and-tooling-owned-pipeline-metadata.md | 71 +++++++++++++++++++ ...nd Virtual Asset Contract Specification.md | 5 +- ...and Deterministic Packing Specification.md | 10 ++- .../FileSystemPackerWorkspaceService.java | 37 +++++----- .../FileSystemPackerWorkspaceServiceTest.java | 6 +- 6 files changed, 107 insertions(+), 25 deletions(-) create mode 100644 discussion/lessons/DSC-0025-packer-pipeline-metadata-ownership/LSN-0039-runtime-header-boundary-and-tooling-owned-pipeline-metadata.md diff --git a/discussion/index.ndjson b/discussion/index.ndjson index e34032c3..f4bdf2ad 100644 --- a/discussion/index.ndjson +++ b/discussion/index.ndjson @@ -1,4 +1,5 @@ -{"type":"meta","next_id":{"DSC":24,"AGD":26,"DEC":24,"PLN":47,"LSN":38,"CLSN":1}} +{"type":"meta","next_id":{"DSC":25,"AGD":28,"DEC":25,"PLN":48,"LSN":40,"CLSN":1}} +{"type":"discussion","id":"DSC-0025","status":"done","ticket":"packer-pipeline-metadata-ownership","title":"Pipeline Metadata Ownership and Runtime Contract","created_at":"2026-04-09","updated_at":"2026-04-10","tags":["packer","metadata","runtime-contract","tooling","studio"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0039","file":"discussion/lessons/DSC-0025-packer-pipeline-metadata-ownership/LSN-0039-runtime-header-boundary-and-tooling-owned-pipeline-metadata.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} {"type":"discussion","id":"DSC-0024","status":"done","ticket":"jacoco-reports-consolidation","title":"JaCoCo Reports Consolidation in Gradle","created_at":"2026-04-07","updated_at":"2026-04-07","tags":["infra","gradle","jacoco","coverage","jenkins"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0038","file":"discussion/lessons/DSC-0024-jacoco-reports-consolidation/LSN-0038-jacoco-reports-consolidation.md","status":"done","created_at":"2026-04-07","updated_at":"2026-04-07"}]} {"type":"discussion","id":"DSC-0001","status":"done","ticket":"studio-docs-import","title":"Import docs/studio into discussion-framework artifacts","created_at":"2026-03-26","updated_at":"2026-03-26","tags":["studio","migration","discussion-framework","docs-import"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0001","file":"discussion/lessons/DSC-0001-studio-docs-import/LSN-0001-assets-workspace-execution-wave-legacy.md","status":"done","created_at":"2026-03-26","updated_at":"2026-03-26"},{"id":"LSN-0002","file":"discussion/lessons/DSC-0001-studio-docs-import/LSN-0002-bank-composition-editor-legacy.md","status":"done","created_at":"2026-03-26","updated_at":"2026-03-26"},{"id":"LSN-0003","file":"discussion/lessons/DSC-0001-studio-docs-import/LSN-0003-mental-model-asset-mutations-legacy.md","status":"done","created_at":"2026-03-26","updated_at":"2026-03-26"},{"id":"LSN-0004","file":"discussion/lessons/DSC-0001-studio-docs-import/LSN-0004-mental-model-assets-workspace-legacy.md","status":"done","created_at":"2026-03-26","updated_at":"2026-03-26"},{"id":"LSN-0005","file":"discussion/lessons/DSC-0001-studio-docs-import/LSN-0005-mental-model-studio-events-and-components-legacy.md","status":"done","created_at":"2026-03-26","updated_at":"2026-03-26"},{"id":"LSN-0006","file":"discussion/lessons/DSC-0001-studio-docs-import/LSN-0006-mental-model-studio-shell-legacy.md","status":"done","created_at":"2026-03-26","updated_at":"2026-03-26"},{"id":"LSN-0007","file":"discussion/lessons/DSC-0001-studio-docs-import/LSN-0007-pack-wizard-shell-legacy.md","status":"done","created_at":"2026-03-26","updated_at":"2026-03-26"},{"id":"LSN-0008","file":"discussion/lessons/DSC-0001-studio-docs-import/LSN-0008-project-scoped-state-and-activity-legacy.md","status":"done","created_at":"2026-03-26","updated_at":"2026-03-26"},{"id":"LSN-0016","file":"discussion/lessons/DSC-0001-studio-docs-import/LSN-0016-studio-docs-import-pattern.md","status":"done","created_at":"2026-03-26","updated_at":"2026-03-26"}]} {"type":"discussion","id":"DSC-0002","status":"open","ticket":"palette-management-in-studio","title":"Palette Management in Studio","created_at":"2026-03-26","updated_at":"2026-03-26","tags":["studio","legacy-import","palette-management","tile-bank","packer-boundary"],"agendas":[{"id":"AGD-0002","file":"AGD-0002-palette-management-in-studio.md","status":"open","created_at":"2026-03-26","updated_at":"2026-03-26"}],"decisions":[],"plans":[],"lessons":[]} diff --git a/discussion/lessons/DSC-0025-packer-pipeline-metadata-ownership/LSN-0039-runtime-header-boundary-and-tooling-owned-pipeline-metadata.md b/discussion/lessons/DSC-0025-packer-pipeline-metadata-ownership/LSN-0039-runtime-header-boundary-and-tooling-owned-pipeline-metadata.md new file mode 100644 index 00000000..b78c27cb --- /dev/null +++ b/discussion/lessons/DSC-0025-packer-pipeline-metadata-ownership/LSN-0039-runtime-header-boundary-and-tooling-owned-pipeline-metadata.md @@ -0,0 +1,71 @@ +--- +id: LSN-0039 +ticket: packer-pipeline-metadata-ownership +title: Runtime Header Boundary and Tooling-Owned Pipeline Metadata +created: 2026-04-10 +tags: [packer, metadata, runtime-contract, tooling, codec, assets-pa] +--- + +## Context + +The packer had drift between implementation and spec around `output.pipeline` ownership. The runtime-facing artifact `assets.pa` was documented as if pipeline metadata should materialize under `asset_table[].metadata.pipeline`, while the implementation already treated at least part of that surface as tooling-oriented and excluded palette payloads from the runtime header. + +This created the wrong architectural pressure: Studio and third-party tools could start depending on `assets.pa` for authoring/build provenance, and runtime-facing specs could silently absorb build-time structures that do not belong in the execution contract. + +The discussion resolved that split explicitly and implemented it end to end in specs, packer materialization, and tests. + +## Key Decisions + +### Tooling Metadata Must Not Leak Into the Runtime Header + +**What:** +`output.pipeline` remains tooling-only metadata in `asset.json` and related sidecar outputs. The runtime-facing `assets.pa` header must not mirror `output.pipeline` by default. Runtime-owned metadata remains under explicit, normalized fields in `asset_table[].metadata`. + +**Why:** +`assets.pa` is the authoritative runtime artifact. If it starts carrying build provenance, editor-oriented structures, or reverse-sync payloads merely because they exist in the authoring manifest, the runtime contract becomes larger, noisier, and harder to evolve safely. + +**Trade-offs:** +Tools that want provenance cannot rely on one binary artifact alone. They must read `asset.json` and/or `build/asset_table_metadata.json`. That is a deliberate trade-off in favor of a cleaner runtime boundary. + +### Promotion Must Be Explicit, Not a Bulk Mirror + +**What:** +If pipeline-derived information is needed at runtime, it must be promoted into an explicit normative field under runtime-owned metadata. The packer must not reintroduce a generic mirrored `pipeline` object under another name. + +**Why:** +Bulk mirroring hides product decisions inside implementation convenience. Explicit promotion forces each field to justify its presence in the runtime contract. + +**Trade-offs:** +This adds small editorial and implementation work whenever a new runtime-facing derived field is introduced, but it keeps the contract intentional and testable. + +### Codec Serialization Must Stay in `SCREAMING_SNAKE_CASE` + +**What:** +Serialized asset entry `codec` values are locked to `SCREAMING_SNAKE_CASE`. The currently known `NONE` value remains serialized as `NONE`. + +**Why:** +The implementation already had a stable manifest/runtime representation through `OutputCodecCatalog`. Recording that casing in spec prevents future drift when more codecs are added. + +**Trade-offs:** +Future codec additions must preserve the established serialized style unless a new decision explicitly revises the contract. + +## Patterns and Algorithms + +- Use separate surfaces for separate consumers: `assets.pa` for runtime contract, `asset_table_metadata.json` for tooling-oriented provenance, and `asset.json` for authoring intent. +- Build runtime metadata from explicit normalized fields first, then derive tooling sidecars from a superset when needed. +- Keep the runtime header free of raw authoring/build objects even if the sidecar keeps them intact for Studio or inspection tools. +- Lock serialized enum casing in spec as soon as the first value ships, even if the current catalog is minimal. + +## Pitfalls + +- Do not treat “already present in the manifest” as sufficient reason to serialize a field into the runtime header. +- Do not let sidecar convenience become implicit runtime compatibility. +- Do not partially filter a tooling object and leave an empty placeholder in runtime metadata unless that placeholder has explicit runtime meaning. +- Do not leave enum casing as an implementation accident. If the manifest value matters, the spec must say so. + +## Takeaways + +- The runtime header is a contract surface, not a metadata dump. +- Tooling provenance belongs in `asset.json` and `build/asset_table_metadata.json`, not in `assets.pa`. +- Runtime-facing derived metadata must be promoted explicitly field by field. +- Stable serialized casing rules should be fixed in spec before the value set grows. diff --git a/docs/specs/packer/3. Asset Declaration and Virtual Asset Contract Specification.md b/docs/specs/packer/3. Asset Declaration and Virtual Asset Contract Specification.md index a22bee4f..54a9a5a1 100644 --- a/docs/specs/packer/3. Asset Declaration and Virtual Asset Contract Specification.md +++ b/docs/specs/packer/3. Asset Declaration and Virtual Asset Contract Specification.md @@ -92,8 +92,9 @@ Rules: - `output.format` defines the semantic/runtime format contract; - `output.codec` defines storage/extraction behavior; +- `output.codec` serialized values must use `SCREAMING_SNAKE_CASE`; - `output.metadata` carries runtime-relevant format-specific detail; -- `output.pipeline` carries pipeline-injected metadata kept separate at authoring time; +- `output.pipeline` carries tooling/build metadata kept separate at authoring time; - codec must remain explicit and must not be hidden inside format naming. ### Explicit Index Collections @@ -139,6 +140,8 @@ Rules: - declaration-time metadata may come from multiple sources under the asset contract (for example, format metadata, codec-related metadata, and build/pipeline-derived declarations); - `output.pipeline` may carry nested pipeline-derived metadata objects; - this segmentation exists for authoring clarity and does not define multiple runtime sinks; +- `output.pipeline` is tooling-only and must not become part of the runtime-facing asset header by default; +- pipeline-derived values required at runtime must be promoted explicitly into normative runtime-owned metadata fields; - runtime consumers must read effective metadata from the runtime asset entry metadata sink (`AssetEntry.metadata`); - convergence/normalization behavior is normative in the build artifact specification. diff --git a/docs/specs/packer/4. Build Artifacts and Deterministic Packing Specification.md b/docs/specs/packer/4. Build Artifacts and Deterministic Packing Specification.md index 10fcb271..4e35b8de 100644 --- a/docs/specs/packer/4. Build Artifacts and Deterministic Packing Specification.md +++ b/docs/specs/packer/4. Build Artifacts and Deterministic Packing Specification.md @@ -95,9 +95,15 @@ Baseline normalized metadata segmentation: - `output.metadata` materializes at the metadata root; - `output.codec_configuration` materializes under `metadata.codec`; -- `output.pipeline` materializes under `metadata.pipeline`; +- `output.pipeline` remains tooling-only and must not materialize under `asset_table[].metadata` by default; +- pipeline-derived runtime-required values must be promoted explicitly into normative runtime-owned metadata fields; - format-specific runtime-required fields may remain directly readable at the metadata root when the runtime consumer requires them there. +Asset entry codec serialization: + +- `asset_table[].codec` MUST use `SCREAMING_SNAKE_CASE`; +- the current known `NONE` value serializes as `NONE`. + ### Preload Preload is emitted deterministically from per-asset declaration. @@ -120,7 +126,7 @@ Rules: - `build/asset_table.json` mirrors `header.asset_table` 1:1; - `build/preload.json` mirrors `header.preload` 1:1; -- `build/asset_table_metadata.json` is tooling-only; +- `build/asset_table_metadata.json` is tooling-only and may retain pipeline/build provenance that is intentionally excluded from the runtime header; - richer tooling data must not be added to the 1:1 mirror files. ## Alignment and Offsets diff --git a/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/services/FileSystemPackerWorkspaceService.java b/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/services/FileSystemPackerWorkspaceService.java index f40e7545..850f2915 100644 --- a/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/services/FileSystemPackerWorkspaceService.java +++ b/prometeu-packer/prometeu-packer-v1/src/main/java/p/packer/services/FileSystemPackerWorkspaceService.java @@ -878,7 +878,7 @@ public final class FileSystemPackerWorkspaceService implements PackerWorkspaceSe "size", packedAsset.payload().length, "decoded_size", packedAsset.decodedSize(), "codec", packedAsset.codec(), - "metadata", packedAsset.metadata()))); + "metadata", packedAsset.runtimeMetadata()))); if (packedAsset.preloadEnabled()) { final int slot = nextPreloadSlotByBankType.getOrDefault(packedAsset.bankType(), 0); nextPreloadSlotByBankType.put(packedAsset.bankType(), slot + 1); @@ -888,7 +888,7 @@ public final class FileSystemPackerWorkspaceService implements PackerWorkspaceSe } assetTableMetadata.add(new LinkedHashMap<>(Map.of( "asset_id", packedAsset.assetId(), - "metadata", packedAsset.metadata()))); + "metadata", packedAsset.toolingMetadata()))); } final byte[] headerBytes = canonicalJsonBytes(Map.of( @@ -960,20 +960,22 @@ public final class FileSystemPackerWorkspaceService implements PackerWorkspaceSe System.arraycopy(packedPixels, 0, payload, 0, packedPixels.length); System.arraycopy(paletteBytes, 0, payload, packedPixels.length, paletteBytes.length); - final LinkedHashMap metadata = new LinkedHashMap<>(); - metadata.put("tile_size", tileSize); - metadata.put("width", width); - metadata.put("height", height); - metadata.put("palette_count", 64); - metadata.put("palette_authored", countAuthoredTileBankPalettes(declaration)); - metadata.put("codec", Map.of()); - metadata.put("pipeline", normalizeTileBankRuntimePipelineMetadata(declaration)); + final LinkedHashMap runtimeMetadata = new LinkedHashMap<>(); + runtimeMetadata.put("tile_size", tileSize); + runtimeMetadata.put("width", width); + runtimeMetadata.put("height", height); + runtimeMetadata.put("palette_count", 64); + runtimeMetadata.put("palette_authored", countAuthoredTileBankPalettes(declaration)); + runtimeMetadata.put("codec", Map.of()); declaration.outputMetadata().forEach((key, value) -> { if (!"tile_size".equals(key)) { - metadata.putIfAbsent(key, value); + runtimeMetadata.putIfAbsent(key, value); } }); + final LinkedHashMap toolingMetadata = new LinkedHashMap<>(runtimeMetadata); + toolingMetadata.put("pipeline", normalizeToolingPipelineMetadata(declaration)); + final PackerRegistryEntry registryEntry = runtimeAsset.registryEntry() .orElseThrow(() -> new IllegalStateException("Packed runtime asset must be registered")); return new PackedAsset( @@ -983,7 +985,8 @@ public final class FileSystemPackerWorkspaceService implements PackerWorkspaceSe "NONE", payload, width * height + 2048, - metadata, + runtimeMetadata, + toolingMetadata, declaration.preloadEnabled()); } @@ -1044,11 +1047,8 @@ public final class FileSystemPackerWorkspaceService implements PackerWorkspaceSe return palettesNode instanceof ArrayNode palettesArray ? palettesArray.size() : 0; } - private Map normalizeTileBankRuntimePipelineMetadata(PackerAssetDeclaration declaration) { - final LinkedHashMap pipeline = new LinkedHashMap<>( - PackerReadMessageMapper.normalizeMetadata(declaration.outputPipelineMetadata())); - pipeline.remove("palettes"); - return pipeline; + private Map normalizeToolingPipelineMetadata(PackerAssetDeclaration declaration) { + return new LinkedHashMap<>(PackerReadMessageMapper.normalizeMetadata(declaration.outputPipelineMetadata())); } private byte[] buildPrelude(int headerLength) { @@ -1111,7 +1111,8 @@ public final class FileSystemPackerWorkspaceService implements PackerWorkspaceSe String codec, byte[] payload, int decodedSize, - Map metadata, + Map runtimeMetadata, + Map toolingMetadata, boolean preloadEnabled) { } diff --git a/prometeu-packer/prometeu-packer-v1/src/test/java/p/packer/services/FileSystemPackerWorkspaceServiceTest.java b/prometeu-packer/prometeu-packer-v1/src/test/java/p/packer/services/FileSystemPackerWorkspaceServiceTest.java index cc9caef6..c63e9ccb 100644 --- a/prometeu-packer/prometeu-packer-v1/src/test/java/p/packer/services/FileSystemPackerWorkspaceServiceTest.java +++ b/prometeu-packer/prometeu-packer-v1/src/test/java/p/packer/services/FileSystemPackerWorkspaceServiceTest.java @@ -337,8 +337,7 @@ final class FileSystemPackerWorkspaceServiceTest { assertEquals(256, assetTable.get(0).path("metadata").path("height").asInt()); assertEquals(64, assetTable.get(0).path("metadata").path("palette_count").asInt()); assertEquals(1, assetTable.get(0).path("metadata").path("palette_authored").asInt()); - assertTrue(assetTable.get(0).path("metadata").path("pipeline").isObject()); - assertTrue(assetTable.get(0).path("metadata").path("pipeline").path("palettes").isMissingNode()); + assertTrue(assetTable.get(0).path("metadata").path("pipeline").isMissingNode()); final var preload = MAPPER.readTree(projectRoot.resolve("build/preload.json").toFile()); assertEquals(1, preload.size()); @@ -348,7 +347,8 @@ final class FileSystemPackerWorkspaceServiceTest { final var assetTableMetadata = MAPPER.readTree(projectRoot.resolve("build/asset_table_metadata.json").toFile()); assertEquals(1, assetTableMetadata.size()); assertEquals(1, assetTableMetadata.get(0).path("metadata").path("palette_authored").asInt()); - assertTrue(assetTableMetadata.get(0).path("metadata").path("pipeline").path("palettes").isMissingNode()); + assertEquals(1, assetTableMetadata.get(0).path("metadata").path("pipeline").path("palettes").size()); + assertEquals(0, assetTableMetadata.get(0).path("metadata").path("pipeline").path("palettes").get(0).path("index").asInt()); } @Test