From 8f37d971cda41a5918eb4731c7b7ab12dd6f78c8 Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Sun, 28 Jun 2026 09:35:08 +0100 Subject: [PATCH 1/9] [PERF] Async Background Work Lanes for Assets and FS --- discussion/index.ndjson | 6 +- ...background-work-lanes-for-assets-and-fs.md | 564 ++++++++++++++++-- ...f-gfx-render-pipeline-and-dirty-regions.md | 409 ------------- ...nc-work-lane-and-asset-backlog-contract.md | 336 +++++++++++ ...ync-work-lane-specification-propagation.md | 118 ++++ ...-async-work-lane-runtime-infrastructure.md | 119 ++++ ...5-asset-backlog-and-stable-slot-handles.md | 127 ++++ ...t-backlog-public-api-and-status-surface.md | 124 ++++ ...e-memcard-and-fs-integration-boundaries.md | 120 ++++ ...t-pipeline-contract-tests-and-telemetry.md | 126 ++++ 10 files changed, 1591 insertions(+), 458 deletions(-) delete mode 100644 discussion/workflow/agendas/AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md create mode 100644 discussion/workflow/decisions/DEC-0034-async-work-lane-and-asset-backlog-contract.md create mode 100644 discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md create mode 100644 discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md create mode 100644 discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md create mode 100644 discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md create mode 100644 discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md create mode 100644 discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md diff --git a/discussion/index.ndjson b/discussion/index.ndjson index f2a5c205..eb2097e5 100644 --- a/discussion/index.ndjson +++ b/discussion/index.ndjson @@ -1,4 +1,4 @@ -{"type":"meta","next_id":{"DSC":43,"AGD":44,"DEC":34,"PLN":123,"LSN":50,"CLSN":1}} +{"type":"meta","next_id":{"DSC":43,"AGD":44,"DEC":35,"PLN":129,"LSN":50,"CLSN":1}} {"type":"discussion","id":"DSC-0039","status":"abandoned","ticket":"render-pipeline-family-and-future-3d","title":"Render Pipeline Family and Future 3D","created_at":"2026-06-04","updated_at":"2026-06-04","tags":["gfx","renderer","runtime","architecture","pipeline"],"agendas":[{"id":"AGD-0039","file":"AGD-0039-render-pipeline-family-and-future-3d.md","status":"abandoned","created_at":"2026-06-04","updated_at":"2026-06-04","_override_reason":"User explicitly chose to close this agenda without a new decision because DSC-0038 already established enough architecture for future extension, and 3D is intentionally deferred."}],"decisions":[],"plans":[],"lessons":[],"_override_reason":"User explicitly chose to close this agenda without a new decision because DSC-0038 already established enough architecture for future extension, and 3D is intentionally deferred."} {"type":"discussion","id":"DSC-0040","status":"done","ticket":"vm-render-parallel-execution-boundary","title":"VM and Render Parallel Execution Boundary","created_at":"2026-06-04","updated_at":"2026-06-06","tags":["runtime","renderer","vm","concurrency","architecture","perf"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0048","file":"discussion/lessons/DSC-0040-vm-render-parallel-execution-boundary/LSN-0048-render-workers-need-a-closed-packet-contract.md","status":"done","created_at":"2026-06-06","updated_at":"2026-06-06"}]} {"type":"discussion","id":"DSC-0041","status":"open","ticket":"foreground-stack-game-pause-shell-vm-backed","title":"Foreground Stack, Game Pause, and VM-Backed Shell Coexistence","created_at":"2026-06-05","updated_at":"2026-06-05","tags":["runtime","os","lifecycle","shell","game","vm","foreground","architecture"],"agendas":[{"id":"AGD-0041","file":"AGD-0041-foreground-stack-game-pause-shell-vm-backed.md","status":"open","created_at":"2026-06-05","updated_at":"2026-06-05"}],"decisions":[],"plans":[],"lessons":[]} @@ -18,9 +18,9 @@ {"type":"discussion","id":"DSC-0006","status":"open","ticket":"vm-owned-random-service","title":"Agenda - VM-Owned Random Service","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0005","file":"AGD-0005-vm-owned-random-service.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0007","status":"open","ticket":"app-home-filesystem-surface-and-semantics","title":"Agenda - App Home Filesystem Surface and Semantics","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0006","file":"AGD-0006-app-home-filesystem-surface-and-semantics.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0008","status":"done","ticket":"perf-runtime-telemetry-hot-path","title":"Agenda - [PERF] Runtime Telemetry Hot Path","created_at":"2026-03-27","updated_at":"2026-06-04","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0026","file":"discussion/lessons/DSC-0008-perf-runtime-telemetry-hot-path/LSN-0026-push-based-telemetry-model.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} -{"type":"discussion","id":"DSC-0009","status":"open","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} +{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} {"type":"discussion","id":"DSC-0010","status":"done","ticket":"perf-host-desktop-frame-pacing-and-presentation","title":"Agenda - [PERF] Host Desktop Frame Pacing and Presentation","created_at":"2026-03-27","updated_at":"2026-04-20","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0036","file":"discussion/lessons/DSC-0010-perf-host-desktop-frame-pacing-and-presentation/LSN-0036-frame-publication-and-host-invalidation-must-be-separate.md","status":"done","created_at":"2026-04-20","updated_at":"2026-04-20"}]} -{"type":"discussion","id":"DSC-0011","status":"open","ticket":"perf-gfx-render-pipeline-and-dirty-regions","title":"Agenda - [PERF] GFX Render Pipeline and Dirty Regions","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0010","file":"AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} +{"type":"discussion","id":"DSC-0011","status":"abandoned","ticket":"perf-gfx-render-pipeline-and-dirty-regions","title":"Agenda - [PERF] GFX Render Pipeline and Dirty Regions","created_at":"2026-03-27","updated_at":"2026-06-28","tags":[],"agendas":[{"id":"AGD-0010","file":"AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md","status":"abandoned","created_at":"2026-03-27","updated_at":"2026-06-28","_override_reason":"User explicitly chose to remove this agenda because it no longer applies to the current render scenario; future optimizations can be discussed when needed."}],"decisions":[],"plans":[],"lessons":[],"_override_reason":"User explicitly chose to abandon this discussion because it no longer applies to the current render scenario; future optimizations can be discussed when needed."} {"type":"discussion","id":"DSC-0012","status":"done","ticket":"perf-runtime-introspection-syscalls","title":"Agenda - [PERF] Runtime Introspection Syscalls","created_at":"2026-03-27","updated_at":"2026-04-19","tags":["perf","runtime","syscall","telemetry","debug","asset"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0034","file":"discussion/lessons/DSC-0012-perf-runtime-introspection-syscalls/LSN-0034-host-owned-debug-boundaries.md","status":"done","created_at":"2026-04-19","updated_at":"2026-04-19"}]} {"type":"discussion","id":"DSC-0013","status":"done","ticket":"perf-host-debug-overlay-isolation","title":"Agenda - [PERF] Host Debug Overlay Isolation","created_at":"2026-03-27","updated_at":"2026-04-10","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0027","file":"discussion/lessons/DSC-0013-perf-host-debug-overlay-isolation/LSN-0027-host-debug-overlay-isolation.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} {"type":"discussion","id":"DSC-0024","status":"done","ticket":"generic-memory-bank-slot-contract","title":"Agenda - Generic Memory Bank Slot Contract","created_at":"2026-04-10","updated_at":"2026-04-10","tags":["runtime","asset","memory-bank","slots","host"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0029","file":"discussion/lessons/DSC-0024-generic-memory-bank-slot-contract/LSN-0029-slot-first-bank-telemetry-belongs-in-asset-manager.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} diff --git a/discussion/workflow/agendas/AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md b/discussion/workflow/agendas/AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md index 28eee281..a7dc7a2e 100644 --- a/discussion/workflow/agendas/AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md +++ b/discussion/workflow/agendas/AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md @@ -2,84 +2,556 @@ id: AGD-0008 ticket: perf-async-background-work-lanes-for-assets-and-fs title: Agenda - [PERF] Async Background Work Lanes for Assets and FS -status: open +status: accepted created: 2026-03-27 -resolved: -decision: -tags: [] +resolved: 2026-06-28 +decision: DEC-0034 +tags: [perf, asset, fs, async, scheduler, runtime] --- # Agenda - [PERF] Async Background Work Lanes for Assets and FS +## Contexto + +Depois das decisoes de render worker, a concorrencia de render saiu do escopo +desta agenda. O problema restante e a lane de trabalho assincrono para IO, +decode e persistencia. + +O modelo de execucao que queremos discutir agora assume uma terceira core/lane +dedicada a trabalhos assincronos de runtime: + +```text +Core/lane 1: VM, firmware, SystemOS e frame logico +Core/lane 2: render worker / rasterizacao assincrona +Core/lane 3: async work lane para asset IO/decode e possivelmente FS +``` + +Essa terceira lane nao deve virar um pool grande e solto. A intencao e ter uma +fila previsivel, com capacidade limitada, telemetria e regras de saturacao. + +Para assets, a necessidade e concreta: `asset.load()` ainda cria uma thread do +SO por requisicao. Para FS, a motivacao provavel era evitar que operacoes de +leitura/escrita/listagem/delete, hoje chamadas de forma sincrona pelo dispatch, +roubem tempo do frame quando o backend real envolver IO. Precisamos verificar +se FS deve usar a mesma lane agora ou apenas reservar compatibilidade. + ## Problema -`asset.load()` hoje cria uma thread do SO por requisicao. Ao mesmo tempo, `fs` ainda nao tem uma politica clara para sync assincrono barato em hardware simples. +`asset.load()` hoje dispara `thread::spawn` por requisicao de asset nao +residente. Isso escala mal, cria jitter e deixa o runtime sem uma politica +unica de backpressure. -O projeto precisa de paralelismo para IO/decode/sync, mas o target inclui handheld DIY e hardware barato, onde um pool grande ou explosao de threads pode piorar a latencia em vez de melhorar. +O dominio `fs` tambem tem operacoes potencialmente bloqueantes (`read`, +`write`, `list_dir`, `delete`, memcard commit/read/write) e a tabela de +syscalls ja as marca como nao deterministicas/custosas. Ainda assim, a +implementacao atual executa essas chamadas diretamente no caminho de dispatch. -## Dor +Sem um contrato explicito, cada dominio pode inventar sua propria concorrencia: +asset criando threads, FS bloqueando o frame, e memcard herdando comportamento +acidental do backend. + +## Pontos Criticos - `thread::spawn` por request escala mal e cria jitter. -- assets e `fs` competem por IO sem uma politica unica de fila/prioridade. -- sem lane dedicada, operacoes de background tendem a vazar custo para o main loop. -- sem disciplina, o host desktop vira referencia errada para hardware fraco. +- A terceira core/lane deve ser um recurso de plataforma controlado, nao uma + permissao geral para criar threads. +- Asset loading precisa de um pool/fila limitada para multiplos loads + pendentes. +- FS pode justificar a mesma lane quando envolver IO real, mas nao devemos + assinar uma API assincrona ampla sem entender a semantica de retorno. +- Commit/install em bancos residentes pode ter restricoes de determinismo e + ownership; decode pode ser assincrono, mas publicacao/instalacao talvez tenha + que cruzar de volta para a lane principal. +- A estrategia precisa funcionar em hardware alvo com poucos cores e tambem no + host desktop sem transformar desktop em referencia de custo errada. +- Render worker ja tem seu proprio contrato e nao deve compartilhar essa lane. ## Hotspots Atuais -- [asset.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-drivers/src/asset.rs#L353) -- [tick.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-system/src/virtual_machine_runtime/tick.rs#L53) -- [runner.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/host/prometeu-host-desktop-winit/src/runner.rs#L315) +- [asset.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-drivers/src/asset.rs#L438) - `AssetManager::load`. +- [asset.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-drivers/src/asset.rs#L507) - `thread::spawn` por load. +- [fs.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-hal/src/syscalls/domains/fs.rs#L4) - `fs` e `mem` marcados como superficies custosas/nao deterministicas. +- [dispatch.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-system/src/services/vm_runtime/dispatch.rs#L549) - chamadas `fs.*` executam no dispatch. +- [virtual_fs.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-system/src/services/fs/virtual_fs.rs#L69) - VFS normaliza path e chama backend diretamente. ## Alvo da Discussao -Fechar um modelo de execucao assincrona para `asset` e `fs` que seja previsivel em hardware simples. +Fechar um modelo de terceira core/lane para trabalho assincrono de runtime, +comecando por asset loading e avaliando se FS deve entrar na mesma lane no v1. + +O alvo nao e criar multitarefa geral, callbacks para guest, ou um scheduler +paralelo da VM. A lane deve executar trabalho de IO/decode/sync e publicar +resultados observaveis por status/polling em pontos previsiveis. ## O Que Precisa Ser Definido -1. Topologia de workers. - Escolher entre: - - uma thread dedicada para `asset` e outra para `fs`; - - um worker unico multiplexando filas; - - um pool minimo e fixo; - - proibicao explicita de `spawn` por request. +1. Topologia da terceira lane. + Definir: + - uma async work lane dedicada, separada da VM e do render worker; + - se a lane e uma thread fixa no host e uma core dedicada no hardware alvo; + - fallback quando nao houver core fisico dedicado; + - proibicao explicita de `thread::spawn` por request no caminho normal. -2. Separacao por dominio. - Decidir se `asset` e `fs` compartilham scheduler/fila ou se cada dominio tem lane propria. +2. Pool/fila de asset loading. + Definir: + - backlog de pedidos sem limite operacional fixo; + - exatamente um job ativo por vez na lane; + - o "pool" significa backlog e estados de job, nao multiplos workers reais; + - politica de substituicao/cancelamento para pedidos que miram o mesmo + bank/slot; + - status para pending/backlog, ativo, progresso, cancelamento, erro de + decode e sucesso. -3. Politica de prioridade. +3. Fases do asset load. + Separar: + - request e alocacao de handle; + - leitura de bytes; + - decode/materializacao; + - staging de resultado; + - install/commit em banco residente. + + Ponto a fechar: quais fases rodam na terceira lane e quais voltam para a + lane principal por determinismo/ownership. + +4. Inclusao de FS na mesma lane. + Verificar se `fs` deve usar a terceira lane no v1 para: + - leituras grandes; + - escritas; + - listagem; + - delete; + - memcard commit/read/write. + + Ponto a fechar: FS sync publico continua bloqueante/status-first, vira + request/poll, ou ganha apenas uma implementacao interna que drena trabalho em + pontos de frame. + +5. Separacao por dominio. + Decidir se `asset` e `fs` compartilham uma fila com classes de prioridade ou + se a terceira lane contem subfilas dedicadas. + +6. Politica de prioridade. Definir: - prioridade de loads visuais vs audio; - - prioridade de sync de save/config; - - limite de trabalho por frame para install/commit. + - prioridade de save/config; + - prioridade de memcard commit; + - starvation prevention; + - limite de trabalho de instalacao/publicacao por frame. -4. Modelo de retorno. - Fechar como o guest observa backlog, cancelamento e saturacao: - - status imediato de fila cheia; - - status de pending; - - metrica de backlog; - - politica de cancelamento. +7. Modelo de retorno e telemetria. + Fechar como guest/runtime observa: + - `pending`; + - `loading`; + - `ready`; + - `canceled`; + - `backend_unavailable`; + - backlog; + - posicao de um asset no backlog; + - progresso percentual do asset ativo; + - drops/retries quando aplicavel. -5. Orçamento para hardware barato. - Definir quantas threads o runtime pode assumir como baseline. +8. Orcamento para hardware barato. + Definir o baseline: + - VM principal; + - render worker; + - terceira async work lane; + - limites de memoria para jobs pendentes e buffers temporarios. -## Open Questions de Arquitetura +## Opcoes -1. O v1 precisa de lanes separadas para `asset` e `fs` ou basta uma fila central com classes de prioridade? -2. Decode de asset fica no mesmo worker do IO ou em fase distinta? -3. Commit/install no device continua no main thread por determinismo? +### Opcao A - Terceira lane unica com subfilas e pool limitado de assets + +- **Abordagem:** criar uma async work lane dedicada. Assets usam um pool/fila + de jobs nessa lane; FS tambem pode submeter jobs na mesma lane, com + prioridade menor ou classe separada. +- **Pro:** modelo simples de budget; evita explosao de threads; encaixa no alvo + de terceira core; permite compartilhar backpressure e telemetria. +- **Contra:** assets e FS podem competir por IO; precisa politica clara para + evitar que save/config fique atras de loads longos. +- **Manutenibilidade:** boa se a lane tiver tipos de job e estados explicitos. + +### Opcao B - Lane de assets agora, FS apenas reservado + +- **Abordagem:** fechar v1 somente para asset loading, removendo `spawn` por + request. FS continua sincrono por enquanto, mas o contrato da lane reserva + compatibilidade para FS no futuro. +- **Pro:** menor escopo; resolve a dor concreta de assets rapidamente; evita + alterar semantica de FS antes da agenda de app home filesystem. +- **Contra:** nao resolve stalls reais de FS/memcard; pode exigir refatorar a + lane depois para aceitar FS. +- **Manutenibilidade:** boa no curto prazo, com risco de segunda migracao. + +### Opcao C - Lanes separadas para asset e FS + +- **Abordagem:** criar uma lane/pool para assets e outra para FS. +- **Pro:** isolamento forte; FS/save nao compete diretamente com loads de + textura/audio/cena. +- **Contra:** consome mais threads/cores; contraria a premissa de terceira core + unica; pior para hardware simples. +- **Manutenibilidade:** media; isolamento ajuda, mas a topologia fica pesada. + +### Opcao D - Pool global pequeno sem nocao de terceira lane + +- **Abordagem:** trocar `spawn` por request por um pool global fixo. +- **Pro:** simples de implementar em host desktop. +- **Contra:** perde a arquitetura de lanes por core; pode mascarar custo em + desktop; fica menos claro como mapear para hardware alvo. +- **Manutenibilidade:** baixa para o modelo de console se virar dependencia + acidental de scheduler do SO. + +## Sugestao / Recomendacao + +Recomendo seguir com a Opcao A como direcao de arquitetura, mas fechar a +decisao em duas camadas: + +1. Contrato obrigatorio agora: + - terceira async work lane como recurso runtime; + - assets deixam de usar `thread::spawn` por requisicao; + - asset loading passa por backlog consultavel/reordenavel; + - status, progresso e telemetria de backlog ficam explicitos. + +2. Contrato condicionado para FS: + - FS deve ser analisado como consumidor legitimo da mesma lane porque suas + operacoes podem envolver IO real e ja sao custosas/nao deterministicas; + - a decisao precisa escolher entre incluir FS no v1 ou reservar a lane para + FS sem mudar ainda a superficie publica; + - memcard precisa ser considerado junto com FS, porque usa a mesma + capacidade e backend de persistencia. + +O ponto importante: a terceira core/lane deve ser a unidade de concorrencia. +O pool de assets deve existir dentro desse modelo, nao como criacao livre de +threads por load. + +## Respostas Consolidadas Ate Aqui + +1. O pool de assets deve ser uma lane serial. + - Ha exatamente um job ativo na async work lane. + - O pool representa backlog, estados e controle de ordem, nao multiplos + workers executando em paralelo. + +2. O backlog de pedidos deve ser semanticamente ilimitado. + - Nao existe `queue_full` como status normal do dominio. + - Como cada pedido mira um `bank_type/slot` e pedidos novos para o mesmo + alvo substituem os anteriores, o backlog efetivo fica limitado pela soma + dos slots dos bancos. + - O contrato nao precisa expor limite de fila, mas a implementacao deve + garantir que metadados de handles cancelados/superseded nao crescam sem + coleta. + +3. Asset loading precisa expor consulta e reordenacao do backlog. + - Deve existir API publica/runtime para consultar onde um asset esta no + backlog. + - Deve existir capacidade de adiantar ou atrasar um pedido de asset na fila. + - O asset ativo deve expor progresso percentual. + - Requests novos para o mesmo `bank_type/slot` devem cancelar pedidos + anteriores pendentes para aquele alvo, evitando carregar asset que ja foi + substituido antes de executar. + - Se o alvo ja estiver carregado com o mesmo `asset_id`, o novo request nao + entra no backlog; ele retorna imediatamente pronto com handle disponivel. + - Se um novo request substituir o alvo atualmente ativo, a lane deve cancelar + em ponto cooperativo quando possivel; se o trabalho nao puder ser + interrompido, o resultado antigo deve ser descartado ao final por geracao + superseded. + - `superseded` significa que um request foi substituido por outro request + mais novo para o mesmo `bank_type/slot`. Nao e erro e nao e cancelamento + manual; e obsolescencia operacional do pedido antigo. + - Handles representam o slot do bank como alvo estavel. Dentro do estado do + handle deve existir uma secao de request/backlog para aquele slot, + informando se ha pedido pendente/ativo, qual asset foi solicitado, + progresso, posicao e geracao. + - Handles devem ser capazes de consultar o estado do alvo `bank_type/slot` + a qualquer momento. Se nao houver asset valido carregado e tambem nao + houver request pendente para aquele alvo, a consulta deve retornar estado + explicito de vazio/invalido, nao falhar por ausencia de request ativo. + +4. Decode/materializacao fria rodam na terceira lane. + - Leitura e decode de asset podem acontecer fora da lane principal. + - O resultado deve voltar como material pronto/staged. + +5. Install/commit em bancos residentes continua na lane principal. + - A async lane prepara trabalho. + - A lane principal publica/instala em pontos previsiveis por ownership, + determinismo e integracao com bancos/render/audio. + +6. FS e memcard entram como consumidores possiveis da mesma lane, mas precisam + ser melhor amarrados com a agenda de app home filesystem. + - A `DSC-0007` / `AGD-0006` deve ser ajustada para estabelecer melhor a + superficie publica e o modelo async/sync de FS. + - Esta agenda deve afirmar a existencia da lane para trabalhos async de IO + desse tipo e permitir que FS a consuma. + - Questoes de API publica de FS, request/poll e semantica de app home devem + ficar a cargo da agenda 6. + - FS, memcard e assets nao devem operar sobrepostos quando isso gerar + concorrencia real de backend; se houver competicao, a fila unica resolve a + ordem. + +7. Prioridade quando houver competicao: + - memcard commit/write; + - FS write/config; + - asset visual/audio load; + - list/read nao critico. + +8. Status-first continua valendo. + - `backend_unavailable`, `canceled`, erro de decode e estados de job sao + status operacionais. + - Cancelamento por superseding de pedido no mesmo bank/slot e resultado + operacional normal, nao fault. + - Nao ha `queue_full` no contrato esperado. + - Trap permanece para violacao estrutural, como argumento invalido, + capability ausente ou handle estruturalmente invalido. + +9. Telemetria minima: + - backlog atual; + - posicao por asset; + - progresso percentual do asset ativo; + - jobs submetidos, concluidos, falhos e cancelados; + - tempo por job; + - percentis por asset enquanto estiver carregando na lane. + +10. Escopo da agenda. + - O foco desta agenda e a lane, assets e memcard/game persistence. + - FS entra como consumidor permitido da lane de IO, mas as decisoes de API + publica de FS pertencem a `AGD-0006`. + +11. API de backlog. + - A API inicial sugerida cobre os casos esperados. + - `asset.backlog_promote(handle)` deve ser mantido como atalho oficial para + `asset.backlog_move(handle, 1)`. + - `asset.backlog_demote(handle)` deve ser mantido como atalho oficial para + mover o pedido para o fim dos pendentes. + - Os atalhos nao criam semantica nova; apenas evitam que o caller calcule + posicoes comuns. + - O handle deve permitir consulta estavel do slot do bank mesmo quando nao ha + asset valido carregado. + - O estado do handle deve separar: + - estado do slot/residencia atual; + - estado do request/backlog atual para aquele slot. + - Operacoes que mudam estado, como commit/cancel/reorder, atuam sobre a + secao de request/backlog do handle e precisam validar geracao/request atual + para nao agir sobre um pedido substituido. + +12. Cancelamento por substituicao. + - Quando um request novo chega para o mesmo `bank_type/slot`, o request + anterior e marcado imediatamente como `superseded`. + - Se o request anterior ainda estiver no backlog, ele sai da fila. + - Se o request anterior estiver ativo, a lane tenta cancelar + cooperativamente quando a fase atual permitir. + - Se a fase ativa nao puder ser interrompida com baixo custo, ela termina e + o resultado e descartado ao final porque a geracao do alvo ja mudou. + - Como o handle pertence ao slot do bank, ele continua consultavel; sua + secao de request deve indicar que a geracao anterior foi `superseded` ou + que nao ha request ativo para aquele alvo. + +13. Fases de progresso. + - `read/decode/stage` e suficiente como modelo minimo para `GLYPH`, `SOUND` + e `SCENE`. + - Subfases futuras podem detalhar progresso dentro de `decode` sem mudar o + contrato geral. + +14. Percentis. + - Percentis devem existir por `bank_type` como telemetria principal. + - Percentis por `asset_id` tambem sao uteis, mas podem usar janela pequena + ou agregador leve. + - Percentis sao atualizados no fechamento do job, nao durante o hot path. + +15. Ajuste esperado na agenda de FS. + - `AGD-0006` deve registrar que existe uma async IO lane compartilhavel por + FS/memcard/assets. + - Esta agenda so decide que FS pode consumir a lane. + - A API publica de FS, incluindo request/poll ou sync aparente, pertence a + `AGD-0006`. + +## API Inicial Sugerida Para Assets + +Esta agenda ainda nao fecha nomes finais de ABI, mas a direcao inicial sugerida +e manter `asset.load/status/commit/cancel` e adicionar uma superficie pequena +para backlog: + +1. `asset.backlog_info() -> (status, pending_count, active_handle, active_asset_id, active_bank_type, active_slot, active_progress)` + - Consulta o estado geral da lane de assets. + - `active_progress` usa escala inteira, preferencialmente `0..10000` para + permitir duas casas decimais sem float. + +2. `asset.backlog_position(handle) -> (status, state, position, progress)` + - Consulta um pedido especifico. + - `position = 0` significa job ativo. + - `position > 0` significa posicao no backlog. + - `position = -1` pode representar nao enfileirado/pronto/cancelado, + dependendo do `state`. + +3. `asset.backlog_move(handle, new_position) -> status` + - Move um pedido pendente para uma posicao absoluta. + - `new_position = 1` significa primeiro pendente depois do ativo. + - Nao interrompe diretamente o job ativo; substituicao do mesmo + `bank_type/slot` usa a regra de superseding. + +4. `asset.backlog_promote(handle) -> status` + - Atalho para mover o pedido para a frente do backlog pendente. + +5. `asset.backlog_demote(handle) -> status` + - Atalho para mover o pedido para o fim do backlog pendente. + +6. `asset.target_status(bank_type, slot) -> (status, asset_id, handle, state, position, progress)` + - Consulta por alvo operacional, que e o que realmente controla superseding. + - Ajuda quando o caller quer saber "o que esta programado para este slot". + +Estados sugeridos para `state`: + +- `empty`; +- `invalid`; +- `queued`; +- `active`; +- `ready`; +- `committed`; +- `canceled`; +- `superseded`; +- `error`; +- `backend_unavailable`. + +Regra de design: APIs de backlog sao operacionais e status-first. Reordenar um +handle inexistente, ja concluido ou ja superseded retorna status operacional +apropriado quando o handle e conhecido; violações estruturais continuam Trap. + +Handles sao observaveis e representam slots de banks. Um handle conhecido pode +ser consultado mesmo quando o slot alvo nao contem asset valido carregado; nesse +caso a consulta deve retornar estado como `empty` ou `invalid`. + +O estado do handle deve conter pelo menos duas partes: + +```text +slot_state: + loaded_asset_id + resident_state + +request_state: + requested_asset_id + generation + queued/active/ready/canceled/superseded/error + backlog_position + progress +``` + +Mutacoes seguem regras mais estritas e devem falhar operacionalmente quando a +secao de request/backlog do handle nao representa a geracao atual. + +## Progresso e Percentis + +### Progresso percentual + +Sugestao: + +- usar progresso em base inteira `0..10000`; +- quando o tamanho total de bytes for conhecido, calcular progresso por fases + ponderadas; +- quando o decode nao for linear ou nao houver tamanho confiavel, expor + progresso por fase com marcos conservadores. + +Modelo inicial de fases: + +```text +queued -> 0 +read -> 0..4000 +decode -> 4000..9000 +stage -> 9000..10000 +ready -> 10000 +``` + +Se uma fase nao consegue reportar progresso interno, ela deve manter o ultimo +marco e saltar para o proximo marco ao concluir. Isso evita progresso falso +sem perder visibilidade de fase. + +### Percentis por asset + +Sugestao: + +- registrar duracoes por asset no fim de cada job, nao por tick/frame; +- manter agregadores pequenos por `asset_id` e por `bank_type`; +- calcular percentis a partir de janela/ring buffer pequeno ou histograma + barato por bucket de duracao; +- nunca atualizar percentis no loop quente de decode por item/pixel; +- o job ativo pode expor tempo decorrido e fase atual; percentis sao atualizados + apenas no fechamento do job. + +## Perguntas em Aberto + +- [x] A agenda esta madura para virar decisao normativa ou ainda falta algum + caso de asset/memcard/lane? +- [x] A estrutura minima de `slot_state` e `request_state` no handle esta + completa ou falta algum campo para commit/cancel/reorder seguro? + +## Resolucao + +A agenda esta madura para decisao normativa. + +A decisao deve formalizar: + +- a terceira async work lane serial; +- um job ativo por vez; +- backlog efetivamente limitado pela unicidade de `bank_type/slot`; +- handles como observadores estaveis de slots de bank; +- `slot_state` e `request_state` dentro do handle; +- cancelamento/superseding por novo request para o mesmo alvo; +- load pronto imediato quando o asset ja esta residente no alvo; +- decode/materializacao na lane async; +- commit/install na lane principal; +- memcard como consumidor prioritario da mesma lane; +- FS como consumidor permitido, deixando API publica de FS para `AGD-0006`; +- progresso por fases e percentis atualizados no fechamento do job. ## Dependencias -- `../specs/09-events-and-concurrency.md` -- `../specs/15-asset-management.md` -- `../specs/16a-syscall-policies.md` -- `014-app-home-filesystem-surface-and-semantics.md` +- `docs/specs/runtime/09-events-and-concurrency.md` +- `docs/specs/runtime/15-asset-management.md` +- `docs/specs/runtime/16a-syscall-policies.md` +- `discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md` -## Criterio de Saida Desta Agenda +## Criterio para Encerrar Pode virar PR quando houver decisao escrita sobre: -- numero e tipo de workers aceitos no baseline; -- fila/prioridade de `asset` e `fs`; -- proibicao ou aceitacao limitada de `thread::spawn` por request; -- modelo de status/telemetria para backlog e saturacao. +- existencia e papel da terceira async work lane; +- relacao dessa lane com a terceira core no hardware alvo e fallback no host; +- desenho do backlog serial de asset loading; +- regra de cancelamento/substituicao para requests no mesmo bank/slot; +- proibicao de `thread::spawn` por load no caminho normal; +- fases de asset load que rodam na lane vs lane principal; +- inclusao condicionada de FS/memcard nessa mesma lane; +- API de consulta e reordenacao do backlog de assets; +- comportamento de request imediato quando o asset ja esta residente no alvo; +- regra de bounded backlog por unicidade de `bank_type/slot`; +- politica de prioridade entre asset, FS e memcard; +- status e telemetria para backlog, progresso, percentis, cancelamento e falha. + +## Discussao + +- 2026-06-28: Agenda atualizada para remover render worker do escopo, assumir + uma terceira core/lane para trabalhos assincronos, exigir pool/fila limitada + para asset loading e reavaliar o papel de FS/memcard como consumidores da + mesma lane. +- 2026-06-28: Discussao consolidou que a async work lane deve ser serial, com + um job ativo, backlog semanticamente ilimitado, consulta/reordenacao publica + de backlog de assets, progresso percentual do asset ativo, decode na lane + async, install/commit na lane principal, e FS/memcard como consumidores a + alinhar com a agenda de app home filesystem. +- 2026-06-28: Acrescentada regra de asset pipe: requests para o mesmo bank/slot + devem cancelar pedidos anteriores pendentes para aquele alvo, tratando o + pedido mais novo como substituto operacional. +- 2026-06-28: Discussao refinou que o alvo canonico e `bank_type/slot`; um + asset ja residente deve retornar handle pronto sem entrar no backlog; o + backlog efetivo e limitado pela soma dos slots dos bancos; FS fica como + consumidor permitido da lane, enquanto API publica de FS permanece na agenda + 6; progresso deve usar fases conservadoras e percentis devem ser atualizados + no fechamento do job, nao no hot path. +- 2026-06-28: Open questions restantes foram consolidadas: API de backlog + aceita com `promote/demote` como atalhos; superseding marca requests + substituidos por outro request para o mesmo `bank_type/slot`; cancelamento do + ativo e cooperativo quando barato e descarte por geracao quando nao for; + `read/decode/stage` cobre GLYPH/SOUND/SCENE; percentis ficam por `bank_type` + e por `asset_id`; API publica de FS permanece em `AGD-0006`. +- 2026-06-28: Adicionada a expectativa de que handles possam consultar o alvo + `bank_type/slot` a qualquer momento, inclusive quando nao houver asset valido + carregado; mutacoes devem continuar protegidas por request/geracao para evitar + commit/cancel acidental de pedidos substituidos. +- 2026-06-28: Refinado o modelo de handle: o handle pertence ao slot do bank e + contem uma secao de estado de request/backlog para aquele slot. Consulta e + residencia ficam estaveis por alvo; mutacoes atuam sobre a geracao atual do + request dentro do handle. diff --git a/discussion/workflow/agendas/AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md b/discussion/workflow/agendas/AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md deleted file mode 100644 index 42307ee1..00000000 --- a/discussion/workflow/agendas/AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md +++ /dev/null @@ -1,409 +0,0 @@ ---- -id: AGD-0010 -ticket: perf-gfx-render-pipeline-and-dirty-regions -title: Agenda - [PERF] GFX Render Pipeline and Dirty Regions -status: open -created: 2026-03-27 -resolved: -decision: -tags: [] ---- - -# Agenda - [PERF] GFX Render Pipeline and Dirty Regions - -## Problema - -O renderer `gfx` recompõe a cena inteira a cada frame logico, mesmo quando a mudanca visual e pequena. - -Hoje `render_all()` reconstrói buckets de sprites, escaneia os 512 sprites, redesenha layers e aplica dois passes fullscreen de fade sem politica de invalidacao. - -Ao mesmo tempo, a arquitetura atual ja opera como um framebuffer destrutivo em memoria: draws escrevem diretamente no buffer de trabalho e operacoes posteriores sobrescrevem o que estiver por baixo. A discussao nao e migrar para um modelo tipo GPU, scene graph ou retained rendering. - -Existe ainda um driver de produto importante: o objetivo continua sendo viabilizar hardware handheld proprio com limitacoes reais de orcamento e com economia de memoria o mais agressiva possivel. Isso significa que ganhos de CPU nao podem ser avaliados isoladamente; custo de RAM adicional, buffers extras e estruturas auxiliares entram diretamente no criterio de aceitacao. - -## Dor - -- custo visual basico cresce demais para hardware simples. -- pequenos updates pagam preco de full redraw. -- fade, HUD e world composition ficam sempre no caminho critico. - -## Hotspots Atuais - -- [gfx.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-drivers/src/gfx.rs#L563) -- [gfx.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-drivers/src/gfx.rs#L671) - -## Alvo da Discussao - -Definir ate onde o v1 pode sofisticar o modelo de desenho destrutivo em memoria para reduzir custo e manter previsibilidade, sem migrar para um renderer de estilo GPU. - -## O Que Precisa Ser Definido - -1. Perfil real de custo. - Medir separadamente: - - rebuild de buckets; - - rasterizacao de layers; - - rasterizacao de sprites; - - fades fullscreen; - - conversao/copia no host; - - apresentacao no host. - -2. Granularidade minima de invalidacao compativel com framebuffer destrutivo. - Escolher entre: - - dirty flag por frame inteiro; - - dirty flags por subsistema (`world`, `hud`, `fade`, `sprites`); - - dirty regions restritas e derivadas de primitivas/sprites; - - combinacao minima viavel. - -3. Buckets e traversal de sprites. - Decidir se os buckets continuam sendo reconstruidos por frame ou se viram estrutura incremental. - -4. Fades. - Definir se fade neutro precisa bypass total e se fade parcial pode operar por regiao/camada. - -5. Cache de composicao. - Delimitar se layers estaticas/HUD podem manter cache intermediario. - -6. Primitivas e contrato operacional. - Definir quais draws devem continuar sendo vistos como escrita destrutiva direta no framebuffer e onde valem aceleracoes internas sem mudar a semantica observavel. - -7. Meta de custo. - Fechar qual teto de pixels/trabalho por frame e aceitavel no baseline. - -8. Orcamento de memoria. - Definir quais otimizacoes aceitam memoria adicional e quais precisam caber no modelo mais economico possivel para handheld barato. - -## Open Questions de Arquitetura - -1. Quais partes do custo atual estao no `render_all()` e quais partes estao na conversao/apresentacao do host? -2. Qual o maior ganho de curto prazo dentro do modelo atual: buckets, fades, HUD, primitivas ou host copy/convert? -3. Dirty regions restritas quebram alguma expectativa de determinismo visivel ou sao apenas um detalhe interno de implementacao? -4. O HUD deve continuar no mesmo pipeline da world scene? -5. Quais primitivas merecem tratamento especial para reduzir overdraw sem mudar a semantica de sobrescrita? -6. Qual o teto aceitavel de memoria extra para caches, buffers separados ou estruturas auxiliares no handheld alvo? - -## Problemas de Medio e Grande Porte Identificados - -### 1. Composicao completa do framebuffer a cada frame logico - -`render_all()` recompõe o `back` inteiro por software, mesmo quando a mudanca visual e pequena. - -Impacto: - -- custo cresce com a quantidade de pixels tocados, nao apenas com a quantidade de estado alterado; -- layers, sprites e fades disputam o mesmo budget de CPU; -- o modelo atual favorece previsibilidade sem ainda ter atalhos internos suficientes. - -### 2. Rasterizacao pixel a pixel de tilemaps - -Cada layer visivel percorre tiles e, para cada tile nao vazio, resolve indices e escreve pixel a pixel no framebuffer. - -Impacto: - -- o custo real esta mais na escrita de pixels do que na manutencao do `TileMap`; -- scroll e barato como estado, mas caro na hora de compor a imagem se tudo for repintado; -- layers estaticas continuam pagando custo de rasterizacao em toda recomposicao. - -### 3. Rasterizacao pixel a pixel de sprites e escolha por bucket full scan - -Os 512 sprites sao varridos para reconstruir buckets e os sprites ativos sao desenhados pixel a pixel. - -Impacto: - -- quando poucos sprites mudam, ainda existe custo fixo de rebuild dos buckets; -- overdraw entre sprites e layers pode explodir o numero de writes no `back`; -- a escolha de sprites ativos/prioridade ainda e simples demais para cenarios com alta ocupacao. - -### 4. Fades fullscreen com custo proporcional ao frame inteiro - -`scene fade` e `hud fade` percorrem o framebuffer inteiro quando ativos. - -Impacto: - -- custo grande e previsivel, mas potencialmente desproporcional; -- fade neutro ja pode ser bypass, mas fade parcial ainda custa uma passada completa; -- dois passes fullscreen no mesmo frame podem virar gargalo antes de layers ou sprites. - -### 5. HUD no mesmo caminho critico da composicao principal - -O HUD e redesenhado como tilemap depois da world scene e antes do `hud fade`. - -Impacto: - -- HUD estatico continua gerando trabalho por frame; -- mistura responsabilidades de world/UI no mesmo budget de composicao; -- dificulta isolar ganho de cache ou dirtying so para interface. - -### 6. Conversao obrigatoria de RGB565 para RGBA8 no host - -No `RedrawRequested`, o host percorre todo o `front_buffer` e converte para o frame do `pixels`. - -Impacto: - -- ha custo de leitura do framebuffer inteiro e escrita de um segundo buffer inteiro; -- esse custo existe mesmo quando o host esta apenas apresentando o mesmo quadro logico; -- pode competir com o custo do renderer sem estar visivel na discussao do `gfx`. - -### 7. Pipeline serializado no host desktop - -`tick`, `render_all()`, conversao de formato e apresentacao ocorrem no mesmo fluxo operacional. - -Impacto: - -- o frame time final acumula custo de runtime, composicao, copy/convert e present; -- falta desacoplamento para esconder latencia entre producao e apresentacao; -- dificulta atribuir gargalo sem instrumentacao por etapa. - -## Estudo Inicial de Possiveis Optimizacoes - -### A. Dirty flags por subsistema em vez de dirty rect generico - -Ideia: - -- flags independentes para `world`, `sprites`, `hud`, `scene_fade`, `hud_fade`, `host_present`. - -Vantagens: - -- preserva a semantica de framebuffer destrutivo; -- reduz recomposicao desnecessaria quando so um subsistema mudou; -- e muito mais simples de validar do que dirty regions arbitrarias. - -Riscos: - -- exige definir dependencias claras entre world, sprites, HUD e fades; -- pode induzir falsos positivos conservadores, o que e aceitavel no v1. - -### B. Rebuild incremental ou condicional dos buckets de sprite - -Ideia: - -- so reconstruir buckets quando algum `GfxSetSprite` mudar atividade, prioridade, banco ou tile; -- opcionalmente manter contadores/sinais de mutacao da OAM. - -Vantagens: - -- elimina custo fixo por frame quando OAM permanece estavel; -- combina com a ideia de sprites como estado, nao como draw list efemera. - -Riscos: - -- precisa invalidacao correta para evitar bucket desatualizado; -- ganho pode ser pequeno se o custo dominante estiver no draw dos sprites, nao no rebuild. - -### C. Separar world e HUD em buffers logicos distintos - -Ideia: - -- manter um buffer da world scene e um buffer do HUD, compondo no final apenas quando necessario. - -Vantagens: - -- HUD estatico deixa de participar da recomposicao da world; -- permite cache e fade especificos para cada dominio; -- se alinha bem com a separacao conceitual entre cena e interface. - -Riscos: - -- aumenta uso de memoria e pontos de sincronizacao; -- precisa manter semantica clara de sobrescrita entre mundo e HUD. -- pode ser inviavel no perfil de handheld barato se exigir buffers adicionais permanentes. - -### D. Cache de layer estatica ou composicao parcial da world - -Ideia: - -- layers que nao mudam podem manter imagem intermediaria pronta; -- scroll, tile updates ou troca de bank invalidam o cache correspondente. - -Vantagens: - -- reduz rasterizacao repetida de cenario estavel; -- aproxima o ganho de hardware tile-based sem abandonar software raster. - -Riscos: - -- cache amplo demais vira complexidade estrutural; -- se scroll muda constantemente, o ganho cai bastante. -- pode consumir memoria demais para um hardware com orcamento agressivo. - -### E. Otimizacao de fades - -Ideia: - -- bypass total para fade neutro; -- opcionalmente aplicar fade apenas sobre buffers relevantes; -- estudar LUTs ou blend mais barato por pixel. - -Vantagens: - -- ataca um custo fullscreen claramente delimitado; -- baixo risco conceitual. - -Riscos: - -- fade parcial por regiao pode complicar demais o contrato; -- LUT ajuda aritmetica, mas nao elimina custo de varrer memoria. - -### F. Melhorias em primitivas e spans de memoria - -Ideia: - -- acelerar `fill_rect`, linhas horizontais/verticais, clears e possiveis blits contiguos; -- explorar caminhos de row spans contiguos em vez de loops mais genericos. - -Vantagens: - -- melhora direta do modelo de desenho destrutivo; -- baixo risco arquitetural; -- cria fundacao para outros atalhos internos. - -Riscos: - -- ganho localizado se o workload dominante vier de tile/sprite compositing; -- precisa medir por primitive class. - -### G. Reduzir custo de copy/convert no host - -Ideia: - -- medir separadamente `draw_rgb565_to_rgba8` e `pixels.render()`; -- estudar formato mais proximo do host ou conversao mais barata; -- evitar redraw host quando nao houver novo front relevante. - -Vantagens: - -- ataca custo fora do `gfx` que ainda entra no frame time total; -- pode render ganho imediato no desktop host. - -Riscos: - -- parte do custo depende da stack `pixels/wgpu`, nao apenas do runtime; -- alguns ganhos podem ser especificos do host desktop e nao do contrato do console. - -## Restricao de Plataforma - -Qualquer recomendacao desta agenda deve ser filtrada por um criterio adicional: - -- priorizar solucoes que melhorem custo sem multiplicar buffers; -- tratar memoria extra como recurso escasso de primeira classe; -- preferir flags, metadados pequenos e estruturas incrementais a caches grandes; -- evitar solucoes cuja performance dependa de assumir um host desktop mais forte do que o handheld alvo. - -Direcao atual da discussao: - -- nesta etapa, buffers extras devem ficar fora; -- o foco deve ser otimizar ao maximo o pipeline existente; -- cache intermediario ou novos buffers so entram em estudo depois que as otimizacoes de baixo custo de memoria forem medidas e esgotadas. - -## Sugestao / Recomendacao Atual - -Priorizar o estudo em camadas, nesta ordem: - -1. instrumentar o pipeline por etapa; -2. validar dirty flags por subsistema como mecanismo minimo de invalidacao; -3. testar rebuild condicional de buckets e bypass/isolamento de fades; -4. otimizar primitivas e caminhos contiguos de escrita no framebuffer atual; -5. estudar separacao world/HUD e cache de layer apenas se os dados justificarem e o teto de memoria permitir; -5. tratar copy/convert do host como frente paralela de otimizacao, nao como substituto da analise do `gfx`. - -## Achados Consolidados Ate Aqui - -1. O contrato base continua sendo framebuffer destrutivo em memoria. - Draws escrevem diretamente no buffer de trabalho e operacoes posteriores sobrescrevem o que estiver por baixo. A agenda nao aponta para migracao a um renderer tipo GPU, retained mode ou compositor sofisticado. - -2. O `present()` atual nao parece ser o problema principal. - No `gfx`, `present()` faz swap de buffers, nao copia completa de pixels. - -3. O custo suspeito esta na recomposicao e na apresentacao, nao na manutencao do estado logico. - Escritas de `TileMap`, scroll e `GfxSetSprite` sao pequenas e pontuais; o custo potencialmente dominante esta em rasterizar pixels, aplicar fades fullscreen e converter o framebuffer para o host. - -4. Os maiores suspeitos atuais de custo sao: - - rasterizacao de layers; - - rasterizacao de sprites; - - fades fullscreen; - - conversao `RGB565 -> RGBA8` no host; - - `pixels.render()` / apresentacao no host. - -5. O pipeline atual deve ser otimizado antes de considerar buffers extras. - A direcao aceita da discussao e esgotar primeiro flags pequenas, bypasses, melhorias de primitive paths e ajustes incrementais no pipeline existente. - -6. Restricao de plataforma pesa tanto quanto CPU. - Como o alvo e um handheld proprio com limitacoes de orcamento e memoria, solucoes que consumam RAM adicional significativa devem ficar fora desta fase. - -7. A instrumentacao minima desejada para retomar o estudo ficou definida. - Quando houver workload representativo, queremos medir ao menos: - - `render_all_total_us` - - `bucket_rebuild_us` - - `layer_raster_us` - - `sprite_raster_us` - - `scene_fade_us` - - `hud_raster_us` - - `hud_fade_us` - - `host_convert_us` - - `host_present_us` - -8. A retencao de metricas deve ser barata e orientada a analise posterior. - A preferencia atual e por acumuladores e ring buffer pequeno de snapshots, sem logging pesado por frame no hot path. - -9. Ring buffer no `TileMap` nao e prioridade nesta fase. - O custo suspeito nao esta na manutencao da grade logica do mapa, e sim na composicao da janela visivel no framebuffer. Mudar a estrutura do mapa so faria sentido se o gargalo principal estivesse em streaming/atualizacao estrutural do cenario, o que nao e a hipotese atual. - -10. A direcao de otimizacao mais promissora e padronizar copias massivas para o framebuffer atual. - Em vez de atacar apenas tilemaps com loops pixel a pixel, a discussao passa a favorecer um padrao geral de blit/copia massiva com: - - calculo previo de offsets e spans; - - fast paths para casos contiguos; - - trabalho por linha/chunk em vez de trabalho atomizado por pixel quando possivel; - - reaproveitamento dessa infraestrutura para tilemaps, sprites e outras operacoes de desenho destrutivo. - -11. Migrar o `back` para `RGBA8888` nao e direcao aceita neste momento. - Isso aumentaria custo de memoria e largura de banda no alvo handheld, alem de deslocar a otimizacao para o host desktop. O contrato interno continua preferencialmente em `RGB565`, mesmo considerando fades e uma futura pipeline de lights. - -## Nova Direcao Tecnica Em Estudo - -Quando esta agenda for reaberta, a linha principal de investigacao deve considerar: - -- manter o `TileMap` simples como estado logico; -- assumir que a janela visivel provavelmente continuara sendo recomposta na maior parte dos frames; -- concentrar a otimizacao em como essa recomposicao escreve no `back`; -- desenhar uma infraestrutura compartilhada de copias massivas/blits para o framebuffer atual; -- usar essa infraestrutura nao apenas para tilemaps, mas como base comum para raster de sprites e outros caminhos de desenho. - -Pergunta orientadora da reabertura: - -- como transformar o renderer de um pipeline de writes atomizados por pixel em um pipeline com fast paths de spans/chunks, sem perder a semantica de framebuffer destrutivo e sem pagar memoria extra relevante? - -## Status de Standby - -Esta agenda deve ficar em espera ate existir um game ou workload real que exercite de forma representativa: - -- tilemaps com scroll e composicao de world; -- sprites em quantidade suficiente para testar buckets e overdraw; -- HUD ativo; -- fades; -- apresentacao completa no host. - -Antes disso, qualquer conclusao sobre gargalo ou prioridade de otimizacao tende a ser prematura. - -## Proximo Gatilho Para Reabrir - -Reabrir esta agenda quando houver: - -- um game jogavel ou cena de teste que percorra o pipeline completo; -- dados de workload mais proximos do uso real; -- necessidade concreta de justificar uma rodada de instrumentacao e profiling. - -## Dependencias - -- `../specs/04-gfx-peripheral.md` -- `../specs/11-portability-and-cross-platform-execution.md` - -## Criterio de Saida Desta Agenda - -Pode virar PR quando houver decisao escrita sobre: - -- filosofia explicita de framebuffer destrutivo como contrato base; -- plano de instrumentacao para localizar o custo dominante do pipeline; -- nivel minimo de invalidacao no v1; -- politica de rebuild de buckets de sprites; -- bypass/cache de fade e HUD; -- politica para otimizar primitivas sem mudar a semantica observavel; -- meta de custo para o render pipeline. diff --git a/discussion/workflow/decisions/DEC-0034-async-work-lane-and-asset-backlog-contract.md b/discussion/workflow/decisions/DEC-0034-async-work-lane-and-asset-backlog-contract.md new file mode 100644 index 00000000..2095c005 --- /dev/null +++ b/discussion/workflow/decisions/DEC-0034-async-work-lane-and-asset-backlog-contract.md @@ -0,0 +1,336 @@ +--- +id: DEC-0034 +ticket: perf-async-background-work-lanes-for-assets-and-fs +title: Async Work Lane and Asset Backlog Contract +status: accepted +created: 2026-06-28 +accepted: 2026-06-28 +ref_agenda: AGD-0008 +plans: [PLN-0123, PLN-0124, PLN-0125, PLN-0126, PLN-0127, PLN-0128] +tags: [perf, asset, fs, async, scheduler, runtime] +--- + +# DEC-0034 - Async Work Lane and Asset Backlog Contract + +## Status + +Accepted. + +## Contexto + +`asset.load()` currently creates an OS thread per non-resident asset request. +That behavior does not match the target runtime model: the machine should have +explicit execution lanes, bounded ownership, observable backpressure, and no +uncontrolled thread creation from guest-visible operations. + +Render worker concurrency is already handled by separate render decisions. This +decision covers the third runtime lane for asynchronous IO/decode/persistence +work: + +```text +Lane 1: VM, firmware, SystemOS, logical frame +Lane 2: render worker / asynchronous rasterization +Lane 3: async work lane for asset IO/decode and game persistence work +``` + +The concrete first consumer is asset loading. Game persistence through memcard +may also consume this lane. FS is allowed to consume this lane, but public FS API +shape belongs to the app-home filesystem agenda. + +## Decisao + +Prometeu MUST introduce a third async work lane as the runtime-owned execution +place for asset IO/decode work and compatible persistence IO work. + +The async work lane MUST be serial: it has exactly one active job at a time. +The runtime MUST NOT create one OS thread per `asset.load` request in the normal +path. + +Asset loading MUST use a backlog keyed by target `bank_type/slot`. A target can +have at most one current request. A newer request for the same `bank_type/slot` +MUST supersede the previous request for that target. + +Asset handles MUST represent stable bank slots, not transient worker threads. +A handle MUST remain queryable even when the slot has no valid loaded asset and +even when there is no active request for that slot. The handle state MUST +separate resident slot state from request/backlog state. + +Asset IO/read and decode/materialization MUST run on the async work lane. +Asset install/commit into resident banks MUST happen on the main runtime lane at +predictable ownership points. + +Memcard work MAY consume the same async work lane and has higher priority than +ordinary asset loads when contention exists. FS MAY consume the same lane for +IO-style work, but this decision does not define the public FS API. + +## Rationale + +A serial lane maps cleanly to the intended third-core mental model and avoids a +desktop-biased generic thread pool. It gives the runtime one place to observe +IO/decode backlog, progress, priority, cancellation, and telemetry. + +Keying requests by `bank_type/slot` makes the backlog naturally bounded by the +sum of bank slots. There is no need for a guest-visible `queue_full` state: +there cannot be unbounded distinct pending targets if each target has at most +one current request. + +Keeping commit/install on the main lane preserves deterministic publication +semantics and avoids letting the async lane mutate resident graphics/audio/scene +state while the VM, render handoff, or frame boundary is observing it. + +Separating `slot_state` and `request_state` inside the handle avoids ambiguity: +callers can inspect the slot at any time, while mutating operations still remain +protected by request generation. + +## Invariantes / Contrato + +### Async Work Lane + +- The async work lane MUST be runtime-owned. +- The async work lane MUST be separate from the render worker. +- The async work lane MUST execute at most one active job at a time. +- The runtime MUST NOT use `thread::spawn` per asset request in the normal + asset loading path. +- The host may implement the lane as a fixed worker thread. Hardware targets may + map it to a dedicated core when available. +- If a physical third core is unavailable, the implementation MUST preserve the + same logical lane contract. + +### Asset Backlog + +- Asset requests MUST target a concrete `bank_type/slot`. +- The backlog is ordered and serial. +- Each `bank_type/slot` MUST have at most one current request. +- A newer request for the same `bank_type/slot` MUST supersede any earlier + pending request for that target. +- If the target already contains the requested `asset_id` as a valid resident + asset, `asset.load` MUST return a handle in ready state without adding a job + to the backlog. +- The effective backlog size is bounded by the sum of targetable bank slots, + even though the contract does not expose a fixed queue limit. +- Implementations MUST collect stale metadata for canceled or superseded + requests so the handle/request history does not grow without bound. + +### Handles + +An asset handle MUST represent a stable bank slot target. The handle state MUST +include at least: + +```text +handle: + bank_type + slot + +slot_state: + loaded_asset_id + resident_state + slot_generation + +request_state: + requested_asset_id + request_generation + state + backlog_position + progress +``` + +The handle MUST be queryable when: + +- the slot is empty; +- the slot contains a valid resident asset; +- a request is queued; +- a request is active; +- a request is ready for commit; +- a request has been canceled; +- a request has been superseded; +- a request ended in error. + +Mutating operations such as commit, cancel, promote, demote, or move MUST act on +the current request generation. They MUST NOT accidentally mutate a newer +request through an older handle view. + +### Superseding + +`superseded` means a request was replaced by a newer request for the same +`bank_type/slot`. + +- Superseding is an operational state, not a fault. +- If the old request is queued, it MUST be removed from the backlog. +- If the old request is active, the lane SHOULD cancel cooperatively when the + current phase supports cheap cancellation. +- If active work cannot be interrupted cheaply, it MAY finish, but its result + MUST be discarded when its generation no longer matches the target's current + request generation. + +### Asset Backlog API Direction + +The existing asset operations remain the base surface: + +```text +asset.load(asset_id, slot) -> (status, handle) +asset.status(handle) -> status +asset.commit(handle) -> status +asset.cancel(handle) -> status +``` + +The asset backlog surface SHOULD add a small status-first API: + +```text +asset.backlog_info() + -> (status, pending_count, active_handle, active_asset_id, + active_bank_type, active_slot, active_progress) + +asset.backlog_position(handle) + -> (status, state, position, progress) + +asset.backlog_move(handle, new_position) + -> status + +asset.backlog_promote(handle) + -> status + +asset.backlog_demote(handle) + -> status + +asset.target_status(bank_type, slot) + -> (status, asset_id, handle, state, position, progress) +``` + +`asset.backlog_promote(handle)` is an official shortcut for moving a queued +request to position `1`, the first pending position after the active job. + +`asset.backlog_demote(handle)` is an official shortcut for moving a queued +request to the end of the pending backlog. + +The final ABI names and exact return shapes may be refined during planning, but +the implementation MUST preserve the capabilities above. + +### Progress + +Progress MUST be represented without floating point. The recommended scale is +`0..10000`. + +The initial phase model is: + +```text +queued -> 0 +read -> 0..4000 +decode -> 4000..9000 +stage -> 9000..10000 +ready -> 10000 +``` + +If a phase cannot report internal progress, it MUST keep the previous progress +mark and advance at phase completion. Implementations MUST NOT invent false +precision for non-linear decode phases. + +### Telemetry + +Telemetry MUST be cheap and must not add hot-path per-item decode cost. + +Minimum telemetry: + +- current backlog depth; +- target/request position; +- active job progress; +- jobs submitted; +- jobs completed; +- jobs failed; +- jobs canceled; +- jobs superseded; +- job duration; +- percentiles by `bank_type`; +- lightweight percentiles or small-window samples by `asset_id`. + +Percentiles MUST be updated when a job closes, not inside the inner decode loop. + +### Priority + +When async lane consumers contend, the initial priority order is: + +1. memcard commit/write; +2. FS write/config work; +3. asset visual/audio/scene load; +4. non-critical list/read work. + +This priority order is a lane arbitration rule. It does not define public FS +syscall semantics. + +### FS Boundary + +FS MAY consume the async work lane for IO-style work. + +This decision MUST NOT define the public FS API, app-home FS semantics, or +whether FS is exposed as request/poll or sync-appearing operations. Those +questions belong to `AGD-0006`. + +`AGD-0006` MUST be updated or interpreted with the existence of this async IO +lane in mind. + +## Impactos + +### Spec + +- `docs/specs/runtime/15-asset-management.md` must absorb the async lane, + backlog, handle, superseding, progress, and telemetry contract. +- `docs/specs/runtime/16-host-abi-and-syscalls.md` must absorb any final public + asset backlog syscall names and return shapes. +- `docs/specs/runtime/16a-syscall-policies.md` may need status catalog updates + for superseded/canceled/backend unavailable states if not already covered by + asset domain status. +- `docs/specs/runtime/09-events-and-concurrency.md` should mention the async + work lane as an implementation-side lane that does not introduce guest + callbacks. +- `AGD-0006` / future FS spec work must account for FS as a possible consumer of + the async IO lane. + +### Runtime + +- Asset loading must stop spawning a thread per request. +- AssetManager needs a serial backlog keyed by `bank_type/slot`. +- Asset handles need stable slot identity plus separated `slot_state` and + `request_state`. +- Commit/install remains a main-lane operation. +- Superseding and generation checks become required correctness mechanisms. + +### Host + +- Desktop host may implement the async work lane with one fixed worker thread. +- Host presentation/render worker remains separate and unaffected. + +### Firmware + +- No direct firmware behavior is required for asset backlog mechanics. +- Firmware flows that rely on preload or memcard may later use lane telemetry or + priority policy if needed. + +### Tooling + +- Test tools may need support for inspecting backlog state and forcing request + ordering. +- Asset fixtures should cover queued, active, ready, canceled, superseded, and + already-resident paths. + +## Referencias + +- `AGD-0008` - Async Background Work Lanes for Assets and FS. +- `docs/specs/runtime/15-asset-management.md` +- `docs/specs/runtime/16-host-abi-and-syscalls.md` +- `docs/specs/runtime/16a-syscall-policies.md` +- `docs/specs/runtime/09-events-and-concurrency.md` +- `docs/specs/runtime/08-save-memory-and-memcard.md` +- `AGD-0006` - App Home Filesystem Surface and Semantics. + +## Propagacao Necessaria + +1. Update asset management specs with the lane/backlog/handle contract. +2. Update syscall specs if backlog APIs become public ABI. +3. Update FS agenda/spec work to acknowledge the async IO lane without deciding + FS public API in this decision. +4. Plan implementation of a serial async work lane and asset backlog. +5. Plan tests for superseding, stable slot handles, immediate-ready resident + assets, progress, telemetry, and main-lane commit. + +## Revision Log + +- 2026-06-28: Initial draft from `AGD-0008`. diff --git a/discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md b/discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md new file mode 100644 index 00000000..f3277ed8 --- /dev/null +++ b/discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md @@ -0,0 +1,118 @@ +--- +id: PLN-0123 +ticket: perf-async-background-work-lanes-for-assets-and-fs +title: Async Work Lane Specification Propagation +status: open +created: 2026-06-28 +completed: +ref_decisions: [DEC-0034] +tags: [perf, asset, fs, async, scheduler, runtime, spec] +--- + +# PLN-0123 - Async Work Lane Specification Propagation + +## Briefing + +`DEC-0034` establishes a third serial async work lane, asset backlog semantics, +stable bank-slot handles, superseding, progress, telemetry, and FS/memcard lane +boundaries. The canonical specs must be updated before code work relies on the +new contract. + +## Decisions de Origem + +- `DEC-0034` - Async Work Lane and Asset Backlog Contract. + +## Alvo + +Publish the accepted async work lane and asset backlog contract in the canonical +runtime specs. + +## Escopo + +Included: + +- Update asset management spec. +- Update syscall ABI spec if public backlog APIs are named there. +- Update syscall policy/status language for operational asset states. +- Update events/concurrency spec with the async work lane as an implementation + lane that does not introduce guest callbacks. +- Add a cross-reference from FS/app-home agenda or notes to the async IO lane. + +Fora de Escopo: + +- Implementing runtime code. +- Finalizing public FS API semantics. +- Changing render worker contracts. + +## Plano de Execucao + +### Step 1 - Update Asset Management + +**What:** Add the third async lane, backlog, handle, superseding, progress, and +telemetry contract. + +**How:** Extend `docs/specs/runtime/15-asset-management.md` around the existing +`asset.load/status/commit/cancel` section. Preserve the existing +`asset.load(asset_id, slot)` identity contract. + +**Files:** `docs/specs/runtime/15-asset-management.md`. + +### Step 2 - Update Host ABI + +**What:** Add the backlog API direction if the ABI spec is the canonical place +for syscall shapes. + +**How:** Extend the asset surface section with the planned backlog operations, +marking exact numeric ids as implementation-plan work if ids are not assigned +yet. + +**Files:** `docs/specs/runtime/16-host-abi-and-syscalls.md`. + +### Step 3 - Update Syscall Policies + +**What:** Clarify that `superseded`, `canceled`, `backend_unavailable`, and +decode failures are operational statuses, not faults. + +**How:** Add asset-domain examples under status-first policy. + +**Files:** `docs/specs/runtime/16a-syscall-policies.md`. + +### Step 4 - Update Events and Concurrency + +**What:** Document that the async work lane is an implementation lane. + +**How:** Add a section next to render worker concurrency stating that asset/IO +lane work does not introduce guest callbacks or hidden guest execution. + +**Files:** `docs/specs/runtime/09-events-and-concurrency.md`. + +### Step 5 - Cross-Reference FS Work + +**What:** Record that FS may consume the async IO lane, while public FS API +semantics remain owned by the app-home filesystem discussion. + +**How:** Add a note in `AGD-0006` or its future decision input, without deciding +FS request/poll shape in this plan. + +**Files:** `discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md`. + +## Criterios de Aceite + +- [ ] Specs state that the async work lane is serial and separate from render. +- [ ] Asset spec states backlog keying by `bank_type/slot`. +- [ ] Asset spec states stable slot handles with `slot_state` and `request_state`. +- [ ] Asset spec states superseding behavior. +- [ ] Asset spec states main-lane commit/install. +- [ ] FS is cross-referenced as a permitted lane consumer without public API decisions. + +## Tests / Validacao + +- Run `discussion validate`. +- Review spec links for broken relative paths. +- No cargo tests are required for this documentation-only plan. + +## Riscos + +- Public ABI names may be refined later; the spec must avoid assigning ids too + early if implementation work has not reserved them. +- FS scope can leak into this work; keep FS public API decisions in `AGD-0006`. diff --git a/discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md b/discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md new file mode 100644 index 00000000..888a10ad --- /dev/null +++ b/discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md @@ -0,0 +1,119 @@ +--- +id: PLN-0124 +ticket: perf-async-background-work-lanes-for-assets-and-fs +title: Async Work Lane Runtime Infrastructure +status: open +created: 2026-06-28 +completed: +ref_decisions: [DEC-0034] +tags: [perf, asset, fs, async, scheduler, runtime] +--- + +# PLN-0124 - Async Work Lane Runtime Infrastructure + +## Briefing + +`DEC-0034` requires one runtime-owned async work lane, separate from VM/main +execution and separate from render worker execution. This plan introduces the +lane infrastructure before migrating asset loading. + +## Decisions de Origem + +- `DEC-0034` - Async Work Lane and Asset Backlog Contract. + +## Alvo + +Create a serial async work lane primitive that can execute one active job at a +time, expose observable state, and support deterministic shutdown. + +## Escopo + +Included: + +- Runtime-owned async work lane type. +- Single active job execution. +- Ordered pending job storage. +- Cooperative cancellation hook. +- Shutdown and join behavior. +- Basic lane telemetry. + +Fora de Escopo: + +- Public asset backlog syscalls. +- Asset handle model migration. +- FS public API. +- Render worker changes. + +## Plano de Execucao + +### Step 1 - Locate Runtime Service Boundary + +**What:** Choose the crate/module boundary for the async lane. + +**How:** Place generic lane infrastructure near VM runtime services, not inside +desktop host code and not inside render worker modules. + +**Files:** likely `crates/console/prometeu-system/src/services/` or +`crates/console/prometeu-system/src/services/vm_runtime/`. + +### Step 2 - Define Job and Lane Types + +**What:** Add a lane type with pending queue, active job metadata, generation, +state, progress, and cancellation flag. + +**How:** Use typed job enums or traits that do not require guest callbacks. +Start with internal asset job support but keep the lane generic enough for +memcard/FS later. + +**Files:** new module under `prometeu-system` service layer. + +### Step 3 - Implement Serial Worker + +**What:** Implement one worker loop that takes one pending job and runs it to +completion or cooperative cancellation. + +**How:** Use a fixed worker thread on host builds. Preserve a logical-lane API +so future hardware can map it to a dedicated core. + +**Files:** async lane module; lifecycle wiring in VM runtime service setup. + +### Step 4 - Implement Shutdown + +**What:** Add explicit stop/shutdown behavior. + +**How:** Wake the lane, stop accepting new jobs, report pending jobs as canceled +or shutdown-discarded, and join the worker with bounded behavior. + +**Files:** async lane module and VM runtime lifecycle modules. + +### Step 5 - Expose Internal Telemetry + +**What:** Add counters for pending depth, active job, submitted, completed, +failed, canceled, and superseded. + +**How:** Use cheap atomics or lane-owned state snapshots. Do not update +expensive percentile data inside inner decode loops. + +**Files:** async lane module; telemetry structs where runtime telemetry lives. + +## Criterios de Aceite + +- [ ] Lane has one active job maximum. +- [ ] Lane is separate from render worker code. +- [ ] Lane can be started and stopped deterministically. +- [ ] Pending jobs are observable. +- [ ] Cooperative cancellation can be signaled. +- [ ] No guest callback path is introduced. + +## Tests / Validacao + +- Unit tests for enqueue, single active execution, ordering, cancellation signal, + and shutdown. +- Concurrency tests must synchronize on real state transitions, not sleeps. +- Run targeted `cargo test` for the owning crate. + +## Riscos + +- Accidentally creating a generic pool instead of a serial lane. +- Coupling the lane to desktop host thread details. +- Reusing render worker handoff semantics where ordered backlog is required. diff --git a/discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md b/discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md new file mode 100644 index 00000000..65bb3669 --- /dev/null +++ b/discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md @@ -0,0 +1,127 @@ +--- +id: PLN-0125 +ticket: perf-async-background-work-lanes-for-assets-and-fs +title: Asset Backlog and Stable Slot Handles +status: open +created: 2026-06-28 +completed: +ref_decisions: [DEC-0034] +tags: [perf, asset, async, runtime] +--- + +# PLN-0125 - Asset Backlog and Stable Slot Handles + +## Briefing + +`DEC-0034` changes asset loading from request-owned thread spawning to a serial +backlog keyed by `bank_type/slot`. Handles become stable bank-slot observers +with separate resident slot state and request state. + +## Decisions de Origem + +- `DEC-0034` - Async Work Lane and Asset Backlog Contract. + +## Alvo + +Refactor `AssetManager` so asset requests are tracked by target bank slot, +deduplicated/superseded by target, and observable through stable handles. + +## Escopo + +Included: + +- Stable handle structure with `bank_type`, `slot`, `slot_state`, and + `request_state`. +- Backlog keyed by `bank_type/slot`. +- Already-resident fast path. +- Superseding for queued and active requests. +- Main-lane commit/install boundary. + +Fora de Escopo: + +- Public backlog syscall wiring. +- Percentile telemetry. +- FS public API. + +## Plano de Execucao + +### Step 1 - Model Slot Handles + +**What:** Replace transient load-operation assumptions with stable bank-slot +handle state. + +**How:** Introduce structures equivalent to: + +```text +handle: bank_type, slot +slot_state: loaded_asset_id, resident_state, slot_generation +request_state: requested_asset_id, request_generation, state, position, progress +``` + +**Files:** `crates/console/prometeu-drivers/src/asset.rs` and HAL asset types if +public type definitions belong there. + +### Step 2 - Key Backlog by Target + +**What:** Ensure at most one current request per `bank_type/slot`. + +**How:** Add a target index from `(bank_type, slot)` to current request +generation/handle state. Enqueue only when the target does not already have the +same resident asset and no newer request supersedes it. + +**Files:** `crates/console/prometeu-drivers/src/asset.rs`. + +### Step 3 - Implement Already-Resident Fast Path + +**What:** Return a ready handle immediately when the requested asset already +resides in the target slot. + +**How:** Compare requested `asset_id` with the target `slot_state.loaded_asset_id` +and resident validity before creating a queued job. + +**Files:** `crates/console/prometeu-drivers/src/asset.rs`. + +### Step 4 - Implement Superseding + +**What:** New request for the same target supersedes previous queued or active +request. + +**How:** Remove queued older request immediately. For active work, update target +generation so the old result is discarded if it cannot stop cooperatively. + +**Files:** `crates/console/prometeu-drivers/src/asset.rs`; async lane module +from `PLN-0124`. + +### Step 5 - Preserve Main-Lane Commit + +**What:** Keep install/commit into resident banks on the main runtime lane. + +**How:** Async lane produces staged materialized result. `asset.commit(handle)` +installs only when the request generation is current and ready. + +**Files:** `crates/console/prometeu-drivers/src/asset.rs`; VM runtime dispatch if +commit behavior changes. + +## Criterios de Aceite + +- [ ] No asset load spawns an OS thread per request. +- [ ] At most one request exists per `bank_type/slot`. +- [ ] Same-target newer request supersedes older queued request. +- [ ] Same-target newer request prevents active stale result from installing. +- [ ] Already-resident target returns ready handle without backlog entry. +- [ ] Handles remain queryable for empty, queued, active, ready, committed, + canceled, superseded, and error states. + +## Tests / Validacao + +- Unit tests for handle state transitions. +- Unit tests for target-keyed superseding. +- Unit tests for already-resident fast path. +- Integration test for stale active result discard. +- Existing asset load/commit tests must continue passing. + +## Riscos + +- Confusing handle identity with request generation. +- Installing a stale async result after superseding. +- Losing compatibility with existing `asset.status/commit/cancel` behavior. diff --git a/discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md b/discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md new file mode 100644 index 00000000..8ea105e7 --- /dev/null +++ b/discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md @@ -0,0 +1,124 @@ +--- +id: PLN-0126 +ticket: perf-async-background-work-lanes-for-assets-and-fs +title: Asset Backlog Public API and Status Surface +status: open +created: 2026-06-28 +completed: +ref_decisions: [DEC-0034] +tags: [perf, asset, async, syscall, runtime] +--- + +# PLN-0126 - Asset Backlog Public API and Status Surface + +## Briefing + +`DEC-0034` keeps `asset.load/status/commit/cancel` and adds a small status-first +backlog inspection and ordering surface. This plan wires that surface through +HAL syscall metadata and runtime dispatch. + +## Decisions de Origem + +- `DEC-0034` - Async Work Lane and Asset Backlog Contract. + +## Alvo + +Expose asset backlog inspection and reordering while preserving status-first +semantics and existing asset load identity. + +## Escopo + +Included: + +- Syscall registry additions for backlog APIs. +- Runtime dispatch handlers. +- Status enum additions or normalization. +- ABI/spec alignment with `PLN-0123`. + +Fora de Escopo: + +- Async lane internals. +- Asset decode refactor. +- FS API. + +## Plano de Execucao + +### Step 1 - Finalize Syscall Shapes + +**What:** Choose final names, arity, and return slots for backlog operations. + +**How:** Start from `DEC-0034`: + +```text +asset.backlog_info() +asset.backlog_position(handle) +asset.backlog_move(handle, new_position) +asset.backlog_promote(handle) +asset.backlog_demote(handle) +asset.target_status(bank_type, slot) +``` + +**Files:** `crates/console/prometeu-hal/src/syscalls/domains/asset.rs`, +`docs/specs/runtime/16-host-abi-and-syscalls.md`. + +### Step 2 - Add Status Values + +**What:** Represent `empty`, `invalid`, `queued`, `active`, `ready`, +`committed`, `canceled`, `superseded`, `error`, and `backend_unavailable`. + +**How:** Extend or map existing `LoadStatus` and `AssetOpStatus` without +turning operational states into traps. + +**Files:** HAL asset status types and `crates/console/prometeu-drivers/src/asset.rs`. + +### Step 3 - Add Dispatch + +**What:** Wire new syscalls through VM runtime dispatch. + +**How:** Add match arms near existing `AssetLoad`, `AssetStatus`, +`AssetCommit`, and `AssetCancel` arms. Return status-first values. + +**Files:** `crates/console/prometeu-system/src/services/vm_runtime/dispatch.rs`. + +### Step 4 - Implement API Methods + +**What:** Add AssetManager methods backing each public operation. + +**How:** Methods must read stable handle/target state and reorder only queued +requests. `promote` and `demote` are shortcuts over move semantics. + +**Files:** `crates/console/prometeu-drivers/src/asset.rs`. + +### Step 5 - Validate ABI Metadata + +**What:** Ensure registry ids, args, returns, capability, cost, and +non-deterministic flags are correct. + +**How:** Update syscall metadata tests and any declared PBS/runtime syscall +tables. + +**Files:** `crates/console/prometeu-hal/src/syscalls/tests.rs` and related +registry files. + +## Criterios de Aceite + +- [ ] Public backlog operations are registered with stable metadata. +- [ ] All backlog operations are status-first. +- [ ] `promote` is equivalent to moving to first pending position. +- [ ] `demote` moves to the end of pending backlog. +- [ ] Reordering a non-queued request returns operational status. +- [ ] Structural ABI misuse remains Trap. + +## Tests / Validacao + +- HAL syscall metadata tests. +- VM runtime dispatch tests for each new syscall. +- AssetManager unit tests for move/promote/demote. +- Tests for `target_status` on empty, queued, active, ready, and superseded + targets. + +## Riscos + +- Adding too many public ABI details before spec propagation is accepted. +- Returning payload values that are ambiguous when status is not success. +- Treating known but non-mutable states as traps instead of statuses. diff --git a/discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md b/discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md new file mode 100644 index 00000000..d04bae32 --- /dev/null +++ b/discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md @@ -0,0 +1,120 @@ +--- +id: PLN-0127 +ticket: perf-async-background-work-lanes-for-assets-and-fs +title: Async Lane Memcard and FS Integration Boundaries +status: open +created: 2026-06-28 +completed: +ref_decisions: [DEC-0034] +tags: [perf, fs, memcard, async, runtime] +--- + +# PLN-0127 - Async Lane Memcard and FS Integration Boundaries + +## Briefing + +`DEC-0034` allows memcard and FS work to consume the async IO lane but keeps +public FS API semantics out of scope. This plan establishes integration +boundaries without deciding app-home FS API shape. + +## Decisions de Origem + +- `DEC-0034` - Async Work Lane and Asset Backlog Contract. + +## Alvo + +Define and implement safe internal boundaries for memcard and FS use of the +async work lane, preserving lane priority and keeping FS public API decisions in +`AGD-0006`. + +## Escopo + +Included: + +- Memcard as a priority lane consumer candidate. +- FS as a permitted internal lane consumer. +- Priority ordering in the lane. +- Updates to `AGD-0006` to acknowledge the lane. + +Fora de Escopo: + +- Public FS request/poll design. +- App-home path/status contract. +- Full memcard async conversion if it requires public ABI changes. + +## Plano de Execucao + +### Step 1 - Update FS Agenda + +**What:** Record that an async IO lane exists and may be used by FS. + +**How:** Add a concise section to `AGD-0006` stating that FS public API remains +owned by that agenda, but implementation may use the async lane from `DEC-0034`. + +**Files:** `discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md`. + +### Step 2 - Identify Memcard Operations + +**What:** List memcard operations that can benefit from async IO. + +**How:** Inspect `crates/console/prometeu-system/src/services/memcard.rs` and +VM runtime dispatch for `mem.*` operations. Classify read, write, commit, and +clear behavior. + +**Files:** memcard service and VM runtime dispatch modules. + +### Step 3 - Define Lane Priority + +**What:** Encode the initial priority order from `DEC-0034`. + +**How:** Add priority classes: + +```text +memcard commit/write +FS write/config +asset load +non-critical list/read +``` + +**Files:** async lane module from `PLN-0124`. + +### Step 4 - Add Internal Consumer Hooks + +**What:** Provide internal APIs for memcard/FS to submit lane jobs later. + +**How:** Add types/interfaces without changing public FS syscall behavior. +No public FS request/poll operations are introduced here. + +**Files:** async lane module; FS/memcard service modules as needed. + +### Step 5 - Preserve Current Public Behavior + +**What:** Ensure existing memcard and FS syscalls keep their current observable +surface unless a later plan explicitly changes it. + +**How:** Do not alter return shapes in this plan. Add tests proving existing +status-first behavior remains intact. + +**Files:** VM runtime FS/memcard tests. + +## Criterios de Aceite + +- [ ] `AGD-0006` acknowledges the async IO lane and keeps FS public API in its + own scope. +- [ ] Async lane supports priority classes required by `DEC-0034`. +- [ ] Memcard and FS can be represented as internal lane consumers. +- [ ] Existing public memcard/FS syscall shapes are unchanged. +- [ ] No FS public request/poll API is introduced by this plan. + +## Tests / Validacao + +- Existing FS/memcard tests continue passing. +- Unit tests for lane priority ordering. +- Tests proving asset jobs do not overlap with higher-priority memcard jobs + once both are submitted through the lane. + +## Riscos + +- Accidentally deciding FS public API in the wrong discussion. +- Making memcard async in a way that changes status-first behavior. +- Starving asset loads if priority rules do not include fairness. diff --git a/discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md b/discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md new file mode 100644 index 00000000..7b60662b --- /dev/null +++ b/discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md @@ -0,0 +1,126 @@ +--- +id: PLN-0128 +ticket: perf-async-background-work-lanes-for-assets-and-fs +title: Async Asset Pipeline Contract Tests and Telemetry +status: open +created: 2026-06-28 +completed: +ref_decisions: [DEC-0034] +tags: [perf, asset, async, telemetry, tests, runtime] +--- + +# PLN-0128 - Async Asset Pipeline Contract Tests and Telemetry + +## Briefing + +`DEC-0034` requires observable progress, backlog state, superseding, and cheap +telemetry. This plan adds the validation and telemetry evidence needed to prove +the new async asset pipeline is correct. + +## Decisions de Origem + +- `DEC-0034` - Async Work Lane and Asset Backlog Contract. + +## Alvo + +Provide contract tests and telemetry for the async asset lane so future +implementations can be changed without regressing lane semantics. + +## Escopo + +Included: + +- Contract tests for lane and asset backlog behavior. +- Progress reporting tests. +- Superseding and stale result discard tests. +- Telemetry counters and percentile updates. +- No-sleep deterministic test helpers where needed. + +Fora de Escopo: + +- Implementing public API wiring if `PLN-0126` has not run. +- Full benchmarking infrastructure. +- Host UI display of telemetry. + +## Plano de Execucao + +### Step 1 - Build Deterministic Test Hooks + +**What:** Add test hooks for blocking a lane job at read/decode/stage. + +**How:** Use channels/barriers or explicit fake job steps. Do not rely on +`thread::sleep` for correctness. + +**Files:** async lane tests; asset test support modules. + +### Step 2 - Test Backlog Ordering + +**What:** Prove queued jobs run serially and in backlog order. + +**How:** Submit multiple target slots and assert active job, pending positions, +promote, demote, and move behavior. + +**Files:** asset manager tests and lane tests. + +### Step 3 - Test Superseding + +**What:** Prove newer request for the same `bank_type/slot` supersedes older +requests. + +**How:** Cover queued superseding and active superseding. For active work, prove +stale completion cannot install. + +**Files:** asset manager tests. + +### Step 4 - Test Stable Handle Observability + +**What:** Prove handles are queryable for empty, queued, active, ready, +committed, canceled, superseded, and error states. + +**How:** Use `target_status` and handle status APIs to inspect both `slot_state` +and `request_state`. + +**Files:** asset manager tests; VM dispatch tests if public API is available. + +### Step 5 - Test Progress + +**What:** Prove progress uses `0..10000` and phase boundaries. + +**How:** Fake read/decode/stage phases with known completion points. Validate +non-linear phases do not report false internal progress. + +**Files:** asset manager tests; telemetry tests. + +### Step 6 - Add Telemetry + +**What:** Add counters and percentile aggregation. + +**How:** Update submitted/completed/failed/canceled/superseded counters when job +state closes. Update percentiles by `bank_type` and lightweight `asset_id` +window only at job completion. + +**Files:** asset manager telemetry, async lane telemetry, existing runtime +telemetry types if shared. + +## Criterios de Aceite + +- [ ] Tests prove one active job at a time. +- [ ] Tests prove backlog order and reordering. +- [ ] Tests prove queued and active superseding. +- [ ] Tests prove stale active results are discarded. +- [ ] Tests prove handles remain queryable across all required states. +- [ ] Tests prove progress phase behavior. +- [ ] Telemetry is updated at job closure, not inside inner decode loops. + +## Tests / Validacao + +- Run targeted cargo tests for asset manager, async lane, and VM runtime + dispatch. +- Run existing asset-bank tests to prove current behavior remains compatible. +- Run `discussion validate` after plan state changes. + +## Riscos + +- Tests that depend on sleeps will be flaky. +- Telemetry may become too expensive if updated inside decode loops. +- Public API tests must wait for `PLN-0126` if syscall wiring is not present. From 54dad3b398af8bfa37565e0a0438f7f04a3f2dbb Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Sun, 28 Jun 2026 13:53:53 +0100 Subject: [PATCH 2/9] implements PLN-0123 async work lane specs --- discussion/index.ndjson | 2 +- ...p-home-filesystem-surface-and-semantics.md | 5 + ...ync-work-lane-specification-propagation.md | 2 +- .../runtime/09-events-and-concurrency.md | 28 +++- docs/specs/runtime/15-asset-management.md | 126 +++++++++++++++++- .../specs/runtime/16-host-abi-and-syscalls.md | 18 +++ docs/specs/runtime/16a-syscall-policies.md | 8 ++ 7 files changed, 183 insertions(+), 6 deletions(-) diff --git a/discussion/index.ndjson b/discussion/index.ndjson index eb2097e5..8ac7a833 100644 --- a/discussion/index.ndjson +++ b/discussion/index.ndjson @@ -18,7 +18,7 @@ {"type":"discussion","id":"DSC-0006","status":"open","ticket":"vm-owned-random-service","title":"Agenda - VM-Owned Random Service","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0005","file":"AGD-0005-vm-owned-random-service.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0007","status":"open","ticket":"app-home-filesystem-surface-and-semantics","title":"Agenda - App Home Filesystem Surface and Semantics","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0006","file":"AGD-0006-app-home-filesystem-surface-and-semantics.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0008","status":"done","ticket":"perf-runtime-telemetry-hot-path","title":"Agenda - [PERF] Runtime Telemetry Hot Path","created_at":"2026-03-27","updated_at":"2026-06-04","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0026","file":"discussion/lessons/DSC-0008-perf-runtime-telemetry-hot-path/LSN-0026-push-based-telemetry-model.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} -{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} +{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} {"type":"discussion","id":"DSC-0010","status":"done","ticket":"perf-host-desktop-frame-pacing-and-presentation","title":"Agenda - [PERF] Host Desktop Frame Pacing and Presentation","created_at":"2026-03-27","updated_at":"2026-04-20","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0036","file":"discussion/lessons/DSC-0010-perf-host-desktop-frame-pacing-and-presentation/LSN-0036-frame-publication-and-host-invalidation-must-be-separate.md","status":"done","created_at":"2026-04-20","updated_at":"2026-04-20"}]} {"type":"discussion","id":"DSC-0011","status":"abandoned","ticket":"perf-gfx-render-pipeline-and-dirty-regions","title":"Agenda - [PERF] GFX Render Pipeline and Dirty Regions","created_at":"2026-03-27","updated_at":"2026-06-28","tags":[],"agendas":[{"id":"AGD-0010","file":"AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md","status":"abandoned","created_at":"2026-03-27","updated_at":"2026-06-28","_override_reason":"User explicitly chose to remove this agenda because it no longer applies to the current render scenario; future optimizations can be discussed when needed."}],"decisions":[],"plans":[],"lessons":[],"_override_reason":"User explicitly chose to abandon this discussion because it no longer applies to the current render scenario; future optimizations can be discussed when needed."} {"type":"discussion","id":"DSC-0012","status":"done","ticket":"perf-runtime-introspection-syscalls","title":"Agenda - [PERF] Runtime Introspection Syscalls","created_at":"2026-03-27","updated_at":"2026-04-19","tags":["perf","runtime","syscall","telemetry","debug","asset"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0034","file":"discussion/lessons/DSC-0012-perf-runtime-introspection-syscalls/LSN-0034-host-owned-debug-boundaries.md","status":"done","created_at":"2026-04-19","updated_at":"2026-04-19"}]} diff --git a/discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md b/discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md index 7ff84a23..01bf40cc 100644 --- a/discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md +++ b/discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md @@ -22,6 +22,9 @@ Sem um contrato claro de `home` por app, a API tende a crescer com semantica inc 1. Todo `app` acessa somente sua `home` logica. 2. Nunca ha acesso direto ao filesystem global do host pela userland. 3. O runtime `fs` interno continua cobrindo tanto `game` quanto `app`. +4. Existe uma async IO lane compartilhavel por assets, memcard e FS. Esta + agenda deve decidir a API publica de FS considerando essa lane, mas a + existencia da lane ja esta fechada por `DEC-0034`. ## Alvo da Discussao @@ -116,6 +119,8 @@ No perfil `app` (`home` sandbox), esta agenda passa a ser a fonte normativa para 2. `rename` entra no MVP ou pode ficar para fase seguinte? 3. Qual conjunto minimo de metadados garante portabilidade real entre hosts? 4. Qual grau de atomicidade e obrigatorio para escrita de arquivo no v1? +5. Quais operacoes de FS devem consumir a async IO lane e quais permanecem + sincrono-aparentes para a userland? ## Dependencias diff --git a/discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md b/discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md index f3277ed8..9fa544cf 100644 --- a/discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md +++ b/discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md @@ -2,7 +2,7 @@ id: PLN-0123 ticket: perf-async-background-work-lanes-for-assets-and-fs title: Async Work Lane Specification Propagation -status: open +status: done created: 2026-06-28 completed: ref_decisions: [DEC-0034] diff --git a/docs/specs/runtime/09-events-and-concurrency.md b/docs/specs/runtime/09-events-and-concurrency.md index 9337ad90..497c25c0 100644 --- a/docs/specs/runtime/09-events-and-concurrency.md +++ b/docs/specs/runtime/09-events-and-concurrency.md @@ -127,7 +127,31 @@ Shutdown is explicit and bounded. A shutdown request wakes a waiting worker, causes pending work that will not be consumed to be discarded, and reports a typed failure if the worker cannot join within the configured timeout. -## 8 Determinism and Best Practices +## 8 Async Asset and IO Work Lane + +The asset/IO async work lane is not a machine-visible event source and does not +introduce guest callbacks. It is an implementation-side lane for asset +IO/decode/materialization and compatible persistence IO work. + +The lane is serial: + +- it has at most one active job; +- it keeps an ordered backlog; +- it is separate from the render worker; +- it must not create one OS thread per guest-visible asset request. + +Asset jobs are keyed by target `bank_type/slot`. A newer request for the same +target supersedes the older request. Superseding is an operational status and +does not execute guest code. + +Asset install/commit into resident banks happens on the main runtime lane at +predictable ownership points. The async lane prepares materialized results; it +does not publish resident graphics/audio/scene state directly. + +FS and game persistence services may consume this lane for IO-style work, but +public FS API shape is defined by the FS/app-home contract, not by this chapter. + +## 9 Determinism and Best Practices PROMETEU encourages: @@ -142,7 +166,7 @@ PROMETEU discourages: - hidden timing channels; - ambiguous out-of-band execution. -## 9 Relationship to Other Specs +## 10 Relationship to Other Specs - [`09a-coroutines-and-cooperative-scheduling.md`](09a-coroutines-and-cooperative-scheduling.md) defines coroutine lifecycle and scheduling behavior. - [`10-debug-inspection-and-profiling.md`](10-debug-inspection-and-profiling.md) defines observability and diagnostics surfaces. diff --git a/docs/specs/runtime/15-asset-management.md b/docs/specs/runtime/15-asset-management.md index d2e52306..699c499d 100644 --- a/docs/specs/runtime/15-asset-management.md +++ b/docs/specs/runtime/15-asset-management.md @@ -393,14 +393,129 @@ Fault boundary: Rules: -- `handle` is valid only when `load` status is `OK`; +- `handle` is returned when `load` status is `OK`; +- `handle` represents a stable bank slot target, not a worker thread; +- a known handle remains queryable even when the slot is empty, has no active + request, or has a superseded request; - failed `load` returns `handle = 0`; - `commit` and `cancel` must not be silent no-op for unknown/invalid handle state. - `asset.load` resolves the target bank type from `asset_table` using `asset_id`; - public callers must not supply `asset_name` or `bank_type` to `asset.load`; - slot validation and residency/lifecycle rejection remain in `asset` status space and are not delegated to `bank`. -### 11.2 Minimum status tables +### 11.2 Async work lane and backlog + +Asset loading uses the runtime async work lane. This lane is separate from the +VM/main lane and separate from the render worker lane. + +The asset async lane is serial: + +- it executes at most one active asset job at a time; +- it keeps an ordered backlog of pending requests; +- it must not create one OS thread per `asset.load` request. + +Asset requests are keyed by the target `bank_type/slot`. + +Rules: + +- each `bank_type/slot` has at most one current request; +- a newer request for the same `bank_type/slot` supersedes the previous request; +- if the previous request is queued, it is removed from the backlog; +- if the previous request is active, the lane should cancel cooperatively when + the current phase supports cheap cancellation; +- if active work cannot stop cheaply, it may finish, but the result must be + discarded when its generation no longer matches the target request generation; +- if the target already contains the requested `asset_id` as a valid resident + asset, `asset.load` returns a ready handle without adding a backlog entry. + +The effective backlog size is bounded by the sum of targetable bank slots, +because only one current request can exist per `bank_type/slot`. The runtime +does not expose a guest-visible `queue_full` status for the normal asset +backlog path. + +### 11.3 Handle state + +An asset handle observes one stable bank slot target. Its observable state has +two parts: + +```text +handle: + bank_type + slot + +slot_state: + loaded_asset_id + resident_state + slot_generation + +request_state: + requested_asset_id + request_generation + state + backlog_position + progress +``` + +`slot_state` describes what is currently resident in the target slot. +`request_state` describes the current or most recent request for that target. + +Mutating operations such as commit, cancel, promote, demote, and move must act +on the current request generation. They must not accidentally mutate a newer +request through an older handle view. + +### 11.4 Backlog inspection and ordering + +The asset backlog surface may expose these status-first operations: + +- `asset.backlog_info() -> (status, pending_count, active_handle, active_asset_id, active_bank_type, active_slot, active_progress)` +- `asset.backlog_position(handle) -> (status, state, position, progress)` +- `asset.backlog_move(handle, new_position) -> status` +- `asset.backlog_promote(handle) -> status` +- `asset.backlog_demote(handle) -> status` +- `asset.target_status(bank_type, slot) -> (status, asset_id, handle, state, position, progress)` + +`asset.backlog_promote(handle)` is a shortcut for moving a queued request to +position `1`, the first pending position after the active job. + +`asset.backlog_demote(handle)` is a shortcut for moving a queued request to the +end of the pending backlog. + +### 11.5 Progress and telemetry + +Asset progress uses integer progress, not floating point. The preferred scale is +`0..10000`. + +The initial phase model is: + +```text +queued -> 0 +read -> 0..4000 +decode -> 4000..9000 +stage -> 9000..10000 +ready -> 10000 +``` + +If a phase cannot report internal progress, it keeps the previous progress mark +and advances at phase completion. The runtime must not invent false precision +for non-linear decode phases. + +Minimum telemetry: + +- current backlog depth; +- target/request position; +- active job progress; +- jobs submitted; +- jobs completed; +- jobs failed; +- jobs canceled; +- jobs superseded; +- job duration; +- percentiles by `bank_type`; +- lightweight percentiles or small-window samples by `asset_id`. + +Percentiles are updated when a job closes, not inside the inner decode loop. + +### 11.6 Minimum status tables `asset.load` request statuses: @@ -418,9 +533,16 @@ Rules: - `4` = `CANCELED` - `5` = `ERROR` - `6` = `UNKNOWN_HANDLE` +- `7` = `QUEUED` +- `8` = `ACTIVE` +- `9` = `SUPERSEDED` +- `10` = `EMPTY` +- `11` = `INVALID` +- `12` = `BACKEND_UNAVAILABLE` `asset.commit` and `asset.cancel` operation statuses: - `0` = `OK` - `1` = `UNKNOWN_HANDLE` - `2` = `INVALID_STATE` +- `3` = `SUPERSEDED` diff --git a/docs/specs/runtime/16-host-abi-and-syscalls.md b/docs/specs/runtime/16-host-abi-and-syscalls.md index 34ac736f..5944b4c4 100644 --- a/docs/specs/runtime/16-host-abi-and-syscalls.md +++ b/docs/specs/runtime/16-host-abi-and-syscalls.md @@ -203,6 +203,12 @@ Canonical operations in v1 are: - `asset.status(handle) -> status` - `asset.commit(handle) -> status` - `asset.cancel(handle) -> status` +- `asset.backlog_info() -> (status, pending_count, active_handle, active_asset_id, active_bank_type, active_slot, active_progress)` +- `asset.backlog_position(handle) -> (status, state, position, progress)` +- `asset.backlog_move(handle, new_position) -> status` +- `asset.backlog_promote(handle) -> status` +- `asset.backlog_demote(handle) -> status` +- `asset.target_status(bank_type, slot) -> (status, asset_id, handle, state, position, progress)` For `asset.load`: @@ -210,6 +216,18 @@ For `asset.load`: - `slot` is the target slot index; - bank kind is resolved from `asset_table` by `asset_id`, not supplied by the caller. +Asset handles represent stable bank slot targets. A handle can be queried even +when its slot has no resident asset or active request. Internally the handle +state separates resident `slot_state` from current `request_state`. + +The asset backlog is keyed by `bank_type/slot`. New requests for the same target +supersede older requests for that target. `superseded` is an operational status, +not a structural trap. + +`asset.backlog_promote(handle)` and `asset.backlog_demote(handle)` are +convenience operations over backlog movement. They do not introduce a second +ordering model. + ### Bank diagnostics surface (`bank`, v1) `DEC-0009` narrows the public bank contract: diff --git a/docs/specs/runtime/16a-syscall-policies.md b/docs/specs/runtime/16a-syscall-policies.md index 1efb10ef..71e58c45 100644 --- a/docs/specs/runtime/16a-syscall-policies.md +++ b/docs/specs/runtime/16a-syscall-policies.md @@ -44,6 +44,9 @@ Normal operational success and operational failure conditions should be represen Examples: - asset not yet loaded; +- asset request queued or active; +- asset request superseded by a newer request for the same bank slot; +- asset backend unavailable; - audio voice unavailable; - persistent storage full. @@ -107,6 +110,11 @@ Game memcard operations (`mem.*`) are status-first and use `fs` capability in v1 `mem` remains layered on runtime `fs`; no parallel persistence channel is introduced. Domain surface, status catalog and slot semantics are defined in [`08-save-memory-and-memcard.md`](08-save-memory-and-memcard.md). +Asset backlog operations are status-first. `queued`, `active`, `ready`, +`canceled`, `superseded`, `empty`, `invalid`, decode failure, and backend +unavailability are operational results. They must not be reclassified as `Trap` +unless the caller violates the structural ABI contract. + ## 3 Interaction with the Garbage Collector The VM heap and host-managed memory are separate. From 7598a15782ef34832a6694e94971e30b5eb68cb5 Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Sun, 28 Jun 2026 13:57:41 +0100 Subject: [PATCH 3/9] implements PLN-0124 async work lane infrastructure --- crates/console/prometeu-system/src/lib.rs | 5 + .../src/services/async_work.rs | 459 ++++++++++++++++++ .../prometeu-system/src/services/mod.rs | 1 + discussion/index.ndjson | 2 +- ...-async-work-lane-runtime-infrastructure.md | 2 +- 5 files changed, 467 insertions(+), 2 deletions(-) create mode 100644 crates/console/prometeu-system/src/services/async_work.rs diff --git a/crates/console/prometeu-system/src/lib.rs b/crates/console/prometeu-system/src/lib.rs index 98c39b85..77e44959 100644 --- a/crates/console/prometeu-system/src/lib.rs +++ b/crates/console/prometeu-system/src/lib.rs @@ -7,6 +7,11 @@ pub use crash_report::CrashReport; pub use os::{LifecycleError, LifecycleOperation, SystemOS}; pub use programs::{NativeShellApp, PrometeuHub, SystemProfileAction, SystemProfileUpdate}; pub use prometeu_hal::{RenderWorkerBackend, RenderWorkerFrameSink}; +pub use services::async_work::{ + AsyncWorkActiveJob, AsyncWorkCancelToken, AsyncWorkJobContext, AsyncWorkJobId, + AsyncWorkJobKind, AsyncWorkJobOutcome, AsyncWorkLane, AsyncWorkLaneConfig, + AsyncWorkLaneController, AsyncWorkLaneError, AsyncWorkLaneTelemetry, +}; pub use services::fs; pub use services::process; pub use services::task; diff --git a/crates/console/prometeu-system/src/services/async_work.rs b/crates/console/prometeu-system/src/services/async_work.rs new file mode 100644 index 00000000..002926b3 --- /dev/null +++ b/crates/console/prometeu-system/src/services/async_work.rs @@ -0,0 +1,459 @@ +use std::collections::VecDeque; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct AsyncWorkJobId(u64); + +impl AsyncWorkJobId { + pub const ZERO: Self = Self(0); + + pub fn new(value: u64) -> Self { + Self(value) + } + + pub fn get(self) -> u64 { + self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AsyncWorkJobKind { + Asset, + Memcard, + Fs, + Test, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AsyncWorkJobOutcome { + Completed, + Canceled, + Failed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AsyncWorkLaneError { + Shutdown, + WorkerPanic, + ShutdownTimeout, + InternalFailure, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AsyncWorkLaneConfig { + pub shutdown_timeout: Duration, +} + +impl Default for AsyncWorkLaneConfig { + fn default() -> Self { + Self { shutdown_timeout: Duration::from_millis(250) } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AsyncWorkActiveJob { + pub id: AsyncWorkJobId, + pub kind: AsyncWorkJobKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct AsyncWorkLaneTelemetry { + pub submitted_jobs: u64, + pub started_jobs: u64, + pub completed_jobs: u64, + pub canceled_jobs: u64, + pub failed_jobs: u64, + pub shutdown_discards: u64, + pub pending_depth: usize, + pub max_pending_depth: usize, + pub active_job_id: Option, + pub last_submitted_job_id: Option, + pub last_started_job_id: Option, + pub last_closed_job_id: Option, +} + +impl AsyncWorkLaneTelemetry { + fn record_submitted(&mut self, job_id: AsyncWorkJobId, pending_depth: usize) { + self.submitted_jobs += 1; + self.pending_depth = pending_depth; + self.max_pending_depth = self.max_pending_depth.max(pending_depth); + self.last_submitted_job_id = Some(job_id); + } + + fn record_started(&mut self, job_id: AsyncWorkJobId, pending_depth: usize) { + self.started_jobs += 1; + self.pending_depth = pending_depth; + self.active_job_id = Some(job_id); + self.last_started_job_id = Some(job_id); + } + + fn record_closed(&mut self, job_id: AsyncWorkJobId, outcome: AsyncWorkJobOutcome) { + match outcome { + AsyncWorkJobOutcome::Completed => self.completed_jobs += 1, + AsyncWorkJobOutcome::Canceled => self.canceled_jobs += 1, + AsyncWorkJobOutcome::Failed => self.failed_jobs += 1, + } + self.active_job_id = None; + self.last_closed_job_id = Some(job_id); + } + + fn record_shutdown_discards(&mut self, count: usize) { + self.shutdown_discards += count as u64; + self.pending_depth = 0; + } +} + +#[derive(Clone, Debug)] +pub struct AsyncWorkCancelToken { + canceled: Arc, +} + +impl AsyncWorkCancelToken { + fn new() -> Self { + Self { canceled: Arc::new(AtomicBool::new(false)) } + } + + pub fn cancel(&self) { + self.canceled.store(true, Ordering::Release); + } + + pub fn is_canceled(&self) -> bool { + self.canceled.load(Ordering::Acquire) + } +} + +#[derive(Clone, Debug)] +pub struct AsyncWorkJobContext { + pub job_id: AsyncWorkJobId, + pub kind: AsyncWorkJobKind, + cancel_token: AsyncWorkCancelToken, +} + +impl AsyncWorkJobContext { + pub fn is_canceled(&self) -> bool { + self.cancel_token.is_canceled() + } +} + +type AsyncWorkFn = Box AsyncWorkJobOutcome + Send + 'static>; + +struct PendingAsyncWorkJob { + id: AsyncWorkJobId, + kind: AsyncWorkJobKind, + cancel_token: AsyncWorkCancelToken, + work: AsyncWorkFn, +} + +struct RunningAsyncWorkJob { + id: AsyncWorkJobId, + kind: AsyncWorkJobKind, + cancel_token: AsyncWorkCancelToken, + work: AsyncWorkFn, +} + +#[derive(Default)] +struct AsyncWorkLaneState { + next_job_id: u64, + pending: VecDeque, + active: Option, + active_cancel_token: Option, + shutdown_requested: bool, + telemetry: AsyncWorkLaneTelemetry, +} + +#[derive(Default)] +pub struct AsyncWorkLane { + state: Mutex, + ready: Condvar, +} + +impl AsyncWorkLane { + pub fn submit( + &self, + kind: AsyncWorkJobKind, + work: impl FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static, + ) -> Result { + let mut state = self.state.lock().map_err(|_| AsyncWorkLaneError::InternalFailure)?; + if state.shutdown_requested { + return Err(AsyncWorkLaneError::Shutdown); + } + + state.next_job_id += 1; + let id = AsyncWorkJobId::new(state.next_job_id); + let cancel_token = AsyncWorkCancelToken::new(); + state.pending.push_back(PendingAsyncWorkJob { + id, + kind, + cancel_token, + work: Box::new(work), + }); + let pending_depth = state.pending.len(); + state.telemetry.record_submitted(id, pending_depth); + self.ready.notify_one(); + Ok(id) + } + + pub fn request_cancel(&self, job_id: AsyncWorkJobId) -> bool { + let state = self.state.lock().unwrap(); + if state.active.is_some_and(|active| active.id == job_id) { + if let Some(token) = &state.active_cancel_token { + token.cancel(); + return true; + } + } + + for pending in &state.pending { + if pending.id == job_id { + pending.cancel_token.cancel(); + return true; + } + } + + false + } + + pub fn active_job(&self) -> Option { + self.state.lock().unwrap().active + } + + pub fn pending_depth(&self) -> usize { + self.state.lock().unwrap().pending.len() + } + + pub fn telemetry(&self) -> AsyncWorkLaneTelemetry { + self.state.lock().unwrap().telemetry + } + + pub fn request_shutdown(&self) { + let mut state = self.state.lock().unwrap(); + state.shutdown_requested = true; + if let Some(token) = &state.active_cancel_token { + token.cancel(); + } + self.ready.notify_all(); + } + + fn wait_take(&self) -> Option { + self.wait_take_with_hook(|| {}) + } + + fn wait_take_with_hook(&self, mut before_wait: impl FnMut()) -> Option { + let mut state = self.state.lock().unwrap(); + loop { + if state.shutdown_requested { + let discarded = state.pending.len(); + state.pending.clear(); + state.telemetry.record_shutdown_discards(discarded); + return None; + } + + if let Some(pending) = state.pending.pop_front() { + let active = AsyncWorkActiveJob { id: pending.id, kind: pending.kind }; + state.active = Some(active); + state.active_cancel_token = Some(pending.cancel_token.clone()); + let pending_depth = state.pending.len(); + state.telemetry.record_started(pending.id, pending_depth); + return Some(RunningAsyncWorkJob { + id: pending.id, + kind: pending.kind, + cancel_token: pending.cancel_token, + work: pending.work, + }); + } + + before_wait(); + state = self.ready.wait(state).unwrap(); + } + } + + fn record_finished(&self, job_id: AsyncWorkJobId, outcome: AsyncWorkJobOutcome) { + let mut state = self.state.lock().unwrap(); + if state.active.is_some_and(|active| active.id == job_id) { + state.active = None; + state.active_cancel_token = None; + } + state.telemetry.record_closed(job_id, outcome); + } +} + +pub struct AsyncWorkLaneController { + config: AsyncWorkLaneConfig, + lane: Arc, + handle: Option>, + done_rx: Receiver>, +} + +impl AsyncWorkLaneController { + pub fn start(config: AsyncWorkLaneConfig, lane: Arc) -> Self { + let worker_lane = Arc::clone(&lane); + let (done_tx, done_rx) = mpsc::channel(); + let handle = thread::spawn(move || { + let result = catch_unwind(AssertUnwindSafe(|| run_async_work_loop(worker_lane))) + .map_err(|_| AsyncWorkLaneError::WorkerPanic); + let _ = done_tx.send(result); + }); + + Self { config, lane, handle: Some(handle), done_rx } + } + + pub fn lane(&self) -> Arc { + Arc::clone(&self.lane) + } + + pub fn stop(&mut self) -> Result<(), AsyncWorkLaneError> { + self.lane.request_shutdown(); + match self.done_rx.recv_timeout(self.config.shutdown_timeout) { + Ok(result) => { + if let Some(handle) = self.handle.take() + && handle.join().is_err() + { + return Err(AsyncWorkLaneError::WorkerPanic); + } + result + } + Err(RecvTimeoutError::Timeout) => Err(AsyncWorkLaneError::ShutdownTimeout), + Err(RecvTimeoutError::Disconnected) => Err(AsyncWorkLaneError::InternalFailure), + } + } +} + +impl Drop for AsyncWorkLaneController { + fn drop(&mut self) { + if self.handle.is_some() { + let _ = self.stop(); + } + } +} + +fn run_async_work_loop(lane: Arc) { + while let Some(job) = lane.wait_take() { + let context = + AsyncWorkJobContext { job_id: job.id, kind: job.kind, cancel_token: job.cancel_token }; + let result = catch_unwind(AssertUnwindSafe(|| (job.work)(context))) + .unwrap_or(AsyncWorkJobOutcome::Failed); + lane.record_finished(job.id, result); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::mpsc; + + #[test] + fn async_work_lane_runs_jobs_in_submission_order() { + let lane = Arc::new(AsyncWorkLane::default()); + let mut controller = + AsyncWorkLaneController::start(AsyncWorkLaneConfig::default(), Arc::clone(&lane)); + let (tx, rx) = mpsc::channel(); + + lane.submit(AsyncWorkJobKind::Test, { + let tx = tx.clone(); + move |_| { + tx.send(1).unwrap(); + AsyncWorkJobOutcome::Completed + } + }) + .unwrap(); + lane.submit(AsyncWorkJobKind::Test, { + let tx = tx.clone(); + move |_| { + tx.send(2).unwrap(); + AsyncWorkJobOutcome::Completed + } + }) + .unwrap(); + + assert_eq!(rx.recv().unwrap(), 1); + assert_eq!(rx.recv().unwrap(), 2); + controller.stop().unwrap(); + + let telemetry = lane.telemetry(); + assert_eq!(telemetry.submitted_jobs, 2); + assert_eq!(telemetry.started_jobs, 2); + assert_eq!(telemetry.completed_jobs, 2); + } + + #[test] + fn async_work_lane_has_one_active_job() { + let lane = Arc::new(AsyncWorkLane::default()); + let mut controller = + AsyncWorkLaneController::start(AsyncWorkLaneConfig::default(), Arc::clone(&lane)); + let (first_started_tx, first_started_rx) = mpsc::channel(); + let (release_first_tx, release_first_rx) = mpsc::channel(); + let (second_started_tx, second_started_rx) = mpsc::channel(); + + let first = lane + .submit(AsyncWorkJobKind::Test, move |_| { + first_started_tx.send(()).unwrap(); + release_first_rx.recv().unwrap(); + AsyncWorkJobOutcome::Completed + }) + .unwrap(); + let second = lane + .submit(AsyncWorkJobKind::Test, move |_| { + second_started_tx.send(()).unwrap(); + AsyncWorkJobOutcome::Completed + }) + .unwrap(); + + first_started_rx.recv().unwrap(); + assert_eq!(lane.active_job().unwrap().id, first); + assert_eq!(lane.pending_depth(), 1); + assert!(second_started_rx.try_recv().is_err()); + + release_first_tx.send(()).unwrap(); + second_started_rx.recv().unwrap(); + assert_eq!(lane.telemetry().last_started_job_id, Some(second)); + controller.stop().unwrap(); + } + + #[test] + fn async_work_lane_supports_cooperative_cancellation() { + let lane = Arc::new(AsyncWorkLane::default()); + let mut controller = + AsyncWorkLaneController::start(AsyncWorkLaneConfig::default(), Arc::clone(&lane)); + let (started_tx, started_rx) = mpsc::channel(); + + let job_id = lane + .submit(AsyncWorkJobKind::Test, move |ctx| { + started_tx.send(()).unwrap(); + while !ctx.is_canceled() { + std::thread::yield_now(); + } + AsyncWorkJobOutcome::Canceled + }) + .unwrap(); + + started_rx.recv().unwrap(); + assert!(lane.request_cancel(job_id)); + controller.stop().unwrap(); + assert_eq!(lane.telemetry().canceled_jobs, 1); + } + + #[test] + fn async_work_lane_rejects_submit_after_shutdown() { + let lane = Arc::new(AsyncWorkLane::default()); + lane.request_shutdown(); + + let result = lane.submit(AsyncWorkJobKind::Test, |_| AsyncWorkJobOutcome::Completed); + assert_eq!(result, Err(AsyncWorkLaneError::Shutdown)); + } + + #[test] + fn async_work_lane_shutdown_discards_pending_jobs() { + let lane = Arc::new(AsyncWorkLane::default()); + lane.submit(AsyncWorkJobKind::Test, |_| AsyncWorkJobOutcome::Completed).unwrap(); + assert_eq!(lane.pending_depth(), 1); + + lane.request_shutdown(); + assert!(lane.wait_take().is_none()); + assert_eq!(lane.pending_depth(), 0); + assert_eq!(lane.telemetry().shutdown_discards, 1); + } +} diff --git a/crates/console/prometeu-system/src/services/mod.rs b/crates/console/prometeu-system/src/services/mod.rs index 6dbdfab5..efe827c6 100644 --- a/crates/console/prometeu-system/src/services/mod.rs +++ b/crates/console/prometeu-system/src/services/mod.rs @@ -1,3 +1,4 @@ +pub mod async_work; pub mod fs; pub mod memcard; pub mod process; diff --git a/discussion/index.ndjson b/discussion/index.ndjson index 8ac7a833..109b9620 100644 --- a/discussion/index.ndjson +++ b/discussion/index.ndjson @@ -18,7 +18,7 @@ {"type":"discussion","id":"DSC-0006","status":"open","ticket":"vm-owned-random-service","title":"Agenda - VM-Owned Random Service","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0005","file":"AGD-0005-vm-owned-random-service.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0007","status":"open","ticket":"app-home-filesystem-surface-and-semantics","title":"Agenda - App Home Filesystem Surface and Semantics","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0006","file":"AGD-0006-app-home-filesystem-surface-and-semantics.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0008","status":"done","ticket":"perf-runtime-telemetry-hot-path","title":"Agenda - [PERF] Runtime Telemetry Hot Path","created_at":"2026-03-27","updated_at":"2026-06-04","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0026","file":"discussion/lessons/DSC-0008-perf-runtime-telemetry-hot-path/LSN-0026-push-based-telemetry-model.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} -{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} +{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} {"type":"discussion","id":"DSC-0010","status":"done","ticket":"perf-host-desktop-frame-pacing-and-presentation","title":"Agenda - [PERF] Host Desktop Frame Pacing and Presentation","created_at":"2026-03-27","updated_at":"2026-04-20","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0036","file":"discussion/lessons/DSC-0010-perf-host-desktop-frame-pacing-and-presentation/LSN-0036-frame-publication-and-host-invalidation-must-be-separate.md","status":"done","created_at":"2026-04-20","updated_at":"2026-04-20"}]} {"type":"discussion","id":"DSC-0011","status":"abandoned","ticket":"perf-gfx-render-pipeline-and-dirty-regions","title":"Agenda - [PERF] GFX Render Pipeline and Dirty Regions","created_at":"2026-03-27","updated_at":"2026-06-28","tags":[],"agendas":[{"id":"AGD-0010","file":"AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md","status":"abandoned","created_at":"2026-03-27","updated_at":"2026-06-28","_override_reason":"User explicitly chose to remove this agenda because it no longer applies to the current render scenario; future optimizations can be discussed when needed."}],"decisions":[],"plans":[],"lessons":[],"_override_reason":"User explicitly chose to abandon this discussion because it no longer applies to the current render scenario; future optimizations can be discussed when needed."} {"type":"discussion","id":"DSC-0012","status":"done","ticket":"perf-runtime-introspection-syscalls","title":"Agenda - [PERF] Runtime Introspection Syscalls","created_at":"2026-03-27","updated_at":"2026-04-19","tags":["perf","runtime","syscall","telemetry","debug","asset"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0034","file":"discussion/lessons/DSC-0012-perf-runtime-introspection-syscalls/LSN-0034-host-owned-debug-boundaries.md","status":"done","created_at":"2026-04-19","updated_at":"2026-04-19"}]} diff --git a/discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md b/discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md index 888a10ad..041c40c3 100644 --- a/discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md +++ b/discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md @@ -2,7 +2,7 @@ id: PLN-0124 ticket: perf-async-background-work-lanes-for-assets-and-fs title: Async Work Lane Runtime Infrastructure -status: open +status: done created: 2026-06-28 completed: ref_decisions: [DEC-0034] From 30c24a4ea7a49fe3de0bf803001d5585290b0c08 Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Sun, 28 Jun 2026 14:02:53 +0100 Subject: [PATCH 4/9] implements PLN-0125 asset backlog stable handles --- crates/console/prometeu-drivers/src/asset.rs | 596 ++++++++++++++---- discussion/index.ndjson | 2 +- ...5-asset-backlog-and-stable-slot-handles.md | 2 +- 3 files changed, 458 insertions(+), 142 deletions(-) diff --git a/crates/console/prometeu-drivers/src/asset.rs b/crates/console/prometeu-drivers/src/asset.rs index 5adb1f12..c1dff3c7 100644 --- a/crates/console/prometeu-drivers/src/asset.rs +++ b/crates/console/prometeu-drivers/src/asset.rs @@ -18,10 +18,10 @@ use prometeu_hal::scene_layer::{ParallaxFactor, SceneLayer}; use prometeu_hal::sound_bank::SoundBank; use prometeu_hal::tile::Tile; use prometeu_hal::tilemap::TileMap; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::io::Read; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Condvar, Mutex, RwLock}; use std::thread; use std::time::Instant; @@ -31,6 +31,8 @@ type StagingMap = HashMap>; type AssetTable = HashMap; type HandleTable = HashMap; +type TargetHandleTable = HashMap; +type TargetGenerationTable = HashMap; #[derive(Clone, Default)] pub struct GlyphAssetSlotIndex { @@ -182,6 +184,8 @@ impl BankPolicy { pub struct AssetManager { assets: Arc>, handles: Arc>, + target_handles: Arc>, + target_generations: Arc>, next_handle_id: Mutex, assets_data: Arc>, @@ -203,14 +207,123 @@ pub struct AssetManager { /// Residency policy for scene banks. scene_policy: BankPolicy, + load_worker: AssetLoadWorker, + // Commits that are ready to be applied at the next frame boundary. - pending_commits: Mutex>, + pending_commits: Mutex>, } struct LoadHandleInfo { _asset_id: AssetId, slot: SlotRef, status: LoadStatus, + request_generation: u64, +} + +struct AssetLoadJob { + handle_id: HandleId, + asset_id: AssetId, + slot: SlotRef, + request_generation: u64, + entry: AssetEntry, +} + +#[derive(Default)] +struct AssetLoadQueueState { + pending: VecDeque, + shutdown: bool, +} + +#[derive(Default)] +struct AssetLoadQueue { + state: Mutex, + ready: Condvar, +} + +impl AssetLoadQueue { + fn submit(&self, job: AssetLoadJob) { + let mut state = self.state.lock().unwrap(); + if state.shutdown { + return; + } + state.pending.push_back(job); + self.ready.notify_one(); + } + + fn next_job(&self) -> Option { + let mut state = self.state.lock().unwrap(); + loop { + if let Some(job) = state.pending.pop_front() { + return Some(job); + } + if state.shutdown { + return None; + } + state = self.ready.wait(state).unwrap(); + } + } + + fn clear_pending(&self) { + self.state.lock().unwrap().pending.clear(); + } + + fn shutdown(&self) { + let mut state = self.state.lock().unwrap(); + state.pending.clear(); + state.shutdown = true; + self.ready.notify_all(); + } +} + +struct AssetLoadWorker { + queue: Arc, + join_handle: Option>, +} + +impl AssetLoadWorker { + fn new( + handles: Arc>, + target_generations: Arc>, + assets_data: Arc>, + gfx_policy: BankPolicy, + sound_policy: BankPolicy, + scene_policy: BankPolicy, + ) -> Self { + let queue = Arc::new(AssetLoadQueue::default()); + let worker_queue = Arc::clone(&queue); + let join_handle = thread::spawn(move || { + while let Some(job) = worker_queue.next_job() { + AssetManager::process_load_job( + job, + &handles, + &target_generations, + &assets_data, + &gfx_policy, + &sound_policy, + &scene_policy, + ); + } + }); + + Self { queue, join_handle: Some(join_handle) } + } + + fn submit(&self, job: AssetLoadJob) { + self.queue.submit(job); + } + + fn clear_pending(&self) { + self.queue.clear_pending(); + } +} + +impl Drop for AssetLoadWorker { + fn drop(&mut self) { + self.queue.shutdown(); + if let Some(join_handle) = self.join_handle.take() { + let _ = join_handle.join(); + } + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -333,6 +446,21 @@ impl AssetManager { asset_map.insert(entry.asset_id, entry); } + let handles = Arc::new(RwLock::new(HashMap::new())); + let target_generations = Arc::new(RwLock::new(HashMap::new())); + let assets_data = Arc::new(RwLock::new(assets_data)); + let gfx_policy = BankPolicy::new(); + let sound_policy = BankPolicy::new(); + let scene_policy = BankPolicy::new(); + let load_worker = AssetLoadWorker::new( + Arc::clone(&handles), + Arc::clone(&target_generations), + Arc::clone(&assets_data), + gfx_policy.clone(), + sound_policy.clone(), + scene_policy.clone(), + ); + Self { assets: Arc::new(RwLock::new(asset_map)), gfx_installer, @@ -342,12 +470,15 @@ impl AssetManager { glyph_slot_index: GlyphAssetSlotIndex::new(), sound_slots: Arc::new(RwLock::new(std::array::from_fn(|_| None))), scene_slots: Arc::new(RwLock::new(std::array::from_fn(|_| None))), - gfx_policy: BankPolicy::new(), - sound_policy: BankPolicy::new(), - scene_policy: BankPolicy::new(), - handles: Arc::new(RwLock::new(HashMap::new())), + gfx_policy, + sound_policy, + scene_policy, + handles, + target_handles: Arc::new(RwLock::new(HashMap::new())), + target_generations, next_handle_id: Mutex::new(1), - assets_data: Arc::new(RwLock::new(assets_data)), + assets_data, + load_worker, pending_commits: Mutex::new(Vec::new()), } } @@ -435,6 +566,51 @@ impl AssetManager { } } + fn handle_for_slot(&self, slot: SlotRef) -> HandleId { + if let Some(handle_id) = self.target_handles.read().unwrap().get(&slot).copied() { + return handle_id; + } + + let mut target_handles = self.target_handles.write().unwrap(); + if let Some(handle_id) = target_handles.get(&slot).copied() { + return handle_id; + } + + let mut next_id = self.next_handle_id.lock().unwrap(); + let handle_id = *next_id; + *next_id += 1; + target_handles.insert(slot, handle_id); + handle_id + } + + fn bump_target_generation(&self, slot: SlotRef) -> u64 { + let mut generations = self.target_generations.write().unwrap(); + let generation = generations.entry(slot).or_insert(0); + *generation += 1; + *generation + } + + fn current_target_generation( + target_generations: &Arc>, + slot: SlotRef, + ) -> u64 { + target_generations.read().unwrap().get(&slot).copied().unwrap_or(0) + } + + fn is_current_target_generation( + target_generations: &Arc>, + slot: SlotRef, + request_generation: u64, + ) -> bool { + Self::current_target_generation(target_generations, slot) == request_generation + } + + fn clear_staging_for_handle(&self, handle_id: HandleId) { + self.gfx_policy.take_staging(handle_id); + self.sound_policy.take_staging(handle_id); + self.scene_policy.take_staging(handle_id); + } + pub fn load(&self, asset_id: AssetId, slot_index: usize) -> Result { if slot_index >= 16 { return Err(AssetLoadError::SlotIndexInvalid); @@ -449,9 +625,9 @@ impl AssetManager { BankType::SCENE => SlotRef::scene(slot_index), }; - let mut next_id = self.next_handle_id.lock().unwrap(); - let handle_id = *next_id; - *next_id += 1; + let handle_id = self.handle_for_slot(slot); + let request_generation = self.bump_target_generation(slot); + self.clear_staging_for_handle(handle_id); // Check if already resident (Dedup) let already_resident = match entry.bank_type { @@ -484,7 +660,12 @@ impl AssetManager { if already_resident { self.handles.write().unwrap().insert( handle_id, - LoadHandleInfo { _asset_id: asset_id, slot, status: LoadStatus::READY }, + LoadHandleInfo { + _asset_id: asset_id, + slot, + status: LoadStatus::READY, + request_generation, + }, ); return Ok(handle_id); } @@ -492,124 +673,173 @@ impl AssetManager { // Not resident, start loading self.handles.write().unwrap().insert( handle_id, - LoadHandleInfo { _asset_id: asset_id, slot, status: LoadStatus::PENDING }, + LoadHandleInfo { + _asset_id: asset_id, + slot, + status: LoadStatus::PENDING, + request_generation, + }, ); - let handles = self.handles.clone(); - let assets_data = self.assets_data.clone(); - let entry_clone = entry.clone(); - - // Capture policies for the worker thread - let gfx_policy = self.gfx_policy.clone(); - let sound_policy = self.sound_policy.clone(); - let scene_policy = self.scene_policy.clone(); - - thread::spawn(move || { - // Update status to LOADING - { - let mut handles_map = handles.write().unwrap(); - if let Some(h) = handles_map.get_mut(&handle_id) { - if h.status == LoadStatus::PENDING { - h.status = LoadStatus::LOADING; - } else { - return; - } - } else { - return; - } - } - - match entry_clone.bank_type { - BankType::GLYPH => { - let result = Self::perform_load_glyph_bank(&entry_clone, assets_data); - if let Ok(tilebank) = result { - let bank_arc = Arc::new(tilebank); - let resident_arc = gfx_policy.put_resident( - asset_id, - bank_arc, - entry_clone.decoded_size as usize, - ); - gfx_policy.stage( - handle_id, - resident_arc, - entry_clone.decoded_size as usize, - ); - - let mut handles_map = handles.write().unwrap(); - if let Some(h) = handles_map.get_mut(&handle_id) { - if h.status == LoadStatus::LOADING { - h.status = LoadStatus::READY; - } - } - } else { - let mut handles_map = handles.write().unwrap(); - if let Some(h) = handles_map.get_mut(&handle_id) { - h.status = LoadStatus::ERROR; - } - } - } - BankType::SOUNDS => { - let result = Self::perform_load_sound_bank(&entry_clone, assets_data); - if let Ok(soundbank) = result { - let bank_arc = Arc::new(soundbank); - let resident_arc = sound_policy.put_resident( - asset_id, - bank_arc, - entry_clone.decoded_size as usize, - ); - sound_policy.stage( - handle_id, - resident_arc, - entry_clone.decoded_size as usize, - ); - - let mut handles_map = handles.write().unwrap(); - if let Some(h) = handles_map.get_mut(&handle_id) { - if h.status == LoadStatus::LOADING { - h.status = LoadStatus::READY; - } - } - } else { - let mut handles_map = handles.write().unwrap(); - if let Some(h) = handles_map.get_mut(&handle_id) { - h.status = LoadStatus::ERROR; - } - } - } - BankType::SCENE => { - let result = Self::perform_load_scene_bank(&entry_clone, assets_data); - if let Ok(scenebank) = result { - let bank_arc = Arc::new(scenebank); - let resident_arc = scene_policy.put_resident( - asset_id, - bank_arc, - entry_clone.decoded_size as usize, - ); - scene_policy.stage( - handle_id, - resident_arc, - entry_clone.decoded_size as usize, - ); - - let mut handles_map = handles.write().unwrap(); - if let Some(h) = handles_map.get_mut(&handle_id) { - if h.status == LoadStatus::LOADING { - h.status = LoadStatus::READY; - } - } - } else { - let mut handles_map = handles.write().unwrap(); - if let Some(h) = handles_map.get_mut(&handle_id) { - h.status = LoadStatus::ERROR; - } - } - } - } + self.load_worker.submit(AssetLoadJob { + handle_id, + asset_id, + slot, + request_generation, + entry, }); Ok(handle_id) } + fn process_load_job( + job: AssetLoadJob, + handles: &Arc>, + target_generations: &Arc>, + assets_data: &Arc>, + gfx_policy: &BankPolicy, + sound_policy: &BankPolicy, + scene_policy: &BankPolicy, + ) { + if !Self::is_current_target_generation(target_generations, job.slot, job.request_generation) + { + return; + } + + { + let mut handles_map = handles.write().unwrap(); + let Some(handle) = handles_map.get_mut(&job.handle_id) else { + return; + }; + if handle.request_generation != job.request_generation + || handle.status != LoadStatus::PENDING + { + return; + } + handle.status = LoadStatus::LOADING; + } + + match job.entry.bank_type { + BankType::GLYPH => { + let result = Self::perform_load_glyph_bank(&job.entry, Arc::clone(assets_data)); + match result { + Ok(tilebank) => { + if !Self::is_current_target_generation( + target_generations, + job.slot, + job.request_generation, + ) { + return; + } + let bank_arc = Arc::new(tilebank); + let resident_arc = gfx_policy.put_resident( + job.asset_id, + bank_arc, + job.entry.decoded_size as usize, + ); + if !Self::is_current_target_generation( + target_generations, + job.slot, + job.request_generation, + ) { + return; + } + gfx_policy.stage( + job.handle_id, + resident_arc, + job.entry.decoded_size as usize, + ); + Self::complete_load_job(handles, &job, LoadStatus::READY); + } + Err(_) => Self::complete_load_job(handles, &job, LoadStatus::ERROR), + } + } + BankType::SOUNDS => { + let result = Self::perform_load_sound_bank(&job.entry, Arc::clone(assets_data)); + match result { + Ok(soundbank) => { + if !Self::is_current_target_generation( + target_generations, + job.slot, + job.request_generation, + ) { + return; + } + let bank_arc = Arc::new(soundbank); + let resident_arc = sound_policy.put_resident( + job.asset_id, + bank_arc, + job.entry.decoded_size as usize, + ); + if !Self::is_current_target_generation( + target_generations, + job.slot, + job.request_generation, + ) { + return; + } + sound_policy.stage( + job.handle_id, + resident_arc, + job.entry.decoded_size as usize, + ); + Self::complete_load_job(handles, &job, LoadStatus::READY); + } + Err(_) => Self::complete_load_job(handles, &job, LoadStatus::ERROR), + } + } + BankType::SCENE => { + let result = Self::perform_load_scene_bank(&job.entry, Arc::clone(assets_data)); + match result { + Ok(scenebank) => { + if !Self::is_current_target_generation( + target_generations, + job.slot, + job.request_generation, + ) { + return; + } + let bank_arc = Arc::new(scenebank); + let resident_arc = scene_policy.put_resident( + job.asset_id, + bank_arc, + job.entry.decoded_size as usize, + ); + if !Self::is_current_target_generation( + target_generations, + job.slot, + job.request_generation, + ) { + return; + } + scene_policy.stage( + job.handle_id, + resident_arc, + job.entry.decoded_size as usize, + ); + Self::complete_load_job(handles, &job, LoadStatus::READY); + } + Err(_) => Self::complete_load_job(handles, &job, LoadStatus::ERROR), + } + } + } + } + + fn complete_load_job( + handles: &Arc>, + job: &AssetLoadJob, + status: LoadStatus, + ) { + let mut handles_map = handles.write().unwrap(); + if let Some(handle) = handles_map.get_mut(&job.handle_id) { + if handle.request_generation == job.request_generation + && handle.status == LoadStatus::LOADING + { + handle.status = status; + } + } + } + fn perform_load_glyph_bank( entry: &AssetEntry, assets_data: Arc>, @@ -960,7 +1190,7 @@ impl AssetManager { return AssetOpStatus::UnknownHandle; }; if h.status == LoadStatus::READY { - self.pending_commits.lock().unwrap().push(handle); + self.pending_commits.lock().unwrap().push((handle, h.request_generation)); AssetOpStatus::Ok } else { AssetOpStatus::InvalidState @@ -969,23 +1199,30 @@ impl AssetManager { pub fn cancel(&self, handle: HandleId) -> AssetOpStatus { let mut final_status = AssetOpStatus::UnknownHandle; - let mut handles_map = self.handles.write().unwrap(); - if let Some(h) = handles_map.get_mut(&handle) { - final_status = match h.status { - LoadStatus::PENDING | LoadStatus::LOADING | LoadStatus::READY => AssetOpStatus::Ok, - LoadStatus::CANCELED => AssetOpStatus::Ok, - _ => AssetOpStatus::InvalidState, - }; - match h.status { - LoadStatus::PENDING | LoadStatus::LOADING | LoadStatus::READY => { - h.status = LoadStatus::CANCELED; + let mut canceled_slot = None; + { + let mut handles_map = self.handles.write().unwrap(); + if let Some(h) = handles_map.get_mut(&handle) { + final_status = match h.status { + LoadStatus::PENDING | LoadStatus::LOADING | LoadStatus::READY => { + AssetOpStatus::Ok + } + LoadStatus::CANCELED => AssetOpStatus::Ok, + _ => AssetOpStatus::InvalidState, + }; + match h.status { + LoadStatus::PENDING | LoadStatus::LOADING | LoadStatus::READY => { + h.status = LoadStatus::CANCELED; + canceled_slot = Some(h.slot); + } + _ => {} } - _ => {} } } - self.gfx_policy.take_staging(handle); - self.sound_policy.take_staging(handle); - self.scene_policy.take_staging(handle); + if let Some(slot) = canceled_slot { + self.bump_target_generation(slot); + } + self.clear_staging_for_handle(handle); final_status } @@ -993,9 +1230,9 @@ impl AssetManager { let mut pending = self.pending_commits.lock().unwrap(); let mut handles = self.handles.write().unwrap(); - for handle_id in pending.drain(..) { + for (handle_id, request_generation) in pending.drain(..) { if let Some(h) = handles.get_mut(&handle_id) { - if h.status == LoadStatus::READY { + if h.status == LoadStatus::READY && h.request_generation == request_generation { match h.slot.asset_type { BankType::GLYPH => { if let Some((bank, _)) = self.gfx_policy.take_staging(handle_id) { @@ -1131,6 +1368,11 @@ impl AssetManager { self.sound_policy.clear(); self.scene_policy.clear(); self.handles.write().unwrap().clear(); + self.target_handles.write().unwrap().clear(); + for generation in self.target_generations.write().unwrap().values_mut() { + *generation += 1; + } + self.load_worker.clear_pending(); self.pending_commits.lock().unwrap().clear(); self.gfx_slots.write().unwrap().fill(None); self.glyph_slot_index.clear(); @@ -1526,6 +1768,80 @@ mod tests { assert!(Arc::ptr_eq(bank1, bank2)); } + #[test] + fn test_asset_load_uses_stable_handle_for_target_slot() { + let banks = Arc::new(MemoryBanks::new()); + let gfx_installer = Arc::clone(&banks) as Arc; + let sound_installer = Arc::clone(&banks) as Arc; + let scene_installer = Arc::clone(&banks) as Arc; + + let data = test_glyph_asset_data(); + let asset_entry = test_glyph_asset_entry("test_glyphs", 16, 16); + + let am = AssetManager::new( + vec![asset_entry], + AssetsPayloadSource::from_bytes(data), + gfx_installer, + sound_installer, + scene_installer, + ); + + let first_handle = am.load(0, 0).unwrap(); + let second_handle = am.load(0, 0).unwrap(); + + assert_eq!(first_handle, second_handle); + + let start = Instant::now(); + while am.status(second_handle) != LoadStatus::READY && start.elapsed().as_secs() < 5 { + thread::sleep(std::time::Duration::from_millis(10)); + } + assert_eq!(am.status(second_handle), LoadStatus::READY); + } + + #[test] + fn test_asset_load_supersedes_previous_request_for_target_slot() { + let banks = Arc::new(MemoryBanks::new()); + let gfx_installer = Arc::clone(&banks) as Arc; + let sound_installer = Arc::clone(&banks) as Arc; + let scene_installer = Arc::clone(&banks) as Arc; + + let first_payload = test_glyph_asset_data(); + let second_payload = test_glyph_asset_data(); + let second_offset = first_payload.len(); + let mut data = first_payload; + data.extend_from_slice(&second_payload); + + let first_asset = test_glyph_asset_entry("first_glyphs", 16, 16); + let mut second_asset = test_glyph_asset_entry("second_glyphs", 16, 16); + second_asset.asset_id = 1; + second_asset.offset = second_offset as u64; + + let am = AssetManager::new( + vec![first_asset, second_asset], + AssetsPayloadSource::from_bytes(data), + gfx_installer, + sound_installer, + scene_installer, + ); + + let first_handle = am.load(0, 0).unwrap(); + let second_handle = am.load(1, 0).unwrap(); + + assert_eq!(first_handle, second_handle); + + let start = Instant::now(); + while am.status(second_handle) != LoadStatus::READY && start.elapsed().as_secs() < 5 { + thread::sleep(std::time::Duration::from_millis(10)); + } + assert_eq!(am.status(second_handle), LoadStatus::READY); + + assert_eq!(am.commit(second_handle), AssetOpStatus::Ok); + am.apply_commits(); + + let slot = am.slot_info(SlotRef::gfx(0)); + assert_eq!(slot.asset_id, Some(1)); + } + #[test] fn test_sound_asset_loading() { let banks = Arc::new(MemoryBanks::new()); diff --git a/discussion/index.ndjson b/discussion/index.ndjson index 109b9620..95b9972c 100644 --- a/discussion/index.ndjson +++ b/discussion/index.ndjson @@ -18,7 +18,7 @@ {"type":"discussion","id":"DSC-0006","status":"open","ticket":"vm-owned-random-service","title":"Agenda - VM-Owned Random Service","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0005","file":"AGD-0005-vm-owned-random-service.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0007","status":"open","ticket":"app-home-filesystem-surface-and-semantics","title":"Agenda - App Home Filesystem Surface and Semantics","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0006","file":"AGD-0006-app-home-filesystem-surface-and-semantics.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0008","status":"done","ticket":"perf-runtime-telemetry-hot-path","title":"Agenda - [PERF] Runtime Telemetry Hot Path","created_at":"2026-03-27","updated_at":"2026-06-04","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0026","file":"discussion/lessons/DSC-0008-perf-runtime-telemetry-hot-path/LSN-0026-push-based-telemetry-model.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} -{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} +{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} {"type":"discussion","id":"DSC-0010","status":"done","ticket":"perf-host-desktop-frame-pacing-and-presentation","title":"Agenda - [PERF] Host Desktop Frame Pacing and Presentation","created_at":"2026-03-27","updated_at":"2026-04-20","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0036","file":"discussion/lessons/DSC-0010-perf-host-desktop-frame-pacing-and-presentation/LSN-0036-frame-publication-and-host-invalidation-must-be-separate.md","status":"done","created_at":"2026-04-20","updated_at":"2026-04-20"}]} {"type":"discussion","id":"DSC-0011","status":"abandoned","ticket":"perf-gfx-render-pipeline-and-dirty-regions","title":"Agenda - [PERF] GFX Render Pipeline and Dirty Regions","created_at":"2026-03-27","updated_at":"2026-06-28","tags":[],"agendas":[{"id":"AGD-0010","file":"AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md","status":"abandoned","created_at":"2026-03-27","updated_at":"2026-06-28","_override_reason":"User explicitly chose to remove this agenda because it no longer applies to the current render scenario; future optimizations can be discussed when needed."}],"decisions":[],"plans":[],"lessons":[],"_override_reason":"User explicitly chose to abandon this discussion because it no longer applies to the current render scenario; future optimizations can be discussed when needed."} {"type":"discussion","id":"DSC-0012","status":"done","ticket":"perf-runtime-introspection-syscalls","title":"Agenda - [PERF] Runtime Introspection Syscalls","created_at":"2026-03-27","updated_at":"2026-04-19","tags":["perf","runtime","syscall","telemetry","debug","asset"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0034","file":"discussion/lessons/DSC-0012-perf-runtime-introspection-syscalls/LSN-0034-host-owned-debug-boundaries.md","status":"done","created_at":"2026-04-19","updated_at":"2026-04-19"}]} diff --git a/discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md b/discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md index 65bb3669..df656308 100644 --- a/discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md +++ b/discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md @@ -2,7 +2,7 @@ id: PLN-0125 ticket: perf-async-background-work-lanes-for-assets-and-fs title: Asset Backlog and Stable Slot Handles -status: open +status: done created: 2026-06-28 completed: ref_decisions: [DEC-0034] From 961557583b6522d4729205f76dbb33f27f96ca3c Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Sun, 28 Jun 2026 14:08:30 +0100 Subject: [PATCH 5/9] implements PLN-0126 asset backlog public api --- crates/console/prometeu-drivers/src/asset.rs | 334 +++++++++++++++++- crates/console/prometeu-hal/src/asset.rs | 36 ++ .../console/prometeu-hal/src/asset_bridge.rs | 11 +- crates/console/prometeu-hal/src/syscalls.rs | 6 + .../src/syscalls/domains/asset.rs | 36 ++ .../prometeu-hal/src/syscalls/registry.rs | 12 + .../prometeu-hal/src/syscalls/tests.rs | 24 ++ .../src/services/vm_runtime/dispatch.rs | 58 +++ .../services/vm_runtime/tests_asset_bank.rs | 96 +++++ discussion/index.ndjson | 2 +- ...t-backlog-public-api-and-status-surface.md | 2 +- 11 files changed, 604 insertions(+), 13 deletions(-) diff --git a/crates/console/prometeu-drivers/src/asset.rs b/crates/console/prometeu-drivers/src/asset.rs index c1dff3c7..4265b092 100644 --- a/crates/console/prometeu-drivers/src/asset.rs +++ b/crates/console/prometeu-drivers/src/asset.rs @@ -2,11 +2,11 @@ use crate::memory_banks::{GlyphBankPoolInstaller, SceneBankPoolInstaller, SoundBankPoolInstaller}; use prometeu_hal::AssetBridge; use prometeu_hal::asset::{ - AssetCodec, AssetEntry, AssetId, AssetLoadError, AssetOpStatus, BankTelemetry, BankType, - HandleId, LoadStatus, PreloadEntry, SCENE_DECODED_LAYER_OVERHEAD_BYTES_V1, - SCENE_HEADER_BYTES_V1, SCENE_LAYER_COUNT_V1, SCENE_LAYER_HEADER_BYTES_V1, - SCENE_PAYLOAD_MAGIC_V1, SCENE_PAYLOAD_VERSION_V1, SCENE_TILE_RECORD_BYTES_V1, SlotRef, - SlotStats, + AssetBacklogInfo, AssetBacklogPosition, AssetCodec, AssetEntry, AssetId, AssetLoadError, + AssetOpStatus, AssetTargetStatus, BankTelemetry, BankType, HandleId, LoadStatus, PreloadEntry, + SCENE_DECODED_LAYER_OVERHEAD_BYTES_V1, SCENE_HEADER_BYTES_V1, SCENE_LAYER_COUNT_V1, + SCENE_LAYER_HEADER_BYTES_V1, SCENE_PAYLOAD_MAGIC_V1, SCENE_PAYLOAD_VERSION_V1, + SCENE_TILE_RECORD_BYTES_V1, SlotRef, SlotStats, }; use prometeu_hal::cartridge::AssetsPayloadSource; use prometeu_hal::color::Color; @@ -220,6 +220,7 @@ struct LoadHandleInfo { request_generation: u64, } +#[derive(Clone)] struct AssetLoadJob { handle_id: HandleId, asset_id: AssetId, @@ -231,6 +232,7 @@ struct AssetLoadJob { #[derive(Default)] struct AssetLoadQueueState { pending: VecDeque, + active: Option, shutdown: bool, } @@ -246,6 +248,7 @@ impl AssetLoadQueue { if state.shutdown { return; } + state.pending.retain(|pending| pending.slot != job.slot); state.pending.push_back(job); self.ready.notify_one(); } @@ -254,6 +257,7 @@ impl AssetLoadQueue { let mut state = self.state.lock().unwrap(); loop { if let Some(job) = state.pending.pop_front() { + state.active = Some(job.clone()); return Some(job); } if state.shutdown { @@ -267,6 +271,51 @@ impl AssetLoadQueue { self.state.lock().unwrap().pending.clear(); } + fn complete_active(&self, handle_id: HandleId, request_generation: u64) { + let mut state = self.state.lock().unwrap(); + if state.active.as_ref().is_some_and(|job| { + job.handle_id == handle_id && job.request_generation == request_generation + }) { + state.active = None; + } + } + + fn pending_count(&self) -> usize { + self.state.lock().unwrap().pending.len() + } + + fn active_job(&self) -> Option { + self.state.lock().unwrap().active.clone() + } + + fn pending_position(&self, handle_id: HandleId) -> Option { + self.state + .lock() + .unwrap() + .pending + .iter() + .position(|job| job.handle_id == handle_id) + .map(|index| index + 1) + } + + fn move_pending(&self, handle_id: HandleId, new_position: usize) -> bool { + if new_position == 0 { + return false; + } + + let mut state = self.state.lock().unwrap(); + let Some(current_index) = state.pending.iter().position(|job| job.handle_id == handle_id) + else { + return false; + }; + let Some(job) = state.pending.remove(current_index) else { + return false; + }; + let insert_index = new_position.saturating_sub(1).min(state.pending.len()); + state.pending.insert(insert_index, job); + true + } + fn shutdown(&self) { let mut state = self.state.lock().unwrap(); state.pending.clear(); @@ -294,7 +343,7 @@ impl AssetLoadWorker { let join_handle = thread::spawn(move || { while let Some(job) = worker_queue.next_job() { AssetManager::process_load_job( - job, + job.clone(), &handles, &target_generations, &assets_data, @@ -302,6 +351,7 @@ impl AssetLoadWorker { &sound_policy, &scene_policy, ); + worker_queue.complete_active(job.handle_id, job.request_generation); } }); @@ -315,6 +365,22 @@ impl AssetLoadWorker { fn clear_pending(&self) { self.queue.clear_pending(); } + + fn pending_count(&self) -> usize { + self.queue.pending_count() + } + + fn active_job(&self) -> Option { + self.queue.active_job() + } + + fn pending_position(&self, handle_id: HandleId) -> Option { + self.queue.pending_position(handle_id) + } + + fn move_pending(&self, handle_id: HandleId, new_position: usize) -> bool { + self.queue.move_pending(handle_id, new_position) + } } impl Drop for AssetLoadWorker { @@ -353,6 +419,24 @@ impl AssetBridge for AssetManager { fn cancel(&self, handle: HandleId) -> AssetOpStatus { self.cancel(handle) } + fn backlog_info(&self) -> AssetBacklogInfo { + self.backlog_info() + } + fn backlog_position(&self, handle: HandleId) -> AssetBacklogPosition { + self.backlog_position(handle) + } + fn backlog_move(&self, handle: HandleId, new_position: usize) -> AssetOpStatus { + self.backlog_move(handle, new_position) + } + fn backlog_promote(&self, handle: HandleId) -> AssetOpStatus { + self.backlog_promote(handle) + } + fn backlog_demote(&self, handle: HandleId) -> AssetOpStatus { + self.backlog_demote(handle) + } + fn target_status(&self, bank_type: BankType, slot: usize) -> AssetTargetStatus { + self.target_status(bank_type, slot) + } fn apply_commits(&self) { self.apply_commits() } @@ -611,6 +695,26 @@ impl AssetManager { self.scene_policy.take_staging(handle_id); } + fn target_for_handle(&self, handle_id: HandleId) -> Option { + self.target_handles + .read() + .unwrap() + .iter() + .find_map(|(slot, target_handle)| (*target_handle == handle_id).then_some(*slot)) + } + + fn slot_asset_id(&self, slot: SlotRef) -> Option { + match slot.asset_type { + BankType::GLYPH => self.gfx_slots.read().unwrap().get(slot.index).and_then(|s| *s), + BankType::SOUNDS => self.sound_slots.read().unwrap().get(slot.index).and_then(|s| *s), + BankType::SCENE => self.scene_slots.read().unwrap().get(slot.index).and_then(|s| *s), + } + } + + fn idle_target_state(&self, slot: SlotRef) -> LoadStatus { + if self.slot_asset_id(slot).is_some() { LoadStatus::COMMITTED } else { LoadStatus::EMPTY } + } + pub fn load(&self, asset_id: AssetId, slot_index: usize) -> Result { if slot_index >= 16 { return Err(AssetLoadError::SlotIndexInvalid); @@ -1176,12 +1280,131 @@ impl AssetManager { } pub fn status(&self, handle: HandleId) -> LoadStatus { - self.handles + if let Some(status) = self.handles.read().unwrap().get(&handle).map(|h| h.status) { + return status; + } + + self.target_for_handle(handle) + .map(|slot| self.idle_target_state(slot)) + .unwrap_or(LoadStatus::UnknownHandle) + } + + pub fn backlog_info(&self) -> AssetBacklogInfo { + let active = self.load_worker.active_job(); + AssetBacklogInfo { + status: AssetOpStatus::Ok, + pending_count: self.load_worker.pending_count() as u32, + active_handle: active.as_ref().map(|job| job.handle_id).unwrap_or(0), + active_asset_id: active.as_ref().map(|job| job.asset_id), + active_bank_type: active.as_ref().map(|job| job.slot.asset_type), + active_slot: active.as_ref().map(|job| job.slot.index), + active_progress: 0, + } + } + + pub fn backlog_position(&self, handle: HandleId) -> AssetBacklogPosition { + let Some(slot) = self.target_for_handle(handle) else { + return AssetBacklogPosition { + status: AssetOpStatus::UnknownHandle, + state: LoadStatus::UnknownHandle, + position: 0, + progress: 0, + }; + }; + + if let Some(position) = self.load_worker.pending_position(handle) { + return AssetBacklogPosition { + status: AssetOpStatus::Ok, + state: LoadStatus::QUEUED, + position: position as u32, + progress: 0, + }; + } + + if self.load_worker.active_job().as_ref().is_some_and(|job| job.handle_id == handle) { + return AssetBacklogPosition { + status: AssetOpStatus::Ok, + state: LoadStatus::ACTIVE, + position: 0, + progress: 0, + }; + } + + AssetBacklogPosition { + status: AssetOpStatus::Ok, + state: self + .handles + .read() + .unwrap() + .get(&handle) + .map(|h| h.status) + .unwrap_or_else(|| self.idle_target_state(slot)), + position: 0, + progress: 0, + } + } + + pub fn backlog_move(&self, handle: HandleId, new_position: usize) -> AssetOpStatus { + if self.target_for_handle(handle).is_none() { + return AssetOpStatus::UnknownHandle; + } + if self.load_worker.move_pending(handle, new_position) { + AssetOpStatus::Ok + } else { + AssetOpStatus::InvalidState + } + } + + pub fn backlog_promote(&self, handle: HandleId) -> AssetOpStatus { + self.backlog_move(handle, 1) + } + + pub fn backlog_demote(&self, handle: HandleId) -> AssetOpStatus { + if self.target_for_handle(handle).is_none() { + return AssetOpStatus::UnknownHandle; + } + let new_position = self.load_worker.pending_count(); + if new_position == 0 { + return AssetOpStatus::InvalidState; + } + self.backlog_move(handle, new_position) + } + + pub fn target_status(&self, bank_type: BankType, slot: usize) -> AssetTargetStatus { + if slot >= 16 { + return AssetTargetStatus { + status: AssetOpStatus::InvalidState, + asset_id: None, + handle: 0, + state: LoadStatus::INVALID, + position: 0, + progress: 0, + }; + } + + let slot_ref = match bank_type { + BankType::GLYPH => SlotRef::gfx(slot), + BankType::SOUNDS => SlotRef::audio(slot), + BankType::SCENE => SlotRef::scene(slot), + }; + let handle = self.handle_for_slot(slot_ref); + let position = self.backlog_position(handle); + let asset_id = self + .handles .read() .unwrap() .get(&handle) - .map(|h| h.status) - .unwrap_or(LoadStatus::UnknownHandle) + .map(|h| h._asset_id) + .or_else(|| self.slot_asset_id(slot_ref)); + + AssetTargetStatus { + status: AssetOpStatus::Ok, + asset_id, + handle, + state: position.state, + position: position.position, + progress: position.progress, + } } pub fn commit(&self, handle: HandleId) -> AssetOpStatus { @@ -1842,6 +2065,99 @@ mod tests { assert_eq!(slot.asset_id, Some(1)); } + #[test] + fn test_asset_backlog_queue_reorders_pending_requests() { + let queue = AssetLoadQueue::default(); + let entry = test_glyph_asset_entry("queued_glyphs", 16, 16); + let job = |handle_id: HandleId, slot: usize| AssetLoadJob { + handle_id, + asset_id: handle_id as AssetId, + slot: SlotRef::gfx(slot), + request_generation: handle_id as u64, + entry: entry.clone(), + }; + + queue.submit(job(1, 0)); + queue.submit(job(2, 1)); + queue.submit(job(3, 2)); + + assert_eq!(queue.pending_position(3), Some(3)); + assert!(queue.move_pending(3, 1)); + assert_eq!(queue.pending_position(3), Some(1)); + assert_eq!(queue.pending_position(1), Some(2)); + assert!(queue.move_pending(3, queue.pending_count())); + assert_eq!(queue.pending_position(3), Some(3)); + } + + #[test] + fn test_asset_target_status_exposes_empty_and_ready_slot_handle() { + let banks = Arc::new(MemoryBanks::new()); + let gfx_installer = Arc::clone(&banks) as Arc; + let sound_installer = Arc::clone(&banks) as Arc; + let scene_installer = Arc::clone(&banks) as Arc; + + let data = test_glyph_asset_data(); + let asset_entry = test_glyph_asset_entry("test_glyphs", 16, 16); + + let am = AssetManager::new( + vec![asset_entry], + AssetsPayloadSource::from_bytes(data), + gfx_installer, + sound_installer, + scene_installer, + ); + + let empty = am.target_status(BankType::GLYPH, 0); + assert_eq!(empty.status, AssetOpStatus::Ok); + assert_eq!(empty.asset_id, None); + assert_eq!(empty.state, LoadStatus::EMPTY); + assert_ne!(empty.handle, 0); + assert_eq!(am.status(empty.handle), LoadStatus::EMPTY); + + let handle = am.load(0, 0).unwrap(); + assert_eq!(handle, empty.handle); + + let start = Instant::now(); + while am.status(handle) != LoadStatus::READY && start.elapsed().as_secs() < 5 { + thread::sleep(std::time::Duration::from_millis(10)); + } + + let ready = am.target_status(BankType::GLYPH, 0); + assert_eq!(ready.status, AssetOpStatus::Ok); + assert_eq!(ready.asset_id, Some(0)); + assert_eq!(ready.handle, handle); + assert_eq!(ready.state, LoadStatus::READY); + } + + #[test] + fn test_asset_backlog_reorder_rejects_non_queued_request() { + let banks = Arc::new(MemoryBanks::new()); + let gfx_installer = Arc::clone(&banks) as Arc; + let sound_installer = Arc::clone(&banks) as Arc; + let scene_installer = Arc::clone(&banks) as Arc; + + let data = test_glyph_asset_data(); + let asset_entry = test_glyph_asset_entry("test_glyphs", 16, 16); + + let am = AssetManager::new( + vec![asset_entry], + AssetsPayloadSource::from_bytes(data), + gfx_installer, + sound_installer, + scene_installer, + ); + + let handle = am.load(0, 0).unwrap(); + let start = Instant::now(); + while am.status(handle) != LoadStatus::READY && start.elapsed().as_secs() < 5 { + thread::sleep(std::time::Duration::from_millis(10)); + } + + assert_eq!(am.backlog_promote(handle), AssetOpStatus::InvalidState); + assert_eq!(am.backlog_demote(handle), AssetOpStatus::InvalidState); + assert_eq!(am.backlog_move(999, 1), AssetOpStatus::UnknownHandle); + } + #[test] fn test_sound_asset_loading() { let banks = Arc::new(MemoryBanks::new()); diff --git a/crates/console/prometeu-hal/src/asset.rs b/crates/console/prometeu-hal/src/asset.rs index aa451b0f..b49483f8 100644 --- a/crates/console/prometeu-hal/src/asset.rs +++ b/crates/console/prometeu-hal/src/asset.rs @@ -112,6 +112,12 @@ pub enum LoadStatus { CANCELED = 4, ERROR = 5, UnknownHandle = 6, + QUEUED = 7, + ACTIVE = 8, + SUPERSEDED = 9, + EMPTY = 10, + INVALID = 11, + BackendUnavailable = 12, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -129,6 +135,36 @@ pub enum AssetOpStatus { Ok = 0, UnknownHandle = 1, InvalidState = 2, + Superseded = 3, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AssetBacklogInfo { + pub status: AssetOpStatus, + pub pending_count: u32, + pub active_handle: HandleId, + pub active_asset_id: Option, + pub active_bank_type: Option, + pub active_slot: Option, + pub active_progress: u8, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AssetBacklogPosition { + pub status: AssetOpStatus, + pub state: LoadStatus, + pub position: u32, + pub progress: u8, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AssetTargetStatus { + pub status: AssetOpStatus, + pub asset_id: Option, + pub handle: HandleId, + pub state: LoadStatus, + pub position: u32, + pub progress: u8, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/console/prometeu-hal/src/asset_bridge.rs b/crates/console/prometeu-hal/src/asset_bridge.rs index 5c80e3fe..ad4f51f3 100644 --- a/crates/console/prometeu-hal/src/asset_bridge.rs +++ b/crates/console/prometeu-hal/src/asset_bridge.rs @@ -1,6 +1,7 @@ use crate::asset::{ - AssetEntry, AssetId, AssetLoadError, AssetOpStatus, BankTelemetry, HandleId, LoadStatus, - PreloadEntry, SlotRef, SlotStats, + AssetBacklogInfo, AssetBacklogPosition, AssetEntry, AssetId, AssetLoadError, AssetOpStatus, + AssetTargetStatus, BankTelemetry, BankType, HandleId, LoadStatus, PreloadEntry, SlotRef, + SlotStats, }; use crate::cartridge::AssetsPayloadSource; @@ -15,6 +16,12 @@ pub trait AssetBridge { fn status(&self, handle: HandleId) -> LoadStatus; fn commit(&self, handle: HandleId) -> AssetOpStatus; fn cancel(&self, handle: HandleId) -> AssetOpStatus; + fn backlog_info(&self) -> AssetBacklogInfo; + fn backlog_position(&self, handle: HandleId) -> AssetBacklogPosition; + fn backlog_move(&self, handle: HandleId, new_position: usize) -> AssetOpStatus; + fn backlog_promote(&self, handle: HandleId) -> AssetOpStatus; + fn backlog_demote(&self, handle: HandleId) -> AssetOpStatus; + fn target_status(&self, bank_type: BankType, slot: usize) -> AssetTargetStatus; fn apply_commits(&self); fn bank_telemetry(&self) -> Vec; fn slot_info(&self, slot: SlotRef) -> SlotStats; diff --git a/crates/console/prometeu-hal/src/syscalls.rs b/crates/console/prometeu-hal/src/syscalls.rs index 73ae59cf..618e20d4 100644 --- a/crates/console/prometeu-hal/src/syscalls.rs +++ b/crates/console/prometeu-hal/src/syscalls.rs @@ -69,6 +69,12 @@ pub enum Syscall { AssetStatus = 0x6002, AssetCommit = 0x6003, AssetCancel = 0x6004, + AssetBacklogInfo = 0x6005, + AssetBacklogPosition = 0x6006, + AssetBacklogMove = 0x6007, + AssetBacklogPromote = 0x6008, + AssetBacklogDemote = 0x6009, + AssetTargetStatus = 0x600A, BankInfo = 0x6101, } diff --git a/crates/console/prometeu-hal/src/syscalls/domains/asset.rs b/crates/console/prometeu-hal/src/syscalls/domains/asset.rs index 26c7ee30..849c7632 100644 --- a/crates/console/prometeu-hal/src/syscalls/domains/asset.rs +++ b/crates/console/prometeu-hal/src/syscalls/domains/asset.rs @@ -24,4 +24,40 @@ pub(crate) const ENTRIES: &[SyscallRegistryEntry] = &[ .caps(caps::ASSET) .non_deterministic() .cost(20), + SyscallRegistryEntry::builder(Syscall::AssetBacklogInfo, "asset", "backlog_info") + .args(0) + .rets(7) + .caps(caps::ASSET) + .non_deterministic() + .cost(5), + SyscallRegistryEntry::builder(Syscall::AssetBacklogPosition, "asset", "backlog_position") + .args(1) + .rets(4) + .caps(caps::ASSET) + .non_deterministic() + .cost(5), + SyscallRegistryEntry::builder(Syscall::AssetBacklogMove, "asset", "backlog_move") + .args(2) + .rets(1) + .caps(caps::ASSET) + .non_deterministic() + .cost(10), + SyscallRegistryEntry::builder(Syscall::AssetBacklogPromote, "asset", "backlog_promote") + .args(1) + .rets(1) + .caps(caps::ASSET) + .non_deterministic() + .cost(10), + SyscallRegistryEntry::builder(Syscall::AssetBacklogDemote, "asset", "backlog_demote") + .args(1) + .rets(1) + .caps(caps::ASSET) + .non_deterministic() + .cost(10), + SyscallRegistryEntry::builder(Syscall::AssetTargetStatus, "asset", "target_status") + .args(2) + .rets(6) + .caps(caps::ASSET) + .non_deterministic() + .cost(5), ]; diff --git a/crates/console/prometeu-hal/src/syscalls/registry.rs b/crates/console/prometeu-hal/src/syscalls/registry.rs index 65e4d3ad..3d0a08a6 100644 --- a/crates/console/prometeu-hal/src/syscalls/registry.rs +++ b/crates/console/prometeu-hal/src/syscalls/registry.rs @@ -53,6 +53,12 @@ impl Syscall { 0x6002 => Some(Self::AssetStatus), 0x6003 => Some(Self::AssetCommit), 0x6004 => Some(Self::AssetCancel), + 0x6005 => Some(Self::AssetBacklogInfo), + 0x6006 => Some(Self::AssetBacklogPosition), + 0x6007 => Some(Self::AssetBacklogMove), + 0x6008 => Some(Self::AssetBacklogPromote), + 0x6009 => Some(Self::AssetBacklogDemote), + 0x600A => Some(Self::AssetTargetStatus), 0x6101 => Some(Self::BankInfo), _ => None, } @@ -109,6 +115,12 @@ impl Syscall { Self::AssetStatus => "AssetStatus", Self::AssetCommit => "AssetCommit", Self::AssetCancel => "AssetCancel", + Self::AssetBacklogInfo => "AssetBacklogInfo", + Self::AssetBacklogPosition => "AssetBacklogPosition", + Self::AssetBacklogMove => "AssetBacklogMove", + Self::AssetBacklogPromote => "AssetBacklogPromote", + Self::AssetBacklogDemote => "AssetBacklogDemote", + Self::AssetTargetStatus => "AssetTargetStatus", Self::BankInfo => "BankInfo", } } diff --git a/crates/console/prometeu-hal/src/syscalls/tests.rs b/crates/console/prometeu-hal/src/syscalls/tests.rs index 3625b5ad..e4e22814 100644 --- a/crates/console/prometeu-hal/src/syscalls/tests.rs +++ b/crates/console/prometeu-hal/src/syscalls/tests.rs @@ -258,6 +258,30 @@ fn status_first_syscall_signatures_are_pinned() { assert_eq!(asset_cancel.arg_slots, 1); assert_eq!(asset_cancel.ret_slots, 1); + let asset_backlog_info = meta_for(Syscall::AssetBacklogInfo); + assert_eq!(asset_backlog_info.arg_slots, 0); + assert_eq!(asset_backlog_info.ret_slots, 7); + + let asset_backlog_position = meta_for(Syscall::AssetBacklogPosition); + assert_eq!(asset_backlog_position.arg_slots, 1); + assert_eq!(asset_backlog_position.ret_slots, 4); + + let asset_backlog_move = meta_for(Syscall::AssetBacklogMove); + assert_eq!(asset_backlog_move.arg_slots, 2); + assert_eq!(asset_backlog_move.ret_slots, 1); + + let asset_backlog_promote = meta_for(Syscall::AssetBacklogPromote); + assert_eq!(asset_backlog_promote.arg_slots, 1); + assert_eq!(asset_backlog_promote.ret_slots, 1); + + let asset_backlog_demote = meta_for(Syscall::AssetBacklogDemote); + assert_eq!(asset_backlog_demote.arg_slots, 1); + assert_eq!(asset_backlog_demote.ret_slots, 1); + + let asset_target_status = meta_for(Syscall::AssetTargetStatus); + assert_eq!(asset_target_status.arg_slots, 2); + assert_eq!(asset_target_status.ret_slots, 6); + let bank_info = meta_for(Syscall::BankInfo); assert_eq!(bank_info.arg_slots, 1); assert_eq!(bank_info.ret_slots, 2); diff --git a/crates/console/prometeu-system/src/services/vm_runtime/dispatch.rs b/crates/console/prometeu-system/src/services/vm_runtime/dispatch.rs index 932b8589..b650f02f 100644 --- a/crates/console/prometeu-system/src/services/vm_runtime/dispatch.rs +++ b/crates/console/prometeu-system/src/services/vm_runtime/dispatch.rs @@ -757,6 +757,64 @@ impl NativeInterface for VmRuntimeHost<'_> { ret.push_int(status as i64); Ok(()) } + Syscall::AssetBacklogInfo => { + let info = platform.assets().backlog_info(); + ret.push_int(info.status as i64); + ret.push_int(info.pending_count as i64); + ret.push_int(info.active_handle as i64); + ret.push_int(info.active_asset_id.unwrap_or(0) as i64); + ret.push_int(info.active_bank_type.map(|bank| bank as i64).unwrap_or(0)); + ret.push_int(info.active_slot.map(|slot| slot as i64).unwrap_or(0)); + ret.push_int(info.active_progress as i64); + Ok(()) + } + Syscall::AssetBacklogPosition => { + let position = platform.assets().backlog_position(expect_int(args, 0)? as u32); + ret.push_int(position.status as i64); + ret.push_int(position.state as i64); + ret.push_int(position.position as i64); + ret.push_int(position.progress as i64); + Ok(()) + } + Syscall::AssetBacklogMove => { + let new_position = expect_non_negative_usize(args, 1, "new_position")?; + let status = + platform.assets().backlog_move(expect_int(args, 0)? as u32, new_position); + ret.push_int(status as i64); + Ok(()) + } + Syscall::AssetBacklogPromote => { + let status = platform.assets().backlog_promote(expect_int(args, 0)? as u32); + ret.push_int(status as i64); + Ok(()) + } + Syscall::AssetBacklogDemote => { + let status = platform.assets().backlog_demote(expect_int(args, 0)? as u32); + ret.push_int(status as i64); + Ok(()) + } + Syscall::AssetTargetStatus => { + let bank_type = match expect_int(args, 0)? { + 0 => BankType::GLYPH, + 1 => BankType::SOUNDS, + 2 => BankType::SCENE, + other => { + return Err(VmFault::Trap( + TRAP_TYPE, + format!("Invalid asset bank type: {}", other), + )); + } + }; + let slot = expect_non_negative_usize(args, 1, "slot")?; + let target = platform.assets().target_status(bank_type, slot); + ret.push_int(target.status as i64); + ret.push_int(target.asset_id.unwrap_or(0) as i64); + ret.push_int(target.handle as i64); + ret.push_int(target.state as i64); + ret.push_int(target.position as i64); + ret.push_int(target.progress as i64); + Ok(()) + } Syscall::BankInfo => { let asset_type = match expect_int(args, 0)? as u32 { 0 => BankType::GLYPH, diff --git a/crates/console/prometeu-system/src/services/vm_runtime/tests_asset_bank.rs b/crates/console/prometeu-system/src/services/vm_runtime/tests_asset_bank.rs index 45ae5a2e..6fd57e59 100644 --- a/crates/console/prometeu-system/src/services/vm_runtime/tests_asset_bank.rs +++ b/crates/console/prometeu-system/src/services/vm_runtime/tests_asset_bank.rs @@ -180,6 +180,102 @@ fn tick_asset_status_unknown_handle_returns_status_not_crash() { assert_eq!(vm.operand_stack_top(1), vec![Value::Int64(LoadStatus::UnknownHandle as i64)]); } +#[test] +fn tick_asset_target_status_empty_slot_returns_status_first_payload() { + let mut runtime = VirtualMachineRuntime::new(None); + let mut log_service = LogService::new(4096); + let mut fs = VirtualFS::new(); + let mut fs_state = FsState::Unmounted; + let mut memcard = MemcardService::new(); + let mut open_files: HashMap = HashMap::new(); + let mut next_handle = 1; + let mut vm = VirtualMachine::default(); + let mut platform = TestPlatform::new(); + let signals = InputSignals::default(); + let code = assemble("PUSH_I32 0\nPUSH_I32 0\nHOSTCALL 0\nHALT").expect("assemble"); + let program = serialized_single_function_module( + code, + vec![SyscallDecl { + module: "asset".into(), + name: "target_status".into(), + version: 1, + arg_slots: 2, + ret_slots: 6, + }], + ); + let cartridge = cartridge_with_program(program, caps::ASSET); + + runtime.initialize_vm(&mut log_service, &mut vm, &cartridge).expect("runtime must initialize"); + let report = runtime.tick( + &mut log_service, + &mut fs, + &mut fs_state, + &mut memcard, + &mut open_files, + &mut next_handle, + &mut vm, + &signals, + &mut platform, + ); + + assert!(report.is_none(), "target_status must not crash for empty slots"); + assert!(vm.is_halted()); + assert_eq!( + vm.operand_stack_top(6), + vec![ + Value::Int64(0), + Value::Int64(0), + Value::Int64(LoadStatus::EMPTY as i64), + Value::Int64(1), + Value::Int64(0), + Value::Int64(AssetOpStatus::Ok as i64), + ] + ); +} + +#[test] +fn tick_asset_backlog_promote_unknown_handle_returns_status_not_crash() { + let mut runtime = VirtualMachineRuntime::new(None); + let mut log_service = LogService::new(4096); + let mut fs = VirtualFS::new(); + let mut fs_state = FsState::Unmounted; + let mut memcard = MemcardService::new(); + let mut open_files: HashMap = HashMap::new(); + let mut next_handle = 1; + let mut vm = VirtualMachine::default(); + let mut platform = TestPlatform::new(); + let signals = InputSignals::default(); + let code = assemble("PUSH_I32 999\nHOSTCALL 0\nHALT").expect("assemble"); + let program = serialized_single_function_module( + code, + vec![SyscallDecl { + module: "asset".into(), + name: "backlog_promote".into(), + version: 1, + arg_slots: 1, + ret_slots: 1, + }], + ); + let cartridge = cartridge_with_program(program, caps::ASSET); + + runtime.initialize_vm(&mut log_service, &mut vm, &cartridge).expect("runtime must initialize"); + let report = runtime.tick( + &mut log_service, + &mut fs, + &mut fs_state, + &mut memcard, + &mut open_files, + &mut next_handle, + &mut vm, + &signals, + &mut platform, + ); + + assert!(report.is_none(), "unknown backlog handle must not crash"); + assert!(vm.is_halted()); + assert_eq!(vm.operand_stack_top(1), vec![Value::Int64(AssetOpStatus::UnknownHandle as i64)]); +} + #[test] fn tick_bank_info_returns_slot_summary_not_json() { let mut runtime = VirtualMachineRuntime::new(None); diff --git a/discussion/index.ndjson b/discussion/index.ndjson index 95b9972c..5cb9d955 100644 --- a/discussion/index.ndjson +++ b/discussion/index.ndjson @@ -18,7 +18,7 @@ {"type":"discussion","id":"DSC-0006","status":"open","ticket":"vm-owned-random-service","title":"Agenda - VM-Owned Random Service","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0005","file":"AGD-0005-vm-owned-random-service.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0007","status":"open","ticket":"app-home-filesystem-surface-and-semantics","title":"Agenda - App Home Filesystem Surface and Semantics","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0006","file":"AGD-0006-app-home-filesystem-surface-and-semantics.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0008","status":"done","ticket":"perf-runtime-telemetry-hot-path","title":"Agenda - [PERF] Runtime Telemetry Hot Path","created_at":"2026-03-27","updated_at":"2026-06-04","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0026","file":"discussion/lessons/DSC-0008-perf-runtime-telemetry-hot-path/LSN-0026-push-based-telemetry-model.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} -{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} +{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} {"type":"discussion","id":"DSC-0010","status":"done","ticket":"perf-host-desktop-frame-pacing-and-presentation","title":"Agenda - [PERF] Host Desktop Frame Pacing and Presentation","created_at":"2026-03-27","updated_at":"2026-04-20","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0036","file":"discussion/lessons/DSC-0010-perf-host-desktop-frame-pacing-and-presentation/LSN-0036-frame-publication-and-host-invalidation-must-be-separate.md","status":"done","created_at":"2026-04-20","updated_at":"2026-04-20"}]} {"type":"discussion","id":"DSC-0011","status":"abandoned","ticket":"perf-gfx-render-pipeline-and-dirty-regions","title":"Agenda - [PERF] GFX Render Pipeline and Dirty Regions","created_at":"2026-03-27","updated_at":"2026-06-28","tags":[],"agendas":[{"id":"AGD-0010","file":"AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md","status":"abandoned","created_at":"2026-03-27","updated_at":"2026-06-28","_override_reason":"User explicitly chose to remove this agenda because it no longer applies to the current render scenario; future optimizations can be discussed when needed."}],"decisions":[],"plans":[],"lessons":[],"_override_reason":"User explicitly chose to abandon this discussion because it no longer applies to the current render scenario; future optimizations can be discussed when needed."} {"type":"discussion","id":"DSC-0012","status":"done","ticket":"perf-runtime-introspection-syscalls","title":"Agenda - [PERF] Runtime Introspection Syscalls","created_at":"2026-03-27","updated_at":"2026-04-19","tags":["perf","runtime","syscall","telemetry","debug","asset"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0034","file":"discussion/lessons/DSC-0012-perf-runtime-introspection-syscalls/LSN-0034-host-owned-debug-boundaries.md","status":"done","created_at":"2026-04-19","updated_at":"2026-04-19"}]} diff --git a/discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md b/discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md index 8ea105e7..4d508601 100644 --- a/discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md +++ b/discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md @@ -2,7 +2,7 @@ id: PLN-0126 ticket: perf-async-background-work-lanes-for-assets-and-fs title: Asset Backlog Public API and Status Surface -status: open +status: done created: 2026-06-28 completed: ref_decisions: [DEC-0034] From 8747c4f4cfce036f71b85b8f84b39a9b9a222458 Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Sun, 28 Jun 2026 14:14:10 +0100 Subject: [PATCH 6/9] implements PLN-0127 async io lane boundaries --- crates/console/prometeu-system/src/lib.rs | 3 +- .../src/services/async_work.rs | 158 +++++++++++++++++- .../prometeu-system/src/services/fs/mod.rs | 2 +- .../src/services/fs/virtual_fs.rs | 38 +++++ .../prometeu-system/src/services/memcard.rs | 50 ++++++ discussion/index.ndjson | 2 +- ...p-home-filesystem-surface-and-semantics.md | 10 ++ ...e-memcard-and-fs-integration-boundaries.md | 2 +- 8 files changed, 252 insertions(+), 13 deletions(-) diff --git a/crates/console/prometeu-system/src/lib.rs b/crates/console/prometeu-system/src/lib.rs index 77e44959..c0c052f0 100644 --- a/crates/console/prometeu-system/src/lib.rs +++ b/crates/console/prometeu-system/src/lib.rs @@ -10,9 +10,10 @@ pub use prometeu_hal::{RenderWorkerBackend, RenderWorkerFrameSink}; pub use services::async_work::{ AsyncWorkActiveJob, AsyncWorkCancelToken, AsyncWorkJobContext, AsyncWorkJobId, AsyncWorkJobKind, AsyncWorkJobOutcome, AsyncWorkLane, AsyncWorkLaneConfig, - AsyncWorkLaneController, AsyncWorkLaneError, AsyncWorkLaneTelemetry, + AsyncWorkLaneController, AsyncWorkLaneError, AsyncWorkLaneTelemetry, AsyncWorkPriority, }; pub use services::fs; +pub use services::memcard::MemcardAsyncLaneOperation; pub use services::process; pub use services::task; pub use services::vm_runtime::{ diff --git a/crates/console/prometeu-system/src/services/async_work.rs b/crates/console/prometeu-system/src/services/async_work.rs index 002926b3..475ef27e 100644 --- a/crates/console/prometeu-system/src/services/async_work.rs +++ b/crates/console/prometeu-system/src/services/async_work.rs @@ -29,6 +29,36 @@ pub enum AsyncWorkJobKind { Test, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AsyncWorkPriority { + MemcardCommitWrite, + FsWriteConfig, + AssetLoad, + NonCriticalReadList, +} + +impl AsyncWorkPriority { + const fn rank(self) -> u8 { + match self { + Self::MemcardCommitWrite => 0, + Self::FsWriteConfig => 1, + Self::AssetLoad => 2, + Self::NonCriticalReadList => 3, + } + } +} + +impl AsyncWorkJobKind { + pub const fn default_priority(self) -> AsyncWorkPriority { + match self { + Self::Memcard => AsyncWorkPriority::MemcardCommitWrite, + Self::Fs => AsyncWorkPriority::FsWriteConfig, + Self::Asset => AsyncWorkPriority::AssetLoad, + Self::Test => AsyncWorkPriority::NonCriticalReadList, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AsyncWorkJobOutcome { Completed, @@ -59,6 +89,7 @@ impl Default for AsyncWorkLaneConfig { pub struct AsyncWorkActiveJob { pub id: AsyncWorkJobId, pub kind: AsyncWorkJobKind, + pub priority: AsyncWorkPriority, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -131,6 +162,7 @@ impl AsyncWorkCancelToken { pub struct AsyncWorkJobContext { pub job_id: AsyncWorkJobId, pub kind: AsyncWorkJobKind, + pub priority: AsyncWorkPriority, cancel_token: AsyncWorkCancelToken, } @@ -145,6 +177,7 @@ type AsyncWorkFn = Box AsyncWorkJobOutcome + struct PendingAsyncWorkJob { id: AsyncWorkJobId, kind: AsyncWorkJobKind, + priority: AsyncWorkPriority, cancel_token: AsyncWorkCancelToken, work: AsyncWorkFn, } @@ -152,6 +185,7 @@ struct PendingAsyncWorkJob { struct RunningAsyncWorkJob { id: AsyncWorkJobId, kind: AsyncWorkJobKind, + priority: AsyncWorkPriority, cancel_token: AsyncWorkCancelToken, work: AsyncWorkFn, } @@ -177,6 +211,15 @@ impl AsyncWorkLane { &self, kind: AsyncWorkJobKind, work: impl FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static, + ) -> Result { + self.submit_with_priority(kind, kind.default_priority(), work) + } + + pub fn submit_with_priority( + &self, + kind: AsyncWorkJobKind, + priority: AsyncWorkPriority, + work: impl FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static, ) -> Result { let mut state = self.state.lock().map_err(|_| AsyncWorkLaneError::InternalFailure)?; if state.shutdown_requested { @@ -186,18 +229,52 @@ impl AsyncWorkLane { state.next_job_id += 1; let id = AsyncWorkJobId::new(state.next_job_id); let cancel_token = AsyncWorkCancelToken::new(); - state.pending.push_back(PendingAsyncWorkJob { - id, - kind, - cancel_token, - work: Box::new(work), - }); + let job = PendingAsyncWorkJob { id, kind, priority, cancel_token, work: Box::new(work) }; + let insert_at = state + .pending + .iter() + .position(|pending| priority.rank() < pending.priority.rank()) + .unwrap_or(state.pending.len()); + state.pending.insert(insert_at, job); let pending_depth = state.pending.len(); state.telemetry.record_submitted(id, pending_depth); self.ready.notify_one(); Ok(id) } + pub fn submit_memcard_commit_write( + &self, + work: impl FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static, + ) -> Result { + self.submit_with_priority( + AsyncWorkJobKind::Memcard, + AsyncWorkPriority::MemcardCommitWrite, + work, + ) + } + + pub fn submit_fs_write_config( + &self, + work: impl FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static, + ) -> Result { + self.submit_with_priority(AsyncWorkJobKind::Fs, AsyncWorkPriority::FsWriteConfig, work) + } + + pub fn submit_asset_load( + &self, + work: impl FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static, + ) -> Result { + self.submit_with_priority(AsyncWorkJobKind::Asset, AsyncWorkPriority::AssetLoad, work) + } + + pub fn submit_non_critical_read_list( + &self, + kind: AsyncWorkJobKind, + work: impl FnOnce(AsyncWorkJobContext) -> AsyncWorkJobOutcome + Send + 'static, + ) -> Result { + self.submit_with_priority(kind, AsyncWorkPriority::NonCriticalReadList, work) + } + pub fn request_cancel(&self, job_id: AsyncWorkJobId) -> bool { let state = self.state.lock().unwrap(); if state.active.is_some_and(|active| active.id == job_id) { @@ -253,7 +330,11 @@ impl AsyncWorkLane { } if let Some(pending) = state.pending.pop_front() { - let active = AsyncWorkActiveJob { id: pending.id, kind: pending.kind }; + let active = AsyncWorkActiveJob { + id: pending.id, + kind: pending.kind, + priority: pending.priority, + }; state.active = Some(active); state.active_cancel_token = Some(pending.cancel_token.clone()); let pending_depth = state.pending.len(); @@ -261,6 +342,7 @@ impl AsyncWorkLane { return Some(RunningAsyncWorkJob { id: pending.id, kind: pending.kind, + priority: pending.priority, cancel_token: pending.cancel_token, work: pending.work, }); @@ -332,8 +414,12 @@ impl Drop for AsyncWorkLaneController { fn run_async_work_loop(lane: Arc) { while let Some(job) = lane.wait_take() { - let context = - AsyncWorkJobContext { job_id: job.id, kind: job.kind, cancel_token: job.cancel_token }; + let context = AsyncWorkJobContext { + job_id: job.id, + kind: job.kind, + priority: job.priority, + cancel_token: job.cancel_token, + }; let result = catch_unwind(AssertUnwindSafe(|| (job.work)(context))) .unwrap_or(AsyncWorkJobOutcome::Failed); lane.record_finished(job.id, result); @@ -379,6 +465,60 @@ mod tests { assert_eq!(telemetry.completed_jobs, 2); } + #[test] + fn async_work_lane_runs_higher_priority_io_before_assets() { + let lane = Arc::new(AsyncWorkLane::default()); + let (tx, rx) = mpsc::channel(); + + lane.submit_asset_load({ + let tx = tx.clone(); + move |ctx| { + tx.send((ctx.kind, ctx.priority)).unwrap(); + AsyncWorkJobOutcome::Completed + } + }) + .unwrap(); + lane.submit_non_critical_read_list(AsyncWorkJobKind::Fs, { + let tx = tx.clone(); + move |ctx| { + tx.send((ctx.kind, ctx.priority)).unwrap(); + AsyncWorkJobOutcome::Completed + } + }) + .unwrap(); + lane.submit_memcard_commit_write({ + let tx = tx.clone(); + move |ctx| { + tx.send((ctx.kind, ctx.priority)).unwrap(); + AsyncWorkJobOutcome::Completed + } + }) + .unwrap(); + lane.submit_fs_write_config({ + let tx = tx.clone(); + move |ctx| { + tx.send((ctx.kind, ctx.priority)).unwrap(); + AsyncWorkJobOutcome::Completed + } + }) + .unwrap(); + + let mut controller = + AsyncWorkLaneController::start(AsyncWorkLaneConfig::default(), Arc::clone(&lane)); + + assert_eq!( + rx.recv().unwrap(), + (AsyncWorkJobKind::Memcard, AsyncWorkPriority::MemcardCommitWrite) + ); + assert_eq!(rx.recv().unwrap(), (AsyncWorkJobKind::Fs, AsyncWorkPriority::FsWriteConfig)); + assert_eq!(rx.recv().unwrap(), (AsyncWorkJobKind::Asset, AsyncWorkPriority::AssetLoad)); + assert_eq!( + rx.recv().unwrap(), + (AsyncWorkJobKind::Fs, AsyncWorkPriority::NonCriticalReadList) + ); + controller.stop().unwrap(); + } + #[test] fn async_work_lane_has_one_active_job() { let lane = Arc::new(AsyncWorkLane::default()); diff --git a/crates/console/prometeu-system/src/services/fs/mod.rs b/crates/console/prometeu-system/src/services/fs/mod.rs index dcb16257..bed0b115 100644 --- a/crates/console/prometeu-system/src/services/fs/mod.rs +++ b/crates/console/prometeu-system/src/services/fs/mod.rs @@ -8,4 +8,4 @@ pub use fs_backend::FsBackend; pub use fs_entry::FsEntry; pub use fs_error::FsError; pub use fs_state::FsState; -pub use virtual_fs::VirtualFS; +pub use virtual_fs::{FsAsyncLaneOperation, VirtualFS}; diff --git a/crates/console/prometeu-system/src/services/fs/virtual_fs.rs b/crates/console/prometeu-system/src/services/fs/virtual_fs.rs index dc09d3f8..fdc6b6f6 100644 --- a/crates/console/prometeu-system/src/services/fs/virtual_fs.rs +++ b/crates/console/prometeu-system/src/services/fs/virtual_fs.rs @@ -1,4 +1,5 @@ use crate::fs::{FsBackend, FsEntry, FsError}; +use crate::services::async_work::{AsyncWorkJobKind, AsyncWorkPriority}; /// Virtual Filesystem (VFS) interface for Prometeu. /// @@ -13,6 +14,29 @@ pub struct VirtualFS { backend: Option>, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FsAsyncLaneOperation { + Write, + Delete, + Config, + Read, + ListDir, + Exists, +} + +impl FsAsyncLaneOperation { + pub const fn job_kind(self) -> AsyncWorkJobKind { + AsyncWorkJobKind::Fs + } + + pub const fn priority(self) -> AsyncWorkPriority { + match self { + Self::Write | Self::Delete | Self::Config => AsyncWorkPriority::FsWriteConfig, + Self::Read | Self::ListDir | Self::Exists => AsyncWorkPriority::NonCriticalReadList, + } + } +} + impl Default for VirtualFS { fn default() -> Self { Self::new() @@ -242,4 +266,18 @@ mod tests { assert!(matches!(vfs.delete("/"), Err(FsError::PermissionDenied))); assert_eq!(calls.delete.load(Ordering::Relaxed), 0); } + + #[test] + fn fs_async_lane_operations_classify_priority_without_public_api_changes() { + assert_eq!(FsAsyncLaneOperation::Write.job_kind(), AsyncWorkJobKind::Fs); + assert_eq!(FsAsyncLaneOperation::Write.priority(), AsyncWorkPriority::FsWriteConfig); + assert_eq!(FsAsyncLaneOperation::Delete.priority(), AsyncWorkPriority::FsWriteConfig); + assert_eq!(FsAsyncLaneOperation::Config.priority(), AsyncWorkPriority::FsWriteConfig); + assert_eq!(FsAsyncLaneOperation::Read.priority(), AsyncWorkPriority::NonCriticalReadList); + assert_eq!( + FsAsyncLaneOperation::ListDir.priority(), + AsyncWorkPriority::NonCriticalReadList + ); + assert_eq!(FsAsyncLaneOperation::Exists.priority(), AsyncWorkPriority::NonCriticalReadList); + } } diff --git a/crates/console/prometeu-system/src/services/memcard.rs b/crates/console/prometeu-system/src/services/memcard.rs index a53b5436..5ade70eb 100644 --- a/crates/console/prometeu-system/src/services/memcard.rs +++ b/crates/console/prometeu-system/src/services/memcard.rs @@ -1,4 +1,5 @@ use crate::fs::{FsError, VirtualFS}; +use crate::services::async_work::{AsyncWorkJobKind, AsyncWorkPriority}; use std::collections::HashMap; pub const MEMCARD_SLOT_COUNT: usize = 32; @@ -50,6 +51,30 @@ pub struct MemcardWriteResult { pub bytes_written: u32, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MemcardAsyncLaneOperation { + SlotWrite, + SlotCommit, + SlotClear, + SlotRead, + SlotStat, +} + +impl MemcardAsyncLaneOperation { + pub const fn job_kind(self) -> AsyncWorkJobKind { + AsyncWorkJobKind::Memcard + } + + pub const fn priority(self) -> AsyncWorkPriority { + match self { + Self::SlotWrite | Self::SlotCommit | Self::SlotClear => { + AsyncWorkPriority::MemcardCommitWrite + } + Self::SlotRead | Self::SlotStat => AsyncWorkPriority::NonCriticalReadList, + } + } +} + #[derive(Debug, Clone)] struct SlotImage { payload: Vec, @@ -428,4 +453,29 @@ mod tests { let stat = mem.slot_stat(&fs, 7, 2); assert_eq!(stat.state, MemcardSlotState::Corrupt); } + + #[test] + fn memcard_async_lane_operations_classify_priority_without_changing_public_api() { + assert_eq!(MemcardAsyncLaneOperation::SlotWrite.job_kind(), AsyncWorkJobKind::Memcard); + assert_eq!( + MemcardAsyncLaneOperation::SlotWrite.priority(), + AsyncWorkPriority::MemcardCommitWrite + ); + assert_eq!( + MemcardAsyncLaneOperation::SlotCommit.priority(), + AsyncWorkPriority::MemcardCommitWrite + ); + assert_eq!( + MemcardAsyncLaneOperation::SlotClear.priority(), + AsyncWorkPriority::MemcardCommitWrite + ); + assert_eq!( + MemcardAsyncLaneOperation::SlotRead.priority(), + AsyncWorkPriority::NonCriticalReadList + ); + assert_eq!( + MemcardAsyncLaneOperation::SlotStat.priority(), + AsyncWorkPriority::NonCriticalReadList + ); + } } diff --git a/discussion/index.ndjson b/discussion/index.ndjson index 5cb9d955..23b25bdc 100644 --- a/discussion/index.ndjson +++ b/discussion/index.ndjson @@ -18,7 +18,7 @@ {"type":"discussion","id":"DSC-0006","status":"open","ticket":"vm-owned-random-service","title":"Agenda - VM-Owned Random Service","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0005","file":"AGD-0005-vm-owned-random-service.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0007","status":"open","ticket":"app-home-filesystem-surface-and-semantics","title":"Agenda - App Home Filesystem Surface and Semantics","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0006","file":"AGD-0006-app-home-filesystem-surface-and-semantics.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0008","status":"done","ticket":"perf-runtime-telemetry-hot-path","title":"Agenda - [PERF] Runtime Telemetry Hot Path","created_at":"2026-03-27","updated_at":"2026-06-04","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0026","file":"discussion/lessons/DSC-0008-perf-runtime-telemetry-hot-path/LSN-0026-push-based-telemetry-model.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} -{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} +{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} {"type":"discussion","id":"DSC-0010","status":"done","ticket":"perf-host-desktop-frame-pacing-and-presentation","title":"Agenda - [PERF] Host Desktop Frame Pacing and Presentation","created_at":"2026-03-27","updated_at":"2026-04-20","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0036","file":"discussion/lessons/DSC-0010-perf-host-desktop-frame-pacing-and-presentation/LSN-0036-frame-publication-and-host-invalidation-must-be-separate.md","status":"done","created_at":"2026-04-20","updated_at":"2026-04-20"}]} {"type":"discussion","id":"DSC-0011","status":"abandoned","ticket":"perf-gfx-render-pipeline-and-dirty-regions","title":"Agenda - [PERF] GFX Render Pipeline and Dirty Regions","created_at":"2026-03-27","updated_at":"2026-06-28","tags":[],"agendas":[{"id":"AGD-0010","file":"AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md","status":"abandoned","created_at":"2026-03-27","updated_at":"2026-06-28","_override_reason":"User explicitly chose to remove this agenda because it no longer applies to the current render scenario; future optimizations can be discussed when needed."}],"decisions":[],"plans":[],"lessons":[],"_override_reason":"User explicitly chose to abandon this discussion because it no longer applies to the current render scenario; future optimizations can be discussed when needed."} {"type":"discussion","id":"DSC-0012","status":"done","ticket":"perf-runtime-introspection-syscalls","title":"Agenda - [PERF] Runtime Introspection Syscalls","created_at":"2026-03-27","updated_at":"2026-04-19","tags":["perf","runtime","syscall","telemetry","debug","asset"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0034","file":"discussion/lessons/DSC-0012-perf-runtime-introspection-syscalls/LSN-0034-host-owned-debug-boundaries.md","status":"done","created_at":"2026-04-19","updated_at":"2026-04-19"}]} diff --git a/discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md b/discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md index 01bf40cc..588a197c 100644 --- a/discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md +++ b/discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md @@ -26,6 +26,16 @@ Sem um contrato claro de `home` por app, a API tende a crescer com semantica inc agenda deve decidir a API publica de FS considerando essa lane, mas a existencia da lane ja esta fechada por `DEC-0034`. +## Fronteira com a Async IO Lane + +`DEC-0034` fecha a existencia de uma lane serial para trabalho async de IO. FS +pode consumir essa lane internamente para escrita, configuracao, leitura ou +listagem, conforme prioridade operacional definida pelo runtime. + +Esta agenda continua dona da decisao sobre API publica de FS para `app home`. +Portanto, a existencia da lane nao implica criar agora request handles, polling +publico ou novas syscalls de FS. Esses shapes permanecem em aberto aqui. + ## Alvo da Discussao Fechar a superficie e semantica funcional de `fs` para perfil `app`, com sandbox por `app_id` e transporte de bytes via decisao `003`. diff --git a/discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md b/discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md index d04bae32..4dd01f7b 100644 --- a/discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md +++ b/discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md @@ -2,7 +2,7 @@ id: PLN-0127 ticket: perf-async-background-work-lanes-for-assets-and-fs title: Async Lane Memcard and FS Integration Boundaries -status: open +status: done created: 2026-06-28 completed: ref_decisions: [DEC-0034] From 0adec90f233ac0282b2fa7b19f47b023aa66823b Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Sun, 28 Jun 2026 14:20:53 +0100 Subject: [PATCH 7/9] implements PLN-0128 asset pipeline telemetry tests --- crates/console/prometeu-drivers/src/asset.rs | 369 +++++++++++++++++- crates/console/prometeu-hal/src/asset.rs | 6 +- discussion/index.ndjson | 2 +- ...t-pipeline-contract-tests-and-telemetry.md | 2 +- 4 files changed, 362 insertions(+), 17 deletions(-) diff --git a/crates/console/prometeu-drivers/src/asset.rs b/crates/console/prometeu-drivers/src/asset.rs index 4265b092..c81654b1 100644 --- a/crates/console/prometeu-drivers/src/asset.rs +++ b/crates/console/prometeu-drivers/src/asset.rs @@ -34,6 +34,134 @@ type HandleTable = HashMap; type TargetHandleTable = HashMap; type TargetGenerationTable = HashMap; +const ASSET_PROGRESS_IDLE: u16 = 0; +const ASSET_PROGRESS_LOADING: u16 = 1_000; +const ASSET_PROGRESS_DONE: u16 = 10_000; +const ASSET_TELEMETRY_SAMPLE_WINDOW: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct AssetLatencyPercentiles { + pub p50_micros: u64, + pub p95_micros: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct AssetPipelineTelemetry { + pub submitted_requests: u64, + pub started_jobs: u64, + pub completed_jobs: u64, + pub failed_jobs: u64, + pub canceled_requests: u64, + pub superseded_requests: u64, + pub stale_results_discarded: u64, + pub current_backlog_depth: usize, + pub max_backlog_depth: usize, + pub active_progress: u16, + pub glyph_latency: AssetLatencyPercentiles, + pub sound_latency: AssetLatencyPercentiles, + pub scene_latency: AssetLatencyPercentiles, + pub last_asset_id: Option, + pub last_asset_latency: AssetLatencyPercentiles, +} + +#[derive(Default)] +struct AssetPipelineTelemetryState { + snapshot: AssetPipelineTelemetry, + bank_samples: HashMap>, + asset_samples: HashMap>, +} + +impl AssetPipelineTelemetryState { + fn record_submitted(&mut self, backlog_depth: usize) { + self.snapshot.submitted_requests += 1; + self.snapshot.current_backlog_depth = backlog_depth; + self.snapshot.max_backlog_depth = self.snapshot.max_backlog_depth.max(backlog_depth); + } + + fn record_started(&mut self, backlog_depth: usize) { + self.snapshot.started_jobs += 1; + self.snapshot.current_backlog_depth = backlog_depth; + self.snapshot.active_progress = ASSET_PROGRESS_LOADING; + } + + fn record_completed(&mut self, asset_id: AssetId, bank_type: BankType, duration_micros: u64) { + self.snapshot.completed_jobs += 1; + self.snapshot.active_progress = ASSET_PROGRESS_IDLE; + self.record_latency_sample(asset_id, bank_type, duration_micros); + } + + fn record_failed(&mut self, asset_id: AssetId, bank_type: BankType, duration_micros: u64) { + self.snapshot.failed_jobs += 1; + self.snapshot.active_progress = ASSET_PROGRESS_IDLE; + self.record_latency_sample(asset_id, bank_type, duration_micros); + } + + fn record_canceled(&mut self) { + self.snapshot.canceled_requests += 1; + } + + fn record_superseded(&mut self, count: usize) { + self.snapshot.superseded_requests += count as u64; + } + + fn record_stale_discard(&mut self) { + self.snapshot.stale_results_discarded += 1; + self.snapshot.superseded_requests += 1; + self.snapshot.active_progress = ASSET_PROGRESS_IDLE; + } + + fn record_latency_sample( + &mut self, + asset_id: AssetId, + bank_type: BankType, + duration_micros: u64, + ) { + push_sample( + self.bank_samples.entry(bank_type).or_default(), + duration_micros, + ASSET_TELEMETRY_SAMPLE_WINDOW, + ); + push_sample( + self.asset_samples.entry(asset_id).or_default(), + duration_micros, + ASSET_TELEMETRY_SAMPLE_WINDOW, + ); + self.snapshot.glyph_latency = percentile_snapshot(self.bank_samples.get(&BankType::GLYPH)); + self.snapshot.sound_latency = percentile_snapshot(self.bank_samples.get(&BankType::SOUNDS)); + self.snapshot.scene_latency = percentile_snapshot(self.bank_samples.get(&BankType::SCENE)); + self.snapshot.last_asset_id = Some(asset_id); + self.snapshot.last_asset_latency = percentile_snapshot(self.asset_samples.get(&asset_id)); + } +} + +fn push_sample(samples: &mut VecDeque, value: u64, window: usize) { + samples.push_back(value); + while samples.len() > window { + samples.pop_front(); + } +} + +fn percentile_snapshot(samples: Option<&VecDeque>) -> AssetLatencyPercentiles { + let Some(samples) = samples else { + return AssetLatencyPercentiles::default(); + }; + if samples.is_empty() { + return AssetLatencyPercentiles::default(); + } + + let mut sorted = samples.iter().copied().collect::>(); + sorted.sort_unstable(); + AssetLatencyPercentiles { + p50_micros: percentile_value(&sorted, 50), + p95_micros: percentile_value(&sorted, 95), + } +} + +fn percentile_value(sorted: &[u64], percentile: usize) -> u64 { + let index = ((sorted.len() - 1) * percentile).div_ceil(100); + sorted[index] +} + #[derive(Clone, Default)] pub struct GlyphAssetSlotIndex { slots_by_asset_id: Arc>>, @@ -188,6 +316,7 @@ pub struct AssetManager { target_generations: Arc>, next_handle_id: Mutex, assets_data: Arc>, + pipeline_telemetry: Arc>, /// Narrow hardware interfaces gfx_installer: Arc, @@ -218,6 +347,7 @@ struct LoadHandleInfo { slot: SlotRef, status: LoadStatus, request_generation: u64, + progress: u16, } #[derive(Clone)] @@ -243,14 +373,17 @@ struct AssetLoadQueue { } impl AssetLoadQueue { - fn submit(&self, job: AssetLoadJob) { + fn submit(&self, job: AssetLoadJob) -> usize { let mut state = self.state.lock().unwrap(); if state.shutdown { - return; + return 0; } + let before_len = state.pending.len(); state.pending.retain(|pending| pending.slot != job.slot); + let removed = before_len - state.pending.len(); state.pending.push_back(job); self.ready.notify_one(); + removed } fn next_job(&self) -> Option { @@ -334,6 +467,7 @@ impl AssetLoadWorker { handles: Arc>, target_generations: Arc>, assets_data: Arc>, + pipeline_telemetry: Arc>, gfx_policy: BankPolicy, sound_policy: BankPolicy, scene_policy: BankPolicy, @@ -347,6 +481,7 @@ impl AssetLoadWorker { &handles, &target_generations, &assets_data, + &pipeline_telemetry, &gfx_policy, &sound_policy, &scene_policy, @@ -358,8 +493,8 @@ impl AssetLoadWorker { Self { queue, join_handle: Some(join_handle) } } - fn submit(&self, job: AssetLoadJob) { - self.queue.submit(job); + fn submit_counting_superseded(&self, job: AssetLoadJob) -> usize { + self.queue.submit(job) } fn clear_pending(&self) { @@ -533,6 +668,7 @@ impl AssetManager { let handles = Arc::new(RwLock::new(HashMap::new())); let target_generations = Arc::new(RwLock::new(HashMap::new())); let assets_data = Arc::new(RwLock::new(assets_data)); + let pipeline_telemetry = Arc::new(Mutex::new(AssetPipelineTelemetryState::default())); let gfx_policy = BankPolicy::new(); let sound_policy = BankPolicy::new(); let scene_policy = BankPolicy::new(); @@ -540,6 +676,7 @@ impl AssetManager { Arc::clone(&handles), Arc::clone(&target_generations), Arc::clone(&assets_data), + Arc::clone(&pipeline_telemetry), gfx_policy.clone(), sound_policy.clone(), scene_policy.clone(), @@ -562,6 +699,7 @@ impl AssetManager { target_generations, next_handle_id: Mutex::new(1), assets_data, + pipeline_telemetry, load_worker, pending_commits: Mutex::new(Vec::new()), } @@ -769,8 +907,11 @@ impl AssetManager { slot, status: LoadStatus::READY, request_generation, + progress: ASSET_PROGRESS_DONE, }, ); + self.pipeline_telemetry.lock().unwrap().record_submitted(0); + self.pipeline_telemetry.lock().unwrap().record_completed(asset_id, entry.bank_type, 0); return Ok(handle_id); } @@ -782,16 +923,21 @@ impl AssetManager { slot, status: LoadStatus::PENDING, request_generation, + progress: ASSET_PROGRESS_IDLE, }, ); - self.load_worker.submit(AssetLoadJob { + let superseded = self.load_worker.submit_counting_superseded(AssetLoadJob { handle_id, asset_id, slot, request_generation, entry, }); + let backlog_depth = self.load_worker.pending_count(); + let mut telemetry = self.pipeline_telemetry.lock().unwrap(); + telemetry.record_submitted(backlog_depth); + telemetry.record_superseded(superseded); Ok(handle_id) } @@ -801,12 +947,14 @@ impl AssetManager { handles: &Arc>, target_generations: &Arc>, assets_data: &Arc>, + pipeline_telemetry: &Arc>, gfx_policy: &BankPolicy, sound_policy: &BankPolicy, scene_policy: &BankPolicy, ) { if !Self::is_current_target_generation(target_generations, job.slot, job.request_generation) { + pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } @@ -821,7 +969,10 @@ impl AssetManager { return; } handle.status = LoadStatus::LOADING; + handle.progress = ASSET_PROGRESS_LOADING; } + let started_at = Instant::now(); + pipeline_telemetry.lock().unwrap().record_started(0); match job.entry.bank_type { BankType::GLYPH => { @@ -833,6 +984,7 @@ impl AssetManager { job.slot, job.request_generation, ) { + pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } let bank_arc = Arc::new(tilebank); @@ -846,6 +998,7 @@ impl AssetManager { job.slot, job.request_generation, ) { + pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } gfx_policy.stage( @@ -854,8 +1007,22 @@ impl AssetManager { job.entry.decoded_size as usize, ); Self::complete_load_job(handles, &job, LoadStatus::READY); + Self::record_load_job_finished( + pipeline_telemetry, + &job, + started_at, + LoadStatus::READY, + ); + } + Err(_) => { + Self::complete_load_job(handles, &job, LoadStatus::ERROR); + Self::record_load_job_finished( + pipeline_telemetry, + &job, + started_at, + LoadStatus::ERROR, + ); } - Err(_) => Self::complete_load_job(handles, &job, LoadStatus::ERROR), } } BankType::SOUNDS => { @@ -867,6 +1034,7 @@ impl AssetManager { job.slot, job.request_generation, ) { + pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } let bank_arc = Arc::new(soundbank); @@ -880,6 +1048,7 @@ impl AssetManager { job.slot, job.request_generation, ) { + pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } sound_policy.stage( @@ -888,8 +1057,22 @@ impl AssetManager { job.entry.decoded_size as usize, ); Self::complete_load_job(handles, &job, LoadStatus::READY); + Self::record_load_job_finished( + pipeline_telemetry, + &job, + started_at, + LoadStatus::READY, + ); + } + Err(_) => { + Self::complete_load_job(handles, &job, LoadStatus::ERROR); + Self::record_load_job_finished( + pipeline_telemetry, + &job, + started_at, + LoadStatus::ERROR, + ); } - Err(_) => Self::complete_load_job(handles, &job, LoadStatus::ERROR), } } BankType::SCENE => { @@ -901,6 +1084,7 @@ impl AssetManager { job.slot, job.request_generation, ) { + pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } let bank_arc = Arc::new(scenebank); @@ -914,6 +1098,7 @@ impl AssetManager { job.slot, job.request_generation, ) { + pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } scene_policy.stage( @@ -922,13 +1107,46 @@ impl AssetManager { job.entry.decoded_size as usize, ); Self::complete_load_job(handles, &job, LoadStatus::READY); + Self::record_load_job_finished( + pipeline_telemetry, + &job, + started_at, + LoadStatus::READY, + ); + } + Err(_) => { + Self::complete_load_job(handles, &job, LoadStatus::ERROR); + Self::record_load_job_finished( + pipeline_telemetry, + &job, + started_at, + LoadStatus::ERROR, + ); } - Err(_) => Self::complete_load_job(handles, &job, LoadStatus::ERROR), } } } } + fn record_load_job_finished( + pipeline_telemetry: &Arc>, + job: &AssetLoadJob, + started_at: Instant, + status: LoadStatus, + ) { + let duration_micros = started_at.elapsed().as_micros() as u64; + let mut telemetry = pipeline_telemetry.lock().unwrap(); + match status { + LoadStatus::READY => { + telemetry.record_completed(job.asset_id, job.entry.bank_type, duration_micros) + } + LoadStatus::ERROR => { + telemetry.record_failed(job.asset_id, job.entry.bank_type, duration_micros) + } + _ => {} + } + } + fn complete_load_job( handles: &Arc>, job: &AssetLoadJob, @@ -940,6 +1158,7 @@ impl AssetManager { && handle.status == LoadStatus::LOADING { handle.status = status; + handle.progress = ASSET_PROGRESS_DONE; } } } @@ -1291,6 +1510,10 @@ impl AssetManager { pub fn backlog_info(&self) -> AssetBacklogInfo { let active = self.load_worker.active_job(); + let active_progress = active + .as_ref() + .and_then(|job| self.handles.read().unwrap().get(&job.handle_id).map(|h| h.progress)) + .unwrap_or(ASSET_PROGRESS_IDLE); AssetBacklogInfo { status: AssetOpStatus::Ok, pending_count: self.load_worker.pending_count() as u32, @@ -1298,7 +1521,7 @@ impl AssetManager { active_asset_id: active.as_ref().map(|job| job.asset_id), active_bank_type: active.as_ref().map(|job| job.slot.asset_type), active_slot: active.as_ref().map(|job| job.slot.index), - active_progress: 0, + active_progress, } } @@ -1317,7 +1540,13 @@ impl AssetManager { status: AssetOpStatus::Ok, state: LoadStatus::QUEUED, position: position as u32, - progress: 0, + progress: self + .handles + .read() + .unwrap() + .get(&handle) + .map(|h| h.progress) + .unwrap_or(ASSET_PROGRESS_IDLE), }; } @@ -1326,7 +1555,13 @@ impl AssetManager { status: AssetOpStatus::Ok, state: LoadStatus::ACTIVE, position: 0, - progress: 0, + progress: self + .handles + .read() + .unwrap() + .get(&handle) + .map(|h| h.progress) + .unwrap_or(ASSET_PROGRESS_LOADING), }; } @@ -1340,7 +1575,13 @@ impl AssetManager { .map(|h| h.status) .unwrap_or_else(|| self.idle_target_state(slot)), position: 0, - progress: 0, + progress: self + .handles + .read() + .unwrap() + .get(&handle) + .map(|h| h.progress) + .unwrap_or(ASSET_PROGRESS_IDLE), } } @@ -1407,6 +1648,13 @@ impl AssetManager { } } + pub fn pipeline_telemetry(&self) -> AssetPipelineTelemetry { + let mut snapshot = self.pipeline_telemetry.lock().unwrap().snapshot.clone(); + snapshot.current_backlog_depth = self.load_worker.pending_count(); + snapshot.active_progress = self.backlog_info().active_progress; + snapshot + } + pub fn commit(&self, handle: HandleId) -> AssetOpStatus { let mut handles_map = self.handles.write().unwrap(); let Some(h) = handles_map.get_mut(&handle) else { @@ -1436,6 +1684,7 @@ impl AssetManager { match h.status { LoadStatus::PENDING | LoadStatus::LOADING | LoadStatus::READY => { h.status = LoadStatus::CANCELED; + h.progress = ASSET_PROGRESS_DONE; canceled_slot = Some(h.slot); } _ => {} @@ -1444,6 +1693,7 @@ impl AssetManager { } if let Some(slot) = canceled_slot { self.bump_target_generation(slot); + self.pipeline_telemetry.lock().unwrap().record_canceled(); } self.clear_staging_for_handle(handle); final_status @@ -2089,6 +2339,26 @@ mod tests { assert_eq!(queue.pending_position(3), Some(3)); } + #[test] + fn test_asset_backlog_queue_keeps_only_latest_request_per_target() { + let queue = AssetLoadQueue::default(); + let entry = test_glyph_asset_entry("queued_glyphs", 16, 16); + let job = |handle_id: HandleId, request_generation: u64| AssetLoadJob { + handle_id, + asset_id: handle_id as AssetId, + slot: SlotRef::gfx(0), + request_generation, + entry: entry.clone(), + }; + + assert_eq!(queue.submit(job(1, 1)), 0); + assert_eq!(queue.submit(job(2, 2)), 1); + + assert_eq!(queue.pending_count(), 1); + assert_eq!(queue.pending_position(1), None); + assert_eq!(queue.pending_position(2), Some(1)); + } + #[test] fn test_asset_target_status_exposes_empty_and_ready_slot_handle() { let banks = Arc::new(MemoryBanks::new()); @@ -2129,6 +2399,81 @@ mod tests { assert_eq!(ready.state, LoadStatus::READY); } + #[test] + fn test_asset_pipeline_progress_and_telemetry_snapshot_for_success() { + let banks = Arc::new(MemoryBanks::new()); + let gfx_installer = Arc::clone(&banks) as Arc; + let sound_installer = Arc::clone(&banks) as Arc; + let scene_installer = Arc::clone(&banks) as Arc; + + let data = test_glyph_asset_data(); + let asset_entry = test_glyph_asset_entry("test_glyphs", 16, 16); + + let am = AssetManager::new( + vec![asset_entry], + AssetsPayloadSource::from_bytes(data), + gfx_installer, + sound_installer, + scene_installer, + ); + + let empty = am.target_status(BankType::GLYPH, 0); + assert_eq!(empty.progress, ASSET_PROGRESS_IDLE); + + let handle = am.load(0, 0).unwrap(); + let start = Instant::now(); + while am.status(handle) != LoadStatus::READY && start.elapsed().as_secs() < 5 { + thread::sleep(std::time::Duration::from_millis(10)); + } + + let position = am.backlog_position(handle); + assert_eq!(position.state, LoadStatus::READY); + assert_eq!(position.progress, ASSET_PROGRESS_DONE); + + let telemetry = am.pipeline_telemetry(); + assert_eq!(telemetry.submitted_requests, 1); + assert_eq!(telemetry.started_jobs, 1); + assert_eq!(telemetry.completed_jobs, 1); + assert_eq!(telemetry.failed_jobs, 0); + assert_eq!(telemetry.active_progress, ASSET_PROGRESS_IDLE); + assert_eq!(telemetry.last_asset_id, Some(0)); + } + + #[test] + fn test_asset_pipeline_progress_and_telemetry_snapshot_for_failure() { + let banks = Arc::new(MemoryBanks::new()); + let gfx_installer = Arc::clone(&banks) as Arc; + let sound_installer = Arc::clone(&banks) as Arc; + let scene_installer = Arc::clone(&banks) as Arc; + + let asset_entry = test_glyph_asset_entry("broken_glyphs", 16, 16); + + let am = AssetManager::new( + vec![asset_entry], + AssetsPayloadSource::from_bytes(Vec::new()), + gfx_installer, + sound_installer, + scene_installer, + ); + + let handle = am.load(0, 0).unwrap(); + let start = Instant::now(); + while am.status(handle) != LoadStatus::ERROR && start.elapsed().as_secs() < 5 { + thread::sleep(std::time::Duration::from_millis(10)); + } + + let position = am.backlog_position(handle); + assert_eq!(position.state, LoadStatus::ERROR); + assert_eq!(position.progress, ASSET_PROGRESS_DONE); + + let telemetry = am.pipeline_telemetry(); + assert_eq!(telemetry.submitted_requests, 1); + assert_eq!(telemetry.started_jobs, 1); + assert_eq!(telemetry.completed_jobs, 0); + assert_eq!(telemetry.failed_jobs, 1); + assert_eq!(telemetry.last_asset_id, Some(0)); + } + #[test] fn test_asset_backlog_reorder_rejects_non_queued_request() { let banks = Arc::new(MemoryBanks::new()); diff --git a/crates/console/prometeu-hal/src/asset.rs b/crates/console/prometeu-hal/src/asset.rs index b49483f8..173cfbe6 100644 --- a/crates/console/prometeu-hal/src/asset.rs +++ b/crates/console/prometeu-hal/src/asset.rs @@ -146,7 +146,7 @@ pub struct AssetBacklogInfo { pub active_asset_id: Option, pub active_bank_type: Option, pub active_slot: Option, - pub active_progress: u8, + pub active_progress: u16, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -154,7 +154,7 @@ pub struct AssetBacklogPosition { pub status: AssetOpStatus, pub state: LoadStatus, pub position: u32, - pub progress: u8, + pub progress: u16, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -164,7 +164,7 @@ pub struct AssetTargetStatus { pub handle: HandleId, pub state: LoadStatus, pub position: u32, - pub progress: u8, + pub progress: u16, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/discussion/index.ndjson b/discussion/index.ndjson index 23b25bdc..96f2e735 100644 --- a/discussion/index.ndjson +++ b/discussion/index.ndjson @@ -18,7 +18,7 @@ {"type":"discussion","id":"DSC-0006","status":"open","ticket":"vm-owned-random-service","title":"Agenda - VM-Owned Random Service","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0005","file":"AGD-0005-vm-owned-random-service.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0007","status":"open","ticket":"app-home-filesystem-surface-and-semantics","title":"Agenda - App Home Filesystem Surface and Semantics","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0006","file":"AGD-0006-app-home-filesystem-surface-and-semantics.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0008","status":"done","ticket":"perf-runtime-telemetry-hot-path","title":"Agenda - [PERF] Runtime Telemetry Hot Path","created_at":"2026-03-27","updated_at":"2026-06-04","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0026","file":"discussion/lessons/DSC-0008-perf-runtime-telemetry-hot-path/LSN-0026-push-based-telemetry-model.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} -{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"open","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} +{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} {"type":"discussion","id":"DSC-0010","status":"done","ticket":"perf-host-desktop-frame-pacing-and-presentation","title":"Agenda - [PERF] Host Desktop Frame Pacing and Presentation","created_at":"2026-03-27","updated_at":"2026-04-20","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0036","file":"discussion/lessons/DSC-0010-perf-host-desktop-frame-pacing-and-presentation/LSN-0036-frame-publication-and-host-invalidation-must-be-separate.md","status":"done","created_at":"2026-04-20","updated_at":"2026-04-20"}]} {"type":"discussion","id":"DSC-0011","status":"abandoned","ticket":"perf-gfx-render-pipeline-and-dirty-regions","title":"Agenda - [PERF] GFX Render Pipeline and Dirty Regions","created_at":"2026-03-27","updated_at":"2026-06-28","tags":[],"agendas":[{"id":"AGD-0010","file":"AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md","status":"abandoned","created_at":"2026-03-27","updated_at":"2026-06-28","_override_reason":"User explicitly chose to remove this agenda because it no longer applies to the current render scenario; future optimizations can be discussed when needed."}],"decisions":[],"plans":[],"lessons":[],"_override_reason":"User explicitly chose to abandon this discussion because it no longer applies to the current render scenario; future optimizations can be discussed when needed."} {"type":"discussion","id":"DSC-0012","status":"done","ticket":"perf-runtime-introspection-syscalls","title":"Agenda - [PERF] Runtime Introspection Syscalls","created_at":"2026-03-27","updated_at":"2026-04-19","tags":["perf","runtime","syscall","telemetry","debug","asset"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0034","file":"discussion/lessons/DSC-0012-perf-runtime-introspection-syscalls/LSN-0034-host-owned-debug-boundaries.md","status":"done","created_at":"2026-04-19","updated_at":"2026-04-19"}]} diff --git a/discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md b/discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md index 7b60662b..9600178c 100644 --- a/discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md +++ b/discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md @@ -2,7 +2,7 @@ id: PLN-0128 ticket: perf-async-background-work-lanes-for-assets-and-fs title: Async Asset Pipeline Contract Tests and Telemetry -status: open +status: done created: 2026-06-28 completed: ref_decisions: [DEC-0034] From b84946a3a04f1c5778234ff48595d9074ad869b5 Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Wed, 1 Jul 2026 10:01:51 +0100 Subject: [PATCH 8/9] [PERF] Async Background Work Lanes for Assets and FS --- crates/console/prometeu-drivers/src/asset.rs | 123 +++++++++--------- .../src/services/async_work.rs | 16 ++- 2 files changed, 74 insertions(+), 65 deletions(-) diff --git a/crates/console/prometeu-drivers/src/asset.rs b/crates/console/prometeu-drivers/src/asset.rs index c81654b1..b80ddff2 100644 --- a/crates/console/prometeu-drivers/src/asset.rs +++ b/crates/console/prometeu-drivers/src/asset.rs @@ -359,6 +359,16 @@ struct AssetLoadJob { entry: AssetEntry, } +struct AssetLoadJobResources<'a> { + handles: &'a Arc>, + target_generations: &'a Arc>, + assets_data: &'a Arc>, + pipeline_telemetry: &'a Arc>, + gfx_policy: &'a BankPolicy, + sound_policy: &'a BankPolicy, + scene_policy: &'a BankPolicy, +} + #[derive(Default)] struct AssetLoadQueueState { pending: VecDeque, @@ -475,17 +485,17 @@ impl AssetLoadWorker { let queue = Arc::new(AssetLoadQueue::default()); let worker_queue = Arc::clone(&queue); let join_handle = thread::spawn(move || { + let resources = AssetLoadJobResources { + handles: &handles, + target_generations: &target_generations, + assets_data: &assets_data, + pipeline_telemetry: &pipeline_telemetry, + gfx_policy: &gfx_policy, + sound_policy: &sound_policy, + scene_policy: &scene_policy, + }; while let Some(job) = worker_queue.next_job() { - AssetManager::process_load_job( - job.clone(), - &handles, - &target_generations, - &assets_data, - &pipeline_telemetry, - &gfx_policy, - &sound_policy, - &scene_policy, - ); + AssetManager::process_load_job(job.clone(), &resources); worker_queue.complete_active(job.handle_id, job.request_generation); } }); @@ -942,24 +952,18 @@ impl AssetManager { Ok(handle_id) } - fn process_load_job( - job: AssetLoadJob, - handles: &Arc>, - target_generations: &Arc>, - assets_data: &Arc>, - pipeline_telemetry: &Arc>, - gfx_policy: &BankPolicy, - sound_policy: &BankPolicy, - scene_policy: &BankPolicy, - ) { - if !Self::is_current_target_generation(target_generations, job.slot, job.request_generation) - { - pipeline_telemetry.lock().unwrap().record_stale_discard(); + fn process_load_job(job: AssetLoadJob, resources: &AssetLoadJobResources<'_>) { + if !Self::is_current_target_generation( + resources.target_generations, + job.slot, + job.request_generation, + ) { + resources.pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } { - let mut handles_map = handles.write().unwrap(); + let mut handles_map = resources.handles.write().unwrap(); let Some(handle) = handles_map.get_mut(&job.handle_id) else { return; }; @@ -972,52 +976,53 @@ impl AssetManager { handle.progress = ASSET_PROGRESS_LOADING; } let started_at = Instant::now(); - pipeline_telemetry.lock().unwrap().record_started(0); + resources.pipeline_telemetry.lock().unwrap().record_started(0); match job.entry.bank_type { BankType::GLYPH => { - let result = Self::perform_load_glyph_bank(&job.entry, Arc::clone(assets_data)); + let result = + Self::perform_load_glyph_bank(&job.entry, Arc::clone(resources.assets_data)); match result { Ok(tilebank) => { if !Self::is_current_target_generation( - target_generations, + resources.target_generations, job.slot, job.request_generation, ) { - pipeline_telemetry.lock().unwrap().record_stale_discard(); + resources.pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } let bank_arc = Arc::new(tilebank); - let resident_arc = gfx_policy.put_resident( + let resident_arc = resources.gfx_policy.put_resident( job.asset_id, bank_arc, job.entry.decoded_size as usize, ); if !Self::is_current_target_generation( - target_generations, + resources.target_generations, job.slot, job.request_generation, ) { - pipeline_telemetry.lock().unwrap().record_stale_discard(); + resources.pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } - gfx_policy.stage( + resources.gfx_policy.stage( job.handle_id, resident_arc, job.entry.decoded_size as usize, ); - Self::complete_load_job(handles, &job, LoadStatus::READY); + Self::complete_load_job(resources.handles, &job, LoadStatus::READY); Self::record_load_job_finished( - pipeline_telemetry, + resources.pipeline_telemetry, &job, started_at, LoadStatus::READY, ); } Err(_) => { - Self::complete_load_job(handles, &job, LoadStatus::ERROR); + Self::complete_load_job(resources.handles, &job, LoadStatus::ERROR); Self::record_load_job_finished( - pipeline_telemetry, + resources.pipeline_telemetry, &job, started_at, LoadStatus::ERROR, @@ -1026,48 +1031,49 @@ impl AssetManager { } } BankType::SOUNDS => { - let result = Self::perform_load_sound_bank(&job.entry, Arc::clone(assets_data)); + let result = + Self::perform_load_sound_bank(&job.entry, Arc::clone(resources.assets_data)); match result { Ok(soundbank) => { if !Self::is_current_target_generation( - target_generations, + resources.target_generations, job.slot, job.request_generation, ) { - pipeline_telemetry.lock().unwrap().record_stale_discard(); + resources.pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } let bank_arc = Arc::new(soundbank); - let resident_arc = sound_policy.put_resident( + let resident_arc = resources.sound_policy.put_resident( job.asset_id, bank_arc, job.entry.decoded_size as usize, ); if !Self::is_current_target_generation( - target_generations, + resources.target_generations, job.slot, job.request_generation, ) { - pipeline_telemetry.lock().unwrap().record_stale_discard(); + resources.pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } - sound_policy.stage( + resources.sound_policy.stage( job.handle_id, resident_arc, job.entry.decoded_size as usize, ); - Self::complete_load_job(handles, &job, LoadStatus::READY); + Self::complete_load_job(resources.handles, &job, LoadStatus::READY); Self::record_load_job_finished( - pipeline_telemetry, + resources.pipeline_telemetry, &job, started_at, LoadStatus::READY, ); } Err(_) => { - Self::complete_load_job(handles, &job, LoadStatus::ERROR); + Self::complete_load_job(resources.handles, &job, LoadStatus::ERROR); Self::record_load_job_finished( - pipeline_telemetry, + resources.pipeline_telemetry, &job, started_at, LoadStatus::ERROR, @@ -1076,48 +1082,49 @@ impl AssetManager { } } BankType::SCENE => { - let result = Self::perform_load_scene_bank(&job.entry, Arc::clone(assets_data)); + let result = + Self::perform_load_scene_bank(&job.entry, Arc::clone(resources.assets_data)); match result { Ok(scenebank) => { if !Self::is_current_target_generation( - target_generations, + resources.target_generations, job.slot, job.request_generation, ) { - pipeline_telemetry.lock().unwrap().record_stale_discard(); + resources.pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } let bank_arc = Arc::new(scenebank); - let resident_arc = scene_policy.put_resident( + let resident_arc = resources.scene_policy.put_resident( job.asset_id, bank_arc, job.entry.decoded_size as usize, ); if !Self::is_current_target_generation( - target_generations, + resources.target_generations, job.slot, job.request_generation, ) { - pipeline_telemetry.lock().unwrap().record_stale_discard(); + resources.pipeline_telemetry.lock().unwrap().record_stale_discard(); return; } - scene_policy.stage( + resources.scene_policy.stage( job.handle_id, resident_arc, job.entry.decoded_size as usize, ); - Self::complete_load_job(handles, &job, LoadStatus::READY); + Self::complete_load_job(resources.handles, &job, LoadStatus::READY); Self::record_load_job_finished( - pipeline_telemetry, + resources.pipeline_telemetry, &job, started_at, LoadStatus::READY, ); } Err(_) => { - Self::complete_load_job(handles, &job, LoadStatus::ERROR); + Self::complete_load_job(resources.handles, &job, LoadStatus::ERROR); Self::record_load_job_finished( - pipeline_telemetry, + resources.pipeline_telemetry, &job, started_at, LoadStatus::ERROR, diff --git a/crates/console/prometeu-system/src/services/async_work.rs b/crates/console/prometeu-system/src/services/async_work.rs index 475ef27e..55c61cd9 100644 --- a/crates/console/prometeu-system/src/services/async_work.rs +++ b/crates/console/prometeu-system/src/services/async_work.rs @@ -150,11 +150,11 @@ impl AsyncWorkCancelToken { } pub fn cancel(&self) { - self.canceled.store(true, Ordering::Release); + self.canceled.store(true, Ordering::Relaxed); } pub fn is_canceled(&self) -> bool { - self.canceled.load(Ordering::Acquire) + self.canceled.load(Ordering::Relaxed) } } @@ -277,11 +277,13 @@ impl AsyncWorkLane { pub fn request_cancel(&self, job_id: AsyncWorkJobId) -> bool { let state = self.state.lock().unwrap(); - if state.active.is_some_and(|active| active.id == job_id) { - if let Some(token) = &state.active_cancel_token { - token.cancel(); - return true; - } + if let Some(token) = state + .active + .filter(|active| active.id == job_id) + .and(state.active_cancel_token.as_ref()) + { + token.cancel(); + return true; } for pending in &state.pending { From 5d3db4c22a64e8938507f30892d852af4ac2ddd6 Mon Sep 17 00:00:00 2001 From: bQUARKz Date: Wed, 1 Jul 2026 10:09:33 +0100 Subject: [PATCH 9/9] [PERF] Async Background Work Lanes for Assets and FS --- discussion/index.ndjson | 4 +- ...al-async-lanes-bound-backlog-complexity.md | 100 ++++ ...background-work-lanes-for-assets-and-fs.md | 557 ------------------ ...nc-work-lane-and-asset-backlog-contract.md | 336 ----------- ...ync-work-lane-specification-propagation.md | 118 ---- ...-async-work-lane-runtime-infrastructure.md | 119 ---- ...5-asset-backlog-and-stable-slot-handles.md | 127 ---- ...t-backlog-public-api-and-status-surface.md | 124 ---- ...e-memcard-and-fs-integration-boundaries.md | 120 ---- ...t-pipeline-contract-tests-and-telemetry.md | 126 ---- 10 files changed, 102 insertions(+), 1629 deletions(-) create mode 100644 discussion/lessons/DSC-0009-perf-async-background-work-lanes-for-assets-and-fs/LSN-0050-serial-async-lanes-bound-backlog-complexity.md delete mode 100644 discussion/workflow/agendas/AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md delete mode 100644 discussion/workflow/decisions/DEC-0034-async-work-lane-and-asset-backlog-contract.md delete mode 100644 discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md delete mode 100644 discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md delete mode 100644 discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md delete mode 100644 discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md delete mode 100644 discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md delete mode 100644 discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md diff --git a/discussion/index.ndjson b/discussion/index.ndjson index 96f2e735..d40b78be 100644 --- a/discussion/index.ndjson +++ b/discussion/index.ndjson @@ -1,4 +1,4 @@ -{"type":"meta","next_id":{"DSC":43,"AGD":44,"DEC":35,"PLN":129,"LSN":50,"CLSN":1}} +{"type":"meta","next_id":{"DSC":43,"AGD":44,"DEC":35,"PLN":129,"LSN":51,"CLSN":1}} {"type":"discussion","id":"DSC-0039","status":"abandoned","ticket":"render-pipeline-family-and-future-3d","title":"Render Pipeline Family and Future 3D","created_at":"2026-06-04","updated_at":"2026-06-04","tags":["gfx","renderer","runtime","architecture","pipeline"],"agendas":[{"id":"AGD-0039","file":"AGD-0039-render-pipeline-family-and-future-3d.md","status":"abandoned","created_at":"2026-06-04","updated_at":"2026-06-04","_override_reason":"User explicitly chose to close this agenda without a new decision because DSC-0038 already established enough architecture for future extension, and 3D is intentionally deferred."}],"decisions":[],"plans":[],"lessons":[],"_override_reason":"User explicitly chose to close this agenda without a new decision because DSC-0038 already established enough architecture for future extension, and 3D is intentionally deferred."} {"type":"discussion","id":"DSC-0040","status":"done","ticket":"vm-render-parallel-execution-boundary","title":"VM and Render Parallel Execution Boundary","created_at":"2026-06-04","updated_at":"2026-06-06","tags":["runtime","renderer","vm","concurrency","architecture","perf"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0048","file":"discussion/lessons/DSC-0040-vm-render-parallel-execution-boundary/LSN-0048-render-workers-need-a-closed-packet-contract.md","status":"done","created_at":"2026-06-06","updated_at":"2026-06-06"}]} {"type":"discussion","id":"DSC-0041","status":"open","ticket":"foreground-stack-game-pause-shell-vm-backed","title":"Foreground Stack, Game Pause, and VM-Backed Shell Coexistence","created_at":"2026-06-05","updated_at":"2026-06-05","tags":["runtime","os","lifecycle","shell","game","vm","foreground","architecture"],"agendas":[{"id":"AGD-0041","file":"AGD-0041-foreground-stack-game-pause-shell-vm-backed.md","status":"open","created_at":"2026-06-05","updated_at":"2026-06-05"}],"decisions":[],"plans":[],"lessons":[]} @@ -18,7 +18,7 @@ {"type":"discussion","id":"DSC-0006","status":"open","ticket":"vm-owned-random-service","title":"Agenda - VM-Owned Random Service","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0005","file":"AGD-0005-vm-owned-random-service.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0007","status":"open","ticket":"app-home-filesystem-surface-and-semantics","title":"Agenda - App Home Filesystem Surface and Semantics","created_at":"2026-03-27","updated_at":"2026-03-27","tags":[],"agendas":[{"id":"AGD-0006","file":"AGD-0006-app-home-filesystem-surface-and-semantics.md","status":"open","created_at":"2026-03-27","updated_at":"2026-03-27"}],"decisions":[],"plans":[],"lessons":[]} {"type":"discussion","id":"DSC-0008","status":"done","ticket":"perf-runtime-telemetry-hot-path","title":"Agenda - [PERF] Runtime Telemetry Hot Path","created_at":"2026-03-27","updated_at":"2026-06-04","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0026","file":"discussion/lessons/DSC-0008-perf-runtime-telemetry-hot-path/LSN-0026-push-based-telemetry-model.md","status":"done","created_at":"2026-04-10","updated_at":"2026-04-10"}]} -{"type":"discussion","id":"DSC-0009","status":"in_progress","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-06-28","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[{"id":"AGD-0008","file":"AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md","status":"accepted","created_at":"2026-03-27","updated_at":"2026-06-28"}],"decisions":[{"id":"DEC-0034","file":"DEC-0034-async-work-lane-and-asset-backlog-contract.md","status":"accepted","created_at":"2026-06-28","updated_at":"2026-06-28","ref_agenda":"AGD-0008"}],"plans":[{"id":"PLN-0123","file":"PLN-0123-async-work-lane-specification-propagation.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0124","file":"PLN-0124-async-work-lane-runtime-infrastructure.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0125","file":"PLN-0125-asset-backlog-and-stable-slot-handles.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0126","file":"PLN-0126-asset-backlog-public-api-and-status-surface.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0127","file":"PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]},{"id":"PLN-0128","file":"PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md","status":"done","created_at":"2026-06-28","updated_at":"2026-06-28","ref_decisions":["DEC-0034"]}],"lessons":[]} +{"type":"discussion","id":"DSC-0009","status":"done","ticket":"perf-async-background-work-lanes-for-assets-and-fs","title":"Agenda - [PERF] Async Background Work Lanes for Assets and FS","created_at":"2026-03-27","updated_at":"2026-07-01","tags":["perf","asset","fs","async","scheduler","runtime"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0050","file":"discussion/lessons/DSC-0009-perf-async-background-work-lanes-for-assets-and-fs/LSN-0050-serial-async-lanes-bound-backlog-complexity.md","status":"done","created_at":"2026-07-01","updated_at":"2026-07-01"}]} {"type":"discussion","id":"DSC-0010","status":"done","ticket":"perf-host-desktop-frame-pacing-and-presentation","title":"Agenda - [PERF] Host Desktop Frame Pacing and Presentation","created_at":"2026-03-27","updated_at":"2026-04-20","tags":[],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0036","file":"discussion/lessons/DSC-0010-perf-host-desktop-frame-pacing-and-presentation/LSN-0036-frame-publication-and-host-invalidation-must-be-separate.md","status":"done","created_at":"2026-04-20","updated_at":"2026-04-20"}]} {"type":"discussion","id":"DSC-0011","status":"abandoned","ticket":"perf-gfx-render-pipeline-and-dirty-regions","title":"Agenda - [PERF] GFX Render Pipeline and Dirty Regions","created_at":"2026-03-27","updated_at":"2026-06-28","tags":[],"agendas":[{"id":"AGD-0010","file":"AGD-0010-perf-gfx-render-pipeline-and-dirty-regions.md","status":"abandoned","created_at":"2026-03-27","updated_at":"2026-06-28","_override_reason":"User explicitly chose to remove this agenda because it no longer applies to the current render scenario; future optimizations can be discussed when needed."}],"decisions":[],"plans":[],"lessons":[],"_override_reason":"User explicitly chose to abandon this discussion because it no longer applies to the current render scenario; future optimizations can be discussed when needed."} {"type":"discussion","id":"DSC-0012","status":"done","ticket":"perf-runtime-introspection-syscalls","title":"Agenda - [PERF] Runtime Introspection Syscalls","created_at":"2026-03-27","updated_at":"2026-04-19","tags":["perf","runtime","syscall","telemetry","debug","asset"],"agendas":[],"decisions":[],"plans":[],"lessons":[{"id":"LSN-0034","file":"discussion/lessons/DSC-0012-perf-runtime-introspection-syscalls/LSN-0034-host-owned-debug-boundaries.md","status":"done","created_at":"2026-04-19","updated_at":"2026-04-19"}]} diff --git a/discussion/lessons/DSC-0009-perf-async-background-work-lanes-for-assets-and-fs/LSN-0050-serial-async-lanes-bound-backlog-complexity.md b/discussion/lessons/DSC-0009-perf-async-background-work-lanes-for-assets-and-fs/LSN-0050-serial-async-lanes-bound-backlog-complexity.md new file mode 100644 index 00000000..71df825b --- /dev/null +++ b/discussion/lessons/DSC-0009-perf-async-background-work-lanes-for-assets-and-fs/LSN-0050-serial-async-lanes-bound-backlog-complexity.md @@ -0,0 +1,100 @@ +--- +id: LSN-0050 +ticket: perf-async-background-work-lanes-for-assets-and-fs +title: Serial Async Lanes Bound Backlog Complexity +created: 2026-07-01 +tags: [runtime, asset, async, scheduler, telemetry] +--- + +## Context + +The async asset work moved from request-owned worker creation toward a +runtime-owned serial lane. The important shift is not just "use a worker +thread"; it is the introduction of a third logical execution lane with explicit +ownership, priority, cancellation, progress, and telemetry. + +This work followed `DEC-0034` and plans `PLN-0123` through `PLN-0128`. The +published model separates: + +- main runtime execution, where guest-visible state is committed; +- render worker execution, where closed render packets are consumed; +- async IO/decode/persistence work, where one background job is active at a + time. + +## Key Decisions + +### Async Work Lane and Asset Backlog Contract + +**What:** Asset IO/decode and compatible persistence work use a runtime-owned +serial async lane. Asset requests are queued by target bank slot, not by +unbounded transient request identity. + +**Why:** A serial lane keeps the runtime observable and portable. It preserves +the intended hardware mental model while avoiding a desktop-biased thread pool +or one OS thread per asset request. + +**Trade-offs:** The lane intentionally sacrifices parallel decode throughput in +exchange for bounded state, deterministic ownership, simpler telemetry, and +clear priority rules. If future implementations add more physical parallelism, +they still need to preserve the logical serial contract where the spec requires +it. + +## Patterns and Algorithms + +The asset backlog is bounded by target identity. A request targets a concrete +bank type and slot. A newer request for the same target supersedes the older +pending request, and an older active result is discarded when its generation is +no longer current. + +Stable handles observe slots. A handle should not be treated as a worker job or +thread token. The durable identity is the bank slot plus request generation: +slot state says what is resident; request state says what is queued, active, +ready, canceled, superseded, or failed. + +Commit remains a main-lane operation. The background lane may read and decode, +but publication into resident runtime state happens at predictable ownership +points. This keeps VM execution, render handoff, and frame observation from +racing with background mutation. + +Progress and telemetry are closure-oriented. Use integer progress and update +expensive aggregate telemetry when jobs close, not inside decode loops. Tests +should synchronize on explicit state transitions rather than sleeps. + +Priority is part of the lane contract. Memcard commit/write work can outrank +ordinary asset loads, FS write/config work can be represented internally, and +non-critical read/list work belongs below asset loads. Public FS semantics +remain owned by the filesystem discussion; sharing the lane is not permission +to decide the FS API here. + +## Pitfalls + +Do not hide concurrency behind per-request `thread::spawn`. That appears simple +locally but loses boundedness, priority, cancellation, and telemetry. + +Do not let a handle mean "the current background job." Handles must survive +empty slots, completed work, cancellation, superseding, errors, and already +resident fast paths. + +Do not install background results directly from the worker. Decode completion is +not the same as publication. Use request generation checks before committing a +result. + +Do not turn operational states into traps. Queued, active, canceled, +superseded, error, and backend-unavailable outcomes belong in status-first +surfaces unless the caller violated the structural ABI. + +Do not reopen FS public API scope while wiring internal async-lane consumers. +The lane can support FS-style work without deciding request/poll semantics. + +## Takeaways + +- A runtime async lane is an ownership boundary, not just an implementation + thread. +- Backlog keying by target slot bounds complexity better than queue length + limits exposed to the guest. +- Stable slot handles plus request generations prevent stale async results from + mutating newer state. +- Main-lane commit keeps background IO/decode compatible with deterministic + runtime publication. +- Telemetry belongs at state transitions and job closure, not inside hot decode + loops. diff --git a/discussion/workflow/agendas/AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md b/discussion/workflow/agendas/AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md deleted file mode 100644 index a7dc7a2e..00000000 --- a/discussion/workflow/agendas/AGD-0008-perf-async-background-work-lanes-for-assets-and-fs.md +++ /dev/null @@ -1,557 +0,0 @@ ---- -id: AGD-0008 -ticket: perf-async-background-work-lanes-for-assets-and-fs -title: Agenda - [PERF] Async Background Work Lanes for Assets and FS -status: accepted -created: 2026-03-27 -resolved: 2026-06-28 -decision: DEC-0034 -tags: [perf, asset, fs, async, scheduler, runtime] ---- - -# Agenda - [PERF] Async Background Work Lanes for Assets and FS - -## Contexto - -Depois das decisoes de render worker, a concorrencia de render saiu do escopo -desta agenda. O problema restante e a lane de trabalho assincrono para IO, -decode e persistencia. - -O modelo de execucao que queremos discutir agora assume uma terceira core/lane -dedicada a trabalhos assincronos de runtime: - -```text -Core/lane 1: VM, firmware, SystemOS e frame logico -Core/lane 2: render worker / rasterizacao assincrona -Core/lane 3: async work lane para asset IO/decode e possivelmente FS -``` - -Essa terceira lane nao deve virar um pool grande e solto. A intencao e ter uma -fila previsivel, com capacidade limitada, telemetria e regras de saturacao. - -Para assets, a necessidade e concreta: `asset.load()` ainda cria uma thread do -SO por requisicao. Para FS, a motivacao provavel era evitar que operacoes de -leitura/escrita/listagem/delete, hoje chamadas de forma sincrona pelo dispatch, -roubem tempo do frame quando o backend real envolver IO. Precisamos verificar -se FS deve usar a mesma lane agora ou apenas reservar compatibilidade. - -## Problema - -`asset.load()` hoje dispara `thread::spawn` por requisicao de asset nao -residente. Isso escala mal, cria jitter e deixa o runtime sem uma politica -unica de backpressure. - -O dominio `fs` tambem tem operacoes potencialmente bloqueantes (`read`, -`write`, `list_dir`, `delete`, memcard commit/read/write) e a tabela de -syscalls ja as marca como nao deterministicas/custosas. Ainda assim, a -implementacao atual executa essas chamadas diretamente no caminho de dispatch. - -Sem um contrato explicito, cada dominio pode inventar sua propria concorrencia: -asset criando threads, FS bloqueando o frame, e memcard herdando comportamento -acidental do backend. - -## Pontos Criticos - -- `thread::spawn` por request escala mal e cria jitter. -- A terceira core/lane deve ser um recurso de plataforma controlado, nao uma - permissao geral para criar threads. -- Asset loading precisa de um pool/fila limitada para multiplos loads - pendentes. -- FS pode justificar a mesma lane quando envolver IO real, mas nao devemos - assinar uma API assincrona ampla sem entender a semantica de retorno. -- Commit/install em bancos residentes pode ter restricoes de determinismo e - ownership; decode pode ser assincrono, mas publicacao/instalacao talvez tenha - que cruzar de volta para a lane principal. -- A estrategia precisa funcionar em hardware alvo com poucos cores e tambem no - host desktop sem transformar desktop em referencia de custo errada. -- Render worker ja tem seu proprio contrato e nao deve compartilhar essa lane. - -## Hotspots Atuais - -- [asset.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-drivers/src/asset.rs#L438) - `AssetManager::load`. -- [asset.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-drivers/src/asset.rs#L507) - `thread::spawn` por load. -- [fs.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-hal/src/syscalls/domains/fs.rs#L4) - `fs` e `mem` marcados como superficies custosas/nao deterministicas. -- [dispatch.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-system/src/services/vm_runtime/dispatch.rs#L549) - chamadas `fs.*` executam no dispatch. -- [virtual_fs.rs](/Users/niltonconstantino/personal/workspace.personal/intrepid/prometeu/runtime/crates/console/prometeu-system/src/services/fs/virtual_fs.rs#L69) - VFS normaliza path e chama backend diretamente. - -## Alvo da Discussao - -Fechar um modelo de terceira core/lane para trabalho assincrono de runtime, -comecando por asset loading e avaliando se FS deve entrar na mesma lane no v1. - -O alvo nao e criar multitarefa geral, callbacks para guest, ou um scheduler -paralelo da VM. A lane deve executar trabalho de IO/decode/sync e publicar -resultados observaveis por status/polling em pontos previsiveis. - -## O Que Precisa Ser Definido - -1. Topologia da terceira lane. - Definir: - - uma async work lane dedicada, separada da VM e do render worker; - - se a lane e uma thread fixa no host e uma core dedicada no hardware alvo; - - fallback quando nao houver core fisico dedicado; - - proibicao explicita de `thread::spawn` por request no caminho normal. - -2. Pool/fila de asset loading. - Definir: - - backlog de pedidos sem limite operacional fixo; - - exatamente um job ativo por vez na lane; - - o "pool" significa backlog e estados de job, nao multiplos workers reais; - - politica de substituicao/cancelamento para pedidos que miram o mesmo - bank/slot; - - status para pending/backlog, ativo, progresso, cancelamento, erro de - decode e sucesso. - -3. Fases do asset load. - Separar: - - request e alocacao de handle; - - leitura de bytes; - - decode/materializacao; - - staging de resultado; - - install/commit em banco residente. - - Ponto a fechar: quais fases rodam na terceira lane e quais voltam para a - lane principal por determinismo/ownership. - -4. Inclusao de FS na mesma lane. - Verificar se `fs` deve usar a terceira lane no v1 para: - - leituras grandes; - - escritas; - - listagem; - - delete; - - memcard commit/read/write. - - Ponto a fechar: FS sync publico continua bloqueante/status-first, vira - request/poll, ou ganha apenas uma implementacao interna que drena trabalho em - pontos de frame. - -5. Separacao por dominio. - Decidir se `asset` e `fs` compartilham uma fila com classes de prioridade ou - se a terceira lane contem subfilas dedicadas. - -6. Politica de prioridade. - Definir: - - prioridade de loads visuais vs audio; - - prioridade de save/config; - - prioridade de memcard commit; - - starvation prevention; - - limite de trabalho de instalacao/publicacao por frame. - -7. Modelo de retorno e telemetria. - Fechar como guest/runtime observa: - - `pending`; - - `loading`; - - `ready`; - - `canceled`; - - `backend_unavailable`; - - backlog; - - posicao de um asset no backlog; - - progresso percentual do asset ativo; - - drops/retries quando aplicavel. - -8. Orcamento para hardware barato. - Definir o baseline: - - VM principal; - - render worker; - - terceira async work lane; - - limites de memoria para jobs pendentes e buffers temporarios. - -## Opcoes - -### Opcao A - Terceira lane unica com subfilas e pool limitado de assets - -- **Abordagem:** criar uma async work lane dedicada. Assets usam um pool/fila - de jobs nessa lane; FS tambem pode submeter jobs na mesma lane, com - prioridade menor ou classe separada. -- **Pro:** modelo simples de budget; evita explosao de threads; encaixa no alvo - de terceira core; permite compartilhar backpressure e telemetria. -- **Contra:** assets e FS podem competir por IO; precisa politica clara para - evitar que save/config fique atras de loads longos. -- **Manutenibilidade:** boa se a lane tiver tipos de job e estados explicitos. - -### Opcao B - Lane de assets agora, FS apenas reservado - -- **Abordagem:** fechar v1 somente para asset loading, removendo `spawn` por - request. FS continua sincrono por enquanto, mas o contrato da lane reserva - compatibilidade para FS no futuro. -- **Pro:** menor escopo; resolve a dor concreta de assets rapidamente; evita - alterar semantica de FS antes da agenda de app home filesystem. -- **Contra:** nao resolve stalls reais de FS/memcard; pode exigir refatorar a - lane depois para aceitar FS. -- **Manutenibilidade:** boa no curto prazo, com risco de segunda migracao. - -### Opcao C - Lanes separadas para asset e FS - -- **Abordagem:** criar uma lane/pool para assets e outra para FS. -- **Pro:** isolamento forte; FS/save nao compete diretamente com loads de - textura/audio/cena. -- **Contra:** consome mais threads/cores; contraria a premissa de terceira core - unica; pior para hardware simples. -- **Manutenibilidade:** media; isolamento ajuda, mas a topologia fica pesada. - -### Opcao D - Pool global pequeno sem nocao de terceira lane - -- **Abordagem:** trocar `spawn` por request por um pool global fixo. -- **Pro:** simples de implementar em host desktop. -- **Contra:** perde a arquitetura de lanes por core; pode mascarar custo em - desktop; fica menos claro como mapear para hardware alvo. -- **Manutenibilidade:** baixa para o modelo de console se virar dependencia - acidental de scheduler do SO. - -## Sugestao / Recomendacao - -Recomendo seguir com a Opcao A como direcao de arquitetura, mas fechar a -decisao em duas camadas: - -1. Contrato obrigatorio agora: - - terceira async work lane como recurso runtime; - - assets deixam de usar `thread::spawn` por requisicao; - - asset loading passa por backlog consultavel/reordenavel; - - status, progresso e telemetria de backlog ficam explicitos. - -2. Contrato condicionado para FS: - - FS deve ser analisado como consumidor legitimo da mesma lane porque suas - operacoes podem envolver IO real e ja sao custosas/nao deterministicas; - - a decisao precisa escolher entre incluir FS no v1 ou reservar a lane para - FS sem mudar ainda a superficie publica; - - memcard precisa ser considerado junto com FS, porque usa a mesma - capacidade e backend de persistencia. - -O ponto importante: a terceira core/lane deve ser a unidade de concorrencia. -O pool de assets deve existir dentro desse modelo, nao como criacao livre de -threads por load. - -## Respostas Consolidadas Ate Aqui - -1. O pool de assets deve ser uma lane serial. - - Ha exatamente um job ativo na async work lane. - - O pool representa backlog, estados e controle de ordem, nao multiplos - workers executando em paralelo. - -2. O backlog de pedidos deve ser semanticamente ilimitado. - - Nao existe `queue_full` como status normal do dominio. - - Como cada pedido mira um `bank_type/slot` e pedidos novos para o mesmo - alvo substituem os anteriores, o backlog efetivo fica limitado pela soma - dos slots dos bancos. - - O contrato nao precisa expor limite de fila, mas a implementacao deve - garantir que metadados de handles cancelados/superseded nao crescam sem - coleta. - -3. Asset loading precisa expor consulta e reordenacao do backlog. - - Deve existir API publica/runtime para consultar onde um asset esta no - backlog. - - Deve existir capacidade de adiantar ou atrasar um pedido de asset na fila. - - O asset ativo deve expor progresso percentual. - - Requests novos para o mesmo `bank_type/slot` devem cancelar pedidos - anteriores pendentes para aquele alvo, evitando carregar asset que ja foi - substituido antes de executar. - - Se o alvo ja estiver carregado com o mesmo `asset_id`, o novo request nao - entra no backlog; ele retorna imediatamente pronto com handle disponivel. - - Se um novo request substituir o alvo atualmente ativo, a lane deve cancelar - em ponto cooperativo quando possivel; se o trabalho nao puder ser - interrompido, o resultado antigo deve ser descartado ao final por geracao - superseded. - - `superseded` significa que um request foi substituido por outro request - mais novo para o mesmo `bank_type/slot`. Nao e erro e nao e cancelamento - manual; e obsolescencia operacional do pedido antigo. - - Handles representam o slot do bank como alvo estavel. Dentro do estado do - handle deve existir uma secao de request/backlog para aquele slot, - informando se ha pedido pendente/ativo, qual asset foi solicitado, - progresso, posicao e geracao. - - Handles devem ser capazes de consultar o estado do alvo `bank_type/slot` - a qualquer momento. Se nao houver asset valido carregado e tambem nao - houver request pendente para aquele alvo, a consulta deve retornar estado - explicito de vazio/invalido, nao falhar por ausencia de request ativo. - -4. Decode/materializacao fria rodam na terceira lane. - - Leitura e decode de asset podem acontecer fora da lane principal. - - O resultado deve voltar como material pronto/staged. - -5. Install/commit em bancos residentes continua na lane principal. - - A async lane prepara trabalho. - - A lane principal publica/instala em pontos previsiveis por ownership, - determinismo e integracao com bancos/render/audio. - -6. FS e memcard entram como consumidores possiveis da mesma lane, mas precisam - ser melhor amarrados com a agenda de app home filesystem. - - A `DSC-0007` / `AGD-0006` deve ser ajustada para estabelecer melhor a - superficie publica e o modelo async/sync de FS. - - Esta agenda deve afirmar a existencia da lane para trabalhos async de IO - desse tipo e permitir que FS a consuma. - - Questoes de API publica de FS, request/poll e semantica de app home devem - ficar a cargo da agenda 6. - - FS, memcard e assets nao devem operar sobrepostos quando isso gerar - concorrencia real de backend; se houver competicao, a fila unica resolve a - ordem. - -7. Prioridade quando houver competicao: - - memcard commit/write; - - FS write/config; - - asset visual/audio load; - - list/read nao critico. - -8. Status-first continua valendo. - - `backend_unavailable`, `canceled`, erro de decode e estados de job sao - status operacionais. - - Cancelamento por superseding de pedido no mesmo bank/slot e resultado - operacional normal, nao fault. - - Nao ha `queue_full` no contrato esperado. - - Trap permanece para violacao estrutural, como argumento invalido, - capability ausente ou handle estruturalmente invalido. - -9. Telemetria minima: - - backlog atual; - - posicao por asset; - - progresso percentual do asset ativo; - - jobs submetidos, concluidos, falhos e cancelados; - - tempo por job; - - percentis por asset enquanto estiver carregando na lane. - -10. Escopo da agenda. - - O foco desta agenda e a lane, assets e memcard/game persistence. - - FS entra como consumidor permitido da lane de IO, mas as decisoes de API - publica de FS pertencem a `AGD-0006`. - -11. API de backlog. - - A API inicial sugerida cobre os casos esperados. - - `asset.backlog_promote(handle)` deve ser mantido como atalho oficial para - `asset.backlog_move(handle, 1)`. - - `asset.backlog_demote(handle)` deve ser mantido como atalho oficial para - mover o pedido para o fim dos pendentes. - - Os atalhos nao criam semantica nova; apenas evitam que o caller calcule - posicoes comuns. - - O handle deve permitir consulta estavel do slot do bank mesmo quando nao ha - asset valido carregado. - - O estado do handle deve separar: - - estado do slot/residencia atual; - - estado do request/backlog atual para aquele slot. - - Operacoes que mudam estado, como commit/cancel/reorder, atuam sobre a - secao de request/backlog do handle e precisam validar geracao/request atual - para nao agir sobre um pedido substituido. - -12. Cancelamento por substituicao. - - Quando um request novo chega para o mesmo `bank_type/slot`, o request - anterior e marcado imediatamente como `superseded`. - - Se o request anterior ainda estiver no backlog, ele sai da fila. - - Se o request anterior estiver ativo, a lane tenta cancelar - cooperativamente quando a fase atual permitir. - - Se a fase ativa nao puder ser interrompida com baixo custo, ela termina e - o resultado e descartado ao final porque a geracao do alvo ja mudou. - - Como o handle pertence ao slot do bank, ele continua consultavel; sua - secao de request deve indicar que a geracao anterior foi `superseded` ou - que nao ha request ativo para aquele alvo. - -13. Fases de progresso. - - `read/decode/stage` e suficiente como modelo minimo para `GLYPH`, `SOUND` - e `SCENE`. - - Subfases futuras podem detalhar progresso dentro de `decode` sem mudar o - contrato geral. - -14. Percentis. - - Percentis devem existir por `bank_type` como telemetria principal. - - Percentis por `asset_id` tambem sao uteis, mas podem usar janela pequena - ou agregador leve. - - Percentis sao atualizados no fechamento do job, nao durante o hot path. - -15. Ajuste esperado na agenda de FS. - - `AGD-0006` deve registrar que existe uma async IO lane compartilhavel por - FS/memcard/assets. - - Esta agenda so decide que FS pode consumir a lane. - - A API publica de FS, incluindo request/poll ou sync aparente, pertence a - `AGD-0006`. - -## API Inicial Sugerida Para Assets - -Esta agenda ainda nao fecha nomes finais de ABI, mas a direcao inicial sugerida -e manter `asset.load/status/commit/cancel` e adicionar uma superficie pequena -para backlog: - -1. `asset.backlog_info() -> (status, pending_count, active_handle, active_asset_id, active_bank_type, active_slot, active_progress)` - - Consulta o estado geral da lane de assets. - - `active_progress` usa escala inteira, preferencialmente `0..10000` para - permitir duas casas decimais sem float. - -2. `asset.backlog_position(handle) -> (status, state, position, progress)` - - Consulta um pedido especifico. - - `position = 0` significa job ativo. - - `position > 0` significa posicao no backlog. - - `position = -1` pode representar nao enfileirado/pronto/cancelado, - dependendo do `state`. - -3. `asset.backlog_move(handle, new_position) -> status` - - Move um pedido pendente para uma posicao absoluta. - - `new_position = 1` significa primeiro pendente depois do ativo. - - Nao interrompe diretamente o job ativo; substituicao do mesmo - `bank_type/slot` usa a regra de superseding. - -4. `asset.backlog_promote(handle) -> status` - - Atalho para mover o pedido para a frente do backlog pendente. - -5. `asset.backlog_demote(handle) -> status` - - Atalho para mover o pedido para o fim do backlog pendente. - -6. `asset.target_status(bank_type, slot) -> (status, asset_id, handle, state, position, progress)` - - Consulta por alvo operacional, que e o que realmente controla superseding. - - Ajuda quando o caller quer saber "o que esta programado para este slot". - -Estados sugeridos para `state`: - -- `empty`; -- `invalid`; -- `queued`; -- `active`; -- `ready`; -- `committed`; -- `canceled`; -- `superseded`; -- `error`; -- `backend_unavailable`. - -Regra de design: APIs de backlog sao operacionais e status-first. Reordenar um -handle inexistente, ja concluido ou ja superseded retorna status operacional -apropriado quando o handle e conhecido; violações estruturais continuam Trap. - -Handles sao observaveis e representam slots de banks. Um handle conhecido pode -ser consultado mesmo quando o slot alvo nao contem asset valido carregado; nesse -caso a consulta deve retornar estado como `empty` ou `invalid`. - -O estado do handle deve conter pelo menos duas partes: - -```text -slot_state: - loaded_asset_id - resident_state - -request_state: - requested_asset_id - generation - queued/active/ready/canceled/superseded/error - backlog_position - progress -``` - -Mutacoes seguem regras mais estritas e devem falhar operacionalmente quando a -secao de request/backlog do handle nao representa a geracao atual. - -## Progresso e Percentis - -### Progresso percentual - -Sugestao: - -- usar progresso em base inteira `0..10000`; -- quando o tamanho total de bytes for conhecido, calcular progresso por fases - ponderadas; -- quando o decode nao for linear ou nao houver tamanho confiavel, expor - progresso por fase com marcos conservadores. - -Modelo inicial de fases: - -```text -queued -> 0 -read -> 0..4000 -decode -> 4000..9000 -stage -> 9000..10000 -ready -> 10000 -``` - -Se uma fase nao consegue reportar progresso interno, ela deve manter o ultimo -marco e saltar para o proximo marco ao concluir. Isso evita progresso falso -sem perder visibilidade de fase. - -### Percentis por asset - -Sugestao: - -- registrar duracoes por asset no fim de cada job, nao por tick/frame; -- manter agregadores pequenos por `asset_id` e por `bank_type`; -- calcular percentis a partir de janela/ring buffer pequeno ou histograma - barato por bucket de duracao; -- nunca atualizar percentis no loop quente de decode por item/pixel; -- o job ativo pode expor tempo decorrido e fase atual; percentis sao atualizados - apenas no fechamento do job. - -## Perguntas em Aberto - -- [x] A agenda esta madura para virar decisao normativa ou ainda falta algum - caso de asset/memcard/lane? -- [x] A estrutura minima de `slot_state` e `request_state` no handle esta - completa ou falta algum campo para commit/cancel/reorder seguro? - -## Resolucao - -A agenda esta madura para decisao normativa. - -A decisao deve formalizar: - -- a terceira async work lane serial; -- um job ativo por vez; -- backlog efetivamente limitado pela unicidade de `bank_type/slot`; -- handles como observadores estaveis de slots de bank; -- `slot_state` e `request_state` dentro do handle; -- cancelamento/superseding por novo request para o mesmo alvo; -- load pronto imediato quando o asset ja esta residente no alvo; -- decode/materializacao na lane async; -- commit/install na lane principal; -- memcard como consumidor prioritario da mesma lane; -- FS como consumidor permitido, deixando API publica de FS para `AGD-0006`; -- progresso por fases e percentis atualizados no fechamento do job. - -## Dependencias - -- `docs/specs/runtime/09-events-and-concurrency.md` -- `docs/specs/runtime/15-asset-management.md` -- `docs/specs/runtime/16a-syscall-policies.md` -- `discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md` - -## Criterio para Encerrar - -Pode virar PR quando houver decisao escrita sobre: - -- existencia e papel da terceira async work lane; -- relacao dessa lane com a terceira core no hardware alvo e fallback no host; -- desenho do backlog serial de asset loading; -- regra de cancelamento/substituicao para requests no mesmo bank/slot; -- proibicao de `thread::spawn` por load no caminho normal; -- fases de asset load que rodam na lane vs lane principal; -- inclusao condicionada de FS/memcard nessa mesma lane; -- API de consulta e reordenacao do backlog de assets; -- comportamento de request imediato quando o asset ja esta residente no alvo; -- regra de bounded backlog por unicidade de `bank_type/slot`; -- politica de prioridade entre asset, FS e memcard; -- status e telemetria para backlog, progresso, percentis, cancelamento e falha. - -## Discussao - -- 2026-06-28: Agenda atualizada para remover render worker do escopo, assumir - uma terceira core/lane para trabalhos assincronos, exigir pool/fila limitada - para asset loading e reavaliar o papel de FS/memcard como consumidores da - mesma lane. -- 2026-06-28: Discussao consolidou que a async work lane deve ser serial, com - um job ativo, backlog semanticamente ilimitado, consulta/reordenacao publica - de backlog de assets, progresso percentual do asset ativo, decode na lane - async, install/commit na lane principal, e FS/memcard como consumidores a - alinhar com a agenda de app home filesystem. -- 2026-06-28: Acrescentada regra de asset pipe: requests para o mesmo bank/slot - devem cancelar pedidos anteriores pendentes para aquele alvo, tratando o - pedido mais novo como substituto operacional. -- 2026-06-28: Discussao refinou que o alvo canonico e `bank_type/slot`; um - asset ja residente deve retornar handle pronto sem entrar no backlog; o - backlog efetivo e limitado pela soma dos slots dos bancos; FS fica como - consumidor permitido da lane, enquanto API publica de FS permanece na agenda - 6; progresso deve usar fases conservadoras e percentis devem ser atualizados - no fechamento do job, nao no hot path. -- 2026-06-28: Open questions restantes foram consolidadas: API de backlog - aceita com `promote/demote` como atalhos; superseding marca requests - substituidos por outro request para o mesmo `bank_type/slot`; cancelamento do - ativo e cooperativo quando barato e descarte por geracao quando nao for; - `read/decode/stage` cobre GLYPH/SOUND/SCENE; percentis ficam por `bank_type` - e por `asset_id`; API publica de FS permanece em `AGD-0006`. -- 2026-06-28: Adicionada a expectativa de que handles possam consultar o alvo - `bank_type/slot` a qualquer momento, inclusive quando nao houver asset valido - carregado; mutacoes devem continuar protegidas por request/geracao para evitar - commit/cancel acidental de pedidos substituidos. -- 2026-06-28: Refinado o modelo de handle: o handle pertence ao slot do bank e - contem uma secao de estado de request/backlog para aquele slot. Consulta e - residencia ficam estaveis por alvo; mutacoes atuam sobre a geracao atual do - request dentro do handle. diff --git a/discussion/workflow/decisions/DEC-0034-async-work-lane-and-asset-backlog-contract.md b/discussion/workflow/decisions/DEC-0034-async-work-lane-and-asset-backlog-contract.md deleted file mode 100644 index 2095c005..00000000 --- a/discussion/workflow/decisions/DEC-0034-async-work-lane-and-asset-backlog-contract.md +++ /dev/null @@ -1,336 +0,0 @@ ---- -id: DEC-0034 -ticket: perf-async-background-work-lanes-for-assets-and-fs -title: Async Work Lane and Asset Backlog Contract -status: accepted -created: 2026-06-28 -accepted: 2026-06-28 -ref_agenda: AGD-0008 -plans: [PLN-0123, PLN-0124, PLN-0125, PLN-0126, PLN-0127, PLN-0128] -tags: [perf, asset, fs, async, scheduler, runtime] ---- - -# DEC-0034 - Async Work Lane and Asset Backlog Contract - -## Status - -Accepted. - -## Contexto - -`asset.load()` currently creates an OS thread per non-resident asset request. -That behavior does not match the target runtime model: the machine should have -explicit execution lanes, bounded ownership, observable backpressure, and no -uncontrolled thread creation from guest-visible operations. - -Render worker concurrency is already handled by separate render decisions. This -decision covers the third runtime lane for asynchronous IO/decode/persistence -work: - -```text -Lane 1: VM, firmware, SystemOS, logical frame -Lane 2: render worker / asynchronous rasterization -Lane 3: async work lane for asset IO/decode and game persistence work -``` - -The concrete first consumer is asset loading. Game persistence through memcard -may also consume this lane. FS is allowed to consume this lane, but public FS API -shape belongs to the app-home filesystem agenda. - -## Decisao - -Prometeu MUST introduce a third async work lane as the runtime-owned execution -place for asset IO/decode work and compatible persistence IO work. - -The async work lane MUST be serial: it has exactly one active job at a time. -The runtime MUST NOT create one OS thread per `asset.load` request in the normal -path. - -Asset loading MUST use a backlog keyed by target `bank_type/slot`. A target can -have at most one current request. A newer request for the same `bank_type/slot` -MUST supersede the previous request for that target. - -Asset handles MUST represent stable bank slots, not transient worker threads. -A handle MUST remain queryable even when the slot has no valid loaded asset and -even when there is no active request for that slot. The handle state MUST -separate resident slot state from request/backlog state. - -Asset IO/read and decode/materialization MUST run on the async work lane. -Asset install/commit into resident banks MUST happen on the main runtime lane at -predictable ownership points. - -Memcard work MAY consume the same async work lane and has higher priority than -ordinary asset loads when contention exists. FS MAY consume the same lane for -IO-style work, but this decision does not define the public FS API. - -## Rationale - -A serial lane maps cleanly to the intended third-core mental model and avoids a -desktop-biased generic thread pool. It gives the runtime one place to observe -IO/decode backlog, progress, priority, cancellation, and telemetry. - -Keying requests by `bank_type/slot` makes the backlog naturally bounded by the -sum of bank slots. There is no need for a guest-visible `queue_full` state: -there cannot be unbounded distinct pending targets if each target has at most -one current request. - -Keeping commit/install on the main lane preserves deterministic publication -semantics and avoids letting the async lane mutate resident graphics/audio/scene -state while the VM, render handoff, or frame boundary is observing it. - -Separating `slot_state` and `request_state` inside the handle avoids ambiguity: -callers can inspect the slot at any time, while mutating operations still remain -protected by request generation. - -## Invariantes / Contrato - -### Async Work Lane - -- The async work lane MUST be runtime-owned. -- The async work lane MUST be separate from the render worker. -- The async work lane MUST execute at most one active job at a time. -- The runtime MUST NOT use `thread::spawn` per asset request in the normal - asset loading path. -- The host may implement the lane as a fixed worker thread. Hardware targets may - map it to a dedicated core when available. -- If a physical third core is unavailable, the implementation MUST preserve the - same logical lane contract. - -### Asset Backlog - -- Asset requests MUST target a concrete `bank_type/slot`. -- The backlog is ordered and serial. -- Each `bank_type/slot` MUST have at most one current request. -- A newer request for the same `bank_type/slot` MUST supersede any earlier - pending request for that target. -- If the target already contains the requested `asset_id` as a valid resident - asset, `asset.load` MUST return a handle in ready state without adding a job - to the backlog. -- The effective backlog size is bounded by the sum of targetable bank slots, - even though the contract does not expose a fixed queue limit. -- Implementations MUST collect stale metadata for canceled or superseded - requests so the handle/request history does not grow without bound. - -### Handles - -An asset handle MUST represent a stable bank slot target. The handle state MUST -include at least: - -```text -handle: - bank_type - slot - -slot_state: - loaded_asset_id - resident_state - slot_generation - -request_state: - requested_asset_id - request_generation - state - backlog_position - progress -``` - -The handle MUST be queryable when: - -- the slot is empty; -- the slot contains a valid resident asset; -- a request is queued; -- a request is active; -- a request is ready for commit; -- a request has been canceled; -- a request has been superseded; -- a request ended in error. - -Mutating operations such as commit, cancel, promote, demote, or move MUST act on -the current request generation. They MUST NOT accidentally mutate a newer -request through an older handle view. - -### Superseding - -`superseded` means a request was replaced by a newer request for the same -`bank_type/slot`. - -- Superseding is an operational state, not a fault. -- If the old request is queued, it MUST be removed from the backlog. -- If the old request is active, the lane SHOULD cancel cooperatively when the - current phase supports cheap cancellation. -- If active work cannot be interrupted cheaply, it MAY finish, but its result - MUST be discarded when its generation no longer matches the target's current - request generation. - -### Asset Backlog API Direction - -The existing asset operations remain the base surface: - -```text -asset.load(asset_id, slot) -> (status, handle) -asset.status(handle) -> status -asset.commit(handle) -> status -asset.cancel(handle) -> status -``` - -The asset backlog surface SHOULD add a small status-first API: - -```text -asset.backlog_info() - -> (status, pending_count, active_handle, active_asset_id, - active_bank_type, active_slot, active_progress) - -asset.backlog_position(handle) - -> (status, state, position, progress) - -asset.backlog_move(handle, new_position) - -> status - -asset.backlog_promote(handle) - -> status - -asset.backlog_demote(handle) - -> status - -asset.target_status(bank_type, slot) - -> (status, asset_id, handle, state, position, progress) -``` - -`asset.backlog_promote(handle)` is an official shortcut for moving a queued -request to position `1`, the first pending position after the active job. - -`asset.backlog_demote(handle)` is an official shortcut for moving a queued -request to the end of the pending backlog. - -The final ABI names and exact return shapes may be refined during planning, but -the implementation MUST preserve the capabilities above. - -### Progress - -Progress MUST be represented without floating point. The recommended scale is -`0..10000`. - -The initial phase model is: - -```text -queued -> 0 -read -> 0..4000 -decode -> 4000..9000 -stage -> 9000..10000 -ready -> 10000 -``` - -If a phase cannot report internal progress, it MUST keep the previous progress -mark and advance at phase completion. Implementations MUST NOT invent false -precision for non-linear decode phases. - -### Telemetry - -Telemetry MUST be cheap and must not add hot-path per-item decode cost. - -Minimum telemetry: - -- current backlog depth; -- target/request position; -- active job progress; -- jobs submitted; -- jobs completed; -- jobs failed; -- jobs canceled; -- jobs superseded; -- job duration; -- percentiles by `bank_type`; -- lightweight percentiles or small-window samples by `asset_id`. - -Percentiles MUST be updated when a job closes, not inside the inner decode loop. - -### Priority - -When async lane consumers contend, the initial priority order is: - -1. memcard commit/write; -2. FS write/config work; -3. asset visual/audio/scene load; -4. non-critical list/read work. - -This priority order is a lane arbitration rule. It does not define public FS -syscall semantics. - -### FS Boundary - -FS MAY consume the async work lane for IO-style work. - -This decision MUST NOT define the public FS API, app-home FS semantics, or -whether FS is exposed as request/poll or sync-appearing operations. Those -questions belong to `AGD-0006`. - -`AGD-0006` MUST be updated or interpreted with the existence of this async IO -lane in mind. - -## Impactos - -### Spec - -- `docs/specs/runtime/15-asset-management.md` must absorb the async lane, - backlog, handle, superseding, progress, and telemetry contract. -- `docs/specs/runtime/16-host-abi-and-syscalls.md` must absorb any final public - asset backlog syscall names and return shapes. -- `docs/specs/runtime/16a-syscall-policies.md` may need status catalog updates - for superseded/canceled/backend unavailable states if not already covered by - asset domain status. -- `docs/specs/runtime/09-events-and-concurrency.md` should mention the async - work lane as an implementation-side lane that does not introduce guest - callbacks. -- `AGD-0006` / future FS spec work must account for FS as a possible consumer of - the async IO lane. - -### Runtime - -- Asset loading must stop spawning a thread per request. -- AssetManager needs a serial backlog keyed by `bank_type/slot`. -- Asset handles need stable slot identity plus separated `slot_state` and - `request_state`. -- Commit/install remains a main-lane operation. -- Superseding and generation checks become required correctness mechanisms. - -### Host - -- Desktop host may implement the async work lane with one fixed worker thread. -- Host presentation/render worker remains separate and unaffected. - -### Firmware - -- No direct firmware behavior is required for asset backlog mechanics. -- Firmware flows that rely on preload or memcard may later use lane telemetry or - priority policy if needed. - -### Tooling - -- Test tools may need support for inspecting backlog state and forcing request - ordering. -- Asset fixtures should cover queued, active, ready, canceled, superseded, and - already-resident paths. - -## Referencias - -- `AGD-0008` - Async Background Work Lanes for Assets and FS. -- `docs/specs/runtime/15-asset-management.md` -- `docs/specs/runtime/16-host-abi-and-syscalls.md` -- `docs/specs/runtime/16a-syscall-policies.md` -- `docs/specs/runtime/09-events-and-concurrency.md` -- `docs/specs/runtime/08-save-memory-and-memcard.md` -- `AGD-0006` - App Home Filesystem Surface and Semantics. - -## Propagacao Necessaria - -1. Update asset management specs with the lane/backlog/handle contract. -2. Update syscall specs if backlog APIs become public ABI. -3. Update FS agenda/spec work to acknowledge the async IO lane without deciding - FS public API in this decision. -4. Plan implementation of a serial async work lane and asset backlog. -5. Plan tests for superseding, stable slot handles, immediate-ready resident - assets, progress, telemetry, and main-lane commit. - -## Revision Log - -- 2026-06-28: Initial draft from `AGD-0008`. diff --git a/discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md b/discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md deleted file mode 100644 index 9fa544cf..00000000 --- a/discussion/workflow/plans/PLN-0123-async-work-lane-specification-propagation.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -id: PLN-0123 -ticket: perf-async-background-work-lanes-for-assets-and-fs -title: Async Work Lane Specification Propagation -status: done -created: 2026-06-28 -completed: -ref_decisions: [DEC-0034] -tags: [perf, asset, fs, async, scheduler, runtime, spec] ---- - -# PLN-0123 - Async Work Lane Specification Propagation - -## Briefing - -`DEC-0034` establishes a third serial async work lane, asset backlog semantics, -stable bank-slot handles, superseding, progress, telemetry, and FS/memcard lane -boundaries. The canonical specs must be updated before code work relies on the -new contract. - -## Decisions de Origem - -- `DEC-0034` - Async Work Lane and Asset Backlog Contract. - -## Alvo - -Publish the accepted async work lane and asset backlog contract in the canonical -runtime specs. - -## Escopo - -Included: - -- Update asset management spec. -- Update syscall ABI spec if public backlog APIs are named there. -- Update syscall policy/status language for operational asset states. -- Update events/concurrency spec with the async work lane as an implementation - lane that does not introduce guest callbacks. -- Add a cross-reference from FS/app-home agenda or notes to the async IO lane. - -Fora de Escopo: - -- Implementing runtime code. -- Finalizing public FS API semantics. -- Changing render worker contracts. - -## Plano de Execucao - -### Step 1 - Update Asset Management - -**What:** Add the third async lane, backlog, handle, superseding, progress, and -telemetry contract. - -**How:** Extend `docs/specs/runtime/15-asset-management.md` around the existing -`asset.load/status/commit/cancel` section. Preserve the existing -`asset.load(asset_id, slot)` identity contract. - -**Files:** `docs/specs/runtime/15-asset-management.md`. - -### Step 2 - Update Host ABI - -**What:** Add the backlog API direction if the ABI spec is the canonical place -for syscall shapes. - -**How:** Extend the asset surface section with the planned backlog operations, -marking exact numeric ids as implementation-plan work if ids are not assigned -yet. - -**Files:** `docs/specs/runtime/16-host-abi-and-syscalls.md`. - -### Step 3 - Update Syscall Policies - -**What:** Clarify that `superseded`, `canceled`, `backend_unavailable`, and -decode failures are operational statuses, not faults. - -**How:** Add asset-domain examples under status-first policy. - -**Files:** `docs/specs/runtime/16a-syscall-policies.md`. - -### Step 4 - Update Events and Concurrency - -**What:** Document that the async work lane is an implementation lane. - -**How:** Add a section next to render worker concurrency stating that asset/IO -lane work does not introduce guest callbacks or hidden guest execution. - -**Files:** `docs/specs/runtime/09-events-and-concurrency.md`. - -### Step 5 - Cross-Reference FS Work - -**What:** Record that FS may consume the async IO lane, while public FS API -semantics remain owned by the app-home filesystem discussion. - -**How:** Add a note in `AGD-0006` or its future decision input, without deciding -FS request/poll shape in this plan. - -**Files:** `discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md`. - -## Criterios de Aceite - -- [ ] Specs state that the async work lane is serial and separate from render. -- [ ] Asset spec states backlog keying by `bank_type/slot`. -- [ ] Asset spec states stable slot handles with `slot_state` and `request_state`. -- [ ] Asset spec states superseding behavior. -- [ ] Asset spec states main-lane commit/install. -- [ ] FS is cross-referenced as a permitted lane consumer without public API decisions. - -## Tests / Validacao - -- Run `discussion validate`. -- Review spec links for broken relative paths. -- No cargo tests are required for this documentation-only plan. - -## Riscos - -- Public ABI names may be refined later; the spec must avoid assigning ids too - early if implementation work has not reserved them. -- FS scope can leak into this work; keep FS public API decisions in `AGD-0006`. diff --git a/discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md b/discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md deleted file mode 100644 index 041c40c3..00000000 --- a/discussion/workflow/plans/PLN-0124-async-work-lane-runtime-infrastructure.md +++ /dev/null @@ -1,119 +0,0 @@ ---- -id: PLN-0124 -ticket: perf-async-background-work-lanes-for-assets-and-fs -title: Async Work Lane Runtime Infrastructure -status: done -created: 2026-06-28 -completed: -ref_decisions: [DEC-0034] -tags: [perf, asset, fs, async, scheduler, runtime] ---- - -# PLN-0124 - Async Work Lane Runtime Infrastructure - -## Briefing - -`DEC-0034` requires one runtime-owned async work lane, separate from VM/main -execution and separate from render worker execution. This plan introduces the -lane infrastructure before migrating asset loading. - -## Decisions de Origem - -- `DEC-0034` - Async Work Lane and Asset Backlog Contract. - -## Alvo - -Create a serial async work lane primitive that can execute one active job at a -time, expose observable state, and support deterministic shutdown. - -## Escopo - -Included: - -- Runtime-owned async work lane type. -- Single active job execution. -- Ordered pending job storage. -- Cooperative cancellation hook. -- Shutdown and join behavior. -- Basic lane telemetry. - -Fora de Escopo: - -- Public asset backlog syscalls. -- Asset handle model migration. -- FS public API. -- Render worker changes. - -## Plano de Execucao - -### Step 1 - Locate Runtime Service Boundary - -**What:** Choose the crate/module boundary for the async lane. - -**How:** Place generic lane infrastructure near VM runtime services, not inside -desktop host code and not inside render worker modules. - -**Files:** likely `crates/console/prometeu-system/src/services/` or -`crates/console/prometeu-system/src/services/vm_runtime/`. - -### Step 2 - Define Job and Lane Types - -**What:** Add a lane type with pending queue, active job metadata, generation, -state, progress, and cancellation flag. - -**How:** Use typed job enums or traits that do not require guest callbacks. -Start with internal asset job support but keep the lane generic enough for -memcard/FS later. - -**Files:** new module under `prometeu-system` service layer. - -### Step 3 - Implement Serial Worker - -**What:** Implement one worker loop that takes one pending job and runs it to -completion or cooperative cancellation. - -**How:** Use a fixed worker thread on host builds. Preserve a logical-lane API -so future hardware can map it to a dedicated core. - -**Files:** async lane module; lifecycle wiring in VM runtime service setup. - -### Step 4 - Implement Shutdown - -**What:** Add explicit stop/shutdown behavior. - -**How:** Wake the lane, stop accepting new jobs, report pending jobs as canceled -or shutdown-discarded, and join the worker with bounded behavior. - -**Files:** async lane module and VM runtime lifecycle modules. - -### Step 5 - Expose Internal Telemetry - -**What:** Add counters for pending depth, active job, submitted, completed, -failed, canceled, and superseded. - -**How:** Use cheap atomics or lane-owned state snapshots. Do not update -expensive percentile data inside inner decode loops. - -**Files:** async lane module; telemetry structs where runtime telemetry lives. - -## Criterios de Aceite - -- [ ] Lane has one active job maximum. -- [ ] Lane is separate from render worker code. -- [ ] Lane can be started and stopped deterministically. -- [ ] Pending jobs are observable. -- [ ] Cooperative cancellation can be signaled. -- [ ] No guest callback path is introduced. - -## Tests / Validacao - -- Unit tests for enqueue, single active execution, ordering, cancellation signal, - and shutdown. -- Concurrency tests must synchronize on real state transitions, not sleeps. -- Run targeted `cargo test` for the owning crate. - -## Riscos - -- Accidentally creating a generic pool instead of a serial lane. -- Coupling the lane to desktop host thread details. -- Reusing render worker handoff semantics where ordered backlog is required. diff --git a/discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md b/discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md deleted file mode 100644 index df656308..00000000 --- a/discussion/workflow/plans/PLN-0125-asset-backlog-and-stable-slot-handles.md +++ /dev/null @@ -1,127 +0,0 @@ ---- -id: PLN-0125 -ticket: perf-async-background-work-lanes-for-assets-and-fs -title: Asset Backlog and Stable Slot Handles -status: done -created: 2026-06-28 -completed: -ref_decisions: [DEC-0034] -tags: [perf, asset, async, runtime] ---- - -# PLN-0125 - Asset Backlog and Stable Slot Handles - -## Briefing - -`DEC-0034` changes asset loading from request-owned thread spawning to a serial -backlog keyed by `bank_type/slot`. Handles become stable bank-slot observers -with separate resident slot state and request state. - -## Decisions de Origem - -- `DEC-0034` - Async Work Lane and Asset Backlog Contract. - -## Alvo - -Refactor `AssetManager` so asset requests are tracked by target bank slot, -deduplicated/superseded by target, and observable through stable handles. - -## Escopo - -Included: - -- Stable handle structure with `bank_type`, `slot`, `slot_state`, and - `request_state`. -- Backlog keyed by `bank_type/slot`. -- Already-resident fast path. -- Superseding for queued and active requests. -- Main-lane commit/install boundary. - -Fora de Escopo: - -- Public backlog syscall wiring. -- Percentile telemetry. -- FS public API. - -## Plano de Execucao - -### Step 1 - Model Slot Handles - -**What:** Replace transient load-operation assumptions with stable bank-slot -handle state. - -**How:** Introduce structures equivalent to: - -```text -handle: bank_type, slot -slot_state: loaded_asset_id, resident_state, slot_generation -request_state: requested_asset_id, request_generation, state, position, progress -``` - -**Files:** `crates/console/prometeu-drivers/src/asset.rs` and HAL asset types if -public type definitions belong there. - -### Step 2 - Key Backlog by Target - -**What:** Ensure at most one current request per `bank_type/slot`. - -**How:** Add a target index from `(bank_type, slot)` to current request -generation/handle state. Enqueue only when the target does not already have the -same resident asset and no newer request supersedes it. - -**Files:** `crates/console/prometeu-drivers/src/asset.rs`. - -### Step 3 - Implement Already-Resident Fast Path - -**What:** Return a ready handle immediately when the requested asset already -resides in the target slot. - -**How:** Compare requested `asset_id` with the target `slot_state.loaded_asset_id` -and resident validity before creating a queued job. - -**Files:** `crates/console/prometeu-drivers/src/asset.rs`. - -### Step 4 - Implement Superseding - -**What:** New request for the same target supersedes previous queued or active -request. - -**How:** Remove queued older request immediately. For active work, update target -generation so the old result is discarded if it cannot stop cooperatively. - -**Files:** `crates/console/prometeu-drivers/src/asset.rs`; async lane module -from `PLN-0124`. - -### Step 5 - Preserve Main-Lane Commit - -**What:** Keep install/commit into resident banks on the main runtime lane. - -**How:** Async lane produces staged materialized result. `asset.commit(handle)` -installs only when the request generation is current and ready. - -**Files:** `crates/console/prometeu-drivers/src/asset.rs`; VM runtime dispatch if -commit behavior changes. - -## Criterios de Aceite - -- [ ] No asset load spawns an OS thread per request. -- [ ] At most one request exists per `bank_type/slot`. -- [ ] Same-target newer request supersedes older queued request. -- [ ] Same-target newer request prevents active stale result from installing. -- [ ] Already-resident target returns ready handle without backlog entry. -- [ ] Handles remain queryable for empty, queued, active, ready, committed, - canceled, superseded, and error states. - -## Tests / Validacao - -- Unit tests for handle state transitions. -- Unit tests for target-keyed superseding. -- Unit tests for already-resident fast path. -- Integration test for stale active result discard. -- Existing asset load/commit tests must continue passing. - -## Riscos - -- Confusing handle identity with request generation. -- Installing a stale async result after superseding. -- Losing compatibility with existing `asset.status/commit/cancel` behavior. diff --git a/discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md b/discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md deleted file mode 100644 index 4d508601..00000000 --- a/discussion/workflow/plans/PLN-0126-asset-backlog-public-api-and-status-surface.md +++ /dev/null @@ -1,124 +0,0 @@ ---- -id: PLN-0126 -ticket: perf-async-background-work-lanes-for-assets-and-fs -title: Asset Backlog Public API and Status Surface -status: done -created: 2026-06-28 -completed: -ref_decisions: [DEC-0034] -tags: [perf, asset, async, syscall, runtime] ---- - -# PLN-0126 - Asset Backlog Public API and Status Surface - -## Briefing - -`DEC-0034` keeps `asset.load/status/commit/cancel` and adds a small status-first -backlog inspection and ordering surface. This plan wires that surface through -HAL syscall metadata and runtime dispatch. - -## Decisions de Origem - -- `DEC-0034` - Async Work Lane and Asset Backlog Contract. - -## Alvo - -Expose asset backlog inspection and reordering while preserving status-first -semantics and existing asset load identity. - -## Escopo - -Included: - -- Syscall registry additions for backlog APIs. -- Runtime dispatch handlers. -- Status enum additions or normalization. -- ABI/spec alignment with `PLN-0123`. - -Fora de Escopo: - -- Async lane internals. -- Asset decode refactor. -- FS API. - -## Plano de Execucao - -### Step 1 - Finalize Syscall Shapes - -**What:** Choose final names, arity, and return slots for backlog operations. - -**How:** Start from `DEC-0034`: - -```text -asset.backlog_info() -asset.backlog_position(handle) -asset.backlog_move(handle, new_position) -asset.backlog_promote(handle) -asset.backlog_demote(handle) -asset.target_status(bank_type, slot) -``` - -**Files:** `crates/console/prometeu-hal/src/syscalls/domains/asset.rs`, -`docs/specs/runtime/16-host-abi-and-syscalls.md`. - -### Step 2 - Add Status Values - -**What:** Represent `empty`, `invalid`, `queued`, `active`, `ready`, -`committed`, `canceled`, `superseded`, `error`, and `backend_unavailable`. - -**How:** Extend or map existing `LoadStatus` and `AssetOpStatus` without -turning operational states into traps. - -**Files:** HAL asset status types and `crates/console/prometeu-drivers/src/asset.rs`. - -### Step 3 - Add Dispatch - -**What:** Wire new syscalls through VM runtime dispatch. - -**How:** Add match arms near existing `AssetLoad`, `AssetStatus`, -`AssetCommit`, and `AssetCancel` arms. Return status-first values. - -**Files:** `crates/console/prometeu-system/src/services/vm_runtime/dispatch.rs`. - -### Step 4 - Implement API Methods - -**What:** Add AssetManager methods backing each public operation. - -**How:** Methods must read stable handle/target state and reorder only queued -requests. `promote` and `demote` are shortcuts over move semantics. - -**Files:** `crates/console/prometeu-drivers/src/asset.rs`. - -### Step 5 - Validate ABI Metadata - -**What:** Ensure registry ids, args, returns, capability, cost, and -non-deterministic flags are correct. - -**How:** Update syscall metadata tests and any declared PBS/runtime syscall -tables. - -**Files:** `crates/console/prometeu-hal/src/syscalls/tests.rs` and related -registry files. - -## Criterios de Aceite - -- [ ] Public backlog operations are registered with stable metadata. -- [ ] All backlog operations are status-first. -- [ ] `promote` is equivalent to moving to first pending position. -- [ ] `demote` moves to the end of pending backlog. -- [ ] Reordering a non-queued request returns operational status. -- [ ] Structural ABI misuse remains Trap. - -## Tests / Validacao - -- HAL syscall metadata tests. -- VM runtime dispatch tests for each new syscall. -- AssetManager unit tests for move/promote/demote. -- Tests for `target_status` on empty, queued, active, ready, and superseded - targets. - -## Riscos - -- Adding too many public ABI details before spec propagation is accepted. -- Returning payload values that are ambiguous when status is not success. -- Treating known but non-mutable states as traps instead of statuses. diff --git a/discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md b/discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md deleted file mode 100644 index 4dd01f7b..00000000 --- a/discussion/workflow/plans/PLN-0127-async-lane-memcard-and-fs-integration-boundaries.md +++ /dev/null @@ -1,120 +0,0 @@ ---- -id: PLN-0127 -ticket: perf-async-background-work-lanes-for-assets-and-fs -title: Async Lane Memcard and FS Integration Boundaries -status: done -created: 2026-06-28 -completed: -ref_decisions: [DEC-0034] -tags: [perf, fs, memcard, async, runtime] ---- - -# PLN-0127 - Async Lane Memcard and FS Integration Boundaries - -## Briefing - -`DEC-0034` allows memcard and FS work to consume the async IO lane but keeps -public FS API semantics out of scope. This plan establishes integration -boundaries without deciding app-home FS API shape. - -## Decisions de Origem - -- `DEC-0034` - Async Work Lane and Asset Backlog Contract. - -## Alvo - -Define and implement safe internal boundaries for memcard and FS use of the -async work lane, preserving lane priority and keeping FS public API decisions in -`AGD-0006`. - -## Escopo - -Included: - -- Memcard as a priority lane consumer candidate. -- FS as a permitted internal lane consumer. -- Priority ordering in the lane. -- Updates to `AGD-0006` to acknowledge the lane. - -Fora de Escopo: - -- Public FS request/poll design. -- App-home path/status contract. -- Full memcard async conversion if it requires public ABI changes. - -## Plano de Execucao - -### Step 1 - Update FS Agenda - -**What:** Record that an async IO lane exists and may be used by FS. - -**How:** Add a concise section to `AGD-0006` stating that FS public API remains -owned by that agenda, but implementation may use the async lane from `DEC-0034`. - -**Files:** `discussion/workflow/agendas/AGD-0006-app-home-filesystem-surface-and-semantics.md`. - -### Step 2 - Identify Memcard Operations - -**What:** List memcard operations that can benefit from async IO. - -**How:** Inspect `crates/console/prometeu-system/src/services/memcard.rs` and -VM runtime dispatch for `mem.*` operations. Classify read, write, commit, and -clear behavior. - -**Files:** memcard service and VM runtime dispatch modules. - -### Step 3 - Define Lane Priority - -**What:** Encode the initial priority order from `DEC-0034`. - -**How:** Add priority classes: - -```text -memcard commit/write -FS write/config -asset load -non-critical list/read -``` - -**Files:** async lane module from `PLN-0124`. - -### Step 4 - Add Internal Consumer Hooks - -**What:** Provide internal APIs for memcard/FS to submit lane jobs later. - -**How:** Add types/interfaces without changing public FS syscall behavior. -No public FS request/poll operations are introduced here. - -**Files:** async lane module; FS/memcard service modules as needed. - -### Step 5 - Preserve Current Public Behavior - -**What:** Ensure existing memcard and FS syscalls keep their current observable -surface unless a later plan explicitly changes it. - -**How:** Do not alter return shapes in this plan. Add tests proving existing -status-first behavior remains intact. - -**Files:** VM runtime FS/memcard tests. - -## Criterios de Aceite - -- [ ] `AGD-0006` acknowledges the async IO lane and keeps FS public API in its - own scope. -- [ ] Async lane supports priority classes required by `DEC-0034`. -- [ ] Memcard and FS can be represented as internal lane consumers. -- [ ] Existing public memcard/FS syscall shapes are unchanged. -- [ ] No FS public request/poll API is introduced by this plan. - -## Tests / Validacao - -- Existing FS/memcard tests continue passing. -- Unit tests for lane priority ordering. -- Tests proving asset jobs do not overlap with higher-priority memcard jobs - once both are submitted through the lane. - -## Riscos - -- Accidentally deciding FS public API in the wrong discussion. -- Making memcard async in a way that changes status-first behavior. -- Starving asset loads if priority rules do not include fairness. diff --git a/discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md b/discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md deleted file mode 100644 index 9600178c..00000000 --- a/discussion/workflow/plans/PLN-0128-async-asset-pipeline-contract-tests-and-telemetry.md +++ /dev/null @@ -1,126 +0,0 @@ ---- -id: PLN-0128 -ticket: perf-async-background-work-lanes-for-assets-and-fs -title: Async Asset Pipeline Contract Tests and Telemetry -status: done -created: 2026-06-28 -completed: -ref_decisions: [DEC-0034] -tags: [perf, asset, async, telemetry, tests, runtime] ---- - -# PLN-0128 - Async Asset Pipeline Contract Tests and Telemetry - -## Briefing - -`DEC-0034` requires observable progress, backlog state, superseding, and cheap -telemetry. This plan adds the validation and telemetry evidence needed to prove -the new async asset pipeline is correct. - -## Decisions de Origem - -- `DEC-0034` - Async Work Lane and Asset Backlog Contract. - -## Alvo - -Provide contract tests and telemetry for the async asset lane so future -implementations can be changed without regressing lane semantics. - -## Escopo - -Included: - -- Contract tests for lane and asset backlog behavior. -- Progress reporting tests. -- Superseding and stale result discard tests. -- Telemetry counters and percentile updates. -- No-sleep deterministic test helpers where needed. - -Fora de Escopo: - -- Implementing public API wiring if `PLN-0126` has not run. -- Full benchmarking infrastructure. -- Host UI display of telemetry. - -## Plano de Execucao - -### Step 1 - Build Deterministic Test Hooks - -**What:** Add test hooks for blocking a lane job at read/decode/stage. - -**How:** Use channels/barriers or explicit fake job steps. Do not rely on -`thread::sleep` for correctness. - -**Files:** async lane tests; asset test support modules. - -### Step 2 - Test Backlog Ordering - -**What:** Prove queued jobs run serially and in backlog order. - -**How:** Submit multiple target slots and assert active job, pending positions, -promote, demote, and move behavior. - -**Files:** asset manager tests and lane tests. - -### Step 3 - Test Superseding - -**What:** Prove newer request for the same `bank_type/slot` supersedes older -requests. - -**How:** Cover queued superseding and active superseding. For active work, prove -stale completion cannot install. - -**Files:** asset manager tests. - -### Step 4 - Test Stable Handle Observability - -**What:** Prove handles are queryable for empty, queued, active, ready, -committed, canceled, superseded, and error states. - -**How:** Use `target_status` and handle status APIs to inspect both `slot_state` -and `request_state`. - -**Files:** asset manager tests; VM dispatch tests if public API is available. - -### Step 5 - Test Progress - -**What:** Prove progress uses `0..10000` and phase boundaries. - -**How:** Fake read/decode/stage phases with known completion points. Validate -non-linear phases do not report false internal progress. - -**Files:** asset manager tests; telemetry tests. - -### Step 6 - Add Telemetry - -**What:** Add counters and percentile aggregation. - -**How:** Update submitted/completed/failed/canceled/superseded counters when job -state closes. Update percentiles by `bank_type` and lightweight `asset_id` -window only at job completion. - -**Files:** asset manager telemetry, async lane telemetry, existing runtime -telemetry types if shared. - -## Criterios de Aceite - -- [ ] Tests prove one active job at a time. -- [ ] Tests prove backlog order and reordering. -- [ ] Tests prove queued and active superseding. -- [ ] Tests prove stale active results are discarded. -- [ ] Tests prove handles remain queryable across all required states. -- [ ] Tests prove progress phase behavior. -- [ ] Telemetry is updated at job closure, not inside inner decode loops. - -## Tests / Validacao - -- Run targeted cargo tests for asset manager, async lane, and VM runtime - dispatch. -- Run existing asset-bank tests to prove current behavior remains compatible. -- Run `discussion validate` after plan state changes. - -## Riscos - -- Tests that depend on sleeps will be flaky. -- Telemetry may become too expensive if updated inside decode loops. -- Public API tests must wait for `PLN-0126` if syscall wiring is not present.