diff --git a/src/Cafe/HW/Latte/Core/FetchShader.cpp b/src/Cafe/HW/Latte/Core/FetchShader.cpp index e705d3d1..0779fac9 100644 --- a/src/Cafe/HW/Latte/Core/FetchShader.cpp +++ b/src/Cafe/HW/Latte/Core/FetchShader.cpp @@ -14,6 +14,8 @@ #include /* SHA1_DIGEST_LENGTH */ #include /* EVP_Digest */ +void LatteSHRC_RemoveShaderStateCacheEntryByKey(uint64 key); + uint32 LatteShaderRecompiler_getAttributeSize(LatteParsedFetchShaderAttribute_t* attrib) { if (attrib->format == FMT_32_32_32_32 || attrib->format == FMT_32_32_32_32_FLOAT) @@ -468,6 +470,9 @@ LatteFetchShader* LatteShaderRecompiler_createFetchShader(LatteFetchShader::Cach LatteFetchShader::~LatteFetchShader() { UnregisterInCache(); + // remove from shader state cache + while (!m_shaderStateCacheKeys.empty()) + LatteSHRC_RemoveShaderStateCacheEntryByKey(m_shaderStateCacheKeys.back()); } struct FetchShaderLookupInfo diff --git a/src/Cafe/HW/Latte/Core/FetchShader.h b/src/Cafe/HW/Latte/Core/FetchShader.h index aac084f2..257cb2f1 100644 --- a/src/Cafe/HW/Latte/Core/FetchShader.h +++ b/src/Cafe/HW/Latte/Core/FetchShader.h @@ -64,11 +64,12 @@ struct LatteFetchShader static bool isValidBufferIndex(const uint32 index) { return index < 0x10; }; - // cache - LatteFetchShader* RegisterInCache(CacheHash fsHash); // Fails if another fetch shader object is already registered with the same fsHash. Returns the previously registered fetch shader or null - void UnregisterInCache(); + // keys in shader state cache + std::vector m_shaderStateCacheKeys; // fetch shader cache (move these to separate Cache class?) + LatteFetchShader* RegisterInCache(CacheHash fsHash); // fails if another fetch shader object is already registered with the same fsHash. Returns the previously registered fetch shader or null + void UnregisterInCache(); static CacheHash CalculateCacheHash(void* programCode, uint32 programSize); static LatteFetchShader* FindInCacheByHash(CacheHash fsHash); static LatteFetchShader* FindByGPUState(); diff --git a/src/Cafe/HW/Latte/Core/LatteCommandProcessor.cpp b/src/Cafe/HW/Latte/Core/LatteCommandProcessor.cpp index 77f9c734..6e377575 100644 --- a/src/Cafe/HW/Latte/Core/LatteCommandProcessor.cpp +++ b/src/Cafe/HW/Latte/Core/LatteCommandProcessor.cpp @@ -1262,7 +1262,6 @@ void LatteCP_processCommandBuffer(DrawPassContext& drawPassCtx) case IT_DRAW_INDEX_2: { drawPassCtx.beginDrawPass(); - //cemuLog_log(LogType::Force, "[CmdBuf] DrawIndex2"); LatteCP_itDrawIndex2(cmdData, nWords, drawPassCtx); // enter fast draw mode drawPassCtx.PushCurrentCommandQueuePos(cmd, cmdStart, cmdEnd); diff --git a/src/Cafe/HW/Latte/Core/LatteOverlay.cpp b/src/Cafe/HW/Latte/Core/LatteOverlay.cpp index 1a1a560c..8fdc8f3e 100644 --- a/src/Cafe/HW/Latte/Core/LatteOverlay.cpp +++ b/src/Cafe/HW/Latte/Core/LatteOverlay.cpp @@ -35,6 +35,8 @@ struct OverlayStats extern std::atomic_int g_compiled_shaders_total; extern std::atomic_int g_compiled_shaders_async; +extern std::atomic_int g_shaderStateCacheSetCount; +extern std::atomic_int g_shaderStateCacheSetAuxCount; std::atomic_int g_compiling_pipelines; std::atomic_int g_compiling_pipelines_async; @@ -110,6 +112,7 @@ void LatteOverlay_renderOverlay(ImVec2& position, ImVec2& pivot, sint32 directio // general debug info ImGui::Text("--- Debug info ---"); ImGui::Text("IndexUploadPerFrame: %dKB", (performanceMonitor.stats.indexDataUploadPerFrame+1023)/1024); + ImGui::Text("SHCSets: %d / %d", g_shaderStateCacheSetCount.load(), g_shaderStateCacheSetAuxCount.load()); // backend specific info g_renderer->AppendOverlayDebugInfo(); } diff --git a/src/Cafe/HW/Latte/Core/LatteRenderTarget.cpp b/src/Cafe/HW/Latte/Core/LatteRenderTarget.cpp index 148a6dce..79b2d694 100644 --- a/src/Cafe/HW/Latte/Core/LatteRenderTarget.cpp +++ b/src/Cafe/HW/Latte/Core/LatteRenderTarget.cpp @@ -689,6 +689,7 @@ void LatteRenderTarget_itHLESwapScanBuffer() performanceMonitor.gpuTime_frameTime.beginMeasuring(); LatteTC_CleanupUnusedTextures(); + LatteSHRC_CleanupShaderStateCache(); #ifdef ENABLE_OPENGL LatteDraw_cleanupAfterFrame(); #endif diff --git a/src/Cafe/HW/Latte/Core/LatteShader.cpp b/src/Cafe/HW/Latte/Core/LatteShader.cpp index dbbf9fbf..38fc5504 100644 --- a/src/Cafe/HW/Latte/Core/LatteShader.cpp +++ b/src/Cafe/HW/Latte/Core/LatteShader.cpp @@ -15,10 +15,10 @@ #include "config/ActiveSettings.h" #include "Cafe/GameProfile/GameProfile.h" #include "util/containers/flat_hash_map.hpp" +#include "util/helpers/StateHasher.h" #ifdef ENABLE_METAL #include "Cafe/HW/Latte/Renderer/Metal/LatteToMtl.h" #endif -#include // experimental new decompiler (WIP) #include "util/Zir/EmitterGLSL/ZpIREmitGLSL.h" @@ -56,6 +56,8 @@ uint64 _shaderBaseHash_ps; std::atomic_int g_compiled_shaders_total = 0; std::atomic_int g_compiled_shaders_async = 0; +std::atomic_int g_shaderStateCacheSetCount = 0; +std::atomic_int g_shaderStateCacheSetAuxCount = 0; LatteFetchShader* LatteSHRC_GetActiveFetchShader() { @@ -87,6 +89,8 @@ inline ska::flat_hash_map& LatteSHRC_GetCacheByT return sPixelShaders; } +void LatteSHRC_RemoveShaderStateCacheEntryByKey(uint64 key); + // calculate hash from shader binary // this algorithm could be more efficient since we could leverage the fact that the size is always aligned to 8 byte // but since this is baked into the shader names used for gfx packs and shader caches we can't really change this @@ -149,7 +153,7 @@ LatteShaderPSInputTable* LatteSHRC_GetPSInputTable() return &_activePSImportTable; } -void LatteSHRC_RemoveFromCache(LatteDecompilerShader* shader) +void LatteSHRC_RemoveFromCaches(LatteDecompilerShader* shader) { bool removed = false; auto& cache = LatteSHRC_GetCacheByType(shader->shaderType); @@ -186,6 +190,10 @@ void LatteSHRC_RemoveFromCache(LatteDecompilerShader* shader) } } cemu_assert(removed); + // remove from shader state cache + // deleting by key means we delete all the other aux variants associated with it too, but it keeps the code simple and cache entries are cheap to recreate anyway + while (!shader->m_shaderStateCacheKeys.empty()) + LatteSHRC_RemoveShaderStateCacheEntryByKey(shader->m_shaderStateCacheKeys.back()); } void LatteSHRC_RemoveFromCacheByHash(uint64 shader_base_hash, uint64 shader_aux_hash, LatteConst::ShaderType type) @@ -198,12 +206,12 @@ void LatteSHRC_RemoveFromCacheByHash(uint64 shader_base_hash, uint64 shader_aux_ else if (type == LatteConst::ShaderType::Pixel) shader = LatteSHRC_FindPixelShader(shader_base_hash, shader_aux_hash); if (shader) - LatteSHRC_RemoveFromCache(shader); + LatteSHRC_RemoveFromCaches(shader); } -void LatteShader_free(LatteDecompilerShader* shader) +void LatteShader_free(LatteDecompilerShader* shader) // todo - make this ~LatteDecompilerShader() { - LatteSHRC_RemoveFromCache(shader); + LatteSHRC_RemoveFromCaches(shader); if (shader->shader) delete shader->shader; shader->shader = nullptr; @@ -436,10 +444,9 @@ LatteDecompilerShader* LatteSHRC_FindPixelShader(uint64 baseHash, uint64 auxHash return LatteSHRC_Get(sPixelShaders, baseHash, auxHash); } -// update the currently active fetch shader -void LatteShaderSHRC_UpdateFetchShader() +LatteFetchShader* LatteSHRC_GetOrCreateFetchShader() { - _activeFetchShader = LatteFetchShader::FindByGPUState(); + return LatteFetchShader::FindByGPUState(); } void LatteShader_CleanupAfterCompile(LatteDecompilerShader* shader) @@ -499,14 +506,14 @@ void LatteShader_DumpRawShader(uint64 baseHash, uint64 auxHash, uint32 type, uin } } -void LatteSHRC_UpdateVSBaseHash(uint8* vertexShaderPtr, uint32 vertexShaderSize, bool usesGeometryShader) +void LatteSHRC_UpdateVSBaseHash(uint8* vertexShaderPtr, uint32 vertexShaderSize, bool usesGeometryShader, LatteFetchShader* fetchShader) { uint32* vsProgramCode = (uint32*)vertexShaderPtr; // update hash from vertex shader data uint64 vsHash1 = 0; uint64 vsHash2 = 0; _calculateShaderProgramHash(vsProgramCode, vertexShaderSize, &hashCacheVS, &vsHash1, &vsHash2); - uint64 vsHash = vsHash1 + vsHash2 + _activeFetchShader->key + _activePSImportTable.key + (usesGeometryShader ? 0x1111ULL : 0ULL); + uint64 vsHash = vsHash1 + vsHash2 + fetchShader->key + _activePSImportTable.key + (usesGeometryShader ? 0x1111ULL : 0ULL); uint32 tmp = LatteGPUState.contextNew.PA_CL_VTE_CNTL.getRawValue() ^ 0x43F; vsHash += tmp; @@ -532,11 +539,11 @@ void LatteSHRC_UpdateVSBaseHash(uint8* vertexShaderPtr, uint32 vertexShaderSize, { bool isRectVertexShader = (primitiveType == Latte::LATTE_VGT_PRIMITIVE_TYPE::E_PRIMITIVE_TYPE::RECTS); - if ((usesGeometryShader || isRectVertexShader) || _activeFetchShader->mtlFetchVertexManually) + if ((usesGeometryShader || isRectVertexShader) || fetchShader->mtlFetchVertexManually) { - for (sint32 g = 0; g < _activeFetchShader->bufferGroups.size(); g++) + for (sint32 g = 0; g < fetchShader->bufferGroups.size(); g++) { - LatteParsedFetchShaderBufferGroup_t& group = _activeFetchShader->bufferGroups[g]; + LatteParsedFetchShaderBufferGroup_t& group = fetchShader->bufferGroups[g]; uint32 bufferIndex = group.attributeBufferIndex; uint32 bufferBaseRegisterIndex = mmSQ_VTX_ATTRIBUTE_BLOCK_START + bufferIndex * 7; uint32 bufferStride = (LatteGPUState.contextRegister[bufferBaseRegisterIndex + 2] >> 11) & 0xFFFF; @@ -552,7 +559,7 @@ void LatteSHRC_UpdateVSBaseHash(uint8* vertexShaderPtr, uint32 vertexShaderSize, vsHash += 51ULL; // Vertex fetch - if (_activeFetchShader->mtlFetchVertexManually) + if (fetchShader->mtlFetchVertexManually) vsHash += 349ULL; } } @@ -881,17 +888,17 @@ LatteDecompilerShader* LatteShader_CompileSeparableVertexShader(uint64 baseHash, return vertexShader; } -LatteDecompilerShader* LatteShader_CompileSeparableGeometryShader(uint64 baseHash, uint8* geometryShaderPtr, uint32 geometryShaderSize, uint8* geometryCopyShader, uint32 geometryCopyShaderSize) +LatteDecompilerShader* LatteShader_CompileSeparableGeometryShader(uint64 baseHash, uint8* geometryShaderPtr, uint32 geometryShaderSize, uint8* geometryCopyShader, uint32 geometryCopyShaderSize, LatteDecompilerShader* vertexShader) { LatteDecompilerOptions options; LatteShader_GetDecompilerOptions(options, LatteConst::ShaderType::Geometry, true); LatteDecompilerOutput_t decompilerOutput{}; - LatteDecompiler_DecompileGeometryShader(_shaderBaseHash_gs, LatteGPUState.contextRegister, geometryShaderPtr, geometryShaderSize, geometryCopyShader, geometryCopyShaderSize, _activeVertexShader->ringParameterCount, options, &decompilerOutput); + LatteDecompiler_DecompileGeometryShader(_shaderBaseHash_gs, LatteGPUState.contextRegister, geometryShaderPtr, geometryShaderSize, geometryCopyShader, geometryCopyShaderSize, vertexShader->ringParameterCount, options, &decompilerOutput); LatteDecompilerShader* geometryShader = LatteShader_CreateShaderFromDecompilerOutput(decompilerOutput, baseHash, true, 0, LatteGPUState.contextRegister); if (geometryShader->hasError == false) { - LatteShaderCache_writeSeparableGeometryShader(geometryShader->baseHash, geometryShader->auxHash, geometryShaderPtr, geometryShaderSize, geometryCopyShader, geometryCopyShaderSize, LatteGPUState.contextRegister, LatteGPUState.contextNew.GetSpecialStateValues(), _activeVertexShader->ringParameterCount); + LatteShaderCache_writeSeparableGeometryShader(geometryShader->baseHash, geometryShader->auxHash, geometryShaderPtr, geometryShaderSize, geometryCopyShader, geometryCopyShaderSize, LatteGPUState.contextRegister, LatteGPUState.contextNew.GetSpecialStateValues(), vertexShader->ringParameterCount); } LatteShader_DumpShader(geometryShader->baseHash, geometryShader->auxHash, geometryShader); LatteShader_DumpRawShader(geometryShader->baseHash, geometryShader->auxHash, SHADER_DUMP_TYPE_GEOMETRY, geometryShaderPtr, geometryShaderSize); @@ -943,10 +950,10 @@ LatteDecompilerShader* LatteShader_CompileSeparablePixelShader(uint64 baseHash, return pixelShader; } -void LatteSHRC_UpdateVertexShader(uint8* vertexShaderPtr, uint32 vertexShaderSize, bool usesGeometryShader) +LatteDecompilerShader* LatteSHRC_GetOrCreateVertexShader(uint8* vertexShaderPtr, uint32 vertexShaderSize, bool usesGeometryShader, LatteFetchShader* fetchShader) { // todo - should include VTX_SEMANTIC table in state - LatteSHRC_UpdateVSBaseHash(vertexShaderPtr, vertexShaderSize, usesGeometryShader); + LatteSHRC_UpdateVSBaseHash(vertexShaderPtr, vertexShaderSize, usesGeometryShader, fetchShader); uint64 vsAuxHash = 0; auto itBaseShader = sVertexShaders.find(_shaderBaseHash_vs); LatteDecompilerShader* vertexShader = nullptr; @@ -956,22 +963,18 @@ void LatteSHRC_UpdateVertexShader(uint8* vertexShaderPtr, uint32 vertexShaderSiz vertexShader = LatteSHRC_GetFromChain(itBaseShader->second, _shaderBaseHash_vs, vsAuxHash); } if (!vertexShader) - vertexShader = LatteShader_CompileSeparableVertexShader(_shaderBaseHash_vs, vsAuxHash, vertexShaderPtr, vertexShaderSize, usesGeometryShader, _activeFetchShader); + vertexShader = LatteShader_CompileSeparableVertexShader(_shaderBaseHash_vs, vsAuxHash, vertexShaderPtr, vertexShaderSize, usesGeometryShader, fetchShader); if (vertexShader->hasError) - { LatteGPUState.activeShaderHasError = true; - return; - } - _activeVertexShader = vertexShader; + return vertexShader; } -void LatteSHRC_UpdateGeometryShader(bool usesGeometryShader, uint8* geometryShaderPtr, uint32 geometryShaderSize, uint8* geometryCopyShader, uint32 geometryCopyShaderSize) +LatteDecompilerShader* LatteSHRC_GetOrCreateGeometryShader(bool usesGeometryShader, uint8* geometryShaderPtr, uint32 geometryShaderSize, uint8* geometryCopyShader, uint32 geometryCopyShaderSize, LatteDecompilerShader* vertexShader) { if (!usesGeometryShader || !_activeVertexShader) { _shaderBaseHash_gs = 0; - _activeGeometryShader = nullptr; - return; + return nullptr; } LatteSHRC_UpdateGSBaseHash(geometryShaderPtr, geometryShaderSize, geometryCopyShader, geometryCopyShaderSize); auto itBaseShader = sGeometryShaders.find(_shaderBaseHash_gs); @@ -985,17 +988,14 @@ void LatteSHRC_UpdateGeometryShader(bool usesGeometryShader, uint8* geometryShad else { // decompile geometry shader - geometryShader = LatteShader_CompileSeparableGeometryShader(_shaderBaseHash_gs, geometryShaderPtr, geometryShaderSize, geometryCopyShader, geometryCopyShaderSize); + geometryShader = LatteShader_CompileSeparableGeometryShader(_shaderBaseHash_gs, geometryShaderPtr, geometryShaderSize, geometryCopyShader, geometryCopyShaderSize, vertexShader); } if (geometryShader->hasError) - { LatteGPUState.activeShaderHasError = true; - return; - } - _activeGeometryShader = geometryShader; + return geometryShader; } -void LatteSHRC_UpdatePixelShader(uint8* pixelShaderPtr, uint32 pixelShaderSize, bool usesGeometryShader) +LatteDecompilerShader* LatteSHRC_GetOrCreatePixelShader(uint8* pixelShaderPtr, uint32 pixelShaderSize, bool usesGeometryShader) { LatteSHRC_UpdatePSBaseHash(pixelShaderPtr, pixelShaderSize, usesGeometryShader); uint64 psAuxHash = 0; @@ -1009,11 +1009,143 @@ void LatteSHRC_UpdatePixelShader(uint8* pixelShaderPtr, uint32 pixelShaderSize, if (!pixelShader) pixelShader = LatteShader_CompileSeparablePixelShader(_shaderBaseHash_ps, psAuxHash, pixelShaderPtr, pixelShaderSize, usesGeometryShader); if (pixelShader->hasError) - { LatteGPUState.activeShaderHasError = true; + return pixelShader; +} + +static inline uint64_t mix64(uint64_t v) +{ + v ^= v >> 30; + v *= 0xbf58476d1ce4e5b9ULL; + v ^= v >> 27; + v *= 0x94d049bb133111ebULL; + v ^= v >> 31; + return v; +} + +struct ShaderStateInfoAuxVariant +{ + uint64 combinedAuxHash{}; + LatteDecompilerShader* vertexShader{nullptr}; + LatteDecompilerShader* pixelShader{nullptr}; + LatteDecompilerShader* geometryShader{nullptr}; + bool hasError{false}; // in case of error just set all the shaders to nullptr? +}; + +struct ShaderStateInfo +{ + // we need any set of shaders from an aux chain to calculate the actual aux variant hash + // so these just match the first one encountered + LatteFetchShader* fetchShader{nullptr}; + LatteDecompilerShader* vertexShader{nullptr}; + LatteDecompilerShader* pixelShader{nullptr}; + LatteDecompilerShader* geometryShader{nullptr}; + uint64 combinedAuxHash{}; + uint32 lastAccessFrameCount{}; + bool shaderError{false}; + std::vector auxVariants; +}; + +struct ShaderStateDirectHash +{ + size_t operator()(uint64 x) const noexcept + { + return x; + } +}; + +std::vector s_shaderStateCacheKeys; +size_t s_shaderStateCacheCleanupIndex = 0; +robin_hood::unordered_flat_map s_shaderStateCache; +// also benchmarked here but didn't perform better or only marginally better and not worth pulling in an extra library: +// jg::dense_hash_map +// folly::F14FastMap +// robin_hood::unordered_flat_map ended up performing close to best and was choosen because we already have it included in the project anyway + +FORCEINLINE uint64 CalcCombinedAuxHash(LatteFetchShader* fetchShader, LatteDecompilerShader* vertexShader, LatteDecompilerShader* pixelShader) +{ + uint64 vsAuxHash = vertexShader ? LatteSHRC_CalcVSAuxHash(vertexShader, LatteGPUState.contextRegister) : 0; + uint64 psAuxHash = pixelShader ? LatteSHRC_CalcPSAuxHash(pixelShader, LatteGPUState.contextRegister) : 0; + uint64 combinedAuxHash = vsAuxHash + mix64(psAuxHash); +#ifdef ENABLE_METAL + if (g_renderer->GetType() == RendererAPI::Metal && fetchShader) + { + for (auto& bufferGroup : fetchShader->bufferGroups) + { + uint32 bufferBaseRegisterIndex = mmSQ_VTX_ATTRIBUTE_BLOCK_START + bufferGroup.attributeBufferIndex * 7; + uint32 bufferStride = (LatteGPUState.contextRegister[bufferBaseRegisterIndex + 2] >> 11) & 0xFFFF; + combinedAuxHash = std::rotl(combinedAuxHash, 7); + combinedAuxHash += bufferStride; + } + } +#endif + return combinedAuxHash; +} + +void LatteSHRC_RemoveShaderStateCacheEntryByKey(uint64 key) +{ + auto it = s_shaderStateCache.find(key); + if (it == s_shaderStateCache.end()) + { + cemu_assert_suspicious(); // shader shouldn't have a key which was already removed return; } - _activePixelShader = pixelShader; + ShaderStateInfo& shaderStateInfo = it->second; + if (shaderStateInfo.fetchShader) + std::erase(shaderStateInfo.fetchShader->m_shaderStateCacheKeys, key); + g_shaderStateCacheSetAuxCount -= shaderStateInfo.auxVariants.size(); + for (auto& auxVariant : shaderStateInfo.auxVariants) + { + if (auxVariant.vertexShader) + std::erase(auxVariant.vertexShader->m_shaderStateCacheKeys, key); + if (auxVariant.pixelShader) + std::erase(auxVariant.pixelShader->m_shaderStateCacheKeys, key); + if (auxVariant.geometryShader) + std::erase(auxVariant.geometryShader->m_shaderStateCacheKeys, key); + } + s_shaderStateCache.erase(it); + --g_shaderStateCacheSetCount; +} + +void LatteSHRC_CleanupShaderStateCache() +{ + constexpr uint32 NUM_FRAMES_UNTIL_EXPIRE = 5 * 60; // entries expire after not being used for 5 seconds at 60 FPS + constexpr sint32 MAX_CHECKS_PER_FRAME = 30; + constexpr sint32 MAX_DELETES_PER_FRAME = 8; + + if (s_shaderStateCache.empty()) + { + s_shaderStateCacheKeys.clear(); + s_shaderStateCacheCleanupIndex = 0; + return; + } + + sint32 deleteCount = 0; + for (sint32 i = 0; i < MAX_CHECKS_PER_FRAME && !s_shaderStateCacheKeys.empty(); i++) + { + if (s_shaderStateCacheCleanupIndex >= s_shaderStateCacheKeys.size()) + s_shaderStateCacheCleanupIndex = 0; + uint64 key = s_shaderStateCacheKeys[s_shaderStateCacheCleanupIndex]; + auto it = s_shaderStateCache.find(key); + if (it == s_shaderStateCache.end()) + { + s_shaderStateCacheKeys[s_shaderStateCacheCleanupIndex] = s_shaderStateCacheKeys.back(); + s_shaderStateCacheKeys.pop_back(); + continue; + } + uint32 framesSinceLastAccess = LatteGPUState.frameCounter - it->second.lastAccessFrameCount; + if (framesSinceLastAccess >= NUM_FRAMES_UNTIL_EXPIRE) + { + if (deleteCount >= MAX_DELETES_PER_FRAME) + break; + LatteSHRC_RemoveShaderStateCacheEntryByKey(key); + s_shaderStateCacheKeys[s_shaderStateCacheCleanupIndex] = s_shaderStateCacheKeys.back(); + s_shaderStateCacheKeys.pop_back(); + deleteCount++; + continue; + } + s_shaderStateCacheCleanupIndex++; + } } void LatteSHRC_UpdateActiveShaders() @@ -1024,66 +1156,158 @@ void LatteSHRC_UpdateActiveShaders() cemu_assert_debug(LatteGPUState.contextNew.VGT_GS_MODE.get_ES_PASSTHRU() == false); // todo: Support for ES passthrough and cut mode in mmVGT_GS_MODE - bool geometryShaderUsed = false; - if (gsMode == Latte::LATTE_VGT_GS_MODE::E_MODE::OFF) - { - geometryShaderUsed = false; - } - else if (gsMode == Latte::LATTE_VGT_GS_MODE::E_MODE::SCENARIO_G) - { - // could also be compute shader? - geometryShaderUsed = true; - } - else - { - cemu_assert_debug(false); - } - // get shader programs - uint8* psProgramCode = (uint8*)memory_getPointerFromPhysicalOffset((LatteGPUState.contextRegister[mmSQ_PGM_START_PS] & 0xFFFFFF) << 8); - uint32 psProgramSize = LatteGPUState.contextRegister[mmSQ_PGM_START_PS + 1] << 3; - uint8* gsProgramCode = (uint8*)memory_getPointerFromPhysicalOffset((LatteGPUState.contextRegister[mmSQ_PGM_START_GS] & 0xFFFFFF) << 8); - uint32 gsProgramSize = LatteGPUState.contextRegister[mmSQ_PGM_START_GS + 1] << 3; + cemu_assert_debug(gsMode == Latte::LATTE_VGT_GS_MODE::E_MODE::OFF || gsMode == Latte::LATTE_VGT_GS_MODE::E_MODE::SCENARIO_G); // other modes are not supported + bool geometryShaderUsed = gsMode != Latte::LATTE_VGT_GS_MODE::E_MODE::OFF; - uint8* vsProgramCode; - uint32 vsProgramSize; - uint8* copyProgramCode = NULL; + // get program pointers and sizes + uint32 fsProgramAddr = LatteGPUState.contextRegister[mmSQ_PGM_START_FS] << 8; + uint32 fsProgramSize = LatteGPUState.contextRegister[mmSQ_PGM_START_FS + 1] << 3; + uint32 vsProgramAddr = LatteGPUState.contextRegister[mmSQ_PGM_START_VS] << 8; + uint32 vsProgramSize = LatteGPUState.contextRegister[mmSQ_PGM_START_VS + 1] << 3; + uint32 psProgramAddr = LatteGPUState.contextRegister[mmSQ_PGM_START_PS] << 8; + uint32 psProgramSize = LatteGPUState.contextRegister[mmSQ_PGM_START_PS + 1] << 3; + uint32 gsProgramAddr = LatteGPUState.contextRegister[mmSQ_PGM_START_GS] << 8; + uint32 gsProgramSize = LatteGPUState.contextRegister[mmSQ_PGM_START_GS + 1] << 3; + uint32 copyProgramAddr = 0; uint32 copyProgramSize = 0; + if (geometryShaderUsed) { - vsProgramCode = (uint8*)memory_getPointerFromPhysicalOffset((LatteGPUState.contextRegister[mmSQ_PGM_START_ES] & 0xFFFFFF) << 8); + // VS parameters come from ES instead + copyProgramAddr = vsProgramAddr; + copyProgramSize = vsProgramSize; + vsProgramAddr = LatteGPUState.contextRegister[mmSQ_PGM_START_ES] << 8; vsProgramSize = LatteGPUState.contextRegister[mmSQ_PGM_START_ES + 1] << 3; - copyProgramCode = (uint8*)memory_getPointerFromPhysicalOffset((LatteGPUState.contextRegister[mmSQ_PGM_START_VS] & 0xFFFFFF) << 8); - if (LatteGPUState.contextRegister[mmSQ_PGM_START_VS] == 0) [[unlikely]] - { - copyProgramCode = NULL; - debug_printf("copyProgram is NULL but used. Might be because of unsupported vertex/geometry mode?"); - } - copyProgramSize = LatteGPUState.contextRegister[mmSQ_PGM_START_VS + 1] << 3; } else { - if (LatteGPUState.contextRegister[mmSQ_PGM_START_VS] == 0) [[unlikely]] + gsProgramAddr = 0; + gsProgramSize = 0; + if (vsProgramAddr == 0 || vsProgramSize == 0) { + // todo - we dont really need to handle this since invalid shader states get baked into the lookup cache anyway debug_printf("No vertex shader program set\n"); LatteGPUState.activeShaderHasError = true; return; } - vsProgramCode = (uint8*)memory_getPointerFromPhysicalOffset((LatteGPUState.contextRegister[mmSQ_PGM_START_VS] & 0xFFFFFF) << 8); - vsProgramSize = LatteGPUState.contextRegister[mmSQ_PGM_START_VS + 1] << 3; } - // set new shaders + // build a hash from the combined shader state + // we use this as a lookup into a "shader state" cache rather than looking up all the shader stages individually + DualStateHasher hasher; + + uint64_t f = ((uint64_t)fsProgramAddr << 32) | (uint64_t)fsProgramSize; + uint64_t a = ((uint64_t)vsProgramAddr << 32) | (uint64_t)vsProgramSize; + uint64_t b = ((uint64_t)psProgramAddr << 32) | (uint64_t)psProgramSize; + uint64_t c = ((uint64_t)gsProgramAddr << 32) | (uint64_t)gsProgramSize; + uint64_t d = ((uint64_t)copyProgramAddr << 32) | (uint64_t)copyProgramSize; + + hasher.MixIn(f, a); + hasher.MixIn(b, c); + + constexpr uint32 PA_CL_VTE_CNTL_MASK = 0x3F; // viewport scale and offset enable bits + constexpr uint32 PA_CL_CLIP_CNTL_MASK = 1 << 19; // DX_CLIP_SPACE_DEF (halfZ) + constexpr uint32 VGT_PRIMITIVE_TYPE_MASK = 0x3F; + constexpr uint32 SPI_PS_IN_CONTROL_0_MASK = 0x3F | (1 << 8) | (0x1F << 10) | (0xF << 15) | (0x7F << 19); + constexpr uint32 SPI_PS_IN_CONTROL_1_MASK = 0x1FF << 8; // front-face settings (gl_FrontFacing) + constexpr uint32 SPI_INTERP_CONTROL_0_MASK = 1 << 1; // point sprite coord enable + + uint64 baseState0 = ((uint64)(LatteGPUState.contextNew.PA_CL_VTE_CNTL.getRawValue() & PA_CL_VTE_CNTL_MASK) << 32) | (LatteGPUState.contextNew.PA_CL_CLIP_CNTL.getRawValue() & PA_CL_CLIP_CNTL_MASK); + uint64 baseState1 = ((uint64)(LatteGPUState.contextNew.VGT_PRIMITIVE_TYPE.getRawValue() & VGT_PRIMITIVE_TYPE_MASK) << 32) | LatteGPUState.contextRegister[mmVGT_STRMOUT_EN]; + hasher.MixIn(baseState0, baseState1); + + LatteShader_UpdatePSInputs(LatteGPUState.contextRegister); // updates _activePSImportTable + + hasher.MixIn(d, _activePSImportTable.key); + hasher.MixIn(LatteGPUState.contextRegister[mmSPI_PS_IN_CONTROL_0] & SPI_PS_IN_CONTROL_0_MASK, LatteGPUState.contextRegister[mmSPI_PS_IN_CONTROL_1] & SPI_PS_IN_CONTROL_1_MASK); + hasher.MixIn(LatteGPUState.contextRegister[mmSPI_INTERP_CONTROL_0] & SPI_INTERP_CONTROL_0_MASK, 0); +#ifdef ENABLE_METAL + if (g_renderer->GetType() == RendererAPI::Metal) + { + uint64 mtlState = LatteGPUState.contextNew.IsRasterizationEnabled() ? 1 : 0; + hasher.MixInSingle(mtlState); + } +#endif + uint64 h = hasher.Finish(); + LatteGPUState.activeShaderHasError = false; - LatteShader_UpdatePSInputs(LatteGPUState.contextRegister); - LatteShaderSHRC_UpdateFetchShader(); - LatteSHRC_UpdateVertexShader(vsProgramCode, vsProgramSize, geometryShaderUsed); - if (LatteGPUState.activeShaderHasError) - return; - LatteSHRC_UpdateGeometryShader(geometryShaderUsed, gsProgramCode, gsProgramSize, copyProgramCode, copyProgramSize); - if (LatteGPUState.activeShaderHasError) - return; - LatteSHRC_UpdatePixelShader(psProgramCode, psProgramSize, geometryShaderUsed); - if (LatteGPUState.activeShaderHasError) - return; + + ShaderStateInfo* shaderStateInfo = nullptr; + + auto it = s_shaderStateCache.find(h); + if (it != s_shaderStateCache.end()) + { + shaderStateInfo = &it->second; + shaderStateInfo->lastAccessFrameCount = LatteGPUState.frameCounter; + uint64 combinedAuxHash = CalcCombinedAuxHash(shaderStateInfo->fetchShader, shaderStateInfo->vertexShader, shaderStateInfo->pixelShader); + if (shaderStateInfo->combinedAuxHash == combinedAuxHash) [[likely]] + { + _activeFetchShader = shaderStateInfo->fetchShader; + _activeVertexShader = shaderStateInfo->vertexShader; + _activePixelShader = shaderStateInfo->pixelShader; + _activeGeometryShader = shaderStateInfo->geometryShader; + return; + } + for (auto& auxVariant : shaderStateInfo->auxVariants) + { + if (auxVariant.combinedAuxHash == combinedAuxHash) + { + _activeFetchShader = shaderStateInfo->fetchShader; + _activeVertexShader = auxVariant.vertexShader; + _activePixelShader = auxVariant.pixelShader; + _activeGeometryShader = auxVariant.geometryShader; + return; + } + } + } + // no cache entry found, get/create shaders individually and add to cache + LatteFetchShader* fetchShader = LatteSHRC_GetOrCreateFetchShader(); + _activeFetchShader = fetchShader; + bool shaderError = LatteGPUState.activeShaderHasError; + LatteDecompilerShader* vertexShader = LatteSHRC_GetOrCreateVertexShader((uint8*)memory_getPointerFromPhysicalOffset(vsProgramAddr), vsProgramSize, geometryShaderUsed, fetchShader); + shaderError |= LatteGPUState.activeShaderHasError; + LatteDecompilerShader* pixelShader = LatteSHRC_GetOrCreatePixelShader((uint8*)memory_getPointerFromPhysicalOffset(psProgramAddr), psProgramSize, geometryShaderUsed); + shaderError |= LatteGPUState.activeShaderHasError; + LatteDecompilerShader* geometryShader = LatteSHRC_GetOrCreateGeometryShader(geometryShaderUsed, (uint8*)memory_getPointerFromPhysicalOffset(gsProgramAddr), gsProgramSize, (uint8*)memory_getPointerFromPhysicalOffset(copyProgramAddr), copyProgramSize, vertexShader); + shaderError |= LatteGPUState.activeShaderHasError; + uint64 combinedAuxHash = CalcCombinedAuxHash(fetchShader, vertexShader, pixelShader); + + if (!shaderStateInfo) + { + // create base entry + shaderStateInfo = &s_shaderStateCache[h]; + s_shaderStateCacheKeys.emplace_back(h); + shaderStateInfo->shaderError = shaderError; + shaderStateInfo->fetchShader = fetchShader; + shaderStateInfo->vertexShader = vertexShader; + shaderStateInfo->pixelShader = pixelShader; + shaderStateInfo->geometryShader = geometryShader; + shaderStateInfo->lastAccessFrameCount = LatteGPUState.frameCounter; + shaderStateInfo->combinedAuxHash = combinedAuxHash; + if (shaderStateInfo->fetchShader) + shaderStateInfo->fetchShader->m_shaderStateCacheKeys.emplace_back(h); + ++g_shaderStateCacheSetCount; + } + + ShaderStateInfoAuxVariant auxVariant; + auxVariant.combinedAuxHash = combinedAuxHash; + auxVariant.vertexShader = vertexShader; + auxVariant.pixelShader = pixelShader; + auxVariant.geometryShader = geometryShader; + auxVariant.hasError = shaderError; + if (auxVariant.vertexShader) + vectorAppendUnique(auxVariant.vertexShader->m_shaderStateCacheKeys, h); + if (auxVariant.pixelShader) + vectorAppendUnique(auxVariant.pixelShader->m_shaderStateCacheKeys, h); + if (auxVariant.geometryShader) + vectorAppendUnique(auxVariant.geometryShader->m_shaderStateCacheKeys, h); + shaderStateInfo->auxVariants.emplace_back(auxVariant); + ++g_shaderStateCacheSetAuxCount; + + // set shaders as active + _activeFetchShader = shaderStateInfo->fetchShader; + _activeVertexShader = auxVariant.vertexShader; + _activePixelShader = auxVariant.pixelShader; + _activeGeometryShader = auxVariant.geometryShader; } // returns the sampler base index for the given shader type @@ -1105,6 +1329,8 @@ void LatteSHRC_Init() cemu_assert_debug(sVertexShaders.empty()); cemu_assert_debug(sGeometryShaders.empty()); cemu_assert_debug(sPixelShaders.empty()); + cemu_assert_debug(s_shaderStateCache.empty()); + cemu_assert_debug(s_shaderStateCacheKeys.empty()); } void LatteSHRC_UnloadAll() @@ -1118,4 +1344,7 @@ void LatteSHRC_UnloadAll() while(!sPixelShaders.empty()) LatteShader_free(sPixelShaders.begin()->second); cemu_assert_debug(sPixelShaders.empty()); + cemu_assert_debug(s_shaderStateCache.empty()); + s_shaderStateCacheKeys.clear(); + s_shaderStateCacheCleanupIndex = 0; } diff --git a/src/Cafe/HW/Latte/Core/LatteShader.h b/src/Cafe/HW/Latte/Core/LatteShader.h index 85d53b01..3844e9f7 100644 --- a/src/Cafe/HW/Latte/Core/LatteShader.h +++ b/src/Cafe/HW/Latte/Core/LatteShader.h @@ -6,9 +6,10 @@ void LatteSHRC_Init(); void LatteSHRC_UnloadAll(); void LatteSHRC_ResetCachedShaderHash(); -void LatteShaderSHRC_UpdateFetchShader(); +LatteFetchShader* LatteSHRC_GetOrCreateFetchShader(); void LatteSHRC_UpdateActiveShaders(); +void LatteSHRC_CleanupShaderStateCache(); struct LatteFetchShader* LatteSHRC_GetActiveFetchShader(); LatteDecompilerShader* LatteSHRC_GetActiveVertexShader(); diff --git a/src/Cafe/HW/Latte/LegacyShaderDecompiler/LatteDecompiler.h b/src/Cafe/HW/Latte/LegacyShaderDecompiler/LatteDecompiler.h index c95f8098..5e1421ef 100644 --- a/src/Cafe/HW/Latte/LegacyShaderDecompiler/LatteDecompiler.h +++ b/src/Cafe/HW/Latte/LegacyShaderDecompiler/LatteDecompiler.h @@ -216,6 +216,8 @@ struct LatteDecompilerShader }; std::vector list_remappedUniformEntries_register; std::vector<_RemappedUniformBufferGroup> list_remappedUniformEntries_bufferGroups; + // keys in shader state cache + std::vector m_shaderStateCacheKeys; }; struct LatteDecompilerOutputUniformOffsets diff --git a/src/util/containers/LookupTableL3.h b/src/util/containers/LookupTableL3.h index 2d93dc06..836c9095 100644 --- a/src/util/containers/LookupTableL3.h +++ b/src/util/containers/LookupTableL3.h @@ -45,13 +45,12 @@ public: // if no match is found a default-constructed object is returned T lookup(uint32 offset) { + uint32 indexX = (offset >> (TBitsZ + TBitsY)) & ((1u << TBitsX) - 1); + auto& a = m_tableXArr[indexX]; + uint32 indexY = (offset >> TBitsZ) & ((1u << TBitsY) - 1); + auto& b = a->arr[indexY]; uint32 indexZ = offset & ((1u << TBitsZ) - 1); - offset >>= TBitsZ; - uint32 indexY = offset & ((1u << TBitsY) - 1); - offset >>= TBitsY; - uint32 indexX = offset & ((1u << TBitsX) - 1); - //offset >>= TBitsX; - return m_tableXArr[indexX]->arr[indexY]->arr[indexZ]; + return b->arr[indexZ]; } void store(uint32 offset, T& t) @@ -77,6 +76,7 @@ private: TableY* tableY = new TableY(); for (auto& itr : tableY->arr) itr = m_placeholderTableZ; + tYCount++; return tableY; } @@ -84,8 +84,11 @@ private: TableZ* GenerateNewTableZ() { TableZ* tableZ = new TableZ(); + tZCount++; return tableZ; } TableY* m_tableXArr[1 << TBitsX]; // x lookup + int tYCount = 0; + int tZCount = 0; }; diff --git a/src/util/helpers/StateHasher.h b/src/util/helpers/StateHasher.h new file mode 100644 index 00000000..e46a49ea --- /dev/null +++ b/src/util/helpers/StateHasher.h @@ -0,0 +1,42 @@ +#pragma once + +// hashes two separate streams at once for better parallelism, combined into one final uint64 hash at the end +// inspired by xxHash and MurmurHash3 +class DualStateHasher +{ +public: + FORCEINLINE DualStateHasher() + { + m_h0 = 0x9E3779B97F4A7C15; + m_h1 = 0xC2B2AE3D27D4EB4F; + }; + + FORCEINLINE void MixIn(uint64 a, uint64 b) + { + uint64 tmp = m_h1; + m_h1 = (m_h0 ^ a) * 0x85EBCA77C2B2AE63ULL; + m_h0 = (tmp ^ b) * 0x165667B19E3779F9ULL; + } + + FORCEINLINE void MixInSingle(uint64 a) + { + uint64 tmp = m_h1; + m_h1 = (m_h0 ^ a) * 0x85EBCA77C2B2AE63ULL; + m_h0 = tmp; + } + + FORCEINLINE uint64 Finish() + { + uint64 combined = m_h0 ^ std::rotl(m_h1, 31); + combined ^= combined >> 33; + combined *= 0xff51afd7ed558ccdULL; + combined ^= combined >> 33; + combined *= 0xc4ceb9fe1a85ec53ULL; + combined ^= combined >> 33; + return combined; + } + +private: + uint64 m_h0; + uint64 m_h1; +};