video_core: implement IT_SET_PREDICATION

src/video_core/amdgpu/liverpool.cpp

@@ -65,6 +65,149 @@ static std::span<const u32> NextPacket(std::span<const u32> span, size_t offset)
     return span.subspan(offset);
 }
 
+Liverpool::PredicateReadResult Liverpool::ReadZpassPredicate(VAddr address, bool nowait_draw) {
+    static constexpr u64 OcclusionCounterValidMask = 0x8000000000000000ULL;
+    static constexpr u64 OcclusionCounterValueMask = ~OcclusionCounterValidMask;
+
+    if (address == 0) {
+        return {.value = true, .fail_open = true};
+    }
+
+    const u64 result_size = u64(num_counter_pairs) * sizeof(u64) * 2;
+
+    if (rasterizer) {
+        const bool resolved = rasterizer->ResolvePixelPipeStats(address, result_size, !nowait_draw);
+        if (!resolved) {
+            if (!nowait_draw) {
+                LOG_WARNING(Render, "IT_SET_PREDICATION: ZPASS result was not resolved");
+            }
+            return {.value = true, .fail_open = true};
+        }
+    }
+
+    auto* memory = Core::Memory::Instance();
+    if (!memory->IsValidMapping(address, result_size)) {
+        LOG_WARNING(Render, "IT_SET_PREDICATION: invalid ZPASS address {:#x}", address);
+        // Execute rather than hide draws on an emulator-side mapping failure.
+        return {.value = true, .fail_open = true};
+    }
+
+    const auto* results = reinterpret_cast<const u64*>(address);
+    bool visible = false;
+    bool all_valid = true;
+    for (u32 i = 0; i < num_counter_pairs; ++i) {
+        const u64 begin = results[i * 2];
+        const u64 end = results[i * 2 + 1];
+        const bool begin_valid = (begin & OcclusionCounterValidMask) != 0;
+        const bool end_valid = (end & OcclusionCounterValidMask) != 0;
+        if (!begin_valid || !end_valid) {
+            all_valid = false;
+            continue;
+        }
+
+        const u64 begin_count = begin & OcclusionCounterValueMask;
+        const u64 end_count = end & OcclusionCounterValueMask;
+        visible |= end_count != begin_count;
+    }
+
+    if (!all_valid) {
+        return {.value = true, .fail_open = true};
+    }
+
+    return {.value = visible};
+}
+
+Liverpool::PredicateReadResult Liverpool::ReadBoolPredicate(VAddr address, u32 width_bytes) {
+    if (address == 0) {
+        return {.value = true, .fail_open = true};
+    }
+
+    if (width_bytes != sizeof(u32) && width_bytes != sizeof(u64)) {
+        UNREACHABLE_MSG("Unsupported boolean predication width {}", width_bytes);
+    }
+
+    auto* memory = Core::Memory::Instance();
+    if (!memory->IsValidMapping(address, width_bytes)) {
+        LOG_WARNING(Render, "IT_SET_PREDICATION: invalid BOOL{} address {:#x}", width_bytes * 8,
+                    address);
+        // Execute rather than hide draws on an emulator-side mapping failure.
+        return {.value = true, .fail_open = true};
+    }
+
+    if (width_bytes == sizeof(u32)) {
+        return {.value = *reinterpret_cast<const u32*>(address) != 0};
+    }
+    return {.value = *reinterpret_cast<const u64*>(address) != 0};
+}
+
+bool Liverpool::IsPredicatedPacketEnabled() const {
+    if (!predication.enabled) {
+        return true;
+    }
+
+    bool predicate = true;
+    switch (predication.op) {
+    case PredicationOp::Clear:
+        predicate = true;
+        break;
+    case PredicationOp::Zpass:
+    case PredicationOp::Bool64:
+    case PredicationOp::Bool32:
+        predicate = predication.result;
+        break;
+    case PredicationOp::PrimCount:
+        // PRIMCOUNT predication is not implemented yet. Keep this path fail-open so an
+        // unsupported predicate cannot incorrectly hide draws.
+        predicate = true;
+        break;
+    default:
+        LOG_WARNING(Render, "Unknown IT_SET_PREDICATION op {}",
+                    static_cast<u32>(predication.op));
+        predicate = true;
+        break;
+    }
+
+    return predication.draw_visible ? predicate : !predicate;
+}
+
+bool Liverpool::ShouldExecutePredicatedPacket(const PM4Header* header, bool is_compute) {
+    if (header->type3.predicate != PM4Predicate::PredEnable) {
+        return true;
+    }
+
+    ++predication_stats.predicated_packets_checked;
+    const bool enabled = IsPredicatedPacketEnabled();
+    if (enabled) {
+        ++predication_stats.predicated_packets_executed;
+    } else {
+        ++predication_stats.predicated_packets_skipped;
+    }
+    return enabled;
+}
+
+void Liverpool::LogPredicationSummary(const char* queue_name) {
+    if (!predication_stats.HasWork()) {
+        return;
+    }
+
+    LOG_DEBUG(Render,
+              "PM4 predication [{}]: set(clear={}, zpass={}, bool32={}, bool64={}, primcount={}), "
+              "zpass(visible={}, occluded={}, fail_open={}), bool(true={}, false={}, "
+              "fail_open={}), packets(checked={}, executed={}, skipped={}), pred_exec(executed={}, "
+              "skipped={}), cond_exec(executed={}, skipped={})",
+              queue_name, predication_stats.clear, predication_stats.zpass,
+              predication_stats.bool32, predication_stats.bool64, predication_stats.prim_count,
+              predication_stats.zpass_visible, predication_stats.zpass_occluded,
+              predication_stats.zpass_fail_open, predication_stats.bool_true,
+              predication_stats.bool_false, predication_stats.bool_fail_open,
+              predication_stats.predicated_packets_checked,
+              predication_stats.predicated_packets_executed,
+              predication_stats.predicated_packets_skipped, predication_stats.pred_exec_executed,
+              predication_stats.pred_exec_skipped, predication_stats.cond_exec_executed,
+              predication_stats.cond_exec_skipped);
+    predication_stats = {};
+}
+
 Liverpool::Liverpool() {
     num_counter_pairs = Libraries::Kernel::sceKernelIsNeoMode() ? 16 : 8;
     process_thread = std::jthread{std::bind_front(&Liverpool::Process, this)};
@@ -254,6 +397,18 @@ Liverpool::Task Liverpool::ProcessGraphics(std::span<const u32> dcb, std::span<c
         case 3:
             const u32 count = header->type3.NumWords();
             const PM4ItOpcode opcode = header->type3.opcode;
+
+            if (!ShouldExecutePredicatedPacket(header, false)) {
+                u32 skip_count = count + 1;
+                if (opcode == PM4ItOpcode::PredExec) {
+                    const auto* pred_exec = reinterpret_cast<const PM4CmdPredExec*>(header);
+                    skip_count += pred_exec->exec_count.Value();
+                    ++predication_stats.pred_exec_skipped;
+                }
+                dcb = NextPacket(dcb, skip_count);
+                continue;
+            }
+
             switch (opcode) {
             case PM4ItOpcode::Nop: {
                 const auto* nop = reinterpret_cast<const PM4CmdNop*>(header);
@@ -409,7 +564,77 @@ Liverpool::Task Liverpool::ProcessGraphics(std::span<const u32> dcb, std::span<c
                 break;
             }
             case PM4ItOpcode::SetPredication: {
-                LOG_WARNING(Render, "Unimplemented IT_SET_PREDICATION");
+                const auto* set_pred = reinterpret_cast<const PM4CmdSetPredication*>(header);
+                const auto pred_op = set_pred->pred_op.Value();
+                switch (pred_op) {
+                case PredicationOp::Clear:
+                    ++predication_stats.clear;
+                    predication = {};
+                    break;
+                case PredicationOp::Zpass: {
+                    const bool nowait_draw = set_pred->hint.Value() == PredicationHint::NowaitDraw;
+                    const auto zpass = ReadZpassPredicate(set_pred->Address(), nowait_draw);
+                    ++predication_stats.zpass;
+                    if (zpass.fail_open) {
+                        ++predication_stats.zpass_fail_open;
+                    } else if (zpass.value) {
+                        ++predication_stats.zpass_visible;
+                    } else {
+                        ++predication_stats.zpass_occluded;
+                    }
+                    predication.enabled = true;
+                    predication.op = PredicationOp::Zpass;
+                    predication.address = set_pred->Address();
+                    predication.draw_visible = set_pred->draw_visible.Value() != 0;
+                    predication.nowait_draw = nowait_draw;
+                    predication.result = set_pred->continue_predication.Value() != 0
+                                             ? predication.result || zpass.value
+                                             : zpass.value;
+                    break;
+                }
+                case PredicationOp::Bool64:
+                case PredicationOp::Bool32: {
+                    const bool nowait_draw = set_pred->hint.Value() == PredicationHint::NowaitDraw;
+                    const u32 width = pred_op == PredicationOp::Bool64 ? sizeof(u64) : sizeof(u32);
+                    const auto boolean = ReadBoolPredicate(set_pred->Address(), width);
+                    if (pred_op == PredicationOp::Bool64) {
+                        ++predication_stats.bool64;
+                    } else {
+                        ++predication_stats.bool32;
+                    }
+                    if (boolean.fail_open) {
+                        ++predication_stats.bool_fail_open;
+                    } else if (boolean.value) {
+                        ++predication_stats.bool_true;
+                    } else {
+                        ++predication_stats.bool_false;
+                    }
+                    predication.enabled = true;
+                    predication.op = pred_op;
+                    predication.address = set_pred->Address();
+                    predication.draw_visible = set_pred->draw_visible.Value() != 0;
+                    predication.nowait_draw = nowait_draw;
+                    predication.result = set_pred->continue_predication.Value() != 0
+                                             ? predication.result || boolean.value
+                                             : boolean.value;
+                    break;
+                }
+                case PredicationOp::PrimCount:
+                    ++predication_stats.prim_count;
+                    predication.enabled = true;
+                    predication.op = PredicationOp::PrimCount;
+                    predication.address = set_pred->Address();
+                    predication.draw_visible = set_pred->draw_visible.Value() != 0;
+                    predication.nowait_draw =
+                        set_pred->hint.Value() == PredicationHint::NowaitDraw;
+                    LOG_WARNING(Render, "Unimplemented IT_SET_PREDICATION PRIMCOUNT addr={:#x}",
+                                predication.address);
+                    break;
+                default:
+                    LOG_WARNING(Render, "Unknown IT_SET_PREDICATION op {}",
+                                static_cast<u32>(pred_op));
+                    break;
+                }
                 break;
             }
             case PM4ItOpcode::IndexType: {
@@ -676,15 +901,36 @@ Liverpool::Task Liverpool::ProcessGraphics(std::span<const u32> dcb, std::span<c
                     // TODO: handle proper synchronization, for now signal that update is done
                     // immediately
                     regs.cp_strmout_cntl.offset_update_done = 1;
-                } else if (event->event_index.Value() == EventIndex::ZpassDone) {
-                    if (event->event_type.Value() == EventType::PixelPipeStatDump) {
+                } else if (event->event_type.Value() == EventType::PixelPipeStatReset) {
+                    if (rasterizer) {
+                        rasterizer->ResetPixelPipeStats();
+                    }
+                } else if (event->event_type.Value() == EventType::PixelPipeStatControl) {
+                    if (rasterizer) {
+                        rasterizer->ControlPixelPipeStats();
+                    }
+                } else if (event->event_index.Value() == EventIndex::ZpassDone &&
+                           event->event_type.Value() == EventType::PixelPipeStatDump) {
+                    const VAddr address = event->Address<VAddr>();
+                    if (rasterizer) {
+                        rasterizer->DumpPixelPipeStats(address, num_counter_pairs);
+                    } else {
                         static constexpr u64 OcclusionCounterValidMask = 0x8000000000000000ULL;
-                        static constexpr u64 OcclusionCounterStep = 0x2FFFFFFULL;
-                        u64* results = event->Address<u64*>();
-                        for (s32 i = 0; i < num_counter_pairs; ++i, results += 2) {
-                            *results = pixel_counter | OcclusionCounterValidMask;
+                        const u64 counter = OcclusionCounterValidMask;
+                        auto* memory = Core::Memory::Instance();
+                        const u64 result_size = u64(num_counter_pairs) * sizeof(u64) * 2;
+                        if (!memory->IsValidMapping(address, result_size)) {
+                            LOG_WARNING(Render,
+                                        "Pixel-pipe stats writeback address is invalid: {:#x}",
+                                        address);
+                            break;
+                        }
+                        for (u32 i = 0; i < num_counter_pairs; ++i) {
+                            auto* dst = reinterpret_cast<void*>(address + i * sizeof(u64) * 2);
+                            if (!memory->TryWriteBacking(dst, &counter, sizeof(counter))) {
+                                std::memcpy(dst, &counter, sizeof(counter));
+                            }
                         }
-                        pixel_counter += OcclusionCounterStep;
                     }
                 }
                 break;
@@ -863,17 +1109,39 @@ Liverpool::Task Liverpool::ProcessGraphics(std::span<const u32> dcb, std::span<c
                 LOG_WARNING(Render_Vulkan, "Unimplemented IT_GET_LOD_STATS");
                 break;
             }
+            case PM4ItOpcode::PredExec: {
+                const auto* pred_exec = reinterpret_cast<const PM4CmdPredExec*>(header);
+                if (!IsPredicatedPacketEnabled()) {
+                    ++predication_stats.pred_exec_skipped;
+                    dcb = NextPacket(dcb, header->type3.NumWords() + 1 +
+                                              pred_exec->exec_count.Value());
+                    continue;
+                }
+                ++predication_stats.pred_exec_executed;
+                break;
+            }
             case PM4ItOpcode::CondExec: {
                 const auto* cond_exec = reinterpret_cast<const PM4CmdCondExec*>(header);
                 if (cond_exec->command.Value() != 0) {
                     LOG_WARNING(Render, "IT_COND_EXEC used a reserved command");
                 }
-                const auto skip = *cond_exec->Address() == false;
+
+                const VAddr address = cond_exec->Address();
+                bool skip = false;
+                auto* memory = Core::Memory::Instance();
+                if (address != 0 && memory->IsValidMapping(address, sizeof(u32))) {
+                    skip = *reinterpret_cast<const u32*>(address) == 0;
+                } else {
+                    LOG_WARNING(Render, "IT_COND_EXEC: invalid BOOL32 address {:#x}", address);
+                }
+
                 if (skip) {
+                    ++predication_stats.cond_exec_skipped;
                     dcb = NextPacket(dcb,
                                      header->type3.NumWords() + 1 + cond_exec->exec_count.Value());
                     continue;
                 }
+                ++predication_stats.cond_exec_executed;
                 break;
             }
             default:
@@ -892,6 +1160,7 @@ Liverpool::Task Liverpool::ProcessGraphics(std::span<const u32> dcb, std::span<c
         ce_task.handle.destroy();
     }
 
+    LogPredicationSummary("graphics");
     FIBER_EXIT;
 }
 
@@ -953,6 +1222,20 @@ Liverpool::Task Liverpool::ProcessCompute(std::span<const u32> acb, u32 vqid) {
 
         const PM4ItOpcode opcode = header->type3.opcode;
 
+        if (!ShouldExecutePredicatedPacket(header, true)) {
+            if (opcode == PM4ItOpcode::PredExec) {
+                const auto* pred_exec = reinterpret_cast<const PM4CmdPredExec*>(header);
+                next_dw_off += pred_exec->exec_count.Value();
+                ++predication_stats.pred_exec_skipped;
+            }
+            acb = NextPacket(acb, next_dw_off);
+            if constexpr (!is_indirect) {
+                *queue.read_addr += next_dw_off;
+                *queue.read_addr %= queue.ring_size_dw;
+            }
+            continue;
+        }
+
         const auto* it_body = reinterpret_cast<const u32*>(header) + 1;
         switch (opcode) {
         case PM4ItOpcode::Nop: {
@@ -1051,6 +1334,38 @@ Liverpool::Task Liverpool::ProcessCompute(std::span<const u32> acb, u32 vqid) {
                         set_data->vqid.Value(), set_data->reg_offset.Value());
             break;
         }
+        case PM4ItOpcode::PredExec: {
+            const auto* pred_exec = reinterpret_cast<const PM4CmdPredExec*>(header);
+            if (!IsPredicatedPacketEnabled()) {
+                next_dw_off += pred_exec->exec_count.Value();
+                ++predication_stats.pred_exec_skipped;
+            } else {
+                ++predication_stats.pred_exec_executed;
+            }
+            break;
+        }
+        case PM4ItOpcode::CondExec: {
+            const auto* cond_exec = reinterpret_cast<const PM4CmdCondExec*>(header);
+            if (cond_exec->command.Value() != 0) {
+                LOG_WARNING(Render, "Compute IT_COND_EXEC used a reserved command");
+            }
+
+            const VAddr address = cond_exec->Address();
+            bool skip = false;
+            auto* memory = Core::Memory::Instance();
+            if (address != 0 && memory->IsValidMapping(address, sizeof(u32))) {
+                skip = *reinterpret_cast<const u32*>(address) == 0;
+            } else {
+                LOG_WARNING(Render, "Compute IT_COND_EXEC: invalid BOOL32 address {:#x}", address);
+            }
+            if (skip) {
+                next_dw_off += cond_exec->exec_count.Value();
+                ++predication_stats.cond_exec_skipped;
+            } else {
+                ++predication_stats.cond_exec_executed;
+            }
+            break;
+        }
         case PM4ItOpcode::DispatchDirect: {
             const auto* dispatch_direct = reinterpret_cast<const PM4CmdDispatchDirect*>(header);
             if (auto it = std::ranges::find(indirect_patches, header, &IndirectPatch::header);
@@ -1163,6 +1478,7 @@ Liverpool::Task Liverpool::ProcessCompute(std::span<const u32> acb, u32 vqid) {
         }
     }
 
+    LogPredicationSummary(is_indirect ? "compute_ib" : "compute");
     FIBER_EXIT;
 }
 

src/video_core/amdgpu/liverpool.h

@@ -30,6 +30,9 @@ struct VideoOutPort;
 
 namespace AmdGpu {
 
+enum class PredicationOp : u32;
+union PM4Header;
+
 struct Liverpool {
     static constexpr u32 GfxQueueId = 0u;
     static constexpr u32 NumGfxRings = 1u;     // actually 2, but HP is reserved by system software
@@ -186,6 +189,17 @@ private:
     void ProcessCommands();
     void Process(std::stop_token stoken);
 
+    struct PredicateReadResult {
+        bool value{};
+        bool fail_open{};
+    };
+
+    bool IsPredicatedPacketEnabled() const;
+    bool ShouldExecutePredicatedPacket(const PM4Header* header, bool is_compute);
+    PredicateReadResult ReadZpassPredicate(VAddr address, bool nowait_draw);
+    PredicateReadResult ReadBoolPredicate(VAddr address, u32 width_bytes);
+    void LogPredicationSummary(const char* queue_name);
+
     struct GpuQueue {
         std::mutex m_access{};
         std::atomic<u32> dcb_buffer_offset;
@@ -200,7 +214,41 @@ private:
 
     VAddr indirect_args_addr{};
     u32 num_counter_pairs{};
-    u64 pixel_counter{};
+
+    struct PredicationState {
+        bool enabled{};
+        PredicationOp op{};
+        VAddr address{};
+        bool draw_visible{};
+        bool nowait_draw{};
+        bool result{};
+    } predication{};
+
+    struct PredicationStats {
+        u64 clear{};
+        u64 zpass{};
+        u64 bool32{};
+        u64 bool64{};
+        u64 prim_count{};
+        u64 zpass_visible{};
+        u64 zpass_occluded{};
+        u64 zpass_fail_open{};
+        u64 bool_true{};
+        u64 bool_false{};
+        u64 bool_fail_open{};
+        u64 predicated_packets_checked{};
+        u64 predicated_packets_executed{};
+        u64 predicated_packets_skipped{};
+        u64 pred_exec_executed{};
+        u64 pred_exec_skipped{};
+        u64 cond_exec_executed{};
+        u64 cond_exec_skipped{};
+        [[nodiscard]] bool HasWork() const {
+            return clear != 0 || zpass != 0 || bool32 != 0 || bool64 != 0 || prim_count != 0 ||
+                   predicated_packets_checked != 0 || pred_exec_executed != 0 ||
+                   pred_exec_skipped != 0 || cond_exec_executed != 0 || cond_exec_skipped != 0;
+        }
+    } predication_stats{};
 
     struct ConstantEngine {
         void Reset() {

src/video_core/amdgpu/pm4_cmds.h

@@ -1215,9 +1215,47 @@ struct PM4CmdCondExec {
                                          ///< if bool pointed to is zero
     };
 
-    bool* Address() const {
-        return std::bit_cast<bool*>(u64(bool_addr_hi.Value()) << 32 | u64(bool_addr_lo.Value())
-                                                                          << 2);
+    VAddr Address() const {
+        return (u64(bool_addr_hi.Value()) << 32) | (u64(bool_addr_lo.Value()) << 2);
+    }
+};
+
+struct PM4CmdPredExec {
+    PM4Type3Header header;
+    union {
+        u32 control;
+        BitField<0, 14, u32> exec_count; ///< Number of following DWords controlled by predicate.
+        BitField<24, 8, u32> device_select;
+    };
+};
+
+enum class PredicationOp : u32 {
+    Clear = 0,
+    Zpass = 1,
+    PrimCount = 2,
+    Bool64 = 3,
+    Bool32 = 4,
+};
+
+enum class PredicationHint : u32 {
+    Wait = 0,
+    NowaitDraw = 1,
+};
+
+struct PM4CmdSetPredication {
+    PM4Type3Header header;
+    u32 start_addr_lo;
+    union {
+        u32 pred_properties;
+        BitField<0, 8, u32> start_addr_hi;
+        BitField<8, 1, u32> draw_visible;
+        BitField<12, 2, PredicationHint> hint;
+        BitField<16, 3, PredicationOp> pred_op;
+        BitField<31, 1, u32> continue_predication;
+    };
+
+    VAddr Address() const {
+        return (u64(start_addr_hi.Value()) << 32) | (u64(start_addr_lo) & 0xfffffff0ULL);
     }
 };
 

src/video_core/renderer_vulkan/vk_instance.h

@@ -89,6 +89,11 @@ public:
         return features.depthBounds;
     }
 
+    /// Returns true if query pools can be reset from the host.
+    bool IsHostQueryResetSupported() const {
+        return vk12_features.hostQueryReset;
+    }
+
     /// Returns true if 16-bit floats are supported in shaders
     bool IsShaderFloat16Supported() const {
         return vk12_features.shaderFloat16;

src/video_core/renderer_vulkan/vk_rasterizer.cpp

@@ -1,6 +1,8 @@
 // SPDX-FileCopyrightText: Copyright 2024-2026 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include <algorithm>
+
 #include "common/debug.h"
 #include "core/emulator_settings.h"
 #include "core/memory.h"
@@ -38,6 +40,13 @@ Rasterizer::Rasterizer(const Instance& instance_, Scheduler& scheduler_,
       texture_cache{instance, scheduler, liverpool_, buffer_cache, page_manager},
       liverpool{liverpool_}, memory{Core::Memory::Instance()},
       pipeline_cache{instance, scheduler, liverpool} {
+    const vk::QueryPoolCreateInfo query_pool_info = {
+        .queryType = vk::QueryType::eOcclusion,
+        .queryCount = PixelPipeQueryPoolSize,
+    };
+    pixel_pipe_query_pool = Check<"create pixel-pipe occlusion query pool">(
+        instance.GetDevice().createQueryPoolUnique(query_pool_info));
+
     if (!EmulatorSettings.IsNullGPU()) {
         liverpool->BindRasterizer(this);
     }
@@ -59,6 +68,285 @@ void Rasterizer::CpSync() {
                            vk::DependencyFlagBits::eByRegion, ib_barrier, {}, {});
 }
 
+std::optional<u32> Rasterizer::AcquirePixelPipeQuery() {
+    CollectRetiredPixelPipeQueries(false);
+
+    for (u32 attempt = 0; attempt < PixelPipeQueryPoolSize; ++attempt) {
+        const u32 query = pixel_pipe_query_cursor++ % PixelPipeQueryPoolSize;
+        if (!pixel_pipe_query_in_use[query]) {
+            pixel_pipe_query_in_use[query] = true;
+            return query;
+        }
+    }
+
+    CollectActivePixelPipeQueries(true);
+    ResolvePendingPixelPipeDumps(true, std::nullopt, 0);
+    CollectRetiredPixelPipeQueries(true);
+
+    for (u32 attempt = 0; attempt < PixelPipeQueryPoolSize; ++attempt) {
+        const u32 query = pixel_pipe_query_cursor++ % PixelPipeQueryPoolSize;
+        if (!pixel_pipe_query_in_use[query]) {
+            pixel_pipe_query_in_use[query] = true;
+            return query;
+        }
+    }
+
+    LOG_WARNING(Render_Vulkan,
+                "Pixel-pipe occlusion query pool exhausted; forcing visible sample");
+    ++pixel_pipe_sample_counter;
+    return std::nullopt;
+}
+
+std::optional<u32> Rasterizer::BeginPixelPipeQuery() {
+    if (!pixel_pipe_stats_enabled || !pixel_pipe_query_pool) {
+        return std::nullopt;
+    }
+
+    const auto query = AcquirePixelPipeQuery();
+    if (!query) {
+        return std::nullopt;
+    }
+
+    if (instance.IsHostQueryResetSupported()) {
+        // vkCmdResetQueryPool is invalid while dynamic rendering is active. The rasterizer keeps
+        // rendering open across consecutive draws, so reset query slots on the host when supported.
+        instance.GetDevice().resetQueryPool(*pixel_pipe_query_pool, *query, 1);
+    } else {
+        scheduler.EndRendering();
+        scheduler.CommandBuffer().resetQueryPool(*pixel_pipe_query_pool, *query, 1);
+    }
+    return query;
+}
+
+void Rasterizer::EndPixelPipeQuery(std::optional<u32> query) {
+    if (!query) {
+        return;
+    }
+    scheduler.CommandBuffer().endQuery(*pixel_pipe_query_pool, *query);
+    active_pixel_pipe_queries.emplace_back(*query);
+}
+
+void Rasterizer::CollectActivePixelPipeQueries(bool wait) {
+    if (active_pixel_pipe_queries.empty()) {
+        return;
+    }
+
+    scheduler.PopPendingOperations();
+    if (wait) {
+        scheduler.Finish();
+    }
+
+    auto device = instance.GetDevice();
+    for (auto it = active_pixel_pipe_queries.begin(); it != active_pixel_pipe_queries.end();) {
+        u64 result{};
+        const auto vk_result = device.getQueryPoolResults(
+            *pixel_pipe_query_pool, *it, 1, sizeof(result), &result, sizeof(result),
+            vk::QueryResultFlagBits::e64);
+        if (vk_result == vk::Result::eSuccess) {
+            pixel_pipe_sample_counter += result;
+            pixel_pipe_query_in_use[*it] = false;
+            it = active_pixel_pipe_queries.erase(it);
+            continue;
+        }
+        if (vk_result == vk::Result::eNotReady && !wait) {
+            ++it;
+            continue;
+        }
+        LOG_WARNING(Render_Vulkan, "Failed to read active pixel-pipe query: {}",
+                    vk::to_string(vk_result));
+        pixel_pipe_query_in_use[*it] = false;
+        it = active_pixel_pipe_queries.erase(it);
+    }
+}
+
+void Rasterizer::CollectRetiredPixelPipeQueries(bool wait) {
+    if (retired_pixel_pipe_queries.empty()) {
+        return;
+    }
+
+    scheduler.PopPendingOperations();
+    if (wait) {
+        scheduler.Finish();
+    }
+
+    auto device = instance.GetDevice();
+    for (auto it = retired_pixel_pipe_queries.begin(); it != retired_pixel_pipe_queries.end();) {
+        u64 result{};
+        const auto vk_result = device.getQueryPoolResults(
+            *pixel_pipe_query_pool, *it, 1, sizeof(result), &result, sizeof(result),
+            vk::QueryResultFlagBits::e64);
+        if (vk_result == vk::Result::eSuccess) {
+            pixel_pipe_query_in_use[*it] = false;
+            it = retired_pixel_pipe_queries.erase(it);
+            continue;
+        }
+        if (vk_result == vk::Result::eNotReady) {
+            ++it;
+            continue;
+        }
+        LOG_WARNING(Render_Vulkan, "Failed to retire pixel-pipe query: {}",
+                    vk::to_string(vk_result));
+        pixel_pipe_query_in_use[*it] = false;
+        it = retired_pixel_pipe_queries.erase(it);
+    }
+}
+
+bool Rasterizer::TryResolvePixelPipeDump(PixelPipeStatsDump& dump) {
+    if (dump.resolved) {
+        return true;
+    }
+
+    auto device = instance.GetDevice();
+    u64 resolved_samples = 0;
+    for (auto it = dump.queries.begin(); it != dump.queries.end();) {
+        u64 result{};
+        const auto vk_result = device.getQueryPoolResults(
+            *pixel_pipe_query_pool, *it, 1, sizeof(result), &result, sizeof(result),
+            vk::QueryResultFlagBits::e64);
+        if (vk_result == vk::Result::eSuccess) {
+            resolved_samples += result;
+            pixel_pipe_query_in_use[*it] = false;
+            it = dump.queries.erase(it);
+            continue;
+        }
+        if (vk_result == vk::Result::eNotReady) {
+            ++it;
+            continue;
+        }
+        LOG_WARNING(Render_Vulkan, "Failed to read pixel-pipe dump query: {}",
+                    vk::to_string(vk_result));
+        pixel_pipe_query_in_use[*it] = false;
+        it = dump.queries.erase(it);
+    }
+
+    dump.base_samples += resolved_samples;
+    if (!dump.queries.empty()) {
+        return false;
+    }
+
+    // If this dump was queued behind an older unresolved dump, its baseline is the
+    // cumulative counter at the time the older dumps resolve.  The samples produced
+    // by this dump's own queries must still be added on top of that deferred baseline.
+    const u64 samples = dump.defer_base_samples
+                            ? pixel_pipe_sample_counter + dump.base_samples
+                            : dump.base_samples;
+    pixel_pipe_sample_counter = std::max(pixel_pipe_sample_counter, samples);
+    WritePixelPipeStats(dump.address, dump.num_counter_pairs, samples);
+    dump.resolved = true;
+    return true;
+}
+
+bool Rasterizer::ResolvePendingPixelPipeDumps(bool wait, std::optional<VAddr> address, u64 size) {
+    scheduler.PopPendingOperations();
+
+    const VAddr end = address ? *address + size : 0;
+    const auto matches = [&](const PixelPipeStatsDump& dump) {
+        return !address || (dump.address >= *address && dump.address < end);
+    };
+
+    auto erase_resolved = [&] {
+        std::erase_if(pending_pixel_pipe_dumps, [](const PixelPipeStatsDump& dump) {
+            return dump.resolved;
+        });
+    };
+
+    auto resolve_until = [&](size_t count) {
+        bool blocked = false;
+        for (size_t i = 0; i < count && i < pending_pixel_pipe_dumps.size(); ++i) {
+            if (!TryResolvePixelPipeDump(pending_pixel_pipe_dumps[i])) {
+                blocked = true;
+                break;
+            }
+        }
+        erase_resolved();
+        return !blocked;
+    };
+
+    size_t required_count = pending_pixel_pipe_dumps.size();
+    if (address) {
+        required_count = 0;
+        for (size_t i = 0; i < pending_pixel_pipe_dumps.size(); ++i) {
+            if (matches(pending_pixel_pipe_dumps[i])) {
+                required_count = i + 1;
+            }
+        }
+    }
+
+    if (required_count == 0) {
+        return true;
+    }
+
+    if (resolve_until(required_count)) {
+        return true;
+    }
+    if (!wait) {
+        return false;
+    }
+
+    scheduler.Finish();
+    // A blocking wait means everything submitted so far is complete. Resolve the whole pending
+    // list in order so consecutive SET_PREDICATION packets do not perform one CPU/GPU sync per
+    // object.
+    return resolve_until(pending_pixel_pipe_dumps.size());
+}
+
+void Rasterizer::WritePixelPipeStats(VAddr address, u32 num_counter_pairs, u64 samples) {
+    static constexpr u64 OcclusionCounterValidMask = 0x8000000000000000ULL;
+    const u64 counter = samples | OcclusionCounterValidMask;
+    auto* memory = Core::Memory::Instance();
+    const u64 result_size = u64(num_counter_pairs) * sizeof(u64) * 2;
+    if (!memory->IsValidMapping(address, result_size)) {
+        LOG_WARNING(Render_Vulkan, "Pixel-pipe stats writeback address is invalid: {:#x}",
+                    address);
+        return;
+    }
+
+    for (u32 i = 0; i < num_counter_pairs; ++i) {
+        auto* dst = reinterpret_cast<void*>(address + i * sizeof(u64) * 2);
+        if (!memory->TryWriteBacking(dst, &counter, sizeof(counter))) {
+            std::memcpy(dst, &counter, sizeof(counter));
+        }
+    }
+}
+
+void Rasterizer::ControlPixelPipeStats() {
+    pixel_pipe_stats_enabled = !pixel_pipe_stats_enabled;
+}
+
+void Rasterizer::ResetPixelPipeStats() {
+    ResolvePendingPixelPipeDumps(true, std::nullopt, 0);
+    retired_pixel_pipe_queries.insert(retired_pixel_pipe_queries.end(),
+                                      active_pixel_pipe_queries.begin(),
+                                      active_pixel_pipe_queries.end());
+    active_pixel_pipe_queries.clear();
+    pixel_pipe_sample_counter = 0;
+}
+
+void Rasterizer::DumpPixelPipeStats(VAddr address, u32 num_counter_pairs) {
+    if (address == 0 || num_counter_pairs == 0) {
+        return;
+    }
+
+    if (active_pixel_pipe_queries.empty() && pending_pixel_pipe_dumps.empty()) {
+        WritePixelPipeStats(address, num_counter_pairs, pixel_pipe_sample_counter);
+        return;
+    }
+
+    const bool defer_base_samples = !pending_pixel_pipe_dumps.empty();
+
+    auto& dump = pending_pixel_pipe_dumps.emplace_back();
+    dump.address = address;
+    dump.num_counter_pairs = num_counter_pairs;
+    dump.defer_base_samples = defer_base_samples;
+    dump.base_samples = defer_base_samples ? 0 : pixel_pipe_sample_counter;
+    dump.queries = std::move(active_pixel_pipe_queries);
+    active_pixel_pipe_queries.clear();
+}
+
+bool Rasterizer::ResolvePixelPipeStats(VAddr address, u64 size, bool wait) {
+    return ResolvePendingPixelPipeDumps(wait, address, size);
+}
+
 bool Rasterizer::FilterDraw() {
     const auto& regs = liverpool->regs;
     if (regs.color_control.mode == AmdGpu::ColorControl::OperationMode::EliminateFastClear) {
@@ -213,6 +501,7 @@ void Rasterizer::Draw(bool is_indexed, u32 index_offset) {
 
     pipeline->BindResources(set_writes, buffer_barriers, push_data);
     UpdateDynamicState(pipeline, is_indexed);
+    const auto pixel_pipe_query = BeginPixelPipeQuery();
     scheduler.BeginRendering(state);
 
     const auto& vs_info = pipeline->GetStage(Shader::LogicalStage::Vertex);
@@ -221,6 +510,9 @@ void Rasterizer::Draw(bool is_indexed, u32 index_offset) {
 
     const auto cmdbuf = scheduler.CommandBuffer();
     cmdbuf.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline->Handle());
+    if (pixel_pipe_query) {
+        cmdbuf.beginQuery(*pixel_pipe_query_pool, *pixel_pipe_query, {});
+    }
 
     if (is_indexed) {
         cmdbuf.drawIndexed(regs.num_indices, regs.num_instances.NumInstances(), 0,
@@ -230,6 +522,7 @@ void Rasterizer::Draw(bool is_indexed, u32 index_offset) {
                     instance_offset);
     }
 
+    EndPixelPipeQuery(pixel_pipe_query);
     ResetBindings();
 }
 
@@ -281,6 +574,7 @@ void Rasterizer::DrawIndirect(bool is_indexed, VAddr arg_address, u32 offset, u3
 
     pipeline->BindResources(set_writes, buffer_barriers, push_data);
     UpdateDynamicState(pipeline, is_indexed);
+    const auto pixel_pipe_query = BeginPixelPipeQuery();
     scheduler.BeginRendering(state);
 
     // We can safely ignore both SGPR UD indices and results of fetch shader parsing, as vertex and
@@ -288,6 +582,9 @@ void Rasterizer::DrawIndirect(bool is_indexed, VAddr arg_address, u32 offset, u3
 
     const auto cmdbuf = scheduler.CommandBuffer();
     cmdbuf.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline->Handle());
+    if (pixel_pipe_query) {
+        cmdbuf.beginQuery(*pixel_pipe_query_pool, *pixel_pipe_query, {});
+    }
 
     if (is_indexed) {
         ASSERT(sizeof(VkDrawIndexedIndirectCommand) == stride);
@@ -309,6 +606,7 @@ void Rasterizer::DrawIndirect(bool is_indexed, VAddr arg_address, u32 offset, u3
         }
     }
 
+    EndPixelPipeQuery(pixel_pipe_query);
     ResetBindings();
 }
 

src/video_core/renderer_vulkan/vk_rasterizer.h

@@ -3,6 +3,11 @@
 
 #pragma once
 
+#include <array>
+#include <deque>
+#include <optional>
+#include <vector>
+
 #include "common/recursive_lock.h"
 #include "common/shared_first_mutex.h"
 #include "video_core/buffer_cache/buffer_cache.h"
@@ -65,6 +70,10 @@ public:
     void UnmapMemory(VAddr addr, u64 size);
 
     void CpSync();
+    void ControlPixelPipeStats();
+    void ResetPixelPipeStats();
+    void DumpPixelPipeStats(VAddr address, u32 num_counter_pairs);
+    bool ResolvePixelPipeStats(VAddr address, u64 size, bool wait);
     u64 Flush();
     void Finish();
     void OnSubmit();
@@ -83,8 +92,18 @@ public:
     }
 
 private:
+    struct PixelPipeStatsDump;
+
     void PrepareRenderState(const GraphicsPipeline* pipeline);
     RenderState BeginRendering(const GraphicsPipeline* pipeline);
+    std::optional<u32> BeginPixelPipeQuery();
+    void EndPixelPipeQuery(std::optional<u32> query);
+    std::optional<u32> AcquirePixelPipeQuery();
+    void CollectActivePixelPipeQueries(bool wait);
+    void CollectRetiredPixelPipeQueries(bool wait);
+    bool TryResolvePixelPipeDump(PixelPipeStatsDump& dump);
+    bool ResolvePendingPixelPipeDumps(bool wait, std::optional<VAddr> address, u64 size);
+    void WritePixelPipeStats(VAddr address, u32 num_counter_pairs, u64 samples);
     void Resolve();
     void DepthStencilCopy(bool is_depth, bool is_stencil);
     void EliminateFastClear();
@@ -146,6 +165,24 @@ private:
     boost::container::static_vector<ImageBindingInfo, Shader::NUM_IMAGES> image_bindings;
     bool fault_process_pending{};
     bool attachment_feedback_loop{};
+
+    static constexpr u32 PixelPipeQueryPoolSize = 4096;
+    vk::UniqueQueryPool pixel_pipe_query_pool;
+    std::array<bool, PixelPipeQueryPoolSize> pixel_pipe_query_in_use{};
+    u32 pixel_pipe_query_cursor{};
+    bool pixel_pipe_stats_enabled{};
+    u64 pixel_pipe_sample_counter{};
+    std::vector<u32> active_pixel_pipe_queries;
+    std::vector<u32> retired_pixel_pipe_queries;
+    struct PixelPipeStatsDump {
+        VAddr address{};
+        u32 num_counter_pairs{};
+        u64 base_samples{};
+        bool defer_base_samples{};
+        bool resolved{};
+        std::vector<u32> queries;
+    };
+    std::deque<PixelPipeStatsDump> pending_pixel_pipe_dumps;
 };
 
 } // namespace Vulkan