From 3f27cb8ff5b428c39f1de57995ab1886dcbc8387 Mon Sep 17 00:00:00 2001 From: Malcolm Date: Sat, 30 May 2026 23:20:34 -0400 Subject: [PATCH] SPU LLVM: Loop large checksum verification --- rpcs3/Emu/Cell/SPULLVMRecompiler.cpp | 179 +++++++++++++++++++++++---- 1 file changed, 158 insertions(+), 21 deletions(-) diff --git a/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp b/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp index bb4aa217af..c459febcdc 100644 --- a/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp +++ b/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp @@ -1846,8 +1846,79 @@ public: // The checksum path is still faster even on narrow hardware. if ((end - starta) >= 192 && !g_cfg.core.precise_spu_verification) { +#ifdef ARCH_ARM64 + constexpr u32 checksum_block_size = 96; +#else + constexpr u32 checksum_block_size = 64; +#endif + constexpr u32 checksum_loop_vectors = 16; + const u32 checksum_vectors_per_block = checksum_block_size / stride; + const u32 checksum_loop_blocks = (checksum_loop_vectors + checksum_vectors_per_block - 1) / checksum_vectors_per_block; + const u32 checksum_loop_size = checksum_block_size * checksum_loop_blocks; + const u32 checksum_loop_end = starta + ((end - starta) / checksum_loop_size) * checksum_loop_size; + + bool use_checksum_loop = (checksum_loop_end - starta) >= checksum_loop_size * 2; + + for (u32 j = starta; use_checksum_loop && j < checksum_loop_end; j += 4) + { + if (!func.data[(j - start) / 4]) + { + use_checksum_loop = false; + } + } + #ifndef ARCH_ARM64 - for (u32 j = starta; j < end; j += 64) + if (use_checksum_loop) + { + const auto acc_init = ConstantAggregateZero::get(get_type()); + const auto loop_block = BasicBlock::Create(m_context, "spu_checksum_loop", m_function); + const auto loop_next = BasicBlock::Create(m_context, "spu_checksum_next", m_function); + const auto loop_preheader = m_ir->GetInsertBlock(); + m_ir->CreateBr(loop_block); + + m_ir->SetInsertPoint(loop_block); + const auto offset = m_ir->CreatePHI(get_type(), 2); + const auto acc0_phi = m_ir->CreatePHI(get_type(), 2); + const auto acc1_phi = m_ir->CreatePHI(get_type(), 2); + + offset->addIncoming(m_ir->getInt32(0), loop_preheader); + acc0_phi->addIncoming(acc_init, loop_preheader); + acc1_phi->addIncoming(acc_init, loop_preheader); + + const auto offset64 = m_ir->CreateZExt(offset, get_type()); + llvm::Value* next_acc0 = acc0_phi; + llvm::Value* next_acc1 = acc1_phi; + + for (u32 block = 0; block < checksum_loop_blocks; block++) + { + const auto vls = m_ir->CreateAlignedLoad(get_type(), _ptr(data_addr, m_ir->CreateAdd(offset64, m_ir->getInt64(block * checksum_block_size))), llvm::MaybeAlign{4}); + + if (block & 1) + { + next_acc1 = m_ir->CreateAdd(next_acc1, vls); + } + else + { + next_acc0 = m_ir->CreateAdd(next_acc0, vls); + } + } + + const auto next_offset = m_ir->CreateAdd(offset, m_ir->getInt32(checksum_loop_size)); + const auto loop_again = m_ir->CreateICmpULT(next_offset, m_ir->getInt32(checksum_loop_end - starta)); + m_ir->CreateCondBr(loop_again, loop_block, loop_next); + + offset->addIncoming(next_offset, loop_block); + acc0_phi->addIncoming(next_acc0, loop_block); + acc1_phi->addIncoming(next_acc1, loop_block); + acc0 = next_acc0; + acc1 = next_acc1; + + check_iterations += (checksum_loop_end - starta) / checksum_block_size; + + m_ir->SetInsertPoint(loop_next); + } + + for (u32 j = use_checksum_loop ? checksum_loop_end : starta; j < end; j += checksum_block_size) { int indices[16]; bool holes = false; @@ -1932,21 +2003,95 @@ public: const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); #else - // - // - // - const auto acc_init = ConstantAggregateZero::get(get_type()); llvm::Value* checksum_parts[4] = {acc_init, acc_init, acc_init, acc_init}; u32 checksum[16] = {0}; - for (u32 j = starta; j < end; j += 64) + const auto update_checksum = [&](const u32* words) { - llvm::Value* vls[4] = {}; - u32 words[16] = {}; - bool any_data = false; + for (u32 i = 0; i < 4; i++) + { + checksum[i] += words[i]; + checksum[4 + i] += words[4 + i] > words[8 + i] ? words[4 + i] - words[8 + i] : words[8 + i] - words[4 + i]; + checksum[8 + i] += words[12 + i]; + checksum[12 + i] += words[16 + i] > words[20 + i] ? words[16 + i] - words[20 + i] : words[20 + i] - words[16 + i]; + } + }; + + if (use_checksum_loop) + { + for (u32 j = starta; j < checksum_loop_end; j += checksum_block_size) + { + u32 words[24]; + + for (u32 i = 0; i < 24; i++) + { + words[i] = func.data[(j + i * 4 - start) / 4]; + } + + update_checksum(words); + } + + const auto loop_block = BasicBlock::Create(m_context, "spu_checksum_loop", m_function); + const auto loop_next = BasicBlock::Create(m_context, "spu_checksum_next", m_function); + const auto loop_preheader = m_ir->GetInsertBlock(); + m_ir->CreateBr(loop_block); + + m_ir->SetInsertPoint(loop_block); + const auto offset = m_ir->CreatePHI(get_type(), 2); + llvm::PHINode* acc_phi[4]; + llvm::Value* next_acc[4]; for (u32 part = 0; part < 4; part++) + { + acc_phi[part] = m_ir->CreatePHI(get_type(), 2); + acc_phi[part]->addIncoming(checksum_parts[part], loop_preheader); + next_acc[part] = acc_phi[part]; + } + + offset->addIncoming(m_ir->getInt32(0), loop_preheader); + + const auto offset64 = m_ir->CreateZExt(offset, get_type()); + + for (u32 block = 0; block < checksum_loop_blocks; block++) + { + llvm::Value* vls[6]; + + for (u32 part = 0; part < 6; part++) + { + vls[part] = m_ir->CreateAlignedLoad(get_type(), _ptr(data_addr, m_ir->CreateAdd(offset64, m_ir->getInt64(block * checksum_block_size + part * 16))), llvm::MaybeAlign{4}); + } + + next_acc[0] = m_ir->CreateAdd(next_acc[0], vls[0]); + next_acc[1] = m_ir->CreateAdd(next_acc[1], m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uabd), {vls[1], vls[2]})); + next_acc[2] = m_ir->CreateAdd(next_acc[2], vls[3]); + next_acc[3] = m_ir->CreateAdd(next_acc[3], m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uabd), {vls[4], vls[5]})); + } + + const auto next_offset = m_ir->CreateAdd(offset, m_ir->getInt32(checksum_loop_size)); + const auto loop_again = m_ir->CreateICmpULT(next_offset, m_ir->getInt32(checksum_loop_end - starta)); + m_ir->CreateCondBr(loop_again, loop_block, loop_next); + + offset->addIncoming(next_offset, loop_block); + + for (u32 part = 0; part < 4; part++) + { + acc_phi[part]->addIncoming(next_acc[part], loop_block); + checksum_parts[part] = next_acc[part]; + } + + check_iterations += (checksum_loop_end - starta) / checksum_block_size; + + m_ir->SetInsertPoint(loop_next); + } + + for (u32 j = use_checksum_loop ? checksum_loop_end : starta; j < end; j += checksum_block_size) + { + llvm::Value* vls[6] = {}; + u32 words[24] = {}; + bool any_data = false; + + for (u32 part = 0; part < 6; part++) { int indices[4]; bool holes = false; @@ -1993,20 +2138,12 @@ public: continue; } - // Keep the checksum logically 512-bit wide while using four NEON vectors. - // The add/uabd pattern is intended to select UABA for the difference lanes. checksum_parts[0] = m_ir->CreateAdd(checksum_parts[0], vls[0]); - checksum_parts[1] = m_ir->CreateAdd(checksum_parts[1], m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uabd), {vls[0], vls[1]})); - checksum_parts[2] = m_ir->CreateAdd(checksum_parts[2], vls[2]); - checksum_parts[3] = m_ir->CreateAdd(checksum_parts[3], m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uabd), {vls[2], vls[3]})); + checksum_parts[1] = m_ir->CreateAdd(checksum_parts[1], m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uabd), {vls[1], vls[2]})); + checksum_parts[2] = m_ir->CreateAdd(checksum_parts[2], vls[3]); + checksum_parts[3] = m_ir->CreateAdd(checksum_parts[3], m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uabd), {vls[4], vls[5]})); - for (u32 i = 0; i < 4; i++) - { - checksum[i] += words[i]; - checksum[4 + i] += words[i] > words[4 + i] ? words[i] - words[4 + i] : words[4 + i] - words[i]; - checksum[8 + i] += words[8 + i]; - checksum[12 + i] += words[8 + i] > words[12 + i] ? words[8 + i] - words[12 + i] : words[12 + i] - words[8 + i]; - } + update_checksum(words); check_iterations++; }