dahjah/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2026-07-09 17:25:18 -06:00
Code Issues Packages Projects Releases Wiki Activity

SPU LLVM: Loop large checksum verification

Browse Source
This commit is contained in:
Malcolm 2026-05-30 23:20:34 -04:00 committed by Elad
parent fa5b899411
commit 3f27cb8ff5
1 changed files with 158 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
rpcs3/Emu/Cell/SPULLVMRecompiler.cpp
View File

@ -1846,8 +1846,79 @@ public:
// The checksum path is still faster even on narrow hardware.
if ((end - starta) >= 192 && !g_cfg.core.precise_spu_verification)
{
#ifdef ARCH_ARM64
constexpr u32 checksum_block_size = 96;
#else
constexpr u32 checksum_block_size = 64;
#endif
constexpr u32 checksum_loop_vectors = 16;
const u32 checksum_vectors_per_block = checksum_block_size / stride;
const u32 checksum_loop_blocks = (checksum_loop_vectors + checksum_vectors_per_block - 1) / checksum_vectors_per_block;
const u32 checksum_loop_size = checksum_block_size * checksum_loop_blocks;
const u32 checksum_loop_end = starta + ((end - starta) / checksum_loop_size) * checksum_loop_size;
bool use_checksum_loop = (checksum_loop_end - starta) >= checksum_loop_size * 2;
for (u32 j = starta; use_checksum_loop && j < checksum_loop_end; j += 4)
{
if (!func.data[(j - start) / 4])
{
use_checksum_loop = false;
}
}
#ifndef ARCH_ARM64
for (u32 j = starta; j < end; j += 64)
if (use_checksum_loop)
{
const auto acc_init = ConstantAggregateZero::get(get_type<u32[16]>());
const auto loop_block = BasicBlock::Create(m_context, "spu_checksum_loop", m_function);
const auto loop_next = BasicBlock::Create(m_context, "spu_checksum_next", m_function);
const auto loop_preheader = m_ir->GetInsertBlock();
m_ir->CreateBr(loop_block);
m_ir->SetInsertPoint(loop_block);
const auto offset = m_ir->CreatePHI(get_type<u32>(), 2);
const auto acc0_phi = m_ir->CreatePHI(get_type<u32[16]>(), 2);
const auto acc1_phi = m_ir->CreatePHI(get_type<u32[16]>(), 2);
offset->addIncoming(m_ir->getInt32(0), loop_preheader);
acc0_phi->addIncoming(acc_init, loop_preheader);
acc1_phi->addIncoming(acc_init, loop_preheader);
const auto offset64 = m_ir->CreateZExt(offset, get_type<u64>());
llvm::Value* next_acc0 = acc0_phi;
llvm::Value* next_acc1 = acc1_phi;
for (u32 block = 0; block < checksum_loop_blocks; block++)
{
const auto vls = m_ir->CreateAlignedLoad(get_type<u32[16]>(), _ptr(data_addr, m_ir->CreateAdd(offset64, m_ir->getInt64(block * checksum_block_size))), llvm::MaybeAlign{4});
if (block & 1)
{
next_acc1 = m_ir->CreateAdd(next_acc1, vls);
}
else
{
next_acc0 = m_ir->CreateAdd(next_acc0, vls);
}
}
const auto next_offset = m_ir->CreateAdd(offset, m_ir->getInt32(checksum_loop_size));
const auto loop_again = m_ir->CreateICmpULT(next_offset, m_ir->getInt32(checksum_loop_end - starta));
m_ir->CreateCondBr(loop_again, loop_block, loop_next);
offset->addIncoming(next_offset, loop_block);
acc0_phi->addIncoming(next_acc0, loop_block);
acc1_phi->addIncoming(next_acc1, loop_block);
acc0 = next_acc0;
acc1 = next_acc1;
check_iterations += (checksum_loop_end - starta) / checksum_block_size;
m_ir->SetInsertPoint(loop_next);
}
for (u32 j = use_checksum_loop ? checksum_loop_end : starta; j < end; j += checksum_block_size)
{
int indices[16];
bool holes = false;
@ -1932,21 +2003,95 @@ public:
const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0));
m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely);
#else
//
//
//
const auto acc_init = ConstantAggregateZero::get(get_type<u32[4]>());
llvm::Value* checksum_parts[4] = {acc_init, acc_init, acc_init, acc_init};
u32 checksum[16] = {0};
for (u32 j = starta; j < end; j += 64)
const auto update_checksum = [&](const u32* words)
{
llvm::Value* vls[4] = {};
u32 words[16] = {};
bool any_data = false;
for (u32 i = 0; i < 4; i++)
{
checksum[i] += words[i];
checksum[4 + i] += words[4 + i] > words[8 + i] ? words[4 + i] - words[8 + i] : words[8 + i] - words[4 + i];
checksum[8 + i] += words[12 + i];
checksum[12 + i] += words[16 + i] > words[20 + i] ? words[16 + i] - words[20 + i] : words[20 + i] - words[16 + i];
}
};
if (use_checksum_loop)
{
for (u32 j = starta; j < checksum_loop_end; j += checksum_block_size)
{
u32 words[24];
for (u32 i = 0; i < 24; i++)
{
words[i] = func.data[(j + i * 4 - start) / 4];
}
update_checksum(words);
}
const auto loop_block = BasicBlock::Create(m_context, "spu_checksum_loop", m_function);
const auto loop_next = BasicBlock::Create(m_context, "spu_checksum_next", m_function);
const auto loop_preheader = m_ir->GetInsertBlock();
m_ir->CreateBr(loop_block);
m_ir->SetInsertPoint(loop_block);
const auto offset = m_ir->CreatePHI(get_type<u32>(), 2);
llvm::PHINode* acc_phi[4];
llvm::Value* next_acc[4];
for (u32 part = 0; part < 4; part++)
{
acc_phi[part] = m_ir->CreatePHI(get_type<u32[4]>(), 2);
acc_phi[part]->addIncoming(checksum_parts[part], loop_preheader);
next_acc[part] = acc_phi[part];
}
offset->addIncoming(m_ir->getInt32(0), loop_preheader);
const auto offset64 = m_ir->CreateZExt(offset, get_type<u64>());
for (u32 block = 0; block < checksum_loop_blocks; block++)
{
llvm::Value* vls[6];
for (u32 part = 0; part < 6; part++)
{
vls[part] = m_ir->CreateAlignedLoad(get_type<u32[4]>(), _ptr(data_addr, m_ir->CreateAdd(offset64, m_ir->getInt64(block * checksum_block_size + part * 16))), llvm::MaybeAlign{4});
}
next_acc[0] = m_ir->CreateAdd(next_acc[0], vls[0]);
next_acc[1] = m_ir->CreateAdd(next_acc[1], m_ir->CreateCall(get_intrinsic<u32[4]>(llvm::Intrinsic::aarch64_neon_uabd), {vls[1], vls[2]}));
next_acc[2] = m_ir->CreateAdd(next_acc[2], vls[3]);
next_acc[3] = m_ir->CreateAdd(next_acc[3], m_ir->CreateCall(get_intrinsic<u32[4]>(llvm::Intrinsic::aarch64_neon_uabd), {vls[4], vls[5]}));
}
const auto next_offset = m_ir->CreateAdd(offset, m_ir->getInt32(checksum_loop_size));
const auto loop_again = m_ir->CreateICmpULT(next_offset, m_ir->getInt32(checksum_loop_end - starta));
m_ir->CreateCondBr(loop_again, loop_block, loop_next);
offset->addIncoming(next_offset, loop_block);
for (u32 part = 0; part < 4; part++)
{
acc_phi[part]->addIncoming(next_acc[part], loop_block);
checksum_parts[part] = next_acc[part];
}
check_iterations += (checksum_loop_end - starta) / checksum_block_size;
m_ir->SetInsertPoint(loop_next);
}
for (u32 j = use_checksum_loop ? checksum_loop_end : starta; j < end; j += checksum_block_size)
{
llvm::Value* vls[6] = {};
u32 words[24] = {};
bool any_data = false;
for (u32 part = 0; part < 6; part++)
{
int indices[4];
bool holes = false;
@ -1993,20 +2138,12 @@ public:
continue;
}
// Keep the checksum logically 512-bit wide while using four NEON vectors.
// The add/uabd pattern is intended to select UABA for the difference lanes.
checksum_parts[0] = m_ir->CreateAdd(checksum_parts[0], vls[0]);
checksum_parts[1] = m_ir->CreateAdd(checksum_parts[1], m_ir->CreateCall(get_intrinsic<u32[4]>(llvm::Intrinsic::aarch64_neon_uabd), {vls[0], vls[1]}));
checksum_parts[2] = m_ir->CreateAdd(checksum_parts[2], vls[2]);
checksum_parts[3] = m_ir->CreateAdd(checksum_parts[3], m_ir->CreateCall(get_intrinsic<u32[4]>(llvm::Intrinsic::aarch64_neon_uabd), {vls[2], vls[3]}));
checksum_parts[1] = m_ir->CreateAdd(checksum_parts[1], m_ir->CreateCall(get_intrinsic<u32[4]>(llvm::Intrinsic::aarch64_neon_uabd), {vls[1], vls[2]}));
checksum_parts[2] = m_ir->CreateAdd(checksum_parts[2], vls[3]);
checksum_parts[3] = m_ir->CreateAdd(checksum_parts[3], m_ir->CreateCall(get_intrinsic<u32[4]>(llvm::Intrinsic::aarch64_neon_uabd), {vls[4], vls[5]}));
for (u32 i = 0; i < 4; i++)
{
checksum[i] += words[i];
checksum[4 + i] += words[i] > words[4 + i] ? words[i] - words[4 + i] : words[4 + i] - words[i];
checksum[8 + i] += words[8 + i];
checksum[12 + i] += words[8 + i] > words[12 + i] ? words[8 + i] - words[12 + i] : words[12 + i] - words[8 + i];
}
update_checksum(words);
check_iterations++;
}