dahjah/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2026-07-09 17:25:18 -06:00
Code Issues Packages Projects Releases Wiki Activity

SPU: Use multiply-accumulate comparisons for cmp_rdata and verification

Browse Source
This commit is contained in:
Malcolm 2026-05-23 21:30:17 -04:00 committed by Elad
parent 537ad399bb
commit ed145540a2
2 changed files with 160 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
rpcs3/Emu/Cell/SPULLVMRecompiler.cpp
View File

@ -1929,6 +1929,144 @@ public:
const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0));
m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely);
}
#ifdef ARCH_ARM64
else
{
const auto acc_init = m_use_dotprod ? ConstantAggregateZero::get(get_type<u32[4]>()) : ConstantAggregateZero::get(get_type<s16[8]>());
llvm::Value* acc0 = acc_init;
llvm::Value* acc1 = acc_init;
llvm::Value* acc2 = acc_init;
llvm::Value* acc3 = acc_init;
llvm::Value** accs[4] = {&acc0, &acc1, &acc2, &acc3};
u32 acc_index = 0;
llvm::Value* pending_cmp = nullptr;
u32 expected_hits = 0;
const auto make_cmp = [&](u32 j) -> llvm::Value*
{
int indices[4];
bool holes = false;
bool data = false;
for (u32 i = 0; i < 4; i++)
{
const u32 k = j + i * 4;
if (k < start || k >= end || !func.data[(k - start) / 4])
{
indices[i] = 4;
holes = true;
}
else
{
indices[i] = i;
data = true;
}
}
if (!data)
{
return nullptr;
}
llvm::Value* vls = m_ir->CreateAlignedLoad(get_type<u32[4]>(), _ptr(data_addr, j - starta), llvm::MaybeAlign{4});
if (holes)
{
vls = m_ir->CreateShuffleVector(vls, ConstantAggregateZero::get(vls->getType()), llvm::ArrayRef(indices, 4));
}
u32 words[4];
for (u32 i = 0; i < 4; i++)
{
const u32 k = j + i * 4;
words[i] = k >= start && k < end ? func.data[(k - start) / 4] : 0;
}
const auto expected = ConstantDataVector::get(m_context, llvm::ArrayRef(words, 4));
if (m_use_dotprod)
{
return m_ir->CreateSExt(m_ir->CreateICmpEQ(
m_ir->CreateBitCast(vls, get_type<u8[16]>()),
m_ir->CreateBitCast(expected, get_type<u8[16]>())), get_type<s8[16]>());
}
return m_ir->CreateSExt(m_ir->CreateICmpEQ(
m_ir->CreateBitCast(vls, get_type<u16[8]>()),
m_ir->CreateBitCast(expected, get_type<u16[8]>())), get_type<s16[8]>());
};
const auto accumulate_pair = [&](llvm::Value* lhs, llvm::Value* rhs)
{
llvm::Value*& acc = *accs[acc_index];
if (m_use_dotprod)
{
acc = m_ir->CreateCall(get_intrinsic<u32[4], u8[16]>(llvm::Intrinsic::aarch64_neon_udot), {acc, lhs, rhs});
}
else
{
acc = m_ir->CreateAdd(acc, m_ir->CreateMul(lhs, rhs));
}
acc_index = (acc_index + 1) & 3;
expected_hits++;
};
for (u32 j = starta; j < end; j += 16)
{
if (const auto cmp = make_cmp(j))
{
if (pending_cmp)
{
accumulate_pair(pending_cmp, cmp);
pending_cmp = nullptr;
}
else
{
pending_cmp = cmp;
}
check_iterations++;
}
}
if (pending_cmp)
{
accumulate_pair(pending_cmp, llvm::ConstantInt::get(pending_cmp->getType(), -1, true));
}
llvm::Value* expected = nullptr;
if (m_use_dotprod)
{
u32 expected_words[4];
expected_words[0] = expected_hits * 4 * 0xff * 0xff;
expected_words[1] = expected_words[0];
expected_words[2] = expected_words[0];
expected_words[3] = expected_words[0];
expected = ConstantDataVector::get(m_context, llvm::ArrayRef(expected_words, 4));
}
else
{
u16 expected_words[8];
std::fill_n(expected_words, 8, static_cast<u16>(expected_hits));
expected = ConstantDataVector::get(m_context, llvm::ArrayRef(expected_words, 8));
}
llvm::Value* acc = m_ir->CreateAdd(m_ir->CreateAdd(acc0, acc1), m_ir->CreateAdd(acc2, acc3));
acc = m_ir->CreateXor(acc, expected);
acc = m_ir->CreateBitCast(acc, get_type<u64[2]>());
llvm::Value* elem = m_ir->CreateExtractElement(acc, u64{0});
elem = m_ir->CreateOr(elem, m_ir->CreateExtractElement(acc, u64{1}));
const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0));
m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely);
}
#else
else
{
for (u32 j = starta; j < end; j += stride)
@ -2031,6 +2169,7 @@ public:
const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0));
m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely);
}
#endif
}
// Increase block counter with statistics

21
rpcs3/Emu/Cell/SPUThread.cpp
View File

@ -245,6 +245,17 @@ static FORCE_INLINE bool cmp_rdata_avx(const __m256i* lhs, const __m256i* rhs)
}
#endif
#if defined(ARCH_ARM64)
static FORCE_INLINE int16x8_t cmp16_pair_accum_arm64(
int16x8_t acc, const v128& lhs0, const v128& rhs0, const v128& lhs1, const v128& rhs1)
{
const int16x8_t eq0 = vreinterpretq_s16_u16(vceqq_u16(static_cast<uint16x8_t>(lhs0), static_cast<uint16x8_t>(rhs0)));
const int16x8_t eq1 = vreinterpretq_s16_u16(vceqq_u16(static_cast<uint16x8_t>(lhs1), static_cast<uint16x8_t>(rhs1)));
return vmlaq_s16(acc, eq0, eq1);
}
#endif
#ifdef _MSC_VER
__forceinline
#endif
@ -261,12 +272,22 @@ extern bool cmp_rdata(const spu_rdata_t& _lhs, const spu_rdata_t& _rhs)
const auto lhs = reinterpret_cast<const v128*>(_lhs);
const auto rhs = reinterpret_cast<const v128*>(_rhs);
#if defined(ARCH_ARM64)
int16x8_t hits = vdupq_n_s16(0);
hits = cmp16_pair_accum_arm64(hits, lhs[0], rhs[0], lhs[1], rhs[1]);
hits = cmp16_pair_accum_arm64(hits, lhs[2], rhs[2], lhs[3], rhs[3]);
hits = cmp16_pair_accum_arm64(hits, lhs[4], rhs[4], lhs[5], rhs[5]);
hits = cmp16_pair_accum_arm64(hits, lhs[6], rhs[6], lhs[7], rhs[7]);
return vaddvq_s16(hits) == 32;
#else
const v128 a = (lhs[0] ^ rhs[0]) | (lhs[1] ^ rhs[1]);
const v128 c = (lhs[4] ^ rhs[4]) | (lhs[5] ^ rhs[5]);
const v128 b = (lhs[2] ^ rhs[2]) | (lhs[3] ^ rhs[3]);
const v128 d = (lhs[6] ^ rhs[6]) | (lhs[7] ^ rhs[7]);
const v128 r = (a | b) | (c | d);
return gv_testz(r);
#endif
}
#if defined(ARCH_X64)