From ed145540a2c7f1578b7f50ea10f9610f100cc320 Mon Sep 17 00:00:00 2001 From: Malcolm Date: Sat, 23 May 2026 21:30:17 -0400 Subject: [PATCH] SPU: Use multiply-accumulate comparisons for cmp_rdata and verification --- rpcs3/Emu/Cell/SPULLVMRecompiler.cpp | 139 +++++++++++++++++++++++++++ rpcs3/Emu/Cell/SPUThread.cpp | 21 ++++ 2 files changed, 160 insertions(+) diff --git a/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp b/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp index 689233e0b3..a417246589 100644 --- a/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp +++ b/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp @@ -1929,6 +1929,144 @@ public: const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); } +#ifdef ARCH_ARM64 + else + { + const auto acc_init = m_use_dotprod ? ConstantAggregateZero::get(get_type()) : ConstantAggregateZero::get(get_type()); + llvm::Value* acc0 = acc_init; + llvm::Value* acc1 = acc_init; + llvm::Value* acc2 = acc_init; + llvm::Value* acc3 = acc_init; + llvm::Value** accs[4] = {&acc0, &acc1, &acc2, &acc3}; + u32 acc_index = 0; + llvm::Value* pending_cmp = nullptr; + u32 expected_hits = 0; + + const auto make_cmp = [&](u32 j) -> llvm::Value* + { + int indices[4]; + bool holes = false; + bool data = false; + + for (u32 i = 0; i < 4; i++) + { + const u32 k = j + i * 4; + + if (k < start || k >= end || !func.data[(k - start) / 4]) + { + indices[i] = 4; + holes = true; + } + else + { + indices[i] = i; + data = true; + } + } + + if (!data) + { + return nullptr; + } + + llvm::Value* vls = m_ir->CreateAlignedLoad(get_type(), _ptr(data_addr, j - starta), llvm::MaybeAlign{4}); + + if (holes) + { + vls = m_ir->CreateShuffleVector(vls, ConstantAggregateZero::get(vls->getType()), llvm::ArrayRef(indices, 4)); + } + + u32 words[4]; + + for (u32 i = 0; i < 4; i++) + { + const u32 k = j + i * 4; + words[i] = k >= start && k < end ? func.data[(k - start) / 4] : 0; + } + + const auto expected = ConstantDataVector::get(m_context, llvm::ArrayRef(words, 4)); + + if (m_use_dotprod) + { + return m_ir->CreateSExt(m_ir->CreateICmpEQ( + m_ir->CreateBitCast(vls, get_type()), + m_ir->CreateBitCast(expected, get_type())), get_type()); + } + + return m_ir->CreateSExt(m_ir->CreateICmpEQ( + m_ir->CreateBitCast(vls, get_type()), + m_ir->CreateBitCast(expected, get_type())), get_type()); + }; + + const auto accumulate_pair = [&](llvm::Value* lhs, llvm::Value* rhs) + { + llvm::Value*& acc = *accs[acc_index]; + + if (m_use_dotprod) + { + acc = m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_udot), {acc, lhs, rhs}); + } + else + { + acc = m_ir->CreateAdd(acc, m_ir->CreateMul(lhs, rhs)); + } + + acc_index = (acc_index + 1) & 3; + expected_hits++; + }; + + for (u32 j = starta; j < end; j += 16) + { + if (const auto cmp = make_cmp(j)) + { + if (pending_cmp) + { + accumulate_pair(pending_cmp, cmp); + pending_cmp = nullptr; + } + else + { + pending_cmp = cmp; + } + + check_iterations++; + } + } + + if (pending_cmp) + { + accumulate_pair(pending_cmp, llvm::ConstantInt::get(pending_cmp->getType(), -1, true)); + } + + llvm::Value* expected = nullptr; + + if (m_use_dotprod) + { + u32 expected_words[4]; + expected_words[0] = expected_hits * 4 * 0xff * 0xff; + expected_words[1] = expected_words[0]; + expected_words[2] = expected_words[0]; + expected_words[3] = expected_words[0]; + expected = ConstantDataVector::get(m_context, llvm::ArrayRef(expected_words, 4)); + } + else + { + u16 expected_words[8]; + std::fill_n(expected_words, 8, static_cast(expected_hits)); + expected = ConstantDataVector::get(m_context, llvm::ArrayRef(expected_words, 8)); + } + + llvm::Value* acc = m_ir->CreateAdd(m_ir->CreateAdd(acc0, acc1), m_ir->CreateAdd(acc2, acc3)); + acc = m_ir->CreateXor(acc, expected); + acc = m_ir->CreateBitCast(acc, get_type()); + + llvm::Value* elem = m_ir->CreateExtractElement(acc, u64{0}); + elem = m_ir->CreateOr(elem, m_ir->CreateExtractElement(acc, u64{1})); + + const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); + m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); + } +#else else { for (u32 j = starta; j < end; j += stride) @@ -2031,6 +2169,7 @@ public: const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); } +#endif } // Increase block counter with statistics diff --git a/rpcs3/Emu/Cell/SPUThread.cpp b/rpcs3/Emu/Cell/SPUThread.cpp index 9ef5da81eb..93af9e8b03 100644 --- a/rpcs3/Emu/Cell/SPUThread.cpp +++ b/rpcs3/Emu/Cell/SPUThread.cpp @@ -245,6 +245,17 @@ static FORCE_INLINE bool cmp_rdata_avx(const __m256i* lhs, const __m256i* rhs) } #endif +#if defined(ARCH_ARM64) +static FORCE_INLINE int16x8_t cmp16_pair_accum_arm64( + int16x8_t acc, const v128& lhs0, const v128& rhs0, const v128& lhs1, const v128& rhs1) +{ + const int16x8_t eq0 = vreinterpretq_s16_u16(vceqq_u16(static_cast(lhs0), static_cast(rhs0))); + const int16x8_t eq1 = vreinterpretq_s16_u16(vceqq_u16(static_cast(lhs1), static_cast(rhs1))); + return vmlaq_s16(acc, eq0, eq1); +} + +#endif + #ifdef _MSC_VER __forceinline #endif @@ -261,12 +272,22 @@ extern bool cmp_rdata(const spu_rdata_t& _lhs, const spu_rdata_t& _rhs) const auto lhs = reinterpret_cast(_lhs); const auto rhs = reinterpret_cast(_rhs); +#if defined(ARCH_ARM64) + int16x8_t hits = vdupq_n_s16(0); + hits = cmp16_pair_accum_arm64(hits, lhs[0], rhs[0], lhs[1], rhs[1]); + hits = cmp16_pair_accum_arm64(hits, lhs[2], rhs[2], lhs[3], rhs[3]); + hits = cmp16_pair_accum_arm64(hits, lhs[4], rhs[4], lhs[5], rhs[5]); + hits = cmp16_pair_accum_arm64(hits, lhs[6], rhs[6], lhs[7], rhs[7]); + + return vaddvq_s16(hits) == 32; +#else const v128 a = (lhs[0] ^ rhs[0]) | (lhs[1] ^ rhs[1]); const v128 c = (lhs[4] ^ rhs[4]) | (lhs[5] ^ rhs[5]); const v128 b = (lhs[2] ^ rhs[2]) | (lhs[3] ^ rhs[3]); const v128 d = (lhs[6] ^ rhs[6]) | (lhs[7] ^ rhs[7]); const v128 r = (a | b) | (c | d); return gv_testz(r); +#endif } #if defined(ARCH_X64)