diff --git a/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp b/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp index 689233e0b3..73396eb08a 100644 --- a/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp +++ b/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp @@ -1835,16 +1835,94 @@ public: llvm::Value* starta_pc = m_ir->CreateAnd(get_pc(starta), 0x3fffc); llvm::Value* data_addr = _ptr(m_lsptr, starta_pc); +#ifndef ARCH_ARM64 llvm::Value* acc0 = nullptr; llvm::Value* acc1 = nullptr; bool toggle = true; +#endif // Use a 512bit simple checksum to verify integrity if size is atleast 512b * 3 // This code uses a 512bit vector for all hardware to ensure behavior matches. // The checksum path is still faster even on narrow hardware. if ((end - starta) >= 192 && !g_cfg.core.precise_spu_verification) { - for (u32 j = starta; j < end; j += 64) +#ifdef ARCH_ARM64 + // Loop if there is at least 288 bytes of data to checksum on ARM. + // Each ARM checksum block consumes 6 NEON vectors: 2 direct adds and 2 UABD accumulates. + constexpr u32 checksum_block_size = 96; +#else + // Loop if there is atleast (16 * stride) bytes of data to checksum to save some instruction cache + constexpr u32 checksum_block_size = 64; +#endif + constexpr u32 checksum_loop_vectors = 16; + const u32 checksum_vectors_per_block = checksum_block_size / stride; + const u32 checksum_loop_blocks = (checksum_loop_vectors + checksum_vectors_per_block - 1) / checksum_vectors_per_block; + const u32 checksum_loop_size = checksum_block_size * checksum_loop_blocks; + const u32 checksum_loop_end = starta + ((end - starta) / checksum_loop_size) * checksum_loop_size; + + bool use_checksum_loop = (checksum_loop_end - starta) >= checksum_loop_size * 2; + + for (u32 j = starta; use_checksum_loop && j < checksum_loop_end; j += 4) + { + if (!func.data[(j - start) / 4]) + { + use_checksum_loop = false; + break; + } + } + +#ifndef ARCH_ARM64 + if (use_checksum_loop) + { + const auto acc_init = ConstantAggregateZero::get(get_type()); + const auto loop_block = BasicBlock::Create(m_context, "spu_checksum_loop", m_function); + const auto loop_next = BasicBlock::Create(m_context, "spu_checksum_next", m_function); + const auto loop_preheader = m_ir->GetInsertBlock(); + m_ir->CreateBr(loop_block); + + m_ir->SetInsertPoint(loop_block); + const auto offset = m_ir->CreatePHI(get_type(), 2); + const auto acc0_phi = m_ir->CreatePHI(get_type(), 2); + const auto acc1_phi = m_ir->CreatePHI(get_type(), 2); + + offset->addIncoming(m_ir->getInt32(0), loop_preheader); + acc0_phi->addIncoming(acc_init, loop_preheader); + acc1_phi->addIncoming(acc_init, loop_preheader); + + const auto offset64 = m_ir->CreateZExt(offset, get_type()); + llvm::Value* next_acc0 = acc0_phi; + llvm::Value* next_acc1 = acc1_phi; + + for (u32 block = 0; block < checksum_loop_blocks; block++) + { + const auto vls = m_ir->CreateAlignedLoad(get_type(), _ptr(data_addr, m_ir->CreateAdd(offset64, m_ir->getInt64(block * checksum_block_size))), llvm::MaybeAlign{4}); + + if (block & 1) + { + next_acc1 = m_ir->CreateAdd(next_acc1, vls); + } + else + { + next_acc0 = m_ir->CreateAdd(next_acc0, vls); + } + } + + const auto next_offset = m_ir->CreateAdd(offset, m_ir->getInt32(checksum_loop_size)); + const auto loop_again = m_ir->CreateICmpULT(next_offset, m_ir->getInt32(checksum_loop_end - starta)); + m_ir->CreateCondBr(loop_again, loop_block, loop_next); + + offset->addIncoming(next_offset, loop_block); + acc0_phi->addIncoming(next_acc0, loop_block); + acc1_phi->addIncoming(next_acc1, loop_block); + acc0 = next_acc0; + acc1 = next_acc1; + + check_iterations += (checksum_loop_end - starta) / checksum_block_size; + + m_ir->SetInsertPoint(loop_next); + } + + for (u32 j = use_checksum_loop ? checksum_loop_end : starta; j < end; j += checksum_block_size) { int indices[16]; bool holes = false; @@ -1928,7 +2006,319 @@ public: // Compare result with zero const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); +#else + // Very cursed "checksumming" code + // 96 bytes per ARM checksum step + //vls[0] -> add + //vls[1], vls[2] -> uaba + //vls[3] -> add + //vls[4], vls[5] -> uaba + //This allows us to save on some ALU ops relative to load instructions + const auto acc_init = ConstantAggregateZero::get(get_type()); + llvm::Value* checksum_parts[4] = {acc_init, acc_init, acc_init, acc_init}; + u32 checksum[16] = {0}; + + const auto update_checksum = [&](const u32* words) + { + for (u32 i = 0; i < 4; i++) + { + checksum[i] += words[i]; + checksum[4 + i] += words[4 + i] > words[8 + i] ? words[4 + i] - words[8 + i] : words[8 + i] - words[4 + i]; + checksum[8 + i] += words[12 + i]; + checksum[12 + i] += words[16 + i] > words[20 + i] ? words[16 + i] - words[20 + i] : words[20 + i] - words[16 + i]; + } + }; + + if (use_checksum_loop) + { + for (u32 j = starta; j < checksum_loop_end; j += checksum_block_size) + { + u32 words[24]; + + for (u32 i = 0; i < 24; i++) + { + words[i] = func.data[(j + i * 4 - start) / 4]; + } + + update_checksum(words); + } + + const auto loop_block = BasicBlock::Create(m_context, "spu_checksum_loop", m_function); + const auto loop_next = BasicBlock::Create(m_context, "spu_checksum_next", m_function); + const auto loop_preheader = m_ir->GetInsertBlock(); + m_ir->CreateBr(loop_block); + + m_ir->SetInsertPoint(loop_block); + const auto offset = m_ir->CreatePHI(get_type(), 2); + llvm::PHINode* acc_phi[4]; + llvm::Value* next_acc[4]; + + for (u32 part = 0; part < 4; part++) + { + acc_phi[part] = m_ir->CreatePHI(get_type(), 2); + acc_phi[part]->addIncoming(checksum_parts[part], loop_preheader); + next_acc[part] = acc_phi[part]; + } + + offset->addIncoming(m_ir->getInt32(0), loop_preheader); + + const auto offset64 = m_ir->CreateZExt(offset, get_type()); + + for (u32 block = 0; block < checksum_loop_blocks; block++) + { + llvm::Value* vls[6]; + + for (u32 part = 0; part < 6; part++) + { + vls[part] = m_ir->CreateAlignedLoad(get_type(), _ptr(data_addr, m_ir->CreateAdd(offset64, m_ir->getInt64(block * checksum_block_size + part * 16))), llvm::MaybeAlign{4}); + } + + next_acc[0] = m_ir->CreateAdd(next_acc[0], vls[0]); + next_acc[1] = m_ir->CreateAdd(next_acc[1], m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uabd), {vls[1], vls[2]})); + next_acc[2] = m_ir->CreateAdd(next_acc[2], vls[3]); + next_acc[3] = m_ir->CreateAdd(next_acc[3], m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uabd), {vls[4], vls[5]})); + } + + const auto next_offset = m_ir->CreateAdd(offset, m_ir->getInt32(checksum_loop_size)); + const auto loop_again = m_ir->CreateICmpULT(next_offset, m_ir->getInt32(checksum_loop_end - starta)); + m_ir->CreateCondBr(loop_again, loop_block, loop_next); + + offset->addIncoming(next_offset, loop_block); + + for (u32 part = 0; part < 4; part++) + { + acc_phi[part]->addIncoming(next_acc[part], loop_block); + checksum_parts[part] = next_acc[part]; + } + + check_iterations += (checksum_loop_end - starta) / checksum_block_size; + + m_ir->SetInsertPoint(loop_next); + } + + for (u32 j = use_checksum_loop ? checksum_loop_end : starta; j < end; j += checksum_block_size) + { + llvm::Value* vls[6] = {}; + u32 words[24] = {}; + bool any_data = false; + + for (u32 part = 0; part < 6; part++) + { + int indices[4]; + bool holes = false; + bool data = false; + + for (u32 i = 0; i < 4; i++) + { + const u32 k = j + (part * 4 + i) * 4; + + if (k < start || k >= end || !func.data[(k - start) / 4]) + { + indices[i] = 4; + holes = true; + } + else + { + indices[i] = i; + data = true; + words[part * 4 + i] = func.data[(k - start) / 4]; + } + } + + if (!data) + { + vls[part] = acc_init; + continue; + } + + any_data = true; + + // Load unaligned code block from LS + vls[part] = m_ir->CreateAlignedLoad(get_type(), _ptr(data_addr, j + part * 16 - starta), llvm::MaybeAlign{4}); + + // Mask if necessary + if (holes) + { + vls[part] = m_ir->CreateShuffleVector(vls[part], acc_init, llvm::ArrayRef(indices, 4)); + } + } + + if (!any_data) + { + // Skip full-sized holes + continue; + } + + checksum_parts[0] = m_ir->CreateAdd(checksum_parts[0], vls[0]); + checksum_parts[1] = m_ir->CreateAdd(checksum_parts[1], m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uabd), {vls[1], vls[2]})); + checksum_parts[2] = m_ir->CreateAdd(checksum_parts[2], vls[3]); + checksum_parts[3] = m_ir->CreateAdd(checksum_parts[3], m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uabd), {vls[4], vls[5]})); + + update_checksum(words); + + check_iterations++; + } + + llvm::Value* elem = nullptr; + + for (u32 part = 0; part < 4; part++) + { + auto* const_vector = ConstantDataVector::get(m_context, llvm::ArrayRef(checksum + part * 4, 4)); + llvm::Value* acc = m_ir->CreateXor(checksum_parts[part], const_vector); + acc = m_ir->CreateBitCast(acc, get_type()); + + for (u32 i = 0; i < 2; i++) + { + const auto lane = m_ir->CreateExtractElement(acc, i); + elem = elem ? m_ir->CreateOr(elem, lane) : lane; + } + } + + // Compare result with zero + const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); + m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); +#endif } +#ifdef ARCH_ARM64 + else + { + const auto acc_init = m_use_dotprod ? ConstantAggregateZero::get(get_type()) : ConstantAggregateZero::get(get_type()); + llvm::Value* acc0 = acc_init; + llvm::Value* acc1 = acc_init; + llvm::Value* acc2 = acc_init; + llvm::Value* acc3 = acc_init; + llvm::Value** accs[4] = {&acc0, &acc1, &acc2, &acc3}; + u32 acc_index = 0; + llvm::Value* pending_cmp = nullptr; + u32 expected_hits = 0; + + const auto make_cmp = [&](u32 j) -> llvm::Value* + { + int indices[4]; + bool holes = false; + bool data = false; + + for (u32 i = 0; i < 4; i++) + { + const u32 k = j + i * 4; + + if (k < start || k >= end || !func.data[(k - start) / 4]) + { + indices[i] = 4; + holes = true; + } + else + { + indices[i] = i; + data = true; + } + } + + if (!data) + { + return nullptr; + } + + llvm::Value* vls = m_ir->CreateAlignedLoad(get_type(), _ptr(data_addr, j - starta), llvm::MaybeAlign{4}); + + if (holes) + { + vls = m_ir->CreateShuffleVector(vls, ConstantAggregateZero::get(vls->getType()), llvm::ArrayRef(indices, 4)); + } + + u32 words[4]; + + for (u32 i = 0; i < 4; i++) + { + const u32 k = j + i * 4; + words[i] = k >= start && k < end ? func.data[(k - start) / 4] : 0; + } + + const auto expected = ConstantDataVector::get(m_context, llvm::ArrayRef(words, 4)); + + if (m_use_dotprod) + { + return m_ir->CreateSExt(m_ir->CreateICmpEQ( + m_ir->CreateBitCast(vls, get_type()), + m_ir->CreateBitCast(expected, get_type())), get_type()); + } + + return m_ir->CreateSExt(m_ir->CreateICmpEQ( + m_ir->CreateBitCast(vls, get_type()), + m_ir->CreateBitCast(expected, get_type())), get_type()); + }; + + // Multiply accumulate based comparison + // See comment above cmp16_pair_accum_arm64 in SPUThread.cpp + // Dotproduct instructions have slightly higher throughput on many common ARM cores + const auto accumulate_pair = [&](llvm::Value* lhs, llvm::Value* rhs) + { + llvm::Value*& acc = *accs[acc_index]; + + if (m_use_dotprod) + { + acc = m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_udot), {acc, lhs, rhs}); + } + else + { + acc = m_ir->CreateAdd(acc, m_ir->CreateMul(lhs, rhs)); + } + + acc_index = (acc_index + 1) & 3; + expected_hits++; + }; + + for (u32 j = starta; j < end; j += 16) + { + if (const auto cmp = make_cmp(j)) + { + if (pending_cmp) + { + accumulate_pair(pending_cmp, cmp); + pending_cmp = nullptr; + } + else + { + pending_cmp = cmp; + } + + check_iterations++; + } + } + + if (pending_cmp) + { + accumulate_pair(pending_cmp, llvm::ConstantInt::get(pending_cmp->getType(), -1, true)); + } + + if (m_use_dotprod) + { + llvm::Value* acc = m_ir->CreateAdd(m_ir->CreateAdd(acc0, acc1), m_ir->CreateAdd(acc2, acc3)); + acc = m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::aarch64_neon_uaddv), {acc}); + + constexpr u64 dot_match_value = 0xff * 0xff; + const u32 expected = static_cast(expected_hits * 16 * dot_match_value); + const auto cond = m_ir->CreateICmpNE(acc, m_ir->getInt32(expected)); + m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); + } + else + { + u16 expected_words[8]; + std::fill_n(expected_words, 8, static_cast(expected_hits)); + const auto expected = ConstantDataVector::get(m_context, llvm::ArrayRef(expected_words, 8)); + + llvm::Value* acc = m_ir->CreateAdd(m_ir->CreateAdd(acc0, acc1), m_ir->CreateAdd(acc2, acc3)); + acc = m_ir->CreateXor(acc, expected); + acc = m_ir->CreateBitCast(acc, get_type()); + + llvm::Value* elem = m_ir->CreateExtractElement(acc, u64{0}); + elem = m_ir->CreateOr(elem, m_ir->CreateExtractElement(acc, u64{1})); + + const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); + m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); + } + } +#else else { for (u32 j = starta; j < end; j += stride) @@ -2031,6 +2421,7 @@ public: const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); } +#endif } // Increase block counter with statistics diff --git a/rpcs3/Emu/Cell/SPUThread.cpp b/rpcs3/Emu/Cell/SPUThread.cpp index 9ef5da81eb..dee3f15309 100644 --- a/rpcs3/Emu/Cell/SPUThread.cpp +++ b/rpcs3/Emu/Cell/SPUThread.cpp @@ -245,6 +245,26 @@ static FORCE_INLINE bool cmp_rdata_avx(const __m256i* lhs, const __m256i* rhs) } #endif +// Insane idea to accelerate comparisons on Neon with a fixed length +// Common ARM chips like the a78 and a715 can Perform 3 128b loads/clock +// But only execute 2 128b instructions on the ALU per clock +// To consume data any faster, we need to use ALU instructions that take 3 inputs +// Idea: compare data, filling each lane with either -1 or 0 +// Then multiply each pair of comparisons together, resulting in 1 if both pairs were -1 +// Accummulate those results, and compare the accumulated value to the expected count +// Benchmarks showed this to be faster even on arm machines that aren't capable of more loads than ALU operations +// Tested on Tensor G1, Snapdragon 8 gen 2, and the Snapdragon 8 Elite gen 5 +#if defined(ARCH_ARM64) +static FORCE_INLINE int16x8_t cmp16_pair_accum_arm64( + int16x8_t acc, const v128& lhs0, const v128& rhs0, const v128& lhs1, const v128& rhs1) +{ + const int16x8_t eq0 = vreinterpretq_s16_u16(vceqq_u16(static_cast(lhs0), static_cast(rhs0))); + const int16x8_t eq1 = vreinterpretq_s16_u16(vceqq_u16(static_cast(lhs1), static_cast(rhs1))); + return vmlaq_s16(acc, eq0, eq1); +} + +#endif + #ifdef _MSC_VER __forceinline #endif @@ -261,12 +281,22 @@ extern bool cmp_rdata(const spu_rdata_t& _lhs, const spu_rdata_t& _rhs) const auto lhs = reinterpret_cast(_lhs); const auto rhs = reinterpret_cast(_rhs); +#if defined(ARCH_ARM64) + int16x8_t hits = vdupq_n_s16(0); + hits = cmp16_pair_accum_arm64(hits, lhs[0], rhs[0], lhs[1], rhs[1]); + hits = cmp16_pair_accum_arm64(hits, lhs[2], rhs[2], lhs[3], rhs[3]); + hits = cmp16_pair_accum_arm64(hits, lhs[4], rhs[4], lhs[5], rhs[5]); + hits = cmp16_pair_accum_arm64(hits, lhs[6], rhs[6], lhs[7], rhs[7]); + + return vaddvq_s16(hits) == 32; +#else const v128 a = (lhs[0] ^ rhs[0]) | (lhs[1] ^ rhs[1]); const v128 c = (lhs[4] ^ rhs[4]) | (lhs[5] ^ rhs[5]); const v128 b = (lhs[2] ^ rhs[2]) | (lhs[3] ^ rhs[3]); const v128 d = (lhs[6] ^ rhs[6]) | (lhs[7] ^ rhs[7]); const v128 r = (a | b) | (c | d); return gv_testz(r); +#endif } #if defined(ARCH_X64) diff --git a/rpcs3/Emu/System.cpp b/rpcs3/Emu/System.cpp index 8f36b9a899..091d45e510 100644 --- a/rpcs3/Emu/System.cpp +++ b/rpcs3/Emu/System.cpp @@ -336,6 +336,20 @@ static void fixup_settings(const psf::registry* _psf) } } +#if defined(ARCH_ARM64) + if (g_cfg.core.spu_decoder == spu_decoder_type::asmjit) + { +#ifdef LLVM_AVAILABLE + constexpr auto arm64_spu_fallback = spu_decoder_type::dynamic; +#else + constexpr auto arm64_spu_fallback = spu_decoder_type::_static; +#endif + sys_log.warning("The setting '%s' is currently not supported on ARM64 builds and will therefore be changed from '%s' to '%s' during emulation.", + g_cfg.core.spu_decoder.get_name(), spu_decoder_type::asmjit, arm64_spu_fallback); + g_cfg.core.spu_decoder.set(arm64_spu_fallback); + } +#endif + if (const u32 psf_resolution = _psf ? psf::get_integer(*_psf, "RESOLUTION", 0) : 0) { const std::map resolutions