dahjah/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2026-07-10 01:44:41 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into uuu

Browse Source
This commit is contained in:
Elad 2026-07-01 14:40:21 +03:00 committed by GitHub
commit 83bf4d748a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 436 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

393
rpcs3/Emu/Cell/SPULLVMRecompiler.cpp
View File

@ -1835,16 +1835,94 @@ public:
llvm::Value* starta_pc = m_ir->CreateAnd(get_pc(starta), 0x3fffc); llvm::Value* starta_pc = m_ir->CreateAnd(get_pc(starta), 0x3fffc);
llvm::Value* data_addr = _ptr(m_lsptr, starta_pc); llvm::Value* data_addr = _ptr(m_lsptr, starta_pc);
#ifndef ARCH_ARM64
llvm::Value* acc0 = nullptr; llvm::Value* acc0 = nullptr;
llvm::Value* acc1 = nullptr; llvm::Value* acc1 = nullptr;
bool toggle = true; bool toggle = true;
#endif
// Use a 512bit simple checksum to verify integrity if size is atleast 512b * 3 // Use a 512bit simple checksum to verify integrity if size is atleast 512b * 3
// This code uses a 512bit vector for all hardware to ensure behavior matches. // This code uses a 512bit vector for all hardware to ensure behavior matches.
// The checksum path is still faster even on narrow hardware. // The checksum path is still faster even on narrow hardware.
if ((end - starta) >= 192 && !g_cfg.core.precise_spu_verification) if ((end - starta) >= 192 && !g_cfg.core.precise_spu_verification)
{ {
for (u32 j = starta; j < end; j += 64) #ifdef ARCH_ARM64
// Loop if there is at least 288 bytes of data to checksum on ARM.
// Each ARM checksum block consumes 6 NEON vectors: 2 direct adds and 2 UABD accumulates.
constexpr u32 checksum_block_size = 96;
#else
// Loop if there is atleast (16 * stride) bytes of data to checksum to save some instruction cache
constexpr u32 checksum_block_size = 64;
#endif
constexpr u32 checksum_loop_vectors = 16;
const u32 checksum_vectors_per_block = checksum_block_size / stride;
const u32 checksum_loop_blocks = (checksum_loop_vectors + checksum_vectors_per_block - 1) / checksum_vectors_per_block;
const u32 checksum_loop_size = checksum_block_size * checksum_loop_blocks;
const u32 checksum_loop_end = starta + ((end - starta) / checksum_loop_size) * checksum_loop_size;
bool use_checksum_loop = (checksum_loop_end - starta) >= checksum_loop_size * 2;
for (u32 j = starta; use_checksum_loop && j < checksum_loop_end; j += 4)
{
if (!func.data[(j - start) / 4])
{
use_checksum_loop = false;
break;
}
}
#ifndef ARCH_ARM64
if (use_checksum_loop)
{
const auto acc_init = ConstantAggregateZero::get(get_type<u32[16]>());
const auto loop_block = BasicBlock::Create(m_context, "spu_checksum_loop", m_function);
const auto loop_next = BasicBlock::Create(m_context, "spu_checksum_next", m_function);
const auto loop_preheader = m_ir->GetInsertBlock();
m_ir->CreateBr(loop_block);
m_ir->SetInsertPoint(loop_block);
const auto offset = m_ir->CreatePHI(get_type<u32>(), 2);
const auto acc0_phi = m_ir->CreatePHI(get_type<u32[16]>(), 2);
const auto acc1_phi = m_ir->CreatePHI(get_type<u32[16]>(), 2);
offset->addIncoming(m_ir->getInt32(0), loop_preheader);
acc0_phi->addIncoming(acc_init, loop_preheader);
acc1_phi->addIncoming(acc_init, loop_preheader);
const auto offset64 = m_ir->CreateZExt(offset, get_type<u64>());
llvm::Value* next_acc0 = acc0_phi;
llvm::Value* next_acc1 = acc1_phi;
for (u32 block = 0; block < checksum_loop_blocks; block++)
{
const auto vls = m_ir->CreateAlignedLoad(get_type<u32[16]>(), _ptr(data_addr, m_ir->CreateAdd(offset64, m_ir->getInt64(block * checksum_block_size))), llvm::MaybeAlign{4});
if (block & 1)
{
next_acc1 = m_ir->CreateAdd(next_acc1, vls);
}
else
{
next_acc0 = m_ir->CreateAdd(next_acc0, vls);
}
}
const auto next_offset = m_ir->CreateAdd(offset, m_ir->getInt32(checksum_loop_size));
const auto loop_again = m_ir->CreateICmpULT(next_offset, m_ir->getInt32(checksum_loop_end - starta));
m_ir->CreateCondBr(loop_again, loop_block, loop_next);
offset->addIncoming(next_offset, loop_block);
acc0_phi->addIncoming(next_acc0, loop_block);
acc1_phi->addIncoming(next_acc1, loop_block);
acc0 = next_acc0;
acc1 = next_acc1;
check_iterations += (checksum_loop_end - starta) / checksum_block_size;
m_ir->SetInsertPoint(loop_next);
}
for (u32 j = use_checksum_loop ? checksum_loop_end : starta; j < end; j += checksum_block_size)
{ {
int indices[16]; int indices[16];
bool holes = false; bool holes = false;
@ -1928,7 +2006,319 @@ public:
// Compare result with zero // Compare result with zero
const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0));
m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely);
#else
// Very cursed "checksumming" code
// 96 bytes per ARM checksum step
//vls[0] -> add
//vls[1], vls[2] -> uaba
//vls[3] -> add
//vls[4], vls[5] -> uaba
//This allows us to save on some ALU ops relative to load instructions
const auto acc_init = ConstantAggregateZero::get(get_type<u32[4]>());
llvm::Value* checksum_parts[4] = {acc_init, acc_init, acc_init, acc_init};
u32 checksum[16] = {0};
const auto update_checksum = [&](const u32* words)
{
for (u32 i = 0; i < 4; i++)
{
checksum[i] += words[i];
checksum[4 + i] += words[4 + i] > words[8 + i] ? words[4 + i] - words[8 + i] : words[8 + i] - words[4 + i];
checksum[8 + i] += words[12 + i];
checksum[12 + i] += words[16 + i] > words[20 + i] ? words[16 + i] - words[20 + i] : words[20 + i] - words[16 + i];
}
};
if (use_checksum_loop)
{
for (u32 j = starta; j < checksum_loop_end; j += checksum_block_size)
{
u32 words[24];
for (u32 i = 0; i < 24; i++)
{
words[i] = func.data[(j + i * 4 - start) / 4];
}
update_checksum(words);
}
const auto loop_block = BasicBlock::Create(m_context, "spu_checksum_loop", m_function);
const auto loop_next = BasicBlock::Create(m_context, "spu_checksum_next", m_function);
const auto loop_preheader = m_ir->GetInsertBlock();
m_ir->CreateBr(loop_block);
m_ir->SetInsertPoint(loop_block);
const auto offset = m_ir->CreatePHI(get_type<u32>(), 2);
llvm::PHINode* acc_phi[4];
llvm::Value* next_acc[4];
for (u32 part = 0; part < 4; part++)
{
acc_phi[part] = m_ir->CreatePHI(get_type<u32[4]>(), 2);
acc_phi[part]->addIncoming(checksum_parts[part], loop_preheader);
next_acc[part] = acc_phi[part];
}
offset->addIncoming(m_ir->getInt32(0), loop_preheader);
const auto offset64 = m_ir->CreateZExt(offset, get_type<u64>());
for (u32 block = 0; block < checksum_loop_blocks; block++)
{
llvm::Value* vls[6];
for (u32 part = 0; part < 6; part++)
{
vls[part] = m_ir->CreateAlignedLoad(get_type<u32[4]>(), _ptr(data_addr, m_ir->CreateAdd(offset64, m_ir->getInt64(block * checksum_block_size + part * 16))), llvm::MaybeAlign{4});
}
next_acc[0] = m_ir->CreateAdd(next_acc[0], vls[0]);
next_acc[1] = m_ir->CreateAdd(next_acc[1], m_ir->CreateCall(get_intrinsic<u32[4]>(llvm::Intrinsic::aarch64_neon_uabd), {vls[1], vls[2]}));
next_acc[2] = m_ir->CreateAdd(next_acc[2], vls[3]);
next_acc[3] = m_ir->CreateAdd(next_acc[3], m_ir->CreateCall(get_intrinsic<u32[4]>(llvm::Intrinsic::aarch64_neon_uabd), {vls[4], vls[5]}));
}
const auto next_offset = m_ir->CreateAdd(offset, m_ir->getInt32(checksum_loop_size));
const auto loop_again = m_ir->CreateICmpULT(next_offset, m_ir->getInt32(checksum_loop_end - starta));
m_ir->CreateCondBr(loop_again, loop_block, loop_next);
offset->addIncoming(next_offset, loop_block);
for (u32 part = 0; part < 4; part++)
{
acc_phi[part]->addIncoming(next_acc[part], loop_block);
checksum_parts[part] = next_acc[part];
}
check_iterations += (checksum_loop_end - starta) / checksum_block_size;
m_ir->SetInsertPoint(loop_next);
}
for (u32 j = use_checksum_loop ? checksum_loop_end : starta; j < end; j += checksum_block_size)
{
llvm::Value* vls[6] = {};
u32 words[24] = {};
bool any_data = false;
for (u32 part = 0; part < 6; part++)
{
int indices[4];
bool holes = false;
bool data = false;
for (u32 i = 0; i < 4; i++)
{
const u32 k = j + (part * 4 + i) * 4;
if (k < start || k >= end || !func.data[(k - start) / 4])
{
indices[i] = 4;
holes = true;
}
else
{
indices[i] = i;
data = true;
words[part * 4 + i] = func.data[(k - start) / 4];
}
}
if (!data)
{
vls[part] = acc_init;
continue;
}
any_data = true;
// Load unaligned code block from LS
vls[part] = m_ir->CreateAlignedLoad(get_type<u32[4]>(), _ptr(data_addr, j + part * 16 - starta), llvm::MaybeAlign{4});
// Mask if necessary
if (holes)
{
vls[part] = m_ir->CreateShuffleVector(vls[part], acc_init, llvm::ArrayRef(indices, 4));
}
}
if (!any_data)
{
// Skip full-sized holes
continue;
}
checksum_parts[0] = m_ir->CreateAdd(checksum_parts[0], vls[0]);
checksum_parts[1] = m_ir->CreateAdd(checksum_parts[1], m_ir->CreateCall(get_intrinsic<u32[4]>(llvm::Intrinsic::aarch64_neon_uabd), {vls[1], vls[2]}));
checksum_parts[2] = m_ir->CreateAdd(checksum_parts[2], vls[3]);
checksum_parts[3] = m_ir->CreateAdd(checksum_parts[3], m_ir->CreateCall(get_intrinsic<u32[4]>(llvm::Intrinsic::aarch64_neon_uabd), {vls[4], vls[5]}));
update_checksum(words);
check_iterations++;
}
llvm::Value* elem = nullptr;
for (u32 part = 0; part < 4; part++)
{
auto* const_vector = ConstantDataVector::get(m_context, llvm::ArrayRef(checksum + part * 4, 4));
llvm::Value* acc = m_ir->CreateXor(checksum_parts[part], const_vector);
acc = m_ir->CreateBitCast(acc, get_type<u64[2]>());
for (u32 i = 0; i < 2; i++)
{
const auto lane = m_ir->CreateExtractElement(acc, i);
elem = elem ? m_ir->CreateOr(elem, lane) : lane;
}
}
// Compare result with zero
const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0));
m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely);
#endif
} }
#ifdef ARCH_ARM64
else
{
const auto acc_init = m_use_dotprod ? ConstantAggregateZero::get(get_type<u32[4]>()) : ConstantAggregateZero::get(get_type<s16[8]>());
llvm::Value* acc0 = acc_init;
llvm::Value* acc1 = acc_init;
llvm::Value* acc2 = acc_init;
llvm::Value* acc3 = acc_init;
llvm::Value** accs[4] = {&acc0, &acc1, &acc2, &acc3};
u32 acc_index = 0;
llvm::Value* pending_cmp = nullptr;
u32 expected_hits = 0;
const auto make_cmp = [&](u32 j) -> llvm::Value*
{
int indices[4];
bool holes = false;
bool data = false;
for (u32 i = 0; i < 4; i++)
{
const u32 k = j + i * 4;
if (k < start || k >= end || !func.data[(k - start) / 4])
{
indices[i] = 4;
holes = true;
}
else
{
indices[i] = i;
data = true;
}
}
if (!data)
{
return nullptr;
}
llvm::Value* vls = m_ir->CreateAlignedLoad(get_type<u32[4]>(), _ptr(data_addr, j - starta), llvm::MaybeAlign{4});
if (holes)
{
vls = m_ir->CreateShuffleVector(vls, ConstantAggregateZero::get(vls->getType()), llvm::ArrayRef(indices, 4));
}
u32 words[4];
for (u32 i = 0; i < 4; i++)
{
const u32 k = j + i * 4;
words[i] = k >= start && k < end ? func.data[(k - start) / 4] : 0;
}
const auto expected = ConstantDataVector::get(m_context, llvm::ArrayRef(words, 4));
if (m_use_dotprod)
{
return m_ir->CreateSExt(m_ir->CreateICmpEQ(
m_ir->CreateBitCast(vls, get_type<u8[16]>()),
m_ir->CreateBitCast(expected, get_type<u8[16]>())), get_type<s8[16]>());
}
return m_ir->CreateSExt(m_ir->CreateICmpEQ(
m_ir->CreateBitCast(vls, get_type<u16[8]>()),
m_ir->CreateBitCast(expected, get_type<u16[8]>())), get_type<s16[8]>());
};
// Multiply accumulate based comparison
// See comment above cmp16_pair_accum_arm64 in SPUThread.cpp
// Dotproduct instructions have slightly higher throughput on many common ARM cores
const auto accumulate_pair = [&](llvm::Value* lhs, llvm::Value* rhs)
{
llvm::Value*& acc = *accs[acc_index];
if (m_use_dotprod)
{
acc = m_ir->CreateCall(get_intrinsic<u32[4], u8[16]>(llvm::Intrinsic::aarch64_neon_udot), {acc, lhs, rhs});
}
else
{
acc = m_ir->CreateAdd(acc, m_ir->CreateMul(lhs, rhs));
}
acc_index = (acc_index + 1) & 3;
expected_hits++;
};
for (u32 j = starta; j < end; j += 16)
{
if (const auto cmp = make_cmp(j))
{
if (pending_cmp)
{
accumulate_pair(pending_cmp, cmp);
pending_cmp = nullptr;
}
else
{
pending_cmp = cmp;
}
check_iterations++;
}
}
if (pending_cmp)
{
accumulate_pair(pending_cmp, llvm::ConstantInt::get(pending_cmp->getType(), -1, true));
}
if (m_use_dotprod)
{
llvm::Value* acc = m_ir->CreateAdd(m_ir->CreateAdd(acc0, acc1), m_ir->CreateAdd(acc2, acc3));
acc = m_ir->CreateCall(get_intrinsic<u32, u32[4]>(llvm::Intrinsic::aarch64_neon_uaddv), {acc});
constexpr u64 dot_match_value = 0xff * 0xff;
const u32 expected = static_cast<u32>(expected_hits * 16 * dot_match_value);
const auto cond = m_ir->CreateICmpNE(acc, m_ir->getInt32(expected));
m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely);
}
else
{
u16 expected_words[8];
std::fill_n(expected_words, 8, static_cast<u16>(expected_hits));
const auto expected = ConstantDataVector::get(m_context, llvm::ArrayRef(expected_words, 8));
llvm::Value* acc = m_ir->CreateAdd(m_ir->CreateAdd(acc0, acc1), m_ir->CreateAdd(acc2, acc3));
acc = m_ir->CreateXor(acc, expected);
acc = m_ir->CreateBitCast(acc, get_type<u64[2]>());
llvm::Value* elem = m_ir->CreateExtractElement(acc, u64{0});
elem = m_ir->CreateOr(elem, m_ir->CreateExtractElement(acc, u64{1}));
const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0));
m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely);
}
}
#else
else else
{ {
for (u32 j = starta; j < end; j += stride) for (u32 j = starta; j < end; j += stride)
@ -2031,6 +2421,7 @@ public:
const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0));
m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely);
} }
#endif
} }
// Increase block counter with statistics // Increase block counter with statistics

30
rpcs3/Emu/Cell/SPUThread.cpp
View File

@ -245,6 +245,26 @@ static FORCE_INLINE bool cmp_rdata_avx(const __m256i* lhs, const __m256i* rhs)
} }
#endif #endif
// Insane idea to accelerate comparisons on Neon with a fixed length
// Common ARM chips like the a78 and a715 can Perform 3 128b loads/clock
// But only execute 2 128b instructions on the ALU per clock
// To consume data any faster, we need to use ALU instructions that take 3 inputs
// Idea: compare data, filling each lane with either -1 or 0
// Then multiply each pair of comparisons together, resulting in 1 if both pairs were -1
// Accummulate those results, and compare the accumulated value to the expected count
// Benchmarks showed this to be faster even on arm machines that aren't capable of more loads than ALU operations
// Tested on Tensor G1, Snapdragon 8 gen 2, and the Snapdragon 8 Elite gen 5
#if defined(ARCH_ARM64)
static FORCE_INLINE int16x8_t cmp16_pair_accum_arm64(
int16x8_t acc, const v128& lhs0, const v128& rhs0, const v128& lhs1, const v128& rhs1)
{
const int16x8_t eq0 = vreinterpretq_s16_u16(vceqq_u16(static_cast<uint16x8_t>(lhs0), static_cast<uint16x8_t>(rhs0)));
const int16x8_t eq1 = vreinterpretq_s16_u16(vceqq_u16(static_cast<uint16x8_t>(lhs1), static_cast<uint16x8_t>(rhs1)));
return vmlaq_s16(acc, eq0, eq1);
}
#endif
#ifdef _MSC_VER #ifdef _MSC_VER
__forceinline __forceinline
#endif #endif
@ -261,12 +281,22 @@ extern bool cmp_rdata(const spu_rdata_t& _lhs, const spu_rdata_t& _rhs)
const auto lhs = reinterpret_cast<const v128*>(_lhs); const auto lhs = reinterpret_cast<const v128*>(_lhs);
const auto rhs = reinterpret_cast<const v128*>(_rhs); const auto rhs = reinterpret_cast<const v128*>(_rhs);
#if defined(ARCH_ARM64)
int16x8_t hits = vdupq_n_s16(0);
hits = cmp16_pair_accum_arm64(hits, lhs[0], rhs[0], lhs[1], rhs[1]);
hits = cmp16_pair_accum_arm64(hits, lhs[2], rhs[2], lhs[3], rhs[3]);
hits = cmp16_pair_accum_arm64(hits, lhs[4], rhs[4], lhs[5], rhs[5]);
hits = cmp16_pair_accum_arm64(hits, lhs[6], rhs[6], lhs[7], rhs[7]);
return vaddvq_s16(hits) == 32;
#else
const v128 a = (lhs[0] ^ rhs[0]) | (lhs[1] ^ rhs[1]); const v128 a = (lhs[0] ^ rhs[0]) | (lhs[1] ^ rhs[1]);
const v128 c = (lhs[4] ^ rhs[4]) | (lhs[5] ^ rhs[5]); const v128 c = (lhs[4] ^ rhs[4]) | (lhs[5] ^ rhs[5]);
const v128 b = (lhs[2] ^ rhs[2]) | (lhs[3] ^ rhs[3]); const v128 b = (lhs[2] ^ rhs[2]) | (lhs[3] ^ rhs[3]);
const v128 d = (lhs[6] ^ rhs[6]) | (lhs[7] ^ rhs[7]); const v128 d = (lhs[6] ^ rhs[6]) | (lhs[7] ^ rhs[7]);
const v128 r = (a | b) | (c | d); const v128 r = (a | b) | (c | d);
return gv_testz(r); return gv_testz(r);
#endif
} }
#if defined(ARCH_X64) #if defined(ARCH_X64)

14
rpcs3/Emu/System.cpp
View File

@ -336,6 +336,20 @@ static void fixup_settings(const psf::registry* _psf)
} }
} }
#if defined(ARCH_ARM64)
if (g_cfg.core.spu_decoder == spu_decoder_type::asmjit)
{
#ifdef LLVM_AVAILABLE
constexpr auto arm64_spu_fallback = spu_decoder_type::dynamic;
#else
constexpr auto arm64_spu_fallback = spu_decoder_type::_static;
#endif
sys_log.warning("The setting '%s' is currently not supported on ARM64 builds and will therefore be changed from '%s' to '%s' during emulation.",
g_cfg.core.spu_decoder.get_name(), spu_decoder_type::asmjit, arm64_spu_fallback);
g_cfg.core.spu_decoder.set(arm64_spu_fallback);
}
#endif
if (const u32 psf_resolution = _psf ? psf::get_integer(*_psf, "RESOLUTION", 0) : 0) if (const u32 psf_resolution = _psf ? psf::get_integer(*_psf, "RESOLUTION", 0) : 0)
{ {
const std::map<video_resolution, u32> resolutions const std::map<video_resolution, u32> resolutions