dahjah/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2026-06-04 13:55:03 -06:00
Code Issues Packages Projects Releases Wiki Activity

Prevent atomic cache line collisions

Browse Source
This commit is contained in:
Elad 2025-12-15 21:30:40 +02:00
parent 16e41f4492
commit 65cd4deb77
9 changed files with 37 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
rpcs3/Emu/CPU/CPUThread.cpp
View File

@ -506,7 +506,7 @@ extern f64 get_cpu_program_usage_percent(u64 hash)
thread_local DECLARE(cpu_thread::g_tls_this_thread) = nullptr; thread_local DECLARE(cpu_thread::g_tls_this_thread) = nullptr;
// Total number of CPU threads // Total number of CPU threads
static atomic_t<u64, 64> s_cpu_counter{0}; static atomic_t<u64, 128> s_cpu_counter{0};
// List of posted tasks for suspend_all // List of posted tasks for suspend_all
//static atomic_t<cpu_thread::suspend_work*> s_cpu_work[128]{}; //static atomic_t<cpu_thread::suspend_work*> s_cpu_work[128]{};

12
rpcs3/Emu/Cell/SPUThread.cpp
View File

@ -488,7 +488,7 @@ waitpkg_func static void __tpause(u32 cycles, u32 cstate)
namespace vm namespace vm
{ {
std::array<atomic_t<reservation_waiter_t>, 2048> g_resrv_waiters_count{}; std::array<atomic_t<reservation_waiter_t, 128>, 1024> g_resrv_waiters_count{};
} }
void do_cell_atomic_128_store(u32 addr, const void* to_write); void do_cell_atomic_128_store(u32 addr, const void* to_write);
@ -499,7 +499,7 @@ const spu_decoder<spu_itype> s_spu_itype;
namespace vm namespace vm
{ {
extern atomic_t<u64, 64> g_range_lock_set[64]; extern atomic_t<u64, 128> g_range_lock_set[64];
// Defined here for performance reasons // Defined here for performance reasons
writer_lock::~writer_lock() noexcept writer_lock::~writer_lock() noexcept
@ -2000,7 +2000,7 @@ void spu_thread::do_dma_transfer(spu_thread* _this, const spu_mfc_cmd& args, u8*
cpu_thread* _cpu = _this ? _this : get_current_cpu_thread(); cpu_thread* _cpu = _this ? _this : get_current_cpu_thread();
atomic_t<u64, 64>* range_lock = nullptr; atomic_t<u64, 128>* range_lock = nullptr;
if (!_this) [[unlikely]] if (!_this) [[unlikely]]
{ {
@ -4928,12 +4928,12 @@ bool spu_thread::reservation_check(u32 addr, const decltype(rdata)& data, u32 cu
return !res; return !res;
} }
bool spu_thread::reservation_check(u32 addr, u32 hash, atomic_t<u64, 64>* range_lock) bool spu_thread::reservation_check(u32 addr, u32 hash, atomic_t<u64, 128>* range_lock)
{ {
if ((addr >> 28) < 2 || (addr >> 28) == 0xd) if ((addr >> 28) < 2 || (addr >> 28) == 0xd)
{ {
// Always-allocated memory does not need strict checking (vm::main or vm::stack) // Always-allocated memory does not need strict checking (vm::main or vm::stack)
return compute_rdata_hash32(*vm::get_super_ptr<decltype(rdata)>(addr)) == hash; return compute_rdata_hash32(*vm::get_super_ptr<decltype(rdata)>(addr)) != hash;
} }
// Ensure data is allocated (HACK: would raise LR event if not) // Ensure data is allocated (HACK: would raise LR event if not)
@ -5067,6 +5067,8 @@ void spu_thread::deregister_cache_line_waiter(usz index)
return; return;
} }
ensure(index < std::size(g_spu_waiters_by_value));
g_spu_waiters_by_value[index].atomic_op([](u64& x) g_spu_waiters_by_value[index].atomic_op([](u64& x)
{ {
x--; x--;

6
rpcs3/Emu/Cell/SPUThread.h
View File

@ -708,7 +708,7 @@ public:
const decltype(rdata)* resrv_mem{}; const decltype(rdata)* resrv_mem{};
// Range Lock pointer // Range Lock pointer
atomic_t<u64, 64>* range_lock{}; atomic_t<u64, 128>* range_lock{};
u32 srr0 = 0; u32 srr0 = 0;
u32 ch_tag_upd = 0; u32 ch_tag_upd = 0;
@ -903,7 +903,7 @@ public:
// It is safe to use on any address, even if not directly accessed by SPU (so it's slower) // It is safe to use on any address, even if not directly accessed by SPU (so it's slower)
// Optionally pass a known allocated address for internal optimization (the current Effective-Address of the MFC command) // Optionally pass a known allocated address for internal optimization (the current Effective-Address of the MFC command)
bool reservation_check(u32 addr, const decltype(rdata)& data, u32 current_eal = 0) const; bool reservation_check(u32 addr, const decltype(rdata)& data, u32 current_eal = 0) const;
static bool reservation_check(u32 addr, u32 hash, atomic_t<u64, 64>* range_lock); static bool reservation_check(u32 addr, u32 hash, atomic_t<u64, 128>* range_lock);
usz register_cache_line_waiter(u32 addr); usz register_cache_line_waiter(u32 addr);
void deregister_cache_line_waiter(usz index); void deregister_cache_line_waiter(usz index);
@ -915,7 +915,7 @@ public:
static atomic_t<u32> g_raw_spu_id[5]; static atomic_t<u32> g_raw_spu_id[5];
static atomic_t<u32> g_spu_work_count; static atomic_t<u32> g_spu_work_count;
static atomic_t<u64> g_spu_waiters_by_value[6]; static atomic_t<u64, 128> g_spu_waiters_by_value[6];
static u32 find_raw_spu(u32 id) static u32 find_raw_spu(u32 id)
{ {

2
rpcs3/Emu/Cell/lv2/lv2.cpp
View File

@ -2260,7 +2260,7 @@ void lv2_obj::notify_all() noexcept
// There may be 6 waiters, but checking them all may be performance expensive // There may be 6 waiters, but checking them all may be performance expensive
// Instead, check 2 at max, but use the CPU ID index to tell which index to start checking so the work would be distributed across all threads // Instead, check 2 at max, but use the CPU ID index to tell which index to start checking so the work would be distributed across all threads
atomic_t<u64, 64>* range_lock = nullptr; atomic_t<u64, 128>* range_lock = nullptr;
if (cpu->get_class() == thread_class::spu) if (cpu->get_class() == thread_class::spu)
{ {

2
rpcs3/Emu/Cell/lv2/sys_mutex.cpp
View File

@ -347,7 +347,7 @@ error_code sys_mutex_unlock(ppu_thread& ppu, u32 mutex_id)
const auto mutex = idm::check<lv2_obj, lv2_mutex>(mutex_id, [&, notify = lv2_obj::notify_all_t()](lv2_mutex& mutex) -> CellError const auto mutex = idm::check<lv2_obj, lv2_mutex>(mutex_id, [&, notify = lv2_obj::notify_all_t()](lv2_mutex& mutex) -> CellError
{ {
// At unlock, we have some time to do other jobs when the thread is unlikely to be in other critical sections // At unlock, we have some time to do other jobs when the thread is unlikely to be in other critical sections
notify.enqueue_on_top(vm::reservation_notifier_notify(ppu.res_notify, ppu.res_notify_time)); notify.enqueue_on_top(vm::reservation_notifier_notify(ppu.res_notify, ppu.res_notify_time, true));
auto result = mutex.try_unlock(ppu); auto result = mutex.try_unlock(ppu);

14
rpcs3/Emu/Memory/vm.cpp
View File

@ -74,7 +74,7 @@ namespace vm
std::array<atomic_t<cpu_thread*>, g_cfg.core.ppu_threads.max> g_locks{}; std::array<atomic_t<cpu_thread*>, g_cfg.core.ppu_threads.max> g_locks{};
// Range lock slot allocation bits // Range lock slot allocation bits
atomic_t<u64, 64> g_range_lock_bits[2]{}; atomic_t<u64, 128> g_range_lock_bits[2]{};
auto& get_range_lock_bits(bool is_exclusive_range) auto& get_range_lock_bits(bool is_exclusive_range)
{ {
@ -82,7 +82,7 @@ namespace vm
} }
// Memory range lock slots (sparse atomics) // Memory range lock slots (sparse atomics)
atomic_t<u64, 64> g_range_lock_set[64]{}; atomic_t<u64, 128> g_range_lock_set[64]{};
// Memory pages // Memory pages
std::array<memory_page, 0x100000000 / 4096> g_pages; std::array<memory_page, 0x100000000 / 4096> g_pages;
@ -142,7 +142,7 @@ namespace vm
} }
} }
atomic_t<u64, 64>* alloc_range_lock() atomic_t<u64, 128>* alloc_range_lock()
{ {
const auto [bits, ok] = get_range_lock_bits(false).fetch_op([](u64& bits) const auto [bits, ok] = get_range_lock_bits(false).fetch_op([](u64& bits)
{ {
@ -167,7 +167,7 @@ namespace vm
template <typename F> template <typename F>
static u64 for_all_range_locks(u64 input, F func); static u64 for_all_range_locks(u64 input, F func);
void range_lock_internal(atomic_t<u64, 64>* range_lock, u32 begin, u32 size) void range_lock_internal(atomic_t<u64, 128>* range_lock, u32 begin, u32 size)
{ {
perf_meter<"RHW_LOCK"_u64> perf0(0); perf_meter<"RHW_LOCK"_u64> perf0(0);
@ -275,7 +275,7 @@ namespace vm
} }
} }
void free_range_lock(atomic_t<u64, 64>* range_lock) noexcept void free_range_lock(atomic_t<u64, 128>* range_lock) noexcept
{ {
if (range_lock < g_range_lock_set || range_lock >= std::end(g_range_lock_set)) if (range_lock < g_range_lock_set || range_lock >= std::end(g_range_lock_set))
{ {
@ -316,7 +316,7 @@ namespace vm
return result; return result;
} }
static atomic_t<u64, 64>* _lock_main_range_lock(u64 flags, u32 addr, u32 size) static atomic_t<u64, 128>* _lock_main_range_lock(u64 flags, u32 addr, u32 size)
{ {
// Shouldn't really happen // Shouldn't really happen
if (size == 0) if (size == 0)
@ -460,7 +460,7 @@ namespace vm
{ {
} }
writer_lock::writer_lock(u32 const addr, atomic_t<u64, 64>* range_lock, u32 const size, u64 const flags) noexcept writer_lock::writer_lock(u32 const addr, atomic_t<u64, 128>* range_lock, u32 const size, u64 const flags) noexcept
: range_lock(range_lock) : range_lock(range_lock)
{ {
cpu_thread* cpu{}; cpu_thread* cpu{};

14
rpcs3/Emu/Memory/vm_locking.h
View File

@ -28,7 +28,7 @@ namespace vm
range_bits = 3, range_bits = 3,
}; };
extern atomic_t<u64, 64> g_range_lock_bits[2]; extern atomic_t<u64, 128> g_range_lock_bits[2];
extern atomic_t<u64> g_shmem[]; extern atomic_t<u64> g_shmem[];
@ -36,13 +36,13 @@ namespace vm
void passive_lock(cpu_thread& cpu); void passive_lock(cpu_thread& cpu);
// Register range lock for further use // Register range lock for further use
atomic_t<u64, 64>* alloc_range_lock(); atomic_t<u64, 128>* alloc_range_lock();
void range_lock_internal(atomic_t<u64, 64>* range_lock, u32 begin, u32 size); void range_lock_internal(atomic_t<u64, 128>* range_lock, u32 begin, u32 size);
// Lock memory range ignoring memory protection (Size!=0 also implies aligned begin) // Lock memory range ignoring memory protection (Size!=0 also implies aligned begin)
template <uint Size = 0> template <uint Size = 0>
FORCE_INLINE void range_lock(atomic_t<u64, 64>* range_lock, u32 begin, u32 _size) FORCE_INLINE void range_lock(atomic_t<u64, 128>* range_lock, u32 begin, u32 _size)
{ {
if constexpr (Size == 0) if constexpr (Size == 0)
{ {
@ -80,7 +80,7 @@ namespace vm
} }
// Release it // Release it
void free_range_lock(atomic_t<u64, 64>*) noexcept; void free_range_lock(atomic_t<u64, 128>*) noexcept;
// Unregister reader // Unregister reader
void passive_unlock(cpu_thread& cpu); void passive_unlock(cpu_thread& cpu);
@ -91,12 +91,12 @@ namespace vm
struct writer_lock final struct writer_lock final
{ {
atomic_t<u64, 64>* range_lock; atomic_t<u64, 128>* range_lock;
writer_lock(const writer_lock&) = delete; writer_lock(const writer_lock&) = delete;
writer_lock& operator=(const writer_lock&) = delete; writer_lock& operator=(const writer_lock&) = delete;
writer_lock() noexcept; writer_lock() noexcept;
writer_lock(u32 addr, atomic_t<u64, 64>* range_lock = nullptr, u32 size = 128, u64 flags = range_locked) noexcept; writer_lock(u32 addr, atomic_t<u64, 128>* range_lock = nullptr, u32 size = 128, u64 flags = range_locked) noexcept;
~writer_lock() noexcept; ~writer_lock() noexcept;
}; };
} // namespace vm } // namespace vm

16
rpcs3/Emu/Memory/vm_reservation.h
View File

@ -34,22 +34,22 @@ namespace vm
void reservation_update(u32 addr); void reservation_update(u32 addr);
std::pair<bool, u64> try_reservation_update(u32 addr); std::pair<bool, u64> try_reservation_update(u32 addr);
struct reservation_waiter_t struct alignas(8) reservation_waiter_t
{ {
u32 wait_flag = 0; u32 wait_flag = 0;
u32 waiters_count = 0; u32 waiters_count = 0;
}; };
static inline atomic_t<reservation_waiter_t>* reservation_notifier(u32 raddr, u64 rtime) static inline atomic_t<reservation_waiter_t, 128>* reservation_notifier(u32 raddr, u64 rtime)
{ {
constexpr u32 wait_vars_for_each = 64; constexpr u32 wait_vars_for_each = 32;
constexpr u32 unique_address_bit_mask = 0b1111; constexpr u32 unique_address_bit_mask = 0b1111;
constexpr u32 unique_rtime_bit_mask = 0b1; constexpr u32 unique_rtime_bit_mask = 0b1;
extern std::array<atomic_t<reservation_waiter_t>, wait_vars_for_each * (unique_address_bit_mask + 1) * (unique_rtime_bit_mask + 1)> g_resrv_waiters_count; extern std::array<atomic_t<reservation_waiter_t, 128>, wait_vars_for_each * (unique_address_bit_mask + 1) * (unique_rtime_bit_mask + 1)> g_resrv_waiters_count;
// Storage efficient method to distinguish different nearby addresses (which are likely) // Storage efficient method to distinguish different nearby addresses (which are likely)
const usz index = std::popcount(raddr & -2048) * (1 << 5) + ((rtime / 128) & unique_rtime_bit_mask) * (1 << 4) + ((raddr / 128) & unique_address_bit_mask); const usz index = std::min<usz>(std::popcount(raddr & -2048), 31) * (1 << 5) + ((rtime / 128) & unique_rtime_bit_mask) * (1 << 4) + ((raddr / 128) & unique_address_bit_mask);
return &g_resrv_waiters_count[index]; return &g_resrv_waiters_count[index];
} }
@ -59,7 +59,7 @@ namespace vm
return reservation_notifier(raddr, rtime)->load().waiters_count; return reservation_notifier(raddr, rtime)->load().waiters_count;
} }
static inline void reservation_notifier_end_wait(atomic_t<reservation_waiter_t>& waiter) static inline void reservation_notifier_end_wait(atomic_t<reservation_waiter_t, 128>& waiter)
{ {
waiter.atomic_op([](reservation_waiter_t& value) waiter.atomic_op([](reservation_waiter_t& value)
{ {
@ -73,9 +73,9 @@ namespace vm
}); });
} }
static inline std::pair<atomic_t<reservation_waiter_t>*, u32> reservation_notifier_begin_wait(u32 raddr, u64 rtime) static inline std::pair<atomic_t<reservation_waiter_t, 128>*, u32> reservation_notifier_begin_wait(u32 raddr, u64 rtime)
{ {
atomic_t<reservation_waiter_t>& waiter = *reservation_notifier(raddr, rtime); atomic_t<reservation_waiter_t, 128>& waiter = *reservation_notifier(raddr, rtime);
u32 wait_flag = 0; u32 wait_flag = 0;

4
rpcs3/util/logs.cpp
View File

@ -89,8 +89,8 @@ namespace logs
z_stream m_zs{}; z_stream m_zs{};
shared_mutex m_m{}; shared_mutex m_m{};
atomic_t<u64, 64> m_buf{0}; // MSB (39 bits): push begin, LSB (25 bis): push size atomic_t<u64, 128> m_buf{0}; // MSB (39 bits): push begin, LSB (25 bis): push size
atomic_t<u64, 64> m_out{0}; // Amount of bytes written to file atomic_t<u64, 128> m_out{0}; // Amount of bytes written to file
uchar m_zout[65536]{}; uchar m_zout[65536]{};