diff --git a/src/core/memory.cpp b/src/core/memory.cpp index a340c3643..9ad207bc1 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright 2025-2026 shadPS4 Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include + #include "common/alignment.h" #include "common/assert.h" #include "common/debug.h" @@ -68,8 +70,26 @@ void MemoryManager::SetupMemoryRegions(u64 flexible_size, bool use_extended_mem1 fmem_map.emplace(total_direct_size, PhysicalMemoryArea{total_direct_size, remaining_physical_space}); + flexible_virtual_base = impl.SystemReservedVirtualBase(); + const u64 flexible_virtual_size = + std::min(total_flexible_size, impl.SystemReservedVirtualSize()); + flexible_virtual_end = flexible_virtual_base + flexible_virtual_size; + + { + std::scoped_lock lk{mutex}; + InvalidateFlexibleMappedRangeCacheLocked(); + RecalculateFlexibleMappedUsageLocked(); + } + LOG_INFO(Kernel_Vmm, "Configured memory regions: flexible size = {:#x}, direct size = {:#x}", total_flexible_size, total_direct_size); + if (Config::debugDump()) { + LOG_DEBUG( + Kernel_Vmm, + "Flexible accounting region: [{:#x}, {:#x}), total = {:#x}, used = {:#x}, free = {:#x}", + flexible_virtual_base, flexible_virtual_end, total_flexible_size, flexible_mapped_usage, + GetAvailableFlexibleSize()); + } } u64 MemoryManager::ClampRangeSize(VAddr virtual_addr, u64 size) { @@ -394,6 +414,7 @@ s32 MemoryManager::PoolCommit(VAddr virtual_addr, u64 size, MemoryProt prot, s32 new_vma.prot = prot; new_vma.name = "anon"; new_vma.type = Core::VMAType::Pooled; + new_vma.is_system_module = false; new_vma.phys_areas.clear(); // Find suitable physical addresses @@ -445,7 +466,8 @@ s32 MemoryManager::PoolCommit(VAddr virtual_addr, u64 size, MemoryProt prot, s32 MemoryManager::VMAHandle MemoryManager::CreateArea(VAddr virtual_addr, u64 size, MemoryProt prot, MemoryMapFlags flags, VMAType type, - std::string_view name, u64 alignment) { + std::string_view name, u64 alignment, + bool is_system_module) { // Locate the VMA representing the requested region auto vma = FindVMA(virtual_addr)->second; if (True(flags & MemoryMapFlags::Fixed)) { @@ -469,6 +491,7 @@ MemoryManager::VMAHandle MemoryManager::CreateArea(VAddr virtual_addr, u64 size, // Create a memory area representing this mapping. const auto new_vma_handle = CarveVMA(virtual_addr, size); auto& new_vma = new_vma_handle->second; + InvalidateFlexibleMappedRangeCacheLocked(); const bool is_exec = True(prot & MemoryProt::CpuExec); if (True(prot & MemoryProt::CpuWrite)) { // On PS4, read is appended to write mappings. @@ -480,13 +503,15 @@ MemoryManager::VMAHandle MemoryManager::CreateArea(VAddr virtual_addr, u64 size, new_vma.prot = prot; new_vma.name = name; new_vma.type = type; + new_vma.is_system_module = is_system_module; new_vma.phys_areas.clear(); return new_vma_handle; } s32 MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, u64 size, MemoryProt prot, MemoryMapFlags flags, VMAType type, std::string_view name, - bool validate_dmem, PAddr phys_addr, u64 alignment) { + bool validate_dmem, PAddr phys_addr, u64 alignment, + bool is_system_module) { // Certain games perform flexible mappings on loop to determine // the available flexible memory size. Questionable but we need to handle this. if (type == VMAType::Flexible && flexible_usage + size > total_flexible_size) { @@ -557,9 +582,11 @@ s32 MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, u64 size, Memo // Acquire writer lock. std::unique_lock lk2{mutex}; + const u64 flexible_before = GetFlexibleMappedBytesInRangeLocked(virtual_addr, size); // Create VMA representing this mapping. - auto new_vma_handle = CreateArea(virtual_addr, size, prot, flags, type, name, alignment); + auto new_vma_handle = + CreateArea(virtual_addr, size, prot, flags, type, name, alignment, is_system_module); auto& new_vma = new_vma_handle->second; auto mapped_addr = new_vma.base; bool is_exec = True(prot & MemoryProt::CpuExec); @@ -643,6 +670,14 @@ s32 MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, u64 size, Memo MergeAdjacent(vma_map, new_vma_handle); } + u64 flexible_after = 0; + if (type == VMAType::Flexible && !new_vma.phys_areas.empty()) { + flexible_after = GetFlexibleRangeOverlapBytesLocked(mapped_addr, size); + } else if (type == VMAType::Code && !is_system_module) { + flexible_after = GetFlexibleRangeOverlapBytesLocked(mapped_addr, size); + } + AdjustFlexibleMappedUsageLocked(flexible_before, flexible_after); + *out_addr = std::bit_cast(mapped_addr); if (type != VMAType::Reserved && type != VMAType::PoolReserved) { // Flexible address space mappings were performed while finding direct memory areas. @@ -731,9 +766,11 @@ s32 MemoryManager::MapFile(void** out_addr, VAddr virtual_addr, u64 size, Memory // Aquire writer lock std::scoped_lock lk2{mutex}; + const u64 flexible_before = GetFlexibleMappedBytesInRangeLocked(virtual_addr, size); // Update VMA map and map to address space. - auto new_vma_handle = CreateArea(virtual_addr, size, prot, flags, VMAType::File, "anon", 0); + auto new_vma_handle = + CreateArea(virtual_addr, size, prot, flags, VMAType::File, "anon", 0, false); auto& new_vma = new_vma_handle->second; new_vma.fd = fd; @@ -742,6 +779,8 @@ s32 MemoryManager::MapFile(void** out_addr, VAddr virtual_addr, u64 size, Memory impl.MapFile(mapped_addr, size, phys_addr, std::bit_cast(prot), handle); + AdjustFlexibleMappedUsageLocked(flexible_before, 0); + *out_addr = std::bit_cast(mapped_addr); return ORBIS_OK; } @@ -814,6 +853,7 @@ s32 MemoryManager::PoolDecommit(VAddr virtual_addr, u64 size) { vma.prot = MemoryProt::NoAccess; vma.disallow_merge = false; vma.name = "anon"; + vma.is_system_module = false; vma.phys_areas.clear(); MergeAdjacent(vma_map, new_it); @@ -848,7 +888,12 @@ s32 MemoryManager::UnmapMemory(VAddr virtual_addr, u64 size) { // Acquire writer lock. std::scoped_lock lk2{mutex}; - return UnmapMemoryImpl(virtual_addr, size); + const u64 flexible_before = GetFlexibleMappedBytesInRangeLocked(virtual_addr, size); + const s32 result = UnmapMemoryImpl(virtual_addr, size); + if (result == ORBIS_OK) { + AdjustFlexibleMappedUsageLocked(flexible_before, 0); + } + return result; } u64 MemoryManager::UnmapBytesFromEntry(VAddr virtual_addr, VirtualMemoryArea vma_base, u64 size) { @@ -904,11 +949,13 @@ u64 MemoryManager::UnmapBytesFromEntry(VAddr virtual_addr, VirtualMemoryArea vma // Mark region as free and attempt to coalesce it with neighbours. const auto new_it = CarveVMA(virtual_addr, size_in_vma); auto& vma = new_it->second; + InvalidateFlexibleMappedRangeCacheLocked(); vma.type = VMAType::Free; vma.prot = MemoryProt::NoAccess; vma.phys_areas.clear(); vma.disallow_merge = false; vma.name = ""; + vma.is_system_module = false; MergeAdjacent(vma_map, new_it); if (vma_type != VMAType::Reserved && vma_type != VMAType::PoolReserved) { @@ -1352,6 +1399,137 @@ void MemoryManager::InvalidateMemory(const VAddr addr, const u64 size) const { } } +void MemoryManager::RecalculateFlexibleUsageForDebug() { + std::scoped_lock lk{mutex}; + RecalculateFlexibleMappedUsageLocked(); +} + +bool MemoryManager::IsFlexibleCountedVmaType(VMAType type) const { + return type == VMAType::Flexible || type == VMAType::Code; +} + +bool MemoryManager::IsFlexibleCommittedVma(const VirtualMemoryArea& vma) const { + if (!vma.IsMapped()) { + return false; + } + + if (vma.type == VMAType::Flexible) { + return !vma.phys_areas.empty(); + } + + if (vma.type == VMAType::Code) { + // System modules should not consume the game's flexible memory. + return !vma.is_system_module; + } + return false; +} + +u64 MemoryManager::GetFlexibleRangeOverlapBytesLocked(VAddr virtual_addr, u64 size) const { + if (!IsFlexibleRegionConfigured() || size == 0) { + return 0; + } + + const VAddr aligned_start = Common::AlignDown(virtual_addr, 16_KB); + const u64 page_offset = virtual_addr - aligned_start; + if (size > std::numeric_limits::max() - page_offset) { + return 0; + } + const u64 aligned_size = Common::AlignUp(size + page_offset, 16_KB); + if (aligned_size == 0) { + return 0; + } + + const VAddr aligned_end = aligned_start + aligned_size; + const VAddr range_start = std::max(aligned_start, flexible_virtual_base); + const VAddr range_end = std::min(aligned_end, flexible_virtual_end); + return range_start < range_end ? range_end - range_start : 0; +} + +u64 MemoryManager::GetFlexibleMappedBytesInRangeLocked(VAddr virtual_addr, u64 size) const { + if (!IsFlexibleRegionConfigured() || size == 0) { + return 0; + } + + const VAddr aligned_start = Common::AlignDown(virtual_addr, 16_KB); + const u64 page_offset = virtual_addr - aligned_start; + if (size > std::numeric_limits::max() - page_offset) { + return 0; + } + const u64 aligned_size = Common::AlignUp(size + page_offset, 16_KB); + if (aligned_size == 0) { + return 0; + } + + const VAddr aligned_end = aligned_start + aligned_size; + const VAddr range_start = std::max(aligned_start, flexible_virtual_base); + const VAddr range_end = std::min(aligned_end, flexible_virtual_end); + if (range_start >= range_end) { + return 0; + } + + if (flexible_mapped_range_cache.valid && flexible_mapped_range_cache.revision == vma_revision && + flexible_mapped_range_cache.range_start == range_start && + flexible_mapped_range_cache.range_end == range_end) { + return flexible_mapped_range_cache.mapped_bytes; + } + + u64 mapped_bytes = 0; + auto it = vma_map.upper_bound(range_start); + if (it != vma_map.begin()) { + it = std::prev(it); + } + while (it != vma_map.end() && it->second.base < range_end) { + const auto& vma = it->second; + const bool counted_type = IsFlexibleCountedVmaType(vma.type); + if (!counted_type) { + ++it; + continue; + } + + const VAddr vma_end = vma.base + vma.size; + const VAddr overlap_start = std::max(range_start, vma.base); + const VAddr overlap_end = std::min(range_end, vma_end); + const bool committed = IsFlexibleCommittedVma(vma); + + if (overlap_start < overlap_end && committed) { + mapped_bytes += overlap_end - overlap_start; + } + + ++it; + } + + flexible_mapped_range_cache.range_start = range_start; + flexible_mapped_range_cache.range_end = range_end; + flexible_mapped_range_cache.mapped_bytes = mapped_bytes; + flexible_mapped_range_cache.revision = vma_revision; + flexible_mapped_range_cache.valid = true; + return mapped_bytes; +} + +void MemoryManager::InvalidateFlexibleMappedRangeCacheLocked() { + ++vma_revision; + flexible_mapped_range_cache.valid = false; +} + +void MemoryManager::AdjustFlexibleMappedUsageLocked(u64 mapped_before, u64 mapped_after) { + if (mapped_after >= mapped_before) { + flexible_mapped_usage += mapped_after - mapped_before; + } else { + const u64 delta = mapped_before - mapped_after; + flexible_mapped_usage = delta > flexible_mapped_usage ? 0 : flexible_mapped_usage - delta; + } +} + +void MemoryManager::RecalculateFlexibleMappedUsageLocked() { + if (!IsFlexibleRegionConfigured()) { + flexible_mapped_usage = 0; + return; + } + + flexible_mapped_usage = GetFlexibleMappedBytesInRangeLocked( + flexible_virtual_base, flexible_virtual_end - flexible_virtual_base); +} + VAddr MemoryManager::SearchFree(VAddr virtual_addr, u64 size, u32 alignment) { // Calculate the minimum and maximum addresses present in our address space. auto min_search_address = impl.SystemManagedVirtualBase(); diff --git a/src/core/memory.h b/src/core/memory.h index f9ae64942..756710a4d 100644 --- a/src/core/memory.h +++ b/src/core/memory.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 shadPS4 Emulator Project +// SPDX-FileCopyrightText: Copyright 2025-2026 shadPS4 Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #pragma once @@ -111,6 +111,7 @@ struct VirtualMemoryArea { std::string name = ""; s32 fd = 0; bool disallow_merge = false; + bool is_system_module = false; bool Contains(VAddr addr, u64 size) const { return addr >= base && (addr + size) <= (base + this->size); @@ -149,6 +150,9 @@ struct VirtualMemoryArea { if (name.compare(next.name) != 0) { return false; } + if (is_system_module != next.is_system_module) { + return false; + } return true; } @@ -157,9 +161,11 @@ struct VirtualMemoryArea { class MemoryManager { using PhysMap = std::map; using PhysHandle = PhysMap::iterator; + using PhysConstHandle = PhysMap::const_iterator; using VMAMap = std::map; using VMAHandle = VMAMap::iterator; + using VMAConstHandle = VMAMap::const_iterator; public: explicit MemoryManager(); @@ -181,8 +187,17 @@ public: return total_flexible_size; } + u64 GetUsedFlexibleSize() const { + return flexible_mapped_usage; + } + u64 GetAvailableFlexibleSize() const { - return total_flexible_size - flexible_usage; + const u64 used = GetUsedFlexibleSize(); + return used < total_flexible_size ? total_flexible_size - used : 0; + } + + bool IsFlexibleRegionConfigured() const { + return flexible_virtual_end > flexible_virtual_base; } VAddr SystemReservedVirtualBase() noexcept { @@ -252,7 +267,8 @@ public: s32 MapMemory(void** out_addr, VAddr virtual_addr, u64 size, MemoryProt prot, MemoryMapFlags flags, VMAType type, std::string_view name = "anon", - bool validate_dmem = false, PAddr phys_addr = -1, u64 alignment = 0); + bool validate_dmem = false, PAddr phys_addr = -1, u64 alignment = 0, + bool is_system_module = false); s32 MapFile(void** out_addr, VAddr virtual_addr, u64 size, MemoryProt prot, MemoryMapFlags flags, s32 fd, s64 phys_addr); @@ -288,26 +304,37 @@ public: void InvalidateMemory(VAddr addr, u64 size) const; + void RecalculateFlexibleUsageForDebug(); + private: VMAHandle FindVMA(VAddr target) { return std::prev(vma_map.upper_bound(target)); } + VMAConstHandle FindVMA(VAddr target) const { + return std::prev(vma_map.upper_bound(target)); + } PhysHandle FindDmemArea(PAddr target) { return std::prev(dmem_map.upper_bound(target)); } + PhysConstHandle FindDmemArea(PAddr target) const { + return std::prev(dmem_map.upper_bound(target)); + } PhysHandle FindFmemArea(PAddr target) { return std::prev(fmem_map.upper_bound(target)); } + PhysConstHandle FindFmemArea(PAddr target) const { + return std::prev(fmem_map.upper_bound(target)); + } - bool HasPhysicalBacking(VirtualMemoryArea vma) { + bool HasPhysicalBacking(const VirtualMemoryArea& vma) const { return vma.type == VMAType::Direct || vma.type == VMAType::Flexible || vma.type == VMAType::Pooled; } VMAHandle CreateArea(VAddr virtual_addr, u64 size, MemoryProt prot, MemoryMapFlags flags, - VMAType type, std::string_view name, u64 alignment); + VMAType type, std::string_view name, u64 alignment, bool is_system_module); VAddr SearchFree(VAddr virtual_addr, u64 size, u32 alignment); @@ -327,6 +354,20 @@ private: s32 UnmapMemoryImpl(VAddr virtual_addr, u64 size); + bool IsFlexibleCountedVmaType(VMAType type) const; + + bool IsFlexibleCommittedVma(const VirtualMemoryArea& vma) const; + + u64 GetFlexibleRangeOverlapBytesLocked(VAddr virtual_addr, u64 size) const; + + u64 GetFlexibleMappedBytesInRangeLocked(VAddr virtual_addr, u64 size) const; + + void InvalidateFlexibleMappedRangeCacheLocked(); + + void AdjustFlexibleMappedUsageLocked(u64 mapped_before, u64 mapped_after); + + void RecalculateFlexibleMappedUsageLocked(); + private: AddressSpace impl; PhysMap dmem_map; @@ -337,6 +378,18 @@ private: u64 total_direct_size{}; u64 total_flexible_size{}; u64 flexible_usage{}; + VAddr flexible_virtual_base{}; + VAddr flexible_virtual_end{}; + u64 flexible_mapped_usage{}; + struct FlexibleMappedRangeCache { + VAddr range_start{}; + VAddr range_end{}; + u64 mapped_bytes{}; + u64 revision{}; + bool valid{}; + }; + mutable FlexibleMappedRangeCache flexible_mapped_range_cache{}; + u64 vma_revision{}; u64 pool_budget{}; s32 sdk_version{}; Vulkan::Rasterizer* rasterizer{}; diff --git a/src/core/module.cpp b/src/core/module.cpp index 7e9d74a09..a356be6f6 100644 --- a/src/core/module.cpp +++ b/src/core/module.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project +// SPDX-FileCopyrightText: Copyright 2024-2026 shadPS4 Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #include "common/alignment.h" @@ -112,12 +112,20 @@ void Module::LoadModuleToMemory(u32& max_tls_index) { // Map module segments (and possible TLS trampolines) void** out_addr = reinterpret_cast(&base_virtual_addr); - memory->MapMemory(out_addr, ModuleLoadBase, aligned_base_size + TrampolineSize, + const bool is_system_module = IsSystemLib(); + memory->MapMemory(out_addr, ModuleLoadBase, aligned_base_size, MemoryProt::CpuReadWrite | MemoryProt::CpuExec, MemoryMapFlags::NoFlags, - VMAType::Code, name); + VMAType::Code, name, false, -1, 0, is_system_module); LOG_INFO(Core_Linker, "Loading module {} to {}", name, fmt::ptr(*out_addr)); #ifdef ARCH_X86_64 + void* trampoline_region = std::bit_cast(base_virtual_addr + aligned_base_size); + const int tramp_ret = memory->MapMemory( + &trampoline_region, base_virtual_addr + aligned_base_size, TrampolineSize, + MemoryProt::CpuReadWrite | MemoryProt::CpuExec, + MemoryMapFlags::Fixed | MemoryMapFlags::NoOverwrite, VMAType::File, "Trampoline"); + ASSERT_MSG(tramp_ret == 0, "Unable to map trampoline memory"); + // Initialize trampoline generator. void* trampoline_addr = std::bit_cast(base_virtual_addr + aligned_base_size); RegisterPatchModule(*out_addr, aligned_base_size, trampoline_addr, TrampolineSize);