From 11050a70320a3c9ed8e11e7ecd82462aa93d17cd Mon Sep 17 00:00:00 2001 From: Vishrut Sachan Date: Sun, 12 Apr 2026 14:37:25 +0530 Subject: [PATCH] ISO: Add metadata cache to speed up game list scanning (#18546) Every launch constructs a fresh iso_archive for each ISO game, which calls iso_form_hierarchy() and walks the full directory tree. On top of that, qt_utils opens a second iso_archive just for icon loading, so every ISO game ends up doing two full directory tree walks on every launch. This adds a metadata cache keyed by ISO path + mtime stored under fs::get_config_dir()/iso_cache/. Each entry stores the raw SFO binary, resolved icon/movie/audio paths and raw icon bytes. - On cache hit, iso_archive construction is skipped entirely for both game list scanning and icon loading - On cache miss archive is scanned as before and the result is persisted to disk - Cache is automatically invalidated when the ISO file's mtime changes Tested with a decrypted PS3 disc ISO (God of War III): - First launch writes cache files correctly to iso_cache/ - Second launch reads from cache with correct title and icon - touch game.iso correctly invalidates the cache and triggers a rescan --- rpcs3/Emu/CMakeLists.txt | 1 + rpcs3/Loader/iso_cache.cpp | 162 ++++++++++++++++++++++++++++++ rpcs3/Loader/iso_cache.h | 32 ++++++ rpcs3/emucore.vcxproj | 1 + rpcs3/rpcs3qt/game_list_frame.cpp | 108 ++++++++++++++++++-- rpcs3/rpcs3qt/game_list_frame.h | 1 + rpcs3/rpcs3qt/qt_utils.cpp | 10 ++ 7 files changed, 307 insertions(+), 8 deletions(-) create mode 100644 rpcs3/Loader/iso_cache.cpp create mode 100644 rpcs3/Loader/iso_cache.h diff --git a/rpcs3/Emu/CMakeLists.txt b/rpcs3/Emu/CMakeLists.txt index d691952fa0..c20b72f694 100644 --- a/rpcs3/Emu/CMakeLists.txt +++ b/rpcs3/Emu/CMakeLists.txt @@ -126,6 +126,7 @@ target_sources(rpcs3_emu PRIVATE ../Loader/PUP.cpp ../Loader/TAR.cpp ../Loader/ISO.cpp + ../Loader/iso_cache.cpp ../Loader/TROPUSR.cpp ../Loader/TRP.cpp ) diff --git a/rpcs3/Loader/iso_cache.cpp b/rpcs3/Loader/iso_cache.cpp new file mode 100644 index 0000000000..a90222a240 --- /dev/null +++ b/rpcs3/Loader/iso_cache.cpp @@ -0,0 +1,162 @@ +#include "stdafx.h" + +#include "iso_cache.h" +#include "Loader/PSF.h" +#include "util/yaml.hpp" +#include "util/fnv_hash.hpp" +#include "Utilities/File.h" + +#include + +LOG_CHANNEL(iso_cache_log, "ISOCache"); + +namespace +{ + std::string get_cache_dir() + { + const std::string dir = fs::get_cache_dir() + "cache/iso_cache/"; + fs::create_path(dir); + return dir; + } + + // FNV-64 hash of the ISO path used as the cache filename stem. + std::string get_cache_stem(const std::string& iso_path) + { + usz hash = rpcs3::fnv_seed; + for (const char c : iso_path) + { + hash ^= static_cast(c); + hash *= rpcs3::fnv_prime; + } + return fmt::format("%016llx", hash); + } +} + +namespace iso_cache +{ + bool load(const std::string& iso_path, iso_metadata_cache_entry& out_entry) + { + fs::stat_t iso_stat{}; + if (!fs::get_stat(iso_path, iso_stat) || iso_stat.is_directory) + { + return false; + } + + const std::string stem = get_cache_stem(iso_path); + const std::string dir = get_cache_dir(); + const std::string yml_path = dir + stem + ".yml"; + const std::string sfo_path = dir + stem + ".sfo"; + const std::string png_path = dir + stem + ".png"; + + const fs::file yml_file(yml_path); + if (!yml_file) + { + return false; + } + + auto [node, error] = yaml_load(yml_file.to_string()); + if (!error.empty()) + { + iso_cache_log.warning("Failed to parse cache YAML for '%s': %s", iso_path, error); + return false; + } + + // Reject stale entries. + const s64 cached_mtime = node["mtime"].as(0); + if (cached_mtime != iso_stat.mtime) + { + return false; + } + + const fs::file sfo_file(sfo_path); + if (!sfo_file) + { + return false; + } + + out_entry.mtime = cached_mtime; + out_entry.psf_data = sfo_file.to_vector(); + out_entry.icon_path = node["icon_path"].as(""); + out_entry.movie_path = node["movie_path"].as(""); + out_entry.audio_path = node["audio_path"].as(""); + + // Icon bytes are optional — game may have no icon. + if (const fs::file png_file(png_path); png_file) + { + out_entry.icon_data = png_file.to_vector(); + } + + return true; + } + + void save(const std::string& iso_path, const iso_metadata_cache_entry& entry) + { + const std::string stem = get_cache_stem(iso_path); + const std::string dir = get_cache_dir(); + const std::string yml_path = dir + stem + ".yml"; + const std::string sfo_path = dir + stem + ".sfo"; + const std::string png_path = dir + stem + ".png"; + + YAML::Emitter out; + out << YAML::BeginMap; + out << YAML::Key << "mtime" << YAML::Value << static_cast(entry.mtime); + out << YAML::Key << "icon_path" << YAML::Value << entry.icon_path; + out << YAML::Key << "movie_path" << YAML::Value << entry.movie_path; + out << YAML::Key << "audio_path" << YAML::Value << entry.audio_path; + out << YAML::EndMap; + + if (fs::pending_file yml_file(yml_path); yml_file.file) + { + yml_file.file.write(out.c_str(), out.size()); + yml_file.commit(); + } + else + { + iso_cache_log.warning("Failed to write cache YAML for '%s'", iso_path); + } + + if (!entry.psf_data.empty()) + { + if (fs::pending_file sfo_file(sfo_path); sfo_file.file) + { + sfo_file.file.write(entry.psf_data); + sfo_file.commit(); + } + } + + if (!entry.icon_data.empty()) + { + if (fs::pending_file png_file(png_path); png_file.file) + { + png_file.file.write(entry.icon_data); + png_file.commit(); + } + } + } + + void cleanup(const std::unordered_set& valid_iso_paths) + { + const std::string dir = get_cache_dir(); + + // Build a set of stems that should exist. + std::unordered_set valid_stems; + for (const std::string& path : valid_iso_paths) + { + valid_stems.insert(get_cache_stem(path)); + } + + // Delete any cache files whose stem is not in the valid set. + fs::dir cache_dir(dir); + fs::dir_entry entry{}; + while (cache_dir.read(entry)) + { + if (entry.name == "." || entry.name == "..") continue; + + const std::string stem = entry.name.substr(0, entry.name.find_last_of('.')); + if (valid_stems.find(stem) == valid_stems.end()) + { + fs::remove_file(dir + entry.name); + } + } + } +} diff --git a/rpcs3/Loader/iso_cache.h b/rpcs3/Loader/iso_cache.h new file mode 100644 index 0000000000..ca2f39e6a4 --- /dev/null +++ b/rpcs3/Loader/iso_cache.h @@ -0,0 +1,32 @@ +#pragma once + +#include "Loader/PSF.h" +#include "Utilities/File.h" +#include "util/types.hpp" + +#include +#include +#include + +// Cached metadata extracted from an ISO during game list scanning. +struct iso_metadata_cache_entry +{ + s64 mtime = 0; + std::vector psf_data{}; + std::string icon_path{}; + std::vector icon_data{}; + std::string movie_path{}; + std::string audio_path{}; +}; + +namespace iso_cache +{ + // Returns false if no valid cache entry exists or mtime has changed. + bool load(const std::string& iso_path, iso_metadata_cache_entry& out_entry); + + // Persists a populated cache entry to disk. + void save(const std::string& iso_path, const iso_metadata_cache_entry& entry); + + // Remove cache entries for ISOs that are no longer in the scanned set. + void cleanup(const std::unordered_set& valid_iso_paths); +} \ No newline at end of file diff --git a/rpcs3/emucore.vcxproj b/rpcs3/emucore.vcxproj index 42aab47a04..c89a066075 100644 --- a/rpcs3/emucore.vcxproj +++ b/rpcs3/emucore.vcxproj @@ -551,6 +551,7 @@ + diff --git a/rpcs3/rpcs3qt/game_list_frame.cpp b/rpcs3/rpcs3qt/game_list_frame.cpp index 36e26cf43d..d2af663b7d 100644 --- a/rpcs3/rpcs3qt/game_list_frame.cpp +++ b/rpcs3/rpcs3qt/game_list_frame.cpp @@ -16,6 +16,7 @@ #include "Emu/system_utils.hpp" #include "Loader/PSF.h" #include "Loader/ISO.h" +#include "Loader/iso_cache.h" #include "util/types.hpp" #include "Utilities/File.h" #include "util/sysinfo.hpp" @@ -530,14 +531,29 @@ void game_list_frame::OnParsingFinished() (const std::string& dir_or_elf) { std::unique_ptr archive; - if (is_file_iso(dir_or_elf)) + iso_metadata_cache_entry cache_entry{}; + const bool is_iso = is_file_iso(dir_or_elf); + + if (is_iso) { - archive = std::make_unique(dir_or_elf); + // Only construct iso_archive (which walks the full directory tree) + // when no valid cache entry exists for this ISO path + mtime. + if (!iso_cache::load(dir_or_elf, cache_entry)) + { + archive = std::make_unique(dir_or_elf); + } + // Track this ISO path for cache cleanup after scan completes. + std::lock_guard lock(m_path_mutex); + m_scanned_iso_paths.insert(dir_or_elf); } - const auto file_exists = [&archive](const std::string& path) + const auto file_exists = [&archive, &cache_entry](const std::string& path) { - return archive ? archive->is_file(path) : fs::is_file(path); + if (archive) return archive->is_file(path); + // On cache hit, paths inside the ISO are not accessible via fs::is_file. + // Return false here — cache hit paths are handled separately. + if (!cache_entry.psf_data.empty()) return false; + return fs::is_file(path); }; gui_game_info game{}; @@ -545,10 +561,34 @@ void game_list_frame::OnParsingFinished() const Localized thread_localized; - const std::string sfo_dir = archive ? "PS3_GAME" : rpcs3::utils::get_sfo_dir_from_game_path(dir_or_elf); + const std::string sfo_dir = (archive || !cache_entry.psf_data.empty()) ? "PS3_GAME" : rpcs3::utils::get_sfo_dir_from_game_path(dir_or_elf); const std::string sfo_path = sfo_dir + "/PARAM.SFO"; - const psf::registry psf = archive ? archive->open_psf(sfo_path) : psf::load_object(sfo_path); + // Load PSF: from archive on cache miss, rehydrate from cached SFO bytes on hit. + psf::registry psf{}; + if (archive) + { + psf = archive->open_psf(sfo_path); + } + else if (!cache_entry.psf_data.empty()) + { + psf = psf::load_object(fs::make_stream>(std::vector(cache_entry.psf_data)), sfo_path); + // Fallback to archive scan if cached PSF is corrupted or missing critical fields. + const bool psf_valid = !psf::get_string(psf, "TITLE_ID", "").empty() + && !psf::get_string(psf, "TITLE", "").empty() + && !psf::get_string(psf, "CATEGORY", "").empty(); + if (!psf_valid) + { + archive = std::make_unique(dir_or_elf); + psf = archive->open_psf(sfo_path); + cache_entry = {}; // Reset so the cache gets rewritten after scan. + } + } + else + { + psf = psf::load_object(sfo_path); + } + const std::string_view title_id = psf::get_string(psf, "TITLE_ID", ""); if (title_id.empty()) @@ -616,19 +656,32 @@ void game_list_frame::OnParsingFinished() if (game.info.icon_path.empty()) { - if (std::string icon_path = sfo_dir + "/" + localized_icon; file_exists(icon_path)) + if (!cache_entry.icon_path.empty()) + { + // Cache hit — icon path already resolved on a previous scan. + game.info.icon_path = cache_entry.icon_path; + game.icon_in_archive = true; + } + else if (std::string icon_path = sfo_dir + "/" + localized_icon; file_exists(icon_path)) { game.info.icon_path = std::move(icon_path); + game.icon_in_archive = archive && archive->exists(game.info.icon_path); } else { game.info.icon_path = sfo_dir + "/ICON0.PNG"; + game.icon_in_archive = archive && archive->exists(game.info.icon_path); } - game.icon_in_archive = archive && archive->exists(game.info.icon_path); } if (play_hover_movies) { + if (!cache_entry.movie_path.empty() && !archive) + { + // Cache hit — restore previously resolved movie path. + game.info.movie_path = cache_entry.movie_path; + game.has_hover_pam = true; + } if (std::string movie_path = game_icon_path + game.info.serial + "/hover.gif"; file_exists(movie_path)) { game.info.movie_path = std::move(movie_path); @@ -648,6 +701,12 @@ void game_list_frame::OnParsingFinished() if (play_hover_music) { + if(!cache_entry.audio_path.empty() && !archive) + { + // Cache hit — restore previously resolved audio path. + game.info.audio_path = cache_entry.audio_path; + game.has_audio_file = true; + } if (std::string audio_path = sfo_dir + "/SND0.AT3"; file_exists(audio_path)) { game.info.audio_path = std::move(audio_path); @@ -655,6 +714,35 @@ void game_list_frame::OnParsingFinished() } } + // On cache miss for an ISO, persist the resolved metadata so subsequent + // launches skip iso_archive construction entirely. + if (archive && is_iso) + { + fs::stat_t iso_stat{}; + if (fs::get_stat(dir_or_elf, iso_stat)) + { + cache_entry.mtime = iso_stat.mtime; + cache_entry.psf_data = psf::save_object(psf); + cache_entry.icon_path = game.info.icon_path; + cache_entry.movie_path = game.info.movie_path; + cache_entry.audio_path = game.info.audio_path; + + // Cache raw icon bytes so load_iso_icon can skip archive open. + if (game.icon_in_archive) + { + auto icon_file = archive->open(game.info.icon_path); + const auto icon_size = icon_file.size(); + if (icon_size > 0) + { + cache_entry.icon_data.resize(icon_size); + icon_file.read(cache_entry.icon_data.data(), icon_size); + } + } + + iso_cache::save(dir_or_elf, cache_entry); + } + } + const QString serial = QString::fromStdString(game.info.serial); m_games_mutex.lock(); @@ -817,6 +905,9 @@ void game_list_frame::OnRefreshFinished() WaitAndAbortSizeCalcThreads(); WaitAndAbortRepaintThreads(); + // Remove cache entries for ISOs that are no longer present in the scanned paths. + iso_cache::cleanup(m_scanned_iso_paths); + for (auto&& g : m_games.pop_all()) { m_game_data.push_back(g); @@ -903,6 +994,7 @@ void game_list_frame::OnRefreshFinished() m_serials.clear(); m_path_list.clear(); m_path_entries.clear(); + m_scanned_iso_paths.clear(); Refresh(); diff --git a/rpcs3/rpcs3qt/game_list_frame.h b/rpcs3/rpcs3qt/game_list_frame.h index 637229bf60..b01dbd5a6e 100644 --- a/rpcs3/rpcs3qt/game_list_frame.h +++ b/rpcs3/rpcs3qt/game_list_frame.h @@ -181,6 +181,7 @@ private: std::vector m_path_entries; shared_mutex m_path_mutex; std::set m_path_list; + std::unordered_set m_scanned_iso_paths; QSet m_serials; QMutex m_games_mutex; lf_queue m_games; diff --git a/rpcs3/rpcs3qt/qt_utils.cpp b/rpcs3/rpcs3qt/qt_utils.cpp index ede9f6be9a..fe7bb6f8d6 100644 --- a/rpcs3/rpcs3qt/qt_utils.cpp +++ b/rpcs3/rpcs3qt/qt_utils.cpp @@ -13,6 +13,7 @@ #include "Emu/system_utils.hpp" #include "Utilities/File.h" #include "Loader/ISO.h" +#include "Loader/iso_cache.h" #include LOG_CHANNEL(gui_log, "GUI"); @@ -709,6 +710,15 @@ namespace gui if (icon_path.empty() || archive_path.empty()) return false; if (!is_file_iso(archive_path)) return false; + // Check cache first — avoids constructing a full iso_archive just for the icon. + iso_metadata_cache_entry cache_entry{}; + if (iso_cache::load(archive_path, cache_entry) && !cache_entry.icon_data.empty()) + { + const QByteArray data(reinterpret_cast(cache_entry.icon_data.data()), + static_cast(cache_entry.icon_data.size())); + return icon.loadFromData(data); + } + iso_archive archive(archive_path); if (!archive.exists(icon_path)) return false;