ISO: Add metadata cache to speed up game list scanning (#18546)

Every launch constructs a fresh iso_archive for each ISO game, which
calls iso_form_hierarchy() and walks the full directory tree. On top of
that, qt_utils opens a second iso_archive just for icon loading, so
every ISO game ends up doing two full directory tree walks on every
launch.
This adds a metadata cache keyed by ISO path + mtime stored under
fs::get_config_dir()/iso_cache/. Each entry stores the raw SFO binary,
resolved icon/movie/audio paths and raw icon bytes.

- On cache hit, iso_archive construction is skipped entirely for both
game list scanning and icon loading
- On cache miss archive is scanned as before and the result is persisted
to disk
- Cache is automatically invalidated when the ISO file's mtime changes

Tested with a decrypted PS3 disc ISO (God of War III):

- First launch writes cache files correctly to iso_cache/
- Second launch reads from cache with correct title and icon
- touch game.iso correctly invalidates the cache and triggers a rescan

rpcs3/Emu/CMakeLists.txt

@@ -126,6 +126,7 @@ target_sources(rpcs3_emu PRIVATE
     ../Loader/PUP.cpp
     ../Loader/TAR.cpp
     ../Loader/ISO.cpp
+    ../Loader/iso_cache.cpp
     ../Loader/TROPUSR.cpp
     ../Loader/TRP.cpp
 )

rpcs3/Loader/iso_cache.cpp

@@ -0,0 +1,162 @@
+#include "stdafx.h"
+
+#include "iso_cache.h"
+#include "Loader/PSF.h"
+#include "util/yaml.hpp"
+#include "util/fnv_hash.hpp"
+#include "Utilities/File.h"
+
+#include <unordered_set>
+
+LOG_CHANNEL(iso_cache_log, "ISOCache");
+
+namespace
+{
+	std::string get_cache_dir()
+	{
+		const std::string dir = fs::get_cache_dir() + "cache/iso_cache/";
+		fs::create_path(dir);
+		return dir;
+	}
+
+	// FNV-64 hash of the ISO path used as the cache filename stem.
+	std::string get_cache_stem(const std::string& iso_path)
+	{
+		usz hash = rpcs3::fnv_seed;
+		for (const char c : iso_path)
+		{
+			hash ^= static_cast<u8>(c);
+			hash *= rpcs3::fnv_prime;
+		}
+		return fmt::format("%016llx", hash);
+	}
+}
+
+namespace iso_cache
+{
+	bool load(const std::string& iso_path, iso_metadata_cache_entry& out_entry)
+	{
+		fs::stat_t iso_stat{};
+		if (!fs::get_stat(iso_path, iso_stat) || iso_stat.is_directory)
+		{
+			return false;
+		}
+
+		const std::string stem     = get_cache_stem(iso_path);
+		const std::string dir      = get_cache_dir();
+		const std::string yml_path = dir + stem + ".yml";
+		const std::string sfo_path = dir + stem + ".sfo";
+		const std::string png_path = dir + stem + ".png";
+
+		const fs::file yml_file(yml_path);
+		if (!yml_file)
+		{
+			return false;
+		}
+
+		auto [node, error] = yaml_load(yml_file.to_string());
+		if (!error.empty())
+		{
+			iso_cache_log.warning("Failed to parse cache YAML for '%s': %s", iso_path, error);
+			return false;
+		}
+
+		// Reject stale entries.
+		const s64 cached_mtime = node["mtime"].as<s64>(0);
+		if (cached_mtime != iso_stat.mtime)
+		{
+			return false;
+		}
+
+		const fs::file sfo_file(sfo_path);
+		if (!sfo_file)
+		{
+			return false;
+		}
+
+		out_entry.mtime      = cached_mtime;
+		out_entry.psf_data   = sfo_file.to_vector<u8>();
+		out_entry.icon_path  = node["icon_path"].as<std::string>("");
+		out_entry.movie_path = node["movie_path"].as<std::string>("");
+		out_entry.audio_path = node["audio_path"].as<std::string>("");
+
+		// Icon bytes are optional — game may have no icon.
+		if (const fs::file png_file(png_path); png_file)
+		{
+			out_entry.icon_data = png_file.to_vector<u8>();
+		}
+
+		return true;
+	}
+
+	void save(const std::string& iso_path, const iso_metadata_cache_entry& entry)
+	{
+		const std::string stem     = get_cache_stem(iso_path);
+		const std::string dir      = get_cache_dir();
+		const std::string yml_path = dir + stem + ".yml";
+		const std::string sfo_path = dir + stem + ".sfo";
+		const std::string png_path = dir + stem + ".png";
+
+		YAML::Emitter out;
+		out << YAML::BeginMap;
+		out << YAML::Key << "mtime"      << YAML::Value << static_cast<long long>(entry.mtime);
+		out << YAML::Key << "icon_path"  << YAML::Value << entry.icon_path;
+		out << YAML::Key << "movie_path" << YAML::Value << entry.movie_path;
+		out << YAML::Key << "audio_path" << YAML::Value << entry.audio_path;
+		out << YAML::EndMap;
+
+		if (fs::pending_file yml_file(yml_path); yml_file.file)
+		{
+			yml_file.file.write(out.c_str(), out.size());
+			yml_file.commit();
+		}
+		else
+		{
+			iso_cache_log.warning("Failed to write cache YAML for '%s'", iso_path);
+		}
+
+		if (!entry.psf_data.empty())
+		{
+			if (fs::pending_file sfo_file(sfo_path); sfo_file.file)
+			{
+				sfo_file.file.write(entry.psf_data);
+				sfo_file.commit();
+			}
+		}
+
+		if (!entry.icon_data.empty())
+		{
+			if (fs::pending_file png_file(png_path); png_file.file)
+			{
+				png_file.file.write(entry.icon_data);
+				png_file.commit();
+			}
+		}
+	}
+
+	void cleanup(const std::unordered_set<std::string>& valid_iso_paths)
+	{
+		const std::string dir = get_cache_dir();
+
+		// Build a set of stems that should exist.
+		std::unordered_set<std::string> valid_stems;
+		for (const std::string& path : valid_iso_paths)
+		{
+			valid_stems.insert(get_cache_stem(path));
+		}
+
+		// Delete any cache files whose stem is not in the valid set.
+		fs::dir cache_dir(dir);
+		fs::dir_entry entry{};
+		while (cache_dir.read(entry))
+		{
+			if (entry.name == "." || entry.name == "..") continue;
+
+			const std::string stem = entry.name.substr(0, entry.name.find_last_of('.'));
+			if (valid_stems.find(stem) == valid_stems.end())
+			{
+				fs::remove_file(dir + entry.name);
+			}
+		}
+	}
+}

rpcs3/Loader/iso_cache.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "Loader/PSF.h"
+#include "Utilities/File.h"
+#include "util/types.hpp"
+
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+// Cached metadata extracted from an ISO during game list scanning.
+struct iso_metadata_cache_entry
+{
+	s64 mtime = 0;
+	std::vector<u8> psf_data{};
+	std::string icon_path{};
+	std::vector<u8> icon_data{};
+	std::string movie_path{};
+	std::string audio_path{};
+};
+
+namespace iso_cache
+{
+	// Returns false if no valid cache entry exists or mtime has changed.
+	bool load(const std::string& iso_path, iso_metadata_cache_entry& out_entry);
+
+	// Persists a populated cache entry to disk.
+	void save(const std::string& iso_path, const iso_metadata_cache_entry& entry);
+
+	// Remove cache entries for ISOs that are no longer in the scanned set.
+	void cleanup(const std::unordered_set<std::string>& valid_iso_paths);
+}

rpcs3/emucore.vcxproj

@@ -551,6 +551,7 @@
     <ClCompile Include="Loader\PUP.cpp" />
     <ClCompile Include="Loader\TAR.cpp" />
     <ClCompile Include="Loader\ISO.cpp" />
+    <ClCompile Include="Loader\iso_cache.cpp" />
     <ClCompile Include="Loader\mself.cpp" />
     <ClCompile Include="Loader\TROPUSR.cpp" />
     <ClCompile Include="Loader\TRP.cpp" />

rpcs3/rpcs3qt/game_list_frame.cpp

@@ -16,6 +16,7 @@
 #include "Emu/system_utils.hpp"
 #include "Loader/PSF.h"
 #include "Loader/ISO.h"
+#include "Loader/iso_cache.h"
 #include "util/types.hpp"
 #include "Utilities/File.h"
 #include "util/sysinfo.hpp"
@@ -530,14 +531,29 @@ void game_list_frame::OnParsingFinished()
 	                       (const std::string& dir_or_elf)
 	{
 		std::unique_ptr<iso_archive> archive;
-		if (is_file_iso(dir_or_elf))
+		iso_metadata_cache_entry cache_entry{};
+		const bool is_iso = is_file_iso(dir_or_elf);
+		
+		if (is_iso)
 		{
-			archive = std::make_unique<iso_archive>(dir_or_elf);
+			// Only construct iso_archive (which walks the full directory tree)
+			// when no valid cache entry exists for this ISO path + mtime.
+			if (!iso_cache::load(dir_or_elf, cache_entry))
+			{
+				archive = std::make_unique<iso_archive>(dir_or_elf);
+			}
+			// Track this ISO path for cache cleanup after scan completes.
+			std::lock_guard lock(m_path_mutex);
+			m_scanned_iso_paths.insert(dir_or_elf);
 		}
 
-		const auto file_exists = [&archive](const std::string& path)
+		const auto file_exists = [&archive, &cache_entry](const std::string& path)
 		{
-			return archive ? archive->is_file(path) : fs::is_file(path);
+			if (archive) return archive->is_file(path);
+			// On cache hit, paths inside the ISO are not accessible via fs::is_file.
+			// Return false here — cache hit paths are handled separately.
+			if (!cache_entry.psf_data.empty()) return false;
+			return fs::is_file(path);
 		};
 
 		gui_game_info game{};
@@ -545,10 +561,34 @@ void game_list_frame::OnParsingFinished()
 
 		const Localized thread_localized;
 
-		const std::string sfo_dir = archive ? "PS3_GAME" : rpcs3::utils::get_sfo_dir_from_game_path(dir_or_elf);
+		const std::string sfo_dir = (archive || !cache_entry.psf_data.empty()) ? "PS3_GAME" : rpcs3::utils::get_sfo_dir_from_game_path(dir_or_elf);
 		const std::string sfo_path = sfo_dir + "/PARAM.SFO";
 
-		const psf::registry psf = archive ? archive->open_psf(sfo_path) : psf::load_object(sfo_path);
+		// Load PSF: from archive on cache miss, rehydrate from cached SFO bytes on hit.
+		psf::registry psf{};
+		if (archive)
+		{
+			psf = archive->open_psf(sfo_path);
+		}
+		else if (!cache_entry.psf_data.empty())
+		{
+			psf = psf::load_object(fs::make_stream<std::vector<u8>>(std::vector<u8>(cache_entry.psf_data)), sfo_path);
+			// Fallback to archive scan if cached PSF is corrupted or missing critical fields.
+			const bool psf_valid = !psf::get_string(psf, "TITLE_ID", "").empty()
+				&& !psf::get_string(psf, "TITLE", "").empty()
+				&& !psf::get_string(psf, "CATEGORY", "").empty();
+			if (!psf_valid)
+			{
+				archive = std::make_unique<iso_archive>(dir_or_elf);
+				psf = archive->open_psf(sfo_path);
+				cache_entry = {}; // Reset so the cache gets rewritten after scan.
+			}
+		}
+		else
+		{
+			psf = psf::load_object(sfo_path);
+		}
+
 		const std::string_view title_id = psf::get_string(psf, "TITLE_ID", "");
 
 		if (title_id.empty())
@@ -616,19 +656,32 @@ void game_list_frame::OnParsingFinished()
 
 		if (game.info.icon_path.empty())
 		{
-			if (std::string icon_path = sfo_dir + "/" + localized_icon; file_exists(icon_path))
+			if (!cache_entry.icon_path.empty())
+			{
+				// Cache hit — icon path already resolved on a previous scan.
+				game.info.icon_path = cache_entry.icon_path;
+				game.icon_in_archive = true;
+			}
+			else if (std::string icon_path = sfo_dir + "/" + localized_icon; file_exists(icon_path))
 			{
 				game.info.icon_path = std::move(icon_path);
+				game.icon_in_archive = archive && archive->exists(game.info.icon_path);
 			}
 			else
 			{
 				game.info.icon_path = sfo_dir + "/ICON0.PNG";
+				game.icon_in_archive = archive && archive->exists(game.info.icon_path);
 			}
-			game.icon_in_archive = archive && archive->exists(game.info.icon_path);
 		}
 
 		if (play_hover_movies)
 		{
+			if (!cache_entry.movie_path.empty() && !archive)
+			{
+				// Cache hit — restore previously resolved movie path.
+				game.info.movie_path = cache_entry.movie_path;
+				game.has_hover_pam = true;
+			}
 			if (std::string movie_path = game_icon_path + game.info.serial + "/hover.gif"; file_exists(movie_path))
 			{
 				game.info.movie_path = std::move(movie_path);
@@ -648,6 +701,12 @@ void game_list_frame::OnParsingFinished()
 
 		if (play_hover_music)
 		{
+			if(!cache_entry.audio_path.empty() && !archive)
+			{
+				// Cache hit — restore previously resolved audio path.
+				game.info.audio_path = cache_entry.audio_path;
+				game.has_audio_file = true;
+			}
 			if (std::string audio_path = sfo_dir + "/SND0.AT3"; file_exists(audio_path))
 			{
 				game.info.audio_path = std::move(audio_path);
@@ -655,6 +714,35 @@ void game_list_frame::OnParsingFinished()
 			}
 		}
 
+		// On cache miss for an ISO, persist the resolved metadata so subsequent
+		// launches skip iso_archive construction entirely.
+		if (archive && is_iso)
+		{
+			fs::stat_t iso_stat{};
+			if (fs::get_stat(dir_or_elf, iso_stat))
+			{
+				cache_entry.mtime      = iso_stat.mtime;
+				cache_entry.psf_data   = psf::save_object(psf);
+				cache_entry.icon_path  = game.info.icon_path;
+				cache_entry.movie_path = game.info.movie_path;
+				cache_entry.audio_path = game.info.audio_path;
+
+				// Cache raw icon bytes so load_iso_icon can skip archive open.
+				if (game.icon_in_archive)
+				{
+					auto icon_file = archive->open(game.info.icon_path);
+					const auto icon_size = icon_file.size();
+					if (icon_size > 0)
+					{
+						cache_entry.icon_data.resize(icon_size);
+						icon_file.read(cache_entry.icon_data.data(), icon_size);
+					}
+				}
+
+				iso_cache::save(dir_or_elf, cache_entry);
+			}
+		}
+
 		const QString serial = QString::fromStdString(game.info.serial);
 
 		m_games_mutex.lock();
@@ -817,6 +905,9 @@ void game_list_frame::OnRefreshFinished()
 	WaitAndAbortSizeCalcThreads();
 	WaitAndAbortRepaintThreads();
 
+	// Remove cache entries for ISOs that are no longer present in the scanned paths.
+	iso_cache::cleanup(m_scanned_iso_paths);
+
 	for (auto&& g : m_games.pop_all())
 	{
 		m_game_data.push_back(g);
@@ -903,6 +994,7 @@ void game_list_frame::OnRefreshFinished()
 	m_serials.clear();
 	m_path_list.clear();
 	m_path_entries.clear();
+	m_scanned_iso_paths.clear();
 
 	Refresh();
 

rpcs3/rpcs3qt/game_list_frame.h

@@ -181,6 +181,7 @@ private:
 	std::vector<path_entry> m_path_entries;
 	shared_mutex m_path_mutex;
 	std::set<std::string> m_path_list;
+	std::unordered_set<std::string> m_scanned_iso_paths;
 	QSet<QString> m_serials;
 	QMutex m_games_mutex;
 	lf_queue<game_info> m_games;

rpcs3/rpcs3qt/qt_utils.cpp

@@ -13,6 +13,7 @@
 #include "Emu/system_utils.hpp"
 #include "Utilities/File.h"
 #include "Loader/ISO.h"
+#include "Loader/iso_cache.h"
 #include <cmath>
 
 LOG_CHANNEL(gui_log, "GUI");
@@ -709,6 +710,15 @@ namespace gui
 			if (icon_path.empty() || archive_path.empty()) return false;
 			if (!is_file_iso(archive_path)) return false;
 
+			// Check cache first — avoids constructing a full iso_archive just for the icon.
+			iso_metadata_cache_entry cache_entry{};
+			if (iso_cache::load(archive_path, cache_entry) && !cache_entry.icon_data.empty())
+			{
+				const QByteArray data(reinterpret_cast<const char*>(cache_entry.icon_data.data()),
+				                      static_cast<qsizetype>(cache_entry.icon_data.size()));
+				return icon.loadFromData(data);
+			}
+
 			iso_archive archive(archive_path);
 			if (!archive.exists(icon_path)) return false;