From 405baed8053be7b6512062b1ba3a0317b8cf439b Mon Sep 17 00:00:00 2001 From: Jordan Woyak Date: Tue, 28 Oct 2025 22:46:56 -0500 Subject: [PATCH 1/4] Common: Add a DirectIOFile class that allows for copies which are entirely thread safe. --- Source/Core/Common/CMakeLists.txt | 2 + Source/Core/Common/DirectIOFile.cpp | 372 +++++++++++++++++++++++ Source/Core/Common/DirectIOFile.h | 150 +++++++++ Source/Core/DolphinLib.props | 2 + Source/UnitTests/Common/FileUtilTest.cpp | 265 ++++++++++++++++ 5 files changed, 791 insertions(+) create mode 100644 Source/Core/Common/DirectIOFile.cpp create mode 100644 Source/Core/Common/DirectIOFile.h diff --git a/Source/Core/Common/CMakeLists.txt b/Source/Core/Common/CMakeLists.txt index cf209ab2279..e833893a9de 100644 --- a/Source/Core/Common/CMakeLists.txt +++ b/Source/Core/Common/CMakeLists.txt @@ -52,6 +52,8 @@ add_library(common Debug/Threads.h Debug/Watches.cpp Debug/Watches.h + DirectIOFile.cpp + DirectIOFile.h DynamicLibrary.cpp DynamicLibrary.h ENet.cpp diff --git a/Source/Core/Common/DirectIOFile.cpp b/Source/Core/Common/DirectIOFile.cpp new file mode 100644 index 00000000000..d5e805529de --- /dev/null +++ b/Source/Core/Common/DirectIOFile.cpp @@ -0,0 +1,372 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "Common/DirectIOFile.h" + +#include + +#if defined(_WIN32) +#include + +#include + +#include "Common/Buffer.h" +#include "Common/CommonFuncs.h" +#include "Common/MathUtil.h" +#include "Common/StringUtil.h" +#else +#include + +#include +#include +#include + +#ifdef ANDROID +#include "jni/AndroidCommon/AndroidCommon.h" + +#include "Common/Lazy.h" +#endif + +#include "Common/FileUtil.h" +#endif + +#include "Common/Assert.h" + +namespace File +{ +DirectIOFile::DirectIOFile() = default; + +DirectIOFile::~DirectIOFile() +{ + Close(); +} + +DirectIOFile::DirectIOFile(const DirectIOFile& other) +{ + *this = other.Duplicate(); +} + +DirectIOFile& DirectIOFile::operator=(const DirectIOFile& other) +{ + return *this = other.Duplicate(); +} + +DirectIOFile::DirectIOFile(DirectIOFile&& other) +{ + Swap(other); +} + +DirectIOFile& DirectIOFile::operator=(DirectIOFile&& other) +{ + Close(); + Swap(other); + return *this; +} + +DirectIOFile::DirectIOFile(const std::string& path, AccessMode access_mode, OpenMode open_mode) +{ + Open(path, access_mode, open_mode); +} + +bool DirectIOFile::Open(const std::string& path, AccessMode access_mode, OpenMode open_mode) +{ + ASSERT(!IsOpen()); + + if (open_mode == OpenMode::Default) + open_mode = (access_mode == AccessMode::Write) ? OpenMode::Truncate : OpenMode::Existing; + + // This is not a sensible combination. Fail here to not rely on OS-specific behaviors. + if (access_mode == AccessMode::Read && open_mode == OpenMode::Truncate) + return false; + +#if defined(_WIN32) + DWORD desired_access = GENERIC_READ | GENERIC_WRITE; + if (access_mode == AccessMode::Read) + desired_access = GENERIC_READ; + else if (access_mode == AccessMode::Write) + desired_access = GENERIC_WRITE; + + // Allow deleting and renaming through our handle. + desired_access |= DELETE; + + // All sharing is allowed to more closely match default behavior on other OSes. + constexpr DWORD share_mode = FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE; + + DWORD creation_disposition = OPEN_ALWAYS; + if (open_mode == OpenMode::Truncate) + creation_disposition = CREATE_ALWAYS; + else if (open_mode == OpenMode::Create) + creation_disposition = CREATE_NEW; + else if (open_mode == OpenMode::Existing) + creation_disposition = OPEN_EXISTING; + + m_handle = CreateFile(UTF8ToTStr(path).c_str(), desired_access, share_mode, nullptr, + creation_disposition, FILE_ATTRIBUTE_NORMAL, nullptr); + if (!IsOpen()) + WARN_LOG_FMT(COMMON, "CreateFile: {}", Common::GetLastErrorString()); + +#else +#if defined(ANDROID) + if (IsPathAndroidContent(path)) + { + // Android documentation says that "w" may or may not truncate. + // In case it does, we'll use "rw" when we don't want truncation. + // This wrongly enables read access when we don't need it. + if (access_mode == AccessMode::Write && open_mode != OpenMode::Truncate) + access_mode = AccessMode::ReadAndWrite; + + std::string open_mode_str = "rw"; + if (access_mode == AccessMode::Read) + open_mode_str = "r"; + else if (access_mode == AccessMode::Write) + open_mode_str = "w"; + + // FYI: File::Exists can be slow on Android. + Common::Lazy file_exists{[&] { return Exists(path); }}; + + // A few features are emulated in a non-atomic manner. + if (open_mode == OpenMode::Existing) + { + if (access_mode != AccessMode::Read && !*file_exists) + return false; + } + else + { + if (open_mode == OpenMode::Truncate) + open_mode_str += 't'; + else if (open_mode == OpenMode::Create && *file_exists) + return false; + + // Modes other than `Existing` may create a file, but "r" won't do that automatically. + if (access_mode == AccessMode::Read && !*file_exists) + CreateEmptyFile(path); + } + + m_fd = OpenAndroidContent(path, open_mode_str); + + return IsOpen(); + } +#endif + int flags = O_RDWR; + if (access_mode == AccessMode::Read) + flags = O_RDONLY; + else if (access_mode == AccessMode::Write) + flags = O_WRONLY; + + if (open_mode == OpenMode::Truncate) + flags |= O_CREAT | O_TRUNC; + else if (open_mode == OpenMode::Create) + flags |= O_CREAT | O_EXCL; + else if (open_mode != OpenMode::Existing) + flags |= O_CREAT; + + m_fd = open(path.c_str(), flags, 0666); + +#endif + + return IsOpen(); +} + +bool DirectIOFile::Close() +{ + if (!IsOpen()) + return false; + + m_current_offset = 0; + +#if defined(_WIN32) + return CloseHandle(std::exchange(m_handle, INVALID_HANDLE_VALUE)) != 0; +#else + return close(std::exchange(m_fd, -1)) == 0; +#endif +} + +bool DirectIOFile::IsOpen() const +{ +#if defined(_WIN32) + return m_handle != INVALID_HANDLE_VALUE; +#else + return m_fd != -1; +#endif +} + +#if defined(_WIN32) +template +static bool OverlappedTransfer(HANDLE handle, u64 offset, auto* data_ptr, u64 size) +{ + // ReadFile/WriteFile take a 32bit size so we must loop to handle our 64bit size. + while (true) + { + OVERLAPPED overlapped{}; + overlapped.Offset = DWORD(offset); + overlapped.OffsetHigh = DWORD(offset >> 32); + + DWORD bytes_transferred{}; + if (TransferFunc(handle, data_ptr, MathUtil::SaturatingCast(size), &bytes_transferred, + &overlapped) == 0) + { + ERROR_LOG_FMT(COMMON, "OverlappedTransfer: {}", Common::GetLastErrorString()); + return false; + } + + size -= bytes_transferred; + + if (size == 0) + return true; + + offset += bytes_transferred; + data_ptr += bytes_transferred; + } +} +#endif + +bool DirectIOFile::OffsetRead(u64 offset, u8* out_ptr, u64 size) +{ +#if defined(_WIN32) + return OverlappedTransfer(m_handle, offset, out_ptr, size); +#else + return pread(m_fd, out_ptr, size, off_t(offset)) == ssize_t(size); +#endif +} + +bool DirectIOFile::OffsetWrite(u64 offset, const u8* in_ptr, u64 size) +{ +#if defined(_WIN32) + return OverlappedTransfer(m_handle, offset, in_ptr, size); +#else + return pwrite(m_fd, in_ptr, size, off_t(offset)) == ssize_t(size); +#endif +} + +u64 DirectIOFile::GetSize() const +{ +#if defined(_WIN32) + LARGE_INTEGER result{}; + if (GetFileSizeEx(m_handle, &result) != 0) + return result.QuadPart; +#else + struct stat st{}; + if (fstat(m_fd, &st) == 0) + return st.st_size; +#endif + + return 0; +} + +bool DirectIOFile::Seek(s64 offset, SeekOrigin origin) +{ + if (!IsOpen()) + return false; + + u64 reference_pos = 0; + switch (origin) + { + case SeekOrigin::Current: + reference_pos = m_current_offset; + break; + case SeekOrigin::End: + reference_pos = GetSize(); + break; + default: + break; + } + + // Don't let our current offset underflow. + if (offset < 0 && u64(-offset) > reference_pos) + return false; + + m_current_offset = reference_pos + offset; + return true; +} + +bool DirectIOFile::Flush() +{ +#if defined(_WIN32) + return FlushFileBuffers(m_handle) != 0; +#else + return fsync(m_fd) == 0; +#endif +} + +void DirectIOFile::Swap(DirectIOFile& other) +{ +#if defined(_WIN32) + std::swap(m_handle, other.m_handle); +#else + std::swap(m_fd, other.m_fd); +#endif + std::swap(m_current_offset, other.m_current_offset); +} + +DirectIOFile DirectIOFile::Duplicate() const +{ + DirectIOFile result; + + if (!IsOpen()) + return result; + +#if defined(_WIN32) + const auto current_process = GetCurrentProcess(); + if (DuplicateHandle(current_process, m_handle, current_process, &result.m_handle, 0, FALSE, + DUPLICATE_SAME_ACCESS) == 0) + { + ERROR_LOG_FMT(COMMON, "DuplicateHandle: {}", Common::GetLastErrorString()); + } +#else + result.m_fd = dup(m_fd); +#endif + + ASSERT(result.IsOpen()); + + result.m_current_offset = m_current_offset; + + return result; +} + +bool Resize(DirectIOFile& file, u64 size) +{ +#if defined(_WIN32) + // This operation is not "atomic", but it's the only thing we're using the file pointer for. + // Concurrent `Resize` would need some external synchronization to prevent race regardless. + const LARGE_INTEGER distance{.QuadPart = LONGLONG(size)}; + return (SetFilePointerEx(file.GetHandle(), distance, nullptr, FILE_BEGIN) != 0) && + (SetEndOfFile(file.GetHandle()) != 0); +#else + return ftruncate(file.GetHandle(), off_t(size)) == 0; +#endif +} + +bool Rename(DirectIOFile& file, const std::string& source_path [[maybe_unused]], + const std::string& destination_path) +{ +#if defined(_WIN32) + const auto dest_name = UTF8ToWString(destination_path); + const auto dest_name_byte_size = DWORD(dest_name.size() * sizeof(WCHAR)); + FILE_RENAME_INFO info{ + .ReplaceIfExists = TRUE, + .FileNameLength = dest_name_byte_size, // The size in bytes, not including null termination. + }; + constexpr auto filename_struct_offset = offsetof(FILE_RENAME_INFO, FileName); + Common::UniqueBuffer buffer(filename_struct_offset + dest_name_byte_size + sizeof(WCHAR)); + std::memcpy(buffer.data(), &info, filename_struct_offset); + std::memcpy(buffer.data() + filename_struct_offset, dest_name.c_str(), + dest_name_byte_size + sizeof(WCHAR)); + return SetFileInformationByHandle(file.GetHandle(), FileRenameInfo, buffer.data(), + DWORD(buffer.size())) != 0; +#else + return file.IsOpen() && Rename(source_path, destination_path); +#endif +} + +bool Delete(DirectIOFile& file, const std::string& filename) +{ +#if defined(_WIN32) + FILE_DISPOSITION_INFO info{.DeleteFile = TRUE}; + return SetFileInformationByHandle(file.GetHandle(), FileDispositionInfo, &info, sizeof(info)) != + 0; +#else + return file.IsOpen() && Delete(filename, IfAbsentBehavior::NoConsoleWarning); +#endif +} + +} // namespace File diff --git a/Source/Core/Common/DirectIOFile.h b/Source/Core/Common/DirectIOFile.h new file mode 100644 index 00000000000..79a5e68a2c1 --- /dev/null +++ b/Source/Core/Common/DirectIOFile.h @@ -0,0 +1,150 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include + +#include "Common/CommonTypes.h" +#include "Common/IOFile.h" + +namespace File +{ +enum class AccessMode +{ + Read, + Write, + ReadAndWrite, +}; + +enum class OpenMode +{ + // Based on the provided `AccessMode`. + // Read: Existing. + // Write: Truncate. + // ReadAndWrite: Existing. + Default, + + // Either create a new file or open an existing file. + Always, + + // Like `Always`, but also erase the contents of an existing file. + Truncate, + + // Require a file to already exist. Fail otherwise. + Existing, + + // Require a new file be created. Fail if one already exists. + Create, +}; + +// This file wrapper avoids use of the underlying system file position. +// It keeps track of its own file position and read/write calls directly use it. +// This makes copied handles entirely thread safe. +class DirectIOFile final +{ +public: + DirectIOFile(); + ~DirectIOFile(); + + // Copies and moves are allowed. + DirectIOFile(const DirectIOFile&); + DirectIOFile& operator=(const DirectIOFile&); + DirectIOFile(DirectIOFile&&); + DirectIOFile& operator=(DirectIOFile&&); + + explicit DirectIOFile(const std::string& path, AccessMode access_mode, + OpenMode open_mode = OpenMode::Default); + + bool Open(const std::string& path, AccessMode access_mode, + OpenMode open_mode = OpenMode::Default); + + bool Close(); + + bool IsOpen() const; + + // An offset from the start of the file may be specified directly. + // These explicit offset versions entirely ignore the current file position. + // They are thread safe, even when used on the same object. + + bool OffsetRead(u64 offset, u8* out_ptr, u64 size); + bool OffsetRead(u64 offset, std::span out_data) + { + return OffsetRead(offset, out_data.data(), out_data.size()); + } + bool OffsetWrite(u64 offset, const u8* in_ptr, u64 size); + bool OffsetWrite(auto offset, std::span in_data) + { + return OffsetWrite(offset, in_data.data(), in_data.size()); + } + + // These Read/Write functions advance the current position on success. + + bool Read(u8* out_ptr, u64 size) + { + if (!OffsetRead(m_current_offset, out_ptr, size)) + return false; + m_current_offset += size; + return true; + } + bool Read(std::span out_data) { return Read(out_data.data(), out_data.size()); } + + bool Write(const u8* in_ptr, u64 size) + { + if (!OffsetWrite(m_current_offset, in_ptr, size)) + return false; + m_current_offset += size; + return true; + } + bool Write(std::span in_data) { return Write(in_data.data(), in_data.size()); } + + // Returns 0 on error. + u64 GetSize() const; + + bool Seek(s64 offset, SeekOrigin origin); + + // Returns 0 when not open. + u64 Tell() const { return m_current_offset; } + + bool Flush(); + + auto GetHandle() const + { +#if defined(_WIN32) + return m_handle; +#else + return m_fd; +#endif + } + +private: + void Swap(DirectIOFile& other); + DirectIOFile Duplicate() const; + +#if defined(_WIN32) + // A workaround to avoid including in this header. + // HANDLE is just void* and INVALID_HANDLE_VALUE is -1. + using HandleType = void*; + HandleType m_handle{HandleType(-1)}; +#else + using HandleType = int; + HandleType m_fd{-1}; +#endif + + u64 m_current_offset{}; +}; + +// These take an open file handle to avoid failures from other processes trying to open our files. +// This is mainly an issue on Windows. + +bool Resize(DirectIOFile& file, u64 size); + +// Attempts to replace a destination file if one already exists. +// Note: Windows uses `file`. Elsewhere uses `source_path`, but `file` is checked for openness. +bool Rename(DirectIOFile& file, const std::string& source_path, + const std::string& destination_path); +// Note: Ditto, only Windows actually uses the file handle. Provide both. +bool Delete(DirectIOFile& file, const std::string& filename); + +} // namespace File diff --git a/Source/Core/DolphinLib.props b/Source/Core/DolphinLib.props index 32f76c8d014..17e44675ac2 100644 --- a/Source/Core/DolphinLib.props +++ b/Source/Core/DolphinLib.props @@ -51,6 +51,7 @@ + @@ -820,6 +821,7 @@ + diff --git a/Source/UnitTests/Common/FileUtilTest.cpp b/Source/UnitTests/Common/FileUtilTest.cpp index 6b3f1168d24..bfdc9128f6b 100644 --- a/Source/UnitTests/Common/FileUtilTest.cpp +++ b/Source/UnitTests/Common/FileUtilTest.cpp @@ -1,9 +1,15 @@ // Copyright 2020 Dolphin Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include #include +#include +#include + #include +#include "Common/BitUtils.h" +#include "Common/DirectIOFile.h" #include "Common/FileUtil.h" class FileUtilTest : public testing::Test @@ -147,3 +153,262 @@ TEST_F(FileUtilTest, CreateFullPath) EXPECT_FALSE(File::CreateFullPath(p3file + "/")); EXPECT_TRUE(File::IsFile(p3file)); } + +TEST_F(FileUtilTest, DirectIOFile) +{ + static constexpr std::array u8_test_data = {42, 7, 99}; + static constexpr std::array u16_test_data = {0xdead, 0xbeef, 0xf00d}; + + static constexpr int u16_data_offset = 73; + + File::DirectIOFile file; + EXPECT_FALSE(file.IsOpen()); + + // Read mode fails with a non-existing file. + EXPECT_FALSE(file.Open(m_file_path, File::AccessMode::Read)); + + std::array u8_buffer = {}; + + // Everything fails when a file isn't open. + EXPECT_FALSE(file.Write(u8_buffer)); + EXPECT_FALSE(file.Read(u8_buffer)); + EXPECT_FALSE(file.Flush()); + EXPECT_FALSE(file.Seek(12, File::SeekOrigin::Begin)); + EXPECT_EQ(file.Tell(), 0); + EXPECT_EQ(file.GetSize(), 0); + + // Fail to open non-existing file. + EXPECT_FALSE(file.Open(m_file_path, File::AccessMode::ReadAndWrite)); + EXPECT_FALSE(file.Open(m_file_path, File::AccessMode::Read, File::OpenMode::Existing)); + EXPECT_FALSE(file.Open(m_file_path, File::AccessMode::Write, File::OpenMode::Existing)); + + // Read mode can create files + EXPECT_TRUE(file.Open(m_file_path, File::AccessMode::Read, File::OpenMode::Create)); + EXPECT_TRUE(file.Close()); + + // Open a file for writing. + EXPECT_TRUE(file.Open(m_file_path, File::AccessMode::Write, File::OpenMode::Always)); + EXPECT_TRUE(file.Flush()); + EXPECT_TRUE(file.IsOpen()); + + // Note: Double Open() currently ASSERTs. It's not obvious if that should succeed or fail. + + EXPECT_TRUE(file.Close()); + EXPECT_FALSE(file.Open(m_file_path, File::AccessMode::Write, File::OpenMode::Create)); + + EXPECT_TRUE(file.Open(m_file_path, File::AccessMode::Write, File::OpenMode::Existing)); + EXPECT_TRUE(file.Close()); + + // Rename, Resize, and Delete fail with a closed handle. + EXPECT_FALSE(Rename(file, m_file_path, "fail")); + EXPECT_FALSE(Resize(file, 72)); + EXPECT_FALSE(Delete(file, m_file_path)); + EXPECT_TRUE(File::Exists(m_file_path)); + + EXPECT_TRUE(file.Open(m_file_path, File::AccessMode::Write)); + + EXPECT_TRUE(file.Write(u8_buffer.data(), 0)); // write of 0 succeeds. + EXPECT_FALSE(file.Read(u8_buffer.data(), 0)); // read of 0 (in write mode) fails. + + // Resize through handle works. + EXPECT_TRUE(Resize(file, 1)); + EXPECT_EQ(file.GetSize(), 1); + file.Close(); + + // Write truncates by default. + EXPECT_TRUE(file.Open(m_file_path, File::AccessMode::Write)); + EXPECT_EQ(file.GetSize(), 0); + + EXPECT_TRUE(file.Write(u8_test_data)); + EXPECT_EQ(file.GetSize(), u8_test_data.size()); // size changed + EXPECT_EQ(file.Tell(), u8_test_data.size()); // file position changed + + // Relative seek works. + EXPECT_TRUE(file.Seek(0, File::SeekOrigin::End)); + EXPECT_EQ(file.Tell(), file.GetSize()); + EXPECT_TRUE(file.Seek(-int(u8_test_data.size()), File::SeekOrigin::Current)); + EXPECT_EQ(file.Tell(), 0); + + // Read while in "write mode" fails + EXPECT_FALSE(file.Read(u8_buffer)); + EXPECT_EQ(file.Tell(), 0); + + // Seeking past the end works. + EXPECT_TRUE(file.Seek(u16_data_offset, File::SeekOrigin::Begin)); + EXPECT_EQ(file.Tell(), u16_data_offset); + EXPECT_EQ(file.GetSize(), u8_test_data.size()); // no change in size + + static constexpr u64 final_file_size = u16_data_offset + sizeof(u16_test_data); + + // Size changes after write. + EXPECT_TRUE(file.Write(Common::AsU8Span(u16_test_data))); + EXPECT_EQ(file.GetSize(), final_file_size); + + // Seek before begin fails. + EXPECT_FALSE(file.Seek(-1, File::SeekOrigin::Begin)); + EXPECT_EQ(file.Tell(), file.GetSize()); // unchanged position + + EXPECT_TRUE(file.Close()); + EXPECT_FALSE(file.IsOpen()); + + EXPECT_EQ(file.Tell(), 0); + EXPECT_EQ(file.GetSize(), 0); + + EXPECT_FALSE(file.Close()); // Double close fails. + + // Open file for reading. + EXPECT_TRUE(file.Open(m_file_path, File::AccessMode::Read, File::OpenMode::Always)); + EXPECT_TRUE(file.Close()); + EXPECT_TRUE(file.Open(m_file_path, File::AccessMode::Read)); + + static constexpr int big_offset = 999; + + // We can seek beyond the end. + EXPECT_TRUE(file.Seek(big_offset, File::SeekOrigin::End)); + EXPECT_EQ(file.Tell(), file.GetSize() + big_offset); + + // Resize through handle fails when in read mode. + EXPECT_FALSE(Resize(file, big_offset)); + EXPECT_EQ(file.GetSize(), final_file_size); + + constexpr size_t thread_count = 4; + + File::DirectIOFile closed_file; + + std::latch do_reads{1}; + + EXPECT_TRUE(file.Seek(u16_data_offset, File::SeekOrigin::Begin)); + + // Concurrent access with copied handles works. + std::vector threads(thread_count); + for (auto& t : threads) + { + t = std::thread{[&do_reads, file, closed_file]() mutable { + EXPECT_FALSE(closed_file.IsOpen()); // a copied closed file is still closed + + EXPECT_EQ(file.Tell(), u16_data_offset); // current position is copied + + std::array one_byte{}; + + constexpr u64 small_offset = 3; + + do_reads.wait(); + + EXPECT_TRUE(file.Seek(small_offset, File::SeekOrigin::Begin)); + EXPECT_EQ(file.Tell(), small_offset); + + // writes fail in read mode. + EXPECT_FALSE(file.Write(u8_test_data)); + EXPECT_FALSE(file.OffsetWrite(0, u8_test_data.data(), 0)); + EXPECT_EQ(file.Tell(), small_offset); + + EXPECT_TRUE(file.Read(one_byte.data(), 0)); // read of zero succeeds. + EXPECT_EQ(file.Tell(), small_offset); // unchanged position + + // Reading at an explicit offset doesn't change the position + EXPECT_TRUE(file.OffsetRead(u8_test_data.size() - 1, one_byte)); + EXPECT_EQ(file.Tell(), small_offset); + EXPECT_EQ(one_byte[0], u8_test_data.back()); + + EXPECT_EQ(file.GetSize(), final_file_size); + EXPECT_TRUE(file.Seek(-int(sizeof(u16_test_data)), File::SeekOrigin::End)); + + // We can't read beyond the end of the file. + EXPECT_FALSE(file.OffsetRead(big_offset, one_byte)); + // A read of zero beyond the end works. + EXPECT_TRUE(file.OffsetRead(big_offset, one_byte.data(), 0)); + + // Reading the previously written data works. + std::array u16_buffer = {}; + EXPECT_TRUE(file.Read(Common::AsWritableU8Span(u16_buffer))); + EXPECT_TRUE(std::ranges::equal(u16_buffer, u16_test_data)); + + // We can't read at the end of the file. + EXPECT_FALSE(file.Read(one_byte)); + EXPECT_EQ(file.Tell(), file.GetSize()); // The position is unchanged. + }}; + } + + // We may close the file on our end before the other threads use it. + file.Close(); + + // ReadAndWrite does not truncate existing files. + EXPECT_TRUE(file.Open(m_file_path, File::AccessMode::ReadAndWrite)); + EXPECT_EQ(file.GetSize(), final_file_size); + file.Close(); + + // Write doesn't truncate when the mode is changed. + EXPECT_TRUE(file.Open(m_file_path, File::AccessMode::Write, File::OpenMode::Existing)); + EXPECT_EQ(file.GetSize(), final_file_size); + + // Open a new handle to the same file. + File::DirectIOFile second_handle(m_file_path, File::AccessMode::Write, File::OpenMode::Always); + EXPECT_TRUE(second_handle.IsOpen()); + + const std::string destination_path_1 = m_file_path + ".dest_1"; + + // Rename through handle works when opened elsewhere. + EXPECT_TRUE(Rename(file, m_file_path, destination_path_1)); + EXPECT_FALSE(File::Exists(m_file_path)); + EXPECT_TRUE(File::Exists(destination_path_1)); + + const std::string destination_path_2 = m_file_path + ".dest_2"; + + // Note: Windows fails the next `Rename` if this file is kept open. + // I don't know if there is a nice way to make that work. + { + File::DirectIOFile another_file{destination_path_2, File::AccessMode::Write}; + EXPECT_TRUE(another_file.IsOpen()); + } + + // Rename overwrites existing files. + EXPECT_TRUE(Rename(file, destination_path_1, destination_path_2)); + EXPECT_FALSE(File::Exists(destination_path_1)); + EXPECT_EQ(File::GetSize(destination_path_2), final_file_size); + + const std::string destination_path_3 = m_file_path + ".dest_3"; + + // Truncate fails in Read mode. + File::Copy(destination_path_2, destination_path_3); + EXPECT_EQ(File::GetSize(destination_path_3), final_file_size); + { + File::DirectIOFile tmp{destination_path_3, File::AccessMode::Read, File::OpenMode::Truncate}; + EXPECT_FALSE(tmp.IsOpen()); + EXPECT_EQ(File::GetSize(destination_path_3), final_file_size); + } + + // Truncate works with ReadAndWrite. + EXPECT_EQ(File::GetSize(destination_path_3), final_file_size); + { + File::DirectIOFile tmp{destination_path_3, File::AccessMode::ReadAndWrite, + File::OpenMode::Truncate}; + EXPECT_EQ(tmp.GetSize(), 0); + Delete(tmp, destination_path_3); + } + + // Truncate works with Write. + File::Copy(destination_path_2, destination_path_3); + EXPECT_EQ(File::GetSize(destination_path_3), final_file_size); + { + File::DirectIOFile tmp{destination_path_3, File::AccessMode::Write, File::OpenMode::Truncate}; + EXPECT_EQ(tmp.GetSize(), 0); + Delete(tmp, destination_path_3); + } + + // Delete through handle works. + EXPECT_TRUE(Delete(file, destination_path_2)); + EXPECT_FALSE(File::Exists(destination_path_2)); + + // The threads can read even after deleting everything. + do_reads.count_down(); + std::ranges::for_each(threads, &std::thread::join); + + file.Close(); + EXPECT_TRUE(file.Open(destination_path_1, File::AccessMode::Read, File::OpenMode::Always)); + + // Required on Windows for the below to succeed. + second_handle.Close(); + + file.Close(); + EXPECT_TRUE(file.Open(destination_path_2, File::AccessMode::Write, File::OpenMode::Always)); +} From b98acb9a37a80b22a0da1bee7be0c4c44adadb56 Mon Sep 17 00:00:00 2001 From: Jordan Woyak Date: Wed, 29 Oct 2025 20:26:33 -0500 Subject: [PATCH 2/4] Common/BitUtils: Add overloads of AsU8Span/AsWritableU8Span that handle conversions from contiguous ranges. --- Source/Core/Common/BitUtils.h | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/Source/Core/Common/BitUtils.h b/Source/Core/Common/BitUtils.h index 6b4835d8aa6..e052c8aeff4 100644 --- a/Source/Core/Common/BitUtils.h +++ b/Source/Core/Common/BitUtils.h @@ -255,15 +255,33 @@ T ExpandValue(T value, size_t left_shift_amount) (T(-ExtractBit<0>(value)) >> (BitSize() - left_shift_amount)); } +// Convert a contiguous range of a trivially-copyable type to a `span` +template +requires(std::is_trivially_copyable_v>) +constexpr auto AsU8Span(const T& range) +{ + return std::span{reinterpret_cast(range.data()), range.size() * sizeof(*range.data())}; +} + +// Convert a contiguous range of a non-const trivially-copyable type to a `span` +template +requires(std::is_trivially_copyable_v>) +constexpr auto AsWritableU8Span(T& range) +{ + return std::span{reinterpret_cast(range.data()), range.size() * sizeof(*range.data())}; +} + +// Convert a trivially-copyable object to a `span` template -requires(std::is_trivially_copyable_v) +requires(!std::ranges::contiguous_range && std::is_trivially_copyable_v) constexpr auto AsU8Span(const T& obj) { return std::span{reinterpret_cast(std::addressof(obj)), sizeof(obj)}; } +// Convert a non-const trivially-copyable object to a `span` template -requires(std::is_trivially_copyable_v) +requires(!std::ranges::contiguous_range && std::is_trivially_copyable_v) constexpr auto AsWritableU8Span(T& obj) { return std::span{reinterpret_cast(std::addressof(obj)), sizeof(obj)}; From 239330017c97fa3f6b4b2670ab570e822d6affb0 Mon Sep 17 00:00:00 2001 From: Jordan Woyak Date: Wed, 29 Oct 2025 20:26:47 -0500 Subject: [PATCH 3/4] DiscIO: Make all BlobReader implementations use DirectIOFile to make CopyReader functionality thread safe. --- Source/Core/DiscIO/Blob.cpp | 9 ++-- Source/Core/DiscIO/CISOBlob.cpp | 21 ++++------ Source/Core/DiscIO/CISOBlob.h | 9 ++-- Source/Core/DiscIO/CompressedBlob.cpp | 45 +++++++++----------- Source/Core/DiscIO/CompressedBlob.h | 8 ++-- Source/Core/DiscIO/DirectoryBlob.cpp | 22 ++++------ Source/Core/DiscIO/DirectoryBlob.h | 2 - Source/Core/DiscIO/FileBlob.cpp | 24 ++++------- Source/Core/DiscIO/FileBlob.h | 9 ++-- Source/Core/DiscIO/NFSBlob.cpp | 60 ++++++++++----------------- Source/Core/DiscIO/NFSBlob.h | 13 +++--- Source/Core/DiscIO/SplitFileBlob.cpp | 25 +++-------- Source/Core/DiscIO/SplitFileBlob.h | 4 +- Source/Core/DiscIO/TGCBlob.cpp | 22 ++++------ Source/Core/DiscIO/TGCBlob.h | 9 ++-- Source/Core/DiscIO/WIABlob.cpp | 55 ++++++++++++------------ Source/Core/DiscIO/WIABlob.h | 23 +++++----- Source/Core/DiscIO/WbfsBlob.cpp | 27 ++++++------ Source/Core/DiscIO/WbfsBlob.h | 14 +++---- Source/Core/DolphinQt/MainWindow.cpp | 1 + 20 files changed, 167 insertions(+), 235 deletions(-) diff --git a/Source/Core/DiscIO/Blob.cpp b/Source/Core/DiscIO/Blob.cpp index 9e1ddb9ead1..aad8b60b5e5 100644 --- a/Source/Core/DiscIO/Blob.cpp +++ b/Source/Core/DiscIO/Blob.cpp @@ -4,14 +4,13 @@ #include "DiscIO/Blob.h" #include -#include -#include #include #include #include +#include "Common/BitUtils.h" #include "Common/CommonTypes.h" -#include "Common/IOFile.h" +#include "Common/DirectIOFile.h" #include "Common/MsgHandler.h" #include "DiscIO/CISOBlob.h" @@ -213,9 +212,9 @@ u32 SectorReader::ReadChunk(u8* buffer, u64 chunk_num) std::unique_ptr CreateBlobReader(const std::string& filename) { - File::IOFile file(filename, "rb"); + File::DirectIOFile file(filename, File::AccessMode::Read); u32 magic; - if (!file.ReadArray(&magic, 1)) + if (!file.Read(Common::AsWritableU8Span(magic))) return nullptr; // Conveniently, every supported file format (except for plain disc images and diff --git a/Source/Core/DiscIO/CISOBlob.cpp b/Source/Core/DiscIO/CISOBlob.cpp index 001c462694d..df32cbaadcf 100644 --- a/Source/Core/DiscIO/CISOBlob.cpp +++ b/Source/Core/DiscIO/CISOBlob.cpp @@ -4,22 +4,20 @@ #include "DiscIO/CISOBlob.h" #include -#include #include #include +#include "Common/BitUtils.h" #include "Common/CommonTypes.h" -#include "Common/IOFile.h" namespace DiscIO { -CISOFileReader::CISOFileReader(File::IOFile file) : m_file(std::move(file)) +CISOFileReader::CISOFileReader(File::DirectIOFile file) : m_file(std::move(file)) { m_size = m_file.GetSize(); CISOHeader header; - m_file.Seek(0, File::SeekOrigin::Begin); - m_file.ReadArray(&header, 1); + m_file.OffsetRead(0, Common::AsWritableU8Span(header)); m_block_size = header.block_size; @@ -28,11 +26,10 @@ CISOFileReader::CISOFileReader(File::IOFile file) : m_file(std::move(file)) m_ciso_map[idx] = (1 == header.map[idx]) ? count++ : UNUSED_BLOCK_ID; } -std::unique_ptr CISOFileReader::Create(File::IOFile file) +std::unique_ptr CISOFileReader::Create(File::DirectIOFile file) { CISOHeader header; - if (file.Seek(0, File::SeekOrigin::Begin) && file.ReadArray(&header, 1) && - header.magic == CISO_MAGIC) + if (file.OffsetRead(0, Common::AsWritableU8Span(header)) && header.magic == CISO_MAGIC) { return std::unique_ptr(new CISOFileReader(std::move(file))); } @@ -42,7 +39,7 @@ std::unique_ptr CISOFileReader::Create(File::IOFile file) std::unique_ptr CISOFileReader::CopyReader() const { - return Create(m_file.Duplicate("rb")); + return Create(m_file); } u64 CISOFileReader::GetDataSize() const @@ -71,12 +68,8 @@ bool CISOFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr) // calculate the base address u64 const file_off = CISO_HEADER_SIZE + m_ciso_map[block] * (u64)m_block_size + data_offset; - if (!(m_file.Seek(file_off, File::SeekOrigin::Begin) && - m_file.ReadArray(out_ptr, bytes_to_read))) - { - m_file.ClearError(); + if (!m_file.OffsetRead(file_off, out_ptr, bytes_to_read)) return false; - } } else { diff --git a/Source/Core/DiscIO/CISOBlob.h b/Source/Core/DiscIO/CISOBlob.h index c32c23bd35e..1611e49ba9d 100644 --- a/Source/Core/DiscIO/CISOBlob.h +++ b/Source/Core/DiscIO/CISOBlob.h @@ -3,12 +3,11 @@ #pragma once -#include #include #include #include "Common/CommonTypes.h" -#include "Common/IOFile.h" +#include "Common/DirectIOFile.h" #include "DiscIO/Blob.h" namespace DiscIO @@ -33,7 +32,7 @@ struct CISOHeader class CISOFileReader final : public BlobReader { public: - static std::unique_ptr Create(File::IOFile file); + static std::unique_ptr Create(File::DirectIOFile file); BlobType GetBlobType() const override { return BlobType::CISO; } std::unique_ptr CopyReader() const override; @@ -50,12 +49,12 @@ public: bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: - CISOFileReader(File::IOFile file); + CISOFileReader(File::DirectIOFile file); typedef u16 MapType; static constexpr MapType UNUSED_BLOCK_ID = UINT16_MAX; - File::IOFile m_file; + File::DirectIOFile m_file; u64 m_size; u32 m_block_size; MapType m_ciso_map[CISO_MAP_SIZE]; diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 99a982efd91..7f86ee23881 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -4,7 +4,6 @@ #include "DiscIO/CompressedBlob.h" #include -#include #include #include #include @@ -19,10 +18,10 @@ #endif #include "Common/Assert.h" +#include "Common/BitUtils.h" #include "Common/CommonTypes.h" #include "Common/FileUtil.h" #include "Common/Hash.h" -#include "Common/IOFile.h" #include "Common/Logging/Log.h" #include "Common/MsgHandler.h" #include "DiscIO/Blob.h" @@ -32,22 +31,23 @@ namespace DiscIO { -bool IsGCZBlob(File::IOFile& file); +bool IsGCZBlob(File::DirectIOFile& file); -CompressedBlobReader::CompressedBlobReader(File::IOFile file, const std::string& filename) +CompressedBlobReader::CompressedBlobReader(File::DirectIOFile file, const std::string& filename) : m_file(std::move(file)), m_file_name(filename) { m_file_size = m_file.GetSize(); m_file.Seek(0, File::SeekOrigin::Begin); - m_file.ReadArray(&m_header, 1); + m_file.Read(Common::AsWritableU8Span(m_header)); SetSectorSize(m_header.block_size); // cache block pointers and hashes m_block_pointers.resize(m_header.num_blocks); - m_file.ReadArray(m_block_pointers.data(), m_header.num_blocks); + m_file.Read(Common::AsWritableU8Span(m_block_pointers)); + m_hashes.resize(m_header.num_blocks); - m_file.ReadArray(m_hashes.data(), m_header.num_blocks); + m_file.Read(Common::AsWritableU8Span(m_hashes)); m_data_offset = (sizeof(CompressedBlobHeader)) + (sizeof(u64)) * m_header.num_blocks // skip block pointers @@ -60,7 +60,7 @@ CompressedBlobReader::CompressedBlobReader(File::IOFile file, const std::string& m_zlib_buffer.resize(zlib_buffer_size); } -std::unique_ptr CompressedBlobReader::Create(File::IOFile file, +std::unique_ptr CompressedBlobReader::Create(File::DirectIOFile file, const std::string& filename) { if (IsGCZBlob(file)) @@ -74,7 +74,7 @@ CompressedBlobReader::~CompressedBlobReader() = default; std::unique_ptr CompressedBlobReader::CopyReader() const { - return Create(m_file.Duplicate("rb"), m_file_name); + return Create(m_file, m_file_name); } // IMPORTANT: Calling this function invalidates all earlier pointers gotten from this function. @@ -107,12 +107,10 @@ bool CompressedBlobReader::GetBlock(u64 block_num, u8* out_ptr) // clear unused part of zlib buffer. maybe this can be deleted when it works fully. memset(&m_zlib_buffer[comp_block_size], 0, m_zlib_buffer.size() - comp_block_size); - m_file.Seek(offset, File::SeekOrigin::Begin); - if (!m_file.ReadBytes(m_zlib_buffer.data(), comp_block_size)) + if (!m_file.OffsetRead(offset, m_zlib_buffer.data(), comp_block_size)) { ERROR_LOG_FMT(DISCIO, "The disc image \"{}\" is truncated, some of the data is missing.", m_file_name); - m_file.ClearError(); return false; } @@ -241,7 +239,7 @@ static ConversionResult Compress(CompressThreadState* state, return std::move(output_parameters); } -static ConversionResultCode Output(OutputParameters parameters, File::IOFile* outfile, +static ConversionResultCode Output(OutputParameters parameters, File::DirectIOFile* outfile, u64* position, std::vector* offsets, int progress_monitor, u32 num_blocks, const CompressCB& callback) { @@ -252,7 +250,7 @@ static ConversionResultCode Output(OutputParameters parameters, File::IOFile* ou *position += parameters.data.size(); - if (!outfile->WriteBytes(parameters.data.data(), parameters.data.size())) + if (!outfile->Write(parameters.data)) return ConversionResultCode::WriteFailed; if (parameters.block_number % progress_monitor == 0) @@ -278,8 +276,8 @@ bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path, { ASSERT(infile->GetDataSizeType() == DataSizeType::Accurate); - File::IOFile outfile(outfile_path, "wb"); - if (!outfile) + File::DirectIOFile outfile(outfile_path, File::AccessMode::Write); + if (!outfile.IsOpen()) { PanicAlertFmtT( "Failed to open the output file \"{0}\".\n" @@ -365,9 +363,9 @@ bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path, { // Okay, go back and fill in headers outfile.Seek(0, File::SeekOrigin::Begin); - outfile.WriteArray(&header, 1); - outfile.WriteArray(offsets.data(), header.num_blocks); - outfile.WriteArray(hashes.data(), header.num_blocks); + outfile.Write(Common::AsU8Span(header)); + outfile.Write(Common::AsU8Span(offsets)); + outfile.Write(Common::AsU8Span(hashes)); callback(Common::GetStringT("Done compressing disc image."), 1.0f); } @@ -385,15 +383,10 @@ bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path, return result == ConversionResultCode::Success; } -bool IsGCZBlob(File::IOFile& file) +bool IsGCZBlob(File::DirectIOFile& file) { - const u64 position = file.Tell(); - if (!file.Seek(0, File::SeekOrigin::Begin)) - return false; CompressedBlobHeader header; - bool is_gcz = file.ReadArray(&header, 1) && header.magic_cookie == GCZ_MAGIC; - file.Seek(position, File::SeekOrigin::Begin); - return is_gcz; + return file.OffsetRead(0, Common::AsWritableU8Span(header)) && header.magic_cookie == GCZ_MAGIC; } } // namespace DiscIO diff --git a/Source/Core/DiscIO/CompressedBlob.h b/Source/Core/DiscIO/CompressedBlob.h index 15eeffc3dbc..5122403f478 100644 --- a/Source/Core/DiscIO/CompressedBlob.h +++ b/Source/Core/DiscIO/CompressedBlob.h @@ -17,7 +17,7 @@ #include #include "Common/CommonTypes.h" -#include "Common/IOFile.h" +#include "Common/DirectIOFile.h" #include "DiscIO/Blob.h" namespace DiscIO @@ -43,7 +43,7 @@ struct CompressedBlobHeader // 32 bytes class CompressedBlobReader final : public SectorReader { public: - static std::unique_ptr Create(File::IOFile file, + static std::unique_ptr Create(File::DirectIOFile file, const std::string& filename); ~CompressedBlobReader() override; @@ -65,13 +65,13 @@ public: bool GetBlock(u64 block_num, u8* out_ptr) override; private: - CompressedBlobReader(File::IOFile file, const std::string& filename); + CompressedBlobReader(File::DirectIOFile file, const std::string& filename); CompressedBlobHeader m_header; std::vector m_block_pointers; std::vector m_hashes; int m_data_offset; - File::IOFile m_file; + File::DirectIOFile m_file; u64 m_file_size; std::vector m_zlib_buffer; std::string m_file_name; diff --git a/Source/Core/DiscIO/DirectoryBlob.cpp b/Source/Core/DiscIO/DirectoryBlob.cpp index d0ee8bf803a..11acc8eb417 100644 --- a/Source/Core/DiscIO/DirectoryBlob.cpp +++ b/Source/Core/DiscIO/DirectoryBlob.cpp @@ -4,9 +4,7 @@ #include "DiscIO/DirectoryBlob.h" #include -#include #include -#include #include #include #include @@ -17,14 +15,12 @@ #include "Common/Align.h" #include "Common/Assert.h" -#include "Common/CommonPaths.h" #include "Common/CommonTypes.h" +#include "Common/DirectIOFile.h" #include "Common/FileUtil.h" -#include "Common/IOFile.h" #include "Common/Logging/Log.h" #include "Common/StringUtil.h" #include "Common/Swap.h" -#include "Core/Boot/DolReader.h" #include "Core/IOS/ES/Formats.h" #include "DiscIO/Blob.h" #include "DiscIO/DiscUtils.h" @@ -97,9 +93,8 @@ bool DiscContent::Read(u64* offset, u64* length, u8** buffer, DirectoryBlobReade if (std::holds_alternative(m_content_source)) { const auto& content = std::get(m_content_source); - File::IOFile file(content.m_filename, "rb"); - if (!file.Seek(content.m_offset + offset_in_content, File::SeekOrigin::Begin) || - !file.ReadBytes(*buffer, bytes_to_read)) + File::DirectIOFile file(content.m_filename, File::AccessMode::Read); + if (!file.OffsetRead(content.m_offset + offset_in_content, *buffer, bytes_to_read)) { return false; } @@ -1013,9 +1008,9 @@ void DirectoryBlobPartition::SetBI2(std::vector bi2) u64 DirectoryBlobPartition::SetApploaderFromFile(const std::string& path) { - File::IOFile file(path, "rb"); + File::DirectIOFile file(path, File::AccessMode::Read); std::vector apploader(file.GetSize()); - file.ReadBytes(apploader.data(), apploader.size()); + file.Read(apploader); return SetApploader(std::move(apploader), path); } @@ -1256,10 +1251,9 @@ void DirectoryBlobPartition::WriteDirectory(std::vector* fst_data, static size_t ReadFileToVector(const std::string& path, std::vector* vector) { - File::IOFile file(path, "rb"); - size_t bytes_read; - file.ReadArray(vector->data(), std::min(file.GetSize(), vector->size()), &bytes_read); - return bytes_read; + File::DirectIOFile file(path, File::AccessMode::Read); + file.Read(vector->data(), std::min(file.GetSize(), vector->size())); + return file.Tell(); } static void PadToAddress(u64 start_address, u64* address, u64* length, u8** buffer) diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index 12d546f9099..baea8d9e85b 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -15,7 +15,6 @@ #include #include "Common/CommonTypes.h" -#include "Common/FileUtil.h" #include "DiscIO/Blob.h" #include "DiscIO/Volume.h" #include "DiscIO/WiiEncryptionCache.h" @@ -23,7 +22,6 @@ namespace File { struct FSTEntry; -class IOFile; } // namespace File namespace DiscIO diff --git a/Source/Core/DiscIO/FileBlob.cpp b/Source/Core/DiscIO/FileBlob.cpp index 9ba572ea77f..3a30e2a2552 100644 --- a/Source/Core/DiscIO/FileBlob.cpp +++ b/Source/Core/DiscIO/FileBlob.cpp @@ -15,14 +15,14 @@ namespace DiscIO { -PlainFileReader::PlainFileReader(File::IOFile file) : m_file(std::move(file)) +PlainFileReader::PlainFileReader(File::DirectIOFile file) : m_file(std::move(file)) { m_size = m_file.GetSize(); } -std::unique_ptr PlainFileReader::Create(File::IOFile file) +std::unique_ptr PlainFileReader::Create(File::DirectIOFile file) { - if (file) + if (file.IsOpen()) return std::unique_ptr(new PlainFileReader(std::move(file))); return nullptr; @@ -30,20 +30,12 @@ std::unique_ptr PlainFileReader::Create(File::IOFile file) std::unique_ptr PlainFileReader::CopyReader() const { - return Create(m_file.Duplicate("rb")); + return Create(m_file); } bool PlainFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr) { - if (m_file.Seek(offset, File::SeekOrigin::Begin) && m_file.ReadBytes(out_ptr, nbytes)) - { - return true; - } - else - { - m_file.ClearError(); - return false; - } + return m_file.OffsetRead(offset, out_ptr, nbytes); } bool ConvertToPlain(BlobReader* infile, const std::string& infile_path, @@ -51,8 +43,8 @@ bool ConvertToPlain(BlobReader* infile, const std::string& infile_path, { ASSERT(infile->GetDataSizeType() == DataSizeType::Accurate); - File::IOFile outfile(outfile_path, "wb"); - if (!outfile) + File::DirectIOFile outfile(outfile_path, File::AccessMode::Write); + if (!outfile.IsOpen()) { PanicAlertFmtT( "Failed to open the output file \"{0}\".\n" @@ -99,7 +91,7 @@ bool ConvertToPlain(BlobReader* infile, const std::string& infile_path, success = false; break; } - if (!outfile.WriteBytes(buffer.data(), sz)) + if (!outfile.Write(buffer.data(), sz)) { PanicAlertFmtT("Failed to write the output file \"{0}\".\n" "Check that you have enough space available on the target drive.", diff --git a/Source/Core/DiscIO/FileBlob.h b/Source/Core/DiscIO/FileBlob.h index 46224d9c3f8..3e3a495343a 100644 --- a/Source/Core/DiscIO/FileBlob.h +++ b/Source/Core/DiscIO/FileBlob.h @@ -3,12 +3,11 @@ #pragma once -#include #include #include #include "Common/CommonTypes.h" -#include "Common/IOFile.h" +#include "Common/DirectIOFile.h" #include "DiscIO/Blob.h" namespace DiscIO @@ -16,7 +15,7 @@ namespace DiscIO class PlainFileReader final : public BlobReader { public: - static std::unique_ptr Create(File::IOFile file); + static std::unique_ptr Create(File::DirectIOFile file); BlobType GetBlobType() const override { return BlobType::PLAIN; } std::unique_ptr CopyReader() const override; @@ -33,9 +32,9 @@ public: bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: - PlainFileReader(File::IOFile file); + PlainFileReader(File::DirectIOFile file); - File::IOFile m_file; + File::DirectIOFile m_file; u64 m_size; }; diff --git a/Source/Core/DiscIO/NFSBlob.cpp b/Source/Core/DiscIO/NFSBlob.cpp index f051ec9832f..df6a65f3216 100644 --- a/Source/Core/DiscIO/NFSBlob.cpp +++ b/Source/Core/DiscIO/NFSBlob.cpp @@ -15,9 +15,9 @@ #include #include "Common/Align.h" +#include "Common/BitUtils.h" #include "Common/CommonTypes.h" #include "Common/Crypto/AES.h" -#include "Common/IOFile.h" #include "Common/Logging/Log.h" #include "Common/StringUtil.h" #include "Common/Swap.h" @@ -39,8 +39,8 @@ bool NFSFileReader::ReadKey(const std::string& path, const std::string& director } const std::string key_path = parent + "code/htk.bin"; - File::IOFile key_file(key_path, "rb"); - if (!key_file.ReadBytes(key_out->data(), key_out->size())) + File::DirectIOFile key_file(key_path, File::AccessMode::Read); + if (!key_file.Read(*key_out)) { ERROR_LOG_FMT(DISCIO, "Failed to read from {}", key_path); return false; @@ -67,13 +67,13 @@ std::vector NFSFileReader::GetLBARanges(const NFSHeader& header) return lba_ranges; } -std::vector NFSFileReader::OpenFiles(const std::string& directory, - File::IOFile first_file, u64 expected_raw_size, - u64* raw_size_out) +std::vector NFSFileReader::OpenFiles(const std::string& directory, + File::DirectIOFile first_file, + u64 expected_raw_size, u64* raw_size_out) { const u64 file_count = Common::AlignUp(expected_raw_size, MAX_FILE_SIZE) / MAX_FILE_SIZE; - std::vector files; + std::vector files; files.reserve(file_count); *raw_size_out = first_file.GetSize(); @@ -82,8 +82,8 @@ std::vector NFSFileReader::OpenFiles(const std::string& directory, for (u64 i = 1; i < file_count; ++i) { const std::string child_path = fmt::format("{}hif_{:06}.nfs", directory, i); - File::IOFile child(child_path, "rb"); - if (!child) + File::DirectIOFile child(child_path, File::AccessMode::Read); + if (!child.IsOpen()) { ERROR_LOG_FMT(DISCIO, "Failed to open {}", child_path); return {}; @@ -123,7 +123,7 @@ u64 NFSFileReader::CalculateExpectedDataSize(const std::vector& lba return u64(greatest_block_index) * BLOCK_SIZE; } -std::unique_ptr NFSFileReader::Create(File::IOFile first_file, +std::unique_ptr NFSFileReader::Create(File::DirectIOFile first_file, const std::string& path) { std::string directory, filename, extension; @@ -136,18 +136,15 @@ std::unique_ptr NFSFileReader::Create(File::IOFile first_file, return nullptr; NFSHeader header; - if (!first_file.Seek(0, File::SeekOrigin::Begin) || !first_file.ReadArray(&header, 1) || - header.magic != NFS_MAGIC) - { + if (!first_file.OffsetRead(0, Common::AsWritableU8Span(header)) || header.magic != NFS_MAGIC) return nullptr; - } std::vector lba_ranges = GetLBARanges(header); const u64 expected_raw_size = CalculateExpectedRawSize(lba_ranges); u64 raw_size; - std::vector files = + std::vector files = OpenFiles(directory, std::move(first_file), expected_raw_size, &raw_size); if (files.empty()) @@ -157,8 +154,8 @@ std::unique_ptr NFSFileReader::Create(File::IOFile first_file, new NFSFileReader(std::move(lba_ranges), std::move(files), key, raw_size)); } -NFSFileReader::NFSFileReader(std::vector lba_ranges, std::vector files, - Key key, u64 raw_size) +NFSFileReader::NFSFileReader(std::vector lba_ranges, + std::vector files, Key key, u64 raw_size) : m_lba_ranges(std::move(lba_ranges)), m_files(std::move(files)), m_aes_context(Common::AES::CreateContextDecrypt(key.data())), m_raw_size(raw_size), m_key(key) { @@ -167,11 +164,7 @@ NFSFileReader::NFSFileReader(std::vector lba_ranges, std::vector NFSFileReader::CopyReader() const { - std::vector new_files{}; - for (const File::IOFile& file : m_files) - new_files.push_back(file.Duplicate("rb")); - return std::unique_ptr( - new NFSFileReader(m_lba_ranges, std::move(new_files), m_key, m_raw_size)); + return std::unique_ptr{new NFSFileReader(m_lba_ranges, m_files, m_key, m_raw_size)}; } u64 NFSFileReader::GetDataSize() const @@ -208,6 +201,7 @@ bool NFSFileReader::ReadEncryptedBlock(u64 physical_block_index) const u64 file_index = physical_block_index / BLOCKS_PER_FILE; const u64 block_in_file = physical_block_index % BLOCKS_PER_FILE; + const u64 offset_in_file = sizeof(NFSHeader) + block_in_file * BLOCK_SIZE; if (block_in_file == BLOCKS_PER_FILE - 1) { @@ -217,33 +211,23 @@ bool NFSFileReader::ReadEncryptedBlock(u64 physical_block_index) constexpr size_t PART_1_SIZE = BLOCK_SIZE - sizeof(NFSHeader); constexpr size_t PART_2_SIZE = sizeof(NFSHeader); - File::IOFile& file_1 = m_files[file_index]; - File::IOFile& file_2 = m_files[file_index + 1]; + File::DirectIOFile& file_1 = m_files[file_index]; + File::DirectIOFile& file_2 = m_files[file_index + 1]; - if (!file_1.Seek(sizeof(NFSHeader) + block_in_file * BLOCK_SIZE, File::SeekOrigin::Begin) || - !file_1.ReadBytes(m_current_block_encrypted.data(), PART_1_SIZE)) - { - file_1.ClearError(); + if (!file_1.OffsetRead(offset_in_file, m_current_block_encrypted.data(), PART_1_SIZE)) return false; - } - if (!file_2.Seek(0, File::SeekOrigin::Begin) || - !file_2.ReadBytes(m_current_block_encrypted.data() + PART_1_SIZE, PART_2_SIZE)) - { - file_2.ClearError(); + if (!file_2.OffsetRead(0, m_current_block_encrypted.data() + PART_1_SIZE, PART_2_SIZE)) return false; - } } else { // Normal case. The read is offset by 0x200 bytes, but it's all within one file. - File::IOFile& file = m_files[file_index]; + File::DirectIOFile& file = m_files[file_index]; - if (!file.Seek(sizeof(NFSHeader) + block_in_file * BLOCK_SIZE, File::SeekOrigin::Begin) || - !file.ReadBytes(m_current_block_encrypted.data(), BLOCK_SIZE)) + if (!file.OffsetRead(offset_in_file, m_current_block_encrypted.data(), BLOCK_SIZE)) { - file.ClearError(); return false; } } diff --git a/Source/Core/DiscIO/NFSBlob.h b/Source/Core/DiscIO/NFSBlob.h index cdcd20011f2..dcbe6bf31a9 100644 --- a/Source/Core/DiscIO/NFSBlob.h +++ b/Source/Core/DiscIO/NFSBlob.h @@ -11,7 +11,7 @@ #include "Common/CommonTypes.h" #include "Common/Crypto/AES.h" -#include "Common/IOFile.h" +#include "Common/DirectIOFile.h" #include "DiscIO/Blob.h" // This is the file format used for Wii games released on the Wii U eShop. @@ -41,7 +41,7 @@ static_assert(sizeof(NFSHeader) == 0x200); class NFSFileReader final : public BlobReader { public: - static std::unique_ptr Create(File::IOFile first_file, + static std::unique_ptr Create(File::DirectIOFile first_file, const std::string& directory_path); BlobType GetBlobType() const override { return BlobType::NFS; } @@ -65,12 +65,13 @@ private: static bool ReadKey(const std::string& path, const std::string& directory, Key* key_out); static std::vector GetLBARanges(const NFSHeader& header); - static std::vector OpenFiles(const std::string& directory, File::IOFile first_file, - u64 expected_raw_size, u64* raw_size_out); + static std::vector OpenFiles(const std::string& directory, + File::DirectIOFile first_file, + u64 expected_raw_size, u64* raw_size_out); static u64 CalculateExpectedRawSize(const std::vector& lba_ranges); static u64 CalculateExpectedDataSize(const std::vector& lba_ranges); - NFSFileReader(std::vector lba_ranges, std::vector files, Key key, + NFSFileReader(std::vector lba_ranges, std::vector files, Key key, u64 raw_size); u64 ToPhysicalBlockIndex(u64 logical_block_index) const; @@ -83,7 +84,7 @@ private: u64 m_current_logical_block_index = std::numeric_limits::max(); std::vector m_lba_ranges; - std::vector m_files; + std::vector m_files; std::unique_ptr m_aes_context; u64 m_raw_size; u64 m_data_size; diff --git a/Source/Core/DiscIO/SplitFileBlob.cpp b/Source/Core/DiscIO/SplitFileBlob.cpp index 4756088a40d..c6a3dbd9870 100644 --- a/Source/Core/DiscIO/SplitFileBlob.cpp +++ b/Source/Core/DiscIO/SplitFileBlob.cpp @@ -4,17 +4,11 @@ #include "DiscIO/SplitFileBlob.h" #include -#include #include #include #include -#include "Common/Assert.h" -#include "Common/FileUtil.h" -#include "Common/IOFile.h" -#include "Common/MsgHandler.h" - namespace DiscIO { SplitPlainFileReader::SplitPlainFileReader(std::vector files) @@ -38,7 +32,7 @@ std::unique_ptr SplitPlainFileReader::Create(std::string_v u64 offset = 0; while (true) { - File::IOFile f(fmt::format("{}.part{}.iso", base_path, index), "rb"); + File::DirectIOFile f(fmt::format("{}.part{}.iso", base_path, index), File::AccessMode::Read); if (!f.IsOpen()) break; const u64 size = f.GetSize(); @@ -58,13 +52,7 @@ std::unique_ptr SplitPlainFileReader::Create(std::string_v std::unique_ptr SplitPlainFileReader::CopyReader() const { - std::vector new_files{}; - for (const SingleFile& file : m_files) - { - new_files.push_back( - {.file = file.file.Duplicate("rb"), .offset = file.offset, .size = file.size}); - } - return std::unique_ptr(new SplitPlainFileReader(std::move(new_files))); + return std::unique_ptr{new SplitPlainFileReader(m_files)}; } bool SplitPlainFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr) @@ -80,13 +68,10 @@ bool SplitPlainFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr) if (current_offset >= file.offset && current_offset < file.offset + file.size) { auto& f = file.file; - const u64 seek_offset = current_offset - file.offset; - const u64 current_read = std::min(file.size - seek_offset, rest); - if (!f.Seek(seek_offset, File::SeekOrigin::Begin) || !f.ReadBytes(out, current_read)) - { - f.ClearError(); + const u64 offset_in_file = current_offset - file.offset; + const u64 current_read = std::min(file.size - offset_in_file, rest); + if (!f.OffsetRead(offset_in_file, out, current_read)) return false; - } rest -= current_read; if (rest == 0) diff --git a/Source/Core/DiscIO/SplitFileBlob.h b/Source/Core/DiscIO/SplitFileBlob.h index d3268dca80d..4c1fe96380a 100644 --- a/Source/Core/DiscIO/SplitFileBlob.h +++ b/Source/Core/DiscIO/SplitFileBlob.h @@ -10,7 +10,7 @@ #include #include "Common/CommonTypes.h" -#include "Common/IOFile.h" +#include "Common/DirectIOFile.h" #include "DiscIO/Blob.h" namespace DiscIO @@ -37,7 +37,7 @@ public: private: struct SingleFile { - File::IOFile file; + File::DirectIOFile file; u64 offset; u64 size; }; diff --git a/Source/Core/DiscIO/TGCBlob.cpp b/Source/Core/DiscIO/TGCBlob.cpp index 6478ea40dac..5b33649fc60 100644 --- a/Source/Core/DiscIO/TGCBlob.cpp +++ b/Source/Core/DiscIO/TGCBlob.cpp @@ -5,12 +5,11 @@ #include #include -#include #include #include #include -#include "Common/IOFile.h" +#include "Common/BitUtils.h" #include "Common/Swap.h" namespace @@ -45,11 +44,10 @@ void Replace(u64 offset, u64 size, u8* out_ptr, u64 replace_offset, const T& rep namespace DiscIO { -std::unique_ptr TGCFileReader::Create(File::IOFile file) +std::unique_ptr TGCFileReader::Create(File::DirectIOFile file) { TGCHeader header; - if (file.Seek(0, File::SeekOrigin::Begin) && file.ReadArray(&header, 1) && - header.magic == TGC_MAGIC) + if (file.OffsetRead(0, Common::AsWritableU8Span(header)) && header.magic == TGC_MAGIC) { return std::unique_ptr(new TGCFileReader(std::move(file))); } @@ -57,18 +55,16 @@ std::unique_ptr TGCFileReader::Create(File::IOFile file) return nullptr; } -TGCFileReader::TGCFileReader(File::IOFile file) : m_file(std::move(file)) +TGCFileReader::TGCFileReader(File::DirectIOFile file) : m_file(std::move(file)) { - m_file.Seek(0, File::SeekOrigin::Begin); - m_file.ReadArray(&m_header, 1); + m_file.OffsetRead(0, Common::AsWritableU8Span(m_header)); m_size = m_file.GetSize(); const u32 fst_offset = Common::swap32(m_header.fst_real_offset); const u32 fst_size = Common::swap32(m_header.fst_size); m_fst.resize(fst_size); - if (!m_file.Seek(fst_offset, File::SeekOrigin::Begin) || - !m_file.ReadBytes(m_fst.data(), m_fst.size())) + if (!m_file.OffsetRead(fst_offset, m_fst)) { m_fst.clear(); } @@ -100,7 +96,7 @@ TGCFileReader::TGCFileReader(File::IOFile file) : m_file(std::move(file)) std::unique_ptr TGCFileReader::CopyReader() const { - return Create(m_file.Duplicate("rb")); + return Create(m_file); } u64 TGCFileReader::GetDataSize() const @@ -112,8 +108,7 @@ bool TGCFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr) { const u32 tgc_header_size = Common::swap32(m_header.tgc_header_size); - if (m_file.Seek(offset + tgc_header_size, File::SeekOrigin::Begin) && - m_file.ReadBytes(out_ptr, nbytes)) + if (m_file.OffsetRead(offset + tgc_header_size, out_ptr, nbytes)) { const u32 replacement_dol_offset = SubtractBE32(m_header.dol_real_offset, tgc_header_size); const u32 replacement_fst_offset = SubtractBE32(m_header.fst_real_offset, tgc_header_size); @@ -126,7 +121,6 @@ bool TGCFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr) return true; } - m_file.ClearError(); return false; } diff --git a/Source/Core/DiscIO/TGCBlob.h b/Source/Core/DiscIO/TGCBlob.h index 2952f2809f9..1271fa9253a 100644 --- a/Source/Core/DiscIO/TGCBlob.h +++ b/Source/Core/DiscIO/TGCBlob.h @@ -4,11 +4,10 @@ #pragma once #include -#include #include #include "Common/CommonTypes.h" -#include "Common/IOFile.h" +#include "Common/DirectIOFile.h" #include "DiscIO/Blob.h" namespace DiscIO @@ -39,7 +38,7 @@ struct TGCHeader class TGCFileReader final : public BlobReader { public: - static std::unique_ptr Create(File::IOFile file); + static std::unique_ptr Create(File::DirectIOFile file); BlobType GetBlobType() const override { return BlobType::TGC; } std::unique_ptr CopyReader() const override; @@ -56,9 +55,9 @@ public: bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: - TGCFileReader(File::IOFile file); + TGCFileReader(File::DirectIOFile file); - File::IOFile m_file; + File::DirectIOFile m_file; u64 m_size; std::vector m_fst; diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 18cdb04d4b1..01baf6a83a4 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -19,10 +19,10 @@ #include "Common/Align.h" #include "Common/Assert.h" +#include "Common/BitUtils.h" #include "Common/CommonTypes.h" #include "Common/Crypto/SHA1.h" #include "Common/FileUtil.h" -#include "Common/IOFile.h" #include "Common/Logging/Log.h" #include "Common/MsgHandler.h" #include "Common/ScopeGuard.h" @@ -78,7 +78,7 @@ std::pair GetAllowedCompressionLevels(WIARVZCompressionType compressio } template -WIARVZFileReader::WIARVZFileReader(File::IOFile file, const std::string& path) +WIARVZFileReader::WIARVZFileReader(File::DirectIOFile file, const std::string& path) : m_file(std::move(file)), m_path(path), m_encryption_cache(this) { m_valid = Initialize(path); @@ -90,8 +90,11 @@ WIARVZFileReader::~WIARVZFileReader() = default; template bool WIARVZFileReader::Initialize(const std::string& path) { - if (!m_file.Seek(0, File::SeekOrigin::Begin) || !m_file.ReadArray(&m_header_1, 1)) + if (!m_file.Seek(0, File::SeekOrigin::Begin) || + !m_file.Read(Common::AsWritableU8Span(m_header_1))) + { return false; + } if ((!RVZ && m_header_1.magic != WIA_MAGIC) || (RVZ && m_header_1.magic != RVZ_MAGIC)) return false; @@ -126,7 +129,7 @@ bool WIARVZFileReader::Initialize(const std::string& path) return false; std::vector header_2(header_2_size); - if (!m_file.ReadBytes(header_2.data(), header_2.size())) + if (!m_file.Read(header_2)) return false; const auto header_2_actual_hash = Common::SHA1::CalculateDigest(header_2); @@ -161,10 +164,10 @@ bool WIARVZFileReader::Initialize(const std::string& path) const size_t number_of_partition_entries = Common::swap32(m_header_2.number_of_partition_entries); const size_t partition_entry_size = Common::swap32(m_header_2.partition_entry_size); std::vector partition_entries(partition_entry_size * number_of_partition_entries); - if (!m_file.Seek(Common::swap64(m_header_2.partition_entries_offset), File::SeekOrigin::Begin)) - return false; - if (!m_file.ReadBytes(partition_entries.data(), partition_entries.size())) + if (!m_file.OffsetRead(Common::swap64(m_header_2.partition_entries_offset), partition_entries)) + { return false; + } const auto partition_entries_actual_hash = Common::SHA1::CalculateDigest(partition_entries); if (m_header_2.partition_entries_hash != partition_entries_actual_hash) @@ -273,7 +276,7 @@ bool WIARVZFileReader::HasDataOverlap() const } template -std::unique_ptr> WIARVZFileReader::Create(File::IOFile file, +std::unique_ptr> WIARVZFileReader::Create(File::DirectIOFile file, const std::string& path) { std::unique_ptr blob(new WIARVZFileReader(std::move(file), path)); @@ -289,7 +292,7 @@ BlobType WIARVZFileReader::GetBlobType() const template std::unique_ptr WIARVZFileReader::CopyReader() const { - return Create(m_file.Duplicate("rb"), m_path); + return Create(m_file, m_path); } template @@ -628,8 +631,8 @@ template WIARVZFileReader::Chunk::Chunk() = default; template -WIARVZFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, - u64 decompressed_size, u32 exception_lists, +WIARVZFileReader::Chunk::Chunk(File::DirectIOFile* file, u64 offset_in_file, + u64 compressed_size, u64 decompressed_size, u32 exception_lists, bool compressed_exception_lists, u32 rvz_packed_size, u64 data_offset, std::unique_ptr decompressor) : m_decompressor(std::move(decompressor)), m_file(file), m_offset_in_file(offset_in_file), @@ -691,9 +694,7 @@ bool WIARVZFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr) return false; } - if (!m_file->Seek(m_offset_in_file, File::SeekOrigin::Begin)) - return false; - if (!m_file->ReadBytes(m_in.data.data() + m_in.bytes_written, bytes_to_read)) + if (!m_file->OffsetRead(m_offset_in_file, m_in.data.data() + m_in.bytes_written, bytes_to_read)) return false; m_offset_in_file += bytes_to_read; @@ -881,7 +882,7 @@ bool WIARVZFileReader::ApplyHashExceptions( } template -bool WIARVZFileReader::PadTo4(File::IOFile* file, u64* bytes_written) +bool WIARVZFileReader::PadTo4(File::DirectIOFile* file, u64* bytes_written) { constexpr u32 ZEROES = 0; const u64 bytes_to_write = Common::AlignUp(*bytes_written, 4) - *bytes_written; @@ -889,7 +890,7 @@ bool WIARVZFileReader::PadTo4(File::IOFile* file, u64* bytes_written) return true; *bytes_written += bytes_to_write; - return file->WriteBytes(&ZEROES, bytes_to_write); + return file->Write(Common::AsU8Span(ZEROES).first(bytes_to_write)); } template @@ -1646,7 +1647,7 @@ WIARVZFileReader::ProcessAndCompress(CompressThreadState* state, CompressPa template ConversionResultCode WIARVZFileReader::Output(std::vector* entries, - File::IOFile* outfile, + File::DirectIOFile* outfile, std::map* reusable_groups, std::mutex* reusable_groups_mutex, GroupEntry* group_entry, u64* bytes_written) @@ -1675,9 +1676,9 @@ ConversionResultCode WIARVZFileReader::Output(std::vectordata_size = Common::swap32(data_size); - if (!outfile->WriteArray(entry.exception_lists.data(), entry.exception_lists.size())) + if (!outfile->Write(entry.exception_lists)) return ConversionResultCode::WriteFailed; - if (!outfile->WriteArray(entry.main_data.data(), entry.main_data.size())) + if (!outfile->Write(entry.main_data)) return ConversionResultCode::WriteFailed; *bytes_written += entry.exception_lists.size() + entry.main_data.size(); @@ -1716,7 +1717,7 @@ ConversionResultCode WIARVZFileReader::RunCallback(size_t groups_written, u } template -bool WIARVZFileReader::WriteHeader(File::IOFile* file, const u8* data, size_t size, +bool WIARVZFileReader::WriteHeader(File::DirectIOFile* file, const u8* data, size_t size, u64 upper_bound, u64* bytes_written, u64* offset_out) { // The first part of the check is to prevent this from running more than once. If *bytes_written @@ -1731,7 +1732,7 @@ bool WIARVZFileReader::WriteHeader(File::IOFile* file, const u8* data, size } *offset_out = *bytes_written; - if (!file->WriteArray(data, size)) + if (!file->Write(data, size)) return false; *bytes_written += size; return PadTo4(file, bytes_written); @@ -1740,7 +1741,7 @@ bool WIARVZFileReader::WriteHeader(File::IOFile* file, const u8* data, size template ConversionResultCode WIARVZFileReader::Convert(BlobReader* infile, const VolumeDisc* infile_volume, - File::IOFile* outfile, WIARVZCompressionType compression_type, + File::DirectIOFile* outfile, WIARVZCompressionType compression_type, int compression_level, int chunk_size, CompressCB callback) { ASSERT(infile->GetDataSizeType() == DataSizeType::Accurate); @@ -1806,7 +1807,7 @@ WIARVZFileReader::Convert(BlobReader* infile, const VolumeDisc* infile_volu std::vector buffer; buffer.resize(headers_size_upper_bound); - outfile->WriteBytes(buffer.data(), buffer.size()); + outfile->Write(buffer); bytes_written = headers_size_upper_bound; if (!infile->Read(0, header_2.disc_header.size(), header_2.disc_header.data())) @@ -2029,9 +2030,9 @@ WIARVZFileReader::Convert(BlobReader* infile, const VolumeDisc* infile_volu if (!outfile->Seek(0, File::SeekOrigin::Begin)) return ConversionResultCode::WriteFailed; - if (!outfile->WriteArray(&header_1, 1)) + if (!outfile->Write(Common::AsU8Span(header_1))) return ConversionResultCode::WriteFailed; - if (!outfile->WriteArray(&header_2, 1)) + if (!outfile->Write(Common::AsU8Span(header_2))) return ConversionResultCode::WriteFailed; return ConversionResultCode::Success; @@ -2042,8 +2043,8 @@ bool ConvertToWIAOrRVZ(BlobReader* infile, const std::string& infile_path, WIARVZCompressionType compression_type, int compression_level, int chunk_size, const CompressCB& callback) { - File::IOFile outfile(outfile_path, "wb"); - if (!outfile) + File::DirectIOFile outfile(outfile_path, File::AccessMode::Write); + if (!outfile.IsOpen()) { PanicAlertFmtT( "Failed to open the output file \"{0}\".\n" diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index f42a7f4c732..5e4bc26c92a 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -13,7 +13,7 @@ #include "Common/CommonTypes.h" #include "Common/Crypto/SHA1.h" -#include "Common/IOFile.h" +#include "Common/DirectIOFile.h" #include "Common/Swap.h" #include "DiscIO/Blob.h" #include "DiscIO/MultithreadedCompressor.h" @@ -46,7 +46,7 @@ class WIARVZFileReader final : public BlobReader public: ~WIARVZFileReader() override; - static std::unique_ptr Create(File::IOFile file, const std::string& path); + static std::unique_ptr Create(File::DirectIOFile file, const std::string& path); BlobType GetBlobType() const override; std::unique_ptr CopyReader() const override; @@ -68,8 +68,9 @@ public: bool ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) override; static ConversionResultCode Convert(BlobReader* infile, const VolumeDisc* infile_volume, - File::IOFile* outfile, WIARVZCompressionType compression_type, - int compression_level, int chunk_size, CompressCB callback); + File::DirectIOFile* outfile, + WIARVZCompressionType compression_type, int compression_level, + int chunk_size, CompressCB callback); private: using WiiKey = std::array; @@ -188,7 +189,7 @@ private: { public: Chunk(); - Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size, + Chunk(File::DirectIOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size, u32 exception_lists, bool compressed_exception_lists, u32 rvz_packed_size, u64 data_offset, std::unique_ptr decompressor); @@ -216,7 +217,7 @@ private: size_t m_in_bytes_read = 0; std::unique_ptr m_decompressor = nullptr; - File::IOFile* m_file = nullptr; + File::DirectIOFile* m_file = nullptr; u64 m_offset_in_file = 0; size_t m_out_bytes_allocated_for_exceptions = 0; @@ -228,7 +229,7 @@ private: u64 m_data_offset = 0; }; - explicit WIARVZFileReader(File::IOFile file, const std::string& path); + explicit WIARVZFileReader(File::DirectIOFile file, const std::string& path); bool Initialize(const std::string& path); bool HasDataOverlap() const; @@ -323,7 +324,7 @@ private: size_t group_index = 0; }; - static bool PadTo4(File::IOFile* file, u64* bytes_written); + static bool PadTo4(File::DirectIOFile* file, u64* bytes_written); static void AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* total_groups, std::vector* raw_data_entries, std::vector* data_entries); @@ -337,7 +338,7 @@ private: std::vector* data_entries, std::vector* partition_file_systems); static std::optional> Compress(Compressor* compressor, const u8* data, size_t size); - static bool WriteHeader(File::IOFile* file, const u8* data, size_t size, u64 upper_bound, + static bool WriteHeader(File::DirectIOFile* file, const u8* data, size_t size, u64 upper_bound, u64* bytes_written, u64* offset_out); static void SetUpCompressor(std::unique_ptr* compressor, @@ -354,7 +355,7 @@ private: u64 exception_lists_per_chunk, bool compressed_exception_lists, bool compression); static ConversionResultCode Output(std::vector* entries, - File::IOFile* outfile, + File::DirectIOFile* outfile, std::map* reusable_groups, std::mutex* reusable_groups_mutex, GroupEntry* group_entry, u64* bytes_written); @@ -365,7 +366,7 @@ private: bool m_valid; WIARVZCompressionType m_compression_type; - File::IOFile m_file; + File::DirectIOFile m_file; std::string m_path; Chunk m_cached_chunk; u64 m_cached_chunk_offset = std::numeric_limits::max(); diff --git a/Source/Core/DiscIO/WbfsBlob.cpp b/Source/Core/DiscIO/WbfsBlob.cpp index 3f5852c46db..6f74497bfae 100644 --- a/Source/Core/DiscIO/WbfsBlob.cpp +++ b/Source/Core/DiscIO/WbfsBlob.cpp @@ -13,8 +13,8 @@ #include "Common/Align.h" #include "Common/Assert.h" +#include "Common/BitUtils.h" #include "Common/CommonTypes.h" -#include "Common/IOFile.h" #include "Common/Logging/Log.h" #include "Common/Swap.h" @@ -24,7 +24,7 @@ static constexpr u64 WII_SECTOR_SIZE = 0x8000; static constexpr u64 WII_SECTOR_COUNT = 143432 * 2; static constexpr u64 WII_DISC_HEADER_SIZE = 256; -WbfsFileReader::WbfsFileReader(File::IOFile file, const std::string& path) +WbfsFileReader::WbfsFileReader(File::DirectIOFile file, const std::string& path) : m_size(0), m_good(false) { if (!AddFileToList(std::move(file))) @@ -39,7 +39,7 @@ WbfsFileReader::WbfsFileReader(File::IOFile file, const std::string& path) m_wlba_table.resize(m_blocks_per_disc); m_files[0].file.Seek(m_hd_sector_size + WII_DISC_HEADER_SIZE /*+ i * m_disc_info_size*/, File::SeekOrigin::Begin); - m_files[0].file.ReadBytes(m_wlba_table.data(), m_blocks_per_disc * sizeof(u16)); + m_files[0].file.Read(Common::AsWritableU8Span(m_wlba_table)); for (size_t i = 0; i < m_blocks_per_disc; i++) m_wlba_table[i] = Common::swap16(m_wlba_table[i]); } @@ -48,10 +48,9 @@ WbfsFileReader::~WbfsFileReader() = default; std::unique_ptr WbfsFileReader::CopyReader() const { - auto retval = - std::unique_ptr(new WbfsFileReader(m_files[0].file.Duplicate("rb"))); + auto retval = std::unique_ptr(new WbfsFileReader(m_files[0].file)); for (size_t ix = 1; ix < m_files.size(); ix++) - retval->AddFileToList(m_files[ix].file.Duplicate("rb")); + retval->AddFileToList(m_files[ix].file); return retval; } @@ -74,12 +73,12 @@ void WbfsFileReader::OpenAdditionalFiles(const std::string& path) return; std::string current_path = path; current_path.back() = static_cast('0' + m_files.size()); - if (!AddFileToList(File::IOFile(current_path, "rb"))) + if (!AddFileToList(File::DirectIOFile(current_path, File::AccessMode::Read))) return; } } -bool WbfsFileReader::AddFileToList(File::IOFile file) +bool WbfsFileReader::AddFileToList(File::DirectIOFile file) { if (!file.IsOpen()) return false; @@ -95,7 +94,7 @@ bool WbfsFileReader::ReadHeader() { // Read hd size info m_files[0].file.Seek(0, File::SeekOrigin::Begin); - m_files[0].file.ReadBytes(&m_header, sizeof(WbfsHeader)); + m_files[0].file.Read(Common::AsWritableU8Span(m_header)); if (m_header.magic != WBFS_MAGIC) return false; @@ -128,14 +127,13 @@ bool WbfsFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr) while (nbytes) { u64 read_size; - File::IOFile& data_file = SeekToCluster(offset, &read_size); + auto& data_file = SeekToCluster(offset, &read_size); if (read_size == 0) return false; read_size = std::min(read_size, nbytes); - if (!data_file.ReadBytes(out_ptr, read_size)) + if (!data_file.Read(out_ptr, read_size)) { - data_file.ClearError(); return false; } @@ -147,7 +145,7 @@ bool WbfsFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr) return true; } -File::IOFile& WbfsFileReader::SeekToCluster(u64 offset, u64* available) +File::DirectIOFile& WbfsFileReader::SeekToCluster(u64 offset, u64* available) { u64 base_cluster = (offset >> m_header.wbfs_sector_shift); if (base_cluster < m_blocks_per_disc) @@ -180,7 +178,8 @@ File::IOFile& WbfsFileReader::SeekToCluster(u64 offset, u64* available) return m_files[0].file; } -std::unique_ptr WbfsFileReader::Create(File::IOFile file, const std::string& path) +std::unique_ptr WbfsFileReader::Create(File::DirectIOFile file, + const std::string& path) { auto reader = std::unique_ptr(new WbfsFileReader(std::move(file), path)); diff --git a/Source/Core/DiscIO/WbfsBlob.h b/Source/Core/DiscIO/WbfsBlob.h index 8e19f4999ea..0423646b97c 100644 --- a/Source/Core/DiscIO/WbfsBlob.h +++ b/Source/Core/DiscIO/WbfsBlob.h @@ -8,7 +8,7 @@ #include #include "Common/CommonTypes.h" -#include "Common/IOFile.h" +#include "Common/DirectIOFile.h" #include "DiscIO/Blob.h" namespace DiscIO @@ -20,7 +20,7 @@ class WbfsFileReader final : public BlobReader public: ~WbfsFileReader() override; - static std::unique_ptr Create(File::IOFile file, const std::string& path); + static std::unique_ptr Create(File::DirectIOFile file, const std::string& path); BlobType GetBlobType() const override { return BlobType::WBFS; } std::unique_ptr CopyReader() const override; @@ -37,22 +37,22 @@ public: bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: - WbfsFileReader(File::IOFile file, const std::string& path = ""); + WbfsFileReader(File::DirectIOFile file, const std::string& path = ""); void OpenAdditionalFiles(const std::string& path); - bool AddFileToList(File::IOFile file); + bool AddFileToList(File::DirectIOFile file); bool ReadHeader(); - File::IOFile& SeekToCluster(u64 offset, u64* available); + File::DirectIOFile& SeekToCluster(u64 offset, u64* available); bool IsGood() const { return m_good; } struct FileEntry { - FileEntry(File::IOFile file_, u64 base_address_, u64 size_) + FileEntry(File::DirectIOFile file_, u64 base_address_, u64 size_) : file(std::move(file_)), base_address(base_address_), size(size_) { } - File::IOFile file; + File::DirectIOFile file; u64 base_address; u64 size; }; diff --git a/Source/Core/DolphinQt/MainWindow.cpp b/Source/Core/DolphinQt/MainWindow.cpp index d7a21da33eb..15342298e01 100644 --- a/Source/Core/DolphinQt/MainWindow.cpp +++ b/Source/Core/DolphinQt/MainWindow.cpp @@ -35,6 +35,7 @@ #endif #include "Common/Config/Config.h" +#include "Common/FileUtil.h" #include "Common/ScopeGuard.h" #include "Common/Version.h" #include "Common/WindowSystemInfo.h" From 9e2fc7f4ddba14b1e7123d1654b385b21c71c744 Mon Sep 17 00:00:00 2001 From: Jordan Woyak Date: Tue, 28 Oct 2025 22:45:21 -0500 Subject: [PATCH 4/4] Common/IOFile: Remove the `Duplicate` function. The duplicate handles shared a read/write position making them effectively not thread-safe. --- Source/Core/Common/IOFile.cpp | 9 --------- Source/Core/Common/IOFile.h | 2 -- 2 files changed, 11 deletions(-) diff --git a/Source/Core/Common/IOFile.cpp b/Source/Core/Common/IOFile.cpp index cc22979bbef..cf7762ea680 100644 --- a/Source/Core/Common/IOFile.cpp +++ b/Source/Core/Common/IOFile.cpp @@ -101,15 +101,6 @@ bool IOFile::Close() return m_good; } -IOFile IOFile::Duplicate(const char openmode[]) const -{ -#ifdef _WIN32 - return IOFile(_fdopen(_dup(_fileno(m_file)), openmode)); -#else // _WIN32 - return IOFile(fdopen(dup(fileno(m_file)), openmode)); -#endif // _WIN32 -} - void IOFile::SetHandle(std::FILE* file) { Close(); diff --git a/Source/Core/Common/IOFile.h b/Source/Core/Common/IOFile.h index 2aeff1916b8..187cbee4cfb 100644 --- a/Source/Core/Common/IOFile.h +++ b/Source/Core/Common/IOFile.h @@ -51,8 +51,6 @@ public: SharedAccess sh = SharedAccess::Default); bool Close(); - IOFile Duplicate(const char openmode[]) const; - template requires(std::is_trivially_copyable_v) bool ReadArray(T* elements, size_t count, size_t* num_read = nullptr)