vfs_real: add file LRU cache for open file limits
This commit is contained in:
parent
e2f6199225
commit
f25236a4d6
|
@ -25,6 +25,8 @@ namespace FS = Common::FS;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
|
constexpr size_t MaxOpenFiles = 512;
|
||||||
|
|
||||||
constexpr FS::FileAccessMode ModeFlagsToFileAccessMode(Mode mode) {
|
constexpr FS::FileAccessMode ModeFlagsToFileAccessMode(Mode mode) {
|
||||||
switch (mode) {
|
switch (mode) {
|
||||||
case Mode::Read:
|
case Mode::Read:
|
||||||
|
@ -73,24 +75,17 @@ VfsEntryType RealVfsFilesystem::GetEntryType(std::string_view path_) const {
|
||||||
VirtualFile RealVfsFilesystem::OpenFile(std::string_view path_, Mode perms) {
|
VirtualFile RealVfsFilesystem::OpenFile(std::string_view path_, Mode perms) {
|
||||||
const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);
|
const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);
|
||||||
|
|
||||||
if (const auto weak_iter = cache.find(path); weak_iter != cache.cend()) {
|
this->EvictSingleReference();
|
||||||
const auto& weak = weak_iter->second;
|
|
||||||
|
|
||||||
if (!weak.expired()) {
|
|
||||||
return std::shared_ptr<RealVfsFile>(new RealVfsFile(*this, weak.lock(), path, perms));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
auto backing = FS::FileOpen(path, ModeFlagsToFileAccessMode(perms), FS::FileType::BinaryFile);
|
auto backing = FS::FileOpen(path, ModeFlagsToFileAccessMode(perms), FS::FileType::BinaryFile);
|
||||||
|
|
||||||
if (!backing) {
|
if (!backing) {
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
cache.insert_or_assign(path, std::move(backing));
|
num_open_files++;
|
||||||
|
auto reference = std::make_unique<FileReference>(std::move(backing));
|
||||||
// Cannot use make_shared as RealVfsFile constructor is private
|
this->InsertReferenceIntoList(*reference);
|
||||||
return std::shared_ptr<RealVfsFile>(new RealVfsFile(*this, backing, path, perms));
|
return std::shared_ptr<RealVfsFile>(new RealVfsFile(*this, std::move(reference), path, perms));
|
||||||
}
|
}
|
||||||
|
|
||||||
VirtualFile RealVfsFilesystem::CreateFile(std::string_view path_, Mode perms) {
|
VirtualFile RealVfsFilesystem::CreateFile(std::string_view path_, Mode perms) {
|
||||||
|
@ -123,51 +118,19 @@ VirtualFile RealVfsFilesystem::CopyFile(std::string_view old_path_, std::string_
|
||||||
VirtualFile RealVfsFilesystem::MoveFile(std::string_view old_path_, std::string_view new_path_) {
|
VirtualFile RealVfsFilesystem::MoveFile(std::string_view old_path_, std::string_view new_path_) {
|
||||||
const auto old_path = FS::SanitizePath(old_path_, FS::DirectorySeparator::PlatformDefault);
|
const auto old_path = FS::SanitizePath(old_path_, FS::DirectorySeparator::PlatformDefault);
|
||||||
const auto new_path = FS::SanitizePath(new_path_, FS::DirectorySeparator::PlatformDefault);
|
const auto new_path = FS::SanitizePath(new_path_, FS::DirectorySeparator::PlatformDefault);
|
||||||
const auto cached_file_iter = cache.find(old_path);
|
if (!FS::RenameFile(old_path, new_path)) {
|
||||||
|
|
||||||
if (cached_file_iter != cache.cend()) {
|
|
||||||
auto file = cached_file_iter->second.lock();
|
|
||||||
|
|
||||||
if (!cached_file_iter->second.expired()) {
|
|
||||||
file->Close();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!FS::RenameFile(old_path, new_path)) {
|
|
||||||
return nullptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
cache.erase(old_path);
|
|
||||||
file->Open(new_path, FS::FileAccessMode::Read, FS::FileType::BinaryFile);
|
|
||||||
if (file->IsOpen()) {
|
|
||||||
cache.insert_or_assign(new_path, std::move(file));
|
|
||||||
} else {
|
|
||||||
LOG_ERROR(Service_FS, "Failed to open path {} in order to re-cache it", new_path);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
ASSERT(false);
|
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
return OpenFile(new_path, Mode::ReadWrite);
|
return OpenFile(new_path, Mode::ReadWrite);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool RealVfsFilesystem::DeleteFile(std::string_view path_) {
|
bool RealVfsFilesystem::DeleteFile(std::string_view path_) {
|
||||||
const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);
|
const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);
|
||||||
const auto cached_iter = cache.find(path);
|
|
||||||
|
|
||||||
if (cached_iter != cache.cend()) {
|
|
||||||
if (!cached_iter->second.expired()) {
|
|
||||||
cached_iter->second.lock()->Close();
|
|
||||||
}
|
|
||||||
cache.erase(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
return FS::RemoveFile(path);
|
return FS::RemoveFile(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
VirtualDir RealVfsFilesystem::OpenDirectory(std::string_view path_, Mode perms) {
|
VirtualDir RealVfsFilesystem::OpenDirectory(std::string_view path_, Mode perms) {
|
||||||
const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);
|
const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);
|
||||||
// Cannot use make_shared as RealVfsDirectory constructor is private
|
|
||||||
return std::shared_ptr<RealVfsDirectory>(new RealVfsDirectory(*this, path, perms));
|
return std::shared_ptr<RealVfsDirectory>(new RealVfsDirectory(*this, path, perms));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -176,7 +139,6 @@ VirtualDir RealVfsFilesystem::CreateDirectory(std::string_view path_, Mode perms
|
||||||
if (!FS::CreateDirs(path)) {
|
if (!FS::CreateDirs(path)) {
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
// Cannot use make_shared as RealVfsDirectory constructor is private
|
|
||||||
return std::shared_ptr<RealVfsDirectory>(new RealVfsDirectory(*this, path, perms));
|
return std::shared_ptr<RealVfsDirectory>(new RealVfsDirectory(*this, path, perms));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -194,73 +156,102 @@ VirtualDir RealVfsFilesystem::MoveDirectory(std::string_view old_path_,
|
||||||
if (!FS::RenameDir(old_path, new_path)) {
|
if (!FS::RenameDir(old_path, new_path)) {
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto& kv : cache) {
|
|
||||||
// If the path in the cache doesn't start with old_path, then bail on this file.
|
|
||||||
if (kv.first.rfind(old_path, 0) != 0) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const auto file_old_path =
|
|
||||||
FS::SanitizePath(kv.first, FS::DirectorySeparator::PlatformDefault);
|
|
||||||
auto file_new_path = FS::SanitizePath(new_path + '/' + kv.first.substr(old_path.size()),
|
|
||||||
FS::DirectorySeparator::PlatformDefault);
|
|
||||||
const auto& cached = cache[file_old_path];
|
|
||||||
|
|
||||||
if (cached.expired()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto file = cached.lock();
|
|
||||||
cache.erase(file_old_path);
|
|
||||||
file->Open(file_new_path, FS::FileAccessMode::Read, FS::FileType::BinaryFile);
|
|
||||||
if (file->IsOpen()) {
|
|
||||||
cache.insert_or_assign(std::move(file_new_path), std::move(file));
|
|
||||||
} else {
|
|
||||||
LOG_ERROR(Service_FS, "Failed to open path {} in order to re-cache it", file_new_path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return OpenDirectory(new_path, Mode::ReadWrite);
|
return OpenDirectory(new_path, Mode::ReadWrite);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool RealVfsFilesystem::DeleteDirectory(std::string_view path_) {
|
bool RealVfsFilesystem::DeleteDirectory(std::string_view path_) {
|
||||||
const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);
|
const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);
|
||||||
|
|
||||||
for (auto& kv : cache) {
|
|
||||||
// If the path in the cache doesn't start with path, then bail on this file.
|
|
||||||
if (kv.first.rfind(path, 0) != 0) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const auto& entry = cache[kv.first];
|
|
||||||
if (!entry.expired()) {
|
|
||||||
entry.lock()->Close();
|
|
||||||
}
|
|
||||||
|
|
||||||
cache.erase(kv.first);
|
|
||||||
}
|
|
||||||
|
|
||||||
return FS::RemoveDirRecursively(path);
|
return FS::RemoveDirRecursively(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
RealVfsFile::RealVfsFile(RealVfsFilesystem& base_, std::shared_ptr<FS::IOFile> backing_,
|
void RealVfsFilesystem::RefreshReference(const std::string& path, Mode perms,
|
||||||
const std::string& path_, Mode perms_)
|
FileReference& reference) {
|
||||||
: base(base_), backing(std::move(backing_)), path(path_), parent_path(FS::GetParentPath(path_)),
|
// Temporarily remove from list.
|
||||||
path_components(FS::SplitPathComponents(path_)), perms(perms_) {}
|
this->RemoveReferenceFromList(reference);
|
||||||
|
|
||||||
RealVfsFile::~RealVfsFile() = default;
|
// Restore file if needed.
|
||||||
|
if (!reference.file) {
|
||||||
|
this->EvictSingleReference();
|
||||||
|
|
||||||
|
reference.file =
|
||||||
|
FS::FileOpen(path, ModeFlagsToFileAccessMode(perms), FS::FileType::BinaryFile);
|
||||||
|
if (reference.file) {
|
||||||
|
num_open_files++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reinsert into list.
|
||||||
|
this->InsertReferenceIntoList(reference);
|
||||||
|
}
|
||||||
|
|
||||||
|
void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference) {
|
||||||
|
// Remove from list.
|
||||||
|
this->RemoveReferenceFromList(*reference);
|
||||||
|
|
||||||
|
// Close the file.
|
||||||
|
if (reference->file) {
|
||||||
|
reference->file.reset();
|
||||||
|
num_open_files--;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void RealVfsFilesystem::EvictSingleReference() {
|
||||||
|
if (num_open_files < MaxOpenFiles || open_references.empty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get and remove from list.
|
||||||
|
auto& reference = open_references.back();
|
||||||
|
this->RemoveReferenceFromList(reference);
|
||||||
|
|
||||||
|
// Close the file.
|
||||||
|
if (reference.file) {
|
||||||
|
reference.file.reset();
|
||||||
|
num_open_files--;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reinsert into closed list.
|
||||||
|
this->InsertReferenceIntoList(reference);
|
||||||
|
}
|
||||||
|
|
||||||
|
void RealVfsFilesystem::InsertReferenceIntoList(FileReference& reference) {
|
||||||
|
if (reference.file) {
|
||||||
|
open_references.push_front(reference);
|
||||||
|
} else {
|
||||||
|
closed_references.push_front(reference);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void RealVfsFilesystem::RemoveReferenceFromList(FileReference& reference) {
|
||||||
|
if (reference.file) {
|
||||||
|
open_references.erase(open_references.iterator_to(reference));
|
||||||
|
} else {
|
||||||
|
closed_references.erase(closed_references.iterator_to(reference));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
RealVfsFile::RealVfsFile(RealVfsFilesystem& base_, std::unique_ptr<FileReference> reference_,
|
||||||
|
const std::string& path_, Mode perms_)
|
||||||
|
: base(base_), reference(std::move(reference_)), path(path_),
|
||||||
|
parent_path(FS::GetParentPath(path_)), path_components(FS::SplitPathComponents(path_)),
|
||||||
|
perms(perms_) {}
|
||||||
|
|
||||||
|
RealVfsFile::~RealVfsFile() {
|
||||||
|
base.DropReference(std::move(reference));
|
||||||
|
}
|
||||||
|
|
||||||
std::string RealVfsFile::GetName() const {
|
std::string RealVfsFile::GetName() const {
|
||||||
return path_components.back();
|
return path_components.back();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::size_t RealVfsFile::GetSize() const {
|
std::size_t RealVfsFile::GetSize() const {
|
||||||
return backing->GetSize();
|
base.RefreshReference(path, perms, *reference);
|
||||||
|
return reference->file ? reference->file->GetSize() : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool RealVfsFile::Resize(std::size_t new_size) {
|
bool RealVfsFile::Resize(std::size_t new_size) {
|
||||||
return backing->SetSize(new_size);
|
base.RefreshReference(path, perms, *reference);
|
||||||
|
return reference->file ? reference->file->SetSize(new_size) : false;
|
||||||
}
|
}
|
||||||
|
|
||||||
VirtualDir RealVfsFile::GetContainingDirectory() const {
|
VirtualDir RealVfsFile::GetContainingDirectory() const {
|
||||||
|
@ -276,27 +267,25 @@ bool RealVfsFile::IsReadable() const {
|
||||||
}
|
}
|
||||||
|
|
||||||
std::size_t RealVfsFile::Read(u8* data, std::size_t length, std::size_t offset) const {
|
std::size_t RealVfsFile::Read(u8* data, std::size_t length, std::size_t offset) const {
|
||||||
if (!backing->Seek(static_cast<s64>(offset))) {
|
base.RefreshReference(path, perms, *reference);
|
||||||
|
if (!reference->file || !reference->file->Seek(static_cast<s64>(offset))) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
return backing->ReadSpan(std::span{data, length});
|
return reference->file->ReadSpan(std::span{data, length});
|
||||||
}
|
}
|
||||||
|
|
||||||
std::size_t RealVfsFile::Write(const u8* data, std::size_t length, std::size_t offset) {
|
std::size_t RealVfsFile::Write(const u8* data, std::size_t length, std::size_t offset) {
|
||||||
if (!backing->Seek(static_cast<s64>(offset))) {
|
base.RefreshReference(path, perms, *reference);
|
||||||
|
if (!reference->file || !reference->file->Seek(static_cast<s64>(offset))) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
return backing->WriteSpan(std::span{data, length});
|
return reference->file->WriteSpan(std::span{data, length});
|
||||||
}
|
}
|
||||||
|
|
||||||
bool RealVfsFile::Rename(std::string_view name) {
|
bool RealVfsFile::Rename(std::string_view name) {
|
||||||
return base.MoveFile(path, parent_path + '/' + std::string(name)) != nullptr;
|
return base.MoveFile(path, parent_path + '/' + std::string(name)) != nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
void RealVfsFile::Close() {
|
|
||||||
backing->Close();
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO(DarkLordZach): MSVC would not let me combine the following two functions using 'if
|
// TODO(DarkLordZach): MSVC would not let me combine the following two functions using 'if
|
||||||
// constexpr' because there is a compile error in the branch not used.
|
// constexpr' because there is a compile error in the branch not used.
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <boost/container/flat_map.hpp>
|
#include "common/intrusive_list.h"
|
||||||
#include "core/file_sys/mode.h"
|
#include "core/file_sys/mode.h"
|
||||||
#include "core/file_sys/vfs.h"
|
#include "core/file_sys/vfs.h"
|
||||||
|
|
||||||
|
@ -14,6 +14,12 @@ class IOFile;
|
||||||
|
|
||||||
namespace FileSys {
|
namespace FileSys {
|
||||||
|
|
||||||
|
struct FileReference : public Common::IntrusiveListBaseNode<FileReference> {
|
||||||
|
FileReference(std::shared_ptr<Common::FS::IOFile>&& f) : file(f) {}
|
||||||
|
std::shared_ptr<Common::FS::IOFile> file;
|
||||||
|
};
|
||||||
|
|
||||||
|
class RealVfsFile;
|
||||||
class RealVfsFilesystem : public VfsFilesystem {
|
class RealVfsFilesystem : public VfsFilesystem {
|
||||||
public:
|
public:
|
||||||
RealVfsFilesystem();
|
RealVfsFilesystem();
|
||||||
|
@ -35,7 +41,20 @@ public:
|
||||||
bool DeleteDirectory(std::string_view path) override;
|
bool DeleteDirectory(std::string_view path) override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
boost::container::flat_map<std::string, std::weak_ptr<Common::FS::IOFile>> cache;
|
using ReferenceListType = Common::IntrusiveListBaseTraits<FileReference>::ListType;
|
||||||
|
ReferenceListType open_references;
|
||||||
|
ReferenceListType closed_references;
|
||||||
|
size_t num_open_files{};
|
||||||
|
|
||||||
|
private:
|
||||||
|
friend class RealVfsFile;
|
||||||
|
void RefreshReference(const std::string& path, Mode perms, FileReference& reference);
|
||||||
|
void DropReference(std::unique_ptr<FileReference>&& reference);
|
||||||
|
void EvictSingleReference();
|
||||||
|
|
||||||
|
private:
|
||||||
|
void InsertReferenceIntoList(FileReference& reference);
|
||||||
|
void RemoveReferenceFromList(FileReference& reference);
|
||||||
};
|
};
|
||||||
|
|
||||||
// An implementation of VfsFile that represents a file on the user's computer.
|
// An implementation of VfsFile that represents a file on the user's computer.
|
||||||
|
@ -57,13 +76,11 @@ public:
|
||||||
bool Rename(std::string_view name) override;
|
bool Rename(std::string_view name) override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
RealVfsFile(RealVfsFilesystem& base, std::shared_ptr<Common::FS::IOFile> backing,
|
RealVfsFile(RealVfsFilesystem& base, std::unique_ptr<FileReference> reference,
|
||||||
const std::string& path, Mode perms = Mode::Read);
|
const std::string& path, Mode perms = Mode::Read);
|
||||||
|
|
||||||
void Close();
|
|
||||||
|
|
||||||
RealVfsFilesystem& base;
|
RealVfsFilesystem& base;
|
||||||
std::shared_ptr<Common::FS::IOFile> backing;
|
std::unique_ptr<FileReference> reference;
|
||||||
std::string path;
|
std::string path;
|
||||||
std::string parent_path;
|
std::string parent_path;
|
||||||
std::vector<std::string> path_components;
|
std::vector<std::string> path_components;
|
||||||
|
|
Reference in New Issue