yuzu-emu
/
yuzu
Archived
1
0
Fork 0
Code

gl_shader_disk_cache: Save GLSL and entries into the precompiled file

This commit is contained in:
ReinUsesLisp 2019-01-15 01:07:57 -03:00
parent e78da8dc1f
commit cfb20c4c9d
9 changed files with 234 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/video_core/renderer_opengl/gl_rasterizer.cpp
View File

@ -1009,22 +1009,20 @@ void RasterizerOpenGL::SetupTextures(Maxwell::ShaderStage stage, const Shader& s
for (u32 bindpoint = 0; bindpoint < entries.size(); ++bindpoint) {
const auto& entry = entries[bindpoint];
const auto texture = maxwell3d.GetStageTexture(stage, entry.GetOffset());
const u32 current_bindpoint = base_bindings.sampler + bindpoint;
auto& unit = state.texture_units[current_bindpoint];
const auto texture = maxwell3d.GetStageTexture(entry.GetStage(), entry.GetOffset());
texture_samplers[current_bindpoint].SyncWithConfig(texture.tsc);
Surface surface = res_cache.GetTextureSurface(texture, entry);
if (surface != nullptr) {
unit.texture =
state.texture_units[current_bindpoint].texture =
entry.IsArray() ? surface->TextureLayer().handle : surface->Texture().handle;
surface->UpdateSwizzle(texture.tic.x_source, texture.tic.y_source, texture.tic.z_source,
texture.tic.w_source);
} else {
// Can occur when texture addr is null or its memory is unmapped/invalid
unit.texture = 0;
state.texture_units[current_bindpoint].texture = 0;
}
}
}

69
src/video_core/renderer_opengl/gl_shader_cache.cpp
View File

@ -143,6 +143,8 @@ GLShader::ProgramResult CreateProgram(Maxwell::ShaderProgram program_type, Progr
// stage here.
setup.SetProgramB(std::move(program_code_b));
}
setup.program.unique_identifier =
GetUniqueIdentifier(program_type, program_code, program_code_b);
switch (program_type) {
case Maxwell::ShaderProgram::VertexA:
@ -348,15 +350,12 @@ void ShaderCacheOpenGL::LoadDiskCache() {
return;
}
std::vector<ShaderDiskCachePrecompiledEntry> precompiled = disk_cache.LoadPrecompiled();
const auto SearchPrecompiled = [&precompiled](const ShaderDiskCacheUsage& usage) {
return std::find_if(
precompiled.begin(), precompiled.end(),
[&usage](const auto& precompiled_entry) { return precompiled_entry.usage == usage; });
};
std::map<u64, ShaderDiskCacheDecompiled> decompiled;
std::map<ShaderDiskCacheUsage, ShaderDiskCacheDump> dumps;
disk_cache.LoadPrecompiled(decompiled, dumps);
const std::set<GLenum> supported_formats{GetSupportedFormats()};
const auto unspecialized{GenerateUnspecializedShaders(raws)};
const auto unspecialized{GenerateUnspecializedShaders(raws, decompiled)};
// Build shaders
for (std::size_t i = 0; i < usages.size(); ++i) {
@ -365,13 +364,17 @@ void ShaderCacheOpenGL::LoadDiskCache() {
i + 1, usages.size());
const auto& unspec{unspecialized.at(usage.unique_identifier)};
const auto precompiled_it = SearchPrecompiled(usage);
const bool is_precompiled = precompiled_it != precompiled.end();
const auto dump_it = dumps.find(usage);
CachedProgram shader;
if (is_precompiled) {
shader = GeneratePrecompiledProgram(precompiled, *precompiled_it, supported_formats);
if (dump_it != dumps.end()) {
// If the shader is dumped, attempt to load it with
shader = GeneratePrecompiledProgram(dump_it->second, supported_formats);
if (!shader) {
// Invalidate the precompiled cache if a shader dumped shader was rejected
disk_cache.InvalidatePrecompiled();
dumps.clear();
}
}
if (!shader) {
shader = SpecializeShader(unspec.code, unspec.entries, unspec.program_type,
@ -385,52 +388,47 @@ void ShaderCacheOpenGL::LoadDiskCache() {
for (std::size_t i = 0; i < usages.size(); ++i) {
const auto& usage{usages[i]};
if (SearchPrecompiled(usage) == precompiled.end()) {
if (dumps.find(usage) == dumps.end()) {
const auto& program = precompiled_programs.at(usage);
disk_cache.SavePrecompiled(usage, program->handle);
disk_cache.SaveDump(usage, program->handle);
}
}
}
CachedProgram ShaderCacheOpenGL::GeneratePrecompiledProgram(
std::vector<ShaderDiskCachePrecompiledEntry>& precompiled,
const ShaderDiskCachePrecompiledEntry& precompiled_entry,
const std::set<GLenum>& supported_formats) {
const ShaderDiskCacheDump& dump, const std::set<GLenum>& supported_formats) {
if (supported_formats.find(precompiled_entry.binary_format) == supported_formats.end()) {
if (supported_formats.find(dump.binary_format) == supported_formats.end()) {
LOG_INFO(Render_OpenGL, "Precompiled cache entry with unsupported format - removing");
disk_cache.InvalidatePrecompiled();
precompiled.clear();
return {};
}
CachedProgram shader = std::make_shared<OGLProgram>();
shader->handle = glCreateProgram();
glProgramBinary(shader->handle, precompiled_entry.binary_format,
precompiled_entry.binary.data(),
static_cast<GLsizei>(precompiled_entry.binary.size()));
glProgramBinary(shader->handle, dump.binary_format, dump.binary.data(),
static_cast<GLsizei>(dump.binary.size()));
GLint link_status{};
glGetProgramiv(shader->handle, GL_LINK_STATUS, &link_status);
if (link_status == GL_FALSE) {
LOG_INFO(Render_OpenGL, "Precompiled cache rejected by the driver - removing");
disk_cache.InvalidatePrecompiled();
precompiled.clear();
shader.reset();
return {};
}
return shader;
}
std::map<u64, UnspecializedShader> ShaderCacheOpenGL::GenerateUnspecializedShaders(
const std::vector<ShaderDiskCacheRaw>& raws) {
const std::vector<ShaderDiskCacheRaw>& raws,
const std::map<u64, ShaderDiskCacheDecompiled>& decompiled) {
std::map<u64, UnspecializedShader> unspecialized;
for (const auto& raw : raws) {
const u64 unique_identifier = raw.GetUniqueIdentifier();
const u64 calculated_hash =
GetUniqueIdentifier(raw.GetProgramType(), raw.GetProgramCode(), raw.GetProgramCodeB());
if (raw.GetUniqueIdentifier() != calculated_hash) {
if (unique_identifier != calculated_hash) {
LOG_ERROR(
Render_OpenGL,
"Invalid hash in entry={:016x} (obtained hash={:016x}) - removing shader cache",
@ -439,10 +437,19 @@ std::map<u64, UnspecializedShader> ShaderCacheOpenGL::GenerateUnspecializedShade
return {};
}
auto result =
GLShader::ProgramResult result;
if (const auto it = decompiled.find(unique_identifier); it != decompiled.end()) {
// If it's stored in the precompiled file, avoid decompiling it here
const auto& stored_decompiled{it->second};
result = {stored_decompiled.code, stored_decompiled.entries};
} else {
// Otherwise decompile the shader at boot and save the result to the decompiled file
result =
CreateProgram(raw.GetProgramType(), raw.GetProgramCode(), raw.GetProgramCodeB());
disk_cache.SaveDecompiled(unique_identifier, result.first, result.second);
}
precompiled_shaders.insert({raw.GetUniqueIdentifier(), result});
precompiled_shaders.insert({unique_identifier, result});
unspecialized.insert(
{raw.GetUniqueIdentifier(),

7
src/video_core/renderer_opengl/gl_shader_cache.h
View File

@ -117,11 +117,10 @@ public:
private:
std::map<u64, UnspecializedShader> GenerateUnspecializedShaders(
const std::vector<ShaderDiskCacheRaw>& raws);
const std::vector<ShaderDiskCacheRaw>& raws,
const std::map<u64, ShaderDiskCacheDecompiled>& decompiled);
CachedProgram GeneratePrecompiledProgram(
std::vector<ShaderDiskCachePrecompiledEntry>& precompiled,
const ShaderDiskCachePrecompiledEntry& precompiled_entry,
CachedProgram GeneratePrecompiledProgram(const ShaderDiskCacheDump& dump,
const std::set<GLenum>& supported_formats);
std::array<Shader, Maxwell::MaxShaderProgram> last_shaders;

7
src/video_core/renderer_opengl/gl_shader_decompiler.cpp
View File

@ -193,13 +193,14 @@ public:
ShaderEntries GetShaderEntries() const {
ShaderEntries entries;
for (const auto& cbuf : ir.GetConstantBuffers()) {
entries.const_buffers.emplace_back(cbuf.second, stage, cbuf.first);
entries.const_buffers.emplace_back(cbuf.second.GetMaxOffset(), cbuf.second.IsIndirect(),
cbuf.first);
}
for (const auto& sampler : ir.GetSamplers()) {
entries.samplers.emplace_back(sampler, stage);
entries.samplers.emplace_back(sampler);
}
for (const auto& gmem : ir.GetGlobalMemoryBases()) {
entries.global_memory_entries.emplace_back(gmem.cbuf_index, gmem.cbuf_offset, stage);
entries.global_memory_entries.emplace_back(gmem.cbuf_index, gmem.cbuf_offset);
}
entries.clip_distances = ir.GetClipDistances();
entries.shader_length = ir.GetLength();

65
src/video_core/renderer_opengl/gl_shader_decompiler.h
View File

@ -23,40 +23,23 @@ using Maxwell = Tegra::Engines::Maxwell3D::Regs;
class ConstBufferEntry : public VideoCommon::Shader::ConstBuffer {
public:
explicit ConstBufferEntry(const VideoCommon::Shader::ConstBuffer& entry,
Maxwell::ShaderStage stage, u32 index)
: VideoCommon::Shader::ConstBuffer{entry}, stage{stage}, index{index} {}
Maxwell::ShaderStage GetStage() const {
return stage;
}
explicit ConstBufferEntry(u32 max_offset, bool is_indirect, u32 index)
: VideoCommon::Shader::ConstBuffer{max_offset, is_indirect}, index{index} {}
u32 GetIndex() const {
return index;
}
private:
Maxwell::ShaderStage stage{};
u32 index{};
};
class SamplerEntry : public VideoCommon::Shader::Sampler {
public:
explicit SamplerEntry(const VideoCommon::Shader::Sampler& entry, Maxwell::ShaderStage stage)
: VideoCommon::Shader::Sampler{entry}, stage{stage} {}
Maxwell::ShaderStage GetStage() const {
return stage;
}
private:
Maxwell::ShaderStage stage{};
};
using SamplerEntry = VideoCommon::Shader::Sampler;
class GlobalMemoryEntry {
public:
explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset, Maxwell::ShaderStage stage)
: cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset}, stage{stage} {}
explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset)
: cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset} {}
u32 GetCbufIndex() const {
return cbuf_index;
@ -66,47 +49,9 @@ public:
return cbuf_offset;
}
Maxwell::ShaderStage GetStage() const {
return stage;
}
private:
u32 cbuf_index{};
u32 cbuf_offset{};
Maxwell::ShaderStage stage{};
std::string name;
};
class GlobalMemoryEntry {
public:
explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset, Maxwell::ShaderStage stage,
std::string name)
: cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset}, stage{stage}, name{std::move(name)} {}
u32 GetCbufIndex() const {
return cbuf_index;
}
u32 GetCbufOffset() const {
return cbuf_offset;
}
const std::string& GetName() const {
return name;
}
Maxwell::ShaderStage GetStage() const {
return stage;
}
u32 GetHash() const {
return (static_cast<u32>(stage) << 24) | (cbuf_index << 16) | cbuf_offset;
}
private:
u32 cbuf_index{};
u32 cbuf_offset{};
Maxwell::ShaderStage stage{};
};
struct ShaderEntries {

173
src/video_core/renderer_opengl/gl_shader_disk_cache.cpp
View File

@ -24,11 +24,16 @@
namespace OpenGL {
enum class EntryKind : u32 {
enum class TransferableEntryKind : u32 {
Raw,
Usage,
};
enum class PrecompiledEntryKind : u32 {
Decompiled,
Dump,
};
constexpr u32 NativeVersion = 1;
constexpr u32 ShaderHashSize = 64;
@ -108,17 +113,17 @@ bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector<ShaderDiskCacheRaw>& ra
// Version is valid, load the shaders
while (file.Tell() < file_size) {
EntryKind kind{};
TransferableEntryKind kind{};
file.ReadBytes(&kind, sizeof(u32));
switch (kind) {
case EntryKind::Raw: {
case TransferableEntryKind::Raw: {
ShaderDiskCacheRaw entry{file};
transferable.insert({entry.GetUniqueIdentifier(), {}});
raws.push_back(std::move(entry));
break;
}
case EntryKind::Usage: {
case TransferableEntryKind::Usage: {
ShaderDiskCacheUsage usage{};
file.ReadBytes(&usage, sizeof(usage));
usages.push_back(std::move(usage));
@ -133,16 +138,19 @@ bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector<ShaderDiskCacheRaw>& ra
return true;
}
std::vector<ShaderDiskCachePrecompiledEntry> ShaderDiskCacheOpenGL::LoadPrecompiled() {
bool ShaderDiskCacheOpenGL::LoadPrecompiled(
std::map<u64, ShaderDiskCacheDecompiled>& decompiled,
std::map<ShaderDiskCacheUsage, ShaderDiskCacheDump>& dumps) {
if (!Settings::values.use_disk_shader_cache) {
return {};
return false;
}
FileUtil::IOFile file(GetPrecompiledPath(), "rb");
if (!file.IsOpen()) {
LOG_INFO(Render_OpenGL, "No precompiled shader cache found for game with title id={}",
GetTitleID());
return {};
return false;
}
const u64 file_size = file.GetSize();
@ -152,24 +160,102 @@ std::vector<ShaderDiskCachePrecompiledEntry> ShaderDiskCacheOpenGL::LoadPrecompi
LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of yuzu - removing");
file.Close();
InvalidatePrecompiled();
return {};
return false;
}
std::vector<ShaderDiskCachePrecompiledEntry> precompiled;
while (file.Tell() < file_size) {
ShaderDiskCachePrecompiledEntry entry;
file.ReadBytes(&entry.usage, sizeof(entry.usage));
PrecompiledEntryKind kind{};
file.ReadBytes(&kind, sizeof(u32));
file.ReadBytes(&entry.binary_format, sizeof(u32));
switch (kind) {
case PrecompiledEntryKind::Decompiled: {
ShaderDiskCacheDecompiled entry;
u64 unique_identifier{};
file.ReadBytes(&unique_identifier, sizeof(u64));
u32 code_size{};
file.ReadBytes(&code_size, sizeof(u32));
std::vector<u8> code(code_size);
file.ReadArray(code.data(), code.size());
entry.code = std::string(reinterpret_cast<char*>(code.data()), code_size);
u32 const_buffers_count{};
file.ReadBytes(&const_buffers_count, sizeof(u32));
for (u32 i = 0; i < const_buffers_count; ++i) {
u32 max_offset{}, index{};
u8 is_indirect{};
file.ReadBytes(&max_offset, sizeof(u32));
file.ReadBytes(&index, sizeof(u32));
file.ReadBytes(&is_indirect, sizeof(u8));
entry.entries.const_buffers.emplace_back(max_offset, is_indirect != 0, index);
}
u32 samplers_count{};
file.ReadBytes(&samplers_count, sizeof(u32));
for (u32 i = 0; i < samplers_count; ++i) {
u64 offset{}, index{};
u32 type{};
u8 is_array{}, is_shadow{};
file.ReadBytes(&offset, sizeof(u64));
file.ReadBytes(&index, sizeof(u64));
file.ReadBytes(&type, sizeof(u32));
file.ReadBytes(&is_array, sizeof(u8));
file.ReadBytes(&is_shadow, sizeof(u8));
entry.entries.samplers.emplace_back(
static_cast<std::size_t>(offset), static_cast<std::size_t>(index),
static_cast<Tegra::Shader::TextureType>(type), is_array != 0, is_shadow != 0);
}
u32 global_memory_count{};
file.ReadBytes(&global_memory_count, sizeof(u32));
for (u32 i = 0; i < global_memory_count; ++i) {
u32 cbuf_index{}, cbuf_offset{};
file.ReadBytes(&cbuf_index, sizeof(u32));
file.ReadBytes(&cbuf_offset, sizeof(u32));
entry.entries.global_memory_entries.emplace_back(cbuf_index, cbuf_offset);
}
for (auto& clip_distance : entry.entries.clip_distances) {
u8 clip_distance_raw{};
file.ReadBytes(&clip_distance_raw, sizeof(u8));
clip_distance = clip_distance_raw != 0;
}
u64 shader_length{};
file.ReadBytes(&shader_length, sizeof(u64));
entry.entries.shader_length = static_cast<std::size_t>(shader_length);
decompiled.insert({unique_identifier, std::move(entry)});
break;
}
case PrecompiledEntryKind::Dump: {
ShaderDiskCacheUsage usage;
file.ReadBytes(&usage, sizeof(usage));
ShaderDiskCacheDump dump;
file.ReadBytes(&dump.binary_format, sizeof(u32));
u32 binary_length{};
file.ReadBytes(&binary_length, sizeof(u32));
entry.binary.resize(binary_length);
file.ReadBytes(entry.binary.data(), entry.binary.size());
dump.binary.resize(binary_length);
file.ReadBytes(dump.binary.data(), dump.binary.size());
precompiled.push_back(entry);
dumps.insert({usage, dump});
break;
}
return precompiled;
default:
LOG_ERROR(Render_OpenGL, "Unknown precompiled shader cache entry kind={} - removing",
static_cast<u32>(kind));
InvalidatePrecompiled();
dumps.clear();
decompiled.clear();
return false;
}
}
return true;
}
void ShaderDiskCacheOpenGL::InvalidateTransferable() const {
@ -196,7 +282,7 @@ void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) {
if (!file.IsOpen()) {
return;
}
file.WriteObject(EntryKind::Raw);
file.WriteObject(TransferableEntryKind::Raw);
entry.Save(file);
transferable.insert({id, {}});
@ -220,11 +306,12 @@ void ShaderDiskCacheOpenGL::SaveUsage(const ShaderDiskCacheUsage& usage) {
if (!file.IsOpen()) {
return;
}
file.WriteObject(EntryKind::Usage);
file.WriteObject(TransferableEntryKind::Usage);
file.WriteObject(usage);
}
void ShaderDiskCacheOpenGL::SavePrecompiled(const ShaderDiskCacheUsage& usage, GLuint program) {
void ShaderDiskCacheOpenGL::SaveDecompiled(u64 unique_identifier, const std::string& code,
const GLShader::ShaderEntries& entries) {
if (!Settings::values.use_disk_shader_cache) {
return;
}
@ -234,6 +321,54 @@ void ShaderDiskCacheOpenGL::SavePrecompiled(const ShaderDiskCacheUsage& usage, G
return;
}
file.WriteObject(static_cast<u32>(PrecompiledEntryKind::Decompiled));
file.WriteObject(unique_identifier);
file.WriteObject(static_cast<u32>(code.size()));
file.WriteArray(code.data(), code.size());
file.WriteObject(static_cast<u32>(entries.const_buffers.size()));
for (const auto& cbuf : entries.const_buffers) {
file.WriteObject(static_cast<u32>(cbuf.GetMaxOffset()));
file.WriteObject(static_cast<u32>(cbuf.GetIndex()));
file.WriteObject(static_cast<u8>(cbuf.IsIndirect() ? 1 : 0));
}
file.WriteObject(static_cast<u32>(entries.samplers.size()));
for (const auto& sampler : entries.samplers) {
file.WriteObject(static_cast<u64>(sampler.GetOffset()));
file.WriteObject(static_cast<u64>(sampler.GetIndex()));
file.WriteObject(static_cast<u32>(sampler.GetType()));
file.WriteObject(static_cast<u8>(sampler.IsArray() ? 1 : 0));
file.WriteObject(static_cast<u8>(sampler.IsShadow() ? 1 : 0));
}
file.WriteObject(static_cast<u32>(entries.global_memory_entries.size()));
for (const auto& gmem : entries.global_memory_entries) {
file.WriteObject(static_cast<u32>(gmem.GetCbufIndex()));
file.WriteObject(static_cast<u32>(gmem.GetCbufOffset()));
}
for (const bool clip_distance : entries.clip_distances) {
file.WriteObject(static_cast<u8>(clip_distance ? 1 : 0));
}
file.WriteObject(static_cast<u64>(entries.shader_length));
}
void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint program) {
if (!Settings::values.use_disk_shader_cache) {
return;
}
FileUtil::IOFile file = AppendPrecompiledFile();
if (!file.IsOpen()) {
return;
}
file.WriteObject(static_cast<u32>(PrecompiledEntryKind::Dump));
file.WriteObject(usage);
GLint binary_length{};

21
src/video_core/renderer_opengl/gl_shader_disk_cache.h
View File

@ -130,14 +130,16 @@ public:
}
};
struct ShaderDiskCachePrecompiledEntry {
ShaderDiskCacheUsage usage;
GLenum binary_format;
std::vector<u8> binary;
struct ShaderDiskCacheDecompiled {
std::string code;
GLShader::ShaderEntries entries;
};
struct ShaderDiskCacheDump {
GLenum binary_format;
std::vector<u8> binary;
};
class ShaderDiskCacheOpenGL {
public:
/// Loads transferable cache. If file has a old version, it deletes it. Returns true on success.
@ -145,7 +147,8 @@ public:
std::vector<ShaderDiskCacheUsage>& usages);
/// Loads current game's precompiled cache. Invalidates if emulator's version has changed.
std::vector<ShaderDiskCachePrecompiledEntry> LoadPrecompiled();
bool LoadPrecompiled(std::map<u64, ShaderDiskCacheDecompiled>& decompiled,
std::map<ShaderDiskCacheUsage, ShaderDiskCacheDump>& dumps);
/// Removes the transferable (and precompiled) cache file.
void InvalidateTransferable() const;
@ -159,8 +162,12 @@ public:
/// Saves shader usage to the transferable file. Does not check for collisions.
void SaveUsage(const ShaderDiskCacheUsage& usage);
/// Saves a precompiled shader entry. Does not check for collisions.
void SavePrecompiled(const ShaderDiskCacheUsage& usage, GLuint program);
/// Saves a decompiled entry to the precompiled file. Does not check for collisions.
void SaveDecompiled(u64 unique_identifier, const std::string& code,
const GLShader::ShaderEntries& entries);
/// Saves a dump entry to the precompiled file. Does not check for collisions.
void SaveDump(const ShaderDiskCacheUsage& usage, GLuint program);
private:
/// Opens current game's transferable file and write it's header if it doesn't exist

2
src/video_core/renderer_opengl/gl_shader_gen.h
View File

@ -26,8 +26,6 @@ struct ShaderSetup {
ProgramCode code;
ProgramCode code_b; // Used for dual vertex shaders
u64 unique_identifier;
std::size_t real_size;
std::size_t real_size_b;
} program;
/// Used in scenarios where we have a dual vertex shaders

9
src/video_core/shader/shader_ir.h
View File

@ -236,6 +236,11 @@ private:
class ConstBuffer {
public:
explicit ConstBuffer(u32 max_offset, bool is_indirect)
: max_offset{max_offset}, is_indirect{is_indirect} {}
ConstBuffer() = default;
void MarkAsUsed(u64 offset) {
max_offset = std::max(max_offset, static_cast<u32>(offset));
}
@ -252,6 +257,10 @@ public:
return max_offset + sizeof(float);
}
u32 GetMaxOffset() const {
return max_offset;
}
private:
u32 max_offset{};
bool is_indirect{};