Merge pull request #2021 from ReinUsesLisp/disk-cache

gl_shader_cache: Disk based shader cache

CMakeLists.txt

@@ -419,19 +419,6 @@ function(create_target_directory_groups target_name)
     endforeach()
 endfunction()
 
-# Gets a UTC timstamp and sets the provided variable to it
-function(get_timestamp _var)
-    string(TIMESTAMP timestamp UTC)
-    set(${_var} "${timestamp}" PARENT_SCOPE)
-endfunction()
-
-# generate git/build information
-include(GetGitRevisionDescription)
-get_git_head_revision(GIT_REF_SPEC GIT_REV)
-git_describe(GIT_DESC --always --long --dirty)
-git_branch_name(GIT_BRANCH)
-get_timestamp(BUILD_DATE)
-
 enable_testing()
 add_subdirectory(externals)
 add_subdirectory(src)

CMakeModules/GenerateSCMRev.cmake

@@ -0,0 +1,94 @@
+# Gets a UTC timstamp and sets the provided variable to it
+function(get_timestamp _var)
+    string(TIMESTAMP timestamp UTC)
+    set(${_var} "${timestamp}" PARENT_SCOPE)
+endfunction()
+
+list(APPEND CMAKE_MODULE_PATH "${SRC_DIR}/externals/cmake-modules")
+# generate git/build information
+include(GetGitRevisionDescription)
+get_git_head_revision(GIT_REF_SPEC GIT_REV)
+git_describe(GIT_DESC --always --long --dirty)
+git_branch_name(GIT_BRANCH)
+get_timestamp(BUILD_DATE)
+
+# Generate cpp with Git revision from template
+# Also if this is a CI build, add the build name (ie: Nightly, Canary) to the scm_rev file as well
+set(REPO_NAME "")
+set(BUILD_VERSION "0")
+if (BUILD_REPOSITORY)
+  # regex capture the string nightly or canary into CMAKE_MATCH_1
+  string(REGEX MATCH "yuzu-emu/yuzu-?(.*)" OUTVAR ${BUILD_REPOSITORY})
+  if (${CMAKE_MATCH_COUNT} GREATER 0)
+    # capitalize the first letter of each word in the repo name.
+    string(REPLACE "-" ";" REPO_NAME_LIST ${CMAKE_MATCH_1})
+    foreach(WORD ${REPO_NAME_LIST})
+      string(SUBSTRING ${WORD} 0 1 FIRST_LETTER)
+      string(SUBSTRING ${WORD} 1 -1 REMAINDER)
+      string(TOUPPER ${FIRST_LETTER} FIRST_LETTER)
+      set(REPO_NAME "${REPO_NAME}${FIRST_LETTER}${REMAINDER}")
+    endforeach()
+    if (BUILD_TAG)
+      string(REGEX MATCH "${CMAKE_MATCH_1}-([0-9]+)" OUTVAR ${BUILD_TAG})
+      if (${CMAKE_MATCH_COUNT} GREATER 0)
+        set(BUILD_VERSION ${CMAKE_MATCH_1})
+      endif()
+      if (BUILD_VERSION)
+        # This leaves a trailing space on the last word, but we actually want that
+        # because of how it's styled in the title bar.
+        set(BUILD_FULLNAME "${REPO_NAME} ${BUILD_VERSION} ")
+      else()
+        set(BUILD_FULLNAME "")
+      endif()
+    endif()
+  endif()
+endif()
+
+# The variable SRC_DIR must be passed into the script (since it uses the current build directory for all values of CMAKE_*_DIR)
+set(VIDEO_CORE "${SRC_DIR}/src/video_core")
+set(HASH_FILES
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.cpp"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.h"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.cpp"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.h"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.cpp"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.h"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.cpp"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.h"
+    "${VIDEO_CORE}/shader/decode/arithmetic.cpp"
+    "${VIDEO_CORE}/shader/decode/arithmetic_half.cpp"
+    "${VIDEO_CORE}/shader/decode/arithmetic_half_immediate.cpp"
+    "${VIDEO_CORE}/shader/decode/arithmetic_immediate.cpp"
+    "${VIDEO_CORE}/shader/decode/arithmetic_integer.cpp"
+    "${VIDEO_CORE}/shader/decode/arithmetic_integer_immediate.cpp"
+    "${VIDEO_CORE}/shader/decode/bfe.cpp"
+    "${VIDEO_CORE}/shader/decode/bfi.cpp"
+    "${VIDEO_CORE}/shader/decode/conversion.cpp"
+    "${VIDEO_CORE}/shader/decode/ffma.cpp"
+    "${VIDEO_CORE}/shader/decode/float_set.cpp"
+    "${VIDEO_CORE}/shader/decode/float_set_predicate.cpp"
+    "${VIDEO_CORE}/shader/decode/half_set.cpp"
+    "${VIDEO_CORE}/shader/decode/half_set_predicate.cpp"
+    "${VIDEO_CORE}/shader/decode/hfma2.cpp"
+    "${VIDEO_CORE}/shader/decode/integer_set.cpp"
+    "${VIDEO_CORE}/shader/decode/integer_set_predicate.cpp"
+    "${VIDEO_CORE}/shader/decode/memory.cpp"
+    "${VIDEO_CORE}/shader/decode/other.cpp"
+    "${VIDEO_CORE}/shader/decode/predicate_set_predicate.cpp"
+    "${VIDEO_CORE}/shader/decode/predicate_set_register.cpp"
+    "${VIDEO_CORE}/shader/decode/register_set_predicate.cpp"
+    "${VIDEO_CORE}/shader/decode/shift.cpp"
+    "${VIDEO_CORE}/shader/decode/video.cpp"
+    "${VIDEO_CORE}/shader/decode/xmad.cpp"
+    "${VIDEO_CORE}/shader/decode.cpp"
+    "${VIDEO_CORE}/shader/shader_ir.cpp"
+    "${VIDEO_CORE}/shader/shader_ir.h"
+    "${VIDEO_CORE}/shader/track.cpp"
+)
+set(COMBINED "")
+foreach (F IN LISTS HASH_FILES)
+    file(READ ${F} TMP)
+    set(COMBINED "${COMBINED}${TMP}")
+endforeach()
+string(MD5 SHADER_CACHE_VERSION "${COMBINED}")
+configure_file("${SRC_DIR}/src/common/scm_rev.cpp.in" "scm_rev.cpp" @ONLY)

src/common/CMakeLists.txt

@@ -1,42 +1,69 @@
-# Generate cpp with Git revision from template
+# Add a custom command to generate a new shader_cache_version hash when any of the following files change
-# Also if this is a CI build, add the build name (ie: Nightly, Canary) to the scm_rev file as well
+# NOTE: This is an approximation of what files affect shader generation, its possible something else
-set(REPO_NAME "")
+# could affect the result, but much more unlikely than the following files. Keeping a list of files
-set(BUILD_VERSION "0")
+# like this allows for much better caching since it doesn't force the user to recompile binary shaders every update
-if ($ENV{CI})
+set(VIDEO_CORE "${CMAKE_SOURCE_DIR}/src/video_core")
-  if ($ENV{TRAVIS})
+if (DEFINED ENV{CI})
+  if (DEFINED ENV{TRAVIS})
     set(BUILD_REPOSITORY $ENV{TRAVIS_REPO_SLUG})
     set(BUILD_TAG $ENV{TRAVIS_TAG})
-  elseif($ENV{APPVEYOR})
+  elseif(DEFINED ENV{APPVEYOR})
     set(BUILD_REPOSITORY $ENV{APPVEYOR_REPO_NAME})
     set(BUILD_TAG $ENV{APPVEYOR_REPO_TAG_NAME})
   endif()
-  # regex capture the string nightly or canary into CMAKE_MATCH_1
-  string(REGEX MATCH "yuzu-emu/yuzu-?(.*)" OUTVAR ${BUILD_REPOSITORY})
-  if (${CMAKE_MATCH_COUNT} GREATER 0)
-    # capitalize the first letter of each word in the repo name.
-    string(REPLACE "-" ";" REPO_NAME_LIST ${CMAKE_MATCH_1})
-    foreach(WORD ${REPO_NAME_LIST})
-      string(SUBSTRING ${WORD} 0 1 FIRST_LETTER)
-      string(SUBSTRING ${WORD} 1 -1 REMAINDER)
-      string(TOUPPER ${FIRST_LETTER} FIRST_LETTER)
-      set(REPO_NAME "${REPO_NAME}${FIRST_LETTER}${REMAINDER}")
-    endforeach()
-    if (BUILD_TAG)
-      string(REGEX MATCH "${CMAKE_MATCH_1}-([0-9]+)" OUTVAR ${BUILD_TAG})
-      if (${CMAKE_MATCH_COUNT} GREATER 0)
-        set(BUILD_VERSION ${CMAKE_MATCH_1})
-      endif()
-      if (BUILD_VERSION)
-        # This leaves a trailing space on the last word, but we actually want that
-        # because of how it's styled in the title bar.
-        set(BUILD_FULLNAME "${REPO_NAME} ${BUILD_VERSION} ")
-      else()
-        set(BUILD_FULLNAME "")
-      endif()
-    endif()
-  endif()
 endif()
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.cpp.in" "${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.cpp" @ONLY)
+add_custom_command(OUTPUT scm_rev.cpp
+    COMMAND ${CMAKE_COMMAND}
+      -DSRC_DIR="${CMAKE_SOURCE_DIR}"
+      -DBUILD_REPOSITORY="${BUILD_REPOSITORY}"
+      -DBUILD_TAG="${BUILD_TAG}"
+      -P "${CMAKE_SOURCE_DIR}/CMakeModules/GenerateSCMRev.cmake"
+    DEPENDS
+      # WARNING! It was too much work to try and make a common location for this list,
+      # so if you need to change it, please update CMakeModules/GenerateSCMRev.cmake as well
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.cpp"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.h"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.cpp"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.h"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.cpp"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.h"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.cpp"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.h"
+      "${VIDEO_CORE}/shader/decode/arithmetic.cpp"
+      "${VIDEO_CORE}/shader/decode/arithmetic_half.cpp"
+      "${VIDEO_CORE}/shader/decode/arithmetic_half_immediate.cpp"
+      "${VIDEO_CORE}/shader/decode/arithmetic_immediate.cpp"
+      "${VIDEO_CORE}/shader/decode/arithmetic_integer.cpp"
+      "${VIDEO_CORE}/shader/decode/arithmetic_integer_immediate.cpp"
+      "${VIDEO_CORE}/shader/decode/bfe.cpp"
+      "${VIDEO_CORE}/shader/decode/bfi.cpp"
+      "${VIDEO_CORE}/shader/decode/conversion.cpp"
+      "${VIDEO_CORE}/shader/decode/ffma.cpp"
+      "${VIDEO_CORE}/shader/decode/float_set.cpp"
+      "${VIDEO_CORE}/shader/decode/float_set_predicate.cpp"
+      "${VIDEO_CORE}/shader/decode/half_set.cpp"
+      "${VIDEO_CORE}/shader/decode/half_set_predicate.cpp"
+      "${VIDEO_CORE}/shader/decode/hfma2.cpp"
+      "${VIDEO_CORE}/shader/decode/integer_set.cpp"
+      "${VIDEO_CORE}/shader/decode/integer_set_predicate.cpp"
+      "${VIDEO_CORE}/shader/decode/memory.cpp"
+      "${VIDEO_CORE}/shader/decode/other.cpp"
+      "${VIDEO_CORE}/shader/decode/predicate_set_predicate.cpp"
+      "${VIDEO_CORE}/shader/decode/predicate_set_register.cpp"
+      "${VIDEO_CORE}/shader/decode/register_set_predicate.cpp"
+      "${VIDEO_CORE}/shader/decode/shift.cpp"
+      "${VIDEO_CORE}/shader/decode/video.cpp"
+      "${VIDEO_CORE}/shader/decode/xmad.cpp"
+      "${VIDEO_CORE}/shader/decode.cpp"
+      "${VIDEO_CORE}/shader/shader_ir.cpp"
+      "${VIDEO_CORE}/shader/shader_ir.h"
+      "${VIDEO_CORE}/shader/track.cpp"
+      # and also check that the scm_rev files haven't changed
+      "${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.cpp.in"
+      "${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.h"
+      # technically we should regenerate if the git version changed, but its not worth the effort imo
+      "${CMAKE_SOURCE_DIR}/CMakeModules/GenerateSCMRev.cmake"
+)
 
 add_library(common STATIC
     alignment.h

src/common/common_paths.h

@@ -35,6 +35,7 @@
 #define KEYS_DIR "keys"
 #define LOAD_DIR "load"
 #define DUMP_DIR "dump"
+#define SHADER_DIR "shader"
 #define LOG_DIR "log"
 
 // Filenames

src/common/file_util.cpp

@@ -710,6 +710,7 @@ const std::string& GetUserPath(UserPath path, const std::string& new_path) {
         paths.emplace(UserPath::NANDDir, user_path + NAND_DIR DIR_SEP);
         paths.emplace(UserPath::LoadDir, user_path + LOAD_DIR DIR_SEP);
         paths.emplace(UserPath::DumpDir, user_path + DUMP_DIR DIR_SEP);
+        paths.emplace(UserPath::ShaderDir, user_path + SHADER_DIR DIR_SEP);
         paths.emplace(UserPath::SysDataDir, user_path + SYSDATA_DIR DIR_SEP);
         paths.emplace(UserPath::KeysDir, user_path + KEYS_DIR DIR_SEP);
         // TODO: Put the logs in a better location for each OS

src/common/file_util.h

@@ -31,6 +31,7 @@ enum class UserPath {
     SDMCDir,
     LoadDir,
     DumpDir,
+    ShaderDir,
     SysDataDir,
     UserDir,
 };

src/common/scm_rev.cpp.in

@@ -11,6 +11,7 @@
 #define BUILD_DATE   "@BUILD_DATE@"
 #define BUILD_FULLNAME "@BUILD_FULLNAME@"
 #define BUILD_VERSION "@BUILD_VERSION@"
+#define SHADER_CACHE_VERSION "@SHADER_CACHE_VERSION@"
 
 namespace Common {
 
@@ -21,6 +22,7 @@ const char g_build_name[]   = BUILD_NAME;
 const char g_build_date[]   = BUILD_DATE;
 const char g_build_fullname[] = BUILD_FULLNAME;
 const char g_build_version[]  = BUILD_VERSION;
+const char g_shader_cache_version[] = SHADER_CACHE_VERSION;
 
 } // namespace
 

src/common/scm_rev.h

@@ -13,5 +13,6 @@ extern const char g_build_name[];
 extern const char g_build_date[];
 extern const char g_build_fullname[];
 extern const char g_build_version[];
+extern const char g_shader_cache_version[];
 
 } // namespace Common

src/core/core.cpp

@@ -123,7 +123,7 @@ struct System::Impl {
         Service::Init(service_manager, *virtual_filesystem);
         GDBStub::Init();
 
-        renderer = VideoCore::CreateRenderer(emu_window);
+        renderer = VideoCore::CreateRenderer(emu_window, system);
         if (!renderer->Init()) {
             return ResultStatus::ErrorVideoCore;
         }
@@ -175,6 +175,7 @@ struct System::Impl {
             return static_cast<ResultStatus>(static_cast<u32>(ResultStatus::ErrorLoader) +
                                              static_cast<u32>(load_result));
         }
+
         status = ResultStatus::Success;
         return status;
     }

src/core/settings.h

@@ -391,6 +391,7 @@ struct Values {
     float resolution_factor;
     bool use_frame_limit;
     u16 frame_limit;
+    bool use_disk_shader_cache;
     bool use_accurate_gpu_emulation;
 
     float bg_red;

src/core/telemetry_session.cpp

@@ -158,6 +158,8 @@ TelemetrySession::TelemetrySession() {
     AddField(Telemetry::FieldType::UserConfig, "Renderer_UseFrameLimit",
              Settings::values.use_frame_limit);
     AddField(Telemetry::FieldType::UserConfig, "Renderer_FrameLimit", Settings::values.frame_limit);
+    AddField(Telemetry::FieldType::UserConfig, "Renderer_UseDiskShaderCache",
+             Settings::values.use_disk_shader_cache);
     AddField(Telemetry::FieldType::UserConfig, "Renderer_UseAccurateGpuEmulation",
              Settings::values.use_accurate_gpu_emulation);
     AddField(Telemetry::FieldType::UserConfig, "System_UseDockedMode",

src/video_core/CMakeLists.txt

@@ -44,6 +44,8 @@ add_library(video_core STATIC
     renderer_opengl/gl_shader_cache.h
     renderer_opengl/gl_shader_decompiler.cpp
     renderer_opengl/gl_shader_decompiler.h
+    renderer_opengl/gl_shader_disk_cache.cpp
+    renderer_opengl/gl_shader_disk_cache.h
     renderer_opengl/gl_shader_gen.cpp
     renderer_opengl/gl_shader_gen.h
     renderer_opengl/gl_shader_manager.cpp
@@ -102,4 +104,4 @@ add_library(video_core STATIC
 create_target_directory_groups(video_core)
 
 target_link_libraries(video_core PUBLIC common core)
-target_link_libraries(video_core PRIVATE glad)
+target_link_libraries(video_core PRIVATE glad lz4_static)

src/video_core/rasterizer_interface.h

@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <functional>
 #include "common/common_types.h"
 #include "video_core/engines/fermi_2d.h"
@@ -61,5 +62,9 @@ public:
 
     /// Increase/decrease the number of object in pages touching the specified region
     virtual void UpdatePagesCachedCount(Tegra::GPUVAddr addr, u64 size, int delta) {}
+
+    /// Initialize disk cached resources for the game being emulated
+    virtual void LoadDiskResources(const std::atomic_bool& stop_loading = false,
+                                   const DiskResourceLoadCallback& callback = {}) {}
 };
 } // namespace VideoCore

src/video_core/renderer_opengl/gl_rasterizer.cpp

@@ -22,6 +22,7 @@
 #include "core/settings.h"
 #include "video_core/engines/maxwell_3d.h"
 #include "video_core/renderer_opengl/gl_rasterizer.h"
+#include "video_core/renderer_opengl/gl_shader_cache.h"
 #include "video_core/renderer_opengl/gl_shader_gen.h"
 #include "video_core/renderer_opengl/maxwell_to_gl.h"
 #include "video_core/renderer_opengl/renderer_opengl.h"
@@ -99,8 +100,9 @@ struct FramebufferCacheKey {
     }
 };
 
-RasterizerOpenGL::RasterizerOpenGL(Core::Frontend::EmuWindow& window, ScreenInfo& info)
+RasterizerOpenGL::RasterizerOpenGL(Core::Frontend::EmuWindow& window, Core::System& system,
-    : res_cache{*this}, shader_cache{*this}, emu_window{window}, screen_info{info},
+                                   ScreenInfo& info)
+    : res_cache{*this}, shader_cache{*this, system}, emu_window{window}, screen_info{info},
       buffer_cache(*this, STREAM_BUFFER_SIZE), global_cache{*this} {
     // Create sampler objects
     for (std::size_t i = 0; i < texture_samplers.size(); ++i) {
@@ -447,7 +449,7 @@ static constexpr auto RangeFromInterval(Map& map, const Interval& interval) {
     return boost::make_iterator_range(map.equal_range(interval));
 }
 
-void RasterizerOpenGL::UpdatePagesCachedCount(VAddr addr, u64 size, int delta) {
+void RasterizerOpenGL::UpdatePagesCachedCount(Tegra::GPUVAddr addr, u64 size, int delta) {
     const u64 page_start{addr >> Memory::PAGE_BITS};
     const u64 page_end{(addr + size + Memory::PAGE_SIZE - 1) >> Memory::PAGE_BITS};
 
@@ -477,6 +479,11 @@ void RasterizerOpenGL::UpdatePagesCachedCount(VAddr addr, u64 size, int delta) {
         cached_pages.add({pages_interval, delta});
 }
 
+void RasterizerOpenGL::LoadDiskResources(const std::atomic_bool& stop_loading,
+                                         const VideoCore::DiskResourceLoadCallback& callback) {
+    shader_cache.LoadDiskCache(stop_loading, callback);
+}
+
 std::pair<bool, bool> RasterizerOpenGL::ConfigureFramebuffers(
     OpenGLState& current_state, bool using_color_fb, bool using_depth_fb, bool preserve_contents,
     std::optional<std::size_t> single_color_target) {
@@ -1004,22 +1011,20 @@ void RasterizerOpenGL::SetupTextures(Maxwell::ShaderStage stage, const Shader& s
 
     for (u32 bindpoint = 0; bindpoint < entries.size(); ++bindpoint) {
         const auto& entry = entries[bindpoint];
+        const auto texture = maxwell3d.GetStageTexture(stage, entry.GetOffset());
         const u32 current_bindpoint = base_bindings.sampler + bindpoint;
-        auto& unit = state.texture_units[current_bindpoint];
-
-        const auto texture = maxwell3d.GetStageTexture(entry.GetStage(), entry.GetOffset());
 
         texture_samplers[current_bindpoint].SyncWithConfig(texture.tsc);
 
         Surface surface = res_cache.GetTextureSurface(texture, entry);
         if (surface != nullptr) {
-            unit.texture =
+            state.texture_units[current_bindpoint].texture =
                 entry.IsArray() ? surface->TextureLayer().handle : surface->Texture().handle;
             surface->UpdateSwizzle(texture.tic.x_source, texture.tic.y_source, texture.tic.z_source,
                                    texture.tic.w_source);
         } else {
             // Can occur when texture addr is null or its memory is unmapped/invalid
-            unit.texture = 0;
+            state.texture_units[current_bindpoint].texture = 0;
         }
     }
 }

src/video_core/renderer_opengl/gl_rasterizer.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <array>
+#include <atomic>
 #include <cstddef>
 #include <map>
 #include <memory>
@@ -33,6 +34,10 @@
 #include "video_core/renderer_opengl/gl_state.h"
 #include "video_core/renderer_opengl/gl_stream_buffer.h"
 
+namespace Core {
+class System;
+}
+
 namespace Core::Frontend {
 class EmuWindow;
 }
@@ -45,7 +50,8 @@ struct FramebufferCacheKey;
 
 class RasterizerOpenGL : public VideoCore::RasterizerInterface {
 public:
-    explicit RasterizerOpenGL(Core::Frontend::EmuWindow& renderer, ScreenInfo& info);
+    explicit RasterizerOpenGL(Core::Frontend::EmuWindow& window, Core::System& system,
+                              ScreenInfo& info);
     ~RasterizerOpenGL() override;
 
     void DrawArrays() override;
@@ -60,6 +66,8 @@ public:
                            u32 pixel_stride) override;
     bool AccelerateDrawBatch(bool is_indexed) override;
     void UpdatePagesCachedCount(Tegra::GPUVAddr addr, u64 size, int delta) override;
+    void LoadDiskResources(const std::atomic_bool& stop_loading,
+                           const VideoCore::DiskResourceLoadCallback& callback) override;
 
     /// Maximum supported size that a constbuffer can have in bytes.
     static constexpr std::size_t MaxConstbufferSize = 0x10000;

src/video_core/renderer_opengl/gl_resource_manager.cpp

@@ -71,7 +71,8 @@ void OGLShader::Release() {
 }
 
 void OGLProgram::CreateFromSource(const char* vert_shader, const char* geo_shader,
-                                  const char* frag_shader, bool separable_program) {
+                                  const char* frag_shader, bool separable_program,
+                                  bool hint_retrievable) {
     OGLShader vert, geo, frag;
     if (vert_shader)
         vert.Create(vert_shader, GL_VERTEX_SHADER);
@@ -81,7 +82,7 @@ void OGLProgram::CreateFromSource(const char* vert_shader, const char* geo_shade
         frag.Create(frag_shader, GL_FRAGMENT_SHADER);
 
     MICROPROFILE_SCOPE(OpenGL_ResourceCreation);
-    Create(separable_program, vert.handle, geo.handle, frag.handle);
+    Create(separable_program, hint_retrievable, vert.handle, geo.handle, frag.handle);
 }
 
 void OGLProgram::Release() {

src/video_core/renderer_opengl/gl_resource_manager.h

@@ -101,15 +101,15 @@ public:
     }
 
     template <typename... T>
-    void Create(bool separable_program, T... shaders) {
+    void Create(bool separable_program, bool hint_retrievable, T... shaders) {
         if (handle != 0)
             return;
-        handle = GLShader::LoadProgram(separable_program, shaders...);
+        handle = GLShader::LoadProgram(separable_program, hint_retrievable, shaders...);
     }
 
     /// Creates a new internal OpenGL resource and stores the handle
     void CreateFromSource(const char* vert_shader, const char* geo_shader, const char* frag_shader,
-                          bool separable_program = false);
+                          bool separable_program = false, bool hint_retrievable = false);
 
     /// Deletes the internal OpenGL resource
     void Release();

src/video_core/renderer_opengl/gl_shader_cache.cpp

@@ -11,6 +11,7 @@
 #include "video_core/renderer_opengl/gl_rasterizer.h"
 #include "video_core/renderer_opengl/gl_shader_cache.h"
 #include "video_core/renderer_opengl/gl_shader_decompiler.h"
+#include "video_core/renderer_opengl/gl_shader_disk_cache.h"
 #include "video_core/renderer_opengl/gl_shader_manager.h"
 #include "video_core/renderer_opengl/utils.h"
 #include "video_core/shader/shader_ir.h"
@@ -19,8 +20,19 @@ namespace OpenGL {
 
 using VideoCommon::Shader::ProgramCode;
 
+// One UBO is always reserved for emulation values
+constexpr u32 RESERVED_UBOS = 1;
+
+struct UnspecializedShader {
+    std::string code;
+    GLShader::ShaderEntries entries;
+    Maxwell::ShaderProgram program_type;
+};
+
+namespace {
+
 /// Gets the address for the specified shader stage program
-static VAddr GetShaderAddress(Maxwell::ShaderProgram program) {
+VAddr GetShaderAddress(Maxwell::ShaderProgram program) {
     const auto& gpu = Core::System::GetInstance().GPU().Maxwell3D();
     const auto& shader_config = gpu.regs.shader_config[static_cast<std::size_t>(program)];
     const auto address = gpu.memory_manager.GpuToCpuAddress(gpu.regs.code_address.CodeAddress() +
@@ -30,7 +42,7 @@ static VAddr GetShaderAddress(Maxwell::ShaderProgram program) {
 }
 
 /// Gets the shader program code from memory for the specified address
-static ProgramCode GetShaderCode(VAddr addr) {
+ProgramCode GetShaderCode(VAddr addr) {
     ProgramCode program_code(VideoCommon::Shader::MAX_PROGRAM_LENGTH);
     Memory::ReadBlock(addr, program_code.data(), program_code.size() * sizeof(u64));
     return program_code;
@@ -51,38 +63,196 @@ constexpr GLenum GetShaderType(Maxwell::ShaderProgram program_type) {
     }
 }
 
-CachedShader::CachedShader(VAddr addr, Maxwell::ShaderProgram program_type)
+/// Gets if the current instruction offset is a scheduler instruction
-    : addr{addr}, program_type{program_type}, setup{GetShaderCode(addr)} {
+constexpr bool IsSchedInstruction(std::size_t offset, std::size_t main_offset) {
+    // Sched instructions appear once every 4 instructions.
+    constexpr std::size_t SchedPeriod = 4;
+    const std::size_t absolute_offset = offset - main_offset;
+    return (absolute_offset % SchedPeriod) == 0;
+}
 
-    GLShader::ProgramResult program_result;
+/// Describes primitive behavior on geometry shaders
+constexpr std::tuple<const char*, const char*, u32> GetPrimitiveDescription(GLenum primitive_mode) {
+    switch (primitive_mode) {
+    case GL_POINTS:
+        return {"points", "Points", 1};
+    case GL_LINES:
+    case GL_LINE_STRIP:
+        return {"lines", "Lines", 2};
+    case GL_LINES_ADJACENCY:
+    case GL_LINE_STRIP_ADJACENCY:
+        return {"lines_adjacency", "LinesAdj", 4};
+    case GL_TRIANGLES:
+    case GL_TRIANGLE_STRIP:
+    case GL_TRIANGLE_FAN:
+        return {"triangles", "Triangles", 3};
+    case GL_TRIANGLES_ADJACENCY:
+    case GL_TRIANGLE_STRIP_ADJACENCY:
+        return {"triangles_adjacency", "TrianglesAdj", 6};
+    default:
+        return {"points", "Invalid", 1};
+    }
+}
 
-    switch (program_type) {
+/// Calculates the size of a program stream
-    case Maxwell::ShaderProgram::VertexA:
+std::size_t CalculateProgramSize(const GLShader::ProgramCode& program) {
+    constexpr std::size_t start_offset = 10;
+    std::size_t offset = start_offset;
+    std::size_t size = start_offset * sizeof(u64);
+    while (offset < program.size()) {
+        const u64 instruction = program[offset];
+        if (!IsSchedInstruction(offset, start_offset)) {
+            if (instruction == 0 || (instruction >> 52) == 0x50b) {
+                // End on Maxwell's "nop" instruction
+                break;
+            }
+        }
+        size += sizeof(u64);
+        offset++;
+    }
+    // The last instruction is included in the program size
+    return std::min(size + sizeof(u64), program.size() * sizeof(u64));
+}
+
+/// Hashes one (or two) program streams
+u64 GetUniqueIdentifier(Maxwell::ShaderProgram program_type, const ProgramCode& code,
+                        const ProgramCode& code_b) {
+    u64 unique_identifier =
+        Common::CityHash64(reinterpret_cast<const char*>(code.data()), CalculateProgramSize(code));
+    if (program_type != Maxwell::ShaderProgram::VertexA) {
+        return unique_identifier;
+    }
+    // VertexA programs include two programs
+
+    std::size_t seed = 0;
+    boost::hash_combine(seed, unique_identifier);
+
+    const u64 identifier_b = Common::CityHash64(reinterpret_cast<const char*>(code_b.data()),
+                                                CalculateProgramSize(code_b));
+    boost::hash_combine(seed, identifier_b);
+    return static_cast<u64>(seed);
+}
+
+/// Creates an unspecialized program from code streams
+GLShader::ProgramResult CreateProgram(Maxwell::ShaderProgram program_type, ProgramCode program_code,
+                                      ProgramCode program_code_b) {
+    GLShader::ShaderSetup setup(program_code);
+    if (program_type == Maxwell::ShaderProgram::VertexA) {
         // VertexB is always enabled, so when VertexA is enabled, we have two vertex shaders.
         // Conventional HW does not support this, so we combine VertexA and VertexB into one
         // stage here.
-        setup.SetProgramB(GetShaderCode(GetShaderAddress(Maxwell::ShaderProgram::VertexB)));
+        setup.SetProgramB(program_code_b);
+    }
+    setup.program.unique_identifier =
+        GetUniqueIdentifier(program_type, program_code, program_code_b);
+
+    switch (program_type) {
+    case Maxwell::ShaderProgram::VertexA:
     case Maxwell::ShaderProgram::VertexB:
-        CalculateProperties();
+        return GLShader::GenerateVertexShader(setup);
-        program_result = GLShader::GenerateVertexShader(setup);
-        break;
     case Maxwell::ShaderProgram::Geometry:
-        CalculateProperties();
+        return GLShader::GenerateGeometryShader(setup);
-        program_result = GLShader::GenerateGeometryShader(setup);
-        break;
     case Maxwell::ShaderProgram::Fragment:
-        CalculateProperties();
+        return GLShader::GenerateFragmentShader(setup);
-        program_result = GLShader::GenerateFragmentShader(setup);
-        break;
     default:
         LOG_CRITICAL(HW_GPU, "Unimplemented program_type={}", static_cast<u32>(program_type));
         UNREACHABLE();
+        return {};
+    }
+}
+
+CachedProgram SpecializeShader(const std::string& code, const GLShader::ShaderEntries& entries,
+                               Maxwell::ShaderProgram program_type, BaseBindings base_bindings,
+                               GLenum primitive_mode, bool hint_retrievable = false) {
+    std::string source = "#version 430 core\n";
+    source += fmt::format("#define EMULATION_UBO_BINDING {}\n", base_bindings.cbuf++);
+
+    for (const auto& cbuf : entries.const_buffers) {
+        source +=
+            fmt::format("#define CBUF_BINDING_{} {}\n", cbuf.GetIndex(), base_bindings.cbuf++);
+    }
+    for (const auto& gmem : entries.global_memory_entries) {
+        source += fmt::format("#define GMEM_BINDING_{}_{} {}\n", gmem.GetCbufIndex(),
+                              gmem.GetCbufOffset(), base_bindings.gmem++);
+    }
+    for (const auto& sampler : entries.samplers) {
+        source += fmt::format("#define SAMPLER_BINDING_{} {}\n", sampler.GetIndex(),
+                              base_bindings.sampler++);
+    }
+
+    if (program_type == Maxwell::ShaderProgram::Geometry) {
+        const auto [glsl_topology, debug_name, max_vertices] =
+            GetPrimitiveDescription(primitive_mode);
+
+        source += "layout (" + std::string(glsl_topology) + ") in;\n";
+        source += "#define MAX_VERTEX_INPUT " + std::to_string(max_vertices) + '\n';
+    }
+
+    source += code;
+
+    OGLShader shader;
+    shader.Create(source.c_str(), GetShaderType(program_type));
+
+    auto program = std::make_shared<OGLProgram>();
+    program->Create(true, hint_retrievable, shader.handle);
+    return program;
+}
+
+std::set<GLenum> GetSupportedFormats() {
+    std::set<GLenum> supported_formats;
+
+    GLint num_formats{};
+    glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &num_formats);
+
+    std::vector<GLint> formats(num_formats);
+    glGetIntegerv(GL_PROGRAM_BINARY_FORMATS, formats.data());
+
+    for (const GLint format : formats)
+        supported_formats.insert(static_cast<GLenum>(format));
+    return supported_formats;
+}
+
+} // namespace
+
+CachedShader::CachedShader(VAddr addr, u64 unique_identifier, Maxwell::ShaderProgram program_type,
+                           ShaderDiskCacheOpenGL& disk_cache,
+                           const PrecompiledPrograms& precompiled_programs,
+                           ProgramCode&& program_code, ProgramCode&& program_code_b)
+    : addr{addr}, unique_identifier{unique_identifier}, program_type{program_type},
+      disk_cache{disk_cache}, precompiled_programs{precompiled_programs} {
+
+    const std::size_t code_size = CalculateProgramSize(program_code);
+    const std::size_t code_size_b =
+        program_code_b.empty() ? 0 : CalculateProgramSize(program_code_b);
+
+    GLShader::ProgramResult program_result =
+        CreateProgram(program_type, program_code, program_code_b);
+    if (program_result.first.empty()) {
+        // TODO(Rodrigo): Unimplemented shader stages hit here, avoid using these for now
         return;
     }
 
     code = program_result.first;
     entries = program_result.second;
     shader_length = entries.shader_length;
+
+    const ShaderDiskCacheRaw raw(unique_identifier, program_type,
+                                 static_cast<u32>(code_size / sizeof(u64)),
+                                 static_cast<u32>(code_size_b / sizeof(u64)),
+                                 std::move(program_code), std::move(program_code_b));
+    disk_cache.SaveRaw(raw);
+}
+
+CachedShader::CachedShader(VAddr addr, u64 unique_identifier, Maxwell::ShaderProgram program_type,
+                           ShaderDiskCacheOpenGL& disk_cache,
+                           const PrecompiledPrograms& precompiled_programs,
+                           GLShader::ProgramResult result)
+    : addr{addr}, unique_identifier{unique_identifier}, program_type{program_type},
+      disk_cache{disk_cache}, precompiled_programs{precompiled_programs} {
+
+    code = std::move(result.first);
+    entries = result.second;
+    shader_length = entries.shader_length;
 }
 
 std::tuple<GLuint, BaseBindings> CachedShader::GetProgramHandle(GLenum primitive_mode,
@@ -94,136 +264,222 @@ std::tuple<GLuint, BaseBindings> CachedShader::GetProgramHandle(GLenum primitive
         const auto [entry, is_cache_miss] = programs.try_emplace(base_bindings);
         auto& program = entry->second;
         if (is_cache_miss) {
-            std::string source = AllocateBindings(base_bindings);
+            program = TryLoadProgram(primitive_mode, base_bindings);
-            source += code;
+            if (!program) {
-
+                program =
-            OGLShader shader;
+                    SpecializeShader(code, entries, program_type, base_bindings, primitive_mode);
-            shader.Create(source.c_str(), GetShaderType(program_type));
+                disk_cache.SaveUsage(GetUsage(primitive_mode, base_bindings));
-            program.Create(true, shader.handle);
-            LabelGLObject(GL_PROGRAM, program.handle, addr);
             }
 
-        handle = program.handle;
+            LabelGLObject(GL_PROGRAM, program->handle, addr);
         }
 
-    // Add const buffer and samplers offset reserved by this shader. One UBO binding is reserved for
+        handle = program->handle;
-    // emulation values
+    }
-    base_bindings.cbuf += static_cast<u32>(entries.const_buffers.size()) + 1;
+
+    base_bindings.cbuf += static_cast<u32>(entries.const_buffers.size()) + RESERVED_UBOS;
     base_bindings.gmem += static_cast<u32>(entries.global_memory_entries.size());
     base_bindings.sampler += static_cast<u32>(entries.samplers.size());
 
     return {handle, base_bindings};
 }
 
-std::string CachedShader::AllocateBindings(BaseBindings base_bindings) {
-    std::string code = "#version 430 core\n";
-    code += fmt::format("#define EMULATION_UBO_BINDING {}\n", base_bindings.cbuf++);
-
-    for (const auto& cbuf : entries.const_buffers) {
-        code += fmt::format("#define CBUF_BINDING_{} {}\n", cbuf.GetIndex(), base_bindings.cbuf++);
-    }
-
-    for (const auto& gmem : entries.global_memory_entries) {
-        code += fmt::format("#define GMEM_BINDING_{}_{} {}\n", gmem.GetCbufIndex(),
-                            gmem.GetCbufOffset(), base_bindings.gmem++);
-    }
-
-    for (const auto& sampler : entries.samplers) {
-        code += fmt::format("#define SAMPLER_BINDING_{} {}\n", sampler.GetIndex(),
-                            base_bindings.sampler++);
-    }
-
-    return code;
-}
-
 GLuint CachedShader::GetGeometryShader(GLenum primitive_mode, BaseBindings base_bindings) {
     const auto [entry, is_cache_miss] = geometry_programs.try_emplace(base_bindings);
     auto& programs = entry->second;
 
     switch (primitive_mode) {
     case GL_POINTS:
-        return LazyGeometryProgram(programs.points, base_bindings, "points", 1, "ShaderPoints");
+        return LazyGeometryProgram(programs.points, base_bindings, primitive_mode);
     case GL_LINES:
     case GL_LINE_STRIP:
-        return LazyGeometryProgram(programs.lines, base_bindings, "lines", 2, "ShaderLines");
+        return LazyGeometryProgram(programs.lines, base_bindings, primitive_mode);
     case GL_LINES_ADJACENCY:
     case GL_LINE_STRIP_ADJACENCY:
-        return LazyGeometryProgram(programs.lines_adjacency, base_bindings, "lines_adjacency", 4,
+        return LazyGeometryProgram(programs.lines_adjacency, base_bindings, primitive_mode);
-                                   "ShaderLinesAdjacency");
     case GL_TRIANGLES:
     case GL_TRIANGLE_STRIP:
     case GL_TRIANGLE_FAN:
-        return LazyGeometryProgram(programs.triangles, base_bindings, "triangles", 3,
+        return LazyGeometryProgram(programs.triangles, base_bindings, primitive_mode);
-                                   "ShaderTriangles");
     case GL_TRIANGLES_ADJACENCY:
     case GL_TRIANGLE_STRIP_ADJACENCY:
-        return LazyGeometryProgram(programs.triangles_adjacency, base_bindings,
+        return LazyGeometryProgram(programs.triangles_adjacency, base_bindings, primitive_mode);
-                                   "triangles_adjacency", 6, "ShaderTrianglesAdjacency");
     default:
         UNREACHABLE_MSG("Unknown primitive mode.");
-        return LazyGeometryProgram(programs.points, base_bindings, "points", 1, "ShaderPoints");
+        return LazyGeometryProgram(programs.points, base_bindings, primitive_mode);
     }
 }
 
-GLuint CachedShader::LazyGeometryProgram(OGLProgram& target_program, BaseBindings base_bindings,
+GLuint CachedShader::LazyGeometryProgram(CachedProgram& target_program, BaseBindings base_bindings,
-                                         const std::string& glsl_topology, u32 max_vertices,
+                                         GLenum primitive_mode) {
-                                         const std::string& debug_name) {
+    if (target_program) {
-    if (target_program.handle != 0) {
+        return target_program->handle;
-        return target_program.handle;
+    }
+    const auto [glsl_name, debug_name, vertices] = GetPrimitiveDescription(primitive_mode);
+    target_program = TryLoadProgram(primitive_mode, base_bindings);
+    if (!target_program) {
+        target_program =
+            SpecializeShader(code, entries, program_type, base_bindings, primitive_mode);
+        disk_cache.SaveUsage(GetUsage(primitive_mode, base_bindings));
     }
-    std::string source = AllocateBindings(base_bindings);
-    source += "layout (" + glsl_topology + ") in;\n";
-    source += "#define MAX_VERTEX_INPUT " + std::to_string(max_vertices) + '\n';
-    source += code;
 
-    OGLShader shader;
+    LabelGLObject(GL_PROGRAM, target_program->handle, addr, debug_name);
-    shader.Create(source.c_str(), GL_GEOMETRY_SHADER);
+
-    target_program.Create(true, shader.handle);
+    return target_program->handle;
-    LabelGLObject(GL_PROGRAM, target_program.handle, addr, debug_name);
-    return target_program.handle;
 };
 
-static bool IsSchedInstruction(std::size_t offset, std::size_t main_offset) {
+CachedProgram CachedShader::TryLoadProgram(GLenum primitive_mode,
-    // sched instructions appear once every 4 instructions.
+                                           BaseBindings base_bindings) const {
-    static constexpr std::size_t SchedPeriod = 4;
+    const auto found = precompiled_programs.find(GetUsage(primitive_mode, base_bindings));
-    const std::size_t absolute_offset = offset - main_offset;
+    if (found == precompiled_programs.end()) {
-    return (absolute_offset % SchedPeriod) == 0;
+        return {};
+    }
+    return found->second;
 }
 
-static std::size_t CalculateProgramSize(const GLShader::ProgramCode& program) {
+ShaderDiskCacheUsage CachedShader::GetUsage(GLenum primitive_mode,
-    constexpr std::size_t start_offset = 10;
+                                            BaseBindings base_bindings) const {
-    std::size_t offset = start_offset;
+    return {unique_identifier, base_bindings, primitive_mode};
-    std::size_t size = start_offset * sizeof(u64);
-    while (offset < program.size()) {
-        const u64 inst = program[offset];
-        if (!IsSchedInstruction(offset, start_offset)) {
-            if (inst == 0 || (inst >> 52) == 0x50b) {
-                break;
-            }
-        }
-        size += sizeof(inst);
-        offset++;
-    }
-    return size;
 }
 
-void CachedShader::CalculateProperties() {
+ShaderCacheOpenGL::ShaderCacheOpenGL(RasterizerOpenGL& rasterizer, Core::System& system)
-    setup.program.real_size = CalculateProgramSize(setup.program.code);
+    : RasterizerCache{rasterizer}, disk_cache{system} {}
-    setup.program.real_size_b = 0;
+
-    setup.program.unique_identifier = Common::CityHash64(
+void ShaderCacheOpenGL::LoadDiskCache(const std::atomic_bool& stop_loading,
-        reinterpret_cast<const char*>(setup.program.code.data()), setup.program.real_size);
+                                      const VideoCore::DiskResourceLoadCallback& callback) {
-    if (program_type == Maxwell::ShaderProgram::VertexA) {
+    const auto transferable = disk_cache.LoadTransferable();
-        std::size_t seed = 0;
+    if (!transferable) {
-        boost::hash_combine(seed, setup.program.unique_identifier);
+        return;
-        setup.program.real_size_b = CalculateProgramSize(setup.program.code_b);
+    }
-        const u64 identifier_b = Common::CityHash64(
+    const auto [raws, usages] = *transferable;
-            reinterpret_cast<const char*>(setup.program.code_b.data()), setup.program.real_size_b);
+
-        boost::hash_combine(seed, identifier_b);
+    auto [decompiled, dumps] = disk_cache.LoadPrecompiled();
-        setup.program.unique_identifier = static_cast<u64>(seed);
+
+    const auto supported_formats{GetSupportedFormats()};
+    const auto unspecialized{
+        GenerateUnspecializedShaders(stop_loading, callback, raws, decompiled)};
+    if (stop_loading)
+        return;
+
+    // Build shaders
+    if (callback)
+        callback(VideoCore::LoadCallbackStage::Build, 0, usages.size());
+    for (std::size_t i = 0; i < usages.size(); ++i) {
+        if (stop_loading)
+            return;
+
+        const auto& usage{usages[i]};
+        LOG_INFO(Render_OpenGL, "Building shader {:016x} ({} of {})", usage.unique_identifier,
+                 i + 1, usages.size());
+
+        const auto& unspec{unspecialized.at(usage.unique_identifier)};
+        const auto dump_it = dumps.find(usage);
+
+        CachedProgram shader;
+        if (dump_it != dumps.end()) {
+            // If the shader is dumped, attempt to load it with
+            shader = GeneratePrecompiledProgram(dump_it->second, supported_formats);
+            if (!shader) {
+                // Invalidate the precompiled cache if a shader dumped shader was rejected
+                disk_cache.InvalidatePrecompiled();
+                dumps.clear();
+            }
+        }
+        if (!shader) {
+            shader = SpecializeShader(unspec.code, unspec.entries, unspec.program_type,
+                                      usage.bindings, usage.primitive, true);
+        }
+        precompiled_programs.insert({usage, std::move(shader)});
+
+        if (callback)
+            callback(VideoCore::LoadCallbackStage::Build, i + 1, usages.size());
+    }
+
+    // TODO(Rodrigo): Do state tracking for transferable shaders and do a dummy draw before
+    // precompiling them
+
+    for (std::size_t i = 0; i < usages.size(); ++i) {
+        const auto& usage{usages[i]};
+        if (dumps.find(usage) == dumps.end()) {
+            const auto& program = precompiled_programs.at(usage);
+            disk_cache.SaveDump(usage, program->handle);
+        }
     }
 }
 
-ShaderCacheOpenGL::ShaderCacheOpenGL(RasterizerOpenGL& rasterizer) : RasterizerCache{rasterizer} {}
+CachedProgram ShaderCacheOpenGL::GeneratePrecompiledProgram(
+    const ShaderDiskCacheDump& dump, const std::set<GLenum>& supported_formats) {
+
+    if (supported_formats.find(dump.binary_format) == supported_formats.end()) {
+        LOG_INFO(Render_OpenGL, "Precompiled cache entry with unsupported format - removing");
+        return {};
+    }
+
+    CachedProgram shader = std::make_shared<OGLProgram>();
+    shader->handle = glCreateProgram();
+    glProgramParameteri(shader->handle, GL_PROGRAM_SEPARABLE, GL_TRUE);
+    glProgramBinary(shader->handle, dump.binary_format, dump.binary.data(),
+                    static_cast<GLsizei>(dump.binary.size()));
+
+    GLint link_status{};
+    glGetProgramiv(shader->handle, GL_LINK_STATUS, &link_status);
+    if (link_status == GL_FALSE) {
+        LOG_INFO(Render_OpenGL, "Precompiled cache rejected by the driver - removing");
+        return {};
+    }
+
+    return shader;
+}
+
+std::unordered_map<u64, UnspecializedShader> ShaderCacheOpenGL::GenerateUnspecializedShaders(
+    const std::atomic_bool& stop_loading, const VideoCore::DiskResourceLoadCallback& callback,
+    const std::vector<ShaderDiskCacheRaw>& raws,
+    const std::unordered_map<u64, ShaderDiskCacheDecompiled>& decompiled) {
+    std::unordered_map<u64, UnspecializedShader> unspecialized;
+
+    if (callback)
+        callback(VideoCore::LoadCallbackStage::Decompile, 0, raws.size());
+
+    for (std::size_t i = 0; i < raws.size(); ++i) {
+        if (stop_loading)
+            return {};
+
+        const auto& raw{raws[i]};
+        const u64 unique_identifier = raw.GetUniqueIdentifier();
+        const u64 calculated_hash =
+            GetUniqueIdentifier(raw.GetProgramType(), raw.GetProgramCode(), raw.GetProgramCodeB());
+        if (unique_identifier != calculated_hash) {
+            LOG_ERROR(
+                Render_OpenGL,
+                "Invalid hash in entry={:016x} (obtained hash={:016x}) - removing shader cache",
+                raw.GetUniqueIdentifier(), calculated_hash);
+            disk_cache.InvalidateTransferable();
+            return {};
+        }
+
+        GLShader::ProgramResult result;
+        if (const auto it = decompiled.find(unique_identifier); it != decompiled.end()) {
+            // If it's stored in the precompiled file, avoid decompiling it here
+            const auto& stored_decompiled{it->second};
+            result = {stored_decompiled.code, stored_decompiled.entries};
+        } else {
+            // Otherwise decompile the shader at boot and save the result to the decompiled file
+            result =
+                CreateProgram(raw.GetProgramType(), raw.GetProgramCode(), raw.GetProgramCodeB());
+            disk_cache.SaveDecompiled(unique_identifier, result.first, result.second);
+        }
+
+        precompiled_shaders.insert({unique_identifier, result});
+
+        unspecialized.insert(
+            {raw.GetUniqueIdentifier(),
+             {std::move(result.first), std::move(result.second), raw.GetProgramType()}});
+
+        if (callback)
+            callback(VideoCore::LoadCallbackStage::Decompile, i, raws.size());
+    }
+    return unspecialized;
+}
 
 Shader ShaderCacheOpenGL::GetStageProgram(Maxwell::ShaderProgram program) {
     if (!Core::System::GetInstance().GPU().Maxwell3D().dirty_flags.shaders) {
@@ -237,7 +493,23 @@ Shader ShaderCacheOpenGL::GetStageProgram(Maxwell::ShaderProgram program) {
 
     if (!shader) {
         // No shader found - create a new one
-        shader = std::make_shared<CachedShader>(program_addr, program);
+        ProgramCode program_code = GetShaderCode(program_addr);
+        ProgramCode program_code_b;
+        if (program == Maxwell::ShaderProgram::VertexA) {
+            program_code_b = GetShaderCode(GetShaderAddress(Maxwell::ShaderProgram::VertexB));
+        }
+        const u64 unique_identifier = GetUniqueIdentifier(program, program_code, program_code_b);
+
+        const auto found = precompiled_shaders.find(unique_identifier);
+        if (found != precompiled_shaders.end()) {
+            shader =
+                std::make_shared<CachedShader>(program_addr, unique_identifier, program, disk_cache,
+                                               precompiled_programs, found->second);
+        } else {
+            shader = std::make_shared<CachedShader>(
+                program_addr, unique_identifier, program, disk_cache, precompiled_programs,
+                std::move(program_code), std::move(program_code_b));
+        }
         Register(shader);
     }
 

src/video_core/renderer_opengl/gl_shader_cache.h

@@ -5,40 +5,49 @@
 #pragma once
 
 #include <array>
-#include <map>
 #include <memory>
+#include <set>
 #include <tuple>
+#include <unordered_map>
 
 #include <glad/glad.h>
 
 #include "common/assert.h"
 #include "common/common_types.h"
 #include "video_core/rasterizer_cache.h"
+#include "video_core/renderer_base.h"
 #include "video_core/renderer_opengl/gl_resource_manager.h"
 #include "video_core/renderer_opengl/gl_shader_decompiler.h"
+#include "video_core/renderer_opengl/gl_shader_disk_cache.h"
 #include "video_core/renderer_opengl/gl_shader_gen.h"
 
+namespace Core {
+class System;
+} // namespace Core
+
 namespace OpenGL {
 
 class CachedShader;
 class RasterizerOpenGL;
+struct UnspecializedShader;
 
 using Shader = std::shared_ptr<CachedShader>;
+using CachedProgram = std::shared_ptr<OGLProgram>;
 using Maxwell = Tegra::Engines::Maxwell3D::Regs;
-
+using PrecompiledPrograms = std::unordered_map<ShaderDiskCacheUsage, CachedProgram>;
-struct BaseBindings {
+using PrecompiledShaders = std::unordered_map<u64, GLShader::ProgramResult>;
-    u32 cbuf{};
-    u32 gmem{};
-    u32 sampler{};
-
-    bool operator<(const BaseBindings& rhs) const {
-        return std::tie(cbuf, gmem, sampler) < std::tie(rhs.cbuf, rhs.gmem, rhs.sampler);
-    }
-};
 
 class CachedShader final : public RasterizerCacheObject {
 public:
-    CachedShader(VAddr addr, Maxwell::ShaderProgram program_type);
+    explicit CachedShader(VAddr addr, u64 unique_identifier, Maxwell::ShaderProgram program_type,
+                          ShaderDiskCacheOpenGL& disk_cache,
+                          const PrecompiledPrograms& precompiled_programs,
+                          ProgramCode&& program_code, ProgramCode&& program_code_b);
+
+    explicit CachedShader(VAddr addr, u64 unique_identifier, Maxwell::ShaderProgram program_type,
+                          ShaderDiskCacheOpenGL& disk_cache,
+                          const PrecompiledPrograms& precompiled_programs,
+                          GLShader::ProgramResult result);
 
     VAddr GetAddr() const override {
         return addr;
@@ -65,49 +74,67 @@ private:
     // declared by the hardware. Workaround this issue by generating a different shader per input
     // topology class.
     struct GeometryPrograms {
-        OGLProgram points;
+        CachedProgram points;
-        OGLProgram lines;
+        CachedProgram lines;
-        OGLProgram lines_adjacency;
+        CachedProgram lines_adjacency;
-        OGLProgram triangles;
+        CachedProgram triangles;
-        OGLProgram triangles_adjacency;
+        CachedProgram triangles_adjacency;
     };
 
-    std::string AllocateBindings(BaseBindings base_bindings);
-
     GLuint GetGeometryShader(GLenum primitive_mode, BaseBindings base_bindings);
 
     /// Generates a geometry shader or returns one that already exists.
-    GLuint LazyGeometryProgram(OGLProgram& target_program, BaseBindings base_bindings,
+    GLuint LazyGeometryProgram(CachedProgram& target_program, BaseBindings base_bindings,
-                               const std::string& glsl_topology, u32 max_vertices,
+                               GLenum primitive_mode);
-                               const std::string& debug_name);
 
-    void CalculateProperties();
+    CachedProgram TryLoadProgram(GLenum primitive_mode, BaseBindings base_bindings) const;
+
+    ShaderDiskCacheUsage GetUsage(GLenum primitive_mode, BaseBindings base_bindings) const;
 
     VAddr addr{};
-    std::size_t shader_length{};
+    u64 unique_identifier{};
     Maxwell::ShaderProgram program_type{};
-    GLShader::ShaderSetup setup;
+    ShaderDiskCacheOpenGL& disk_cache;
+    const PrecompiledPrograms& precompiled_programs;
+
+    std::size_t shader_length{};
     GLShader::ShaderEntries entries;
 
     std::string code;
 
-    std::map<BaseBindings, OGLProgram> programs;
+    std::unordered_map<BaseBindings, CachedProgram> programs;
-    std::map<BaseBindings, GeometryPrograms> geometry_programs;
+    std::unordered_map<BaseBindings, GeometryPrograms> geometry_programs;
 
-    std::map<u32, GLuint> cbuf_resource_cache;
+    std::unordered_map<u32, GLuint> cbuf_resource_cache;
-    std::map<u32, GLuint> gmem_resource_cache;
+    std::unordered_map<u32, GLuint> gmem_resource_cache;
-    std::map<u32, GLint> uniform_cache;
+    std::unordered_map<u32, GLint> uniform_cache;
 };
 
 class ShaderCacheOpenGL final : public RasterizerCache<Shader> {
 public:
-    explicit ShaderCacheOpenGL(RasterizerOpenGL& rasterizer);
+    explicit ShaderCacheOpenGL(RasterizerOpenGL& rasterizer, Core::System& system);
+
+    /// Loads disk cache for the current game
+    void LoadDiskCache(const std::atomic_bool& stop_loading,
+                       const VideoCore::DiskResourceLoadCallback& callback);
 
     /// Gets the current specified shader stage program
     Shader GetStageProgram(Maxwell::ShaderProgram program);
 
 private:
+    std::unordered_map<u64, UnspecializedShader> GenerateUnspecializedShaders(
+        const std::atomic_bool& stop_loading, const VideoCore::DiskResourceLoadCallback& callback,
+        const std::vector<ShaderDiskCacheRaw>& raws,
+        const std::unordered_map<u64, ShaderDiskCacheDecompiled>& decompiled);
+
+    CachedProgram GeneratePrecompiledProgram(const ShaderDiskCacheDump& dump,
+                                             const std::set<GLenum>& supported_formats);
+
     std::array<Shader, Maxwell::MaxShaderProgram> last_shaders;
+
+    ShaderDiskCacheOpenGL disk_cache;
+    PrecompiledShaders precompiled_shaders;
+    PrecompiledPrograms precompiled_programs;
 };
 
 } // namespace OpenGL

src/video_core/renderer_opengl/gl_shader_decompiler.cpp

@@ -193,15 +193,14 @@ public:
     ShaderEntries GetShaderEntries() const {
         ShaderEntries entries;
         for (const auto& cbuf : ir.GetConstantBuffers()) {
-            entries.const_buffers.emplace_back(cbuf.second, stage, GetConstBufferBlock(cbuf.first),
+            entries.const_buffers.emplace_back(cbuf.second.GetMaxOffset(), cbuf.second.IsIndirect(),
                                                cbuf.first);
         }
         for (const auto& sampler : ir.GetSamplers()) {
-            entries.samplers.emplace_back(sampler, stage, GetSampler(sampler));
+            entries.samplers.emplace_back(sampler);
         }
         for (const auto& gmem : ir.GetGlobalMemoryBases()) {
-            entries.global_memory_entries.emplace_back(gmem.cbuf_index, gmem.cbuf_offset, stage,
+            entries.global_memory_entries.emplace_back(gmem.cbuf_index, gmem.cbuf_offset);
-                                                       GetGlobalMemoryBlock(gmem));
         }
         entries.clip_distances = ir.GetClipDistances();
         entries.shader_length = ir.GetLength();

src/video_core/renderer_opengl/gl_shader_decompiler.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <array>
+#include <set>
 #include <string>
 #include <utility>
 #include <vector>
@@ -18,56 +19,29 @@ class ShaderIR;
 
 namespace OpenGL::GLShader {
 
+struct ShaderEntries;
+
 using Maxwell = Tegra::Engines::Maxwell3D::Regs;
+using ProgramResult = std::pair<std::string, ShaderEntries>;
+using SamplerEntry = VideoCommon::Shader::Sampler;
 
 class ConstBufferEntry : public VideoCommon::Shader::ConstBuffer {
 public:
-    explicit ConstBufferEntry(const VideoCommon::Shader::ConstBuffer& entry,
+    explicit ConstBufferEntry(u32 max_offset, bool is_indirect, u32 index)
-                              Maxwell::ShaderStage stage, const std::string& name, u32 index)
+        : VideoCommon::Shader::ConstBuffer{max_offset, is_indirect}, index{index} {}
-        : VideoCommon::Shader::ConstBuffer{entry}, stage{stage}, name{name}, index{index} {}
-
-    const std::string& GetName() const {
-        return name;
-    }
-
-    Maxwell::ShaderStage GetStage() const {
-        return stage;
-    }
 
     u32 GetIndex() const {
         return index;
     }
 
 private:
-    std::string name;
-    Maxwell::ShaderStage stage{};
     u32 index{};
 };
 
-class SamplerEntry : public VideoCommon::Shader::Sampler {
-public:
-    explicit SamplerEntry(const VideoCommon::Shader::Sampler& entry, Maxwell::ShaderStage stage,
-                          const std::string& name)
-        : VideoCommon::Shader::Sampler{entry}, stage{stage}, name{name} {}
-
-    const std::string& GetName() const {
-        return name;
-    }
-
-    Maxwell::ShaderStage GetStage() const {
-        return stage;
-    }
-
-private:
-    std::string name;
-    Maxwell::ShaderStage stage{};
-};
-
 class GlobalMemoryEntry {
 public:
-    explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset, Maxwell::ShaderStage stage,
+    explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset)
-                               std::string name)
+        : cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset} {}
-        : cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset}, stage{stage}, name{std::move(name)} {}
 
     u32 GetCbufIndex() const {
         return cbuf_index;
@@ -77,19 +51,9 @@ public:
         return cbuf_offset;
     }
 
-    const std::string& GetName() const {
-        return name;
-    }
-
-    Maxwell::ShaderStage GetStage() const {
-        return stage;
-    }
-
 private:
     u32 cbuf_index{};
     u32 cbuf_offset{};
-    Maxwell::ShaderStage stage{};
-    std::string name;
 };
 
 struct ShaderEntries {
@@ -100,8 +64,6 @@ struct ShaderEntries {
     std::size_t shader_length{};
 };
 
-using ProgramResult = std::pair<std::string, ShaderEntries>;
-
 std::string GetCommonDeclarations();
 
 ProgramResult Decompile(const VideoCommon::Shader::ShaderIR& ir, Maxwell::ShaderStage stage,

src/video_core/renderer_opengl/gl_shader_disk_cache.cpp

@@ -0,0 +1,656 @@
+// Copyright 2019 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <cstring>
+#include <fmt/format.h>
+#include <lz4.h>
+
+#include "common/assert.h"
+#include "common/common_paths.h"
+#include "common/common_types.h"
+#include "common/file_util.h"
+#include "common/logging/log.h"
+#include "common/scm_rev.h"
+
+#include "core/core.h"
+#include "core/hle/kernel/process.h"
+#include "core/settings.h"
+
+#include "video_core/renderer_opengl/gl_shader_cache.h"
+#include "video_core/renderer_opengl/gl_shader_disk_cache.h"
+
+namespace OpenGL {
+
+using ShaderCacheVersionHash = std::array<u8, 64>;
+
+enum class TransferableEntryKind : u32 {
+    Raw,
+    Usage,
+};
+
+enum class PrecompiledEntryKind : u32 {
+    Decompiled,
+    Dump,
+};
+
+constexpr u32 NativeVersion = 1;
+
+// Making sure sizes doesn't change by accident
+static_assert(sizeof(BaseBindings) == 12);
+static_assert(sizeof(ShaderDiskCacheUsage) == 24);
+
+namespace {
+
+ShaderCacheVersionHash GetShaderCacheVersionHash() {
+    ShaderCacheVersionHash hash{};
+    const std::size_t length = std::min(std::strlen(Common::g_shader_cache_version), hash.size());
+    std::memcpy(hash.data(), Common::g_shader_cache_version, length);
+    return hash;
+}
+
+template <typename T>
+std::vector<u8> CompressData(const T* source, std::size_t source_size) {
+    if (source_size > LZ4_MAX_INPUT_SIZE) {
+        // Source size exceeds LZ4 maximum input size
+        return {};
+    }
+    const auto source_size_int = static_cast<int>(source_size);
+    const int max_compressed_size = LZ4_compressBound(source_size_int);
+    std::vector<u8> compressed(max_compressed_size);
+    const int compressed_size = LZ4_compress_default(reinterpret_cast<const char*>(source),
+                                                     reinterpret_cast<char*>(compressed.data()),
+                                                     source_size_int, max_compressed_size);
+    if (compressed_size <= 0) {
+        // Compression failed
+        return {};
+    }
+    compressed.resize(compressed_size);
+    return compressed;
+}
+
+std::vector<u8> DecompressData(const std::vector<u8>& compressed, std::size_t uncompressed_size) {
+    std::vector<u8> uncompressed(uncompressed_size);
+    const int size_check = LZ4_decompress_safe(reinterpret_cast<const char*>(compressed.data()),
+                                               reinterpret_cast<char*>(uncompressed.data()),
+                                               static_cast<int>(compressed.size()),
+                                               static_cast<int>(uncompressed.size()));
+    if (static_cast<int>(uncompressed_size) != size_check) {
+        // Decompression failed
+        return {};
+    }
+    return uncompressed;
+}
+
+} // namespace
+
+ShaderDiskCacheRaw::ShaderDiskCacheRaw(u64 unique_identifier, Maxwell::ShaderProgram program_type,
+                                       u32 program_code_size, u32 program_code_size_b,
+                                       ProgramCode program_code, ProgramCode program_code_b)
+    : unique_identifier{unique_identifier}, program_type{program_type},
+      program_code_size{program_code_size}, program_code_size_b{program_code_size_b},
+      program_code{std::move(program_code)}, program_code_b{std::move(program_code_b)} {}
+
+ShaderDiskCacheRaw::ShaderDiskCacheRaw() = default;
+
+ShaderDiskCacheRaw::~ShaderDiskCacheRaw() = default;
+
+bool ShaderDiskCacheRaw::Load(FileUtil::IOFile& file) {
+    if (file.ReadBytes(&unique_identifier, sizeof(u64)) != sizeof(u64) ||
+        file.ReadBytes(&program_type, sizeof(u32)) != sizeof(u32)) {
+        return false;
+    }
+    u32 program_code_size{};
+    u32 program_code_size_b{};
+    if (file.ReadBytes(&program_code_size, sizeof(u32)) != sizeof(u32) ||
+        file.ReadBytes(&program_code_size_b, sizeof(u32)) != sizeof(u32)) {
+        return false;
+    }
+
+    program_code.resize(program_code_size);
+    program_code_b.resize(program_code_size_b);
+
+    if (file.ReadArray(program_code.data(), program_code_size) != program_code_size)
+        return false;
+
+    if (HasProgramA() &&
+        file.ReadArray(program_code_b.data(), program_code_size_b) != program_code_size_b) {
+        return false;
+    }
+    return true;
+}
+
+bool ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const {
+    if (file.WriteObject(unique_identifier) != 1 ||
+        file.WriteObject(static_cast<u32>(program_type)) != 1 ||
+        file.WriteObject(program_code_size) != 1 || file.WriteObject(program_code_size_b) != 1) {
+        return false;
+    }
+
+    if (file.WriteArray(program_code.data(), program_code_size) != program_code_size)
+        return false;
+
+    if (HasProgramA() &&
+        file.WriteArray(program_code_b.data(), program_code_size_b) != program_code_size_b) {
+        return false;
+    }
+    return true;
+}
+
+ShaderDiskCacheOpenGL::ShaderDiskCacheOpenGL(Core::System& system) : system{system} {}
+
+std::optional<std::pair<std::vector<ShaderDiskCacheRaw>, std::vector<ShaderDiskCacheUsage>>>
+ShaderDiskCacheOpenGL::LoadTransferable() {
+    // Skip games without title id
+    const bool has_title_id = system.CurrentProcess()->GetTitleID() != 0;
+    if (!Settings::values.use_disk_shader_cache || !has_title_id)
+        return {};
+    tried_to_load = true;
+
+    FileUtil::IOFile file(GetTransferablePath(), "rb");
+    if (!file.IsOpen()) {
+        LOG_INFO(Render_OpenGL, "No transferable shader cache found for game with title id={}",
+                 GetTitleID());
+        return {};
+    }
+
+    u32 version{};
+    if (file.ReadBytes(&version, sizeof(version)) != sizeof(version)) {
+        LOG_ERROR(Render_OpenGL,
+                  "Failed to get transferable cache version for title id={} - skipping",
+                  GetTitleID());
+        return {};
+    }
+
+    if (version < NativeVersion) {
+        LOG_INFO(Render_OpenGL, "Transferable shader cache is old - removing");
+        file.Close();
+        InvalidateTransferable();
+        return {};
+    }
+    if (version > NativeVersion) {
+        LOG_WARNING(Render_OpenGL, "Transferable shader cache was generated with a newer version "
+                                   "of the emulator - skipping");
+        return {};
+    }
+
+    // Version is valid, load the shaders
+    std::vector<ShaderDiskCacheRaw> raws;
+    std::vector<ShaderDiskCacheUsage> usages;
+    while (file.Tell() < file.GetSize()) {
+        TransferableEntryKind kind{};
+        if (file.ReadBytes(&kind, sizeof(u32)) != sizeof(u32)) {
+            LOG_ERROR(Render_OpenGL, "Failed to read transferable file - skipping");
+            return {};
+        }
+
+        switch (kind) {
+        case TransferableEntryKind::Raw: {
+            ShaderDiskCacheRaw entry;
+            if (!entry.Load(file)) {
+                LOG_ERROR(Render_OpenGL, "Failed to load transferable raw entry - skipping");
+                return {};
+            }
+            transferable.insert({entry.GetUniqueIdentifier(), {}});
+            raws.push_back(std::move(entry));
+            break;
+        }
+        case TransferableEntryKind::Usage: {
+            ShaderDiskCacheUsage usage{};
+            if (file.ReadBytes(&usage, sizeof(usage)) != sizeof(usage)) {
+                LOG_ERROR(Render_OpenGL, "Failed to load transferable usage entry - skipping");
+                return {};
+            }
+            usages.push_back(std::move(usage));
+            break;
+        }
+        default:
+            LOG_ERROR(Render_OpenGL, "Unknown transferable shader cache entry kind={} - skipping",
+                      static_cast<u32>(kind));
+            return {};
+        }
+    }
+    return {{raws, usages}};
+}
+
+std::pair<std::unordered_map<u64, ShaderDiskCacheDecompiled>,
+          std::unordered_map<ShaderDiskCacheUsage, ShaderDiskCacheDump>>
+ShaderDiskCacheOpenGL::LoadPrecompiled() {
+    if (!IsUsable())
+        return {};
+
+    FileUtil::IOFile file(GetPrecompiledPath(), "rb");
+    if (!file.IsOpen()) {
+        LOG_INFO(Render_OpenGL, "No precompiled shader cache found for game with title id={}",
+                 GetTitleID());
+        return {};
+    }
+
+    const auto result = LoadPrecompiledFile(file);
+    if (!result) {
+        LOG_INFO(Render_OpenGL,
+                 "Failed to load precompiled cache for game with title id={} - removing",
+                 GetTitleID());
+        file.Close();
+        InvalidatePrecompiled();
+        return {};
+    }
+    return *result;
+}
+
+std::optional<std::pair<std::unordered_map<u64, ShaderDiskCacheDecompiled>,
+                        std::unordered_map<ShaderDiskCacheUsage, ShaderDiskCacheDump>>>
+ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) {
+    ShaderCacheVersionHash file_hash{};
+    if (file.ReadArray(file_hash.data(), file_hash.size()) != file_hash.size()) {
+        return {};
+    }
+    if (GetShaderCacheVersionHash() != file_hash) {
+        LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of the emulator");
+        return {};
+    }
+
+    std::unordered_map<u64, ShaderDiskCacheDecompiled> decompiled;
+    std::unordered_map<ShaderDiskCacheUsage, ShaderDiskCacheDump> dumps;
+    while (file.Tell() < file.GetSize()) {
+        PrecompiledEntryKind kind{};
+        if (file.ReadBytes(&kind, sizeof(u32)) != sizeof(u32)) {
+            return {};
+        }
+
+        switch (kind) {
+        case PrecompiledEntryKind::Decompiled: {
+            u64 unique_identifier{};
+            if (file.ReadBytes(&unique_identifier, sizeof(u64)) != sizeof(u64))
+                return {};
+
+            const auto entry = LoadDecompiledEntry(file);
+            if (!entry)
+                return {};
+            decompiled.insert({unique_identifier, std::move(*entry)});
+            break;
+        }
+        case PrecompiledEntryKind::Dump: {
+            ShaderDiskCacheUsage usage;
+            if (file.ReadBytes(&usage, sizeof(usage)) != sizeof(usage))
+                return {};
+
+            ShaderDiskCacheDump dump;
+            if (file.ReadBytes(&dump.binary_format, sizeof(u32)) != sizeof(u32))
+                return {};
+
+            u32 binary_length{};
+            u32 compressed_size{};
+            if (file.ReadBytes(&binary_length, sizeof(u32)) != sizeof(u32) ||
+                file.ReadBytes(&compressed_size, sizeof(u32)) != sizeof(u32)) {
+                return {};
+            }
+
+            std::vector<u8> compressed_binary(compressed_size);
+            if (file.ReadArray(compressed_binary.data(), compressed_binary.size()) !=
+                compressed_binary.size()) {
+                return {};
+            }
+
+            dump.binary = DecompressData(compressed_binary, binary_length);
+            if (dump.binary.empty()) {
+                return {};
+            }
+
+            dumps.insert({usage, dump});
+            break;
+        }
+        default:
+            return {};
+        }
+    }
+    return {{decompiled, dumps}};
+}
+
+std::optional<ShaderDiskCacheDecompiled> ShaderDiskCacheOpenGL::LoadDecompiledEntry(
+    FileUtil::IOFile& file) {
+    u32 code_size{};
+    u32 compressed_code_size{};
+    if (file.ReadBytes(&code_size, sizeof(u32)) != sizeof(u32) ||
+        file.ReadBytes(&compressed_code_size, sizeof(u32)) != sizeof(u32)) {
+        return {};
+    }
+
+    std::vector<u8> compressed_code(compressed_code_size);
+    if (file.ReadArray(compressed_code.data(), compressed_code.size()) != compressed_code.size()) {
+        return {};
+    }
+
+    const std::vector<u8> code = DecompressData(compressed_code, code_size);
+    if (code.empty()) {
+        return {};
+    }
+    ShaderDiskCacheDecompiled entry;
+    entry.code = std::string(reinterpret_cast<const char*>(code.data()), code_size);
+
+    u32 const_buffers_count{};
+    if (file.ReadBytes(&const_buffers_count, sizeof(u32)) != sizeof(u32))
+        return {};
+    for (u32 i = 0; i < const_buffers_count; ++i) {
+        u32 max_offset{};
+        u32 index{};
+        u8 is_indirect{};
+        if (file.ReadBytes(&max_offset, sizeof(u32)) != sizeof(u32) ||
+            file.ReadBytes(&index, sizeof(u32)) != sizeof(u32) ||
+            file.ReadBytes(&is_indirect, sizeof(u8)) != sizeof(u8)) {
+            return {};
+        }
+        entry.entries.const_buffers.emplace_back(max_offset, is_indirect != 0, index);
+    }
+
+    u32 samplers_count{};
+    if (file.ReadBytes(&samplers_count, sizeof(u32)) != sizeof(u32))
+        return {};
+    for (u32 i = 0; i < samplers_count; ++i) {
+        u64 offset{};
+        u64 index{};
+        u32 type{};
+        u8 is_array{};
+        u8 is_shadow{};
+        if (file.ReadBytes(&offset, sizeof(u64)) != sizeof(u64) ||
+            file.ReadBytes(&index, sizeof(u64)) != sizeof(u64) ||
+            file.ReadBytes(&type, sizeof(u32)) != sizeof(u32) ||
+            file.ReadBytes(&is_array, sizeof(u8)) != sizeof(u8) ||
+            file.ReadBytes(&is_shadow, sizeof(u8)) != sizeof(u8)) {
+            return {};
+        }
+        entry.entries.samplers.emplace_back(
+            static_cast<std::size_t>(offset), static_cast<std::size_t>(index),
+            static_cast<Tegra::Shader::TextureType>(type), is_array != 0, is_shadow != 0);
+    }
+
+    u32 global_memory_count{};
+    if (file.ReadBytes(&global_memory_count, sizeof(u32)) != sizeof(u32))
+        return {};
+    for (u32 i = 0; i < global_memory_count; ++i) {
+        u32 cbuf_index{};
+        u32 cbuf_offset{};
+        if (file.ReadBytes(&cbuf_index, sizeof(u32)) != sizeof(u32) ||
+            file.ReadBytes(&cbuf_offset, sizeof(u32)) != sizeof(u32)) {
+            return {};
+        }
+        entry.entries.global_memory_entries.emplace_back(cbuf_index, cbuf_offset);
+    }
+
+    for (auto& clip_distance : entry.entries.clip_distances) {
+        u8 clip_distance_raw{};
+        if (file.ReadBytes(&clip_distance_raw, sizeof(u8)) != sizeof(u8))
+            return {};
+        clip_distance = clip_distance_raw != 0;
+    }
+
+    u64 shader_length{};
+    if (file.ReadBytes(&shader_length, sizeof(u64)) != sizeof(u64))
+        return {};
+    entry.entries.shader_length = static_cast<std::size_t>(shader_length);
+
+    return entry;
+}
+
+bool ShaderDiskCacheOpenGL::SaveDecompiledFile(FileUtil::IOFile& file, u64 unique_identifier,
+                                               const std::string& code,
+                                               const std::vector<u8>& compressed_code,
+                                               const GLShader::ShaderEntries& entries) {
+    if (file.WriteObject(static_cast<u32>(PrecompiledEntryKind::Decompiled)) != 1 ||
+        file.WriteObject(unique_identifier) != 1 ||
+        file.WriteObject(static_cast<u32>(code.size())) != 1 ||
+        file.WriteObject(static_cast<u32>(compressed_code.size())) != 1 ||
+        file.WriteArray(compressed_code.data(), compressed_code.size()) != compressed_code.size()) {
+        return false;
+    }
+
+    if (file.WriteObject(static_cast<u32>(entries.const_buffers.size())) != 1)
+        return false;
+    for (const auto& cbuf : entries.const_buffers) {
+        if (file.WriteObject(static_cast<u32>(cbuf.GetMaxOffset())) != 1 ||
+            file.WriteObject(static_cast<u32>(cbuf.GetIndex())) != 1 ||
+            file.WriteObject(static_cast<u8>(cbuf.IsIndirect() ? 1 : 0)) != 1) {
+            return false;
+        }
+    }
+
+    if (file.WriteObject(static_cast<u32>(entries.samplers.size())) != 1)
+        return false;
+    for (const auto& sampler : entries.samplers) {
+        if (file.WriteObject(static_cast<u64>(sampler.GetOffset())) != 1 ||
+            file.WriteObject(static_cast<u64>(sampler.GetIndex())) != 1 ||
+            file.WriteObject(static_cast<u32>(sampler.GetType())) != 1 ||
+            file.WriteObject(static_cast<u8>(sampler.IsArray() ? 1 : 0)) != 1 ||
+            file.WriteObject(static_cast<u8>(sampler.IsShadow() ? 1 : 0)) != 1) {
+            return false;
+        }
+    }
+
+    if (file.WriteObject(static_cast<u32>(entries.global_memory_entries.size())) != 1)
+        return false;
+    for (const auto& gmem : entries.global_memory_entries) {
+        if (file.WriteObject(static_cast<u32>(gmem.GetCbufIndex())) != 1 ||
+            file.WriteObject(static_cast<u32>(gmem.GetCbufOffset())) != 1) {
+            return false;
+        }
+    }
+
+    for (const bool clip_distance : entries.clip_distances) {
+        if (file.WriteObject(static_cast<u8>(clip_distance ? 1 : 0)) != 1)
+            return false;
+    }
+
+    return file.WriteObject(static_cast<u64>(entries.shader_length)) == 1;
+}
+
+void ShaderDiskCacheOpenGL::InvalidateTransferable() const {
+    if (!FileUtil::Delete(GetTransferablePath())) {
+        LOG_ERROR(Render_OpenGL, "Failed to invalidate transferable file={}",
+                  GetTransferablePath());
+    }
+    InvalidatePrecompiled();
+}
+
+void ShaderDiskCacheOpenGL::InvalidatePrecompiled() const {
+    if (!FileUtil::Delete(GetPrecompiledPath())) {
+        LOG_ERROR(Render_OpenGL, "Failed to invalidate precompiled file={}", GetPrecompiledPath());
+    }
+}
+
+void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) {
+    if (!IsUsable())
+        return;
+
+    const u64 id = entry.GetUniqueIdentifier();
+    if (transferable.find(id) != transferable.end()) {
+        // The shader already exists
+        return;
+    }
+
+    FileUtil::IOFile file = AppendTransferableFile();
+    if (!file.IsOpen())
+        return;
+    if (file.WriteObject(TransferableEntryKind::Raw) != 1 || !entry.Save(file)) {
+        LOG_ERROR(Render_OpenGL, "Failed to save raw transferable cache entry - removing");
+        file.Close();
+        InvalidateTransferable();
+        return;
+    }
+    transferable.insert({id, {}});
+}
+
+void ShaderDiskCacheOpenGL::SaveUsage(const ShaderDiskCacheUsage& usage) {
+    if (!IsUsable())
+        return;
+
+    const auto it = transferable.find(usage.unique_identifier);
+    ASSERT_MSG(it != transferable.end(), "Saving shader usage without storing raw previously");
+
+    auto& usages{it->second};
+    ASSERT(usages.find(usage) == usages.end());
+    usages.insert(usage);
+
+    FileUtil::IOFile file = AppendTransferableFile();
+    if (!file.IsOpen())
+        return;
+
+    if (file.WriteObject(TransferableEntryKind::Usage) != 1 || file.WriteObject(usage) != 1) {
+        LOG_ERROR(Render_OpenGL, "Failed to save usage transferable cache entry - removing");
+        file.Close();
+        InvalidateTransferable();
+        return;
+    }
+}
+
+void ShaderDiskCacheOpenGL::SaveDecompiled(u64 unique_identifier, const std::string& code,
+                                           const GLShader::ShaderEntries& entries) {
+    if (!IsUsable())
+        return;
+
+    const std::vector<u8> compressed_code{CompressData(code.data(), code.size())};
+    if (compressed_code.empty()) {
+        LOG_ERROR(Render_OpenGL, "Failed to compress GLSL code - skipping shader {:016x}",
+                  unique_identifier);
+        return;
+    }
+
+    FileUtil::IOFile file = AppendPrecompiledFile();
+    if (!file.IsOpen())
+        return;
+
+    if (!SaveDecompiledFile(file, unique_identifier, code, compressed_code, entries)) {
+        LOG_ERROR(Render_OpenGL,
+                  "Failed to save decompiled entry to the precompiled file - removing");
+        file.Close();
+        InvalidatePrecompiled();
+    }
+}
+
+void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint program) {
+    if (!IsUsable())
+        return;
+
+    GLint binary_length{};
+    glGetProgramiv(program, GL_PROGRAM_BINARY_LENGTH, &binary_length);
+
+    GLenum binary_format{};
+    std::vector<u8> binary(binary_length);
+    glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data());
+
+    const std::vector<u8> compressed_binary = CompressData(binary.data(), binary.size());
+    if (compressed_binary.empty()) {
+        LOG_ERROR(Render_OpenGL, "Failed to compress binary program in shader={:016x}",
+                  usage.unique_identifier);
+        return;
+    }
+
+    FileUtil::IOFile file = AppendPrecompiledFile();
+    if (!file.IsOpen())
+        return;
+
+    if (file.WriteObject(static_cast<u32>(PrecompiledEntryKind::Dump)) != 1 ||
+        file.WriteObject(usage) != 1 || file.WriteObject(static_cast<u32>(binary_format)) != 1 ||
+        file.WriteObject(static_cast<u32>(binary_length)) != 1 ||
+        file.WriteObject(static_cast<u32>(compressed_binary.size())) != 1 ||
+        file.WriteArray(compressed_binary.data(), compressed_binary.size()) !=
+            compressed_binary.size()) {
+        LOG_ERROR(Render_OpenGL, "Failed to save binary program file in shader={:016x} - removing",
+                  usage.unique_identifier);
+        file.Close();
+        InvalidatePrecompiled();
+        return;
+    }
+}
+
+bool ShaderDiskCacheOpenGL::IsUsable() const {
+    return tried_to_load && Settings::values.use_disk_shader_cache;
+}
+
+FileUtil::IOFile ShaderDiskCacheOpenGL::AppendTransferableFile() const {
+    if (!EnsureDirectories())
+        return {};
+
+    const auto transferable_path{GetTransferablePath()};
+    const bool existed = FileUtil::Exists(transferable_path);
+
+    FileUtil::IOFile file(transferable_path, "ab");
+    if (!file.IsOpen()) {
+        LOG_ERROR(Render_OpenGL, "Failed to open transferable cache in path={}", transferable_path);
+        return {};
+    }
+    if (!existed || file.GetSize() == 0) {
+        // If the file didn't exist, write its version
+        if (file.WriteObject(NativeVersion) != 1) {
+            LOG_ERROR(Render_OpenGL, "Failed to write transferable cache version in path={}",
+                      transferable_path);
+            return {};
+        }
+    }
+    return file;
+}
+
+FileUtil::IOFile ShaderDiskCacheOpenGL::AppendPrecompiledFile() const {
+    if (!EnsureDirectories())
+        return {};
+
+    const auto precompiled_path{GetPrecompiledPath()};
+    const bool existed = FileUtil::Exists(precompiled_path);
+
+    FileUtil::IOFile file(precompiled_path, "ab");
+    if (!file.IsOpen()) {
+        LOG_ERROR(Render_OpenGL, "Failed to open precompiled cache in path={}", precompiled_path);
+        return {};
+    }
+
+    if (!existed || file.GetSize() == 0) {
+        const auto hash{GetShaderCacheVersionHash()};
+        if (file.WriteArray(hash.data(), hash.size()) != hash.size()) {
+            LOG_ERROR(Render_OpenGL, "Failed to write precompiled cache version hash in path={}",
+                      precompiled_path);
+            return {};
+        }
+    }
+    return file;
+}
+
+bool ShaderDiskCacheOpenGL::EnsureDirectories() const {
+    const auto CreateDir = [](const std::string& dir) {
+        if (!FileUtil::CreateDir(dir)) {
+            LOG_ERROR(Render_OpenGL, "Failed to create directory={}", dir);
+            return false;
+        }
+        return true;
+    };
+
+    return CreateDir(FileUtil::GetUserPath(FileUtil::UserPath::ShaderDir)) &&
+           CreateDir(GetBaseDir()) && CreateDir(GetTransferableDir()) &&
+           CreateDir(GetPrecompiledDir());
+}
+
+std::string ShaderDiskCacheOpenGL::GetTransferablePath() const {
+    return FileUtil::SanitizePath(GetTransferableDir() + DIR_SEP_CHR + GetTitleID() + ".bin");
+}
+
+std::string ShaderDiskCacheOpenGL::GetPrecompiledPath() const {
+    return FileUtil::SanitizePath(GetPrecompiledDir() + DIR_SEP_CHR + GetTitleID() + ".bin");
+}
+
+std::string ShaderDiskCacheOpenGL::GetTransferableDir() const {
+    return GetBaseDir() + DIR_SEP "transferable";
+}
+
+std::string ShaderDiskCacheOpenGL::GetPrecompiledDir() const {
+    return GetBaseDir() + DIR_SEP "precompiled";
+}
+
+std::string ShaderDiskCacheOpenGL::GetBaseDir() const {
+    return FileUtil::GetUserPath(FileUtil::UserPath::ShaderDir) + DIR_SEP "opengl";
+}
+
+std::string ShaderDiskCacheOpenGL::GetTitleID() const {
+    return fmt::format("{:016X}", system.CurrentProcess()->GetTitleID());
+}
+
+} // namespace OpenGL

src/video_core/renderer_opengl/gl_shader_disk_cache.h

@@ -0,0 +1,245 @@
+// Copyright 2019 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <optional>
+#include <string>
+#include <tuple>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include <glad/glad.h>
+
+#include "common/assert.h"
+#include "common/common_types.h"
+#include "video_core/engines/maxwell_3d.h"
+#include "video_core/renderer_opengl/gl_shader_gen.h"
+
+namespace Core {
+class System;
+}
+
+namespace FileUtil {
+class IOFile;
+}
+
+namespace OpenGL {
+
+using ProgramCode = std::vector<u64>;
+using Maxwell = Tegra::Engines::Maxwell3D::Regs;
+
+/// Allocated bindings used by an OpenGL shader program
+struct BaseBindings {
+    u32 cbuf{};
+    u32 gmem{};
+    u32 sampler{};
+
+    bool operator==(const BaseBindings& rhs) const {
+        return std::tie(cbuf, gmem, sampler) == std::tie(rhs.cbuf, rhs.gmem, rhs.sampler);
+    }
+
+    bool operator!=(const BaseBindings& rhs) const {
+        return !operator==(rhs);
+    }
+};
+
+/// Describes how a shader is used
+struct ShaderDiskCacheUsage {
+    u64 unique_identifier{};
+    BaseBindings bindings;
+    GLenum primitive{};
+
+    bool operator==(const ShaderDiskCacheUsage& rhs) const {
+        return std::tie(unique_identifier, bindings, primitive) ==
+               std::tie(rhs.unique_identifier, rhs.bindings, rhs.primitive);
+    }
+
+    bool operator!=(const ShaderDiskCacheUsage& rhs) const {
+        return !operator==(rhs);
+    }
+};
+
+} // namespace OpenGL
+
+namespace std {
+
+template <>
+struct hash<OpenGL::BaseBindings> {
+    std::size_t operator()(const OpenGL::BaseBindings& bindings) const {
+        return bindings.cbuf | bindings.gmem << 8 | bindings.sampler << 16;
+    }
+};
+
+template <>
+struct hash<OpenGL::ShaderDiskCacheUsage> {
+    std::size_t operator()(const OpenGL::ShaderDiskCacheUsage& usage) const {
+        return static_cast<std::size_t>(usage.unique_identifier) ^
+               std::hash<OpenGL::BaseBindings>()(usage.bindings) ^ usage.primitive << 16;
+    }
+};
+
+} // namespace std
+
+namespace OpenGL {
+
+/// Describes a shader how it's used by the guest GPU
+class ShaderDiskCacheRaw {
+public:
+    explicit ShaderDiskCacheRaw(u64 unique_identifier, Maxwell::ShaderProgram program_type,
+                                u32 program_code_size, u32 program_code_size_b,
+                                ProgramCode program_code, ProgramCode program_code_b);
+    ShaderDiskCacheRaw();
+    ~ShaderDiskCacheRaw();
+
+    bool Load(FileUtil::IOFile& file);
+
+    bool Save(FileUtil::IOFile& file) const;
+
+    u64 GetUniqueIdentifier() const {
+        return unique_identifier;
+    }
+
+    bool HasProgramA() const {
+        return program_type == Maxwell::ShaderProgram::VertexA;
+    }
+
+    Maxwell::ShaderProgram GetProgramType() const {
+        return program_type;
+    }
+
+    Maxwell::ShaderStage GetProgramStage() const {
+        switch (program_type) {
+        case Maxwell::ShaderProgram::VertexA:
+        case Maxwell::ShaderProgram::VertexB:
+            return Maxwell::ShaderStage::Vertex;
+        case Maxwell::ShaderProgram::TesselationControl:
+            return Maxwell::ShaderStage::TesselationControl;
+        case Maxwell::ShaderProgram::TesselationEval:
+            return Maxwell::ShaderStage::TesselationEval;
+        case Maxwell::ShaderProgram::Geometry:
+            return Maxwell::ShaderStage::Geometry;
+        case Maxwell::ShaderProgram::Fragment:
+            return Maxwell::ShaderStage::Fragment;
+        }
+        UNREACHABLE();
+    }
+
+    const ProgramCode& GetProgramCode() const {
+        return program_code;
+    }
+
+    const ProgramCode& GetProgramCodeB() const {
+        return program_code_b;
+    }
+
+private:
+    u64 unique_identifier{};
+    Maxwell::ShaderProgram program_type{};
+    u32 program_code_size{};
+    u32 program_code_size_b{};
+
+    ProgramCode program_code;
+    ProgramCode program_code_b;
+};
+
+/// Contains decompiled data from a shader
+struct ShaderDiskCacheDecompiled {
+    std::string code;
+    GLShader::ShaderEntries entries;
+};
+
+/// Contains an OpenGL dumped binary program
+struct ShaderDiskCacheDump {
+    GLenum binary_format;
+    std::vector<u8> binary;
+};
+
+class ShaderDiskCacheOpenGL {
+public:
+    explicit ShaderDiskCacheOpenGL(Core::System& system);
+
+    /// Loads transferable cache. If file has a old version or on failure, it deletes the file.
+    std::optional<std::pair<std::vector<ShaderDiskCacheRaw>, std::vector<ShaderDiskCacheUsage>>>
+    LoadTransferable();
+
+    /// Loads current game's precompiled cache. Invalidates on failure.
+    std::pair<std::unordered_map<u64, ShaderDiskCacheDecompiled>,
+              std::unordered_map<ShaderDiskCacheUsage, ShaderDiskCacheDump>>
+    LoadPrecompiled();
+
+    /// Removes the transferable (and precompiled) cache file.
+    void InvalidateTransferable() const;
+
+    /// Removes the precompiled cache file.
+    void InvalidatePrecompiled() const;
+
+    /// Saves a raw dump to the transferable file. Checks for collisions.
+    void SaveRaw(const ShaderDiskCacheRaw& entry);
+
+    /// Saves shader usage to the transferable file. Does not check for collisions.
+    void SaveUsage(const ShaderDiskCacheUsage& usage);
+
+    /// Saves a decompiled entry to the precompiled file. Does not check for collisions.
+    void SaveDecompiled(u64 unique_identifier, const std::string& code,
+                        const GLShader::ShaderEntries& entries);
+
+    /// Saves a dump entry to the precompiled file. Does not check for collisions.
+    void SaveDump(const ShaderDiskCacheUsage& usage, GLuint program);
+
+private:
+    /// Loads the transferable cache. Returns empty on failure.
+    std::optional<std::pair<std::unordered_map<u64, ShaderDiskCacheDecompiled>,
+                            std::unordered_map<ShaderDiskCacheUsage, ShaderDiskCacheDump>>>
+    LoadPrecompiledFile(FileUtil::IOFile& file);
+
+    /// Loads a decompiled cache entry from the passed file. Returns empty on failure.
+    std::optional<ShaderDiskCacheDecompiled> LoadDecompiledEntry(FileUtil::IOFile& file);
+
+    /// Saves a decompiled entry to the passed file. Returns true on success.
+    bool SaveDecompiledFile(FileUtil::IOFile& file, u64 unique_identifier, const std::string& code,
+                            const std::vector<u8>& compressed_code,
+                            const GLShader::ShaderEntries& entries);
+
+    /// Returns if the cache can be used
+    bool IsUsable() const;
+
+    /// Opens current game's transferable file and write it's header if it doesn't exist
+    FileUtil::IOFile AppendTransferableFile() const;
+
+    /// Opens current game's precompiled file and write it's header if it doesn't exist
+    FileUtil::IOFile AppendPrecompiledFile() const;
+
+    /// Create shader disk cache directories. Returns true on success.
+    bool EnsureDirectories() const;
+
+    /// Gets current game's transferable file path
+    std::string GetTransferablePath() const;
+
+    /// Gets current game's precompiled file path
+    std::string GetPrecompiledPath() const;
+
+    /// Get user's transferable directory path
+    std::string GetTransferableDir() const;
+
+    /// Get user's precompiled directory path
+    std::string GetPrecompiledDir() const;
+
+    /// Get user's shader directory path
+    std::string GetBaseDir() const;
+
+    /// Get current game's title id
+    std::string GetTitleID() const;
+
+    // Copre system
+    Core::System& system;
+    // Stored transferable shaders
+    std::map<u64, std::unordered_set<ShaderDiskCacheUsage>> transferable;
+    // The cache has been loaded at boot
+    bool tried_to_load{};
+};
+
+} // namespace OpenGL

src/video_core/renderer_opengl/gl_shader_gen.h

@@ -26,12 +26,10 @@ struct ShaderSetup {
         ProgramCode code;
         ProgramCode code_b; // Used for dual vertex shaders
         u64 unique_identifier;
-        std::size_t real_size;
-        std::size_t real_size_b;
     } program;
 
     /// Used in scenarios where we have a dual vertex shaders
-    void SetProgramB(ProgramCode&& program_b) {
+    void SetProgramB(ProgramCode program_b) {
         program.code_b = std::move(program_b);
         has_program_b = true;
     }

src/video_core/renderer_opengl/gl_shader_util.h

@@ -47,7 +47,7 @@ GLuint LoadShader(const char* source, GLenum type);
  * @returns Handle of the newly created OpenGL program object
  */
 template <typename... T>
-GLuint LoadProgram(bool separable_program, T... shaders) {
+GLuint LoadProgram(bool separable_program, bool hint_retrievable, T... shaders) {
     // Link the program
     LOG_DEBUG(Render_OpenGL, "Linking program...");
 
@@ -58,6 +58,9 @@ GLuint LoadProgram(bool separable_program, T... shaders) {
     if (separable_program) {
         glProgramParameteri(program_id, GL_PROGRAM_SEPARABLE, GL_TRUE);
     }
+    if (hint_retrievable) {
+        glProgramParameteri(program_id, GL_PROGRAM_BINARY_RETRIEVABLE_HINT, GL_TRUE);
+    }
 
     glLinkProgram(program_id);
 

src/video_core/renderer_opengl/renderer_opengl.cpp

@@ -98,8 +98,8 @@ static std::array<GLfloat, 3 * 2> MakeOrthographicMatrix(const float width, cons
     return matrix;
 }
 
-RendererOpenGL::RendererOpenGL(Core::Frontend::EmuWindow& window)
+RendererOpenGL::RendererOpenGL(Core::Frontend::EmuWindow& window, Core::System& system)
-    : VideoCore::RendererBase{window} {}
+    : VideoCore::RendererBase{window}, system{system} {}
 
 RendererOpenGL::~RendererOpenGL() = default;
 
@@ -250,7 +250,7 @@ void RendererOpenGL::CreateRasterizer() {
     }
     // Initialize sRGB Usage
     OpenGLState::ClearsRGBUsed();
-    rasterizer = std::make_unique<RasterizerOpenGL>(render_window, screen_info);
+    rasterizer = std::make_unique<RasterizerOpenGL>(render_window, system, screen_info);
 }
 
 void RendererOpenGL::ConfigureFramebufferTexture(TextureInfo& texture,

src/video_core/renderer_opengl/renderer_opengl.h

@@ -12,6 +12,10 @@
 #include "video_core/renderer_opengl/gl_resource_manager.h"
 #include "video_core/renderer_opengl/gl_state.h"
 
+namespace Core {
+class System;
+}
+
 namespace Core::Frontend {
 class EmuWindow;
 }
@@ -41,7 +45,7 @@ struct ScreenInfo {
 
 class RendererOpenGL : public VideoCore::RendererBase {
 public:
-    explicit RendererOpenGL(Core::Frontend::EmuWindow& window);
+    explicit RendererOpenGL(Core::Frontend::EmuWindow& window, Core::System& system);
     ~RendererOpenGL() override;
 
     /// Swap buffers (render frame)
@@ -72,6 +76,8 @@ private:
     void LoadColorToActiveGLTexture(u8 color_r, u8 color_g, u8 color_b, u8 color_a,
                                     const TextureInfo& texture);
 
+    Core::System& system;
+
     OpenGLState state;
 
     // OpenGL object IDs

src/video_core/shader/shader_ir.h

@@ -236,6 +236,11 @@ private:
 
 class ConstBuffer {
 public:
+    explicit ConstBuffer(u32 max_offset, bool is_indirect)
+        : max_offset{max_offset}, is_indirect{is_indirect} {}
+
+    ConstBuffer() = default;
+
     void MarkAsUsed(u64 offset) {
         max_offset = std::max(max_offset, static_cast<u32>(offset));
     }
@@ -252,6 +257,10 @@ public:
         return max_offset + sizeof(float);
     }
 
+    u32 GetMaxOffset() const {
+        return max_offset;
+    }
+
 private:
     u32 max_offset{};
     bool is_indirect{};

src/video_core/video_core.cpp

@@ -11,8 +11,9 @@
 
 namespace VideoCore {
 
-std::unique_ptr<RendererBase> CreateRenderer(Core::Frontend::EmuWindow& emu_window) {
+std::unique_ptr<RendererBase> CreateRenderer(Core::Frontend::EmuWindow& emu_window,
-    return std::make_unique<OpenGL::RendererOpenGL>(emu_window);
+                                             Core::System& system) {
+    return std::make_unique<OpenGL::RendererOpenGL>(emu_window, system);
 }
 
 u16 GetResolutionScaleFactor(const RendererBase& renderer) {

src/video_core/video_core.h

@@ -6,6 +6,10 @@
 
 #include <memory>
 
+namespace Core {
+class System;
+}
+
 namespace Core::Frontend {
 class EmuWindow;
 }
@@ -20,7 +24,8 @@ class RendererBase;
  * @note The returned renderer instance is simply allocated. Its Init()
  *       function still needs to be called to fully complete its setup.
  */
-std::unique_ptr<RendererBase> CreateRenderer(Core::Frontend::EmuWindow& emu_window);
+std::unique_ptr<RendererBase> CreateRenderer(Core::Frontend::EmuWindow& emu_window,
+                                             Core::System& system);
 
 u16 GetResolutionScaleFactor(const RendererBase& renderer);
 

src/yuzu/bootmanager.cpp

@@ -29,6 +29,15 @@ void EmuThread::run() {
 
     stop_run = false;
 
+    emit LoadProgress(VideoCore::LoadCallbackStage::Prepare, 0, 0);
+
+    Core::System::GetInstance().Renderer().Rasterizer().LoadDiskResources(
+        stop_run, [this](VideoCore::LoadCallbackStage stage, std::size_t value, std::size_t total) {
+            emit LoadProgress(stage, value, total);
+        });
+
+    emit LoadProgress(VideoCore::LoadCallbackStage::Complete, 0, 0);
+
     // holds whether the cpu was running during the last iteration,
     // so that the DebugModeLeft signal can be emitted before the
     // next execution step

src/yuzu/bootmanager.h

@@ -22,6 +22,10 @@ class GGLWidgetInternal;
 class GMainWindow;
 class GRenderWindow;
 
+namespace VideoCore {
+enum class LoadCallbackStage;
+}
+
 class EmuThread : public QThread {
     Q_OBJECT
 
@@ -75,7 +79,7 @@ public:
 private:
     bool exec_step = false;
     bool running = false;
-    std::atomic<bool> stop_run{false};
+    std::atomic_bool stop_run{false};
     std::mutex running_mutex;
     std::condition_variable running_cv;
 
@@ -101,6 +105,8 @@ signals:
     void DebugModeLeft();
 
     void ErrorThrown(Core::System::ResultStatus, std::string);
+
+    void LoadProgress(VideoCore::LoadCallbackStage stage, std::size_t value, std::size_t total);
 };
 
 class GRenderWindow : public QWidget, public Core::Frontend::EmuWindow {

src/yuzu/configuration/config.cpp

@@ -370,6 +370,8 @@ void Config::ReadValues() {
     Settings::values.resolution_factor = qt_config->value("resolution_factor", 1.0).toFloat();
     Settings::values.use_frame_limit = qt_config->value("use_frame_limit", true).toBool();
     Settings::values.frame_limit = qt_config->value("frame_limit", 100).toInt();
+    Settings::values.use_disk_shader_cache =
+        qt_config->value("use_disk_shader_cache", false).toBool();
     Settings::values.use_accurate_gpu_emulation =
         qt_config->value("use_accurate_gpu_emulation", false).toBool();
 
@@ -629,6 +631,7 @@ void Config::SaveValues() {
     qt_config->setValue("resolution_factor", (double)Settings::values.resolution_factor);
     qt_config->setValue("use_frame_limit", Settings::values.use_frame_limit);
     qt_config->setValue("frame_limit", Settings::values.frame_limit);
+    qt_config->setValue("use_disk_shader_cache", Settings::values.use_disk_shader_cache);
     qt_config->setValue("use_accurate_gpu_emulation", Settings::values.use_accurate_gpu_emulation);
 
     // Cast to double because Qt's written float values are not human-readable

src/yuzu/configuration/configure_graphics.cpp

@@ -73,6 +73,7 @@ void ConfigureGraphics::setConfiguration() {
         static_cast<int>(FromResolutionFactor(Settings::values.resolution_factor)));
     ui->toggle_frame_limit->setChecked(Settings::values.use_frame_limit);
     ui->frame_limit->setValue(Settings::values.frame_limit);
+    ui->use_disk_shader_cache->setChecked(Settings::values.use_disk_shader_cache);
     ui->use_accurate_gpu_emulation->setChecked(Settings::values.use_accurate_gpu_emulation);
     UpdateBackgroundColorButton(QColor::fromRgbF(Settings::values.bg_red, Settings::values.bg_green,
                                                  Settings::values.bg_blue));
@@ -83,6 +84,7 @@ void ConfigureGraphics::applyConfiguration() {
         ToResolutionFactor(static_cast<Resolution>(ui->resolution_factor_combobox->currentIndex()));
     Settings::values.use_frame_limit = ui->toggle_frame_limit->isChecked();
     Settings::values.frame_limit = ui->frame_limit->value();
+    Settings::values.use_disk_shader_cache = ui->use_disk_shader_cache->isChecked();
     Settings::values.use_accurate_gpu_emulation = ui->use_accurate_gpu_emulation->isChecked();
     Settings::values.bg_red = static_cast<float>(bg_color.redF());
     Settings::values.bg_green = static_cast<float>(bg_color.greenF());

src/yuzu/configuration/configure_graphics.ui

@@ -49,6 +49,13 @@
           </item>
          </layout>
         </item>
+        <item>
+         <widget class="QCheckBox" name="use_disk_shader_cache">
+          <property name="text">
+           <string>Use disk shader cache</string>
+          </property>
+         </widget>
+        </item>
         <item>
          <widget class="QCheckBox" name="use_accurate_gpu_emulation">
           <property name="text">

src/yuzu/loading_screen.cpp

@@ -43,6 +43,7 @@ QProgressBar {
 }
 QProgressBar::chunk {
   background-color: #0ab9e6;
+  width: 1px;
 })";
 
 constexpr const char PROGRESSBAR_STYLE_BUILD[] = R"(
@@ -54,6 +55,7 @@ QProgressBar {
 }
 QProgressBar::chunk {
   background-color: #ff3c28;
+  width: 1px;
 })";
 
 constexpr const char PROGRESSBAR_STYLE_COMPLETE[] = R"(

src/yuzu/main.cpp

@@ -887,6 +887,9 @@ void GMainWindow::BootGame(const QString& filename) {
     connect(emu_thread.get(), &EmuThread::DebugModeLeft, waitTreeWidget,
             &WaitTreeWidget::OnDebugModeLeft, Qt::BlockingQueuedConnection);
 
+    connect(emu_thread.get(), &EmuThread::LoadProgress, loading_screen,
+            &LoadingScreen::OnLoadProgress, Qt::QueuedConnection);
+
     // Update the GUI
     if (ui.action_Single_Window_Mode->isChecked()) {
         game_list->hide();

src/yuzu_cmd/config.cpp

@@ -350,6 +350,8 @@ void Config::ReadValues() {
     Settings::values.use_frame_limit = sdl2_config->GetBoolean("Renderer", "use_frame_limit", true);
     Settings::values.frame_limit =
         static_cast<u16>(sdl2_config->GetInteger("Renderer", "frame_limit", 100));
+    Settings::values.use_disk_shader_cache =
+        sdl2_config->GetBoolean("Renderer", "use_disk_shader_cache", false);
     Settings::values.use_accurate_gpu_emulation =
         sdl2_config->GetBoolean("Renderer", "use_accurate_gpu_emulation", false);
 

src/yuzu_cmd/default_ini.h

@@ -110,6 +110,10 @@ use_frame_limit =
 # 1 - 9999: Speed limit as a percentage of target game speed. 100 (default)
 frame_limit =
 
+# Whether to use disk based shader cache
+# 0 (default): Off, 1 : On
+use_disk_shader_cache =
+
 # Whether to use accurate GPU emulation
 # 0 (default): Off (fast), 1 : On (slow)
 use_accurate_gpu_emulation =

src/yuzu_cmd/yuzu.cpp

@@ -28,6 +28,7 @@
 #include "core/loader/loader.h"
 #include "core/settings.h"
 #include "core/telemetry_session.h"
+#include "video_core/renderer_base.h"
 #include "yuzu_cmd/config.h"
 #include "yuzu_cmd/emu_window/emu_window_sdl2.h"
 
@@ -217,6 +218,8 @@ int main(int argc, char** argv) {
 
     Core::Telemetry().AddField(Telemetry::FieldType::App, "Frontend", "SDL");
 
+    system.Renderer().Rasterizer().LoadDiskResources();
+
     while (emu_window->IsOpen()) {
         system.RunLoop();
     }