diff --git a/src/common/settings.cpp b/src/common/settings.cpp
index 12fdb0f9b..bc2c8c7d7 100644
--- a/src/common/settings.cpp
+++ b/src/common/settings.cpp
@@ -109,10 +109,12 @@ float Volume() {
 void UpdateRescalingInfo() {
     const auto setup = values.resolution_setup.GetValue();
     auto& info = values.resolution_info;
+    info.downscale = false;
     switch (setup) {
     case ResolutionSetup::Res1_2X:
         info.up_scale = 1;
         info.down_shift = 1;
+        info.downscale = true;
         break;
     case ResolutionSetup::Res1X:
         info.up_scale = 1;
diff --git a/src/common/settings.h b/src/common/settings.h
index 09f7cdd84..a09db0822 100644
--- a/src/common/settings.h
+++ b/src/common/settings.h
@@ -72,6 +72,7 @@ struct ResolutionScalingInfo {
     f32 up_factor{1.0f};
     f32 down_factor{1.0f};
     bool active{};
+    bool downscale{};
 
     s32 ScaleUp(s32 value) const {
         if (value == 0) {
diff --git a/src/video_core/texture_cache/image_info.cpp b/src/video_core/texture_cache/image_info.cpp
index 7fa8fd4fe..d8e414247 100644
--- a/src/video_core/texture_cache/image_info.cpp
+++ b/src/video_core/texture_cache/image_info.cpp
@@ -102,6 +102,7 @@ ImageInfo::ImageInfo(const TICEntry& config) noexcept {
         layer_stride = CalculateLayerStride(*this);
         maybe_unaligned_layer_stride = CalculateLayerSize(*this);
         rescaleable &= (block.depth == 0) && resources.levels == 1;
+        downscaleable = size.height > 512;
     }
 }
 
@@ -135,6 +136,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Maxwell3D::Regs& regs, size_t index)
         size.depth = rt.depth;
     } else {
         rescaleable = block.depth == 0 && size.height > 256;
+        downscaleable = size.height > 512;
         type = ImageType::e2D;
         resources.layers = rt.depth;
     }
@@ -164,6 +166,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Maxwell3D::Regs& regs) noexcept {
         size.depth = regs.zeta_depth;
     } else {
         rescaleable = block.depth == 0 && size.height > 256;
+        downscaleable = size.height > 512;
         type = ImageType::e2D;
         resources.layers = regs.zeta_depth;
     }
@@ -197,6 +200,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Fermi2D::Surface& config) noexcept {
             .depth = 1,
         };
         rescaleable = block.depth == 0 && size.height > 256;
+        downscaleable = size.height > 512;
     }
 }
 
diff --git a/src/video_core/texture_cache/image_info.h b/src/video_core/texture_cache/image_info.h
index e874d2870..5932dcaba 100644
--- a/src/video_core/texture_cache/image_info.h
+++ b/src/video_core/texture_cache/image_info.h
@@ -34,6 +34,7 @@ struct ImageInfo {
     u32 num_samples = 1;
     u32 tile_width_spacing = 0;
     bool rescaleable = false;
+    bool downscaleable = false;
 };
 
 } // namespace VideoCommon
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index c1fb12679..261cb6c48 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -798,6 +798,9 @@ bool TextureCache<P>::ImageCanRescale(ImageBase& image) {
     if (!image.info.rescaleable || True(image.flags & ImageFlagBits::Blacklisted)) {
         return false;
     }
+    if (Settings::values.resolution_info.downscale && !image.info.downscaleable) {
+        return false;
+    }
     if (True(image.flags & (ImageFlagBits::Rescaled | ImageFlagBits::CheckingRescalable))) {
         return true;
     }