diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index dd9553806..26ab857c9 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -878,10 +878,6 @@ bool TextureCache<P>::ScaleDown(Image& image) {
     if (!rescaled) {
         return false;
     }
-    const bool has_copy = image.HasScaled();
-    if (!has_copy) {
-        total_used_memory -= GetScaledImageSizeBytes(image);
-    }
     InvalidateScale(image);
     return true;
 }