Merge pull request #10084 from FernandoS27/yuzu-goes-broom-broom
Y.F.C Buffer Cache Revamp
This commit is contained in:
commit
47938541c2
|
@ -15,7 +15,7 @@ add_executable(tests
|
|||
core/core_timing.cpp
|
||||
core/internal_network/network.cpp
|
||||
precompiled_headers.h
|
||||
video_core/buffer_base.cpp
|
||||
video_core/memory_tracker.cpp
|
||||
input_common/calibration_configuration_job.cpp
|
||||
)
|
||||
|
||||
|
|
|
@ -1,549 +0,0 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include <stdexcept>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
#include "common/alignment.h"
|
||||
#include "common/common_types.h"
|
||||
#include "video_core/buffer_cache/buffer_base.h"
|
||||
|
||||
namespace {
|
||||
using VideoCommon::BufferBase;
|
||||
using Range = std::pair<u64, u64>;
|
||||
|
||||
constexpr u64 PAGE = 4096;
|
||||
constexpr u64 WORD = 4096 * 64;
|
||||
|
||||
constexpr VAddr c = 0x1328914000;
|
||||
|
||||
class RasterizerInterface {
|
||||
public:
|
||||
void UpdatePagesCachedCount(VAddr addr, u64 size, int delta) {
|
||||
const u64 page_start{addr >> Core::Memory::YUZU_PAGEBITS};
|
||||
const u64 page_end{(addr + size + Core::Memory::YUZU_PAGESIZE - 1) >>
|
||||
Core::Memory::YUZU_PAGEBITS};
|
||||
for (u64 page = page_start; page < page_end; ++page) {
|
||||
int& value = page_table[page];
|
||||
value += delta;
|
||||
if (value < 0) {
|
||||
throw std::logic_error{"negative page"};
|
||||
}
|
||||
if (value == 0) {
|
||||
page_table.erase(page);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] int Count(VAddr addr) const noexcept {
|
||||
const auto it = page_table.find(addr >> Core::Memory::YUZU_PAGEBITS);
|
||||
return it == page_table.end() ? 0 : it->second;
|
||||
}
|
||||
|
||||
[[nodiscard]] unsigned Count() const noexcept {
|
||||
unsigned count = 0;
|
||||
for (const auto& [index, value] : page_table) {
|
||||
count += value;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private:
|
||||
std::unordered_map<u64, int> page_table;
|
||||
};
|
||||
} // Anonymous namespace
|
||||
|
||||
TEST_CASE("BufferBase: Small buffer", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == WORD / PAGE);
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c, WORD) == Range{0, 0});
|
||||
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE, 1);
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c, WORD) == Range{PAGE * 1, PAGE * 2});
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Large buffer", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 32);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 32);
|
||||
buffer.MarkRegionAsCpuModified(c + 4096, WORD * 4);
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c, WORD + PAGE * 2) == Range{PAGE, WORD + PAGE * 2});
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c + PAGE * 2, PAGE * 6) == Range{PAGE * 2, PAGE * 8});
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c, WORD * 32) == Range{PAGE, WORD * 4 + PAGE});
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 4, PAGE) == Range{WORD * 4, WORD * 4 + PAGE});
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 3 + PAGE * 63, PAGE) ==
|
||||
Range{WORD * 3 + PAGE * 63, WORD * 4});
|
||||
|
||||
buffer.MarkRegionAsCpuModified(c + WORD * 5 + PAGE * 6, PAGE);
|
||||
buffer.MarkRegionAsCpuModified(c + WORD * 5 + PAGE * 8, PAGE);
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 5, WORD) ==
|
||||
Range{WORD * 5 + PAGE * 6, WORD * 5 + PAGE * 9});
|
||||
|
||||
buffer.UnmarkRegionAsCpuModified(c + WORD * 5 + PAGE * 8, PAGE);
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 5, WORD) ==
|
||||
Range{WORD * 5 + PAGE * 6, WORD * 5 + PAGE * 7});
|
||||
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE, WORD * 31 + PAGE * 63);
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c, WORD * 32) == Range{PAGE, WORD * 32});
|
||||
|
||||
buffer.UnmarkRegionAsCpuModified(c + PAGE * 4, PAGE);
|
||||
buffer.UnmarkRegionAsCpuModified(c + PAGE * 6, PAGE);
|
||||
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 32);
|
||||
REQUIRE(buffer.ModifiedCpuRegion(c, WORD * 32) == Range{0, 0});
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Rasterizer counting", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, PAGE * 2);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
buffer.UnmarkRegionAsCpuModified(c, PAGE);
|
||||
REQUIRE(rasterizer.Count() == 1);
|
||||
buffer.MarkRegionAsCpuModified(c, PAGE * 2);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
buffer.UnmarkRegionAsCpuModified(c, PAGE);
|
||||
buffer.UnmarkRegionAsCpuModified(c + PAGE, PAGE);
|
||||
REQUIRE(rasterizer.Count() == 2);
|
||||
buffer.MarkRegionAsCpuModified(c, PAGE * 2);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Basic range", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
buffer.MarkRegionAsCpuModified(c, PAGE);
|
||||
int num = 0;
|
||||
buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == 0U);
|
||||
REQUIRE(size == PAGE);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1U);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Border upload", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 2);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 2);
|
||||
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
buffer.ForEachUploadRange(c, WORD * 2, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD - PAGE);
|
||||
REQUIRE(size == PAGE * 2);
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Border upload range", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 2);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 2);
|
||||
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
buffer.ForEachUploadRange(c + WORD - PAGE, PAGE * 2, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD - PAGE);
|
||||
REQUIRE(size == PAGE * 2);
|
||||
});
|
||||
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
buffer.ForEachUploadRange(c + WORD - PAGE, PAGE, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD - PAGE);
|
||||
REQUIRE(size == PAGE);
|
||||
});
|
||||
buffer.ForEachUploadRange(c + WORD, PAGE, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD);
|
||||
REQUIRE(size == PAGE);
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Border upload partial range", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 2);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 2);
|
||||
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
buffer.ForEachUploadRange(c + WORD - 1, 2, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD - PAGE);
|
||||
REQUIRE(size == PAGE * 2);
|
||||
});
|
||||
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
buffer.ForEachUploadRange(c + WORD - 1, 1, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD - PAGE);
|
||||
REQUIRE(size == PAGE);
|
||||
});
|
||||
buffer.ForEachUploadRange(c + WORD + 50, 1, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD);
|
||||
REQUIRE(size == PAGE);
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Partial word uploads", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, 0x9d000);
|
||||
int num = 0;
|
||||
buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == 0U);
|
||||
REQUIRE(size == WORD);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1);
|
||||
buffer.ForEachUploadRange(c + WORD, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD);
|
||||
REQUIRE(size == WORD);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 2);
|
||||
buffer.ForEachUploadRange(c + 0x79000, 0x24000, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD * 2);
|
||||
REQUIRE(size == PAGE * 0x1d);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 3);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Partial page upload", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
int num = 0;
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 2, PAGE);
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 9, PAGE);
|
||||
buffer.ForEachUploadRange(c, PAGE * 3, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == PAGE * 2);
|
||||
REQUIRE(size == PAGE);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1);
|
||||
buffer.ForEachUploadRange(c + PAGE * 7, PAGE * 3, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == PAGE * 9);
|
||||
REQUIRE(size == PAGE);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 2);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Partial page upload with multiple words on the right") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 8);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 8);
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 13, WORD * 7);
|
||||
int num = 0;
|
||||
buffer.ForEachUploadRange(c + PAGE * 10, WORD * 7, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == PAGE * 13);
|
||||
REQUIRE(size == WORD * 7 - PAGE * 3);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1);
|
||||
buffer.ForEachUploadRange(c + PAGE, WORD * 8, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD * 7 + PAGE * 10);
|
||||
REQUIRE(size == PAGE * 3);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 2);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Partial page upload with multiple words on the left", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 8);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 8);
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 13, WORD * 7);
|
||||
int num = 0;
|
||||
buffer.ForEachUploadRange(c + PAGE * 16, WORD * 7, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == PAGE * 16);
|
||||
REQUIRE(size == WORD * 7 - PAGE * 3);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1);
|
||||
buffer.ForEachUploadRange(c + PAGE, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == PAGE * 13);
|
||||
REQUIRE(size == PAGE * 3);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 2);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Partial page upload with multiple words in the middle", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 8);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 8);
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 13, PAGE * 140);
|
||||
int num = 0;
|
||||
buffer.ForEachUploadRange(c + PAGE * 16, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == PAGE * 16);
|
||||
REQUIRE(size == WORD);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1);
|
||||
buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == PAGE * 13);
|
||||
REQUIRE(size == PAGE * 3);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 2);
|
||||
buffer.ForEachUploadRange(c, WORD * 8, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD + PAGE * 16);
|
||||
REQUIRE(size == PAGE * 73);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 3);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Empty right bits", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 2048);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 2048);
|
||||
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
buffer.ForEachUploadRange(c, WORD * 2048, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == WORD - PAGE);
|
||||
REQUIRE(size == PAGE * 2);
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Out of bound ranges 1", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
buffer.MarkRegionAsCpuModified(c, PAGE);
|
||||
int num = 0;
|
||||
buffer.ForEachUploadRange(c - WORD, WORD, [&](u64 offset, u64 size) { ++num; });
|
||||
buffer.ForEachUploadRange(c + WORD, WORD, [&](u64 offset, u64 size) { ++num; });
|
||||
buffer.ForEachUploadRange(c - PAGE, PAGE, [&](u64 offset, u64 size) { ++num; });
|
||||
REQUIRE(num == 0);
|
||||
buffer.ForEachUploadRange(c - PAGE, PAGE * 2, [&](u64 offset, u64 size) { ++num; });
|
||||
REQUIRE(num == 1);
|
||||
buffer.MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Out of bound ranges 2", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, 0x22000);
|
||||
REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c + 0x22000, PAGE));
|
||||
REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c + 0x28000, PAGE));
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c + 0x21100, PAGE - 0x100));
|
||||
REQUIRE(rasterizer.Count() == 1);
|
||||
REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c - 0x1000, PAGE * 2));
|
||||
buffer.UnmarkRegionAsCpuModified(c - 0x3000, PAGE * 2);
|
||||
buffer.UnmarkRegionAsCpuModified(c - 0x2000, PAGE * 2);
|
||||
REQUIRE(rasterizer.Count() == 2);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Out of bound ranges 3", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, 0x310720);
|
||||
buffer.UnmarkRegionAsCpuModified(c, 0x310720);
|
||||
REQUIRE(rasterizer.Count(c) == 1);
|
||||
REQUIRE(rasterizer.Count(c + PAGE) == 1);
|
||||
REQUIRE(rasterizer.Count(c + WORD) == 1);
|
||||
REQUIRE(rasterizer.Count(c + WORD + PAGE) == 1);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Sparse regions 1", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 1, PAGE);
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 3, PAGE * 4);
|
||||
buffer.ForEachUploadRange(c, WORD, [i = 0](u64 offset, u64 size) mutable {
|
||||
static constexpr std::array<u64, 2> offsets{PAGE, PAGE * 3};
|
||||
static constexpr std::array<u64, 2> sizes{PAGE, PAGE * 4};
|
||||
REQUIRE(offset == offsets.at(i));
|
||||
REQUIRE(size == sizes.at(i));
|
||||
++i;
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Sparse regions 2", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, 0x22000);
|
||||
buffer.UnmarkRegionAsCpuModified(c, 0x22000);
|
||||
REQUIRE(rasterizer.Count() == 0x22);
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 0x1B, PAGE);
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 0x21, PAGE);
|
||||
buffer.ForEachUploadRange(c, WORD, [i = 0](u64 offset, u64 size) mutable {
|
||||
static constexpr std::array<u64, 2> offsets{PAGE * 0x1B, PAGE * 0x21};
|
||||
static constexpr std::array<u64, 2> sizes{PAGE, PAGE};
|
||||
REQUIRE(offset == offsets.at(i));
|
||||
REQUIRE(size == sizes.at(i));
|
||||
++i;
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Single page modified range", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, PAGE);
|
||||
REQUIRE(buffer.IsRegionCpuModified(c, PAGE));
|
||||
buffer.UnmarkRegionAsCpuModified(c, PAGE);
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c, PAGE));
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Two page modified range", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, PAGE * 2);
|
||||
REQUIRE(buffer.IsRegionCpuModified(c, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c, PAGE * 2));
|
||||
buffer.UnmarkRegionAsCpuModified(c, PAGE);
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c, PAGE));
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Multi word modified ranges", "[video_core]") {
|
||||
for (int offset = 0; offset < 4; ++offset) {
|
||||
const VAddr address = c + WORD * offset;
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, address, WORD * 4);
|
||||
REQUIRE(buffer.IsRegionCpuModified(address, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 48, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 56, PAGE));
|
||||
|
||||
buffer.UnmarkRegionAsCpuModified(address + PAGE * 32, PAGE);
|
||||
REQUIRE(buffer.IsRegionCpuModified(address + PAGE, WORD));
|
||||
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 31, PAGE));
|
||||
REQUIRE(!buffer.IsRegionCpuModified(address + PAGE * 32, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 33, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 31, PAGE * 2));
|
||||
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 32, PAGE * 2));
|
||||
|
||||
buffer.UnmarkRegionAsCpuModified(address + PAGE * 33, PAGE);
|
||||
REQUIRE(!buffer.IsRegionCpuModified(address + PAGE * 32, PAGE * 2));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Single page in large buffer", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 16);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 16);
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c, WORD * 16));
|
||||
|
||||
buffer.MarkRegionAsCpuModified(c + WORD * 12 + PAGE * 8, PAGE);
|
||||
REQUIRE(buffer.IsRegionCpuModified(c, WORD * 16));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 10, WORD * 2));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 11, WORD * 2));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12, WORD * 2));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 4, PAGE * 8));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 6, PAGE * 8));
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 6, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 7, PAGE * 2));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 8, PAGE * 2));
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Out of bounds region query") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 16);
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c - PAGE, PAGE));
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c - PAGE * 2, PAGE));
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + WORD * 16, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 16 - PAGE, WORD * 64));
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + WORD * 16, WORD * 64));
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Wrap word regions") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD * 2);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD * 2);
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 63, PAGE * 2);
|
||||
REQUIRE(buffer.IsRegionCpuModified(c, WORD * 2));
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 62, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 63, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 64, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 63, PAGE * 2));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 63, PAGE * 8));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 60, PAGE * 8));
|
||||
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 127, WORD * 16));
|
||||
buffer.MarkRegionAsCpuModified(c + PAGE * 127, PAGE);
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 127, WORD * 16));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 127, PAGE));
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 126, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 126, PAGE * 2));
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 128, WORD * 16));
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Unaligned page region query") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
buffer.MarkRegionAsCpuModified(c + 4000, 1000);
|
||||
REQUIRE(buffer.IsRegionCpuModified(c, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + 4000, 1000));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + 4000, 1));
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Cached write") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
buffer.CachedCpuWrite(c + PAGE, PAGE);
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
buffer.FlushCachedWrites();
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
buffer.MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Multiple cached write") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
buffer.CachedCpuWrite(c + PAGE, PAGE);
|
||||
buffer.CachedCpuWrite(c + PAGE * 3, PAGE);
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 3, PAGE));
|
||||
buffer.FlushCachedWrites();
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 3, PAGE));
|
||||
buffer.MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Cached write unmarked") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
buffer.CachedCpuWrite(c + PAGE, PAGE);
|
||||
buffer.UnmarkRegionAsCpuModified(c + PAGE, PAGE);
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
buffer.FlushCachedWrites();
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
buffer.MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Cached write iterated") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
buffer.CachedCpuWrite(c + PAGE, PAGE);
|
||||
int num = 0;
|
||||
buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; });
|
||||
REQUIRE(num == 0);
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
buffer.FlushCachedWrites();
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
buffer.MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("BufferBase: Cached write downloads") {
|
||||
RasterizerInterface rasterizer;
|
||||
BufferBase buffer(rasterizer, c, WORD);
|
||||
buffer.UnmarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 64);
|
||||
buffer.CachedCpuWrite(c + PAGE, PAGE);
|
||||
REQUIRE(rasterizer.Count() == 63);
|
||||
buffer.MarkRegionAsGpuModified(c + PAGE, PAGE);
|
||||
int num = 0;
|
||||
buffer.ForEachDownloadRangeAndClear(c, WORD, [&](u64 offset, u64 size) { ++num; });
|
||||
buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; });
|
||||
REQUIRE(num == 0);
|
||||
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(!buffer.IsRegionGpuModified(c + PAGE, PAGE));
|
||||
buffer.FlushCachedWrites();
|
||||
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(!buffer.IsRegionGpuModified(c + PAGE, PAGE));
|
||||
buffer.MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
|
@ -0,0 +1,549 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
#include "common/alignment.h"
|
||||
#include "common/common_types.h"
|
||||
#include "video_core/buffer_cache/memory_tracker_base.h"
|
||||
|
||||
namespace {
|
||||
using Range = std::pair<u64, u64>;
|
||||
|
||||
constexpr u64 PAGE = 4096;
|
||||
constexpr u64 WORD = 4096 * 64;
|
||||
constexpr u64 HIGH_PAGE_BITS = 22;
|
||||
constexpr u64 HIGH_PAGE_SIZE = 1ULL << HIGH_PAGE_BITS;
|
||||
|
||||
constexpr VAddr c = 16 * HIGH_PAGE_SIZE;
|
||||
|
||||
class RasterizerInterface {
|
||||
public:
|
||||
void UpdatePagesCachedCount(VAddr addr, u64 size, int delta) {
|
||||
const u64 page_start{addr >> Core::Memory::YUZU_PAGEBITS};
|
||||
const u64 page_end{(addr + size + Core::Memory::YUZU_PAGESIZE - 1) >>
|
||||
Core::Memory::YUZU_PAGEBITS};
|
||||
for (u64 page = page_start; page < page_end; ++page) {
|
||||
int& value = page_table[page];
|
||||
value += delta;
|
||||
if (value < 0) {
|
||||
throw std::logic_error{"negative page"};
|
||||
}
|
||||
if (value == 0) {
|
||||
page_table.erase(page);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] int Count(VAddr addr) const noexcept {
|
||||
const auto it = page_table.find(addr >> Core::Memory::YUZU_PAGEBITS);
|
||||
return it == page_table.end() ? 0 : it->second;
|
||||
}
|
||||
|
||||
[[nodiscard]] unsigned Count() const noexcept {
|
||||
unsigned count = 0;
|
||||
for (const auto& [index, value] : page_table) {
|
||||
count += value;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private:
|
||||
std::unordered_map<u64, int> page_table;
|
||||
};
|
||||
} // Anonymous namespace
|
||||
|
||||
using MemoryTracker = VideoCommon::MemoryTrackerBase<RasterizerInterface>;
|
||||
|
||||
TEST_CASE("MemoryTracker: Small region", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == WORD / PAGE);
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD) == Range{0, 0});
|
||||
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE, 1);
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD) == Range{c + PAGE * 1, c + PAGE * 2});
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Large region", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 32);
|
||||
memory_track->MarkRegionAsCpuModified(c + 4096, WORD * 4);
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD + PAGE * 2) ==
|
||||
Range{c + PAGE, c + WORD + PAGE * 2});
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c + PAGE * 2, PAGE * 6) ==
|
||||
Range{c + PAGE * 2, c + PAGE * 8});
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD * 32) == Range{c + PAGE, c + WORD * 4 + PAGE});
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c + WORD * 4, PAGE) ==
|
||||
Range{c + WORD * 4, c + WORD * 4 + PAGE});
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c + WORD * 3 + PAGE * 63, PAGE) ==
|
||||
Range{c + WORD * 3 + PAGE * 63, c + WORD * 4});
|
||||
|
||||
memory_track->MarkRegionAsCpuModified(c + WORD * 5 + PAGE * 6, PAGE);
|
||||
memory_track->MarkRegionAsCpuModified(c + WORD * 5 + PAGE * 8, PAGE);
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c + WORD * 5, WORD) ==
|
||||
Range{c + WORD * 5 + PAGE * 6, c + WORD * 5 + PAGE * 9});
|
||||
|
||||
memory_track->UnmarkRegionAsCpuModified(c + WORD * 5 + PAGE * 8, PAGE);
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c + WORD * 5, WORD) ==
|
||||
Range{c + WORD * 5 + PAGE * 6, c + WORD * 5 + PAGE * 7});
|
||||
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE, WORD * 31 + PAGE * 63);
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD * 32) == Range{c + PAGE, c + WORD * 32});
|
||||
|
||||
memory_track->UnmarkRegionAsCpuModified(c + PAGE * 4, PAGE);
|
||||
memory_track->UnmarkRegionAsCpuModified(c + PAGE * 6, PAGE);
|
||||
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 32);
|
||||
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD * 32) == Range{0, 0});
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Rasterizer counting", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
memory_track->UnmarkRegionAsCpuModified(c, PAGE);
|
||||
REQUIRE(rasterizer.Count() == 1);
|
||||
memory_track->MarkRegionAsCpuModified(c, PAGE * 2);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
memory_track->UnmarkRegionAsCpuModified(c, PAGE);
|
||||
memory_track->UnmarkRegionAsCpuModified(c + PAGE, PAGE);
|
||||
REQUIRE(rasterizer.Count() == 2);
|
||||
memory_track->MarkRegionAsCpuModified(c, PAGE * 2);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Basic range", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD);
|
||||
memory_track->MarkRegionAsCpuModified(c, PAGE);
|
||||
int num = 0;
|
||||
memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c);
|
||||
REQUIRE(size == PAGE);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1U);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Border upload", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 2);
|
||||
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
memory_track->ForEachUploadRange(c, WORD * 2, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD - PAGE);
|
||||
REQUIRE(size == PAGE * 2);
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Border upload range", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 2);
|
||||
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
memory_track->ForEachUploadRange(c + WORD - PAGE, PAGE * 2, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD - PAGE);
|
||||
REQUIRE(size == PAGE * 2);
|
||||
});
|
||||
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
memory_track->ForEachUploadRange(c + WORD - PAGE, PAGE, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD - PAGE);
|
||||
REQUIRE(size == PAGE);
|
||||
});
|
||||
memory_track->ForEachUploadRange(c + WORD, PAGE, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD);
|
||||
REQUIRE(size == PAGE);
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Border upload partial range", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 2);
|
||||
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
memory_track->ForEachUploadRange(c + WORD - 1, 2, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD - PAGE);
|
||||
REQUIRE(size == PAGE * 2);
|
||||
});
|
||||
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
memory_track->ForEachUploadRange(c + WORD - 1, 1, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD - PAGE);
|
||||
REQUIRE(size == PAGE);
|
||||
});
|
||||
memory_track->ForEachUploadRange(c + WORD + 50, 1, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD);
|
||||
REQUIRE(size == PAGE);
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Partial word uploads", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
int num = 0;
|
||||
memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c);
|
||||
REQUIRE(size == WORD);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1);
|
||||
memory_track->ForEachUploadRange(c + WORD, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD);
|
||||
REQUIRE(size == WORD);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 2);
|
||||
memory_track->ForEachUploadRange(c + 0x79000, 0x24000, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD * 2);
|
||||
REQUIRE(size == PAGE * 0x1d);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 3);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Partial page upload", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD);
|
||||
int num = 0;
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 2, PAGE);
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 9, PAGE);
|
||||
memory_track->ForEachUploadRange(c, PAGE * 3, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + PAGE * 2);
|
||||
REQUIRE(size == PAGE);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1);
|
||||
memory_track->ForEachUploadRange(c + PAGE * 7, PAGE * 3, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + PAGE * 9);
|
||||
REQUIRE(size == PAGE);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 2);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Partial page upload with multiple words on the right") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 9);
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 13, WORD * 7);
|
||||
int num = 0;
|
||||
memory_track->ForEachUploadRange(c + PAGE * 10, WORD * 7, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + PAGE * 13);
|
||||
REQUIRE(size == WORD * 7 - PAGE * 3);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1);
|
||||
memory_track->ForEachUploadRange(c + PAGE, WORD * 8, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD * 7 + PAGE * 10);
|
||||
REQUIRE(size == PAGE * 3);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 2);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Partial page upload with multiple words on the left", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 8);
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 13, WORD * 7);
|
||||
int num = 0;
|
||||
memory_track->ForEachUploadRange(c + PAGE * 16, WORD * 7, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + PAGE * 16);
|
||||
REQUIRE(size == WORD * 7 - PAGE * 3);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1);
|
||||
memory_track->ForEachUploadRange(c + PAGE, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + PAGE * 13);
|
||||
REQUIRE(size == PAGE * 3);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 2);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Partial page upload with multiple words in the middle", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 8);
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 13, PAGE * 140);
|
||||
int num = 0;
|
||||
memory_track->ForEachUploadRange(c + PAGE * 16, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + PAGE * 16);
|
||||
REQUIRE(size == WORD);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 1);
|
||||
memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + PAGE * 13);
|
||||
REQUIRE(size == PAGE * 3);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 2);
|
||||
memory_track->ForEachUploadRange(c, WORD * 8, [&](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD + PAGE * 16);
|
||||
REQUIRE(size == PAGE * 73);
|
||||
++num;
|
||||
});
|
||||
REQUIRE(num == 3);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Empty right bits", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 2048);
|
||||
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
|
||||
memory_track->ForEachUploadRange(c, WORD * 2048, [](u64 offset, u64 size) {
|
||||
REQUIRE(offset == c + WORD - PAGE);
|
||||
REQUIRE(size == PAGE * 2);
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Out of bound ranges 1", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c - WORD, 3 * WORD);
|
||||
memory_track->MarkRegionAsCpuModified(c, PAGE);
|
||||
REQUIRE(rasterizer.Count() == (3 * WORD - PAGE) / PAGE);
|
||||
int num = 0;
|
||||
memory_track->ForEachUploadRange(c - WORD, WORD, [&](u64 offset, u64 size) { ++num; });
|
||||
memory_track->ForEachUploadRange(c + WORD, WORD, [&](u64 offset, u64 size) { ++num; });
|
||||
memory_track->ForEachUploadRange(c - PAGE, PAGE, [&](u64 offset, u64 size) { ++num; });
|
||||
REQUIRE(num == 0);
|
||||
memory_track->ForEachUploadRange(c - PAGE, PAGE * 2, [&](u64 offset, u64 size) { ++num; });
|
||||
REQUIRE(num == 1);
|
||||
memory_track->MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 2 * WORD / PAGE);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Out of bound ranges 2", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
REQUIRE_NOTHROW(memory_track->UnmarkRegionAsCpuModified(c + 0x22000, PAGE));
|
||||
REQUIRE_NOTHROW(memory_track->UnmarkRegionAsCpuModified(c + 0x28000, PAGE));
|
||||
REQUIRE(rasterizer.Count() == 2);
|
||||
REQUIRE_NOTHROW(memory_track->UnmarkRegionAsCpuModified(c + 0x21100, PAGE - 0x100));
|
||||
REQUIRE(rasterizer.Count() == 3);
|
||||
REQUIRE_NOTHROW(memory_track->UnmarkRegionAsCpuModified(c - PAGE, PAGE * 2));
|
||||
memory_track->UnmarkRegionAsCpuModified(c - PAGE * 3, PAGE * 2);
|
||||
memory_track->UnmarkRegionAsCpuModified(c - PAGE * 2, PAGE * 2);
|
||||
REQUIRE(rasterizer.Count() == 7);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Out of bound ranges 3", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, 0x310720);
|
||||
REQUIRE(rasterizer.Count(c) == 1);
|
||||
REQUIRE(rasterizer.Count(c + PAGE) == 1);
|
||||
REQUIRE(rasterizer.Count(c + WORD) == 1);
|
||||
REQUIRE(rasterizer.Count(c + WORD + PAGE) == 1);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Sparse regions 1", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD);
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 1, PAGE);
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 3, PAGE * 4);
|
||||
memory_track->ForEachUploadRange(c, WORD, [i = 0](u64 offset, u64 size) mutable {
|
||||
static constexpr std::array<u64, 2> offsets{c + PAGE, c + PAGE * 3};
|
||||
static constexpr std::array<u64, 2> sizes{PAGE, PAGE * 4};
|
||||
REQUIRE(offset == offsets.at(i));
|
||||
REQUIRE(size == sizes.at(i));
|
||||
++i;
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Sparse regions 2", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, PAGE * 0x23);
|
||||
REQUIRE(rasterizer.Count() == 0x23);
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 0x1B, PAGE);
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 0x21, PAGE);
|
||||
memory_track->ForEachUploadRange(c, PAGE * 0x23, [i = 0](u64 offset, u64 size) mutable {
|
||||
static constexpr std::array<u64, 3> offsets{c + PAGE * 0x1B, c + PAGE * 0x21};
|
||||
static constexpr std::array<u64, 3> sizes{PAGE, PAGE};
|
||||
REQUIRE(offset == offsets.at(i));
|
||||
REQUIRE(size == sizes.at(i));
|
||||
++i;
|
||||
});
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Single page modified range", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c, PAGE));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, PAGE);
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c, PAGE));
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Two page modified range", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c, PAGE * 2));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, PAGE);
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c, PAGE));
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Multi word modified ranges", "[video_core]") {
|
||||
for (int offset = 0; offset < 4; ++offset) {
|
||||
const VAddr address = c + WORD * offset;
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(address, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 48, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 56, PAGE));
|
||||
|
||||
memory_track->UnmarkRegionAsCpuModified(address + PAGE * 32, PAGE);
|
||||
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE, WORD));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 31, PAGE));
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(address + PAGE * 32, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 33, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 31, PAGE * 2));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 32, PAGE * 2));
|
||||
|
||||
memory_track->UnmarkRegionAsCpuModified(address + PAGE * 33, PAGE);
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(address + PAGE * 32, PAGE * 2));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Single page in large region", "[video_core]") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 16);
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c, WORD * 16));
|
||||
|
||||
memory_track->MarkRegionAsCpuModified(c + WORD * 12 + PAGE * 8, PAGE);
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c, WORD * 16));
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + WORD * 10, WORD * 2));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 11, WORD * 2));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12, WORD * 2));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 4, PAGE * 8));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 6, PAGE * 8));
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 6, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 7, PAGE * 2));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 8, PAGE * 2));
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Wrap word regions") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD * 32);
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 63, PAGE * 2);
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c, WORD * 2));
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE * 62, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 63, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 64, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 63, PAGE * 2));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 63, PAGE * 8));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 60, PAGE * 8));
|
||||
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE * 127, WORD * 16));
|
||||
memory_track->MarkRegionAsCpuModified(c + PAGE * 127, PAGE);
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 127, WORD * 16));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 127, PAGE));
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE * 126, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 126, PAGE * 2));
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE * 128, WORD * 16));
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Unaligned page region query") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD);
|
||||
memory_track->MarkRegionAsCpuModified(c + 4000, 1000);
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + 4000, 1000));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + 4000, 1));
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Cached write") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD);
|
||||
memory_track->CachedCpuWrite(c + PAGE, c + PAGE);
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
memory_track->FlushCachedWrites();
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
memory_track->MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Multiple cached write") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD);
|
||||
memory_track->CachedCpuWrite(c + PAGE, PAGE);
|
||||
memory_track->CachedCpuWrite(c + PAGE * 3, PAGE);
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE * 3, PAGE));
|
||||
memory_track->FlushCachedWrites();
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 3, PAGE));
|
||||
memory_track->MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Cached write unmarked") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD);
|
||||
memory_track->CachedCpuWrite(c + PAGE, PAGE);
|
||||
memory_track->UnmarkRegionAsCpuModified(c + PAGE, PAGE);
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
memory_track->FlushCachedWrites();
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
memory_track->MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Cached write iterated") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD);
|
||||
memory_track->CachedCpuWrite(c + PAGE, PAGE);
|
||||
int num = 0;
|
||||
memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; });
|
||||
REQUIRE(num == 0);
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
memory_track->FlushCachedWrites();
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
memory_track->MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("MemoryTracker: Cached write downloads") {
|
||||
RasterizerInterface rasterizer;
|
||||
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
|
||||
memory_track->UnmarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 64);
|
||||
memory_track->CachedCpuWrite(c + PAGE, PAGE);
|
||||
REQUIRE(rasterizer.Count() == 63);
|
||||
memory_track->MarkRegionAsGpuModified(c + PAGE, PAGE);
|
||||
int num = 0;
|
||||
memory_track->ForEachDownloadRangeAndClear(c, WORD, [&](u64 offset, u64 size) { ++num; });
|
||||
REQUIRE(num == 1);
|
||||
num = 0;
|
||||
memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; });
|
||||
REQUIRE(num == 0);
|
||||
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(!memory_track->IsRegionGpuModified(c + PAGE, PAGE));
|
||||
memory_track->FlushCachedWrites();
|
||||
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
|
||||
REQUIRE(!memory_track->IsRegionGpuModified(c + PAGE, PAGE));
|
||||
memory_track->MarkRegionAsCpuModified(c, WORD);
|
||||
REQUIRE(rasterizer.Count() == 0);
|
||||
}
|
|
@ -11,8 +11,11 @@ endif()
|
|||
|
||||
add_library(video_core STATIC
|
||||
buffer_cache/buffer_base.h
|
||||
buffer_cache/buffer_cache_base.h
|
||||
buffer_cache/buffer_cache.cpp
|
||||
buffer_cache/buffer_cache.h
|
||||
buffer_cache/memory_tracker_base.h
|
||||
buffer_cache/word_manager.h
|
||||
cache_types.h
|
||||
cdma_pusher.cpp
|
||||
cdma_pusher.h
|
||||
|
@ -104,6 +107,7 @@ add_library(video_core STATIC
|
|||
renderer_null/renderer_null.h
|
||||
renderer_opengl/blit_image.cpp
|
||||
renderer_opengl/blit_image.h
|
||||
renderer_opengl/gl_buffer_cache_base.cpp
|
||||
renderer_opengl/gl_buffer_cache.cpp
|
||||
renderer_opengl/gl_buffer_cache.h
|
||||
renderer_opengl/gl_compute_pipeline.cpp
|
||||
|
@ -154,6 +158,7 @@ add_library(video_core STATIC
|
|||
renderer_vulkan/renderer_vulkan.cpp
|
||||
renderer_vulkan/vk_blit_screen.cpp
|
||||
renderer_vulkan/vk_blit_screen.h
|
||||
renderer_vulkan/vk_buffer_cache_base.cpp
|
||||
renderer_vulkan/vk_buffer_cache.cpp
|
||||
renderer_vulkan/vk_buffer_cache.h
|
||||
renderer_vulkan/vk_command_pool.cpp
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
|
@ -11,9 +11,7 @@
|
|||
#include "common/alignment.h"
|
||||
#include "common/common_funcs.h"
|
||||
#include "common/common_types.h"
|
||||
#include "common/div_ceil.h"
|
||||
#include "common/settings.h"
|
||||
#include "core/memory.h"
|
||||
#include "video_core/buffer_cache/word_manager.h"
|
||||
|
||||
namespace VideoCommon {
|
||||
|
||||
|
@ -36,116 +34,12 @@ struct NullBufferParams {};
|
|||
*/
|
||||
template <class RasterizerInterface>
|
||||
class BufferBase {
|
||||
static constexpr u64 PAGES_PER_WORD = 64;
|
||||
static constexpr u64 BYTES_PER_PAGE = Core::Memory::YUZU_PAGESIZE;
|
||||
static constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE;
|
||||
|
||||
/// Vector tracking modified pages tightly packed with small vector optimization
|
||||
union WordsArray {
|
||||
/// Returns the pointer to the words state
|
||||
[[nodiscard]] const u64* Pointer(bool is_short) const noexcept {
|
||||
return is_short ? &stack : heap;
|
||||
}
|
||||
|
||||
/// Returns the pointer to the words state
|
||||
[[nodiscard]] u64* Pointer(bool is_short) noexcept {
|
||||
return is_short ? &stack : heap;
|
||||
}
|
||||
|
||||
u64 stack = 0; ///< Small buffers storage
|
||||
u64* heap; ///< Not-small buffers pointer to the storage
|
||||
};
|
||||
|
||||
struct Words {
|
||||
explicit Words() = default;
|
||||
explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} {
|
||||
if (IsShort()) {
|
||||
cpu.stack = ~u64{0};
|
||||
gpu.stack = 0;
|
||||
cached_cpu.stack = 0;
|
||||
untracked.stack = ~u64{0};
|
||||
} else {
|
||||
// Share allocation between CPU and GPU pages and set their default values
|
||||
const size_t num_words = NumWords();
|
||||
u64* const alloc = new u64[num_words * 4];
|
||||
cpu.heap = alloc;
|
||||
gpu.heap = alloc + num_words;
|
||||
cached_cpu.heap = alloc + num_words * 2;
|
||||
untracked.heap = alloc + num_words * 3;
|
||||
std::fill_n(cpu.heap, num_words, ~u64{0});
|
||||
std::fill_n(gpu.heap, num_words, 0);
|
||||
std::fill_n(cached_cpu.heap, num_words, 0);
|
||||
std::fill_n(untracked.heap, num_words, ~u64{0});
|
||||
}
|
||||
// Clean up tailing bits
|
||||
const u64 last_word_size = size_bytes % BYTES_PER_WORD;
|
||||
const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE);
|
||||
const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD;
|
||||
const u64 last_word = (~u64{0} << shift) >> shift;
|
||||
cpu.Pointer(IsShort())[NumWords() - 1] = last_word;
|
||||
untracked.Pointer(IsShort())[NumWords() - 1] = last_word;
|
||||
}
|
||||
|
||||
~Words() {
|
||||
Release();
|
||||
}
|
||||
|
||||
Words& operator=(Words&& rhs) noexcept {
|
||||
Release();
|
||||
size_bytes = rhs.size_bytes;
|
||||
cpu = rhs.cpu;
|
||||
gpu = rhs.gpu;
|
||||
cached_cpu = rhs.cached_cpu;
|
||||
untracked = rhs.untracked;
|
||||
rhs.cpu.heap = nullptr;
|
||||
return *this;
|
||||
}
|
||||
|
||||
Words(Words&& rhs) noexcept
|
||||
: size_bytes{rhs.size_bytes}, cpu{rhs.cpu}, gpu{rhs.gpu},
|
||||
cached_cpu{rhs.cached_cpu}, untracked{rhs.untracked} {
|
||||
rhs.cpu.heap = nullptr;
|
||||
}
|
||||
|
||||
Words& operator=(const Words&) = delete;
|
||||
Words(const Words&) = delete;
|
||||
|
||||
/// Returns true when the buffer fits in the small vector optimization
|
||||
[[nodiscard]] bool IsShort() const noexcept {
|
||||
return size_bytes <= BYTES_PER_WORD;
|
||||
}
|
||||
|
||||
/// Returns the number of words of the buffer
|
||||
[[nodiscard]] size_t NumWords() const noexcept {
|
||||
return Common::DivCeil(size_bytes, BYTES_PER_WORD);
|
||||
}
|
||||
|
||||
/// Release buffer resources
|
||||
void Release() {
|
||||
if (!IsShort()) {
|
||||
// CPU written words is the base for the heap allocation
|
||||
delete[] cpu.heap;
|
||||
}
|
||||
}
|
||||
|
||||
u64 size_bytes = 0;
|
||||
WordsArray cpu;
|
||||
WordsArray gpu;
|
||||
WordsArray cached_cpu;
|
||||
WordsArray untracked;
|
||||
};
|
||||
|
||||
enum class Type {
|
||||
CPU,
|
||||
GPU,
|
||||
CachedCPU,
|
||||
Untracked,
|
||||
};
|
||||
|
||||
public:
|
||||
explicit BufferBase(RasterizerInterface& rasterizer_, VAddr cpu_addr_, u64 size_bytes)
|
||||
: rasterizer{&rasterizer_}, cpu_addr{Common::AlignDown(cpu_addr_, BYTES_PER_PAGE)},
|
||||
words(Common::AlignUp(size_bytes + (cpu_addr_ - cpu_addr), BYTES_PER_PAGE)) {}
|
||||
static constexpr u64 BASE_PAGE_BITS = 16;
|
||||
static constexpr u64 BASE_PAGE_SIZE = 1ULL << BASE_PAGE_BITS;
|
||||
|
||||
explicit BufferBase(RasterizerInterface& rasterizer_, VAddr cpu_addr_, u64 size_bytes_)
|
||||
: cpu_addr{cpu_addr_}, size_bytes{size_bytes_} {}
|
||||
|
||||
explicit BufferBase(NullBufferParams) {}
|
||||
|
||||
|
@ -155,100 +49,6 @@ public:
|
|||
BufferBase& operator=(BufferBase&&) = default;
|
||||
BufferBase(BufferBase&&) = default;
|
||||
|
||||
/// Returns the inclusive CPU modified range in a begin end pair
|
||||
[[nodiscard]] std::pair<u64, u64> ModifiedCpuRegion(VAddr query_cpu_addr,
|
||||
u64 query_size) const noexcept {
|
||||
const u64 offset = query_cpu_addr - cpu_addr;
|
||||
return ModifiedRegion<Type::CPU>(offset, query_size);
|
||||
}
|
||||
|
||||
/// Returns the inclusive GPU modified range in a begin end pair
|
||||
[[nodiscard]] std::pair<u64, u64> ModifiedGpuRegion(VAddr query_cpu_addr,
|
||||
u64 query_size) const noexcept {
|
||||
const u64 offset = query_cpu_addr - cpu_addr;
|
||||
return ModifiedRegion<Type::GPU>(offset, query_size);
|
||||
}
|
||||
|
||||
/// Returns true if a region has been modified from the CPU
|
||||
[[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept {
|
||||
const u64 offset = query_cpu_addr - cpu_addr;
|
||||
return IsRegionModified<Type::CPU>(offset, query_size);
|
||||
}
|
||||
|
||||
/// Returns true if a region has been modified from the GPU
|
||||
[[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept {
|
||||
const u64 offset = query_cpu_addr - cpu_addr;
|
||||
return IsRegionModified<Type::GPU>(offset, query_size);
|
||||
}
|
||||
|
||||
/// Mark region as CPU modified, notifying the rasterizer about this change
|
||||
void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) {
|
||||
ChangeRegionState<Type::CPU, true>(dirty_cpu_addr, size);
|
||||
}
|
||||
|
||||
/// Unmark region as CPU modified, notifying the rasterizer about this change
|
||||
void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) {
|
||||
ChangeRegionState<Type::CPU, false>(dirty_cpu_addr, size);
|
||||
}
|
||||
|
||||
/// Mark region as modified from the host GPU
|
||||
void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept {
|
||||
ChangeRegionState<Type::GPU, true>(dirty_cpu_addr, size);
|
||||
}
|
||||
|
||||
/// Unmark region as modified from the host GPU
|
||||
void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept {
|
||||
ChangeRegionState<Type::GPU, false>(dirty_cpu_addr, size);
|
||||
}
|
||||
|
||||
/// Mark region as modified from the CPU
|
||||
/// but don't mark it as modified until FlusHCachedWrites is called.
|
||||
void CachedCpuWrite(VAddr dirty_cpu_addr, u64 size) {
|
||||
flags |= BufferFlagBits::CachedWrites;
|
||||
ChangeRegionState<Type::CachedCPU, true>(dirty_cpu_addr, size);
|
||||
}
|
||||
|
||||
/// Flushes cached CPU writes, and notify the rasterizer about the deltas
|
||||
void FlushCachedWrites() noexcept {
|
||||
flags &= ~BufferFlagBits::CachedWrites;
|
||||
const u64 num_words = NumWords();
|
||||
u64* const cached_words = Array<Type::CachedCPU>();
|
||||
u64* const untracked_words = Array<Type::Untracked>();
|
||||
u64* const cpu_words = Array<Type::CPU>();
|
||||
for (u64 word_index = 0; word_index < num_words; ++word_index) {
|
||||
const u64 cached_bits = cached_words[word_index];
|
||||
NotifyRasterizer<false>(word_index, untracked_words[word_index], cached_bits);
|
||||
untracked_words[word_index] |= cached_bits;
|
||||
cpu_words[word_index] |= cached_bits;
|
||||
if (!Settings::values.use_pessimistic_flushes) {
|
||||
cached_words[word_index] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Call 'func' for each CPU modified range and unmark those pages as CPU modified
|
||||
template <typename Func>
|
||||
void ForEachUploadRange(VAddr query_cpu_range, u64 size, Func&& func) {
|
||||
ForEachModifiedRange<Type::CPU>(query_cpu_range, size, true, func);
|
||||
}
|
||||
|
||||
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
|
||||
template <typename Func>
|
||||
void ForEachDownloadRange(VAddr query_cpu_range, u64 size, bool clear, Func&& func) {
|
||||
ForEachModifiedRange<Type::GPU>(query_cpu_range, size, clear, func);
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 size, Func&& func) {
|
||||
ForEachModifiedRange<Type::GPU>(query_cpu_range, size, true, func);
|
||||
}
|
||||
|
||||
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
|
||||
template <typename Func>
|
||||
void ForEachDownloadRange(Func&& func) {
|
||||
ForEachModifiedRange<Type::GPU>(cpu_addr, SizeBytes(), true, func);
|
||||
}
|
||||
|
||||
/// Mark buffer as picked
|
||||
void Pick() noexcept {
|
||||
flags |= BufferFlagBits::Picked;
|
||||
|
@ -295,11 +95,6 @@ public:
|
|||
return static_cast<u32>(other_cpu_addr - cpu_addr);
|
||||
}
|
||||
|
||||
/// Returns the size in bytes of the buffer
|
||||
[[nodiscard]] u64 SizeBytes() const noexcept {
|
||||
return words.size_bytes;
|
||||
}
|
||||
|
||||
size_t getLRUID() const noexcept {
|
||||
return lru_id;
|
||||
}
|
||||
|
@ -308,305 +103,16 @@ public:
|
|||
lru_id = lru_id_;
|
||||
}
|
||||
|
||||
size_t SizeBytes() const {
|
||||
return size_bytes;
|
||||
}
|
||||
|
||||
private:
|
||||
template <Type type>
|
||||
u64* Array() noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return words.cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return words.gpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::CachedCPU) {
|
||||
return words.cached_cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return words.untracked.Pointer(IsShort());
|
||||
}
|
||||
}
|
||||
|
||||
template <Type type>
|
||||
const u64* Array() const noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return words.cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return words.gpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::CachedCPU) {
|
||||
return words.cached_cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return words.untracked.Pointer(IsShort());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Change the state of a range of pages
|
||||
*
|
||||
* @param dirty_addr Base address to mark or unmark as modified
|
||||
* @param size Size in bytes to mark or unmark as modified
|
||||
*/
|
||||
template <Type type, bool enable>
|
||||
void ChangeRegionState(u64 dirty_addr, s64 size) noexcept(type == Type::GPU) {
|
||||
const s64 difference = dirty_addr - cpu_addr;
|
||||
const u64 offset = std::max<s64>(difference, 0);
|
||||
size += std::min<s64>(difference, 0);
|
||||
if (offset >= SizeBytes() || size < 0) {
|
||||
return;
|
||||
}
|
||||
u64* const untracked_words = Array<Type::Untracked>();
|
||||
u64* const state_words = Array<type>();
|
||||
const u64 offset_end = std::min(offset + size, SizeBytes());
|
||||
const u64 begin_page_index = offset / BYTES_PER_PAGE;
|
||||
const u64 begin_word_index = begin_page_index / PAGES_PER_WORD;
|
||||
const u64 end_page_index = Common::DivCeil(offset_end, BYTES_PER_PAGE);
|
||||
const u64 end_word_index = Common::DivCeil(end_page_index, PAGES_PER_WORD);
|
||||
u64 page_index = begin_page_index % PAGES_PER_WORD;
|
||||
u64 word_index = begin_word_index;
|
||||
while (word_index < end_word_index) {
|
||||
const u64 next_word_first_page = (word_index + 1) * PAGES_PER_WORD;
|
||||
const u64 left_offset =
|
||||
std::min(next_word_first_page - end_page_index, PAGES_PER_WORD) % PAGES_PER_WORD;
|
||||
const u64 right_offset = page_index;
|
||||
u64 bits = ~u64{0};
|
||||
bits = (bits >> right_offset) << right_offset;
|
||||
bits = (bits << left_offset) >> left_offset;
|
||||
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
|
||||
NotifyRasterizer<!enable>(word_index, untracked_words[word_index], bits);
|
||||
}
|
||||
if constexpr (enable) {
|
||||
state_words[word_index] |= bits;
|
||||
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
|
||||
untracked_words[word_index] |= bits;
|
||||
}
|
||||
} else {
|
||||
state_words[word_index] &= ~bits;
|
||||
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
|
||||
untracked_words[word_index] &= ~bits;
|
||||
}
|
||||
}
|
||||
page_index = 0;
|
||||
++word_index;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify rasterizer about changes in the CPU tracking state of a word in the buffer
|
||||
*
|
||||
* @param word_index Index to the word to notify to the rasterizer
|
||||
* @param current_bits Current state of the word
|
||||
* @param new_bits New state of the word
|
||||
*
|
||||
* @tparam add_to_rasterizer True when the rasterizer should start tracking the new pages
|
||||
*/
|
||||
template <bool add_to_rasterizer>
|
||||
void NotifyRasterizer(u64 word_index, u64 current_bits, u64 new_bits) const {
|
||||
u64 changed_bits = (add_to_rasterizer ? current_bits : ~current_bits) & new_bits;
|
||||
VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
|
||||
while (changed_bits != 0) {
|
||||
const int empty_bits = std::countr_zero(changed_bits);
|
||||
addr += empty_bits * BYTES_PER_PAGE;
|
||||
changed_bits >>= empty_bits;
|
||||
|
||||
const u32 continuous_bits = std::countr_one(changed_bits);
|
||||
const u64 size = continuous_bits * BYTES_PER_PAGE;
|
||||
const VAddr begin_addr = addr;
|
||||
addr += size;
|
||||
changed_bits = continuous_bits < PAGES_PER_WORD ? (changed_bits >> continuous_bits) : 0;
|
||||
rasterizer->UpdatePagesCachedCount(begin_addr, size, add_to_rasterizer ? 1 : -1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loop over each page in the given range, turn off those bits and notify the rasterizer if
|
||||
* needed. Call the given function on each turned off range.
|
||||
*
|
||||
* @param query_cpu_range Base CPU address to loop over
|
||||
* @param size Size in bytes of the CPU range to loop over
|
||||
* @param func Function to call for each turned off region
|
||||
*/
|
||||
template <Type type, typename Func>
|
||||
void ForEachModifiedRange(VAddr query_cpu_range, s64 size, bool clear, Func&& func) {
|
||||
static_assert(type != Type::Untracked);
|
||||
|
||||
const s64 difference = query_cpu_range - cpu_addr;
|
||||
const u64 query_begin = std::max<s64>(difference, 0);
|
||||
size += std::min<s64>(difference, 0);
|
||||
if (query_begin >= SizeBytes() || size < 0) {
|
||||
return;
|
||||
}
|
||||
u64* const untracked_words = Array<Type::Untracked>();
|
||||
u64* const state_words = Array<type>();
|
||||
const u64 query_end = query_begin + std::min(static_cast<u64>(size), SizeBytes());
|
||||
u64* const words_begin = state_words + query_begin / BYTES_PER_WORD;
|
||||
u64* const words_end = state_words + Common::DivCeil(query_end, BYTES_PER_WORD);
|
||||
|
||||
const auto modified = [](u64 word) { return word != 0; };
|
||||
const auto first_modified_word = std::find_if(words_begin, words_end, modified);
|
||||
if (first_modified_word == words_end) {
|
||||
// Exit early when the buffer is not modified
|
||||
return;
|
||||
}
|
||||
const auto last_modified_word = std::find_if_not(first_modified_word, words_end, modified);
|
||||
|
||||
const u64 word_index_begin = std::distance(state_words, first_modified_word);
|
||||
const u64 word_index_end = std::distance(state_words, last_modified_word);
|
||||
|
||||
const unsigned local_page_begin = std::countr_zero(*first_modified_word);
|
||||
const unsigned local_page_end =
|
||||
static_cast<unsigned>(PAGES_PER_WORD) - std::countl_zero(last_modified_word[-1]);
|
||||
const u64 word_page_begin = word_index_begin * PAGES_PER_WORD;
|
||||
const u64 word_page_end = (word_index_end - 1) * PAGES_PER_WORD;
|
||||
const u64 query_page_begin = query_begin / BYTES_PER_PAGE;
|
||||
const u64 query_page_end = Common::DivCeil(query_end, BYTES_PER_PAGE);
|
||||
const u64 page_index_begin = std::max(word_page_begin + local_page_begin, query_page_begin);
|
||||
const u64 page_index_end = std::min(word_page_end + local_page_end, query_page_end);
|
||||
const u64 first_word_page_begin = page_index_begin % PAGES_PER_WORD;
|
||||
const u64 last_word_page_end = (page_index_end - 1) % PAGES_PER_WORD + 1;
|
||||
|
||||
u64 page_begin = first_word_page_begin;
|
||||
u64 current_base = 0;
|
||||
u64 current_size = 0;
|
||||
bool on_going = false;
|
||||
for (u64 word_index = word_index_begin; word_index < word_index_end; ++word_index) {
|
||||
const bool is_last_word = word_index + 1 == word_index_end;
|
||||
const u64 page_end = is_last_word ? last_word_page_end : PAGES_PER_WORD;
|
||||
const u64 right_offset = page_begin;
|
||||
const u64 left_offset = PAGES_PER_WORD - page_end;
|
||||
u64 bits = ~u64{0};
|
||||
bits = (bits >> right_offset) << right_offset;
|
||||
bits = (bits << left_offset) >> left_offset;
|
||||
|
||||
const u64 current_word = state_words[word_index] & bits;
|
||||
if (clear) {
|
||||
state_words[word_index] &= ~bits;
|
||||
}
|
||||
|
||||
if constexpr (type == Type::CPU) {
|
||||
const u64 current_bits = untracked_words[word_index] & bits;
|
||||
untracked_words[word_index] &= ~bits;
|
||||
NotifyRasterizer<true>(word_index, current_bits, ~u64{0});
|
||||
}
|
||||
// Exclude CPU modified pages when visiting GPU pages
|
||||
const u64 word = current_word & ~(type == Type::GPU ? untracked_words[word_index] : 0);
|
||||
u64 page = page_begin;
|
||||
page_begin = 0;
|
||||
|
||||
while (page < page_end) {
|
||||
const int empty_bits = std::countr_zero(word >> page);
|
||||
if (on_going && empty_bits != 0) {
|
||||
InvokeModifiedRange(func, current_size, current_base);
|
||||
current_size = 0;
|
||||
on_going = false;
|
||||
}
|
||||
if (empty_bits == PAGES_PER_WORD) {
|
||||
break;
|
||||
}
|
||||
page += empty_bits;
|
||||
|
||||
const int continuous_bits = std::countr_one(word >> page);
|
||||
if (!on_going && continuous_bits != 0) {
|
||||
current_base = word_index * PAGES_PER_WORD + page;
|
||||
on_going = true;
|
||||
}
|
||||
current_size += continuous_bits;
|
||||
page += continuous_bits;
|
||||
}
|
||||
}
|
||||
if (on_going && current_size > 0) {
|
||||
InvokeModifiedRange(func, current_size, current_base);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void InvokeModifiedRange(Func&& func, u64 current_size, u64 current_base) {
|
||||
const u64 current_size_bytes = current_size * BYTES_PER_PAGE;
|
||||
const u64 offset_begin = current_base * BYTES_PER_PAGE;
|
||||
const u64 offset_end = std::min(offset_begin + current_size_bytes, SizeBytes());
|
||||
func(offset_begin, offset_end - offset_begin);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when a region has been modified
|
||||
*
|
||||
* @param offset Offset in bytes from the start of the buffer
|
||||
* @param size Size in bytes of the region to query for modifications
|
||||
*/
|
||||
template <Type type>
|
||||
[[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept {
|
||||
static_assert(type != Type::Untracked);
|
||||
|
||||
const u64* const untracked_words = Array<Type::Untracked>();
|
||||
const u64* const state_words = Array<type>();
|
||||
const u64 num_query_words = size / BYTES_PER_WORD + 1;
|
||||
const u64 word_begin = offset / BYTES_PER_WORD;
|
||||
const u64 word_end = std::min<u64>(word_begin + num_query_words, NumWords());
|
||||
const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE);
|
||||
u64 page_index = (offset / BYTES_PER_PAGE) % PAGES_PER_WORD;
|
||||
for (u64 word_index = word_begin; word_index < word_end; ++word_index, page_index = 0) {
|
||||
const u64 off_word = type == Type::GPU ? untracked_words[word_index] : 0;
|
||||
const u64 word = state_words[word_index] & ~off_word;
|
||||
if (word == 0) {
|
||||
continue;
|
||||
}
|
||||
const u64 page_end = std::min((word_index + 1) * PAGES_PER_WORD, page_limit);
|
||||
const u64 local_page_end = page_end % PAGES_PER_WORD;
|
||||
const u64 page_end_shift = (PAGES_PER_WORD - local_page_end) % PAGES_PER_WORD;
|
||||
if (((word >> page_index) << page_index) << page_end_shift != 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a begin end pair with the inclusive modified region
|
||||
*
|
||||
* @param offset Offset in bytes from the start of the buffer
|
||||
* @param size Size in bytes of the region to query for modifications
|
||||
*/
|
||||
template <Type type>
|
||||
[[nodiscard]] std::pair<u64, u64> ModifiedRegion(u64 offset, u64 size) const noexcept {
|
||||
static_assert(type != Type::Untracked);
|
||||
|
||||
const u64* const untracked_words = Array<Type::Untracked>();
|
||||
const u64* const state_words = Array<type>();
|
||||
const u64 num_query_words = size / BYTES_PER_WORD + 1;
|
||||
const u64 word_begin = offset / BYTES_PER_WORD;
|
||||
const u64 word_end = std::min<u64>(word_begin + num_query_words, NumWords());
|
||||
const u64 page_base = offset / BYTES_PER_PAGE;
|
||||
const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE);
|
||||
u64 begin = std::numeric_limits<u64>::max();
|
||||
u64 end = 0;
|
||||
for (u64 word_index = word_begin; word_index < word_end; ++word_index) {
|
||||
const u64 off_word = type == Type::GPU ? untracked_words[word_index] : 0;
|
||||
const u64 word = state_words[word_index] & ~off_word;
|
||||
if (word == 0) {
|
||||
continue;
|
||||
}
|
||||
const u64 local_page_begin = std::countr_zero(word);
|
||||
const u64 local_page_end = PAGES_PER_WORD - std::countl_zero(word);
|
||||
const u64 page_index = word_index * PAGES_PER_WORD;
|
||||
const u64 page_begin = std::max(page_index + local_page_begin, page_base);
|
||||
const u64 page_end = std::min(page_index + local_page_end, page_limit);
|
||||
begin = std::min(begin, page_begin);
|
||||
end = std::max(end, page_end);
|
||||
}
|
||||
static constexpr std::pair<u64, u64> EMPTY{0, 0};
|
||||
return begin < end ? std::make_pair(begin * BYTES_PER_PAGE, end * BYTES_PER_PAGE) : EMPTY;
|
||||
}
|
||||
|
||||
/// Returns the number of words of the buffer
|
||||
[[nodiscard]] size_t NumWords() const noexcept {
|
||||
return words.NumWords();
|
||||
}
|
||||
|
||||
/// Returns true when the buffer fits in the small vector optimization
|
||||
[[nodiscard]] bool IsShort() const noexcept {
|
||||
return words.IsShort();
|
||||
}
|
||||
|
||||
RasterizerInterface* rasterizer = nullptr;
|
||||
VAddr cpu_addr = 0;
|
||||
Words words;
|
||||
BufferFlagBits flags{};
|
||||
int stream_score = 0;
|
||||
size_t lru_id = SIZE_MAX;
|
||||
size_t size_bytes = 0;
|
||||
};
|
||||
|
||||
} // namespace VideoCommon
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
#include "common/microprofile.h"
|
||||
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,580 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <numeric>
|
||||
#include <span>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/container/small_vector.hpp>
|
||||
#define BOOST_NO_MT
|
||||
#include <boost/pool/detail/mutex.hpp>
|
||||
#undef BOOST_NO_MT
|
||||
#include <boost/icl/interval.hpp>
|
||||
#include <boost/icl/interval_base_set.hpp>
|
||||
#include <boost/icl/interval_set.hpp>
|
||||
#include <boost/icl/split_interval_map.hpp>
|
||||
#include <boost/pool/pool.hpp>
|
||||
#include <boost/pool/pool_alloc.hpp>
|
||||
#include <boost/pool/poolfwd.hpp>
|
||||
|
||||
#include "common/common_types.h"
|
||||
#include "common/div_ceil.h"
|
||||
#include "common/literals.h"
|
||||
#include "common/lru_cache.h"
|
||||
#include "common/microprofile.h"
|
||||
#include "common/scope_exit.h"
|
||||
#include "common/settings.h"
|
||||
#include "core/memory.h"
|
||||
#include "video_core/buffer_cache/buffer_base.h"
|
||||
#include "video_core/control/channel_state_cache.h"
|
||||
#include "video_core/delayed_destruction_ring.h"
|
||||
#include "video_core/dirty_flags.h"
|
||||
#include "video_core/engines/draw_manager.h"
|
||||
#include "video_core/engines/kepler_compute.h"
|
||||
#include "video_core/engines/maxwell_3d.h"
|
||||
#include "video_core/memory_manager.h"
|
||||
#include "video_core/rasterizer_interface.h"
|
||||
#include "video_core/surface.h"
|
||||
#include "video_core/texture_cache/slot_vector.h"
|
||||
#include "video_core/texture_cache/types.h"
|
||||
|
||||
namespace boost {
|
||||
template <typename T>
|
||||
class fast_pool_allocator<T, default_user_allocator_new_delete, details::pool::null_mutex, 4096, 0>;
|
||||
}
|
||||
|
||||
namespace VideoCommon {
|
||||
|
||||
MICROPROFILE_DECLARE(GPU_PrepareBuffers);
|
||||
MICROPROFILE_DECLARE(GPU_BindUploadBuffers);
|
||||
MICROPROFILE_DECLARE(GPU_DownloadMemory);
|
||||
|
||||
using BufferId = SlotId;
|
||||
|
||||
using VideoCore::Surface::PixelFormat;
|
||||
using namespace Common::Literals;
|
||||
|
||||
constexpr u32 NUM_VERTEX_BUFFERS = 32;
|
||||
constexpr u32 NUM_TRANSFORM_FEEDBACK_BUFFERS = 4;
|
||||
constexpr u32 NUM_GRAPHICS_UNIFORM_BUFFERS = 18;
|
||||
constexpr u32 NUM_COMPUTE_UNIFORM_BUFFERS = 8;
|
||||
constexpr u32 NUM_STORAGE_BUFFERS = 16;
|
||||
constexpr u32 NUM_TEXTURE_BUFFERS = 16;
|
||||
constexpr u32 NUM_STAGES = 5;
|
||||
|
||||
using UniformBufferSizes = std::array<std::array<u32, NUM_GRAPHICS_UNIFORM_BUFFERS>, NUM_STAGES>;
|
||||
using ComputeUniformBufferSizes = std::array<u32, NUM_COMPUTE_UNIFORM_BUFFERS>;
|
||||
|
||||
enum class ObtainBufferSynchronize : u32 {
|
||||
NoSynchronize = 0,
|
||||
FullSynchronize = 1,
|
||||
SynchronizeNoDirty = 2,
|
||||
};
|
||||
|
||||
enum class ObtainBufferOperation : u32 {
|
||||
DoNothing = 0,
|
||||
MarkAsWritten = 1,
|
||||
DiscardWrite = 2,
|
||||
MarkQuery = 3,
|
||||
};
|
||||
|
||||
template <typename P>
|
||||
class BufferCache : public VideoCommon::ChannelSetupCaches<VideoCommon::ChannelInfo> {
|
||||
// Page size for caching purposes.
|
||||
// This is unrelated to the CPU page size and it can be changed as it seems optimal.
|
||||
static constexpr u32 CACHING_PAGEBITS = 16;
|
||||
static constexpr u64 CACHING_PAGESIZE = u64{1} << CACHING_PAGEBITS;
|
||||
|
||||
static constexpr bool IS_OPENGL = P::IS_OPENGL;
|
||||
static constexpr bool HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS =
|
||||
P::HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS;
|
||||
static constexpr bool HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT =
|
||||
P::HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT;
|
||||
static constexpr bool NEEDS_BIND_UNIFORM_INDEX = P::NEEDS_BIND_UNIFORM_INDEX;
|
||||
static constexpr bool NEEDS_BIND_STORAGE_INDEX = P::NEEDS_BIND_STORAGE_INDEX;
|
||||
static constexpr bool USE_MEMORY_MAPS = P::USE_MEMORY_MAPS;
|
||||
static constexpr bool SEPARATE_IMAGE_BUFFERS_BINDINGS = P::SEPARATE_IMAGE_BUFFER_BINDINGS;
|
||||
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = P::IMPLEMENTS_ASYNC_DOWNLOADS;
|
||||
|
||||
static constexpr BufferId NULL_BUFFER_ID{0};
|
||||
|
||||
static constexpr s64 DEFAULT_EXPECTED_MEMORY = 512_MiB;
|
||||
static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB;
|
||||
static constexpr s64 TARGET_THRESHOLD = 4_GiB;
|
||||
|
||||
// Debug Flags.
|
||||
|
||||
static constexpr bool DISABLE_DOWNLOADS = true;
|
||||
|
||||
using Maxwell = Tegra::Engines::Maxwell3D::Regs;
|
||||
|
||||
using Runtime = typename P::Runtime;
|
||||
using Buffer = typename P::Buffer;
|
||||
using Async_Buffer = typename P::Async_Buffer;
|
||||
using MemoryTracker = typename P::MemoryTracker;
|
||||
|
||||
using IntervalCompare = std::less<VAddr>;
|
||||
using IntervalInstance = boost::icl::interval_type_default<VAddr, std::less>;
|
||||
using IntervalAllocator = boost::fast_pool_allocator<VAddr>;
|
||||
using IntervalSet = boost::icl::interval_set<VAddr>;
|
||||
using IntervalType = typename IntervalSet::interval_type;
|
||||
|
||||
template <typename Type>
|
||||
struct counter_add_functor : public boost::icl::identity_based_inplace_combine<Type> {
|
||||
// types
|
||||
typedef counter_add_functor<Type> type;
|
||||
typedef boost::icl::identity_based_inplace_combine<Type> base_type;
|
||||
|
||||
// public member functions
|
||||
void operator()(Type& current, const Type& added) const {
|
||||
current += added;
|
||||
if (current < base_type::identity_element()) {
|
||||
current = base_type::identity_element();
|
||||
}
|
||||
}
|
||||
|
||||
// public static functions
|
||||
static void version(Type&){};
|
||||
};
|
||||
|
||||
using OverlapCombine = counter_add_functor<int>;
|
||||
using OverlapSection = boost::icl::inter_section<int>;
|
||||
using OverlapCounter = boost::icl::split_interval_map<VAddr, int>;
|
||||
|
||||
struct Empty {};
|
||||
|
||||
struct OverlapResult {
|
||||
std::vector<BufferId> ids;
|
||||
VAddr begin;
|
||||
VAddr end;
|
||||
bool has_stream_leap = false;
|
||||
};
|
||||
|
||||
struct Binding {
|
||||
VAddr cpu_addr{};
|
||||
u32 size{};
|
||||
BufferId buffer_id;
|
||||
};
|
||||
|
||||
struct TextureBufferBinding : Binding {
|
||||
PixelFormat format;
|
||||
};
|
||||
|
||||
static constexpr Binding NULL_BINDING{
|
||||
.cpu_addr = 0,
|
||||
.size = 0,
|
||||
.buffer_id = NULL_BUFFER_ID,
|
||||
};
|
||||
|
||||
public:
|
||||
static constexpr u32 DEFAULT_SKIP_CACHE_SIZE = static_cast<u32>(4_KiB);
|
||||
|
||||
explicit BufferCache(VideoCore::RasterizerInterface& rasterizer_,
|
||||
Core::Memory::Memory& cpu_memory_, Runtime& runtime_);
|
||||
|
||||
void TickFrame();
|
||||
|
||||
void WriteMemory(VAddr cpu_addr, u64 size);
|
||||
|
||||
void CachedWriteMemory(VAddr cpu_addr, u64 size);
|
||||
|
||||
void DownloadMemory(VAddr cpu_addr, u64 size);
|
||||
|
||||
bool InlineMemory(VAddr dest_address, size_t copy_size, std::span<const u8> inlined_buffer);
|
||||
|
||||
void BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr, u32 size);
|
||||
|
||||
void DisableGraphicsUniformBuffer(size_t stage, u32 index);
|
||||
|
||||
void UpdateGraphicsBuffers(bool is_indexed);
|
||||
|
||||
void UpdateComputeBuffers();
|
||||
|
||||
void BindHostGeometryBuffers(bool is_indexed);
|
||||
|
||||
void BindHostStageBuffers(size_t stage);
|
||||
|
||||
void BindHostComputeBuffers();
|
||||
|
||||
void SetUniformBuffersState(const std::array<u32, NUM_STAGES>& mask,
|
||||
const UniformBufferSizes* sizes);
|
||||
|
||||
void SetComputeUniformBufferState(u32 mask, const ComputeUniformBufferSizes* sizes);
|
||||
|
||||
void UnbindGraphicsStorageBuffers(size_t stage);
|
||||
|
||||
void BindGraphicsStorageBuffer(size_t stage, size_t ssbo_index, u32 cbuf_index, u32 cbuf_offset,
|
||||
bool is_written);
|
||||
|
||||
void UnbindGraphicsTextureBuffers(size_t stage);
|
||||
|
||||
void BindGraphicsTextureBuffer(size_t stage, size_t tbo_index, GPUVAddr gpu_addr, u32 size,
|
||||
PixelFormat format, bool is_written, bool is_image);
|
||||
|
||||
void UnbindComputeStorageBuffers();
|
||||
|
||||
void BindComputeStorageBuffer(size_t ssbo_index, u32 cbuf_index, u32 cbuf_offset,
|
||||
bool is_written);
|
||||
|
||||
void UnbindComputeTextureBuffers();
|
||||
|
||||
void BindComputeTextureBuffer(size_t tbo_index, GPUVAddr gpu_addr, u32 size, PixelFormat format,
|
||||
bool is_written, bool is_image);
|
||||
|
||||
[[nodiscard]] std::pair<Buffer*, u32> ObtainBuffer(GPUVAddr gpu_addr, u32 size,
|
||||
ObtainBufferSynchronize sync_info,
|
||||
ObtainBufferOperation post_op);
|
||||
void FlushCachedWrites();
|
||||
|
||||
/// Return true when there are uncommitted buffers to be downloaded
|
||||
[[nodiscard]] bool HasUncommittedFlushes() const noexcept;
|
||||
|
||||
void AccumulateFlushes();
|
||||
|
||||
/// Return true when the caller should wait for async downloads
|
||||
[[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept;
|
||||
|
||||
/// Commit asynchronous downloads
|
||||
void CommitAsyncFlushes();
|
||||
void CommitAsyncFlushesHigh();
|
||||
|
||||
/// Pop asynchronous downloads
|
||||
void PopAsyncFlushes();
|
||||
void PopAsyncBuffers();
|
||||
|
||||
bool DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount);
|
||||
|
||||
bool DMAClear(GPUVAddr src_address, u64 amount, u32 value);
|
||||
|
||||
/// Return true when a CPU region is modified from the GPU
|
||||
[[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size);
|
||||
|
||||
/// Return true when a region is registered on the cache
|
||||
[[nodiscard]] bool IsRegionRegistered(VAddr addr, size_t size);
|
||||
|
||||
/// Return true when a CPU region is modified from the CPU
|
||||
[[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size);
|
||||
|
||||
void SetDrawIndirect(
|
||||
const Tegra::Engines::DrawManager::IndirectParams* current_draw_indirect_) {
|
||||
current_draw_indirect = current_draw_indirect_;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::pair<Buffer*, u32> GetDrawIndirectCount();
|
||||
|
||||
[[nodiscard]] std::pair<Buffer*, u32> GetDrawIndirectBuffer();
|
||||
|
||||
std::recursive_mutex mutex;
|
||||
Runtime& runtime;
|
||||
|
||||
private:
|
||||
template <typename Func>
|
||||
static void ForEachEnabledBit(u32 enabled_mask, Func&& func) {
|
||||
for (u32 index = 0; enabled_mask != 0; ++index, enabled_mask >>= 1) {
|
||||
const int disabled_bits = std::countr_zero(enabled_mask);
|
||||
index += disabled_bits;
|
||||
enabled_mask >>= disabled_bits;
|
||||
func(index);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void ForEachBufferInRange(VAddr cpu_addr, u64 size, Func&& func) {
|
||||
const u64 page_end = Common::DivCeil(cpu_addr + size, CACHING_PAGESIZE);
|
||||
for (u64 page = cpu_addr >> CACHING_PAGEBITS; page < page_end;) {
|
||||
const BufferId buffer_id = page_table[page];
|
||||
if (!buffer_id) {
|
||||
++page;
|
||||
continue;
|
||||
}
|
||||
Buffer& buffer = slot_buffers[buffer_id];
|
||||
func(buffer_id, buffer);
|
||||
|
||||
const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes();
|
||||
page = Common::DivCeil(end_addr, CACHING_PAGESIZE);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void ForEachInRangeSet(IntervalSet& current_range, VAddr cpu_addr, u64 size, Func&& func) {
|
||||
const VAddr start_address = cpu_addr;
|
||||
const VAddr end_address = start_address + size;
|
||||
const IntervalType search_interval{start_address, end_address};
|
||||
auto it = current_range.lower_bound(search_interval);
|
||||
if (it == current_range.end()) {
|
||||
return;
|
||||
}
|
||||
auto end_it = current_range.upper_bound(search_interval);
|
||||
for (; it != end_it; it++) {
|
||||
VAddr inter_addr_end = it->upper();
|
||||
VAddr inter_addr = it->lower();
|
||||
if (inter_addr_end > end_address) {
|
||||
inter_addr_end = end_address;
|
||||
}
|
||||
if (inter_addr < start_address) {
|
||||
inter_addr = start_address;
|
||||
}
|
||||
func(inter_addr, inter_addr_end);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void ForEachInOverlapCounter(OverlapCounter& current_range, VAddr cpu_addr, u64 size,
|
||||
Func&& func) {
|
||||
const VAddr start_address = cpu_addr;
|
||||
const VAddr end_address = start_address + size;
|
||||
const IntervalType search_interval{start_address, end_address};
|
||||
auto it = current_range.lower_bound(search_interval);
|
||||
if (it == current_range.end()) {
|
||||
return;
|
||||
}
|
||||
auto end_it = current_range.upper_bound(search_interval);
|
||||
for (; it != end_it; it++) {
|
||||
auto& inter = it->first;
|
||||
VAddr inter_addr_end = inter.upper();
|
||||
VAddr inter_addr = inter.lower();
|
||||
if (inter_addr_end > end_address) {
|
||||
inter_addr_end = end_address;
|
||||
}
|
||||
if (inter_addr < start_address) {
|
||||
inter_addr = start_address;
|
||||
}
|
||||
func(inter_addr, inter_addr_end, it->second);
|
||||
}
|
||||
}
|
||||
|
||||
void RemoveEachInOverlapCounter(OverlapCounter& current_range,
|
||||
const IntervalType search_interval, int subtract_value) {
|
||||
bool any_removals = false;
|
||||
current_range.add(std::make_pair(search_interval, subtract_value));
|
||||
do {
|
||||
any_removals = false;
|
||||
auto it = current_range.lower_bound(search_interval);
|
||||
if (it == current_range.end()) {
|
||||
return;
|
||||
}
|
||||
auto end_it = current_range.upper_bound(search_interval);
|
||||
for (; it != end_it; it++) {
|
||||
if (it->second <= 0) {
|
||||
any_removals = true;
|
||||
current_range.erase(it);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} while (any_removals);
|
||||
}
|
||||
|
||||
static bool IsRangeGranular(VAddr cpu_addr, size_t size) {
|
||||
return (cpu_addr & ~Core::Memory::YUZU_PAGEMASK) ==
|
||||
((cpu_addr + size) & ~Core::Memory::YUZU_PAGEMASK);
|
||||
}
|
||||
|
||||
void RunGarbageCollector();
|
||||
|
||||
void WaitOnAsyncFlushes(VAddr cpu_addr, u64 size);
|
||||
|
||||
void BindHostIndexBuffer();
|
||||
|
||||
void BindHostVertexBuffers();
|
||||
|
||||
void BindHostDrawIndirectBuffers();
|
||||
|
||||
void BindHostGraphicsUniformBuffers(size_t stage);
|
||||
|
||||
void BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind);
|
||||
|
||||
void BindHostGraphicsStorageBuffers(size_t stage);
|
||||
|
||||
void BindHostGraphicsTextureBuffers(size_t stage);
|
||||
|
||||
void BindHostTransformFeedbackBuffers();
|
||||
|
||||
void BindHostComputeUniformBuffers();
|
||||
|
||||
void BindHostComputeStorageBuffers();
|
||||
|
||||
void BindHostComputeTextureBuffers();
|
||||
|
||||
void DoUpdateGraphicsBuffers(bool is_indexed);
|
||||
|
||||
void DoUpdateComputeBuffers();
|
||||
|
||||
void UpdateIndexBuffer();
|
||||
|
||||
void UpdateVertexBuffers();
|
||||
|
||||
void UpdateVertexBuffer(u32 index);
|
||||
|
||||
void UpdateDrawIndirect();
|
||||
|
||||
void UpdateUniformBuffers(size_t stage);
|
||||
|
||||
void UpdateStorageBuffers(size_t stage);
|
||||
|
||||
void UpdateTextureBuffers(size_t stage);
|
||||
|
||||
void UpdateTransformFeedbackBuffers();
|
||||
|
||||
void UpdateTransformFeedbackBuffer(u32 index);
|
||||
|
||||
void UpdateComputeUniformBuffers();
|
||||
|
||||
void UpdateComputeStorageBuffers();
|
||||
|
||||
void UpdateComputeTextureBuffers();
|
||||
|
||||
void MarkWrittenBuffer(BufferId buffer_id, VAddr cpu_addr, u32 size);
|
||||
|
||||
[[nodiscard]] BufferId FindBuffer(VAddr cpu_addr, u32 size);
|
||||
|
||||
[[nodiscard]] OverlapResult ResolveOverlaps(VAddr cpu_addr, u32 wanted_size);
|
||||
|
||||
void JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, bool accumulate_stream_score);
|
||||
|
||||
[[nodiscard]] BufferId CreateBuffer(VAddr cpu_addr, u32 wanted_size);
|
||||
|
||||
void Register(BufferId buffer_id);
|
||||
|
||||
void Unregister(BufferId buffer_id);
|
||||
|
||||
template <bool insert>
|
||||
void ChangeRegister(BufferId buffer_id);
|
||||
|
||||
void TouchBuffer(Buffer& buffer, BufferId buffer_id) noexcept;
|
||||
|
||||
bool SynchronizeBuffer(Buffer& buffer, VAddr cpu_addr, u32 size);
|
||||
|
||||
bool SynchronizeBufferImpl(Buffer& buffer, VAddr cpu_addr, u32 size);
|
||||
|
||||
bool SynchronizeBufferNoModified(Buffer& buffer, VAddr cpu_addr, u32 size);
|
||||
|
||||
void UploadMemory(Buffer& buffer, u64 total_size_bytes, u64 largest_copy,
|
||||
std::span<BufferCopy> copies);
|
||||
|
||||
void ImmediateUploadMemory(Buffer& buffer, u64 largest_copy,
|
||||
std::span<const BufferCopy> copies);
|
||||
|
||||
void MappedUploadMemory(Buffer& buffer, u64 total_size_bytes, std::span<BufferCopy> copies);
|
||||
|
||||
void DownloadBufferMemory(Buffer& buffer_id);
|
||||
|
||||
void DownloadBufferMemory(Buffer& buffer_id, VAddr cpu_addr, u64 size);
|
||||
|
||||
void DeleteBuffer(BufferId buffer_id, bool do_not_mark = false);
|
||||
|
||||
void NotifyBufferDeletion();
|
||||
|
||||
[[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index,
|
||||
bool is_written) const;
|
||||
|
||||
[[nodiscard]] TextureBufferBinding GetTextureBufferBinding(GPUVAddr gpu_addr, u32 size,
|
||||
PixelFormat format);
|
||||
|
||||
[[nodiscard]] std::span<const u8> ImmediateBufferWithData(VAddr cpu_addr, size_t size);
|
||||
|
||||
[[nodiscard]] std::span<u8> ImmediateBuffer(size_t wanted_capacity);
|
||||
|
||||
[[nodiscard]] bool HasFastUniformBufferBound(size_t stage, u32 binding_index) const noexcept;
|
||||
|
||||
void ClearDownload(IntervalType subtract_interval);
|
||||
|
||||
VideoCore::RasterizerInterface& rasterizer;
|
||||
Core::Memory::Memory& cpu_memory;
|
||||
|
||||
SlotVector<Buffer> slot_buffers;
|
||||
DelayedDestructionRing<Buffer, 8> delayed_destruction_ring;
|
||||
|
||||
const Tegra::Engines::DrawManager::IndirectParams* current_draw_indirect{};
|
||||
|
||||
u32 last_index_count = 0;
|
||||
|
||||
Binding index_buffer;
|
||||
std::array<Binding, NUM_VERTEX_BUFFERS> vertex_buffers;
|
||||
std::array<std::array<Binding, NUM_GRAPHICS_UNIFORM_BUFFERS>, NUM_STAGES> uniform_buffers;
|
||||
std::array<std::array<Binding, NUM_STORAGE_BUFFERS>, NUM_STAGES> storage_buffers;
|
||||
std::array<std::array<TextureBufferBinding, NUM_TEXTURE_BUFFERS>, NUM_STAGES> texture_buffers;
|
||||
std::array<Binding, NUM_TRANSFORM_FEEDBACK_BUFFERS> transform_feedback_buffers;
|
||||
Binding count_buffer_binding;
|
||||
Binding indirect_buffer_binding;
|
||||
|
||||
std::array<Binding, NUM_COMPUTE_UNIFORM_BUFFERS> compute_uniform_buffers;
|
||||
std::array<Binding, NUM_STORAGE_BUFFERS> compute_storage_buffers;
|
||||
std::array<TextureBufferBinding, NUM_TEXTURE_BUFFERS> compute_texture_buffers;
|
||||
|
||||
std::array<u32, NUM_STAGES> enabled_uniform_buffer_masks{};
|
||||
u32 enabled_compute_uniform_buffer_mask = 0;
|
||||
|
||||
const UniformBufferSizes* uniform_buffer_sizes{};
|
||||
const ComputeUniformBufferSizes* compute_uniform_buffer_sizes{};
|
||||
|
||||
std::array<u32, NUM_STAGES> enabled_storage_buffers{};
|
||||
std::array<u32, NUM_STAGES> written_storage_buffers{};
|
||||
u32 enabled_compute_storage_buffers = 0;
|
||||
u32 written_compute_storage_buffers = 0;
|
||||
|
||||
std::array<u32, NUM_STAGES> enabled_texture_buffers{};
|
||||
std::array<u32, NUM_STAGES> written_texture_buffers{};
|
||||
std::array<u32, NUM_STAGES> image_texture_buffers{};
|
||||
u32 enabled_compute_texture_buffers = 0;
|
||||
u32 written_compute_texture_buffers = 0;
|
||||
u32 image_compute_texture_buffers = 0;
|
||||
|
||||
std::array<u32, 16> uniform_cache_hits{};
|
||||
std::array<u32, 16> uniform_cache_shots{};
|
||||
|
||||
u32 uniform_buffer_skip_cache_size = DEFAULT_SKIP_CACHE_SIZE;
|
||||
|
||||
bool has_deleted_buffers = false;
|
||||
|
||||
std::conditional_t<HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS, std::array<u32, NUM_STAGES>, Empty>
|
||||
dirty_uniform_buffers{};
|
||||
std::conditional_t<IS_OPENGL, std::array<u32, NUM_STAGES>, Empty> fast_bound_uniform_buffers{};
|
||||
std::conditional_t<HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS,
|
||||
std::array<std::array<u32, NUM_GRAPHICS_UNIFORM_BUFFERS>, NUM_STAGES>, Empty>
|
||||
uniform_buffer_binding_sizes{};
|
||||
|
||||
std::vector<BufferId> cached_write_buffer_ids;
|
||||
|
||||
MemoryTracker memory_tracker;
|
||||
IntervalSet uncommitted_ranges;
|
||||
IntervalSet common_ranges;
|
||||
IntervalSet cached_ranges;
|
||||
IntervalSet pending_ranges;
|
||||
std::deque<IntervalSet> committed_ranges;
|
||||
|
||||
// Async Buffers
|
||||
OverlapCounter async_downloads;
|
||||
std::deque<std::optional<Async_Buffer>> async_buffers;
|
||||
std::deque<boost::container::small_vector<BufferCopy, 4>> pending_downloads;
|
||||
std::optional<Async_Buffer> current_buffer;
|
||||
|
||||
std::deque<Async_Buffer> async_buffers_death_ring;
|
||||
|
||||
size_t immediate_buffer_capacity = 0;
|
||||
Common::ScratchBuffer<u8> immediate_buffer_alloc;
|
||||
|
||||
struct LRUItemParams {
|
||||
using ObjectType = BufferId;
|
||||
using TickType = u64;
|
||||
};
|
||||
Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
|
||||
u64 frame_tick = 0;
|
||||
u64 total_used_memory = 0;
|
||||
u64 minimum_memory = 0;
|
||||
u64 critical_memory = 0;
|
||||
BufferId inline_buffer_id;
|
||||
|
||||
bool active_async_buffers = false;
|
||||
|
||||
std::array<BufferId, ((1ULL << 39) >> CACHING_PAGEBITS)> page_table;
|
||||
};
|
||||
|
||||
} // namespace VideoCommon
|
|
@ -0,0 +1,271 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <bit>
|
||||
#include <deque>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
|
||||
#include "common/alignment.h"
|
||||
#include "common/common_types.h"
|
||||
#include "video_core/buffer_cache/word_manager.h"
|
||||
|
||||
namespace VideoCommon {
|
||||
|
||||
template <class RasterizerInterface>
|
||||
class MemoryTrackerBase {
|
||||
static constexpr size_t MAX_CPU_PAGE_BITS = 39;
|
||||
static constexpr size_t HIGHER_PAGE_BITS = 22;
|
||||
static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS;
|
||||
static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL;
|
||||
static constexpr size_t NUM_HIGH_PAGES = 1ULL << (MAX_CPU_PAGE_BITS - HIGHER_PAGE_BITS);
|
||||
static constexpr size_t MANAGER_POOL_SIZE = 32;
|
||||
static constexpr size_t WORDS_STACK_NEEDED = HIGHER_PAGE_SIZE / BYTES_PER_WORD;
|
||||
using Manager = WordManager<RasterizerInterface, WORDS_STACK_NEEDED>;
|
||||
|
||||
public:
|
||||
MemoryTrackerBase(RasterizerInterface& rasterizer_) : rasterizer{&rasterizer_} {}
|
||||
~MemoryTrackerBase() = default;
|
||||
|
||||
/// Returns the inclusive CPU modified range in a begin end pair
|
||||
[[nodiscard]] std::pair<u64, u64> ModifiedCpuRegion(VAddr query_cpu_addr,
|
||||
u64 query_size) noexcept {
|
||||
return IteratePairs<true>(
|
||||
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
|
||||
return manager->template ModifiedRegion<Type::CPU>(offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Returns the inclusive GPU modified range in a begin end pair
|
||||
[[nodiscard]] std::pair<u64, u64> ModifiedGpuRegion(VAddr query_cpu_addr,
|
||||
u64 query_size) noexcept {
|
||||
return IteratePairs<false>(
|
||||
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
|
||||
return manager->template ModifiedRegion<Type::GPU>(offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Returns true if a region has been modified from the CPU
|
||||
[[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
|
||||
return IteratePages<true>(
|
||||
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
|
||||
return manager->template IsRegionModified<Type::CPU>(offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Returns true if a region has been modified from the GPU
|
||||
[[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
|
||||
return IteratePages<false>(
|
||||
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
|
||||
return manager->template IsRegionModified<Type::GPU>(offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Mark region as CPU modified, notifying the rasterizer about this change
|
||||
void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::CPU, true>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Unmark region as CPU modified, notifying the rasterizer about this change
|
||||
void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::CPU, false>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Mark region as modified from the host GPU
|
||||
void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::GPU, true>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Unmark region as modified from the host GPU
|
||||
void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
|
||||
IteratePages<true>(dirty_cpu_addr, query_size,
|
||||
[](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ChangeRegionState<Type::GPU, false>(
|
||||
manager->GetCpuAddr() + offset, size);
|
||||
});
|
||||
}
|
||||
|
||||
/// Mark region as modified from the CPU
|
||||
/// but don't mark it as modified until FlusHCachedWrites is called.
|
||||
void CachedCpuWrite(VAddr dirty_cpu_addr, u64 query_size) {
|
||||
IteratePages<true>(
|
||||
dirty_cpu_addr, query_size, [this](Manager* manager, u64 offset, size_t size) {
|
||||
const VAddr cpu_address = manager->GetCpuAddr() + offset;
|
||||
manager->template ChangeRegionState<Type::CachedCPU, true>(cpu_address, size);
|
||||
cached_pages.insert(static_cast<u32>(cpu_address >> HIGHER_PAGE_BITS));
|
||||
});
|
||||
}
|
||||
|
||||
/// Flushes cached CPU writes, and notify the rasterizer about the deltas
|
||||
void FlushCachedWrites(VAddr query_cpu_addr, u64 query_size) noexcept {
|
||||
IteratePages<false>(query_cpu_addr, query_size,
|
||||
[](Manager* manager, [[maybe_unused]] u64 offset,
|
||||
[[maybe_unused]] size_t size) { manager->FlushCachedWrites(); });
|
||||
}
|
||||
|
||||
void FlushCachedWrites() noexcept {
|
||||
for (auto id : cached_pages) {
|
||||
top_tier[id]->FlushCachedWrites();
|
||||
}
|
||||
cached_pages.clear();
|
||||
}
|
||||
|
||||
/// Call 'func' for each CPU modified range and unmark those pages as CPU modified
|
||||
template <typename Func>
|
||||
void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
|
||||
IteratePages<true>(query_cpu_range, query_size,
|
||||
[&func](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ForEachModifiedRange<Type::CPU, true>(
|
||||
manager->GetCpuAddr() + offset, size, func);
|
||||
});
|
||||
}
|
||||
|
||||
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
|
||||
template <typename Func>
|
||||
void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, bool clear, Func&& func) {
|
||||
IteratePages<false>(query_cpu_range, query_size,
|
||||
[&func, clear](Manager* manager, u64 offset, size_t size) {
|
||||
if (clear) {
|
||||
manager->template ForEachModifiedRange<Type::GPU, true>(
|
||||
manager->GetCpuAddr() + offset, size, func);
|
||||
} else {
|
||||
manager->template ForEachModifiedRange<Type::GPU, false>(
|
||||
manager->GetCpuAddr() + offset, size, func);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 query_size, Func&& func) {
|
||||
IteratePages<false>(query_cpu_range, query_size,
|
||||
[&func](Manager* manager, u64 offset, size_t size) {
|
||||
manager->template ForEachModifiedRange<Type::GPU, true>(
|
||||
manager->GetCpuAddr() + offset, size, func);
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
template <bool create_region_on_fail, typename Func>
|
||||
bool IteratePages(VAddr cpu_address, size_t size, Func&& func) {
|
||||
using FuncReturn = typename std::invoke_result<Func, Manager*, u64, size_t>::type;
|
||||
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
|
||||
std::size_t remaining_size{size};
|
||||
std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
|
||||
u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
|
||||
while (remaining_size > 0) {
|
||||
const std::size_t copy_amount{std::min(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
|
||||
auto* manager{top_tier[page_index]};
|
||||
if (manager) {
|
||||
if constexpr (BOOL_BREAK) {
|
||||
if (func(manager, page_offset, copy_amount)) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
func(manager, page_offset, copy_amount);
|
||||
}
|
||||
} else if constexpr (create_region_on_fail) {
|
||||
CreateRegion(page_index);
|
||||
manager = top_tier[page_index];
|
||||
if constexpr (BOOL_BREAK) {
|
||||
if (func(manager, page_offset, copy_amount)) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
func(manager, page_offset, copy_amount);
|
||||
}
|
||||
}
|
||||
page_index++;
|
||||
page_offset = 0;
|
||||
remaining_size -= copy_amount;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
template <bool create_region_on_fail, typename Func>
|
||||
std::pair<u64, u64> IteratePairs(VAddr cpu_address, size_t size, Func&& func) {
|
||||
std::size_t remaining_size{size};
|
||||
std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
|
||||
u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
|
||||
u64 begin = std::numeric_limits<u64>::max();
|
||||
u64 end = 0;
|
||||
while (remaining_size > 0) {
|
||||
const std::size_t copy_amount{std::min(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
|
||||
auto* manager{top_tier[page_index]};
|
||||
const auto execute = [&] {
|
||||
auto [new_begin, new_end] = func(manager, page_offset, copy_amount);
|
||||
if (new_begin != 0 || new_end != 0) {
|
||||
const u64 base_address = page_index << HIGHER_PAGE_BITS;
|
||||
begin = std::min(new_begin + base_address, begin);
|
||||
end = std::max(new_end + base_address, end);
|
||||
}
|
||||
};
|
||||
if (manager) {
|
||||
execute();
|
||||
} else if constexpr (create_region_on_fail) {
|
||||
CreateRegion(page_index);
|
||||
manager = top_tier[page_index];
|
||||
execute();
|
||||
}
|
||||
page_index++;
|
||||
page_offset = 0;
|
||||
remaining_size -= copy_amount;
|
||||
}
|
||||
if (begin < end) {
|
||||
return std::make_pair(begin, end);
|
||||
} else {
|
||||
return std::make_pair(0ULL, 0ULL);
|
||||
}
|
||||
}
|
||||
|
||||
void CreateRegion(std::size_t page_index) {
|
||||
const VAddr base_cpu_addr = page_index << HIGHER_PAGE_BITS;
|
||||
top_tier[page_index] = GetNewManager(base_cpu_addr);
|
||||
}
|
||||
|
||||
Manager* GetNewManager(VAddr base_cpu_addess) {
|
||||
const auto on_return = [&] {
|
||||
auto* new_manager = free_managers.front();
|
||||
new_manager->SetCpuAddress(base_cpu_addess);
|
||||
free_managers.pop_front();
|
||||
return new_manager;
|
||||
};
|
||||
if (!free_managers.empty()) {
|
||||
return on_return();
|
||||
}
|
||||
manager_pool.emplace_back();
|
||||
auto& last_pool = manager_pool.back();
|
||||
for (size_t i = 0; i < MANAGER_POOL_SIZE; i++) {
|
||||
new (&last_pool[i]) Manager(0, *rasterizer, HIGHER_PAGE_SIZE);
|
||||
free_managers.push_back(&last_pool[i]);
|
||||
}
|
||||
return on_return();
|
||||
}
|
||||
|
||||
std::deque<std::array<Manager, MANAGER_POOL_SIZE>> manager_pool;
|
||||
std::deque<Manager*> free_managers;
|
||||
|
||||
std::array<Manager*, NUM_HIGH_PAGES> top_tier{};
|
||||
|
||||
std::unordered_set<u32> cached_pages;
|
||||
|
||||
RasterizerInterface* rasterizer = nullptr;
|
||||
};
|
||||
|
||||
} // namespace VideoCommon
|
|
@ -0,0 +1,462 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <bit>
|
||||
#include <limits>
|
||||
#include <span>
|
||||
#include <utility>
|
||||
|
||||
#include "common/alignment.h"
|
||||
#include "common/common_funcs.h"
|
||||
#include "common/common_types.h"
|
||||
#include "common/div_ceil.h"
|
||||
#include "core/memory.h"
|
||||
|
||||
namespace VideoCommon {
|
||||
|
||||
constexpr u64 PAGES_PER_WORD = 64;
|
||||
constexpr u64 BYTES_PER_PAGE = Core::Memory::YUZU_PAGESIZE;
|
||||
constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE;
|
||||
|
||||
enum class Type {
|
||||
CPU,
|
||||
GPU,
|
||||
CachedCPU,
|
||||
Untracked,
|
||||
};
|
||||
|
||||
/// Vector tracking modified pages tightly packed with small vector optimization
|
||||
template <size_t stack_words = 1>
|
||||
struct WordsArray {
|
||||
/// Returns the pointer to the words state
|
||||
[[nodiscard]] const u64* Pointer(bool is_short) const noexcept {
|
||||
return is_short ? stack.data() : heap;
|
||||
}
|
||||
|
||||
/// Returns the pointer to the words state
|
||||
[[nodiscard]] u64* Pointer(bool is_short) noexcept {
|
||||
return is_short ? stack.data() : heap;
|
||||
}
|
||||
|
||||
std::array<u64, stack_words> stack{}; ///< Small buffers storage
|
||||
u64* heap; ///< Not-small buffers pointer to the storage
|
||||
};
|
||||
|
||||
template <size_t stack_words = 1>
|
||||
struct Words {
|
||||
explicit Words() = default;
|
||||
explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} {
|
||||
num_words = Common::DivCeil(size_bytes, BYTES_PER_WORD);
|
||||
if (IsShort()) {
|
||||
cpu.stack.fill(~u64{0});
|
||||
gpu.stack.fill(0);
|
||||
cached_cpu.stack.fill(0);
|
||||
untracked.stack.fill(~u64{0});
|
||||
} else {
|
||||
// Share allocation between CPU and GPU pages and set their default values
|
||||
u64* const alloc = new u64[num_words * 4];
|
||||
cpu.heap = alloc;
|
||||
gpu.heap = alloc + num_words;
|
||||
cached_cpu.heap = alloc + num_words * 2;
|
||||
untracked.heap = alloc + num_words * 3;
|
||||
std::fill_n(cpu.heap, num_words, ~u64{0});
|
||||
std::fill_n(gpu.heap, num_words, 0);
|
||||
std::fill_n(cached_cpu.heap, num_words, 0);
|
||||
std::fill_n(untracked.heap, num_words, ~u64{0});
|
||||
}
|
||||
// Clean up tailing bits
|
||||
const u64 last_word_size = size_bytes % BYTES_PER_WORD;
|
||||
const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE);
|
||||
const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD;
|
||||
const u64 last_word = (~u64{0} << shift) >> shift;
|
||||
cpu.Pointer(IsShort())[NumWords() - 1] = last_word;
|
||||
untracked.Pointer(IsShort())[NumWords() - 1] = last_word;
|
||||
}
|
||||
|
||||
~Words() {
|
||||
Release();
|
||||
}
|
||||
|
||||
Words& operator=(Words&& rhs) noexcept {
|
||||
Release();
|
||||
size_bytes = rhs.size_bytes;
|
||||
num_words = rhs.num_words;
|
||||
cpu = rhs.cpu;
|
||||
gpu = rhs.gpu;
|
||||
cached_cpu = rhs.cached_cpu;
|
||||
untracked = rhs.untracked;
|
||||
rhs.cpu.heap = nullptr;
|
||||
return *this;
|
||||
}
|
||||
|
||||
Words(Words&& rhs) noexcept
|
||||
: size_bytes{rhs.size_bytes}, num_words{rhs.num_words}, cpu{rhs.cpu}, gpu{rhs.gpu},
|
||||
cached_cpu{rhs.cached_cpu}, untracked{rhs.untracked} {
|
||||
rhs.cpu.heap = nullptr;
|
||||
}
|
||||
|
||||
Words& operator=(const Words&) = delete;
|
||||
Words(const Words&) = delete;
|
||||
|
||||
/// Returns true when the buffer fits in the small vector optimization
|
||||
[[nodiscard]] bool IsShort() const noexcept {
|
||||
return num_words <= stack_words;
|
||||
}
|
||||
|
||||
/// Returns the number of words of the buffer
|
||||
[[nodiscard]] size_t NumWords() const noexcept {
|
||||
return num_words;
|
||||
}
|
||||
|
||||
/// Release buffer resources
|
||||
void Release() {
|
||||
if (!IsShort()) {
|
||||
// CPU written words is the base for the heap allocation
|
||||
delete[] cpu.heap;
|
||||
}
|
||||
}
|
||||
|
||||
template <Type type>
|
||||
std::span<u64> Span() noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return std::span<u64>(cpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return std::span<u64>(gpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::CachedCPU) {
|
||||
return std::span<u64>(cached_cpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return std::span<u64>(untracked.Pointer(IsShort()), num_words);
|
||||
}
|
||||
}
|
||||
|
||||
template <Type type>
|
||||
std::span<const u64> Span() const noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return std::span<const u64>(cpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return std::span<const u64>(gpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::CachedCPU) {
|
||||
return std::span<const u64>(cached_cpu.Pointer(IsShort()), num_words);
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return std::span<const u64>(untracked.Pointer(IsShort()), num_words);
|
||||
}
|
||||
}
|
||||
|
||||
u64 size_bytes = 0;
|
||||
size_t num_words = 0;
|
||||
WordsArray<stack_words> cpu;
|
||||
WordsArray<stack_words> gpu;
|
||||
WordsArray<stack_words> cached_cpu;
|
||||
WordsArray<stack_words> untracked;
|
||||
};
|
||||
|
||||
template <class RasterizerInterface, size_t stack_words = 1>
|
||||
class WordManager {
|
||||
public:
|
||||
explicit WordManager(VAddr cpu_addr_, RasterizerInterface& rasterizer_, u64 size_bytes)
|
||||
: cpu_addr{cpu_addr_}, rasterizer{&rasterizer_}, words{size_bytes} {}
|
||||
|
||||
explicit WordManager() = default;
|
||||
|
||||
void SetCpuAddress(VAddr new_cpu_addr) {
|
||||
cpu_addr = new_cpu_addr;
|
||||
}
|
||||
|
||||
VAddr GetCpuAddr() const {
|
||||
return cpu_addr;
|
||||
}
|
||||
|
||||
static u64 ExtractBits(u64 word, size_t page_start, size_t page_end) {
|
||||
constexpr size_t number_bits = sizeof(u64) * 8;
|
||||
const size_t limit_page_end = number_bits - std::min(page_end, number_bits);
|
||||
u64 bits = (word >> page_start) << page_start;
|
||||
bits = (bits << limit_page_end) >> limit_page_end;
|
||||
return bits;
|
||||
}
|
||||
|
||||
static std::pair<size_t, size_t> GetWordPage(VAddr address) {
|
||||
const size_t converted_address = static_cast<size_t>(address);
|
||||
const size_t word_number = converted_address / BYTES_PER_WORD;
|
||||
const size_t amount_pages = converted_address % BYTES_PER_WORD;
|
||||
return std::make_pair(word_number, amount_pages / BYTES_PER_PAGE);
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void IterateWords(size_t offset, size_t size, Func&& func) const {
|
||||
using FuncReturn = std::invoke_result_t<Func, std::size_t, u64>;
|
||||
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
|
||||
const size_t start = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset), 0LL));
|
||||
const size_t end = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset + size), 0LL));
|
||||
if (start >= SizeBytes() || end <= start) {
|
||||
return;
|
||||
}
|
||||
auto [start_word, start_page] = GetWordPage(start);
|
||||
auto [end_word, end_page] = GetWordPage(end + BYTES_PER_PAGE - 1ULL);
|
||||
const size_t num_words = NumWords();
|
||||
start_word = std::min(start_word, num_words);
|
||||
end_word = std::min(end_word, num_words);
|
||||
const size_t diff = end_word - start_word;
|
||||
end_word += (end_page + PAGES_PER_WORD - 1ULL) / PAGES_PER_WORD;
|
||||
end_word = std::min(end_word, num_words);
|
||||
end_page += diff * PAGES_PER_WORD;
|
||||
constexpr u64 base_mask{~0ULL};
|
||||
for (size_t word_index = start_word; word_index < end_word; word_index++) {
|
||||
const u64 mask = ExtractBits(base_mask, start_page, end_page);
|
||||
start_page = 0;
|
||||
end_page -= PAGES_PER_WORD;
|
||||
if constexpr (BOOL_BREAK) {
|
||||
if (func(word_index, mask)) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
func(word_index, mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Func>
|
||||
void IteratePages(u64 mask, Func&& func) const {
|
||||
size_t offset = 0;
|
||||
while (mask != 0) {
|
||||
const size_t empty_bits = std::countr_zero(mask);
|
||||
offset += empty_bits;
|
||||
mask = mask >> empty_bits;
|
||||
|
||||
const size_t continuous_bits = std::countr_one(mask);
|
||||
func(offset, continuous_bits);
|
||||
mask = continuous_bits < PAGES_PER_WORD ? (mask >> continuous_bits) : 0;
|
||||
offset += continuous_bits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Change the state of a range of pages
|
||||
*
|
||||
* @param dirty_addr Base address to mark or unmark as modified
|
||||
* @param size Size in bytes to mark or unmark as modified
|
||||
*/
|
||||
template <Type type, bool enable>
|
||||
void ChangeRegionState(u64 dirty_addr, u64 size) noexcept(type == Type::GPU) {
|
||||
std::span<u64> state_words = words.template Span<type>();
|
||||
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
|
||||
[[maybe_unused]] std::span<u64> cached_words = words.template Span<Type::CachedCPU>();
|
||||
IterateWords(dirty_addr - cpu_addr, size, [&](size_t index, u64 mask) {
|
||||
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
|
||||
NotifyRasterizer<!enable>(index, untracked_words[index], mask);
|
||||
}
|
||||
if constexpr (enable) {
|
||||
state_words[index] |= mask;
|
||||
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
|
||||
untracked_words[index] |= mask;
|
||||
}
|
||||
if constexpr (type == Type::CPU) {
|
||||
cached_words[index] &= ~mask;
|
||||
}
|
||||
} else {
|
||||
if constexpr (type == Type::CPU) {
|
||||
const u64 word = state_words[index] & mask;
|
||||
cached_words[index] &= ~word;
|
||||
}
|
||||
state_words[index] &= ~mask;
|
||||
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
|
||||
untracked_words[index] &= ~mask;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Loop over each page in the given range, turn off those bits and notify the rasterizer if
|
||||
* needed. Call the given function on each turned off range.
|
||||
*
|
||||
* @param query_cpu_range Base CPU address to loop over
|
||||
* @param size Size in bytes of the CPU range to loop over
|
||||
* @param func Function to call for each turned off region
|
||||
*/
|
||||
template <Type type, bool clear, typename Func>
|
||||
void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) {
|
||||
static_assert(type != Type::Untracked);
|
||||
|
||||
std::span<u64> state_words = words.template Span<type>();
|
||||
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
|
||||
[[maybe_unused]] std::span<u64> cached_words = words.template Span<Type::CachedCPU>();
|
||||
const size_t offset = query_cpu_range - cpu_addr;
|
||||
bool pending = false;
|
||||
size_t pending_offset{};
|
||||
size_t pending_pointer{};
|
||||
const auto release = [&]() {
|
||||
func(cpu_addr + pending_offset * BYTES_PER_PAGE,
|
||||
(pending_pointer - pending_offset) * BYTES_PER_PAGE);
|
||||
};
|
||||
IterateWords(offset, size, [&](size_t index, u64 mask) {
|
||||
const u64 word = state_words[index] & mask;
|
||||
if constexpr (clear) {
|
||||
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
|
||||
NotifyRasterizer<true>(index, untracked_words[index], mask);
|
||||
}
|
||||
state_words[index] &= ~mask;
|
||||
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
|
||||
untracked_words[index] &= ~mask;
|
||||
}
|
||||
if constexpr (type == Type::CPU) {
|
||||
cached_words[index] &= ~word;
|
||||
}
|
||||
}
|
||||
const size_t base_offset = index * PAGES_PER_WORD;
|
||||
IteratePages(word, [&](size_t pages_offset, size_t pages_size) {
|
||||
const auto reset = [&]() {
|
||||
pending_offset = base_offset + pages_offset;
|
||||
pending_pointer = base_offset + pages_offset + pages_size;
|
||||
};
|
||||
if (!pending) {
|
||||
reset();
|
||||
pending = true;
|
||||
return;
|
||||
}
|
||||
if (pending_pointer == base_offset + pages_offset) {
|
||||
pending_pointer += pages_size;
|
||||
return;
|
||||
}
|
||||
release();
|
||||
reset();
|
||||
});
|
||||
});
|
||||
if (pending) {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when a region has been modified
|
||||
*
|
||||
* @param offset Offset in bytes from the start of the buffer
|
||||
* @param size Size in bytes of the region to query for modifications
|
||||
*/
|
||||
template <Type type>
|
||||
[[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept {
|
||||
static_assert(type != Type::Untracked);
|
||||
|
||||
const std::span<const u64> state_words = words.template Span<type>();
|
||||
bool result = false;
|
||||
IterateWords(offset, size, [&](size_t index, u64 mask) {
|
||||
const u64 word = state_words[index] & mask;
|
||||
if (word != 0) {
|
||||
result = true;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a begin end pair with the inclusive modified region
|
||||
*
|
||||
* @param offset Offset in bytes from the start of the buffer
|
||||
* @param size Size in bytes of the region to query for modifications
|
||||
*/
|
||||
template <Type type>
|
||||
[[nodiscard]] std::pair<u64, u64> ModifiedRegion(u64 offset, u64 size) const noexcept {
|
||||
static_assert(type != Type::Untracked);
|
||||
const std::span<const u64> state_words = words.template Span<type>();
|
||||
u64 begin = std::numeric_limits<u64>::max();
|
||||
u64 end = 0;
|
||||
IterateWords(offset, size, [&](size_t index, u64 mask) {
|
||||
const u64 word = state_words[index] & mask;
|
||||
if (word == 0) {
|
||||
return;
|
||||
}
|
||||
const u64 local_page_begin = std::countr_zero(word);
|
||||
const u64 local_page_end = PAGES_PER_WORD - std::countl_zero(word);
|
||||
const u64 page_index = index * PAGES_PER_WORD;
|
||||
begin = std::min(begin, page_index + local_page_begin);
|
||||
end = page_index + local_page_end;
|
||||
});
|
||||
static constexpr std::pair<u64, u64> EMPTY{0, 0};
|
||||
return begin < end ? std::make_pair(begin * BYTES_PER_PAGE, end * BYTES_PER_PAGE) : EMPTY;
|
||||
}
|
||||
|
||||
/// Returns the number of words of the manager
|
||||
[[nodiscard]] size_t NumWords() const noexcept {
|
||||
return words.NumWords();
|
||||
}
|
||||
|
||||
/// Returns the size in bytes of the manager
|
||||
[[nodiscard]] u64 SizeBytes() const noexcept {
|
||||
return words.size_bytes;
|
||||
}
|
||||
|
||||
/// Returns true when the buffer fits in the small vector optimization
|
||||
[[nodiscard]] bool IsShort() const noexcept {
|
||||
return words.IsShort();
|
||||
}
|
||||
|
||||
void FlushCachedWrites() noexcept {
|
||||
const u64 num_words = NumWords();
|
||||
u64* const cached_words = Array<Type::CachedCPU>();
|
||||
u64* const untracked_words = Array<Type::Untracked>();
|
||||
u64* const cpu_words = Array<Type::CPU>();
|
||||
for (u64 word_index = 0; word_index < num_words; ++word_index) {
|
||||
const u64 cached_bits = cached_words[word_index];
|
||||
NotifyRasterizer<false>(word_index, untracked_words[word_index], cached_bits);
|
||||
untracked_words[word_index] |= cached_bits;
|
||||
cpu_words[word_index] |= cached_bits;
|
||||
cached_words[word_index] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
template <Type type>
|
||||
u64* Array() noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return words.cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return words.gpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::CachedCPU) {
|
||||
return words.cached_cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return words.untracked.Pointer(IsShort());
|
||||
}
|
||||
}
|
||||
|
||||
template <Type type>
|
||||
const u64* Array() const noexcept {
|
||||
if constexpr (type == Type::CPU) {
|
||||
return words.cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::GPU) {
|
||||
return words.gpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::CachedCPU) {
|
||||
return words.cached_cpu.Pointer(IsShort());
|
||||
} else if constexpr (type == Type::Untracked) {
|
||||
return words.untracked.Pointer(IsShort());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify rasterizer about changes in the CPU tracking state of a word in the buffer
|
||||
*
|
||||
* @param word_index Index to the word to notify to the rasterizer
|
||||
* @param current_bits Current state of the word
|
||||
* @param new_bits New state of the word
|
||||
*
|
||||
* @tparam add_to_rasterizer True when the rasterizer should start tracking the new pages
|
||||
*/
|
||||
template <bool add_to_rasterizer>
|
||||
void NotifyRasterizer(u64 word_index, u64 current_bits, u64 new_bits) const {
|
||||
u64 changed_bits = (add_to_rasterizer ? current_bits : ~current_bits) & new_bits;
|
||||
VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
|
||||
IteratePages(changed_bits, [&](size_t offset, size_t size) {
|
||||
rasterizer->UpdatePagesCachedCount(addr + offset * BYTES_PER_PAGE,
|
||||
size * BYTES_PER_PAGE, add_to_rasterizer ? 1 : -1);
|
||||
});
|
||||
}
|
||||
|
||||
VAddr cpu_addr = 0;
|
||||
RasterizerInterface* rasterizer = nullptr;
|
||||
Words<stack_words> words;
|
||||
};
|
||||
|
||||
} // namespace VideoCommon
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
#include "common/common_types.h"
|
||||
#include "video_core/buffer_cache/buffer_cache.h"
|
||||
#include "video_core/buffer_cache/memory_tracker_base.h"
|
||||
#include "video_core/rasterizer_interface.h"
|
||||
#include "video_core/renderer_opengl/gl_device.h"
|
||||
#include "video_core/renderer_opengl/gl_resource_manager.h"
|
||||
|
@ -200,6 +201,8 @@ private:
|
|||
struct BufferCacheParams {
|
||||
using Runtime = OpenGL::BufferCacheRuntime;
|
||||
using Buffer = OpenGL::Buffer;
|
||||
using Async_Buffer = u32;
|
||||
using MemoryTracker = VideoCommon::MemoryTrackerBase<VideoCore::RasterizerInterface>;
|
||||
|
||||
static constexpr bool IS_OPENGL = true;
|
||||
static constexpr bool HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS = true;
|
||||
|
@ -208,6 +211,7 @@ struct BufferCacheParams {
|
|||
static constexpr bool NEEDS_BIND_STORAGE_INDEX = true;
|
||||
static constexpr bool USE_MEMORY_MAPS = false;
|
||||
static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = true;
|
||||
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = false;
|
||||
};
|
||||
|
||||
using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
#include "video_core/buffer_cache/buffer_cache.h"
|
||||
#include "video_core/renderer_opengl/gl_buffer_cache.h"
|
||||
|
||||
namespace VideoCommon {
|
||||
template class VideoCommon::BufferCache<OpenGL::BufferCacheParams>;
|
||||
}
|
|
@ -314,8 +314,12 @@ StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) {
|
|||
return staging_pool.Request(size, MemoryUsage::Upload);
|
||||
}
|
||||
|
||||
StagingBufferRef BufferCacheRuntime::DownloadStagingBuffer(size_t size) {
|
||||
return staging_pool.Request(size, MemoryUsage::Download);
|
||||
StagingBufferRef BufferCacheRuntime::DownloadStagingBuffer(size_t size, bool deferred) {
|
||||
return staging_pool.Request(size, MemoryUsage::Download, deferred);
|
||||
}
|
||||
|
||||
void BufferCacheRuntime::FreeDeferredStagingBuffer(StagingBufferRef& ref) {
|
||||
staging_pool.FreeDeferred(ref);
|
||||
}
|
||||
|
||||
u64 BufferCacheRuntime::GetDeviceLocalMemory() const {
|
||||
|
|
|
@ -3,7 +3,8 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "video_core/buffer_cache/buffer_cache.h"
|
||||
#include "video_core/buffer_cache/buffer_cache_base.h"
|
||||
#include "video_core/buffer_cache/memory_tracker_base.h"
|
||||
#include "video_core/engines/maxwell_3d.h"
|
||||
#include "video_core/renderer_vulkan/vk_compute_pass.h"
|
||||
#include "video_core/renderer_vulkan/vk_staging_buffer_pool.h"
|
||||
|
@ -75,7 +76,9 @@ public:
|
|||
|
||||
[[nodiscard]] StagingBufferRef UploadStagingBuffer(size_t size);
|
||||
|
||||
[[nodiscard]] StagingBufferRef DownloadStagingBuffer(size_t size);
|
||||
[[nodiscard]] StagingBufferRef DownloadStagingBuffer(size_t size, bool deferred = false);
|
||||
|
||||
void FreeDeferredStagingBuffer(StagingBufferRef& ref);
|
||||
|
||||
void PreCopyBarrier();
|
||||
|
||||
|
@ -142,6 +145,8 @@ private:
|
|||
struct BufferCacheParams {
|
||||
using Runtime = Vulkan::BufferCacheRuntime;
|
||||
using Buffer = Vulkan::Buffer;
|
||||
using Async_Buffer = Vulkan::StagingBufferRef;
|
||||
using MemoryTracker = VideoCommon::MemoryTrackerBase<VideoCore::RasterizerInterface>;
|
||||
|
||||
static constexpr bool IS_OPENGL = false;
|
||||
static constexpr bool HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS = false;
|
||||
|
@ -150,6 +155,7 @@ struct BufferCacheParams {
|
|||
static constexpr bool NEEDS_BIND_STORAGE_INDEX = false;
|
||||
static constexpr bool USE_MEMORY_MAPS = true;
|
||||
static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = false;
|
||||
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true;
|
||||
};
|
||||
|
||||
using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "video_core/buffer_cache/buffer_cache.h"
|
||||
#include "video_core/renderer_vulkan/vk_buffer_cache.h"
|
||||
|
||||
namespace VideoCommon {
|
||||
template class VideoCommon::BufferCache<Vulkan::BufferCacheParams>;
|
||||
}
|
Reference in New Issue