Merge pull request #9415 from liamwhite/dc
memory: correct semantics of data cache management operations
This commit is contained in:
commit
d5684dbe7d
|
@ -34,8 +34,6 @@ add_library(common STATIC
|
||||||
bit_util.h
|
bit_util.h
|
||||||
cityhash.cpp
|
cityhash.cpp
|
||||||
cityhash.h
|
cityhash.h
|
||||||
cache_management.cpp
|
|
||||||
cache_management.h
|
|
||||||
common_funcs.h
|
common_funcs.h
|
||||||
common_precompiled_headers.h
|
common_precompiled_headers.h
|
||||||
common_types.h
|
common_types.h
|
||||||
|
|
|
@ -1,59 +0,0 @@
|
||||||
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
|
||||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
||||||
|
|
||||||
#include <cstdint>
|
|
||||||
#include <cstring>
|
|
||||||
|
|
||||||
#include "common/cache_management.h"
|
|
||||||
|
|
||||||
namespace Common {
|
|
||||||
|
|
||||||
#if defined(ARCHITECTURE_x86_64)
|
|
||||||
|
|
||||||
// Most cache operations are no-ops on x86
|
|
||||||
|
|
||||||
void DataCacheLineCleanByVAToPoU(void* start, size_t size) {}
|
|
||||||
void DataCacheLineCleanAndInvalidateByVAToPoC(void* start, size_t size) {}
|
|
||||||
void DataCacheLineCleanByVAToPoC(void* start, size_t size) {}
|
|
||||||
void DataCacheZeroByVA(void* start, size_t size) {
|
|
||||||
std::memset(start, 0, size);
|
|
||||||
}
|
|
||||||
|
|
||||||
#elif defined(ARCHITECTURE_arm64)
|
|
||||||
|
|
||||||
// BS/DminLine is log2(cache size in words), we want size in bytes
|
|
||||||
#define EXTRACT_DMINLINE(ctr_el0) (1 << ((((ctr_el0) >> 16) & 0xf) + 2))
|
|
||||||
#define EXTRACT_BS(dczid_el0) (1 << (((dczid_el0)&0xf) + 2))
|
|
||||||
|
|
||||||
#define DEFINE_DC_OP(op_name, function_name) \
|
|
||||||
void function_name(void* start, size_t size) { \
|
|
||||||
size_t ctr_el0; \
|
|
||||||
asm volatile("mrs %[ctr_el0], ctr_el0\n\t" : [ctr_el0] "=r"(ctr_el0)); \
|
|
||||||
size_t cacheline_size = EXTRACT_DMINLINE(ctr_el0); \
|
|
||||||
uintptr_t va_start = reinterpret_cast<uintptr_t>(start); \
|
|
||||||
uintptr_t va_end = va_start + size; \
|
|
||||||
for (uintptr_t va = va_start; va < va_end; va += cacheline_size) { \
|
|
||||||
asm volatile("dc " #op_name ", %[va]\n\t" : : [va] "r"(va) : "memory"); \
|
|
||||||
} \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define DEFINE_DC_OP_DCZID(op_name, function_name) \
|
|
||||||
void function_name(void* start, size_t size) { \
|
|
||||||
size_t dczid_el0; \
|
|
||||||
asm volatile("mrs %[dczid_el0], dczid_el0\n\t" : [dczid_el0] "=r"(dczid_el0)); \
|
|
||||||
size_t cacheline_size = EXTRACT_BS(dczid_el0); \
|
|
||||||
uintptr_t va_start = reinterpret_cast<uintptr_t>(start); \
|
|
||||||
uintptr_t va_end = va_start + size; \
|
|
||||||
for (uintptr_t va = va_start; va < va_end; va += cacheline_size) { \
|
|
||||||
asm volatile("dc " #op_name ", %[va]\n\t" : : [va] "r"(va) : "memory"); \
|
|
||||||
} \
|
|
||||||
}
|
|
||||||
|
|
||||||
DEFINE_DC_OP(cvau, DataCacheLineCleanByVAToPoU);
|
|
||||||
DEFINE_DC_OP(civac, DataCacheLineCleanAndInvalidateByVAToPoC);
|
|
||||||
DEFINE_DC_OP(cvac, DataCacheLineCleanByVAToPoC);
|
|
||||||
DEFINE_DC_OP_DCZID(zva, DataCacheZeroByVA);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
||||||
} // namespace Common
|
|
|
@ -1,27 +0,0 @@
|
||||||
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
|
||||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
||||||
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include <cstddef>
|
|
||||||
|
|
||||||
namespace Common {
|
|
||||||
|
|
||||||
// Data cache instructions enabled at EL0 by SCTLR_EL1.UCI.
|
|
||||||
// VA = virtual address
|
|
||||||
// PoC = point of coherency
|
|
||||||
// PoU = point of unification
|
|
||||||
|
|
||||||
// dc cvau
|
|
||||||
void DataCacheLineCleanByVAToPoU(void* start, size_t size);
|
|
||||||
|
|
||||||
// dc civac
|
|
||||||
void DataCacheLineCleanAndInvalidateByVAToPoC(void* start, size_t size);
|
|
||||||
|
|
||||||
// dc cvac
|
|
||||||
void DataCacheLineCleanByVAToPoC(void* start, size_t size);
|
|
||||||
|
|
||||||
// dc zva
|
|
||||||
void DataCacheZeroByVA(void* start, size_t size);
|
|
||||||
|
|
||||||
} // namespace Common
|
|
|
@ -6,7 +6,6 @@
|
||||||
|
|
||||||
#include "common/assert.h"
|
#include "common/assert.h"
|
||||||
#include "common/atomic_ops.h"
|
#include "common/atomic_ops.h"
|
||||||
#include "common/cache_management.h"
|
|
||||||
#include "common/common_types.h"
|
#include "common/common_types.h"
|
||||||
#include "common/logging/log.h"
|
#include "common/logging/log.h"
|
||||||
#include "common/page_table.h"
|
#include "common/page_table.h"
|
||||||
|
@ -340,10 +339,9 @@ struct Memory::Impl {
|
||||||
LOG_ERROR(HW_Memory, "Unmapped cache maintenance @ {:#018X}", current_vaddr);
|
LOG_ERROR(HW_Memory, "Unmapped cache maintenance @ {:#018X}", current_vaddr);
|
||||||
throw InvalidMemoryException();
|
throw InvalidMemoryException();
|
||||||
},
|
},
|
||||||
[&](const std::size_t block_size, u8* const host_ptr) { cb(block_size, host_ptr); },
|
[&](const std::size_t block_size, u8* const host_ptr) {},
|
||||||
[&](const VAddr current_vaddr, const std::size_t block_size, u8* const host_ptr) {
|
[&](const VAddr current_vaddr, const std::size_t block_size, u8* const host_ptr) {
|
||||||
system.GPU().FlushRegion(current_vaddr, block_size);
|
cb(current_vaddr, block_size);
|
||||||
cb(block_size, host_ptr);
|
|
||||||
},
|
},
|
||||||
[](const std::size_t block_size) {});
|
[](const std::size_t block_size) {});
|
||||||
} catch (InvalidMemoryException&) {
|
} catch (InvalidMemoryException&) {
|
||||||
|
@ -354,27 +352,30 @@ struct Memory::Impl {
|
||||||
}
|
}
|
||||||
|
|
||||||
Result InvalidateDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size) {
|
Result InvalidateDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size) {
|
||||||
auto perform = [&](const std::size_t block_size, u8* const host_ptr) {
|
auto on_rasterizer = [&](const VAddr current_vaddr, const std::size_t block_size) {
|
||||||
// Do nothing; this operation (dc ivac) cannot be supported
|
// dc ivac: Invalidate to point of coherency
|
||||||
// from EL0
|
// GPU flush -> CPU invalidate
|
||||||
|
system.GPU().FlushRegion(current_vaddr, block_size);
|
||||||
};
|
};
|
||||||
return PerformCacheOperation(process, dest_addr, size, perform);
|
return PerformCacheOperation(process, dest_addr, size, on_rasterizer);
|
||||||
}
|
}
|
||||||
|
|
||||||
Result StoreDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size) {
|
Result StoreDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size) {
|
||||||
auto perform = [&](const std::size_t block_size, u8* const host_ptr) {
|
auto on_rasterizer = [&](const VAddr current_vaddr, const std::size_t block_size) {
|
||||||
// dc cvac: Store to point of coherency
|
// dc cvac: Store to point of coherency
|
||||||
Common::DataCacheLineCleanByVAToPoC(host_ptr, block_size);
|
// CPU flush -> GPU invalidate
|
||||||
|
system.GPU().InvalidateRegion(current_vaddr, block_size);
|
||||||
};
|
};
|
||||||
return PerformCacheOperation(process, dest_addr, size, perform);
|
return PerformCacheOperation(process, dest_addr, size, on_rasterizer);
|
||||||
}
|
}
|
||||||
|
|
||||||
Result FlushDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size) {
|
Result FlushDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size) {
|
||||||
auto perform = [&](const std::size_t block_size, u8* const host_ptr) {
|
auto on_rasterizer = [&](const VAddr current_vaddr, const std::size_t block_size) {
|
||||||
// dc civac: Store to point of coherency, and invalidate from cache
|
// dc civac: Store to point of coherency, and invalidate from cache
|
||||||
Common::DataCacheLineCleanAndInvalidateByVAToPoC(host_ptr, block_size);
|
// CPU flush -> GPU invalidate
|
||||||
|
system.GPU().InvalidateRegion(current_vaddr, block_size);
|
||||||
};
|
};
|
||||||
return PerformCacheOperation(process, dest_addr, size, perform);
|
return PerformCacheOperation(process, dest_addr, size, on_rasterizer);
|
||||||
}
|
}
|
||||||
|
|
||||||
void MarkRegionDebug(VAddr vaddr, u64 size, bool debug) {
|
void MarkRegionDebug(VAddr vaddr, u64 size, bool debug) {
|
||||||
|
|
Reference in New Issue