Merge pull request #9234 from liamwhite/data-cash-money
kernel: implement data cache management operations
This commit is contained in:
commit
1fb33bd1e1
|
@ -34,6 +34,8 @@ add_library(common STATIC
|
||||||
bit_util.h
|
bit_util.h
|
||||||
cityhash.cpp
|
cityhash.cpp
|
||||||
cityhash.h
|
cityhash.h
|
||||||
|
cache_management.cpp
|
||||||
|
cache_management.h
|
||||||
common_funcs.h
|
common_funcs.h
|
||||||
common_types.h
|
common_types.h
|
||||||
concepts.h
|
concepts.h
|
||||||
|
|
|
@ -0,0 +1,60 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#include <cstring>
|
||||||
|
|
||||||
|
#include "alignment.h"
|
||||||
|
#include "cache_management.h"
|
||||||
|
#include "common_types.h"
|
||||||
|
|
||||||
|
namespace Common {
|
||||||
|
|
||||||
|
#if defined(ARCHITECTURE_x86_64)
|
||||||
|
|
||||||
|
// Most cache operations are no-ops on x86
|
||||||
|
|
||||||
|
void DataCacheLineCleanByVAToPoU(void* start, size_t size) {}
|
||||||
|
void DataCacheLineCleanAndInvalidateByVAToPoC(void* start, size_t size) {}
|
||||||
|
void DataCacheLineCleanByVAToPoC(void* start, size_t size) {}
|
||||||
|
void DataCacheZeroByVA(void* start, size_t size) {
|
||||||
|
std::memset(start, 0, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
#elif defined(ARCHITECTURE_arm64)
|
||||||
|
|
||||||
|
// BS/DminLine is log2(cache size in words), we want size in bytes
|
||||||
|
#define EXTRACT_DMINLINE(ctr_el0) (1 << ((((ctr_el0) >> 16) & 0xf) + 2))
|
||||||
|
#define EXTRACT_BS(dczid_el0) (1 << (((dczid_el0)&0xf) + 2))
|
||||||
|
|
||||||
|
#define DEFINE_DC_OP(op_name, function_name) \
|
||||||
|
void function_name(void* start, size_t size) { \
|
||||||
|
size_t ctr_el0; \
|
||||||
|
asm volatile("mrs %[ctr_el0], ctr_el0\n\t" : [ctr_el0] "=r"(ctr_el0)); \
|
||||||
|
size_t cacheline_size = EXTRACT_DMINLINE(ctr_el0); \
|
||||||
|
uintptr_t va_start = reinterpret_cast<uintptr_t>(start); \
|
||||||
|
uintptr_t va_end = va_start + size; \
|
||||||
|
for (uintptr_t va = va_start; va < va_end; va += cacheline_size) { \
|
||||||
|
asm volatile("dc " #op_name ", %[va]\n\t" : : [va] "r"(va) : "memory"); \
|
||||||
|
} \
|
||||||
|
}
|
||||||
|
|
||||||
|
#define DEFINE_DC_OP_DCZID(op_name, function_name) \
|
||||||
|
void function_name(void* start, size_t size) { \
|
||||||
|
size_t dczid_el0; \
|
||||||
|
asm volatile("mrs %[dczid_el0], dczid_el0\n\t" : [dczid_el0] "=r"(dczid_el0)); \
|
||||||
|
size_t cacheline_size = EXTRACT_BS(dczid_el0); \
|
||||||
|
uintptr_t va_start = reinterpret_cast<uintptr_t>(start); \
|
||||||
|
uintptr_t va_end = va_start + size; \
|
||||||
|
for (uintptr_t va = va_start; va < va_end; va += cacheline_size) { \
|
||||||
|
asm volatile("dc " #op_name ", %[va]\n\t" : : [va] "r"(va) : "memory"); \
|
||||||
|
} \
|
||||||
|
}
|
||||||
|
|
||||||
|
DEFINE_DC_OP(cvau, DataCacheLineCleanByVAToPoU);
|
||||||
|
DEFINE_DC_OP(civac, DataCacheLineCleanAndInvalidateByVAToPoC);
|
||||||
|
DEFINE_DC_OP(cvac, DataCacheLineCleanByVAToPoC);
|
||||||
|
DEFINE_DC_OP_DCZID(zva, DataCacheZeroByVA);
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
} // namespace Common
|
|
@ -0,0 +1,27 @@
|
||||||
|
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
||||||
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "stdlib.h"
|
||||||
|
|
||||||
|
namespace Common {
|
||||||
|
|
||||||
|
// Data cache instructions enabled at EL0 by SCTLR_EL1.UCI.
|
||||||
|
// VA = virtual address
|
||||||
|
// PoC = point of coherency
|
||||||
|
// PoU = point of unification
|
||||||
|
|
||||||
|
// dc cvau
|
||||||
|
void DataCacheLineCleanByVAToPoU(void* start, size_t size);
|
||||||
|
|
||||||
|
// dc civac
|
||||||
|
void DataCacheLineCleanAndInvalidateByVAToPoC(void* start, size_t size);
|
||||||
|
|
||||||
|
// dc cvac
|
||||||
|
void DataCacheLineCleanByVAToPoC(void* start, size_t size);
|
||||||
|
|
||||||
|
// dc zva
|
||||||
|
void DataCacheZeroByVA(void* start, size_t size);
|
||||||
|
|
||||||
|
} // namespace Common
|
|
@ -2701,14 +2701,24 @@ static Result GetThreadList(Core::System& system, u32* out_num_threads, VAddr ou
|
||||||
return ResultSuccess;
|
return ResultSuccess;
|
||||||
}
|
}
|
||||||
|
|
||||||
static Result FlushProcessDataCache32([[maybe_unused]] Core::System& system,
|
static Result FlushProcessDataCache32(Core::System& system, Handle process_handle, u64 address,
|
||||||
[[maybe_unused]] Handle handle, [[maybe_unused]] u32 address,
|
u64 size) {
|
||||||
[[maybe_unused]] u32 size) {
|
// Validate address/size.
|
||||||
// Note(Blinkhawk): For emulation purposes of the data cache this is mostly a no-op,
|
R_UNLESS(size > 0, ResultInvalidSize);
|
||||||
// as all emulation is done in the same cache level in host architecture, thus data cache
|
R_UNLESS(address == static_cast<uintptr_t>(address), ResultInvalidCurrentMemory);
|
||||||
// does not need flushing.
|
R_UNLESS(size == static_cast<size_t>(size), ResultInvalidCurrentMemory);
|
||||||
LOG_DEBUG(Kernel_SVC, "called");
|
|
||||||
return ResultSuccess;
|
// Get the process from its handle.
|
||||||
|
KScopedAutoObject process =
|
||||||
|
system.Kernel().CurrentProcess()->GetHandleTable().GetObject<KProcess>(process_handle);
|
||||||
|
R_UNLESS(process.IsNotNull(), ResultInvalidHandle);
|
||||||
|
|
||||||
|
// Verify the region is within range.
|
||||||
|
auto& page_table = process->PageTable();
|
||||||
|
R_UNLESS(page_table.Contains(address, size), ResultInvalidCurrentMemory);
|
||||||
|
|
||||||
|
// Perform the operation.
|
||||||
|
R_RETURN(system.Memory().FlushDataCache(*process, address, size));
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
|
@ -722,4 +722,12 @@ void SvcWrap32(Core::System& system) {
|
||||||
FuncReturn(system, retval);
|
FuncReturn(system, retval);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Used by Invalidate/Store/FlushProcessDataCache32
|
||||||
|
template <Result func(Core::System&, Handle, u64, u64)>
|
||||||
|
void SvcWrap32(Core::System& system) {
|
||||||
|
const u64 address = (Param(system, 3) << 32) | Param(system, 2);
|
||||||
|
const u64 size = (Param(system, 4) << 32) | Param(system, 1);
|
||||||
|
FuncReturn32(system, func(system, Param32(system, 0), address, size).raw);
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace Kernel
|
} // namespace Kernel
|
||||||
|
|
|
@ -6,6 +6,7 @@
|
||||||
|
|
||||||
#include "common/assert.h"
|
#include "common/assert.h"
|
||||||
#include "common/atomic_ops.h"
|
#include "common/atomic_ops.h"
|
||||||
|
#include "common/cache_management.h"
|
||||||
#include "common/common_types.h"
|
#include "common/common_types.h"
|
||||||
#include "common/logging/log.h"
|
#include "common/logging/log.h"
|
||||||
#include "common/page_table.h"
|
#include "common/page_table.h"
|
||||||
|
@ -329,6 +330,55 @@ struct Memory::Impl {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <typename Callback>
|
||||||
|
Result PerformCacheOperation(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size,
|
||||||
|
Callback&& cb) {
|
||||||
|
class InvalidMemoryException : public std::exception {};
|
||||||
|
|
||||||
|
try {
|
||||||
|
WalkBlock(
|
||||||
|
process, dest_addr, size,
|
||||||
|
[&](const std::size_t block_size, const VAddr current_vaddr) {
|
||||||
|
LOG_ERROR(HW_Memory, "Unmapped cache maintenance @ {:#018X}", current_vaddr);
|
||||||
|
throw InvalidMemoryException();
|
||||||
|
},
|
||||||
|
[&](const std::size_t block_size, u8* const host_ptr) { cb(block_size, host_ptr); },
|
||||||
|
[&](const VAddr current_vaddr, const std::size_t block_size, u8* const host_ptr) {
|
||||||
|
system.GPU().FlushRegion(current_vaddr, block_size);
|
||||||
|
cb(block_size, host_ptr);
|
||||||
|
},
|
||||||
|
[](const std::size_t block_size) {});
|
||||||
|
} catch (InvalidMemoryException&) {
|
||||||
|
return Kernel::ResultInvalidCurrentMemory;
|
||||||
|
}
|
||||||
|
|
||||||
|
return ResultSuccess;
|
||||||
|
}
|
||||||
|
|
||||||
|
Result InvalidateDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size) {
|
||||||
|
auto perform = [&](const std::size_t block_size, u8* const host_ptr) {
|
||||||
|
// Do nothing; this operation (dc ivac) cannot be supported
|
||||||
|
// from EL0
|
||||||
|
};
|
||||||
|
return PerformCacheOperation(process, dest_addr, size, perform);
|
||||||
|
}
|
||||||
|
|
||||||
|
Result StoreDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size) {
|
||||||
|
auto perform = [&](const std::size_t block_size, u8* const host_ptr) {
|
||||||
|
// dc cvac: Store to point of coherency
|
||||||
|
Common::DataCacheLineCleanByVAToPoC(host_ptr, block_size);
|
||||||
|
};
|
||||||
|
return PerformCacheOperation(process, dest_addr, size, perform);
|
||||||
|
}
|
||||||
|
|
||||||
|
Result FlushDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size) {
|
||||||
|
auto perform = [&](const std::size_t block_size, u8* const host_ptr) {
|
||||||
|
// dc civac: Store to point of coherency, and invalidate from cache
|
||||||
|
Common::DataCacheLineCleanAndInvalidateByVAToPoC(host_ptr, block_size);
|
||||||
|
};
|
||||||
|
return PerformCacheOperation(process, dest_addr, size, perform);
|
||||||
|
}
|
||||||
|
|
||||||
void MarkRegionDebug(VAddr vaddr, u64 size, bool debug) {
|
void MarkRegionDebug(VAddr vaddr, u64 size, bool debug) {
|
||||||
if (vaddr == 0) {
|
if (vaddr == 0) {
|
||||||
return;
|
return;
|
||||||
|
@ -786,6 +836,21 @@ void Memory::ZeroBlock(const Kernel::KProcess& process, VAddr dest_addr, const s
|
||||||
impl->ZeroBlock(process, dest_addr, size);
|
impl->ZeroBlock(process, dest_addr, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Result Memory::InvalidateDataCache(const Kernel::KProcess& process, VAddr dest_addr,
|
||||||
|
const std::size_t size) {
|
||||||
|
return impl->InvalidateDataCache(process, dest_addr, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
Result Memory::StoreDataCache(const Kernel::KProcess& process, VAddr dest_addr,
|
||||||
|
const std::size_t size) {
|
||||||
|
return impl->StoreDataCache(process, dest_addr, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
Result Memory::FlushDataCache(const Kernel::KProcess& process, VAddr dest_addr,
|
||||||
|
const std::size_t size) {
|
||||||
|
return impl->FlushDataCache(process, dest_addr, size);
|
||||||
|
}
|
||||||
|
|
||||||
void Memory::RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) {
|
void Memory::RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) {
|
||||||
impl->RasterizerMarkRegionCached(vaddr, size, cached);
|
impl->RasterizerMarkRegionCached(vaddr, size, cached);
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include "common/common_types.h"
|
#include "common/common_types.h"
|
||||||
|
#include "core/hle/result.h"
|
||||||
|
|
||||||
namespace Common {
|
namespace Common {
|
||||||
struct PageTable;
|
struct PageTable;
|
||||||
|
@ -449,6 +450,39 @@ public:
|
||||||
*/
|
*/
|
||||||
void ZeroBlock(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size);
|
void ZeroBlock(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Invalidates a range of bytes within the current process' address space at the specified
|
||||||
|
* virtual address.
|
||||||
|
*
|
||||||
|
* @param process The process that will have data invalidated within its address space.
|
||||||
|
* @param dest_addr The destination virtual address to invalidate the data from.
|
||||||
|
* @param size The size of the range to invalidate, in bytes.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
Result InvalidateDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stores a range of bytes within the current process' address space at the specified
|
||||||
|
* virtual address.
|
||||||
|
*
|
||||||
|
* @param process The process that will have data stored within its address space.
|
||||||
|
* @param dest_addr The destination virtual address to store the data from.
|
||||||
|
* @param size The size of the range to store, in bytes.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
Result StoreDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flushes a range of bytes within the current process' address space at the specified
|
||||||
|
* virtual address.
|
||||||
|
*
|
||||||
|
* @param process The process that will have data flushed within its address space.
|
||||||
|
* @param dest_addr The destination virtual address to flush the data from.
|
||||||
|
* @param size The size of the range to flush, in bytes.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
Result FlushDataCache(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Marks each page within the specified address range as cached or uncached.
|
* Marks each page within the specified address range as cached or uncached.
|
||||||
*
|
*
|
||||||
|
|
Reference in New Issue