shader_ir/warp: Implement SHFL
This commit is contained in:
parent
393cc3ef2f
commit
0526bf1895
|
@ -566,6 +566,13 @@ enum class ImageAtomicOperation : u64 {
|
||||||
Exch = 8,
|
Exch = 8,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
enum class ShuffleOperation : u64 {
|
||||||
|
Idx = 0, // shuffleNV
|
||||||
|
Up = 1, // shuffleUpNV
|
||||||
|
Down = 2, // shuffleDownNV
|
||||||
|
Bfly = 3, // shuffleXorNV
|
||||||
|
};
|
||||||
|
|
||||||
union Instruction {
|
union Instruction {
|
||||||
Instruction& operator=(const Instruction& instr) {
|
Instruction& operator=(const Instruction& instr) {
|
||||||
value = instr.value;
|
value = instr.value;
|
||||||
|
@ -599,6 +606,15 @@ union Instruction {
|
||||||
BitField<42, 1, u64> negate_value;
|
BitField<42, 1, u64> negate_value;
|
||||||
} vote;
|
} vote;
|
||||||
|
|
||||||
|
union {
|
||||||
|
BitField<30, 2, ShuffleOperation> operation;
|
||||||
|
BitField<48, 3, u64> pred48;
|
||||||
|
BitField<28, 1, u64> is_index_imm;
|
||||||
|
BitField<29, 1, u64> is_mask_imm;
|
||||||
|
BitField<20, 5, u64> index_imm;
|
||||||
|
BitField<34, 13, u64> mask_imm;
|
||||||
|
} shfl;
|
||||||
|
|
||||||
union {
|
union {
|
||||||
BitField<8, 8, Register> gpr;
|
BitField<8, 8, Register> gpr;
|
||||||
BitField<20, 24, s64> offset;
|
BitField<20, 24, s64> offset;
|
||||||
|
@ -1542,6 +1558,7 @@ public:
|
||||||
BRK,
|
BRK,
|
||||||
DEPBAR,
|
DEPBAR,
|
||||||
VOTE,
|
VOTE,
|
||||||
|
SHFL,
|
||||||
BFE_C,
|
BFE_C,
|
||||||
BFE_R,
|
BFE_R,
|
||||||
BFE_IMM,
|
BFE_IMM,
|
||||||
|
@ -1833,6 +1850,7 @@ private:
|
||||||
INST("111000110000----", Id::EXIT, Type::Flow, "EXIT"),
|
INST("111000110000----", Id::EXIT, Type::Flow, "EXIT"),
|
||||||
INST("1111000011110---", Id::DEPBAR, Type::Synch, "DEPBAR"),
|
INST("1111000011110---", Id::DEPBAR, Type::Synch, "DEPBAR"),
|
||||||
INST("0101000011011---", Id::VOTE, Type::Warp, "VOTE"),
|
INST("0101000011011---", Id::VOTE, Type::Warp, "VOTE"),
|
||||||
|
INST("1110111100010---", Id::SHFL, Type::Warp, "SHFL"),
|
||||||
INST("1110111111011---", Id::LD_A, Type::Memory, "LD_A"),
|
INST("1110111111011---", Id::LD_A, Type::Memory, "LD_A"),
|
||||||
INST("1110111101001---", Id::LD_S, Type::Memory, "LD_S"),
|
INST("1110111101001---", Id::LD_S, Type::Memory, "LD_S"),
|
||||||
INST("1110111101000---", Id::LD_L, Type::Memory, "LD_L"),
|
INST("1110111101000---", Id::LD_L, Type::Memory, "LD_L"),
|
||||||
|
|
|
@ -214,7 +214,8 @@ CachedProgram SpecializeShader(const std::string& code, const GLShader::ShaderEn
|
||||||
std::string source = "#version 430 core\n"
|
std::string source = "#version 430 core\n"
|
||||||
"#extension GL_ARB_separate_shader_objects : enable\n"
|
"#extension GL_ARB_separate_shader_objects : enable\n"
|
||||||
"#extension GL_NV_gpu_shader5 : enable\n"
|
"#extension GL_NV_gpu_shader5 : enable\n"
|
||||||
"#extension GL_NV_shader_thread_group : enable\n";
|
"#extension GL_NV_shader_thread_group : enable\n"
|
||||||
|
"#extension GL_NV_shader_thread_shuffle : enable\n";
|
||||||
if (entries.shader_viewport_layer_array) {
|
if (entries.shader_viewport_layer_array) {
|
||||||
source += "#extension GL_ARB_shader_viewport_layer_array : enable\n";
|
source += "#extension GL_ARB_shader_viewport_layer_array : enable\n";
|
||||||
}
|
}
|
||||||
|
|
|
@ -1934,8 +1934,7 @@ private:
|
||||||
Expression BallotThread(Operation operation) {
|
Expression BallotThread(Operation operation) {
|
||||||
const std::string value = VisitOperand(operation, 0).AsBool();
|
const std::string value = VisitOperand(operation, 0).AsBool();
|
||||||
if (!device.HasWarpIntrinsics()) {
|
if (!device.HasWarpIntrinsics()) {
|
||||||
LOG_ERROR(Render_OpenGL,
|
LOG_ERROR(Render_OpenGL, "Nvidia vote intrinsics are required by this shader");
|
||||||
"Nvidia warp intrinsics are not available and its required by a shader");
|
|
||||||
// Stub on non-Nvidia devices by simulating all threads voting the same as the active
|
// Stub on non-Nvidia devices by simulating all threads voting the same as the active
|
||||||
// one.
|
// one.
|
||||||
return {fmt::format("({} ? 0xFFFFFFFFU : 0U)", value), Type::Uint};
|
return {fmt::format("({} ? 0xFFFFFFFFU : 0U)", value), Type::Uint};
|
||||||
|
@ -1946,8 +1945,7 @@ private:
|
||||||
Expression Vote(Operation operation, const char* func) {
|
Expression Vote(Operation operation, const char* func) {
|
||||||
const std::string value = VisitOperand(operation, 0).AsBool();
|
const std::string value = VisitOperand(operation, 0).AsBool();
|
||||||
if (!device.HasWarpIntrinsics()) {
|
if (!device.HasWarpIntrinsics()) {
|
||||||
LOG_ERROR(Render_OpenGL,
|
LOG_ERROR(Render_OpenGL, "Nvidia vote intrinsics are required by this shader");
|
||||||
"Nvidia vote intrinsics are not available and its required by a shader");
|
|
||||||
// Stub with a warp size of one.
|
// Stub with a warp size of one.
|
||||||
return {value, Type::Bool};
|
return {value, Type::Bool};
|
||||||
}
|
}
|
||||||
|
@ -1964,15 +1962,54 @@ private:
|
||||||
|
|
||||||
Expression VoteEqual(Operation operation) {
|
Expression VoteEqual(Operation operation) {
|
||||||
if (!device.HasWarpIntrinsics()) {
|
if (!device.HasWarpIntrinsics()) {
|
||||||
LOG_ERROR(Render_OpenGL,
|
LOG_ERROR(Render_OpenGL, "Nvidia vote intrinsics are required by this shader");
|
||||||
"Nvidia vote intrinsics are not available and its required by a shader");
|
// We must return true here since a stub for a theoretical warp size of 1.
|
||||||
// We must return true here since a stub for a theoretical warp size of 1 will always
|
// This will always return an equal result across all votes.
|
||||||
// return an equal result for all its votes.
|
|
||||||
return {"true", Type::Bool};
|
return {"true", Type::Bool};
|
||||||
}
|
}
|
||||||
return Vote(operation, "allThreadsEqualNV");
|
return Vote(operation, "allThreadsEqualNV");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <const std::string_view& func>
|
||||||
|
Expression Shuffle(Operation operation) {
|
||||||
|
const std::string value = VisitOperand(operation, 0).AsFloat();
|
||||||
|
if (!device.HasWarpIntrinsics()) {
|
||||||
|
LOG_ERROR(Render_OpenGL, "Nvidia shuffle intrinsics are required by this shader");
|
||||||
|
// On a "single-thread" device we are either on the same thread or out of bounds. Both
|
||||||
|
// cases return the passed value.
|
||||||
|
return {value, Type::Float};
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string index = VisitOperand(operation, 1).AsUint();
|
||||||
|
const std::string width = VisitOperand(operation, 2).AsUint();
|
||||||
|
return {fmt::format("{}({}, {}, {})", func, value, index, width), Type::Float};
|
||||||
|
}
|
||||||
|
|
||||||
|
template <const std::string_view& func>
|
||||||
|
Expression InRangeShuffle(Operation operation) {
|
||||||
|
const std::string index = VisitOperand(operation, 0).AsUint();
|
||||||
|
const std::string width = VisitOperand(operation, 1).AsUint();
|
||||||
|
if (!device.HasWarpIntrinsics()) {
|
||||||
|
// On a "single-thread" device we are only in bounds when the requested index is 0.
|
||||||
|
return {fmt::format("({} == 0U)", index), Type::Bool};
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string in_range = code.GenerateTemporary();
|
||||||
|
code.AddLine("bool {};", in_range);
|
||||||
|
code.AddLine("{}(0U, {}, {}, {});", func, index, width, in_range);
|
||||||
|
return {in_range, Type::Bool};
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Func final {
|
||||||
|
Func() = delete;
|
||||||
|
~Func() = delete;
|
||||||
|
|
||||||
|
static constexpr std::string_view ShuffleIndexed = "shuffleNV";
|
||||||
|
static constexpr std::string_view ShuffleUp = "shuffleUpNV";
|
||||||
|
static constexpr std::string_view ShuffleDown = "shuffleDownNV";
|
||||||
|
static constexpr std::string_view ShuffleButterfly = "shuffleXorNV";
|
||||||
|
};
|
||||||
|
|
||||||
static constexpr std::array operation_decompilers = {
|
static constexpr std::array operation_decompilers = {
|
||||||
&GLSLDecompiler::Assign,
|
&GLSLDecompiler::Assign,
|
||||||
|
|
||||||
|
@ -2135,6 +2172,16 @@ private:
|
||||||
&GLSLDecompiler::VoteAll,
|
&GLSLDecompiler::VoteAll,
|
||||||
&GLSLDecompiler::VoteAny,
|
&GLSLDecompiler::VoteAny,
|
||||||
&GLSLDecompiler::VoteEqual,
|
&GLSLDecompiler::VoteEqual,
|
||||||
|
|
||||||
|
&GLSLDecompiler::Shuffle<Func::ShuffleIndexed>,
|
||||||
|
&GLSLDecompiler::Shuffle<Func::ShuffleUp>,
|
||||||
|
&GLSLDecompiler::Shuffle<Func::ShuffleDown>,
|
||||||
|
&GLSLDecompiler::Shuffle<Func::ShuffleButterfly>,
|
||||||
|
|
||||||
|
&GLSLDecompiler::InRangeShuffle<Func::ShuffleIndexed>,
|
||||||
|
&GLSLDecompiler::InRangeShuffle<Func::ShuffleUp>,
|
||||||
|
&GLSLDecompiler::InRangeShuffle<Func::ShuffleDown>,
|
||||||
|
&GLSLDecompiler::InRangeShuffle<Func::ShuffleButterfly>,
|
||||||
};
|
};
|
||||||
static_assert(operation_decompilers.size() == static_cast<std::size_t>(OperationCode::Amount));
|
static_assert(operation_decompilers.size() == static_cast<std::size_t>(OperationCode::Amount));
|
||||||
|
|
||||||
|
|
|
@ -1127,6 +1127,46 @@ private:
|
||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Id ShuffleIndexed(Operation) {
|
||||||
|
UNIMPLEMENTED();
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
Id ShuffleUp(Operation) {
|
||||||
|
UNIMPLEMENTED();
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
Id ShuffleDown(Operation) {
|
||||||
|
UNIMPLEMENTED();
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
Id ShuffleButterfly(Operation) {
|
||||||
|
UNIMPLEMENTED();
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
Id InRangeShuffleIndexed(Operation) {
|
||||||
|
UNIMPLEMENTED();
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
Id InRangeShuffleUp(Operation) {
|
||||||
|
UNIMPLEMENTED();
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
Id InRangeShuffleDown(Operation) {
|
||||||
|
UNIMPLEMENTED();
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
Id InRangeShuffleButterfly(Operation) {
|
||||||
|
UNIMPLEMENTED();
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
Id DeclareBuiltIn(spv::BuiltIn builtin, spv::StorageClass storage, Id type,
|
Id DeclareBuiltIn(spv::BuiltIn builtin, spv::StorageClass storage, Id type,
|
||||||
const std::string& name) {
|
const std::string& name) {
|
||||||
const Id id = OpVariable(type, storage);
|
const Id id = OpVariable(type, storage);
|
||||||
|
@ -1431,6 +1471,16 @@ private:
|
||||||
&SPIRVDecompiler::VoteAll,
|
&SPIRVDecompiler::VoteAll,
|
||||||
&SPIRVDecompiler::VoteAny,
|
&SPIRVDecompiler::VoteAny,
|
||||||
&SPIRVDecompiler::VoteEqual,
|
&SPIRVDecompiler::VoteEqual,
|
||||||
|
|
||||||
|
&SPIRVDecompiler::ShuffleIndexed,
|
||||||
|
&SPIRVDecompiler::ShuffleUp,
|
||||||
|
&SPIRVDecompiler::ShuffleDown,
|
||||||
|
&SPIRVDecompiler::ShuffleButterfly,
|
||||||
|
|
||||||
|
&SPIRVDecompiler::InRangeShuffleIndexed,
|
||||||
|
&SPIRVDecompiler::InRangeShuffleUp,
|
||||||
|
&SPIRVDecompiler::InRangeShuffleDown,
|
||||||
|
&SPIRVDecompiler::InRangeShuffleButterfly,
|
||||||
};
|
};
|
||||||
static_assert(operation_decompilers.size() == static_cast<std::size_t>(OperationCode::Amount));
|
static_assert(operation_decompilers.size() == static_cast<std::size_t>(OperationCode::Amount));
|
||||||
|
|
||||||
|
|
|
@ -13,6 +13,7 @@ namespace VideoCommon::Shader {
|
||||||
using Tegra::Shader::Instruction;
|
using Tegra::Shader::Instruction;
|
||||||
using Tegra::Shader::OpCode;
|
using Tegra::Shader::OpCode;
|
||||||
using Tegra::Shader::Pred;
|
using Tegra::Shader::Pred;
|
||||||
|
using Tegra::Shader::ShuffleOperation;
|
||||||
using Tegra::Shader::VoteOperation;
|
using Tegra::Shader::VoteOperation;
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
@ -44,6 +45,52 @@ u32 ShaderIR::DecodeWarp(NodeBlock& bb, u32 pc) {
|
||||||
SetPredicate(bb, instr.vote.dest_pred, vote);
|
SetPredicate(bb, instr.vote.dest_pred, vote);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
case OpCode::Id::SHFL: {
|
||||||
|
Node mask = instr.shfl.is_mask_imm ? Immediate(static_cast<u32>(instr.shfl.mask_imm))
|
||||||
|
: GetRegister(instr.gpr39);
|
||||||
|
Node width = [&] {
|
||||||
|
// Convert the obscure SHFL mask back into GL_NV_shader_thread_shuffle's width. This has
|
||||||
|
// been done reversing Nvidia's math. It won't work on all cases due to SHFL having
|
||||||
|
// different parameters that don't properly map to GLSL's interface, but it should work
|
||||||
|
// for cases emitted by Nvidia's compiler.
|
||||||
|
if (instr.shfl.operation == ShuffleOperation::Up) {
|
||||||
|
return Operation(
|
||||||
|
OperationCode::ILogicalShiftRight,
|
||||||
|
Operation(OperationCode::IAdd, std::move(mask), Immediate(-0x2000)),
|
||||||
|
Immediate(8));
|
||||||
|
} else {
|
||||||
|
return Operation(OperationCode::ILogicalShiftRight,
|
||||||
|
Operation(OperationCode::IAdd, Immediate(0x201F),
|
||||||
|
Operation(OperationCode::INegate, std::move(mask))),
|
||||||
|
Immediate(8));
|
||||||
|
}
|
||||||
|
}();
|
||||||
|
|
||||||
|
const auto [operation, in_range] = [instr]() -> std::pair<OperationCode, OperationCode> {
|
||||||
|
switch (instr.shfl.operation) {
|
||||||
|
case ShuffleOperation::Idx:
|
||||||
|
return {OperationCode::ShuffleIndexed, OperationCode::InRangeShuffleIndexed};
|
||||||
|
case ShuffleOperation::Up:
|
||||||
|
return {OperationCode::ShuffleUp, OperationCode::InRangeShuffleUp};
|
||||||
|
case ShuffleOperation::Down:
|
||||||
|
return {OperationCode::ShuffleDown, OperationCode::InRangeShuffleDown};
|
||||||
|
case ShuffleOperation::Bfly:
|
||||||
|
return {OperationCode::ShuffleButterfly, OperationCode::InRangeShuffleButterfly};
|
||||||
|
}
|
||||||
|
UNREACHABLE_MSG("Invalid SHFL operation: {}",
|
||||||
|
static_cast<u64>(instr.shfl.operation.Value()));
|
||||||
|
return {};
|
||||||
|
}();
|
||||||
|
|
||||||
|
// Setting the predicate before the register is intentional to avoid overwriting.
|
||||||
|
Node index = instr.shfl.is_index_imm ? Immediate(static_cast<u32>(instr.shfl.index_imm))
|
||||||
|
: GetRegister(instr.gpr20);
|
||||||
|
SetPredicate(bb, instr.shfl.pred48, Operation(in_range, index, width));
|
||||||
|
SetRegister(
|
||||||
|
bb, instr.gpr0,
|
||||||
|
Operation(operation, GetRegister(instr.gpr8), std::move(index), std::move(width)));
|
||||||
|
break;
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
UNIMPLEMENTED_MSG("Unhandled warp instruction: {}", opcode->get().GetName());
|
UNIMPLEMENTED_MSG("Unhandled warp instruction: {}", opcode->get().GetName());
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -181,6 +181,16 @@ enum class OperationCode {
|
||||||
VoteAny, /// (bool) -> bool
|
VoteAny, /// (bool) -> bool
|
||||||
VoteEqual, /// (bool) -> bool
|
VoteEqual, /// (bool) -> bool
|
||||||
|
|
||||||
|
ShuffleIndexed, /// (uint value, uint index, uint width) -> uint
|
||||||
|
ShuffleUp, /// (uint value, uint index, uint width) -> uint
|
||||||
|
ShuffleDown, /// (uint value, uint index, uint width) -> uint
|
||||||
|
ShuffleButterfly, /// (uint value, uint index, uint width) -> uint
|
||||||
|
|
||||||
|
InRangeShuffleIndexed, /// (uint index, uint width) -> bool
|
||||||
|
InRangeShuffleUp, /// (uint index, uint width) -> bool
|
||||||
|
InRangeShuffleDown, /// (uint index, uint width) -> bool
|
||||||
|
InRangeShuffleButterfly, /// (uint index, uint width) -> bool
|
||||||
|
|
||||||
Amount,
|
Amount,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Reference in New Issue