From 9de860a419853bbaf913c07b1aae239c91c56d9e Mon Sep 17 00:00:00 2001 From: David Marcec Date: Wed, 22 Apr 2020 13:03:58 +1000 Subject: [PATCH 1/3] audio_renderer: Better voice mixing and 6 channel downmixing Supersedes #3738 and #3321 --- src/audio_core/audio_renderer.cpp | 96 +++++++++++++++++++++++++++---- src/audio_core/audio_renderer.h | 10 ++++ src/audio_core/common.h | 1 + 3 files changed, 96 insertions(+), 11 deletions(-) diff --git a/src/audio_core/audio_renderer.cpp b/src/audio_core/audio_renderer.cpp index d18ef6940..f54ce48c5 100644 --- a/src/audio_core/audio_renderer.cpp +++ b/src/audio_core/audio_renderer.cpp @@ -37,9 +37,11 @@ public: } void SetWaveIndex(std::size_t index); - std::vector DequeueSamples(std::size_t sample_count, Core::Memory::Memory& memory); + std::vector DequeueSamples(std::size_t sample_count, Core::Memory::Memory& memory, + std::array voice_resources); void UpdateState(); - void RefreshBuffer(Core::Memory::Memory& memory); + void RefreshBuffer(Core::Memory::Memory& memory, + std::array voice_resources); private: bool is_in_use{}; @@ -79,7 +81,7 @@ AudioRenderer::AudioRenderer(Core::Timing::CoreTiming& core_timing, Core::Memory std::shared_ptr buffer_event, std::size_t instance_number) : worker_params{params}, buffer_event{buffer_event}, voices(params.voice_count), - effects(params.effect_count), memory{memory_} { + voice_resources(params.voice_count), effects(params.effect_count), memory{memory_} { behavior_info.SetUserRevision(params.revision); audio_out = std::make_unique(); stream = audio_out->OpenStream(core_timing, STREAM_SAMPLE_RATE, STREAM_NUM_CHANNELS, @@ -127,6 +129,12 @@ ResultVal> AudioRenderer::UpdateAudioRenderer(const std::vector< input_params.data() + sizeof(UpdateDataHeader) + config.behavior_size, memory_pool_count * sizeof(MemoryPoolInfo)); + // Copy voice resources + const std::size_t voice_resource_offset{sizeof(UpdateDataHeader) + config.behavior_size + + config.memory_pools_size}; + std::memcpy(voice_resources.data(), input_params.data() + voice_resource_offset, + sizeof(VoiceResourceInformation) * voice_resources.size()); + // Copy VoiceInfo structs std::size_t voice_offset{sizeof(UpdateDataHeader) + config.behavior_size + config.memory_pools_size + config.voice_resource_size}; @@ -220,14 +228,15 @@ void AudioRenderer::VoiceState::SetWaveIndex(std::size_t index) { is_refresh_pending = true; } -std::vector AudioRenderer::VoiceState::DequeueSamples(std::size_t sample_count, - Core::Memory::Memory& memory) { +std::vector AudioRenderer::VoiceState::DequeueSamples( + std::size_t sample_count, Core::Memory::Memory& memory, + std::array voice_resources) { if (!IsPlaying()) { return {}; } if (is_refresh_pending) { - RefreshBuffer(memory); + RefreshBuffer(memory, voice_resources); } const std::size_t max_size{samples.size() - offset}; @@ -271,7 +280,8 @@ void AudioRenderer::VoiceState::UpdateState() { is_in_use = info.is_in_use; } -void AudioRenderer::VoiceState::RefreshBuffer(Core::Memory::Memory& memory) { +void AudioRenderer::VoiceState::RefreshBuffer( + Core::Memory::Memory& memory, std::array voice_resources) { const auto wave_buffer_address = info.wave_buffer[wave_index].buffer_addr; const auto wave_buffer_size = info.wave_buffer[wave_index].buffer_sz; std::vector new_samples(wave_buffer_size / sizeof(s16)); @@ -296,17 +306,75 @@ void AudioRenderer::VoiceState::RefreshBuffer(Core::Memory::Memory& memory) { } switch (info.channel_count) { - case 1: + case 1: { // 1 channel is upsampled to 2 channel samples.resize(new_samples.size() * 2); + for (std::size_t index = 0; index < new_samples.size(); ++index) { - samples[index * 2] = new_samples[index]; - samples[index * 2 + 1] = new_samples[index]; + auto sample = static_cast(new_samples[index]); + if (voice_resources[0]->in_use) { + sample *= voice_resources[0]->mix_volumes[0]; + } + + samples[index * 2] = static_cast(sample); + samples[index * 2 + 1] = static_cast(sample); } break; + } case 2: { // 2 channel is played as is samples = std::move(new_samples); + const std::size_t sample_count = (samples.size() / 2); + for (std::size_t index = 0; index < sample_count; ++index) { + const std::size_t index_l = index * 2; + const std::size_t index_r = index * 2 + 1; + + auto sample_l = static_cast(samples[index_l]); + auto sample_r = static_cast(samples[index_r]); + + if (voice_resources[0]->in_use) { + sample_l *= voice_resources[0]->mix_volumes[0]; + } + + if (voice_resources[1]->in_use) { + sample_l *= voice_resources[1]->mix_volumes[1]; + } + + samples[index_l] = static_cast(sample_l); + samples[index_r] = static_cast(sample_r); + } + break; + } + case 6: { + samples.resize((new_samples.size() / 6) * 2); + const std::size_t sample_count = samples.size() / 2; + + for (std::size_t index = 0; index < sample_count; ++index) { + auto FL = static_cast(new_samples[index * 6]); + auto FR = static_cast(new_samples[index * 6 + 1]); + auto FC = static_cast(new_samples[index * 6 + 2]); + auto BL = static_cast(new_samples[index * 6 + 4]); + auto BR = static_cast(new_samples[index * 6 + 5]); + + if (voice_resources[0]->in_use) { + FL *= voice_resources[0]->mix_volumes[0]; + } + if (voice_resources[1]->in_use) { + FR *= voice_resources[1]->mix_volumes[1]; + } + if (voice_resources[2]->in_use) { + FC *= voice_resources[2]->mix_volumes[2]; + } + if (voice_resources[4]->in_use) { + BL *= voice_resources[4]->mix_volumes[4]; + } + if (voice_resources[5]->in_use) { + BR *= voice_resources[5]->mix_volumes[5]; + } + + samples[index * 2] = static_cast(0.3694f * FL + 0.2612f * FC + 0.3694f * BL); + samples[index * 2 + 1] = static_cast(0.3694f * FR + 0.2612f * FC + 0.3694f * BR); + } break; } default: @@ -352,11 +420,17 @@ void AudioRenderer::QueueMixedBuffer(Buffer::Tag tag) { if (!voice.IsPlaying()) { continue; } + std::array resources{}; + for (u32 channel = 0; channel < voice.GetInfo().channel_count; channel++) { + const auto channel_resource_id = voice.GetInfo().voice_channel_resource_ids[channel]; + resources[channel] = &voice_resources[channel_resource_id]; + } std::size_t offset{}; s64 samples_remaining{BUFFER_SIZE}; while (samples_remaining > 0) { - const std::vector samples{voice.DequeueSamples(samples_remaining, memory)}; + const std::vector samples{ + voice.DequeueSamples(samples_remaining, memory, resources)}; if (samples.empty()) { break; diff --git a/src/audio_core/audio_renderer.h b/src/audio_core/audio_renderer.h index b42770fae..1f9114c07 100644 --- a/src/audio_core/audio_renderer.h +++ b/src/audio_core/audio_renderer.h @@ -9,6 +9,7 @@ #include #include "audio_core/behavior_info.h" +#include "audio_core/common.h" #include "audio_core/stream.h" #include "common/common_funcs.h" #include "common/common_types.h" @@ -116,6 +117,14 @@ struct WaveBuffer { }; static_assert(sizeof(WaveBuffer) == 0x38, "WaveBuffer has wrong size"); +struct VoiceResourceInformation { + s32_le id{}; + std::array mix_volumes{}; + bool in_use{}; + INSERT_PADDING_BYTES(11); +}; +static_assert(sizeof(VoiceResourceInformation) == 0x70, "VoiceResourceInformation has wrong size"); + struct VoiceInfo { u32_le id; u32_le node_id; @@ -244,6 +253,7 @@ private: AudioRendererParameter worker_params; std::shared_ptr buffer_event; std::vector voices; + std::vector voice_resources; std::vector effects; std::unique_ptr audio_out; StreamPtr stream; diff --git a/src/audio_core/common.h b/src/audio_core/common.h index 98478b66b..7bb145c53 100644 --- a/src/audio_core/common.h +++ b/src/audio_core/common.h @@ -14,6 +14,7 @@ constexpr ResultCode ERR_INVALID_PARAMETERS{ErrorModule::Audio, 41}; } constexpr u32_le CURRENT_PROCESS_REVISION = Common::MakeMagic('R', 'E', 'V', '8'); +constexpr std::size_t MAX_MIX_BUFFERS = 24; static constexpr u32 VersionFromRevision(u32_le rev) { // "REV7" -> 7 From c4e7ec7a996d49517852d1f5d46160cfc2bd6032 Mon Sep 17 00:00:00 2001 From: David Marcec Date: Wed, 22 Apr 2020 13:12:58 +1000 Subject: [PATCH 2/3] pass by const ref instead --- src/audio_core/audio_renderer.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/audio_core/audio_renderer.cpp b/src/audio_core/audio_renderer.cpp index f54ce48c5..fc6e70f6b 100644 --- a/src/audio_core/audio_renderer.cpp +++ b/src/audio_core/audio_renderer.cpp @@ -17,7 +17,7 @@ namespace AudioCore { constexpr u32 STREAM_SAMPLE_RATE{48000}; constexpr u32 STREAM_NUM_CHANNELS{2}; - +using VoiceChannelHolder = std::array; class AudioRenderer::VoiceState { public: bool IsPlaying() const { @@ -38,10 +38,9 @@ public: void SetWaveIndex(std::size_t index); std::vector DequeueSamples(std::size_t sample_count, Core::Memory::Memory& memory, - std::array voice_resources); + const VoiceChannelHolder& voice_resources); void UpdateState(); - void RefreshBuffer(Core::Memory::Memory& memory, - std::array voice_resources); + void RefreshBuffer(Core::Memory::Memory& memory, const VoiceChannelHolder& voice_resources); private: bool is_in_use{}; @@ -230,7 +229,7 @@ void AudioRenderer::VoiceState::SetWaveIndex(std::size_t index) { std::vector AudioRenderer::VoiceState::DequeueSamples( std::size_t sample_count, Core::Memory::Memory& memory, - std::array voice_resources) { + const VoiceChannelHolder& voice_resources) { if (!IsPlaying()) { return {}; } @@ -280,8 +279,8 @@ void AudioRenderer::VoiceState::UpdateState() { is_in_use = info.is_in_use; } -void AudioRenderer::VoiceState::RefreshBuffer( - Core::Memory::Memory& memory, std::array voice_resources) { +void AudioRenderer::VoiceState::RefreshBuffer(Core::Memory::Memory& memory, + const VoiceChannelHolder& voice_resources) { const auto wave_buffer_address = info.wave_buffer[wave_index].buffer_addr; const auto wave_buffer_size = info.wave_buffer[wave_index].buffer_sz; std::vector new_samples(wave_buffer_size / sizeof(s16)); @@ -420,7 +419,7 @@ void AudioRenderer::QueueMixedBuffer(Buffer::Tag tag) { if (!voice.IsPlaying()) { continue; } - std::array resources{}; + VoiceChannelHolder resources{}; for (u32 channel = 0; channel < voice.GetInfo().channel_count; channel++) { const auto channel_resource_id = voice.GetInfo().voice_channel_resource_ids[channel]; resources[channel] = &voice_resources[channel_resource_id]; From 16c0373adc5bd76db97d5d2231e80463dda242dd Mon Sep 17 00:00:00 2001 From: David Marcec Date: Wed, 22 Apr 2020 14:36:08 +1000 Subject: [PATCH 3/3] fix logic error & scale sample volume based on voice volume --- src/audio_core/audio_renderer.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/audio_core/audio_renderer.cpp b/src/audio_core/audio_renderer.cpp index fc6e70f6b..50846a854 100644 --- a/src/audio_core/audio_renderer.cpp +++ b/src/audio_core/audio_renderer.cpp @@ -315,8 +315,8 @@ void AudioRenderer::VoiceState::RefreshBuffer(Core::Memory::Memory& memory, sample *= voice_resources[0]->mix_volumes[0]; } - samples[index * 2] = static_cast(sample); - samples[index * 2 + 1] = static_cast(sample); + samples[index * 2] = static_cast(sample * info.volume); + samples[index * 2 + 1] = static_cast(sample * info.volume); } break; } @@ -336,11 +336,11 @@ void AudioRenderer::VoiceState::RefreshBuffer(Core::Memory::Memory& memory, } if (voice_resources[1]->in_use) { - sample_l *= voice_resources[1]->mix_volumes[1]; + sample_r *= voice_resources[1]->mix_volumes[1]; } - samples[index_l] = static_cast(sample_l); - samples[index_r] = static_cast(sample_r); + samples[index_l] = static_cast(sample_l * info.volume); + samples[index_r] = static_cast(sample_r * info.volume); } break; } @@ -371,8 +371,10 @@ void AudioRenderer::VoiceState::RefreshBuffer(Core::Memory::Memory& memory, BR *= voice_resources[5]->mix_volumes[5]; } - samples[index * 2] = static_cast(0.3694f * FL + 0.2612f * FC + 0.3694f * BL); - samples[index * 2 + 1] = static_cast(0.3694f * FR + 0.2612f * FC + 0.3694f * BR); + samples[index * 2] = + static_cast((0.3694f * FL + 0.2612f * FC + 0.3694f * BL) * info.volume); + samples[index * 2 + 1] = + static_cast((0.3694f * FR + 0.2612f * FC + 0.3694f * BR) * info.volume); } break; }