MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
SileroVadModel.cpp
Go to the documentation of this file.
1#include "SileroVadModel.hpp"
2
8
9namespace MayaFlux::Yantra {
10
11namespace {
12
13 enum Slot : size_t {
14 StftWeight,
15 Conv1Weight,
16 Conv1Bias,
17 Conv2Weight,
18 Conv2Bias,
19 Conv3Weight,
20 Conv3Bias,
21 Conv4Weight,
22 Conv4Bias,
23 LstmInputWeight,
24 LstmHiddenWeight,
25 LstmInputBias,
26 LstmHiddenBias,
27 HeadWeight,
28 HeadBias,
29 SlotCount
30 };
31
33
34 const std::array<TensorRequirement, SlotCount>& slot_specs()
35 {
36 static const std::array<TensorRequirement, SlotCount> specs { {
37 { "stft_conv.weight", { 258, 1, 256 } },
38 { "conv1.weight", { 128, 129, 3 } },
39 { "conv1.bias", { 128 } },
40 { "conv2.weight", { 64, 128, 3 } },
41 { "conv2.bias", { 64 } },
42 { "conv3.weight", { 64, 64, 3 } },
43 { "conv3.bias", { 64 } },
44 { "conv4.weight", { 128, 64, 3 } },
45 { "conv4.bias", { 128 } },
46 { "lstm_cell.weight_ih", { 512, 128 } },
47 { "lstm_cell.weight_hh", { 512, 128 } },
48 { "lstm_cell.bias_ih", { 512 } },
49 { "lstm_cell.bias_hh", { 512 } },
50 { "final_conv.weight", { 1, 128, 1 } },
51 { "final_conv.bias", { 1 } },
52 } };
53 return specs;
54 }
55
56 enum Buffer : uint32_t {
57 Audio,
58 Frames,
59 Transform,
60 Magnitude,
61 Encoded1,
62 Encoded2,
63 Encoded3,
64 Encoded4,
65 Hidden,
66 Probability,
67 Weights,
68 State
69 };
70
71 constexpr uint32_t k_reflect_pad = 64;
72 constexpr uint32_t k_frame = SileroVadModel::context_samples + SileroVadModel::chunk_samples + k_reflect_pad;
73 constexpr uint32_t k_hidden = 128;
74 constexpr size_t k_max_chunks = 4096;
75
76}
77
79 std::shared_ptr<GpuDispatchRecorder> recording;
80 std::unique_ptr<ShaderExecutionContext<>> context;
82 size_t capacity {};
83 std::vector<float> pending;
84 std::vector<float> tail;
85 size_t resident { 0 };
86 bool keep_hidden { false };
87 std::vector<float> kept_hidden;
88
89 Impl(TensorKernels::PackedWeights packed, std::shared_ptr<GpuDispatchRecorder> shared)
90 : recording(shared ? std::move(shared) : std::make_shared<GpuDispatchRecorder>(true))
91 , context(std::make_unique<ShaderExecutionContext<>>(
92 GpuComputeConfig { .shader_path = TensorKernels::frame_audio_shader, .workgroup_size = { TensorKernels::element_workgroup, 1, 1 }, .push_constant_size = sizeof(TensorKernels::FrameAudioPC) }))
93 , tail(context_samples, 0.0F)
94 {
95 context->set_recording(recording);
96 if (!context->ensure_gpu_ready()) {
98 std::source_location::current(), "SileroVadModel needs a graphics device and none is available");
99 }
100 TensorKernels::upload_packed_weights(*context, 0, Weights, packed);
101 packed.values = std::vector<float> {};
102 layout = std::move(packed);
103
104 context->ensure_shared_buffer(0, State, 2 * k_hidden, GpuBufferBinding::ElementType::FLOAT32, Portal::Graphics::BufferUsageHint::COMPUTE);
105 clear_state();
106 }
107
109 {
110 const std::vector<float> zeros(2 * k_hidden, 0.0F);
111 context->upload_shared_raw(0, State, reinterpret_cast<const uint8_t*>(zeros.data()), zeros.size() * sizeof(float));
112 }
113
114 void ensure_capacity(size_t chunks)
115 {
116 if (chunks <= capacity) {
117 return;
118 }
120 constexpr auto type = GpuBufferBinding::ElementType::FLOAT32;
121 context->ensure_shared_buffer(0, Audio, context_samples + chunks * chunk_samples, type, BufferUsageHint::COMPUTE);
122 context->ensure_shared_buffer(0, Frames, chunks * k_frame, type, BufferUsageHint::COMPUTE);
123 context->ensure_shared_buffer(0, Transform, chunks * 258 * 4, type, BufferUsageHint::COMPUTE);
124 context->ensure_shared_buffer(0, Magnitude, chunks * 129 * 4, type, BufferUsageHint::COMPUTE);
125 context->ensure_shared_buffer(0, Encoded1, chunks * 128 * 4, type, BufferUsageHint::COMPUTE);
126 context->ensure_shared_buffer(0, Encoded2, chunks * 64 * 2, type, BufferUsageHint::COMPUTE);
127 context->ensure_shared_buffer(0, Encoded3, chunks * 64, type, BufferUsageHint::COMPUTE);
128 context->ensure_shared_buffer(0, Encoded4, chunks * k_hidden, type, BufferUsageHint::COMPUTE);
129 context->ensure_shared_buffer(0, Hidden, chunks * k_hidden, type, BufferUsageHint::COMPUTE);
130 context->ensure_shared_buffer(0, Probability, chunks, type);
131 capacity = chunks;
132 }
133
134 [[nodiscard]] uint64_t address(uint32_t buffer) const
135 {
136 return context->shared_buffer_address(0, buffer);
137 }
138
139 [[nodiscard]] uint64_t weight_address(size_t slot) const
140 {
141 return layout.address(address(Weights), slot);
142 }
143
144 [[nodiscard]] std::vector<float> run(std::span<const float> audio, size_t chunks, bool read = true)
145 {
146 ensure_capacity(chunks);
147 const auto n = static_cast<uint32_t>(chunks);
148 context->upload_shared_raw(0, Audio, reinterpret_cast<const uint8_t*>(audio.data()), audio.size_bytes());
149
150 std::vector<DependencyStage> stages;
151 stages.push_back(TensorKernels::frame_audio_stage(address(Audio), address(Frames), n, chunk_samples, context_samples, k_reflect_pad));
152
153 uint32_t t = k_frame;
154 const auto convolve = [&](const TensorKernels::Conv1d& layer, uint64_t src, size_t weight, std::optional<size_t> bias, uint64_t dst) {
155 stages.push_back(TensorKernels::conv1d_stage(layer, src, weight_address(weight), bias ? weight_address(*bias) : 0, dst, n, t));
156 t = TensorKernels::conv1d_length(layer, t);
157 };
158
159 convolve({ .c_in = 1, .c_out = 258, .kernel = 256, .stride = 128 }, address(Frames), StftWeight, std::nullopt, address(Transform));
160 stages.push_back(TensorKernels::magnitude_stage(address(Transform), address(Magnitude), n, 129, t));
161
162 constexpr auto relu = TensorKernels::Activation::Relu;
163 convolve({ .c_in = 129, .c_out = 128, .kernel = 3, .pad = 1, .activation = relu }, address(Magnitude), Conv1Weight, Conv1Bias, address(Encoded1));
164 convolve({ .c_in = 128, .c_out = 64, .kernel = 3, .stride = 2, .pad = 1, .activation = relu }, address(Encoded1), Conv2Weight, Conv2Bias, address(Encoded2));
165 convolve({ .c_in = 64, .c_out = 64, .kernel = 3, .stride = 2, .pad = 1, .activation = relu }, address(Encoded2), Conv3Weight, Conv3Bias, address(Encoded3));
166 convolve({ .c_in = 64, .c_out = k_hidden, .kernel = 3, .pad = 1, .activation = relu }, address(Encoded3), Conv4Weight, Conv4Bias, address(Encoded4));
167
168 stages.push_back(TensorKernels::lstm_sequence_stage(address(Encoded4), weight_address(LstmInputWeight), weight_address(LstmHiddenWeight),
169 weight_address(LstmInputBias), weight_address(LstmHiddenBias), address(State), address(Hidden), n, k_hidden));
170
171 t = 1;
172 convolve({ .c_in = k_hidden, .c_out = 1, .kernel = 1, .relu_input = true, .activation = TensorKernels::Activation::Sigmoid },
173 address(Hidden), HeadWeight, HeadBias, address(Probability));
174
175 context->record_stages(stages);
176 resident = chunks;
177 if (!read) {
178 return {};
179 }
180
181 std::vector<float> probabilities(chunks);
182 context->download_shared(0, Probability, probabilities.data(), probabilities.size() * sizeof(float));
183 if (keep_hidden) {
184 const size_t held = kept_hidden.size();
185 kept_hidden.resize(held + chunks * k_hidden);
186 context->download_shared(0, Hidden, kept_hidden.data() + held, chunks * k_hidden * sizeof(float));
187 }
188 return probabilities;
189 }
190
191 [[nodiscard]] std::vector<float> consume(size_t chunks, bool read = true)
192 {
193 std::vector<float> audio;
194 audio.reserve(context_samples + chunks * chunk_samples);
195 audio.insert(audio.end(), tail.begin(), tail.end());
196 audio.insert(audio.end(), pending.begin(), pending.begin() + static_cast<std::ptrdiff_t>(chunks * chunk_samples));
197
198 auto probabilities = run(audio, chunks, read);
199
200 tail.assign(audio.end() - static_cast<std::ptrdiff_t>(context_samples), audio.end());
201 pending.erase(pending.begin(), pending.begin() + static_cast<std::ptrdiff_t>(chunks * chunk_samples));
202 return probabilities;
203 }
204};
205
207{
208 return TensorKernels::check_tensors(weights, slot_specs());
209}
210
215
217 : SileroVadModel(weights, nullptr)
218{
219}
220
221SileroVadModel::SileroVadModel(const Kakshya::TensorSet& weights, std::shared_ptr<GpuDispatchRecorder> recording)
222{
223 auto packed = TensorKernels::pack_weights(weights, slot_specs());
224 if (!packed) {
226 std::source_location::current(), "SileroVadModel: {}", packed.error());
227 }
228 m_impl = std::make_unique<Impl>(std::move(*packed), std::move(recording));
229}
230
232
233std::vector<float> SileroVadModel::process(std::span<const float> samples)
234{
235 m_impl->pending.insert(m_impl->pending.end(), samples.begin(), samples.end());
236
237 std::vector<float> probabilities;
238 for (size_t chunks = m_impl->pending.size() / chunk_samples; chunks > 0; chunks = m_impl->pending.size() / chunk_samples) {
239 const auto part = m_impl->consume(std::min(chunks, k_max_chunks));
240 probabilities.insert(probabilities.end(), part.begin(), part.end());
241 }
242 return probabilities;
243}
244
246{
247 m_impl->ensure_capacity(std::max<size_t>(m_impl->capacity, 1));
248 return { .context = m_impl->context.get(), .set = 0, .binding = Probability };
249}
250
252{
253 m_impl->ensure_capacity(std::max<size_t>(m_impl->capacity, 1));
254 return { .context = m_impl->context.get(), .set = 0, .binding = Hidden };
255}
256
258{
259 m_impl->keep_hidden = keep;
260 if (!keep) {
261 m_impl->kept_hidden.clear();
262 }
263}
264
265std::vector<float> SileroVadModel::take_hidden()
266{
267 return std::exchange(m_impl->kept_hidden, {});
268}
269
270size_t SileroVadModel::record(std::span<const float> samples)
271{
272 m_impl->pending.insert(m_impl->pending.end(), samples.begin(), samples.end());
273
274 const size_t chunks = std::min(m_impl->pending.size() / chunk_samples, k_max_chunks);
275 if (chunks == 0) {
276 return 0;
277 }
278 static_cast<void>(m_impl->consume(chunks, false));
279 return chunks;
280}
281
282size_t SileroVadModel::resident_count() const noexcept
283{
284 return m_impl->resident;
285}
286
287std::vector<float> SileroVadModel::flush()
288{
289 if (m_impl->pending.empty()) {
290 return {};
291 }
292 m_impl->pending.resize(chunk_samples, 0.0F);
293 return m_impl->consume(1);
294}
295
297{
298 m_impl->pending.clear();
299 m_impl->tail.assign(context_samples, 0.0F);
300 m_impl->clear_state();
301 m_impl->resident = 0;
302 m_impl->kept_hidden.clear();
303}
304
305size_t SileroVadModel::pending() const noexcept
306{
307 return m_impl->pending.size();
308}
309
310} // namespace MayaFlux::Yantra
The Silero voice activity network run on the GPU, one probability per 512 sample chunk.
The stages that run the tensor kernels.
Checking a set of tensors against what a network needs, and packing it into one float32 buffer.
Shared command buffer for GPU dispatch work.
Concrete GpuExecutionContext for a single fixed shader with fixed bindings.
size_t pending() const noexcept
Samples received but not yet part of a completed chunk.
ProbabilityBuffer hidden_buffer()
Where the recurrent output of the latest run is on the device.
static constexpr uint32_t sample_rate
Sample rate in Hz the network was trained and runs at.
ProbabilityBuffer probability_buffer()
The device buffer of the probabilities.
static std::string diagnose(const Kakshya::TensorSet &weights)
Check that a tensor set holds the weights the network needs.
static constexpr uint32_t hidden_width
Width of the recurrent state the network holds for each chunk.
size_t record(std::span< const float > samples)
Run every chunk the samples complete, without waiting for the result.
static constexpr uint32_t context_samples
void keep_hidden(bool keep)
Keep the recurrent output of every chunk process and flush run, for take_hidden.
SileroVadModel(const Kakshya::TensorSet &weights)
Upload the weights and prepare the GPU resources.
size_t resident_count() const noexcept
Probabilities the latest run left in probability_buffer.
void reset()
Forget the recurrent state, the context and any pending samples.
std::vector< float > flush()
Run the partial chunk, padded with zeros to a whole one.
std::vector< float > process(std::span< const float > samples)
Append samples and run every chunk they complete.
static StreamFraming framing() noexcept
The framing of this network.
std::vector< float > take_hidden()
The recurrent output kept since the last call, hidden_width values per chunk, chunk after chunk.
static constexpr uint32_t chunk_samples
void run()
Definition main.cpp:21
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP).
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
BufferUsageHint
Semantic usage hint for buffer allocation and memory properties.
@ COMPUTE
Storage buffer for compute shaders (device-local).
DependencyStage frame_audio_stage(uint64_t samples, uint64_t frames, uint32_t chunks, uint32_t chunk_size, uint32_t context, uint32_t pad)
One nn_frame_audio dispatch: cut a sample stream into overlapping frames, reflect padded on the right...
Definition Stages.hpp:267
DependencyStage lstm_sequence_stage(uint64_t x, uint64_t w_ih, uint64_t w_hh, uint64_t b_ih, uint64_t b_hh, uint64_t state, uint64_t hs, uint32_t steps, uint32_t hidden)
One nn_lstm_sequence dispatch: an LSTM cell over a whole sequence in one workgroup.
Definition Stages.hpp:311
std::string check_tensors(const Kakshya::TensorSet &weights, std::span< const TensorRequirement > required)
Whether a set holds every required tensor with the required shape.
constexpr uint32_t element_workgroup
Invocations per workgroup of the element-wise kernels.
Definition Layouts.hpp:15
uint32_t conv1d_length(const Conv1d &layer, uint32_t t_in)
Output length of a convolution, 0 when the window does not fit the padded input.
Definition Layers.hpp:88
DependencyStage magnitude_stage(uint64_t src, uint64_t dst, uint32_t batch, uint32_t channels, uint32_t time)
One nn_magnitude dispatch: sqrt(re^2 + im^2) of a complex signal stored as two channel halves.
Definition Stages.hpp:286
std::expected< PackedWeights, std::string > pack_weights(const Kakshya::TensorSet &weights, std::span< const TensorRequirement > required)
Convert the required tensors to float32 and lay them out in the order of required.
DependencyStage conv1d_stage(const Conv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d dispatch: [batch, c_in, t_in] to [batch, c_out, conv1d_length].
Definition Stages.hpp:70
void upload_packed_weights(GpuDispatchCore &context, uint32_t set, size_t binding, const PackedWeights &packed)
Allocate a shared buffer on a context and fill it with packed weights.
static constexpr DomainSpec Audio
Domain constant for Audio domain.
Definition Creator.hpp:497
Named tensors that belong together, such as a model's weights or I/O.
Definition Tensor.hpp:167
enum MayaFlux::Portal::Graphics::GpuBufferBinding::ElementType FLOAT32
Plain-data description of the compute shader to dispatch.
uint64_t weight_address(size_t slot) const
Impl(TensorKernels::PackedWeights packed, std::shared_ptr< GpuDispatchRecorder > shared)
std::vector< float > consume(size_t chunks, bool read=true)
uint64_t address(uint32_t buffer) const
std::shared_ptr< GpuDispatchRecorder > recording
std::unique_ptr< ShaderExecutionContext<> > context
std::vector< float > run(std::span< const float > audio, size_t chunks, bool read=true)
TensorKernels::PackedWeights layout
Where the probabilities of the latest run are on the device.
How a stream classifier cuts its input: sample rate, chunk length, the context carried into each chun...
One convolution layer over [batch, c_in, t_in], with weights [c_out, c_in, kernel].
Definition Layers.hpp:31
Push constants of nn_frame_audio.
Definition Layouts.hpp:69
The required tensors as float32 in one run, in the order they were required.