MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches

◆ run()

std::vector< float > MayaFlux::Yantra::SileroVadModel::Impl::run ( std::span< const float > audio,
size_t chunks,
bool read = true )
inlinenodiscard

Definition at line 144 of file SileroVadModel.cpp.

145 {
146 ensure_capacity(chunks);
147 const auto n = static_cast<uint32_t>(chunks);
148 context->upload_shared_raw(0, Audio, reinterpret_cast<const uint8_t*>(audio.data()), audio.size_bytes());
149
150 std::vector<DependencyStage> stages;
151 stages.push_back(TensorKernels::frame_audio_stage(address(Audio), address(Frames), n, chunk_samples, context_samples, k_reflect_pad));
152
153 uint32_t t = k_frame;
154 const auto convolve = [&](const TensorKernels::Conv1d& layer, uint64_t src, size_t weight, std::optional<size_t> bias, uint64_t dst) {
155 stages.push_back(TensorKernels::conv1d_stage(layer, src, weight_address(weight), bias ? weight_address(*bias) : 0, dst, n, t));
156 t = TensorKernels::conv1d_length(layer, t);
157 };
158
159 convolve({ .c_in = 1, .c_out = 258, .kernel = 256, .stride = 128 }, address(Frames), StftWeight, std::nullopt, address(Transform));
160 stages.push_back(TensorKernels::magnitude_stage(address(Transform), address(Magnitude), n, 129, t));
161
162 constexpr auto relu = TensorKernels::Activation::Relu;
163 convolve({ .c_in = 129, .c_out = 128, .kernel = 3, .pad = 1, .activation = relu }, address(Magnitude), Conv1Weight, Conv1Bias, address(Encoded1));
164 convolve({ .c_in = 128, .c_out = 64, .kernel = 3, .stride = 2, .pad = 1, .activation = relu }, address(Encoded1), Conv2Weight, Conv2Bias, address(Encoded2));
165 convolve({ .c_in = 64, .c_out = 64, .kernel = 3, .stride = 2, .pad = 1, .activation = relu }, address(Encoded2), Conv3Weight, Conv3Bias, address(Encoded3));
166 convolve({ .c_in = 64, .c_out = k_hidden, .kernel = 3, .pad = 1, .activation = relu }, address(Encoded3), Conv4Weight, Conv4Bias, address(Encoded4));
167
168 stages.push_back(TensorKernels::lstm_sequence_stage(address(Encoded4), weight_address(LstmInputWeight), weight_address(LstmHiddenWeight),
169 weight_address(LstmInputBias), weight_address(LstmHiddenBias), address(State), address(Hidden), n, k_hidden));
170
171 t = 1;
172 convolve({ .c_in = k_hidden, .c_out = 1, .kernel = 1, .relu_input = true, .activation = TensorKernels::Activation::Sigmoid },
173 address(Hidden), HeadWeight, HeadBias, address(Probability));
174
175 context->record_stages(stages);
176 resident = chunks;
177 if (!read) {
178 return {};
179 }
180
181 std::vector<float> probabilities(chunks);
182 context->download_shared(0, Probability, probabilities.data(), probabilities.size() * sizeof(float));
183 if (keep_hidden) {
184 const size_t held = kept_hidden.size();
185 kept_hidden.resize(held + chunks * k_hidden);
186 context->download_shared(0, Hidden, kept_hidden.data() + held, chunks * k_hidden * sizeof(float));
187 }
188 return probabilities;
189 }
static constexpr uint32_t context_samples
static constexpr uint32_t chunk_samples
std::vector< double > convolve(std::span< const double > src, std::span< const double > ir)
Linear FFT convolution.
DependencyStage frame_audio_stage(uint64_t samples, uint64_t frames, uint32_t chunks, uint32_t chunk_size, uint32_t context, uint32_t pad)
One nn_frame_audio dispatch: cut a sample stream into overlapping frames, reflect padded on the right...
Definition Stages.hpp:267
DependencyStage lstm_sequence_stage(uint64_t x, uint64_t w_ih, uint64_t w_hh, uint64_t b_ih, uint64_t b_hh, uint64_t state, uint64_t hs, uint32_t steps, uint32_t hidden)
One nn_lstm_sequence dispatch: an LSTM cell over a whole sequence in one workgroup.
Definition Stages.hpp:311
uint32_t conv1d_length(const Conv1d &layer, uint32_t t_in)
Output length of a convolution, 0 when the window does not fit the padded input.
Definition Layers.hpp:88
DependencyStage magnitude_stage(uint64_t src, uint64_t dst, uint32_t batch, uint32_t channels, uint32_t time)
One nn_magnitude dispatch: sqrt(re^2 + im^2) of a complex signal stored as two channel halves.
Definition Stages.hpp:286
DependencyStage conv1d_stage(const Conv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d dispatch: [batch, c_in, t_in] to [batch, c_out, conv1d_length].
Definition Stages.hpp:70
static constexpr DomainSpec Audio
Domain constant for Audio domain.
Definition Creator.hpp:497
uint64_t weight_address(size_t slot) const
uint64_t address(uint32_t buffer) const
std::unique_ptr< ShaderExecutionContext<> > context

References address(), MayaFlux::Audio, MayaFlux::Yantra::SileroVadModel::chunk_samples, context, MayaFlux::Yantra::SileroVadModel::context_samples, MayaFlux::Yantra::TensorKernels::conv1d_length(), MayaFlux::Yantra::TensorKernels::conv1d_stage(), ensure_capacity(), MayaFlux::Yantra::TensorKernels::frame_audio_stage(), keep_hidden, kept_hidden, MayaFlux::Yantra::TensorKernels::lstm_sequence_stage(), MayaFlux::Yantra::TensorKernels::magnitude_stage(), MayaFlux::Yantra::TensorKernels::Relu, resident, MayaFlux::Yantra::TensorKernels::Sigmoid, and weight_address().

Here is the call graph for this function: