34 const std::array<TensorRequirement, SlotCount>& slot_specs()
36 static const std::array<TensorRequirement, SlotCount> specs { {
37 {
"stft_conv.weight", { 258, 1, 256 } },
38 {
"conv1.weight", { 128, 129, 3 } },
39 {
"conv1.bias", { 128 } },
40 {
"conv2.weight", { 64, 128, 3 } },
41 {
"conv2.bias", { 64 } },
42 {
"conv3.weight", { 64, 64, 3 } },
43 {
"conv3.bias", { 64 } },
44 {
"conv4.weight", { 128, 64, 3 } },
45 {
"conv4.bias", { 128 } },
46 {
"lstm_cell.weight_ih", { 512, 128 } },
47 {
"lstm_cell.weight_hh", { 512, 128 } },
48 {
"lstm_cell.bias_ih", { 512 } },
49 {
"lstm_cell.bias_hh", { 512 } },
50 {
"final_conv.weight", { 1, 128, 1 } },
51 {
"final_conv.bias", { 1 } },
56 enum Buffer : uint32_t {
71 constexpr uint32_t k_reflect_pad = 64;
73 constexpr uint32_t k_hidden = 128;
74 constexpr size_t k_max_chunks = 4096;
80 std::unique_ptr<ShaderExecutionContext<>>
context;
93 , tail(context_samples, 0.0F)
95 context->set_recording(recording);
96 if (!context->ensure_gpu_ready()) {
98 std::source_location::current(),
"SileroVadModel needs a graphics device and none is available");
101 packed.values = std::vector<float> {};
102 layout = std::move(packed);
110 const std::vector<float> zeros(2 * k_hidden, 0.0F);
111 context->upload_shared_raw(0, State,
reinterpret_cast<const uint8_t*
>(zeros.data()), zeros.size() *
sizeof(
float));
122 context->ensure_shared_buffer(0, Frames, chunks * k_frame, type, BufferUsageHint::COMPUTE);
123 context->ensure_shared_buffer(0, Transform, chunks * 258 * 4, type, BufferUsageHint::COMPUTE);
124 context->ensure_shared_buffer(0, Magnitude, chunks * 129 * 4, type, BufferUsageHint::COMPUTE);
125 context->ensure_shared_buffer(0, Encoded1, chunks * 128 * 4, type, BufferUsageHint::COMPUTE);
126 context->ensure_shared_buffer(0, Encoded2, chunks * 64 * 2, type, BufferUsageHint::COMPUTE);
127 context->ensure_shared_buffer(0, Encoded3, chunks * 64, type, BufferUsageHint::COMPUTE);
128 context->ensure_shared_buffer(0, Encoded4, chunks * k_hidden, type, BufferUsageHint::COMPUTE);
129 context->ensure_shared_buffer(0, Hidden, chunks * k_hidden, type, BufferUsageHint::COMPUTE);
130 context->ensure_shared_buffer(0, Probability, chunks, type);
134 [[nodiscard]] uint64_t
address(uint32_t buffer)
const
136 return context->shared_buffer_address(0, buffer);
144 [[nodiscard]] std::vector<float>
run(std::span<const float> audio,
size_t chunks,
bool read =
true)
147 const auto n =
static_cast<uint32_t
>(chunks);
148 context->upload_shared_raw(0,
Audio,
reinterpret_cast<const uint8_t*
>(audio.data()), audio.size_bytes());
150 std::vector<DependencyStage> stages;
153 uint32_t t = k_frame;
154 const auto convolve = [&](
const TensorKernels::Conv1d& layer, uint64_t src,
size_t weight, std::optional<size_t> bias, uint64_t dst) {
159 convolve({ .c_in = 1, .c_out = 258, .kernel = 256, .stride = 128 },
address(Frames), StftWeight, std::nullopt,
address(Transform));
163 convolve({ .c_in = 129, .c_out = 128, .kernel = 3, .pad = 1, .activation = relu },
address(Magnitude), Conv1Weight, Conv1Bias,
address(Encoded1));
164 convolve({ .c_in = 128, .c_out = 64, .kernel = 3, .stride = 2, .pad = 1, .activation = relu },
address(Encoded1), Conv2Weight, Conv2Bias,
address(Encoded2));
165 convolve({ .c_in = 64, .c_out = 64, .kernel = 3, .stride = 2, .pad = 1, .activation = relu },
address(Encoded2), Conv3Weight, Conv3Bias,
address(Encoded3));
166 convolve({ .c_in = 64, .c_out = k_hidden, .kernel = 3, .pad = 1, .activation = relu },
address(Encoded3), Conv4Weight, Conv4Bias,
address(Encoded4));
175 context->record_stages(stages);
181 std::vector<float> probabilities(chunks);
182 context->download_shared(0, Probability, probabilities.data(), probabilities.size() *
sizeof(
float));
186 context->download_shared(0, Hidden,
kept_hidden.data() + held, chunks * k_hidden *
sizeof(
float));
188 return probabilities;
191 [[nodiscard]] std::vector<float>
consume(
size_t chunks,
bool read =
true)
193 std::vector<float> audio;
195 audio.insert(audio.end(),
tail.begin(),
tail.end());
198 auto probabilities =
run(audio, chunks, read);
202 return probabilities;
226 std::source_location::current(),
"SileroVadModel: {}", packed.error());
228 m_impl = std::make_unique<Impl>(std::move(*packed), std::move(recording));
235 m_impl->pending.insert(
m_impl->pending.end(), samples.begin(), samples.end());
237 std::vector<float> probabilities;
239 const auto part =
m_impl->consume(std::min(chunks, k_max_chunks));
240 probabilities.insert(probabilities.end(), part.begin(), part.end());
242 return probabilities;
247 m_impl->ensure_capacity(std::max<size_t>(
m_impl->capacity, 1));
248 return { .context =
m_impl->context.get(), .set = 0, .binding = Probability };
253 m_impl->ensure_capacity(std::max<size_t>(
m_impl->capacity, 1));
254 return { .context =
m_impl->context.get(), .set = 0, .binding = Hidden };
259 m_impl->keep_hidden = keep;
261 m_impl->kept_hidden.clear();
267 return std::exchange(
m_impl->kept_hidden, {});
272 m_impl->pending.insert(
m_impl->pending.end(), samples.begin(), samples.end());
278 static_cast<void>(
m_impl->consume(chunks,
false));
289 if (
m_impl->pending.empty()) {
293 return m_impl->consume(1);
302 m_impl->kept_hidden.clear();
307 return m_impl->pending.size();
The Silero voice activity network run on the GPU, one probability per 512 sample chunk.
The stages that run the tensor kernels.
Checking a set of tensors against what a network needs, and packing it into one float32 buffer.
Shared command buffer for GPU dispatch work.
Concrete GpuExecutionContext for a single fixed shader with fixed bindings.
size_t pending() const noexcept
Samples received but not yet part of a completed chunk.
ProbabilityBuffer hidden_buffer()
Where the recurrent output of the latest run is on the device.
static constexpr uint32_t sample_rate
Sample rate in Hz the network was trained and runs at.
ProbabilityBuffer probability_buffer()
The device buffer of the probabilities.
std::unique_ptr< Impl > m_impl
static std::string diagnose(const Kakshya::TensorSet &weights)
Check that a tensor set holds the weights the network needs.
static constexpr uint32_t hidden_width
Width of the recurrent state the network holds for each chunk.
size_t record(std::span< const float > samples)
Run every chunk the samples complete, without waiting for the result.
static constexpr uint32_t context_samples
void keep_hidden(bool keep)
Keep the recurrent output of every chunk process and flush run, for take_hidden.
SileroVadModel(const Kakshya::TensorSet &weights)
Upload the weights and prepare the GPU resources.
size_t resident_count() const noexcept
Probabilities the latest run left in probability_buffer.
void reset()
Forget the recurrent state, the context and any pending samples.
std::vector< float > flush()
Run the partial chunk, padded with zeros to a whole one.
std::vector< float > process(std::span< const float > samples)
Append samples and run every chunk they complete.
static StreamFraming framing() noexcept
The framing of this network.
std::vector< float > take_hidden()
The recurrent output kept since the last call, hidden_width values per chunk, chunk after chunk.
static constexpr uint32_t chunk_samples
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP).
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
BufferUsageHint
Semantic usage hint for buffer allocation and memory properties.
@ COMPUTE
Storage buffer for compute shaders (device-local).
DependencyStage frame_audio_stage(uint64_t samples, uint64_t frames, uint32_t chunks, uint32_t chunk_size, uint32_t context, uint32_t pad)
One nn_frame_audio dispatch: cut a sample stream into overlapping frames, reflect padded on the right...
DependencyStage lstm_sequence_stage(uint64_t x, uint64_t w_ih, uint64_t w_hh, uint64_t b_ih, uint64_t b_hh, uint64_t state, uint64_t hs, uint32_t steps, uint32_t hidden)
One nn_lstm_sequence dispatch: an LSTM cell over a whole sequence in one workgroup.
std::string check_tensors(const Kakshya::TensorSet &weights, std::span< const TensorRequirement > required)
Whether a set holds every required tensor with the required shape.
constexpr uint32_t element_workgroup
Invocations per workgroup of the element-wise kernels.
uint32_t conv1d_length(const Conv1d &layer, uint32_t t_in)
Output length of a convolution, 0 when the window does not fit the padded input.
DependencyStage magnitude_stage(uint64_t src, uint64_t dst, uint32_t batch, uint32_t channels, uint32_t time)
One nn_magnitude dispatch: sqrt(re^2 + im^2) of a complex signal stored as two channel halves.
std::expected< PackedWeights, std::string > pack_weights(const Kakshya::TensorSet &weights, std::span< const TensorRequirement > required)
Convert the required tensors to float32 and lay them out in the order of required.
DependencyStage conv1d_stage(const Conv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d dispatch: [batch, c_in, t_in] to [batch, c_out, conv1d_length].
void upload_packed_weights(GpuDispatchCore &context, uint32_t set, size_t binding, const PackedWeights &packed)
Allocate a shared buffer on a context and fill it with packed weights.
static constexpr DomainSpec Audio
Domain constant for Audio domain.
Named tensors that belong together, such as a model's weights or I/O.
enum MayaFlux::Portal::Graphics::GpuBufferBinding::ElementType FLOAT32
Plain-data description of the compute shader to dispatch.
std::vector< float > kept_hidden
uint64_t weight_address(size_t slot) const
std::vector< float > tail
Impl(TensorKernels::PackedWeights packed, std::shared_ptr< GpuDispatchRecorder > shared)
void ensure_capacity(size_t chunks)
std::vector< float > consume(size_t chunks, bool read=true)
uint64_t address(uint32_t buffer) const
std::shared_ptr< GpuDispatchRecorder > recording
std::unique_ptr< ShaderExecutionContext<> > context
std::vector< float > pending
std::vector< float > run(std::span< const float > audio, size_t chunks, bool read=true)
TensorKernels::PackedWeights layout
Where the probabilities of the latest run are on the device.
How a stream classifier cuts its input: sample rate, chunk length, the context carried into each chun...
One convolution layer over [batch, c_in, t_in], with weights [c_out, c_in, kernel].
Push constants of nn_frame_audio.
The required tensors as float32 in one run, in the order they were required.
One tensor a network needs.