MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
VadAnalyzer.hpp
Go to the documentation of this file.
1#pragma once
2
5
6#include "Analyzer.hpp"
7#include <Eigen/Dense>
8
10
12
13/**
14 * @file VadAnalyzer.hpp
15 * @brief Voice activity probability per 32 ms chunk from a learned network, run on the GPU.
16 */
17
18namespace MayaFlux::Yantra {
19
20/**
21 * @struct VadAnalyzerConfig
22 * @brief Settings of a VadAnalyzer.
23 *
24 * The network fixes the chunk at 512 samples of 16 kHz audio, so the window and
25 * hop of the base config are both 512 and are not settings. sample_rate states
26 * the rate of the input and exists so that a wrong one is refused with its
27 * reason instead of giving wrong probabilities: the analyzer does not convert.
28 * Input at another rate is converted first, for example with
29 * Kinesis::Discrete::Resampler.
30 */
33 {
34 const auto framing = SileroVadModel::framing();
35 window_size = framing.chunk_samples;
36 hop_size = framing.chunk_samples;
37 }
38
39 /** Also run the partial chunk left at the end of each call, padded with zeros. Off for a stream, on for a whole clip. */
40 bool flush_tail { false };
41 /** Sample rate in Hz of the input. The network runs at 16000 only, so any other value is refused. */
42 double sample_rate { static_cast<double>(SileroVadModel::framing().sample_rate) };
43 /** Also read back the network's recurrent output of every chunk, SileroVadModel::hidden_width values each. */
44 bool keep_embeddings { false };
45};
46
47MF_DESCRIBE(VadAnalyzerConfig)
48
49/**
50 * @struct VadAnalysis
51 * @brief Analysis result of a VadAnalyzer.
52 */
54 /** Settings the analysis was produced under. */
56 /** Speech probability of each chunk completed by this call, 0 to 1. */
57 std::vector<float> probabilities;
58 /** First and one past the last sample of each chunk, counted from the start of the stream. */
59 std::vector<std::pair<size_t, size_t>> chunk_positions;
60 /**
61 * Recurrent output of the network for each chunk, SileroVadModel::hidden_width values each, kept on request.
62 * It is what the probability is derived from; the network was trained only to tell speech from other sound.
63 */
64 std::vector<float> embeddings;
65};
66
68
69/**
70 * @class VadAnalyzer
71 * @brief Whether each 32 ms chunk of a mono 16 kHz stream is voice activity, from a learned network.
72 *
73 * Where WaveformAnalyzer measures the shape of the waveform, this says what
74 * the sound is: a sung note and a fan hum are as periodic as a vowel, and only
75 * a learned model separates them from speech. The probabilities are the
76 * network's own, before any thresholding.
77 *
78 * The analyzer is named for the task. Today it runs one network, Silero VAD at
79 * 16 kHz, recognised by its tensors; weights for any other network are
80 * rejected with the reason.
81 *
82 * The analyzer is a stream. The recurrent state, the 64 samples of context
83 * before the next chunk and any partial chunk persist between calls, so
84 * successive calls continue one signal. Call reset for a new one. For a whole
85 * clip, set flush_tail so the final partial chunk is run too. One channel only.
86 *
87 * The weights are an explicit dependency: the caller loads the network's
88 * tensors and passes them. The network runs in native float on the vulkan
89 * target and has no CPU twin. It is not registered with the default registry,
90 * because it has no selection to resolve by, and its result arrives a chunk
91 * late, so it must not run on the audio thread.
92 */
93template <ComputeData InputType = std::vector<Kakshya::DataVariant>, ComputeData OutputType = Eigen::MatrixXd>
94class VadAnalyzer : public Analyzer<InputType, OutputType> {
95public:
100
101 static constexpr Target target = targets::vulkan;
102
103 /**
104 * @brief Construct from the network weights.
105 * @param weights The tensors of the network. Silero VAD at 16 kHz is the one supported, as in silero_vad_16k.safetensors.
106 * @param config Analysis settings.
107 * @throws std::invalid_argument When diagnose refuses the config, or the weights are missing a tensor or one has the wrong shape.
108 */
110 {
111 if (const std::string reason = diagnose(config); !reason.empty()) {
113 std::source_location::current(), "VadAnalyzer: {}", reason);
114 }
115 if (const std::string reason = SileroVadModel::diagnose(weights); !reason.empty()) {
117 std::source_location::current(), "VadAnalyzer: {}", reason);
118 }
119 m_weights = std::make_shared<const Kakshya::TensorSet>(weights);
120 this->set_config(std::move(config));
121 }
122
123 [[nodiscard]] ConfigRefs configs() const override
124 {
125 return this->stored_configs(supports {});
126 }
127
128 /**
129 * @brief Why a config cannot be analysed.
130 * @return The reason, empty when the config is usable: its sample_rate is the 16000 Hz the network runs at.
131 */
132 [[nodiscard]] static std::string diagnose(const VadAnalyzerConfig& config)
133 {
134 const auto framing = SileroVadModel::framing();
135 if (config.sample_rate != static_cast<double>(framing.sample_rate)) {
136 return std::format("sample_rate {} is not the {} Hz the network runs at; convert the input first, for example with Kinesis::Discrete::Resampler",
137 config.sample_rate, framing.sample_rate);
138 }
139 return {};
140 }
141
142 /**
143 * @brief Type-safe speech probability analysis
144 */
146 {
147 auto result = this->analyze_data(data);
148 return safe_any_cast_or_throw<VadAnalysis>(result);
149 }
150
151 VadAnalysis analyze_speech(const InputType& data)
152 {
153 return this->analyze_speech(input_type { data });
154 }
155
156 [[nodiscard]] VadAnalysis get_speech_analysis() const
157 {
158 return safe_any_cast_or_throw<VadAnalysis>(this->get_current_analysis());
159 }
160
161 /**
162 * @brief Append to a recording the caller shares with other work.
163 *
164 * With a shared recording the network goes to the device in the same
165 * submission as the work recorded around it, and analyze_speech submits
166 * and waits for all of it. Call it before the first analysis or record. The
167 * recording must be compute isolated.
168 *
169 * @throws std::logic_error when the network has already been created.
170 */
171 void share_recording(std::shared_ptr<GpuDispatchRecorder> recording)
172 {
173 if (m_model) {
175 std::source_location::current(), "The recording can only be shared before the network starts");
176 }
177 m_recording = std::move(recording);
178 }
179
180 /**
181 * @brief Run every chunk the samples complete without waiting for the result.
182 *
183 * The probabilities stay on the device, in probability_buffer, and are
184 * recorded into the analyzer's recording without a submission. A consumer
185 * on the same recording, such as an IntervalExtractor, reads them in place
186 * and submits. Whole chunks only: a partial chunk stays pending, and
187 * flush_tail does not apply. At most 4096 chunks run per call, so read the
188 * buffer before the next call.
189 *
190 * @param samples Mono samples at 16 kHz, continuing the stream.
191 * @return The number of chunks recorded.
192 * @throws std::invalid_argument when the config cannot be analysed.
193 */
194 [[nodiscard]] size_t record(std::span<const float> samples)
195 {
196 const auto& cfg = this->template config<VadAnalyzerConfig>();
197 if (const std::string reason = diagnose(cfg); !reason.empty()) {
199 std::source_location::current(), "VadAnalyzer: {}", reason);
200 }
201 const size_t chunks = network().record(samples);
202 m_chunks_done += chunks;
203 return chunks;
204 }
205
206 /**
207 * @brief Where the probabilities of the latest run are on the device, for a consumer on the same recording.
208 */
213
214 /**
215 * @brief Where the network's recurrent output of the latest run is on the device, SileroVadModel::hidden_width values per chunk.
216 */
218 {
219 return network().hidden_buffer();
220 }
221
222 /** @brief Probabilities the latest run left in probability_buffer. */
223 [[nodiscard]] size_t resident_count() const noexcept
224 {
225 return m_model ? m_model->resident_count() : 0;
226 }
227
228 /**
229 * @brief Start a new signal: forget the recurrent state, the context and any partial chunk.
230 */
231 void reset()
232 {
233 if (m_model) {
234 m_model->reset();
235 }
236 m_chunks_done = 0;
237 }
238
239 [[nodiscard]] AnalysisType get_analysis_type() const override
240 {
242 }
243
244 [[nodiscard]] std::string get_name() const override
245 {
246 return "VadAnalyzer";
247 }
248
249 /**
250 * @brief One while a device is up, zero otherwise.
251 */
252 static float fitness(const input_type& /*input*/)
253 {
255 }
256
257protected:
259 {
260 try {
261 const auto& cfg = this->template config<VadAnalyzerConfig>();
262 if (const std::string reason = diagnose(cfg); !reason.empty()) {
264 std::source_location::current(), "VadAnalyzer: {}", reason);
265 }
266
267 auto [data_span, structure_info] = OperationHelper::extract_structured_double(
268 const_cast<input_type&>(input));
269
270 if (data_span.size() != 1) {
272 std::source_location::current(), "VadAnalyzer analyses one channel and was given {}", data_span.size());
273 }
274
275 const std::vector<float> samples(data_span[0].begin(), data_span[0].end());
276
277 VadAnalysis analysis;
278 analysis.config = cfg;
279 network().keep_hidden(cfg.keep_embeddings);
280 analysis.probabilities = network().process(samples);
281 if (cfg.flush_tail) {
282 const auto tail = network().flush();
283 analysis.probabilities.insert(analysis.probabilities.end(), tail.begin(), tail.end());
284 }
285 analysis.embeddings = network().take_hidden();
286
287 analysis.chunk_positions.reserve(analysis.probabilities.size());
288 for (size_t i = 0; i < analysis.probabilities.size(); ++i) {
289 const size_t start = (m_chunks_done + i) * SileroVadModel::chunk_samples;
290 analysis.chunk_positions.emplace_back(start, start + SileroVadModel::chunk_samples);
291 }
292 m_chunks_done += analysis.probabilities.size();
293
294 this->store_current_analysis(analysis);
295
296 std::vector<std::vector<double>> channel_values(1);
297 channel_values[0].assign(analysis.probabilities.begin(), analysis.probabilities.end());
298
299 output_type output = this->convert_result(channel_values, structure_info);
300 output.metadata = input.metadata;
301 output.facets = input.facets;
302 output.facets.set(analysis);
303 return output;
304 } catch (const std::exception& e) {
305 MF_ERROR(Journal::Component::Yantra, Journal::Context::ComputeMatrix, "Voice activity analysis failed: {}", e.what());
306 output_type error_result;
307 error_result.metadata = input.metadata;
308 error_result.facets = input.facets;
309 return error_result;
310 }
311 }
312
313private:
315 {
316 if (!m_model) {
317 m_model = std::make_unique<SileroVadModel>(*m_weights, m_recording);
318 }
319 return *m_model;
320 }
321
322 std::shared_ptr<const Kakshya::TensorSet> m_weights;
323 std::shared_ptr<GpuDispatchRecorder> m_recording;
324 mutable std::unique_ptr<SileroVadModel> m_model;
325 size_t m_chunks_done { 0 };
326};
327
328/// Standard VAD analyzer: DataVariant -> MatrixXd
330
331/// Container VAD analyzer: SignalContainer -> MatrixXd
333
334} // namespace MayaFlux::Yantra
Modern, digital-first analyzer framework for Maya Flux.
#define MF_ERROR(comp, ctx,...)
#define MF_DESCRIBE(...)
Requests a generated member description for a struct at namespace scope.
Definition Describe.hpp:14
The Silero voice activity network run on the GPU, one probability per 512 sample chunk.
void set(T value)
Stores a struct under its type identity.
Definition Facets.hpp:107
bool is_initialized() const
Check if compiler is initialized.
void store_current_analysis(AnalysisResultType &&result) const
Definition Analyzer.hpp:234
std::any get_current_analysis() const
Access cached analysis from last operation.
Definition Analyzer.hpp:175
std::any analyze_data(const input_type &data)
User-facing analysis method - returns analysis results directly.
Definition Analyzer.hpp:138
Template-flexible analyzer base with instance-defined I/O types.
Definition Analyzer.hpp:101
void set_config(Config config)
Stores a config struct, whole, under its own type.
ConfigRefs stored_configs(Supports< Configs... >) const
The stored configs among those listed, in listed order.
output_type convert_result(std::vector< std::vector< double > > &result_data, DataStructureInfo &metadata)
Convert processed double data back to OutputType using metadata and optional callback.
static std::tuple< NumericChannels, DataStructureInfo > extract_structured_double(T &compute_data)
Extract structured double data from Datum container or direct ComputeData with automatic container ha...
ProbabilityBuffer hidden_buffer()
Where the recurrent output of the latest run is on the device.
ProbabilityBuffer probability_buffer()
The device buffer of the probabilities.
static std::string diagnose(const Kakshya::TensorSet &weights)
Check that a tensor set holds the weights the network needs.
size_t record(std::span< const float > samples)
Run every chunk the samples complete, without waiting for the result.
void keep_hidden(bool keep)
Keep the recurrent output of every chunk process and flush run, for take_hidden.
std::vector< float > flush()
Run the partial chunk, padded with zeros to a whole one.
std::vector< float > process(std::span< const float > samples)
Append samples and run every chunk they complete.
static StreamFraming framing() noexcept
The framing of this network.
std::vector< float > take_hidden()
The recurrent output kept since the last call, hidden_width values per chunk, chunk after chunk.
static constexpr uint32_t chunk_samples
Silero VAD at 16 kHz as a stream: samples in, one speech probability per chunk out.
static float fitness(const input_type &)
One while a device is up, zero otherwise.
VadAnalysis get_speech_analysis() const
void reset()
Start a new signal: forget the recurrent state, the context and any partial chunk.
ConfigRefs configs() const override
The base view of the config structs this operation holds.
std::string get_name() const override
Get operation name for debugging/introspection.
Datum< OutputType > output_type
size_t record(std::span< const float > samples)
Run every chunk the samples complete without waiting for the result.
Supports< VadAnalyzerConfig > supports
VadAnalysis analyze_speech(const input_type &data)
Type-safe speech probability analysis.
VadAnalyzer(const Kakshya::TensorSet &weights, VadAnalyzerConfig config={})
Construct from the network weights.
SileroVadModel::ProbabilityBuffer probability_buffer()
Where the probabilities of the latest run are on the device, for a consumer on the same recording.
AnalysisType get_analysis_type() const override
Gets the analysis type category for this analyzer.
std::shared_ptr< const Kakshya::TensorSet > m_weights
SileroVadModel::ProbabilityBuffer hidden_buffer()
Where the network's recurrent output of the latest run is on the device, SileroVadModel::hidden_width...
VadAnalysis analyze_speech(const InputType &data)
Analyzer< InputType, OutputType > base_type
void share_recording(std::shared_ptr< GpuDispatchRecorder > recording)
Append to a recording the caller shares with other work.
Datum< InputType > input_type
SileroVadModel & network() const
output_type analyze_implementation(const input_type &input) override
Pure virtual analysis implementation - derived classes implement this.
size_t resident_count() const noexcept
Probabilities the latest run left in probability_buffer.
static std::string diagnose(const VadAnalyzerConfig &config)
Why a config cannot be analysed.
Whether each 32 ms chunk of a mono 16 kHz stream is voice activity, from a learned network.
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP).
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
MAYAFLUX_API ShaderFoundry & get_shader_foundry()
Get the global shader compiler instance.
AnalysisType
Categories of analysis operations for discovery and organization.
Definition Analyzer.hpp:33
@ CUSTOM
User-defined analysis types.
Definition Analyzer.hpp:41
VadAnalyzer< std::vector< Kakshya::DataVariant >, Eigen::MatrixXd > StandardVadAnalyzer
Standard VAD analyzer: DataVariant -> MatrixXd.
VadAnalyzer< std::shared_ptr< Kakshya::SignalSourceContainer >, Eigen::MatrixXd > ContainerVadAnalyzer
Container VAD analyzer: SignalContainer -> MatrixXd.
std::vector< std::reference_wrapper< const OperationConfig > > ConfigRefs
Borrowed views of the configs an operation holds.
Named tensors that belong together, such as a model's weights or I/O.
Definition Tensor.hpp:167
uint32_t window_size
Analysis window length in samples.
Definition Analyzer.hpp:64
uint32_t hop_size
Distance between successive windows in samples.
Definition Analyzer.hpp:66
Settings every analyzer family accepts.
Definition Analyzer.hpp:62
Facets facets
Typed results that travel with the data, read by type.
Definition DataIO.hpp:68
std::unordered_map< std::string, std::any > metadata
Associated metadata.
Definition DataIO.hpp:67
Input/Output container for computation pipeline data flow with structure preservation.
Definition DataIO.hpp:63
Where the probabilities of the latest run are on the device.
uint32_t sample_rate
Sample rate in Hz the network runs at.
The config structs an operation class accepts, declared as using supports.
A set of (device, runtime) pairs.
std::vector< float > embeddings
Recurrent output of the network for each chunk, SileroVadModel::hidden_width values each,...
std::vector< float > probabilities
Speech probability of each chunk completed by this call, 0 to 1.
VadAnalyzerConfig config
Settings the analysis was produced under.
std::vector< std::pair< size_t, size_t > > chunk_positions
First and one past the last sample of each chunk, counted from the start of the stream.
Analysis result of a VadAnalyzer.
bool flush_tail
Also run the partial chunk left at the end of each call, padded with zeros.
bool keep_embeddings
Also read back the network's recurrent output of every chunk, SileroVadModel::hidden_width values eac...
double sample_rate
Sample rate in Hz of the input.
Settings of a VadAnalyzer.