81 std::vector<std::pair<size_t, size_t>>
speech;
85 std::vector<double>
f0;
99 scan.scan.enter_threshold = speech.
threshold;
103 scan.scan.padding = speech.
padding;
104 scan.scan.max_run = speech.
max_run;
106 scan.length =
static_cast<uint32_t
>(samples);
118 std::vector<double>
f0;
144 std::span<const double> samples,
double threshold, uint32_t hop)
150 std::vector<Kakshya::DataVariant> { std::vector<double>(samples.begin(), samples.end()) });
151 if (analysis.channels.empty()) {
154 const auto& waveform = analysis.
channels[0];
156 std::vector<size_t> cells;
157 for (
size_t window = 0; window < waveform.periodicity.size() && window < waveform.window_positions.size(); ++window) {
158 if (waveform.periodicity[window] >=
threshold) {
159 const auto& [first, last] = waveform.window_positions[window];
160 const size_t centre = (first + last) / 2;
161 cells.push_back(centre >= hop / 2 ? centre - hop / 2 : 0);
177 const std::vector<std::pair<size_t, size_t>>& speech,
double voicing_threshold)
181 voicing.
f0.assign(speech.size(), 0.0);
187 std::vector<Kakshya::DataVariant> { std::vector<double>(samples.begin(), samples.end()) });
188 if (analysis.channels.empty()) {
191 const auto& waveform = analysis.
channels[0];
193 std::vector<size_t> starts;
194 starts.reserve(waveform.window_positions.size());
195 for (
const auto& position : waveform.window_positions) {
196 starts.push_back(position.first);
199 waveform.periodicity, starts, speech, voicing_threshold);
201 std::vector<double> voiced_frequency;
202 std::vector<size_t> voiced_starts;
203 for (
size_t window = 0; window < starts.size() && window < waveform.periodicity.size(); ++window) {
204 if (waveform.periodicity[window] >= voicing_threshold) {
205 voiced_frequency.push_back(waveform.frequency[window]);
206 voiced_starts.push_back(starts[window]);
231template <ComputeData InputType = std::vector<Kakshya::DataVariant>, ComputeData OutputType = Kakshya::RegionGroup>
250 if (
const std::string reason =
diagnose(
config); !reason.empty()) {
252 std::source_location::current(),
"SpeechSegmenter: {}", reason);
274 return std::format(
"sample_rate {} is not the {} Hz the network runs at; convert the input first, for example with Kinesis::Discrete::resample",
277 if (!std::isfinite(
config.speech.voicing_threshold) ||
config.speech.voicing_threshold <= 0.0) {
278 return "voicing_threshold must be finite and greater than 0";
293 if (
const std::string reason =
diagnose(
config); !reason.empty()) {
295 std::source_location::current(),
"SpeechSegmenter: {}", reason);
300 if (samples.empty()) {
304 if (
config.speech.measure) {
317 const auto*
record = result.facets.template get<SpeechSegmentation>();
320 std::source_location::current(),
"Speech segmentation produced no result");
341 return "SpeechSegmenter";
350 if (channels.size() != 1) {
352 std::source_location::current(),
"SpeechSegmenter analyses one channel and was given {}", channels.size());
354 const std::span<const double> samples { channels[0].data(), channels[0].size() };
362 }
catch (
const std::exception& e) {
383 const auto probabilities =
m_vad.analyze_speech(
384 std::vector<Kakshya::DataVariant> { std::vector<double>(samples.begin(), samples.end()) })
386 if (probabilities.empty()) {
390 Eigen::MatrixXd scores(
static_cast<Eigen::Index
>(probabilities.size()), 1);
391 for (
size_t chunk = 0; chunk < probabilities.size(); ++chunk) {
392 scores(
static_cast<Eigen::Index
>(chunk), 0) = probabilities[chunk];
395 const auto intervals =
m_intervals.extract_intervals(scores);
396 if (!intervals.channels.empty()) {
397 record.speech = intervals.channels[0].intervals;
407 record.voiced_fraction = std::move(voicing.voiced_fraction);
408 record.f0 = std::move(voicing.f0);
413 if constexpr (std::is_same_v<OutputType, Kakshya::RegionGroup>) {
#define MF_ERROR(comp, ctx,...)
#define MF_DESCRIBE(...)
Requests a generated member description for a struct at namespace scope.
Runs of a score sequence that stay above a threshold, found in order as a stream, for MayaFlux::Kines...
Voice activity probability per 32 ms chunk from a learned network, run on the GPU.
void set(T value)
Stores a struct under its type identity.
bool is_initialized() const
Check if compiler is initialized.
const Config & config() const
void set_config(Config config)
Stores a config struct, whole, under its own type.
output_type apply_operation(const input_type &input)
Public synchronous execution interface.
ConfigRefs stored_configs(Supports< Configs... >) const
The stored configs among those listed, in listed order.
output_type convert_result(std::vector< std::vector< double > > &result_data, DataStructureInfo &metadata)
Convert processed double data back to OutputType using metadata and optional callback.
static std::tuple< NumericChannels, DataStructureInfo > extract_structured_double(T &compute_data)
Extract structured double data from Datum container or direct ComputeData with automatic container ha...
static constexpr uint32_t sample_rate
Sample rate in Hz the network was trained and runs at.
static constexpr uint32_t chunk_samples
Datum< OutputType > output_type
StandardVadAnalyzer m_vad
void find_speech(std::span< const double > samples, const SpeechSegmenterConfig &cfg, SpeechSegmentation &record)
static float fitness(const input_type &)
One while a device is up, zero otherwise.
output_type build_output(const SpeechSegmentation &record, std::span< const double > samples, DataStructureInfo &info)
Datum< InputType > input_type
SpeechSegmentation extract_segments(const InputType &data)
static VadAnalyzerConfig vad_settings(const SpeechSegmenterConfig &config)
Extractor< InputType, OutputType > base_type
void measure_speech(std::span< const double > samples, const SpeechSegmenterConfig &cfg, SpeechSegmentation &record)
std::optional< StandardWaveformAnalyzer > m_waveform
static constexpr Target target
output_type extract_implementation(const input_type &input) override
Pure virtual extraction implementation - derived classes implement this.
Supports< SpeechSegmenterConfig > supports
ConfigRefs configs() const override
The base view of the config structs this operation holds.
ExtractionType get_extraction_type() const override
Gets the extraction type category for this extractor.
MatrixIntervalExtractor m_intervals
SpeechSegmenter(const Kakshya::TensorSet &vad_weights, SpeechSegmenterConfig config={})
Construct from the weights of the speech network.
static std::string diagnose(const SpeechSegmenterConfig &config)
Why a config cannot be segmented with.
std::string get_extractor_name() const override
Get extractor-specific name (derived classes override this).
SpeechSegmentation segment(std::span< const double > samples, const SpeechSegmenterConfig &config)
Segment samples under the given settings, without the Datum wrapper.
SpeechSegmentation extract_segments(const input_type &data)
Type-safe segmentation.
The speech intervals of a whole mono 16 kHz recording.
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP).
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
std::string diagnose_interval_scan(const IntervalScanConfig &config)
Why a config cannot be scanned.
std::vector< double > slice_intervals(std::span< const double > data, const std::vector< std::pair< size_t, size_t > > &intervals)
Copy data from a set of half-open intervals into a flat vector.
std::vector< std::pair< size_t, size_t > > intervals_from_window_starts(const std::vector< size_t > &window_starts, uint32_t window_size, size_t data_size)
Build [start, end) intervals from window start indices.
std::vector< double > interval_share_above(std::span< const double > values, std::span< const size_t > positions, const std::vector< std::pair< size_t, size_t > > &intervals, double threshold)
Share of the values inside each interval that are at or above a threshold.
std::vector< std::pair< size_t, size_t > > merge_intervals(const std::vector< std::pair< size_t, size_t > > &intervals)
Merge overlapping or adjacent half-open intervals.
std::vector< double > interval_median(std::span< const double > values, std::span< const size_t > positions, const std::vector< std::pair< size_t, size_t > > &intervals)
Median of the values inside each interval.
void threshold(std::span< const float > gray, std::span< float > dst, float value)
Global threshold writing into caller-supplied buffer.
MAYAFLUX_API ShaderFoundry & get_shader_foundry()
Get the global shader compiler instance.
ExtractionType
Categories of extraction operations for discovery and organization.
@ PATTERN_BASED
Extract based on pattern recognition.
VadAnalyzer< std::vector< Kakshya::DataVariant >, Eigen::MatrixXd > StandardVadAnalyzer
Standard VAD analyzer: DataVariant -> MatrixXd.
std::vector< std::pair< size_t, size_t > > voiced_intervals(StandardWaveformAnalyzer &analyzer, std::span< const double > samples, double threshold, uint32_t hop)
The stretches of a recording where the waveform is voiced.
IntervalExtractor< Eigen::MatrixXd, std::vector< std::vector< double > > > MatrixIntervalExtractor
Matrix interval extractor: the MatrixXd an analyzer returns, one score channel per column -> interval...
Kinesis::Discrete::IntervalScanConfig interval_scan_config(const IntervalScanSettings &settings, uint32_t length)
The host scan configuration of one column's settings over a stream of the given length.
IntervalExtractorConfig speech_scan_settings(const SpeechSettings &speech, size_t samples)
The interval scan settings of speech runs, for a recording of the given length.
SpeechSegmenter< std::vector< Kakshya::DataVariant >, Kakshya::RegionGroup > RegionSpeechSegmenter
Regions: one region per speech interval.
WaveformAnalyzer< std::vector< Kakshya::DataVariant >, Eigen::MatrixXd > StandardWaveformAnalyzer
Standard waveform analyzer: DataVariant -> MatrixXd.
SpeechSegmenter< std::vector< Kakshya::DataVariant >, std::vector< Kakshya::DataVariant > > AudioSpeechSegmenter
Audio: the speech intervals, concatenated.
SpeechVoicing measure_voicing(StandardWaveformAnalyzer &analyzer, std::span< const double > samples, const std::vector< std::pair< size_t, size_t > > &speech, double voicing_threshold)
Voiced share and median frequency of the waveform windows that start in each interval.
std::vector< std::reference_wrapper< const OperationConfig > > ConfigRefs
Borrowed views of the configs an operation holds.
Kakshya::RegionGroup interval_regions(const ChannelIntervals &intervals, uint32_t first_channel, uint32_t last_channel, std::string name="intervals")
One region per interval, spanning the given channels.
WaveformAnalyzerConfig voicing_analyzer_config(double sample_rate, uint32_t hop=256)
The waveform analyzer settings the voicing measure uses at the given rate and hop.
std::pair< std::shared_ptr< Kriya::BufferPipeline >, std::shared_ptr< Buffers::AudioBuffer > > record(const std::shared_ptr< Kakshya::DynamicSoundStream > &stream, uint32_t input_channel, double seconds, std::optional< uint32_t > monitor, uint32_t channel)
Record a live input channel into a stream for a number of seconds.
Organizes related signal regions into a categorized collection.
Named tensors that belong together, such as a model's weights or I/O.
uint32_t window_size
Analysis window length in samples.
uint32_t hop_size
Distance between successive windows in samples.
std::vector< std::pair< size_t, size_t > > intervals
Intervals of one channel, half open sample ranges in stream order.
Dimensions, modality, source type, and optional Composite field schema.
T data
The actual computation data.
Facets facets
Typed results that travel with the data, read by type.
std::unordered_map< std::string, std::any > metadata
Associated metadata.
Input/Output container for computation pipeline data flow with structure preservation.
std::vector< double > f0
Median frequency of each interval's voiced windows, empty unless measured.
std::vector< double > voiced_fraction
Voiced share of each interval's windows, empty unless measured.
SpeechSegmenterConfig config
Settings the segmentation was produced under.
std::vector< std::pair< size_t, size_t > > speech
Speech intervals, half open sample ranges.
Record of one SpeechSegmenter run, stored on the output's facets.
double sample_rate
Sample rate in Hz of the input.
SpeechSettings speech
How speech is found and measured.
Settings of a SpeechSegmenter.
double threshold
Speech probability at or above which speech starts.
bool measure
Also measure the voiced fraction and pitch of each speech interval.
uint32_t max_run
Longest speech interval in samples, 0 for no cap.
uint32_t min_speech
Shortest speech run kept, in samples.
uint32_t padding
Samples added on each side of a speech run.
double voicing_threshold
Periodicity at or above which a window is voiced.
uint32_t min_silence
Shortest silence that ends a speech run, in samples.
How speech runs are found and measured.
std::vector< double > voiced_fraction
Voiced share of each interval's windows.
std::vector< double > f0
Median frequency of each interval's voiced windows.
Voicing of each interval, as SpeechSegmentation holds it.
The config structs an operation class accepts, declared as using supports.
A set of (device, runtime) pairs.
bool flush_tail
Also run the partial chunk left at the end of each call, padded with zeros.
double sample_rate
Sample rate in Hz of the input.
Settings of a VadAnalyzer.